Files
ViperC/App/lib/core/Phase1Main.vp
T
TermiNexus f5c4be3995 refactor(core): Phase1 拆 3 组(Scan/Topo/Main,1801→46 行)
直接移动+改调用方:7 个函数按功能移至 3 个子模块,主文件仅保留
常量/_mbuddy。
- 子模块函数体内裸调用映射子模块前缀,全局符号经 Phase1._xxx 访问
- 调用方 main.vp 的 Phase1.func( → 子模块前缀并补 import
- 全 App 语法/引用/import 验证通过
2026-08-17 08:14:45 +08:00

955 lines
42 KiB
Plaintext
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import t, c
from stdint import *
import string
import stdlib
import memhub
import viperlib
import w32.fileio as fileio
import w32.win32file
import w32.win32base
import ast
import llvmlite
import lib.core.VLogger as VLogger
import lib.core.Handles.HandlesTranslator as HandlesTranslator
import lib.core.Handles.HandlesStruct as HandlesStruct
import lib.core.Handles.HandlesType as HandlesType
import lib.core.Handles.HandlesExprCall as HandlesExprCall
import lib.core.Handles.HandlesImports as HandlesImports
import lib.core.Handles.HandlesClassDef as HandlesClassDef
import lib.core.IncludesScanner as IncludesScanner
import lib.core.StubMerger as StubMerger
import lib.core.Gen.Generator as GenGenerator
import lib.core.CompilerFactory as CompilerFactory
import lib.core.BuildPipeline as BuildPipeline
import lib.Projectrans.Config as Config
import lib.StubGen.Converter as StubConverter
import lib.core.StubMergerBuild as StubMergerBuild
import lib.core.StubMergerCollect as StubMergerCollect
import lib.core.StubMergerIR as StubMergerIR
import lib.core.StubMergerStore as StubMergerStore
import lib.core.Handles.ExprCallCtor as ExprCallCtor
import lib.core.Phase1Scan as Phase1Scan
import lib.core.Phase1Topo as Phase1Topo
import lib.core.Phase1 as Phase1
# ============================================================
# Phase1Main - Phase1 拆分(阶段 E):Phase1Main
#
# 直接移动模式:函数迁至此模块,原 Phase1.vp 仅保留常量/_mbuddy;
# 调用方经本模块前缀访问。全局符号经 Phase1._xxx 跨模块访问。
# ============================================================
# ============================================================
# RunPhase1 - Phase1: 扫描 includes 目录,按需翻译并生成 stub
#
# 对每个 .vp/.py 文件检查 stub 是否已存在,若不存在则翻译并分离 stub。
# "按需翻译":仅对 stub 不存在的文件执行翻译,避免重复工作。
#
# Args:
# mb: 内存池
# includes_dir: includes 目录路径
# temp_dir: 临时目录(保存 stub)
# log: 日志器
#
# Returns:
# 0 成功,非 0 失败
# ============================================================
def RunPhase1(mb: memhub.MemBuddy | t.CPtr, includes_dir: str, temp_dir: str,
log: VLogger.Logger | t.CPtr) -> int:
"""Phase1: 扫描 includes 目录,按需翻译并生成 stub"""
if includes_dir is None or temp_dir is None:
VLogger.error("includes_dir 或 temp_dir 为空,无法运行 Phase1", "Phase1")
return 1
# 设置全局 temp_dir(供跨模块 CDefine 查找使用)
HandlesType.set_temp_dir(temp_dir)
# 同步设置 HandlesExprCall 的 temp_dir(供 vpi stub 返回注解缓存按需加载)。
# 必须在 includes 翻译前设置:Phase1 翻译 json 等 includes 时,"同模块方法
# 调用(定义在后)"(如 json.__setitem__ 调 _pool_alloc)需要读
# {temp_dir}/{sha1[0:2]}/{sha1}.vpi 兜底,而 Phase2 的 SetTempDir 太晚。
ExprCallCtor.SetTempDir(temp_dir)
# 确保 temp 目录存在(build_dir/temp 可能尚未创建)
BuildPipeline.ensure_dir(temp_dir)
if log is not None:
log.banner("Phase1: 扫描 includes(按需翻译)")
# 扫描 includes 目录
result: IncludesScanner.ScanResult | t.CPtr = IncludesScanner.scan_includes(mb, includes_dir)
if result is None:
VLogger.error("扫描失败", "Phase1")
return 1
# 从扫描结果直接构建 SHA1 集合(不依赖 _sha1_map.txt 文件)
# Phase1 作为编译器自身,应从内存中的扫描结果获取 SHA1,避免外部文件依赖
sha1_set: str = stdlib.malloc(StubMerger.MAX_INCLUDES_SHA1 * 17)
if sha1_set is None:
VLogger.error("sha1_set 分配失败", "Phase1")
return 1
string.memset(sha1_set, 0, StubMerger.MAX_INCLUDES_SHA1 * 17)
set_count: int = 0
p1_entry_size_init: t.CSizeT = IncludesScanner.FileEntry.__sizeof__()
for se_i in range(result.Count):
if set_count >= StubMerger.MAX_INCLUDES_SHA1:
break
se_addr: t.CUInt64T = t.CUInt64T(result.Entries) + se_i * p1_entry_size_init
se_ent: IncludesScanner.FileEntry | t.CPtr = (IncludesScanner.FileEntry | t.CPtr)(t.CVoid(se_addr, t.CPtr))
if se_ent is None or se_ent.Sha1 is None:
continue
string.strcpy(sha1_set + set_count * 17, se_ent.Sha1)
set_count += 1
# 先填充全局 SHA1 映射存储器(即使 set_count==0 也填充,确保 Phase2 能获取 includes 条目)
# 注意:PopulateSha1MapStore 会先 free 旧存储器再重新分配
StubMergerStore.PopulateSha1MapStore(result)
# 写入 _sha1_map.txt(人类可读输出,程序内部不读取;机器分析使用 PopulateSha1MapStore 内存存储器)
StubMergerCollect.WriteIncludesSha1Map(mb, temp_dir, result, None, 0)
if set_count <= 0:
VLogger.warning("扫描结果无有效 SHA1,跳过 Phase1(项目可能无 includes 依赖)", "Phase1")
stdlib.free(sha1_set)
return 0
# 模块 SHA1 映射已由 PopulateSha1MapStore 填充到 StubMerger 全局存储器,
# HandlesExprCall._lookup_module_sha1 直接查全局存储器,不再需要独立副本。
# 初始化 AST 表(只需一次)
ast._init_tables(mb)
# FileEntry 结构体大小(Phase 1a 和 1b 共用)
entry_size: t.CSizeT = IncludesScanner.FileEntry.__sizeof__()
# ============================================================
# Phase 1a-pre: 扫描所有 includes 文件的 import 依赖
#
# 只处理 Import/ImportFrom 语句,跳过 ClassDef/FunctionDef 等,
# 不调用 resolve_annotation_type,避免 list[...] 等不支持的语法触发 crash。
# 生成 .deps.txt 供依赖图按需翻译使用。
# ============================================================
p1a_registered: int = 0
p1a_skipped: int = 0
p1a_failed: int = 0
for i in range(result.Count):
# 文件级 mark:import 依赖收集完(deps 写盘 + PopulateIncludesDeps
# 内部 stdlib 深拷贝)后立即回收本文件 AST——98 includes 全量 parse
# 的 AST 若驻留到 mark1 → 1GB 主池耗尽 OOM 根因之一
scan_mark_a: t.CSizeT = mb.mark()
entry_addr_a: t.CUInt64T = t.CUInt64T(result.Entries) + i * entry_size
entry_a: IncludesScanner.FileEntry | t.CPtr = (IncludesScanner.FileEntry | t.CPtr)(t.CVoid(entry_addr_a, t.CPtr))
if entry_a is None:
p1a_failed += 1
continue
sha1_a: str = entry_a.Sha1
if sha1_a is None:
p1a_failed += 1
continue
if StubMergerCollect._is_in_sha1_set(sha1_a, sha1_set, set_count) == 0:
p1a_skipped += 1
continue
# 读取文件内容
file_path_a: str = entry_a.Path
f_a: fileio.File | t.CPtr = fileio.File(file_path_a, fileio.MODE.R)
if f_a.closed:
p1a_failed += 1
continue
src_buf_a: bytes = stdlib.malloc(Phase1.SRC_BUF_SIZE)
if src_buf_a is None:
f_a.close()
p1a_failed += 1
continue
bytes_read_a: LONG = f_a.read_all(src_buf_a, Phase1.SRC_BUF_SIZE)
f_a.close()
if bytes_read_a <= 0:
stdlib.free(src_buf_a)
p1a_failed += 1
continue
if bytes_read_a < Phase1.SRC_BUF_SIZE:
src_buf_a[bytes_read_a] = 0
else:
src_buf_a[Phase1.SRC_BUF_SIZE - 1] = 0
# 解析 AST
lx_a: ast.Lexer | t.CPtr = ast.new_lexer(mb)
if lx_a is None:
stdlib.free(src_buf_a)
p1a_failed += 1
continue
ast._lexer_init(lx_a, src_buf_a, mb)
tokens_a: ast.Token | t.CPtr = ast.tokenize(lx_a)
tree_a: ast.AST | t.CPtr = ast.parse_tokens(mb, tokens_a)
if tree_a is None:
stdlib.free(src_buf_a)
p1a_failed += 1
continue
# 创建 Translator,设置 _declare_only=2(只扫描 import,跳过 ClassDef)
tr_a: HandlesTranslator.Translator | t.CPtr = HandlesTranslator.Translator()
if tr_a is None:
stdlib.free(src_buf_a)
p1a_failed += 1
continue
tr_a.ModuleSha1 = sha1_a
tr_a._declare_only = 2
tr_a.CurrentPackage = HandlesImports.compute_package_from_relpath(mb, entry_a.RelPath)
HandlesType.set_current_file(file_path_a)
HandlesImports.set_current_file(file_path_a)
HandlesType.set_current_module_sha1(sha1_a)
HandlesType.clear_cdefine_constants()
HandlesType.clear_ctypedefs()
HandlesStruct.reset_visible_structs(mb, 0)
ret_a: int = tr_a.translate(tree_a)
if ret_a != 0:
p1a_failed += 1
else:
p1a_registered += 1
# 容器模板按需纳入依赖:includes 使用 list/dict 泛型/构造/字面量
# (如 base.vp 的 list[AST | CPtr])时追加 _list/_dict 到
# _imported_modules → 进 deps → _BuildReachableSha1Set 的可达集合
# 自动包含(Phase1 与 Phase2 都受益),否则泛型模板未注册 →
# "泛型类 'list' 未找到模板"。对齐 App 层 _inject_auto_container_visibility。
cl_box: t.CPtr = stdlib.malloc(4)
cd_box: t.CPtr = stdlib.malloc(4)
if cl_box is not None and cd_box is not None:
cl_box[0] = 0
cd_box[0] = 0
Phase1Scan._ScanTreeForContainers(tree_a, cl_box, cd_box)
if cl_box[0] != 0:
if tr_a._imported_modules is None or \
HandlesImports.is_module_imported(tr_a._imported_modules, "_list") == 0:
tr_a._imported_modules = HandlesImports.add_imported_module(
mb, tr_a._imported_modules, "_list")
if cd_box[0] != 0:
if tr_a._imported_modules is None or \
HandlesImports.is_module_imported(tr_a._imported_modules, "_dict") == 0:
tr_a._imported_modules = HandlesImports.add_imported_module(
mb, tr_a._imported_modules, "_dict")
if cl_box is not None:
stdlib.free(cl_box)
if cd_box is not None:
stdlib.free(cd_box)
# 生成 .deps.txt(记录依赖模块名,供依赖图按需翻译使用)
# 同时填充内存依赖存储器(PopulateIncludesDeps),供 _BuildReachableSha1Set
# 直接从内存读取,不依赖 .deps.txt 过程文件
td_len_a: t.CSizeT = string.strlen(temp_dir)
deps_path_a: str = StubMergerCollect._sliced_path(temp_dir, td_len_a, sha1_a, "deps.txt")
if deps_path_a is not None:
df_a: fileio.File | t.CPtr = fileio.File(deps_path_a, fileio.MODE.W)
if not df_a.closed:
if tr_a._imported_modules is not None:
dl_a: t.CSizeT = string.strlen(tr_a._imported_modules)
df_a.write(tr_a._imported_modules, dl_a)
df_a.close()
stdlib.free(deps_path_a)
# 填充内存依赖存储器(供 _BuildReachableSha1Set 使用,不读 .deps.txt)
if tr_a._imported_modules is not None:
StubMergerStore.PopulateIncludesDeps(sha1_a, tr_a._imported_modules)
# 释放 Translator 的 C malloc 资源
if tr_a._global_names is not None:
stdlib.free(tr_a._global_names)
if tr_a._nonlocal_names is not None:
stdlib.free(tr_a._nonlocal_names)
stdlib.free(src_buf_a)
# 回收本文件 AST/lexer/token(deps 已写盘、依赖存储器已深拷贝)
mb.reset_to(scan_mark_a)
if p1a_failed > 0:
fb_p1apre: str = VLogger.fmt_buf()
if fb_p1apre is not None:
viperlib.snprintf(fb_p1apre, 1024, "Phase 1a-pre 有 %d 个文件扫描失败", p1a_failed)
VLogger.error(fb_p1apre, "Phase1")
return 1
# ============================================================
# 依赖图按需翻译:构建可达 SHA1 集合
#
# 从 Config.SourceDir 的源文件开始,解析 import 语句,递归收集
# 可达的 includes 文件 SHA1。Phase 1b 只翻译可达集合中的文件,
# 避免翻译不需要的 includes(如 Test 不依赖 llvmlite,则不翻译)。
# 如果 Config.SourceDir 不可用(仅运行 Phase1),回退到 sha1_set。
# ============================================================
reachable_set: str = None
reachable_count: int = 0
use_reachable: int = 0
if Config.SourceDir is not None:
reachable_set = stdlib.malloc(StubMerger.MAX_INCLUDES_SHA1 * 17)
if reachable_set is not None:
string.memset(reachable_set, 0, StubMerger.MAX_INCLUDES_SHA1 * 17)
reachable_count = StubMergerBuild._BuildReachableSha1Set(mb, Config.SourceDir, temp_dir, reachable_set)
if reachable_count < 0:
VLogger.error("可达集合构建失败,终止", "Phase1")
return 1
if reachable_count > 0:
use_reachable = 1
# 用可达集合重新生成 _sha1_map.txt(按图求索的最终产物)
# Phase B+ 只遍历这些条目,避免编译不需要的 includes(如 asm.vp)
StubMergerCollect.WriteIncludesSha1Map(mb, temp_dir, result, reachable_set, reachable_count)
# 重新加载 sha1_set,使后续 Phase 1a-pre/1a/1b 的过滤也使用可达集合
string.memset(sha1_set, 0, StubMerger.MAX_INCLUDES_SHA1 * 17)
set_count = StubMergerCollect._load_includes_sha1_set(mb, temp_dir, sha1_set)
else:
VLogger.warning("可达集合为空,回退全量扫描", "Phase1")
stdlib.free(reachable_set)
reachable_set = None
# ============================================================
# Phase 1a: 注册可达 includes 文件的 struct/enum/union
#
# 解决字母序依赖问题:按字母顺序翻译时,后面的 struct 未注册
# 前面就需要用。Phase 1a 先注册所有 struct/enum/union 到全局表,
# Phase 1b 全量翻译时 struct 已注册,走 existing 路径只翻译方法体。
# 只处理可达文件,避免翻译不需要的 includes(如 Test 不依赖 ast 模块)。
#
# 拓扑排序:预扫描所有文件收集类继承关系,按依赖顺序处理文件,
# 确保父类所在文件先于子类所在文件处理(如 base.vp 先于 astaux.vp)。
# ============================================================
p1a_reg: int = 0
p1a_skp: int = 0
p1a_fl: int = 0
# 预扫描类继承关系,构建拓扑顺序(用扁平字节数组,避免结构体 str 指针 NULL 崩溃)
class_names_buf: bytes = mb.alloc(Phase1.MAX_CLASSES * Phase1.CLASS_NAME_LEN)
parent_names_buf: bytes = mb.alloc(Phase1.MAX_CLASSES * Phase1.CLASS_NAME_LEN)
def_sha1s_buf: bytes = mb.alloc(Phase1.MAX_CLASSES * Phase1.SHA1_LEN)
topo_order: t.CPtr = mb.alloc(4 * Phase1.MAX_FILE_DEPS)
topo_count_box: t.CPtr = mb.alloc(4)
generic_file_indices: t.CPtr = mb.alloc(4 * Phase1.MAX_FILE_DEPS)
generic_file_count_box: t.CPtr = mb.alloc(4)
if generic_file_indices is not None:
string.memset(generic_file_indices, 0, 4 * Phase1.MAX_FILE_DEPS)
if generic_file_count_box is not None:
string.memset(generic_file_count_box, 0, 4)
topo_count: int = 0
if class_names_buf is not None and parent_names_buf is not None and def_sha1s_buf is not None and topo_order is not None and topo_count_box is not None:
string.memset(class_names_buf, 0, Phase1.MAX_CLASSES * Phase1.CLASS_NAME_LEN)
string.memset(parent_names_buf, 0, Phase1.MAX_CLASSES * Phase1.CLASS_NAME_LEN)
string.memset(def_sha1s_buf, 0, Phase1.MAX_CLASSES * Phase1.SHA1_LEN)
string.memset(topo_order, 0, 4 * Phase1.MAX_FILE_DEPS)
string.memset(topo_count_box, 0, 4)
if generic_file_indices is not None:
string.memset(generic_file_indices, 0, 4 * Phase1.MAX_FILE_DEPS)
if generic_file_count_box is not None:
string.memset(generic_file_count_box, 0, 4)
class_count_scanned: int = Phase1Scan._ScanClassInheritance(
mb, result, reachable_set, reachable_count,
use_reachable, sha1_set, set_count,
class_names_buf, parent_names_buf, def_sha1s_buf,
generic_file_indices, generic_file_count_box)
if class_count_scanned > 0:
td_len_ts_call: t.CSizeT = string.strlen(temp_dir)
ret_topo: int = Phase1Topo._TopoSortFiles(
mb, result, reachable_set, reachable_count,
use_reachable, sha1_set, set_count,
class_names_buf, parent_names_buf, def_sha1s_buf, class_count_scanned,
topo_order, topo_count_box,
temp_dir, td_len_ts_call)
if ret_topo == 0:
tc_ptr: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(topo_count_box, t.CPtr))
if tc_ptr is not None:
topo_count = tc_ptr[0]
fb_topo: str = VLogger.fmt_buf()
if fb_topo is not None:
viperlib.snprintf(fb_topo, 1024,
"拓扑排序: %d 个类, %d 个文件按依赖顺序处理",
class_count_scanned, topo_count)
VLogger.info(fb_topo, "Phase1")
else:
VLogger.error("拓扑排序失败(类继承依赖无法解析),终止编译", "Phase1")
return 1
else:
VLogger.warning("类继承预扫描无结果,回退到字母序", "Phase1")
topo_count = 0
# ============================================================
# Phase 1a-pre-generic-all: mark1 前全量预注册 includes 泛型模板
#
# 1a-pre-generic 只覆盖"可达"泛型文件;非可达泛型文件(如 FuncTest
# 场景的 linkedlist/llvmlite/_list)的模板要等 Phase2 Phase A-pre-inc
# (mark2 之后)才注册 → reset 回收表/节点 → _find_generic_template
# strcmp 悬空崩溃。此处对全部 includes 轻量预注册,使模板表与节点
# 都落在 mark1 之前(mark2 > mark1,reset_to(mark2) 不回收)。
# ============================================================
Phase1Scan._PreRegisterAllGenericTemplates(mb, result)
# ============================================================
# Phase 1a: 注册 struct/enum/union(mark1 前,全量 includes)
#
# OOM 治理:1a 对全部 includes parse 完整 AST(主 pool 分配),若
# 驻留到 mark1(循环后)会累积 98 文件 × 每文件 AST 树 → 1GB 主池
# 耗尽(NewAstTest alloc 失败 0xBE 根因)。现改为:
# 1) StdlibAlloc=1:1a 注册的跨文件持久数据(StructTable 表、
# Fields/Name/ModuleSha1、struct_ty 深拷贝、枚举/Renum/泛型
# 模板等)全部落在 perm_pool(永不 reset),不占主 pool;
# 2) 每文件翻译完 reset_to(文件级 mark):仅回收该文件的 AST/
# lexer/token/IR 等瞬时数据,注册数据不受影响。
# mark1 仍在 1a 结束后打,1b 每文件 reset_to(mark1) 语义不变。
# ============================================================
HandlesStruct.set_stdlib_alloc(1)
# 遍历文件:优先使用拓扑顺序,回退到字母序
iter_count: int = topo_count if topo_count > 0 else result.Count
for i in range(iter_count):
# 每文件级 mark:翻译完回收该文件 AST(注册数据在 perm_pool)
p1a_mark: t.CSizeT = mb.mark()
# 获取文件索引:拓扑顺序或字母序
file_idx: int = i
if topo_count > 0:
od_addr: t.CUInt64T = t.CUInt64T(topo_order) + i * 4
od_ptr: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(od_addr, t.CPtr))
if od_ptr is not None:
file_idx = od_ptr[0]
entry_addr_r: t.CUInt64T = t.CUInt64T(result.Entries) + file_idx * entry_size
entry_r: IncludesScanner.FileEntry | t.CPtr = (IncludesScanner.FileEntry | t.CPtr)(t.CVoid(entry_addr_r, t.CPtr))
if entry_r is None:
p1a_fl += 1
continue
sha1_r: str = entry_r.Sha1
if sha1_r is None:
p1a_fl += 1
continue
# 按需翻译过滤(拓扑排序已过滤,但字母序回退需要检查)
if topo_count == 0:
if use_reachable != 0:
if StubMergerCollect._is_in_sha1_set(sha1_r, reachable_set, reachable_count) == 0:
p1a_skp += 1
continue
else:
if StubMergerCollect._is_in_sha1_set(sha1_r, sha1_set, set_count) == 0:
p1a_skp += 1
continue
# 读取文件内容
file_path_r: str = entry_r.Path
f_r: fileio.File | t.CPtr = fileio.File(file_path_r, fileio.MODE.R)
if f_r.closed:
p1a_fl += 1
continue
src_buf_r: bytes = stdlib.malloc(Phase1.SRC_BUF_SIZE)
if src_buf_r is None:
f_r.close()
p1a_fl += 1
continue
bytes_read_r: LONG = f_r.read_all(src_buf_r, Phase1.SRC_BUF_SIZE)
f_r.close()
if bytes_read_r <= 0:
stdlib.free(src_buf_r)
p1a_fl += 1
continue
if bytes_read_r < Phase1.SRC_BUF_SIZE:
src_buf_r[bytes_read_r] = 0
else:
src_buf_r[Phase1.SRC_BUF_SIZE - 1] = 0
# 解析 AST
lx_r: ast.Lexer | t.CPtr = ast.new_lexer(mb)
if lx_r is None:
stdlib.free(src_buf_r)
p1a_fl += 1
continue
ast._lexer_init(lx_r, src_buf_r, mb)
tokens_r: ast.Token | t.CPtr = ast.tokenize(lx_r)
tree_r: ast.AST | t.CPtr = ast.parse_tokens(mb, tokens_r)
if tree_r is None:
stdlib.free(src_buf_r)
p1a_fl += 1
continue
# 创建 Translator,设置 _declare_only=1(注册 struct/enum/union)
tr_r: HandlesTranslator.Translator | t.CPtr = HandlesTranslator.Translator()
if tr_r is None:
stdlib.free(src_buf_r)
p1a_fl += 1
continue
tr_r.ModuleSha1 = sha1_r
tr_r._declare_only = 1
tr_r.CurrentPackage = HandlesImports.compute_package_from_relpath(mb, entry_r.RelPath)
HandlesType.set_current_file(file_path_r)
HandlesImports.set_current_file(file_path_r)
HandlesType.set_current_module_sha1(sha1_r)
HandlesType.clear_cdefine_constants()
HandlesType.clear_ctypedefs()
HandlesStruct.reset_visible_structs(mb, 0)
ret_r: int = tr_r.translate(tree_r)
if ret_r != 0:
p1a_fl += 1
else:
p1a_reg += 1
# 释放 Translator 的 C malloc 资源
if tr_r._global_names is not None:
stdlib.free(tr_r._global_names)
if tr_r._nonlocal_names is not None:
stdlib.free(tr_r._nonlocal_names)
stdlib.free(src_buf_r)
# 回收本文件 AST/lexer/token/IR 等瞬时数据(注册数据已在 perm_pool,
# reset 不影响;IncludesScanner result 等在 p1a_mark 前分配同样保留)
mb.reset_to(p1a_mark)
if p1a_fl > 0:
fb_p1ar: str = VLogger.fmt_buf()
if fb_p1ar is not None:
viperlib.snprintf(fb_p1ar, 1024, "Phase 1a 有 %d 个文件 struct 注册失败", p1a_fl)
VLogger.error(fb_p1ar, "Phase1")
return 1
# 释放拓扑排序资源(topo_order/topo_count_box 保留到 Phase 1b 结束,
# 1b 按拓扑序翻译需要它们;class/def 数组 1b 不再使用)
if class_names_buf is not None:
mb.free(class_names_buf)
if parent_names_buf is not None:
mb.free(parent_names_buf)
if def_sha1s_buf is not None:
mb.free(def_sha1s_buf)
if generic_file_indices is not None:
mb.free(generic_file_indices)
if generic_file_count_box is not None:
mb.free(generic_file_count_box)
# ============================================================
# Phase 1b: 全量翻译(struct 已注册,走 existing 路径翻译方法体)
# ============================================================
# OOM 回收:1a-pre/泛型预注册/1a 全部结束后打 mark1,
# 所有跨文件持久表(_struct_table/_enum_table/_generic_class_nodes/
# _renum_*/关键字表等)都落在 mark1 之前。
# 1b 每文件翻译完后 reset_to(mark1) 回收该文件的 AST/IR/Handler/dump 临时,
# 避免 90 文件累积导致 1GB 池 OOM。
mark1: t.CSizeT = mb.mark()
# 1b 阶段新注册的 struct/enum 条目改 stdlib 分配(跨文件永久存活,不被 reset 回收)
HandlesStruct.set_stdlib_alloc(1)
# ============================================================
# Phase 1b-pre: 预生成所有待翻译文件的 .vpi stub
#
# 依赖模块的 .vpi 必须在翻译前可用:1b 主循环按扫描顺序翻译(非拓扑),
# 若 argparse 先于 hashtable 翻译,hashtable.vpi 尚未生成,
# LoadDepVpiIntoModule 读不到 → declare 未注入 → 跨模块调用
# 返回类型推断错误(如 HashTable.__getitem__ 被推断为 Ptr(HashTable)
# 而 define 返回 i8*)→ 运行时返回值截断崩溃。
# ============================================================
p1bpre_failed: int = 0
dir_len_pv: t.CSizeT = string.strlen(temp_dir)
for i_pv in range(result.Count):
entry_addr_pv: t.CUInt64T = t.CUInt64T(result.Entries) + i_pv * entry_size
entry_pv: IncludesScanner.FileEntry | t.CPtr = (IncludesScanner.FileEntry | t.CPtr)(t.CVoid(entry_addr_pv, t.CPtr))
if entry_pv is None or entry_pv.Sha1 is None:
p1bpre_failed += 1
continue
sha1_pv: str = entry_pv.Sha1
in_set_pv: int = 0
if use_reachable != 0:
in_set_pv = StubMergerCollect._is_in_sha1_set(sha1_pv, reachable_set, reachable_count)
else:
in_set_pv = StubMergerCollect._is_in_sha1_set(sha1_pv, sha1_set, set_count)
if in_set_pv == 0:
continue
# vpi 已存在则跳过
vpi_path_pv: str = StubMergerCollect._sliced_path(temp_dir, dir_len_pv, sha1_pv, "vpi")
if vpi_path_pv is None:
p1bpre_failed += 1
continue
vf_pv: fileio.File | t.CPtr = fileio.File(vpi_path_pv, fileio.MODE.R)
if not vf_pv.closed:
vf_pv.close()
stdlib.free(vpi_path_pv)
continue
stdlib.free(vpi_path_pv)
# 读取文件、解析 AST、生成 vpi
file_path_pv: str = entry_pv.Path
f_pv: fileio.File | t.CPtr = fileio.File(file_path_pv, fileio.MODE.R)
if f_pv.closed:
p1bpre_failed += 1
continue
src_buf_pv: bytes = stdlib.malloc(Phase1.SRC_BUF_SIZE)
if src_buf_pv is None:
f_pv.close()
p1bpre_failed += 1
continue
bytes_read_pv: LONG = f_pv.read_all(src_buf_pv, Phase1.SRC_BUF_SIZE)
f_pv.close()
if bytes_read_pv <= 0:
stdlib.free(src_buf_pv)
p1bpre_failed += 1
continue
if bytes_read_pv < Phase1.SRC_BUF_SIZE:
src_buf_pv[bytes_read_pv] = 0
else:
src_buf_pv[Phase1.SRC_BUF_SIZE - 1] = 0
lx_pv: ast.Lexer | t.CPtr = ast.new_lexer(mb)
if lx_pv is None:
stdlib.free(src_buf_pv)
p1bpre_failed += 1
continue
ast._lexer_init(lx_pv, src_buf_pv, mb)
tokens_pv: ast.Token | t.CPtr = ast.tokenize(lx_pv)
tree_pv: ast.AST | t.CPtr = ast.parse_tokens(mb, tokens_pv)
if tree_pv is None:
stdlib.free(src_buf_pv)
p1bpre_failed += 1
continue
pyi_buf_pv: bytes = stdlib.malloc(Phase1.PYI_BUF_SIZE)
if pyi_buf_pv is not None:
pyi_pos_pv: t.CSizeT = StubConverter._GeneratePyiFromAst(mb, tree_pv, entry_pv.RelPath, pyi_buf_pv, Phase1.PYI_BUF_SIZE)
if pyi_pos_pv > 0:
pyi_path_pv: str = StubMergerCollect._sliced_path(temp_dir, dir_len_pv, sha1_pv, "vpi")
if pyi_path_pv is not None:
pf_pv: fileio.File | t.CPtr = fileio.File(pyi_path_pv, fileio.MODE.W)
if not pf_pv.closed:
pf_pv.write(pyi_buf_pv, pyi_pos_pv)
pf_pv.close()
stdlib.free(pyi_path_pv)
stdlib.free(pyi_buf_pv)
else:
p1bpre_failed += 1
stdlib.free(src_buf_pv)
# 回收本文件 AST(vpi 已写盘,AST 不再需要;mark1 后无其他持久数据)
mb.reset_to(mark1)
if p1bpre_failed > 0:
fb_p1bpre: str = VLogger.fmt_buf()
if fb_p1bpre is not None:
viperlib.snprintf(fb_p1bpre, 1024, "Phase 1b-pre 有 %d 个文件 vpi 预生成失败", p1bpre_failed)
VLogger.error(fb_p1bpre, "Phase1")
return 1
# 遍历每个文件(拓扑序优先:父类所在文件先翻译,确保跨模块 vtable
# 生成时依赖模块的函数 TypeStr 已注册;topo_count==0 回退扫描顺序)
translated: int = 0
skipped: int = 0
failed: int = 0
loop_n_b: int = result.Count
if topo_count > 0:
loop_n_b = topo_count
for i in range(loop_n_b):
# 获取 entry(拓扑序:file_idx 从 topo_order 取,否则用扫描索引)
file_idx_b: int = i
if topo_count > 0:
od_addr_b: t.CUInt64T = t.CUInt64T(topo_order) + t.CSizeT(i) * 4
od_ptr_b: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(od_addr_b, t.CPtr))
if od_ptr_b is not None:
file_idx_b = od_ptr_b[0]
entry_addr: t.CUInt64T = t.CUInt64T(result.Entries) + t.CSizeT(file_idx_b) * entry_size
entry: IncludesScanner.FileEntry | t.CPtr = (IncludesScanner.FileEntry | t.CPtr)(t.CVoid(entry_addr, t.CPtr))
if entry is None:
failed += 1
continue
sha1: str = entry.Sha1
if sha1 is None:
failed += 1
continue
# 检查 SHA1 是否在翻译集合中(拓扑序已过滤可达;仅回退扫描顺序时检查)
in_set: int = 1
if topo_count == 0:
in_set = 0
if use_reachable != 0:
in_set = StubMergerCollect._is_in_sha1_set(sha1, reachable_set, reachable_count)
else:
in_set = StubMergerCollect._is_in_sha1_set(sha1, sha1_set, set_count)
if in_set == 0:
skipped += 1
continue
# 构造 stub 路径: {temp_dir}/{sha1}.stub.ll
dir_len: t.CSizeT = string.strlen(temp_dir)
sha1_len: t.CSizeT = string.strlen(sha1)
stub_path: str = StubMergerCollect._sliced_path(temp_dir, dir_len, sha1, "stub.ll")
if stub_path is None:
failed += 1
continue
# 检查 stub 是否已存在(按需翻译:跳过已存在的)
sf: fileio.File | t.CPtr = fileio.File(stub_path, fileio.MODE.R)
if not sf.closed:
sf.close()
# 检查 text.ll 是否也存在(虚表扫描需要 text.ll)
text_path: str = StubMergerCollect._sliced_path(temp_dir, dir_len, sha1, "text.ll")
if text_path is not None:
tf: fileio.File | t.CPtr = fileio.File(text_path, fileio.MODE.R)
if not tf.closed:
tf.close()
stdlib.free(text_path)
stdlib.free(stub_path)
continue
stdlib.free(text_path)
# text.ll 不存在,需要重新翻译
rp: str = entry.RelPath
fb: str = VLogger.fmt_buf()
if fb is not None:
viperlib.snprintf(fb, 1024, "text.ll 不存在,重新翻译: %s(sha1=%s)", rp, sha1)
VLogger.info(fb, "Phase1")
else:
# stub 不存在,需要翻译
rp: str = entry.RelPath
fb: str = VLogger.fmt_buf()
if fb is not None:
viperlib.snprintf(fb, 1024, "翻译: %s(sha1=%s)", rp, sha1)
VLogger.info(fb, "Phase1")
stdlib.free(stub_path)
# 读取文件内容
file_path: str = entry.Path
f: fileio.File | t.CPtr = fileio.File(file_path, fileio.MODE.R)
if f is None:
fb: str = VLogger.fmt_buf()
if fb is not None:
viperlib.snprintf(fb, 1024, "无法打开(None): %s", file_path)
VLogger.error(fb, "Phase1")
failed += 1
continue
if f.closed:
fb: str = VLogger.fmt_buf()
if fb is not None:
viperlib.snprintf(fb, 1024, "无法打开: %s", file_path)
VLogger.error(fb, "Phase1")
failed += 1
continue
src_buf: bytes = stdlib.malloc(Phase1.SRC_BUF_SIZE)
if src_buf is None:
f.close()
failed += 1
continue
bytes_read: LONG = f.read_all(src_buf, Phase1.SRC_BUF_SIZE)
f.close()
if bytes_read <= 0:
fb: str = VLogger.fmt_buf()
if fb is not None:
viperlib.snprintf(fb, 1024, "读取失败: %s", file_path)
VLogger.error(fb, "Phase1")
stdlib.free(src_buf)
failed += 1
continue
if bytes_read < Phase1.SRC_BUF_SIZE:
src_buf[bytes_read] = 0
else:
src_buf[Phase1.SRC_BUF_SIZE - 1] = 0
# 解析 AST
lx: ast.Lexer | t.CPtr = ast.new_lexer(mb)
if lx is None:
stdlib.free(src_buf)
failed += 1
continue
ast._lexer_init(lx, src_buf, mb)
tokens: ast.Token | t.CPtr = ast.tokenize(lx)
tree: ast.AST | t.CPtr = ast.parse_tokens(mb, tokens)
if tree is None:
fb: str = VLogger.fmt_buf()
if fb is not None:
viperlib.snprintf(fb, 1024, "AST 解析失败: %s", file_path)
VLogger.error(fb, "Phase1")
stdlib.free(src_buf)
failed += 1
continue
# 生成 .vpi 存根文件(直接遍历 AST,不依赖 PythonToStubConverter)
pyi_buf: bytes = stdlib.malloc(Phase1.PYI_BUF_SIZE)
if pyi_buf is not None:
pyi_pos: t.CSizeT = StubConverter._GeneratePyiFromAst(mb, tree, entry.RelPath, pyi_buf, Phase1.PYI_BUF_SIZE)
if pyi_pos > 0:
pyi_path: str = StubMergerCollect._sliced_path(temp_dir, dir_len, sha1, "vpi")
if pyi_path is not None:
pf: fileio.File | t.CPtr = fileio.File(pyi_path, fileio.MODE.W)
if not pf.closed:
pf.write(pyi_buf, pyi_pos)
pf.close()
stdlib.free(pyi_path)
stdlib.free(pyi_buf)
# 翻译 AST → LLVM IR
tr: HandlesTranslator.Translator | t.CPtr = HandlesTranslator.Translator()
if tr is None:
stdlib.free(src_buf)
failed += 1
continue
tr.ModuleSha1 = sha1
tr.CurrentPackage = HandlesImports.compute_package_from_relpath(mb, entry.RelPath)
HandlesType.set_current_file(file_path)
HandlesImports.set_current_file(file_path)
HandlesType.set_current_module_sha1(sha1)
HandlesType.clear_cdefine_constants()
HandlesType.clear_ctypedefs()
HandlesStruct.reset_visible_structs(mb, 0)
# 翻译前哨兵:AST 解析(lexer/tokenize/parse_tokens)同样在 pool 分配,
# 若解析器越界写破坏了存活块 header,此处立即定位到当前文件
# (崩溃点漂移排查:把"翻译过程中随机崩"变成确定性定位)
pre_rc: t.CInt = mb.check_alive_blocks(mark1)
if pre_rc != 0:
fb: str = VLogger.fmt_buf()
if fb is not None:
viperlib.snprintf(fb, 1024,
"内存损坏(翻译前存活块, rc=%d): %s", pre_rc, file_path)
VLogger.error(fb, "Phase1")
stdlib.exit(1)
ret: int = tr.translate(tree)
# 诊断:translate 正常返回(若挂起在 translate 内部,此行不会打印,
# 用于区分"translate 内部挂起"与"后续 dump 挂起")
if ret == 0:
fb_done: str = VLogger.fmt_buf()
if fb_done is not None:
viperlib.snprintf(fb_done, 1024, "翻译完成(translate返回): %s", file_path)
VLogger.info(fb_done, "Phase1")
if ret != 0:
fb: str = VLogger.fmt_buf()
if fb is not None:
viperlib.snprintf(fb, 1024, "翻译失败: %s", file_path)
VLogger.error(fb, "Phase1")
stdlib.free(src_buf)
if tr._global_names is not None:
stdlib.free(tr._global_names)
if tr._nonlocal_names is not None:
stdlib.free(tr._nonlocal_names)
failed += 1
continue
# dump stub IR(declarations only)
PHASE1_IR_SIZE: t.CSizeT = 1048576
stub_buf: bytes = stdlib.malloc(PHASE1_IR_SIZE)
if stub_buf is None:
stdlib.free(src_buf)
if tr._global_names is not None:
stdlib.free(tr._global_names)
if tr._nonlocal_names is not None:
stdlib.free(tr._nonlocal_names)
failed += 1
continue
tr.dump_ir(stub_buf, PHASE1_IR_SIZE, llvmlite.OUTPUT_STUB)
stub_len: t.CSizeT = string.strlen(stub_buf)
# save stub.ll
dir_len_p1: t.CSizeT = string.strlen(temp_dir)
stub_path_p1: str = StubMergerCollect._sliced_path(temp_dir, dir_len_p1, sha1, "stub.ll")
if stub_path_p1 is not None:
sf_p1: fileio.File | t.CPtr = fileio.File(stub_path_p1, fileio.MODE.W)
if not sf_p1.closed:
sf_p1.write(stub_buf, stub_len)
sf_p1.close()
stdlib.free(stub_path_p1)
stdlib.free(stub_buf)
# dump text IR(definitions only)
text_buf: bytes = stdlib.malloc(PHASE1_IR_SIZE)
if text_buf is None:
stdlib.free(src_buf)
if tr._global_names is not None:
stdlib.free(tr._global_names)
if tr._nonlocal_names is not None:
stdlib.free(tr._nonlocal_names)
failed += 1
continue
tr.dump_ir(text_buf, PHASE1_IR_SIZE, llvmlite.OUTPUT_TEXT)
text_len: t.CSizeT = string.strlen(text_buf)
# save text.ll
text_path_p1: str = StubMergerCollect._sliced_path(temp_dir, dir_len_p1, sha1, "text.ll")
if text_path_p1 is not None:
tf_p1: fileio.File | t.CPtr = fileio.File(text_path_p1, fileio.MODE.W)
if not tf_p1.closed:
tf_p1.write(text_buf, text_len)
tf_p1.close()
stdlib.free(text_path_p1)
stdlib.free(text_buf)
# save dependencies(_imported_modules) for Phase B
deps_path_p1: str = StubMergerCollect._sliced_path(temp_dir, dir_len_p1, sha1, "deps.txt")
if deps_path_p1 is not None:
df_p1: fileio.File | t.CPtr = fileio.File(deps_path_p1, fileio.MODE.W)
if not df_p1.closed:
if tr._imported_modules is not None:
dl: t.CSizeT = string.strlen(tr._imported_modules)
df_p1.write(tr._imported_modules, dl)
df_p1.close()
stdlib.free(deps_path_p1)
# 释放 Translator 的 C malloc 资源
if tr._global_names is not None:
stdlib.free(tr._global_names)
if tr._nonlocal_names is not None:
stdlib.free(tr._nonlocal_names)
# 所有 dump_ir 完成,src_buf 不再需要
stdlib.free(src_buf)
translated += 1
# OOM 回收:回收本文件翻译产生的瞬时数据(AST/tokens/lexer/IR/Handler/dump 临时)
# 持久表(struct/enum/REnum/spec 缓存/全局函数名等)已迁出 reset 区间(stdlib),
# reset_to(mark1) 只回收 mark1 之后分配、未被单独 free 的块。
#
# 内存哨兵(崩溃点漂移排查):reset 前检查存活块 header(抓越界写),
# reset 后检查 free list 完整性(抓链表损坏)。任一损坏立即终止,
# 把"随机崩溃漂移"变成"确定性定位到当前文件"。
# rc 语义(check_alive_blocks):1=块地址低于 usable,2=高于 usable+size,
# 3=order 为负,4=order 超 max_order,5=块未对齐到 order 大小,
# 6=块尾 canary 损坏(越界写穿数据区——事发地定位)
# rc 语义(self_check):1/2=越界,3=对齐错,4=free list 成环/超步数
chk_rc: t.CInt = mb.check_alive_blocks(mark1)
if chk_rc != 0:
fb: str = VLogger.fmt_buf()
if fb is not None:
viperlib.snprintf(fb, 1024,
"内存损坏(存活块 header, rc=%d): %s", chk_rc, file_path)
VLogger.error(fb, "Phase1")
stdlib.exit(1)
# A/B 实验结论(已确认):禁用 reset_to 后 stdarg 仍崩 → 越界写是根因,
# 与回收/UAF 无关。恢复 reset_to,改用块尾 canary(check_alive_blocks 校验)
# 定位越界写事发地到具体文件。
mb.reset_to(mark1)
sc_rc: t.CInt = mb.self_check()
if sc_rc != 0:
fb: str = VLogger.fmt_buf()
if fb is not None:
viperlib.snprintf(fb, 1024,
"内存损坏(free list, rc=%d): %s", sc_rc, file_path)
VLogger.error(fb, "Phase1")
stdlib.exit(1)
# 释放 Phase 1b 使用的拓扑排序资源
if topo_order is not None:
mb.free(topo_order)
if topo_count_box is not None:
mb.free(topo_count_box)
fb: str = VLogger.fmt_buf()
if fb is not None:
viperlib.snprintf(fb, 1024, "完成: 翻译=%d 跳过=%d 失败=%d", translated, skipped, failed)
VLogger.info(fb, "Phase1")
if failed > 0:
fb_fail: str = VLogger.fmt_buf()
if fb_fail is not None:
viperlib.snprintf(fb_fail, 1024, "Phase1 有 %d 个文件翻译失败", failed)
VLogger.error(fb_fail, "Phase1")
return 1
# 释放可达 SHA1 集合(如果分配了)
if reachable_set is not None:
stdlib.free(reachable_set)
return 0