Files
ViperC/App/lib/Phase/Phase1/Phase1Main.vp
T
TermiNexus d82f24e17a fix(diag): AST 解析失败日志行内联 行/列 + 原因
四处解析失败上报点(Phase2 的 [Phase A-pre-vpi]、Phase1 的 [Phase 1b-pre] 与单文件、main.vp 单文件)此前只打 AST 解析失败,看不到定位与原因;现内联 第 N 行第 M 列: <原因>,与 dump_parse_traceback 的完整 traceback 同源(原因由 includes/ast/parser.vp 的 _record_* 与 parse_tokens 出口兜底补记提供)。另:App/main.vp 同时带上你工作区既有的 define 迁移/版本号改动。
2026-09-27 17:32:34 +08:00

1194 lines
58 KiB
Plaintext
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import t, c
from stdint import *
import string
import stdlib
import memhub
import viperlib
import w32.fileio as fileio
import w32.win32base
import ast
import llvmlite
import lib.core.VLogger as VLogger
import lib.core.Gen.Translator as HandlesTranslator
import lib.core.IncludesScanner as IncludesScanner
import lib.Stub.StubMerger as StubMerger
import lib.core.Gen.Generator as GenGenerator
import lib.core.CompilerFactory as CompilerFactory
import lib.Phase.BuildPipeline as BuildPipeline
import lib.Projectrans.Config as Config
import lib.Stub.StubGen.Converter as StubConverter
import lib.Stub.StubMerger.StubMergerBuild as StubMergerBuild
import lib.Stub.StubMerger.StubMergerCollect as StubMergerCollect
import lib.Stub.StubMerger.StubMergerIR as StubMergerIR
import lib.Stub.StubMerger.StubMergerStore as StubMergerStore
import lib.Phase.Phase1.Phase1Scan as Phase1Scan
import lib.Phase.Phase1.Phase1Topo as Phase1Topo
import lib.Phase.Phase1 as Phase1
import lib.core.Gen.GenApi as GenApi
import lib.core.Gen.DebugInfoGen as DebugInfoGen
# ============================================================
# _InjectedCreatesCycle - 容器模块自动注入前的破环检查
#
# 注入 _list/_dict 前检查被注入模块自身的依赖是否已含当前模块,命中(如 memhub↔_list)则跳过注入——
# 否则 Phase A 多遍收敛死锁("无法完成翻译");去掉环边后 _list 仍因当前模块被其依赖而处于闭包,list 类仍由 Phase1b 注册。
# ============================================================
def _InjectedCreatesCycle(entry: rawptr[IncludesScanner.FileEntry],
mod_name: str) -> int:
"""注入 mod_name 到当前模块前检查是否成环,返回 1=成环需跳过注入"""
if entry is None or entry.RelPath is None or mod_name is None:
return 0
inj_sha1: str = StubMergerStore.LookupSha1ByModName(mod_name)
if inj_sha1 is None:
return 0
inj_deps: str = StubMergerStore.LookupIncludesDeps(inj_sha1)
if inj_deps is None:
return 0
rp: str = entry.RelPath
# 当前模块名 = RelPath 基名(最后 '/' 或 '\\' 之后)去扩展名
base: str = string.strrchr(rp, '/')
if base is None:
base = string.strrchr(rp, '\\')
if base is not None:
base = base + 1
else:
base = rp
dot: str = string.strchr(base, '.')
nm_buf: bytes = stdlib.malloc(128)
if nm_buf is None:
return 0
if dot is not None:
nm_len: sizet = sizet(uint64(dot) - uint64(base))
if nm_len > 127:
nm_len = 127
string.strncpy(nm_buf, base, nm_len)
nm_buf[nm_len] = '\0'
else:
string.strncpy(nm_buf, base, 127)
nm_buf[127] = '\0'
nm_len2: sizet = string.strlen(nm_buf)
d_len: sizet = string.strlen(inj_deps)
dp: sizet = 0
while dp < d_len:
ts: sizet = dp
while dp < d_len and inj_deps[dp] != ' ':
dp += 1
tl: sizet = dp - ts
if dp < d_len:
dp += 1
if tl > 0 and tl == nm_len2 and string.strncmp(inj_deps + ts, nm_buf, tl) == 0:
stdlib.free(nm_buf)
return 1
stdlib.free(nm_buf)
return 0
def _DepsContainsName(deps: str, name: str) -> int:
"""deps 字符串(空格分隔模块名)是否含 name token(词边界精确匹配)"""
if deps is None or name is None:
return 0
nl: sizet = string.strlen(name)
dl: sizet = string.strlen(deps)
dp: sizet = 0
while dp < dl:
ts: sizet = dp
while dp < dl and deps[dp] != ' ':
dp += 1
tl: sizet = dp - ts
if dp < dl:
dp += 1
if tl == nl and string.strncmp(deps + ts, name, nl) == 0:
return 1
return 0
def _DepsRemoveName(deps: str, name: str) -> int:
"""从 deps 字符串移除 name token(原地重建,返回 1=有改动)"""
if deps is None or name is None:
return 0
nl: sizet = string.strlen(name)
dl: sizet = string.strlen(deps)
buf: bytes = stdlib.malloc(StubMerger.MAX_DEPS_STR_LEN)
if buf is None:
return 0
buf[0] = '\0'
bl: sizet = 0
dp: sizet = 0
while dp < dl:
ts: sizet = dp
while dp < dl and deps[dp] != ' ':
dp += 1
tl: sizet = dp - ts
if dp < dl:
dp += 1
if tl == 0:
continue
if tl == nl and string.strncmp(deps + ts, name, nl) == 0:
continue
if bl > 0:
buf[bl] = ' '
bl += 1
string.strncpy(buf + bl, deps + ts, tl)
bl += tl
buf[bl] = '\0'
string.strcpy(deps, buf)
stdlib.free(buf)
return 1
def _RewriteDepsTxt(temp_dir: str, sha1: str, deps_str: str) -> None:
"""重写 {temp_dir}/{sha1前2位}/{sha1}.deps.txt(内存破环后同步磁盘)"""
if temp_dir is None or sha1 is None or deps_str is None:
return
td_len: sizet = string.strlen(temp_dir)
dpath: str = StubMergerCollect._sliced_path(temp_dir, td_len, sha1, "deps.txt")
if dpath is None:
return
df: rawptr[fileio.File] = fileio.File(dpath, fileio.MODE.W)
if not df.closed:
dl: sizet = string.strlen(deps_str)
if dl > 0:
df.write(deps_str, dl)
df.close()
stdlib.free(dpath)
def _BreakContainerCyclesInDeps(temp_dir: str) -> int:
"""扫描后统一清理 _list/_dict 注入环边。
Phase1a-pre 全量扫描用 FindFirstFileA(文件系统序),注入 _list/_dict 时
被注入模块(_list/_dict)自身的 deps 未必已注册 → _InjectedCreatesCycle
内联检查可能漏判(如 _list 显式 import memhub,而 memhub 的 list[str]
注入 _list 未被拦截)→ memhub↔_list 2-环 → Phase A 永不收敛。此处扫描
结束后全量检查:凡某模块 M 的 deps 含 _list/_dict,且 _list/_dict 的 deps
含 M → 从 M 的 deps 移除注入边(list 类仍由 Phase1b 经闭包注册)。
内存破环后同步重写 {temp_dir}/deps.txt,保证磁盘与内存一致(Phase2 依赖
图诊断回退读 deps.txt)。
"""
if StubMerger._includes_deps_sha1 is None or StubMerger._includes_deps_str is None:
return 0
cnt: sizet = StubMerger._includes_deps_count
ci: sizet
for ci in range(cnt):
sha1_m: str = StubMerger._includes_deps_sha1 + sizet(ci) * 17
deps_m: str = StubMerger._includes_deps_str + sizet(ci) * StubMerger.MAX_DEPS_STR_LEN
if sha1_m is None or deps_m is None:
continue
if string.strstr(deps_m, "_list") is None and string.strstr(deps_m, "_dict") is None:
continue
# 该模块名(RelPath 基名去扩展名)
rel_m: str = StubMergerStore.LookupSha1RelPath(sha1_m)
if rel_m is None:
continue
base_m: str = string.strrchr(rel_m, '/')
if base_m is None:
base_m = string.strrchr(rel_m, '\\')
if base_m is not None:
base_m = base_m + 1
else:
base_m = rel_m
dot_m: str = string.strchr(base_m, '.')
if dot_m is not None:
nm_len_m: sizet = sizet(uint64(dot_m) - uint64(base_m))
nm_m: bytes = stdlib.malloc(nm_len_m + 1)
if nm_m is None:
continue
string.strncpy(nm_m, base_m, nm_len_m)
nm_m[nm_len_m] = '\0'
else:
nm_m: bytes = stdlib.malloc(129)
if nm_m is None:
continue
string.strncpy(nm_m, base_m, 128)
nm_m[128] = '\0'
# _list ↔ M 环
if string.strstr(deps_m, "_list") is not None:
lst_sha1: str = StubMergerStore.LookupSha1ByModName("_list")
if lst_sha1 is not None:
lst_deps: str = StubMergerStore.LookupIncludesDeps(lst_sha1)
if lst_deps is not None and _DepsContainsName(lst_deps, nm_m) != 0:
if _DepsRemoveName(deps_m, "_list") != 0:
# 内存破环后同步重写 deps.txt(诊断回退路径读文件)
_RewriteDepsTxt(temp_dir, sha1_m, deps_m)
# _dict ↔ M 环
if string.strstr(deps_m, "_dict") is not None:
dct_sha1: str = StubMergerStore.LookupSha1ByModName("_dict")
if dct_sha1 is not None:
dct_deps: str = StubMergerStore.LookupIncludesDeps(dct_sha1)
if dct_deps is not None and _DepsContainsName(dct_deps, nm_m) != 0:
if _DepsRemoveName(deps_m, "_dict") != 0:
_RewriteDepsTxt(temp_dir, sha1_m, deps_m)
stdlib.free(nm_m)
return 0
# ============================================================
# _BuildReachableResult - 从 Sha1Store 构建可达 FileEntry 列表
#
# 依赖图按需扫描(StubMergerBuild._BuildReachableSha1Set)只把「可达 includes + App」登记到全局 Sha1Store;
# 本函数把 incdir 非空条目(includes)重建为 Phase1 1a/1b-pre/1b 需要的 FileEntry 列表。
# App 条目(incdir 空)不进入 result——Phase B 已编译其 .obj,加入会导致 Phase B+ 重复编译 → 重复符号。
#
# Args: mb 内存池;includes_dir includes 根目录(仅用于失败诊断,路径以 store 为准)
# Returns: ScanResult 指针(stdlib 分配,调用方负责释放);失败返回 None
# ============================================================
def _BuildReachableResult(mb: memhub.MemBuddy*,
includes_dir: str) -> rawptr[IncludesScanner.ScanResult]:
"""从 Sha1Store(仅可达 includes + App)构建可达 FileEntry 列表"""
if mb is None:
return None
store_count: sizet = sizet(StubMerger._sha1_store_count)
if store_count <= 0 or StubMerger._sha1_store_arr is None:
return None
result: rawptr[IncludesScanner.ScanResult] = IncludesScanner.create_scan_result(mb)
if result is None:
return None
entry_size: sizet = IncludesScanner.FileEntry.__sizeof__()
i: sizet
for i in range(store_count):
sidx: sizet = sizet(i) * 17
if StubMerger._sha1_store_arr[sidx] == '\0':
continue
# 仅 includes 条目(incdir 非空);App 条目不进入 result
iidx: sizet = sizet(i) * StubMerger.MAX_INC_DIR_LEN
if StubMerger._sha1_store_incdir[iidx] == '\0':
continue
eaddr: uint64 = uint64(result.Entries) + uint64(result.Count) * entry_size
entry: rawptr[IncludesScanner.FileEntry] = rawptr[IncludesScanner.FileEntry](eaddr)
if entry is None:
continue
# SHA1 深拷贝(store 槽 17 字节)
sha1_buf: bytes = stdlib.malloc(17)
if sha1_buf is None:
continue
string.strcpy(sha1_buf, StubMerger._sha1_store_arr + sidx)
entry.Sha1 = sha1_buf
# RelPath / IncDir 深拷贝
ridx: sizet = sizet(i) * StubMerger.MAX_REL_PATH_LEN
rel_len: sizet = string.strlen(StubMerger._sha1_store_rel + ridx)
inc_len: sizet = string.strlen(StubMerger._sha1_store_incdir + iidx)
rel_buf: bytes = stdlib.malloc(rel_len + 1)
inc_buf: bytes = stdlib.malloc(inc_len + 1)
if rel_buf is None or inc_buf is None:
if rel_buf is not None:
stdlib.free(rel_buf)
if inc_buf is not None:
stdlib.free(inc_buf)
stdlib.free(sha1_buf)
continue
string.strcpy(rel_buf, StubMerger._sha1_store_rel + ridx)
string.strcpy(inc_buf, StubMerger._sha1_store_incdir + iidx)
entry.RelPath = rel_buf
entry.IncDir = inc_buf
# 绝对路径 = inc_dir + '/' + rel_path(与 _TryResolveModuleFile 拼接一致)
abs_buf: bytes = stdlib.malloc(inc_len + 1 + rel_len + 1)
if abs_buf is None:
stdlib.free(sha1_buf)
stdlib.free(rel_buf)
stdlib.free(inc_buf)
continue
string.strcpy(abs_buf, inc_buf)
abs_buf[inc_len] = '/'
string.strcpy(abs_buf + inc_len + 1, rel_buf)
entry.Path = abs_buf
# 模块名(store 登记时保存的绝对模块名)
midx: sizet = sizet(i) * 64
mod_len: sizet = string.strlen(StubMerger._sha1_store_mod + midx)
if mod_len > 0:
mod_buf: bytes = stdlib.malloc(mod_len + 1)
if mod_buf is not None:
string.strcpy(mod_buf, StubMerger._sha1_store_mod + midx)
entry.ModuleName = mod_buf
result.Count += 1
return result
# ============================================================
# Phase1Main - Phase1 主流程(扫描 → 预注册 → 1a/1b 翻译 → 生成 stub)
#
# 函数实现直接位于本模块,调用方经本模块前缀访问;全局符号经 Phase1._xxx 跨模块访问。
# ============================================================
# ============================================================
# RunPhase1 - Phase1: 扫描 includes 目录,按需翻译并生成 stub
#
# 对每个 .vp/.py 文件检查 stub 是否已存在,不存在才翻译并分离 stub(避免重复工作)。
#
# Args: mb 内存池;includes_dir includes 目录路径;temp_dir 临时目录(保存 stub);log 日志器
# Returns: 0 成功,非 0 失败
# ============================================================
def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str,
log: rawptr[VLogger.Logger]) -> int:
"""Phase1: 扫描 includes 目录,按需翻译并生成 stub"""
if includes_dir is None or temp_dir is None:
VLogger.error("includes_dir 或 temp_dir 为空,无法运行 Phase1", "Phase1")
return 1
# 设置全局 temp_dir(供跨模块 CDefine 查找使用)
GenApi.set_temp_dir(temp_dir)
# 同步设置 HandlesExprCall 的 temp_dir(供 vpi stub 返回注解缓存按需加载)。
# 必须在 includes 翻译前设置:Phase1 翻译 includes 时「同模块方法调用(定义在后)」需读 {temp_dir}/{sha1[0:2]}/{sha1}.vpi 兜底,Phase2 的 SetTempDir 太晚。
GenApi.SetTempDir(temp_dir)
# 确保 temp 目录存在(build_dir/temp 可能尚未创建)
BuildPipeline.ensure_dir(temp_dir)
log.banner("Phase1: 扫描 includes(按需翻译)")
# 初始化 AST 表(只需一次):必须在依赖图 BFS 扫描之前调用。
# _init_tables 懒分配的关键字/操作符表(全局 _kw_head 等)若首次触发在 BFS 的文件级 mark 之后,reset_to 会回收表节点 → 全局悬空 → 后续 tokenize 崩溃。
ast._init_tables(mb)
# 依赖图按需扫描(App 优先模型):先扫 App 源文件(Config.SourceDir)的 import 依赖作为初始队列,
# BFS 按文件系统解析 includes 文件(只对可达者读文件+算 SHA1+解析 import)并登记 Sha1Store/deps/写 deps.txt;
# 未引用的 includes 从不读取。
t_scan0: uint64 = w32.win32base.GetTickCount64()
reachable_set: str = None
reachable_count: int = 0
use_reachable: int = 0
if Config.SourceDir is not None:
reachable_set = stdlib.malloc(StubMerger.MAX_INCLUDES_SHA1 * 17)
if reachable_set is not None:
string.memset(reachable_set, 0, StubMerger.MAX_INCLUDES_SHA1 * 17)
reachable_count = StubMergerBuild._BuildReachableSha1Set(
mb, Config.SourceDir, temp_dir, reachable_set, None, includes_dir)
if reachable_count < 0:
VLogger.error("可达集合构建失败(依赖图扫描失败),终止编译", "Phase1")
return 1
if reachable_count <= 0:
VLogger.warning("App 无可达 includes 依赖,跳过 Phase1", "Phase1")
stdlib.free(reachable_set)
reachable_set = None
return 0
use_reachable = 1
else:
VLogger.error("reachable_set 分配失败,终止编译", "Phase1")
return 1
else:
VLogger.warning("SourceDir 不可用,无法构建依赖图,跳过 Phase1", "Phase1")
return 0
t_scan1: uint64 = w32.win32base.GetTickCount64()
VLogger.info(f"依赖图扫描总耗时: {t_scan1 - t_scan0} ms(可达 {reachable_count} 个 includes)", "Phase1")
# 从 Sha1Store(仅可达 includes + App)构造可达 FileEntry 列表:
# 后续 1a/1b-pre/1b 直接遍历 result(incdir 非空条目=includes),
# use_reachable 过滤幂等保留(result 即可达子集)
result: rawptr[IncludesScanner.ScanResult] = _BuildReachableResult(mb, includes_dir)
if result is None:
VLogger.error("可达 FileEntry 列表构建失败,终止编译", "Phase1")
return 1
VLogger.info(f"可达 includes 条目: {result.Count}", "Phase1")
# 扫描后统一清理 _list/_dict 注入环边(_list↔memhub 等 2-环)。
# 依赖图按需扫描顺序不可靠,内联检查可能漏判,此处全量清理保证
# 依赖图无注入环,Phase A 多遍可收敛。破环后同步重写 deps.txt。
_BreakContainerCyclesInDeps(temp_dir)
# 写 _sha1_map.txt(仅可达 includes,人类可读输出;机器分析使用内存存储器)
StubMergerCollect.WriteIncludesSha1Map(mb, temp_dir, result, reachable_set, reachable_count)
# 兼容旧引用:sha1_set/set_count 对齐可达集合(use_reachable=1 时不被使用)
sha1_set: str = reachable_set
set_count: int = reachable_count
# 模块 SHA1 映射已由依赖图扫描登记到 StubMerger 全局存储器(仅可达 includes + App),
# HandlesExprCall._lookup_module_sha1 直接查全局存储器。
# FileEntry 结构体大小(Phase 1a 和 1b 共用)
entry_size: sizet = IncludesScanner.FileEntry.__sizeof__()
# ============================================================
# Phase 1a: 注册可达 includes 文件的 struct/enum/union
#
# 解决字母序依赖问题:1a 先把所有 struct/enum/union 注册到全局表,1b 全量翻译时 struct 已注册、走 existing 路径只翻译方法体;
# 只处理可达文件;处理前按类继承关系拓扑排序,确保父类所在文件先于子类所在文件。
# ============================================================
p1a_reg: int = 0
p1a_skp: int = 0
p1a_fl: int = 0
# ============================================================
# [临时取证打点] Phase1 早期扁平缓冲分配失败现场快照(定位后清理);仅只读 mb 状态 + 打印,不改变任何分配行为与逻辑。
# 三维度判定:① OOM(dump_stats 显示 used≈total、free≈0);② mb 损坏(usable 基址/大小荒谬、self_check != 0);③ 本次请求 size 算出的 order > max_order。
# 用 VLogger.info(非终止;VLogger.error 会 sys.exit(1) 掩盖后续打点)。
# ============================================================
if mb is not None:
# 先做带环检测的 self_check:free list 被写坏成环时 dump_stats 的
# free_bytes/free_count 无步数上限会死循环(打点自身卡死),故
# self_check 返回 4(成环)时跳过 dump_stats。
p1_sc: int = mb.self_check()
if p1_sc != 4:
mb.dump_stats("phase1-pre")
else:
VLogger.info("[取证] self_check=4(free list 成环),跳过 dump_stats 防死循环", "Phase1")
p1_mo: int = mb.max_order
if p1_mo < 0 or p1_mo > 32:
p1_mo = 32 # 钳制:max_order 被写坏时防本打点自身越界读 free_lists
p1_fl_head: uint64 = uint64(mb.free_lists[p1_mo])
p1_mb_addr: uint64 = uint64(mb)
p1_base_addr: uint64 = uint64(mb.base)
p1_usable_addr: uint64 = uint64(mb.usable)
p1_chkbase_addr: uint64 = uint64(mb.chk_base)
VLogger.info(
f"[取证] mb={p1_mb_addr} base={p1_base_addr} usable={p1_usable_addr} "
f"usable_size={mb.usable_size} max_order={mb.max_order} self_check={p1_sc} "
f"detection={mb.detection}", "Phase1")
VLogger.info(
f"[取证] chk_base={p1_chkbase_addr} chk_depth={mb.chk_depth} "
f"chk_cap={mb.chk_cap} free_lists[clamp={p1_mo}]={p1_fl_head}",
"Phase1")
p1_sz_cn: sizet = sizet(Phase1.MAX_CLASSES * Phase1.CLASS_NAME_LEN)
p1_sz_ds: sizet = sizet(Phase1.MAX_CLASSES * Phase1.SHA1_LEN)
p1_sz_tp: sizet = sizet(4 * Phase1.MAX_FILE_DEPS)
p1_sz_bx: sizet = sizet(4)
p1_o_cn: int = mb._order_for_size(p1_sz_cn + 16)
p1_o_ds: int = mb._order_for_size(p1_sz_ds + 16)
p1_o_tp: int = mb._order_for_size(p1_sz_tp + 16)
p1_o_bx: int = mb._order_for_size(p1_sz_bx + 16)
p1_fl_cn: uint64 = uint64(mb.free_lists[p1_o_cn])
p1_fl_ds: uint64 = uint64(mb.free_lists[p1_o_ds])
VLogger.info(
f"[取证] order(class_names {p1_sz_cn}B+16)={p1_o_cn} fl={p1_fl_cn} "
f"order(def_sha1s {p1_sz_ds}B+16)={p1_o_ds} fl={p1_fl_ds} "
f"order(topo {p1_sz_tp}B+16)={p1_o_tp} order(box {p1_sz_bx}B+16)={p1_o_bx} "
f"(fl==0 只说明该阶空、仍可由更高阶拆分;若 alloc 失败而 dump_stats 的 free>0 才是元数据损坏铁证,"
f"任一 order>max_order={mb.max_order} 则是 size 异常)",
"Phase1")
# 预扫描类继承关系,构建拓扑顺序(用扁平字节数组,避免结构体 str 指针 NULL 崩溃)
class_names_buf: bytes = mb.alloc(Phase1.MAX_CLASSES * Phase1.CLASS_NAME_LEN)
parent_names_buf: bytes = mb.alloc(Phase1.MAX_CLASSES * Phase1.CLASS_NAME_LEN)
def_sha1s_buf: bytes = mb.alloc(Phase1.MAX_CLASSES * Phase1.SHA1_LEN)
topo_order: t.CPtr = mb.alloc(4 * Phase1.MAX_FILE_DEPS)
topo_count_box: t.CPtr = mb.alloc(4)
generic_file_indices: t.CPtr = mb.alloc(4 * Phase1.MAX_FILE_DEPS)
generic_file_count_box: t.CPtr = mb.alloc(4)
if generic_file_indices is not None:
string.memset(generic_file_indices, 0, 4 * Phase1.MAX_FILE_DEPS)
if generic_file_count_box is not None:
string.memset(generic_file_count_box, 0, 4)
topo_count: int = 0
# 三块扁平缓冲是 Phase1Scan._ScanClassInheritance 的契约前置缓冲:该函数入口对 NULL fail-fast(静默跳过整段会退化为字母序并掩盖 OOM 等内存故障),
# 故分配失败时逐块显式报错终止(VLogger.error 内部 sys.exit(1)),绝不静默回退。
# 注:topo_order/topo_count_box 仅供 Phase1Topo._TopoSortFiles 使用,有效性由其入口自身校验(NULL 即返回 -1 并终止),不并入此处终止逻辑。
if class_names_buf is None:
VLogger.error("类继承预扫描扁平缓冲分配失败:class_names_buf 为 NULL,终止编译", "Phase1")
if parent_names_buf is None:
VLogger.error("类继承预扫描扁平缓冲分配失败:parent_names_buf 为 NULL,终止编译", "Phase1")
if def_sha1s_buf is None:
VLogger.error("类继承预扫描扁平缓冲分配失败:def_sha1s_buf 为 NULL,终止编译", "Phase1")
string.memset(class_names_buf, 0, Phase1.MAX_CLASSES * Phase1.CLASS_NAME_LEN)
string.memset(parent_names_buf, 0, Phase1.MAX_CLASSES * Phase1.CLASS_NAME_LEN)
string.memset(def_sha1s_buf, 0, Phase1.MAX_CLASSES * Phase1.SHA1_LEN)
if topo_order is not None:
string.memset(topo_order, 0, 4 * Phase1.MAX_FILE_DEPS)
if topo_count_box is not None:
string.memset(topo_count_box, 0, 4)
if generic_file_indices is not None:
string.memset(generic_file_indices, 0, 4 * Phase1.MAX_FILE_DEPS)
if generic_file_count_box is not None:
string.memset(generic_file_count_box, 0, 4)
class_count_scanned: int = Phase1Scan._ScanClassInheritance(
mb, result, reachable_set, reachable_count,
use_reachable, sha1_set, set_count,
class_names_buf, parent_names_buf, def_sha1s_buf,
generic_file_indices, generic_file_count_box)
if class_count_scanned > 0:
td_len_ts_call: sizet = string.strlen(temp_dir)
ret_topo: int = Phase1Topo._TopoSortFiles(
mb, result, reachable_set, reachable_count,
use_reachable, sha1_set, set_count,
class_names_buf, parent_names_buf, def_sha1s_buf, class_count_scanned,
topo_order, topo_count_box,
temp_dir, td_len_ts_call)
if ret_topo == 0:
tc_ptr: t.CPtr = (int | t.CPtr)(topo_count_box)
if tc_ptr is not None:
topo_count = tc_ptr[0]
VLogger.info(f"拓扑排序: {class_count_scanned} 个类, {topo_count} 个文件按依赖顺序处理", "Phase1")
else:
VLogger.error("拓扑排序失败(类继承依赖无法解析),终止编译", "Phase1")
return 1
else:
VLogger.warning("类继承预扫描无结果,回退到字母序", "Phase1")
topo_count = 0
# ============================================================
# Phase 1a-pre-generic-all: mark1 前全量预注册 includes 泛型模板
#
# 非可达泛型文件(_list/_dict/linkedlist/llvmlite 等)的模板若在 Phase2(mark2 之后)才注册会被 reset 回收 → list[int]/dict[...] 特化失败。
# 故对全部 includes 轻量预注册,使模板表与节点都落在 mark1 之前(_PreRegisterAllGenericTemplates 内有 generic marker 文本预筛,不 parse 全部 AST,OOM 风险可控)。
full_scan_g: rawptr[IncludesScanner.ScanResult] = IncludesScanner.scan_includes(mb, includes_dir)
if full_scan_g is not None:
Phase1Scan._PreRegisterAllGenericTemplates(mb, full_scan_g)
else:
Phase1Scan._PreRegisterAllGenericTemplates(mb, result)
# ============================================================
# Phase 1a: 注册 struct/enum/union(mark1 前,全量 includes)
#
# OOM 治理:1a 对全部 includes parse 完整 AST,若驻留到 mark1 会累积至 1GB 主池耗尽。故:
# 1) StdlibAlloc=1:1a 注册的跨文件持久数据(StructTable 表、Fields/Name/ModuleSha1、struct_ty 深拷贝、枚举/Renum/泛型模板等)全部落 perm_pool(永不 reset);
# 2) 每文件翻译完 reset_to(文件级 mark):只回收该文件的 AST/lexer/token/IR 等瞬时数据,注册数据不受影响。
# mark1 仍在 1a 结束后打,1b 每文件 reset_to(mark1) 语义不变。
# ============================================================
GenApi.set_stdlib_alloc(1)
# 遍历文件:优先使用拓扑顺序,回退到字母序
iter_count: int = topo_count if topo_count > 0 else result.Count
# 1a 阶段总耗时起点(struct/enum/union 注册)
t_a0: uint64 = w32.win32base.GetTickCount64()
for i in range(iter_count):
# 每文件级 mark:翻译完回收该文件 AST(注册数据在 perm_pool)
p1a_mark: sizet = mb.mark()
# 获取文件索引:拓扑顺序或字母序
file_idx: int = i
if topo_count > 0:
od_addr: uint64 = uint64(topo_order) + i * 4
od_ptr: t.CPtr = (int | t.CPtr)(t.CPtr(od_addr))
if od_ptr is not None:
file_idx = od_ptr[0]
entry_addr_r: uint64 = uint64(result.Entries) + file_idx * entry_size
entry_r: rawptr[IncludesScanner.FileEntry] = rawptr[IncludesScanner.FileEntry](entry_addr_r)
if entry_r is None:
p1a_fl += 1
continue
sha1_r: str = entry_r.Sha1
if sha1_r is None:
p1a_fl += 1
continue
# 按需翻译过滤(拓扑排序已过滤,但字母序回退需要检查)
if topo_count == 0:
if use_reachable != 0:
if StubMergerCollect._is_in_sha1_set(sha1_r, reachable_set, reachable_count) == 0:
p1a_skp += 1
continue
else:
if StubMergerCollect._is_in_sha1_set(sha1_r, sha1_set, set_count) == 0:
p1a_skp += 1
continue
# 单文件计时起点(读文件 + parse 段,定位 1a 卡点)
t_fa0: uint64 = w32.win32base.GetTickCount64()
# 读取文件内容
file_path_r: str = entry_r.Path
f_r: rawptr[fileio.File] = fileio.File(file_path_r, fileio.MODE.R)
if f_r.closed:
p1a_fl += 1
continue
# read_all 的容量参数含终止符(最多读 cap-1 字节), 文件更长会被静默截断
if f_r.size() >= Phase1.SRC_BUF_SIZE:
VLogger.error(f"源文件超过 {Phase1.SRC_BUF_SIZE} 字节上限, 拒绝读取: {file_path_r}", "Phase1")
f_r.close()
p1a_fl += 1
continue
src_buf_r: bytes = stdlib.malloc(Phase1.SRC_BUF_SIZE)
if src_buf_r is None:
f_r.close()
p1a_fl += 1
continue
bytes_read_r: LONG = f_r.read_all(src_buf_r, Phase1.SRC_BUF_SIZE)
f_r.close()
if bytes_read_r <= 0:
stdlib.free(src_buf_r)
p1a_fl += 1
continue
if bytes_read_r < Phase1.SRC_BUF_SIZE:
src_buf_r[bytes_read_r] = 0
# 解析 AST
lx_r: ast.Lexer* = ast.new_lexer(mb)
if lx_r is None:
stdlib.free(src_buf_r)
p1a_fl += 1
continue
ast._lexer_init(lx_r, src_buf_r, mb)
tokens_r: ast.Token* = ast.tokenize(lx_r)
tree_r: ast.AST* = ast.parse_tokens(mb, tokens_r)
if tree_r is None:
stdlib.free(src_buf_r)
p1a_fl += 1
continue
# 计时点 1:读文件 + parse 完成
t_fa1: uint64 = w32.win32base.GetTickCount64()
# 创建 Translator,设置 _declare_only=1(注册 struct/enum/union)
tr_r: rawptr[HandlesTranslator.Translator] = HandlesTranslator.Translator()
if tr_r is None:
stdlib.free(src_buf_r)
p1a_fl += 1
continue
tr_r.ModuleSha1 = sha1_r
tr_r._declare_only = 1
tr_r.CurrentPackage = GenApi.compute_package_from_relpath(mb, entry_r.RelPath)
GenApi.set_current_file_type(file_path_r)
GenApi.set_current_file_imports(file_path_r)
GenApi.set_current_module_sha1(sha1_r)
GenApi.clear_cdefine_constants()
GenApi.clear_ctypedefs()
GenApi.reset_visible_structs(mb, 0)
ret_r: int = tr_r.translate(tree_r)
# 计时点 2:translate(declare=1) 返回
t_fa2: uint64 = w32.win32base.GetTickCount64()
if ret_r != 0:
p1a_fl += 1
else:
p1a_reg += 1
# 逐文件进度日志(parse/translate 两段,定位 1a 卡点)
VLogger.info(f"[{i+1}/{iter_count}] 1a: {file_path_r} (parse={t_fa1-t_fa0} ms, translate={t_fa2-t_fa1} ms, total={t_fa2-t_fa0} ms)", "Phase1")
# 释放 Translator 的 C malloc 资源
if tr_r._global_names is not None:
stdlib.free(tr_r._global_names)
if tr_r._nonlocal_names is not None:
stdlib.free(tr_r._nonlocal_names)
stdlib.free(src_buf_r)
# [TEMP-DIAG] 1a 每文件哨兵:校验本文件窗口 [p1a_mark, chk_depth) 内所有存活块的块尾 canary。
# 位置必须在 reset_to 之前:reset_to 会把 chk_depth 截断回 p1a_mark,之后再查同一 mark 时 check_alive_blocks 直接 return 0(窗口为空、必然漏报)。
# p1a_mark 在循环体首行捕获(本文件 L597),故窗口恰为「本文件 parse + translate(declare=1)」期间分配的块。
# 仅 --detection-deep(mb.deep_detect != 0)时执行(每文件 O(n) 遍历活块);读法同本文件既有 mb.detection:直接读 MemBuddy 实例的公开 int 字段。
if mb.deep_detect != 0:
chk_rc_1a: int = mb.check_alive_blocks(p1a_mark)
if chk_rc_1a != 0:
VLogger.error(f"内存损坏(1a 每文件后续, rc={chk_rc_1a}): {file_path_r}", "Phase1")
stdlib.exit(1)
# 回收本文件 AST/lexer/token/IR 等瞬时数据(注册数据已在 perm_pool,
# reset 不影响;IncludesScanner result 等在 p1a_mark 前分配同样保留)
mb.reset_to(p1a_mark)
# 1a 阶段总耗时(struct/enum/union 注册,含每文件读+parse+翻译声明)
t_a1: uint64 = w32.win32base.GetTickCount64()
VLogger.info(f"1a 总耗时: {t_a1-t_a0} ms (注册 struct/enum, reg={p1a_reg} skp={p1a_skp})", "Phase1")
if p1a_fl > 0:
VLogger.error(f"Phase 1a 有 {p1a_fl} 个文件 struct 注册失败", "Phase1")
return 1
# 释放拓扑排序资源(topo_order/topo_count_box 保留到 Phase 1b 结束,
# 1b 按拓扑序翻译需要它们;class/def 数组 1b 不再使用)
if class_names_buf is not None:
mb.free(class_names_buf)
if parent_names_buf is not None:
mb.free(parent_names_buf)
if def_sha1s_buf is not None:
mb.free(def_sha1s_buf)
if generic_file_indices is not None:
mb.free(generic_file_indices)
if generic_file_count_box is not None:
mb.free(generic_file_count_box)
# ============================================================
# Phase 1b: 全量翻译(struct 已注册,走 existing 路径翻译方法体)
# ============================================================
# OOM 回收:mark1 在 1a-pre/泛型预注册/1a 全部结束后打,所有跨文件持久表(_struct_table/_enum_table/_generic_class_nodes/_renum_*/关键字表等)都落在 mark1 之前;
# 1b 每文件翻译完后 reset_to(mark1) 回收该文件的 AST/IR/Handler/dump 临时,避免 90 文件累积导致 1GB 池 OOM。
mark1: sizet = mb.mark()
# 1b 阶段新注册的 struct/enum 条目改 stdlib 分配(跨文件永久存活,不被 reset 回收)
GenApi.set_stdlib_alloc(1)
# ============================================================
# Phase 1b-pre: 预生成所有待翻译文件的 .vpi stub
#
# 依赖模块的 .vpi 必须在翻译前可用:1b 主循环按扫描顺序(非拓扑)翻译,若依赖模块后翻译则 LoadDepVpiIntoModule 读不到 .vpi
# → declare 未注入 → 跨模块调用的返回类型推断错误(如误推为 Ptr 而 define 返回 i8*)→ 运行时返回值截断崩溃。
# ============================================================
p1bpre_failed: int = 0
dir_len_pv: sizet = string.strlen(temp_dir)
# 1b-pre 阶段总耗时起点(vpi 预生成)
t_pv0: uint64 = w32.win32base.GetTickCount64()
for i_pv in range(result.Count):
entry_addr_pv: uint64 = uint64(result.Entries) + i_pv * entry_size
entry_pv: rawptr[IncludesScanner.FileEntry] = rawptr[IncludesScanner.FileEntry](entry_addr_pv)
if entry_pv is None or entry_pv.Sha1 is None:
p1bpre_failed += 1
continue
sha1_pv: str = entry_pv.Sha1
in_set_pv: int = 0
if use_reachable != 0:
in_set_pv = StubMergerCollect._is_in_sha1_set(sha1_pv, reachable_set, reachable_count)
else:
in_set_pv = StubMergerCollect._is_in_sha1_set(sha1_pv, sha1_set, set_count)
if in_set_pv == 0:
continue
# vpi 已存在则跳过
vpi_path_pv: str = StubMergerCollect._sliced_path(temp_dir, dir_len_pv, sha1_pv, "vpi")
if vpi_path_pv is None:
p1bpre_failed += 1
VLogger.error(f"[Phase 1b-pre] {entry_pv.Path} | vpi 路径生成失败", "Phase1")
continue
vf_pv: rawptr[fileio.File] = fileio.File(vpi_path_pv, fileio.MODE.R)
if not vf_pv.closed:
vf_pv.close()
stdlib.free(vpi_path_pv)
continue
stdlib.free(vpi_path_pv)
# 读取文件、解析 AST、生成 vpi
file_path_pv: str = entry_pv.Path
f_pv: rawptr[fileio.File] = fileio.File(file_path_pv, fileio.MODE.R)
if f_pv.closed:
p1bpre_failed += 1
VLogger.error(f"[Phase 1b-pre] {file_path_pv} | 无法打开源文件", "Phase1")
continue
# read_all 的容量参数含终止符(最多读 cap-1 字节), 文件更长会被静默截断
if f_pv.size() >= Phase1.SRC_BUF_SIZE:
p1bpre_failed += 1
VLogger.error(f"[Phase 1b-pre] {file_path_pv} | 源文件超过 {Phase1.SRC_BUF_SIZE} 字节上限", "Phase1")
f_pv.close()
continue
src_buf_pv: bytes = stdlib.malloc(Phase1.SRC_BUF_SIZE)
if src_buf_pv is None:
f_pv.close()
p1bpre_failed += 1
VLogger.error(f"[Phase 1b-pre] {file_path_pv} | 源文件缓冲分配失败(OOM)", "Phase1")
continue
bytes_read_pv: LONG = f_pv.read_all(src_buf_pv, Phase1.SRC_BUF_SIZE)
f_pv.close()
if bytes_read_pv <= 0:
stdlib.free(src_buf_pv)
p1bpre_failed += 1
VLogger.error(f"[Phase 1b-pre] {file_path_pv} | 文件读取失败或为空", "Phase1")
continue
if bytes_read_pv < Phase1.SRC_BUF_SIZE:
src_buf_pv[bytes_read_pv] = 0
lx_pv: ast.Lexer* = ast.new_lexer(mb)
if lx_pv is None:
stdlib.free(src_buf_pv)
p1bpre_failed += 1
VLogger.error(f"[Phase 1b-pre] {file_path_pv} | 词法器分配失败(OOM)", "Phase1")
continue
ast._lexer_init(lx_pv, src_buf_pv, mb)
tokens_pv: ast.Token* = ast.tokenize(lx_pv)
tree_pv: ast.AST* = ast.parse_tokens(mb, tokens_pv)
if tree_pv is None:
stdlib.free(src_buf_pv)
p1bpre_failed += 1
# 解析失败:**日志行内联 行/列 + 原因**,并附完整 Python 风格 traceback
# (Traceback + File/行 + 源码行 + caret + SyntaxError: 消息;消息含 token
# 类型与前/后 token 附注,见 includes/ast/parser.vp 的 _record_parse_err_ctx
# / parse_tokens 兜底补记),与单文件管线 BuildPipeline 同一条上报路径。
VLogger.error(f"[Phase 1b-pre] {file_path_pv} | AST 解析失败(第 {ast.get_parse_err_line()} 行第 {ast.get_parse_err_col()} 列: {ast.get_parse_err_msg()})", "Phase1")
GenApi.dump_parse_traceback(file_path_pv)
continue
pyi_buf_pv: bytes = stdlib.malloc(Phase1.PYI_BUF_SIZE)
if pyi_buf_pv is not None:
pyi_pos_pv: sizet = StubConverter._GeneratePyiFromAst(mb, tree_pv, entry_pv.RelPath, pyi_buf_pv, Phase1.PYI_BUF_SIZE)
if pyi_pos_pv > 0:
pyi_path_pv: str = StubMergerCollect._sliced_path(temp_dir, dir_len_pv, sha1_pv, "vpi")
if pyi_path_pv is not None:
pf_pv: rawptr[fileio.File] = fileio.File(pyi_path_pv, fileio.MODE.W)
if not pf_pv.closed:
pf_pv.write(pyi_buf_pv, pyi_pos_pv)
pf_pv.close()
stdlib.free(pyi_path_pv)
stdlib.free(pyi_buf_pv)
else:
p1bpre_failed += 1
VLogger.error(f"[Phase 1b-pre] {file_path_pv} | pyi 缓冲分配失败(OOM)", "Phase1")
stdlib.free(src_buf_pv)
# [TEMP-DIAG] 1b-pre 每文件哨兵(同上):校验本文件窗口 [mark1, chk_depth) 内所有存活块的块尾 canary。
# 同样必须在 reset_to(mark1) 之前(否则窗口被截断为空)。mark1 在 1b 段前捕获(本文件 L756),是 1b/1b-pre 的统一窗口起点。
# 仅 --detection-deep(mb.deep_detect != 0)时执行,读法同上方 1a 哨兵。
if mb.deep_detect != 0:
chk_rc_pv: int = mb.check_alive_blocks(mark1)
if chk_rc_pv != 0:
VLogger.error(f"内存损坏(1b-pre 每文件后续, rc={chk_rc_pv}): {file_path_pv}", "Phase1")
stdlib.exit(1)
# 回收本文件 AST(vpi 已写盘,AST 不再需要;mark1 后无其他持久数据)
mb.reset_to(mark1)
# 1b-pre 阶段总耗时(vpi 预生成)
t_pv1: uint64 = w32.win32base.GetTickCount64()
VLogger.info(f"1b-pre 总耗时: {t_pv1-t_pv0} ms (vpi 预生成)", "Phase1")
if p1bpre_failed > 0:
VLogger.error(f"Phase 1b-pre 有 {p1bpre_failed} 个文件 vpi 预生成失败(失败文件与原因见上方逐条 [Phase 1b-pre] 日志)", "Phase1")
return 1
# 遍历每个文件(拓扑序优先:父类所在文件先翻译,确保跨模块 vtable
# 生成时依赖模块的函数 TypeStr 已注册;topo_count==0 回退扫描顺序)
translated: int = 0
skipped: int = 0
failed: int = 0
# 1b 阶段总耗时起点(全量翻译)
t_b0: uint64 = w32.win32base.GetTickCount64()
loop_n_b: int = result.Count
if topo_count > 0:
loop_n_b = topo_count
for i in range(loop_n_b):
# 获取 entry(拓扑序:file_idx 从 topo_order 取,否则用扫描索引)
file_idx_b: int = i
if topo_count > 0:
od_addr_b: uint64 = uint64(topo_order) + sizet(i) * 4
od_ptr_b: t.CPtr = (int | t.CPtr)(od_addr_b)
if od_ptr_b is not None:
file_idx_b = od_ptr_b[0]
entry_addr: uint64 = uint64(result.Entries) + sizet(file_idx_b) * entry_size
entry: rawptr[IncludesScanner.FileEntry] = rawptr[IncludesScanner.FileEntry](entry_addr)
if entry is None:
failed += 1
continue
sha1: str = entry.Sha1
if sha1 is None:
failed += 1
continue
# 检查 SHA1 是否在翻译集合中(拓扑序已过滤可达;仅回退扫描顺序时检查)
in_set: int = 1
if topo_count == 0:
in_set = 0
if use_reachable != 0:
in_set = StubMergerCollect._is_in_sha1_set(sha1, reachable_set, reachable_count)
else:
in_set = StubMergerCollect._is_in_sha1_set(sha1, sha1_set, set_count)
if in_set == 0:
skipped += 1
continue
# 构造 stub 路径: {temp_dir}/{sha1}.stub.ll
dir_len: sizet = string.strlen(temp_dir)
sha1_len: sizet = string.strlen(sha1)
stub_path: str = StubMergerCollect._sliced_path(temp_dir, dir_len, sha1, "stub.ll")
if stub_path is None:
failed += 1
continue
# 检查 stub 是否已存在(按需翻译:跳过已存在的)
# 例外:含 class 定义的模块必须翻译——类注册只在翻译时发生,增量编译跳过会导致跨模块类构造 find_struct_by_module 落空 → 返回类型 fail-fast。
# 判据:扫描 1b-pre 生成的 vpi 是否有 "class " 行(vpi 为解析后存根,精确无注释/字符串误判)。
force_translate: int = 0
vpi_path_1b: str = StubMergerCollect._sliced_path(temp_dir, dir_len, sha1, "vpi")
if vpi_path_1b is not None:
vf_1b: rawptr[fileio.File] = fileio.File(vpi_path_1b, fileio.MODE.R)
if not vf_1b.closed:
vbuf_1b: bytes = stdlib.malloc(1048576)
if vbuf_1b is not None:
vrd_1b: LONG = vf_1b.read_all(vbuf_1b, 1048576)
vf_1b.close()
if vrd_1b > 0:
if vrd_1b < 1048576:
vbuf_1b[vrd_1b] = 0
vtotal_1b: sizet = string.strlen(vbuf_1b)
vpos_1b: sizet = 0
while vpos_1b < vtotal_1b:
vls_1b: sizet = vpos_1b
while vpos_1b < vtotal_1b and vbuf_1b[vpos_1b] != '\n':
vpos_1b += 1
vll_1b: sizet = vpos_1b - vls_1b
if vpos_1b < vtotal_1b:
vpos_1b += 1
vi2_1b: sizet = vls_1b
while vi2_1b < vls_1b + vll_1b and vbuf_1b[vi2_1b] == ' ':
vi2_1b += 1
if vll_1b >= 6 and string.strncmp(vbuf_1b + vi2_1b, "class ", 6) == 0:
force_translate = 1
break
stdlib.free(vbuf_1b)
else:
vf_1b.close()
stdlib.free(vpi_path_1b)
if force_translate == 1:
# 含 class:强制翻译(覆盖 stub/text,重新注册类)
rp: str = entry.RelPath
VLogger.info(f"含 class 定义,强制翻译: {rp}(sha1={sha1})", "Phase1")
else:
# 检查 stub 是否已存在(按需翻译:跳过已存在的)
sf: rawptr[fileio.File] = fileio.File(stub_path, fileio.MODE.R)
if not sf.closed:
sf.close()
# 检查 text.ll 是否也存在(虚表扫描需要 text.ll)
text_path: str = StubMergerCollect._sliced_path(temp_dir, dir_len, sha1, "text.ll")
if text_path is not None:
tf: rawptr[fileio.File] = fileio.File(text_path, fileio.MODE.R)
if not tf.closed:
tf.close()
stdlib.free(text_path)
stdlib.free(stub_path)
continue
stdlib.free(text_path)
# text.ll 不存在,需要重新翻译
rp: str = entry.RelPath
VLogger.info(f"text.ll 不存在,重新翻译: {rp}(sha1={sha1})", "Phase1")
else:
# stub 不存在,需要翻译
rp: str = entry.RelPath
VLogger.info(f"翻译: {rp}(sha1={sha1})", "Phase1")
stdlib.free(stub_path)
# 单文件计时起点:读文件(parse/translate/dump 三段,对齐 Phase2 插桩)
t_f0: uint64 = w32.win32base.GetTickCount64()
# 读取文件内容
file_path: str = entry.Path
f: rawptr[fileio.File] = fileio.File(file_path, fileio.MODE.R)
if f is None:
VLogger.error(f"无法打开(f=None): {file_path}", "Phase1")
failed += 1
continue
if f.closed:
VLogger.error(f"无法打开: {file_path}", "Phase1")
failed += 1
continue
# read_all 的容量参数含终止符(最多读 cap-1 字节), 文件更长会被静默截断
if f.size() >= Phase1.SRC_BUF_SIZE:
VLogger.error(f"源文件超过 {Phase1.SRC_BUF_SIZE} 字节上限, 拒绝读取: {file_path}", "Phase1")
f.close()
failed += 1
continue
src_buf: bytes = stdlib.malloc(Phase1.SRC_BUF_SIZE)
if src_buf is None:
f.close()
failed += 1
continue
bytes_read: LONG = f.read_all(src_buf, Phase1.SRC_BUF_SIZE)
f.close()
if bytes_read <= 0:
VLogger.error(f"读取失败: {file_path}", "Phase1")
stdlib.free(src_buf)
failed += 1
continue
if bytes_read < Phase1.SRC_BUF_SIZE:
src_buf[bytes_read] = 0
# 解析 AST
lx: ast.Lexer* = ast.new_lexer(mb)
if lx is None:
stdlib.free(src_buf)
failed += 1
continue
ast._lexer_init(lx, src_buf, mb)
tokens: ast.Token* = ast.tokenize(lx)
tree: ast.AST* = ast.parse_tokens(mb, tokens)
if tree is None:
# 解析失败:行/列 + 原因一并进日志行(原因由 parser.vp 的 _record_* /
# parse_tokens 兜底补记提供)
VLogger.error(f"AST 解析失败: {file_path}(第 {ast.get_parse_err_line()} 行第 {ast.get_parse_err_col()} 列: {ast.get_parse_err_msg()})", "Phase1")
stdlib.free(src_buf)
failed += 1
continue
# 计时点 1:读文件 + lexer/tokenize/parse 完成
t_f1: uint64 = w32.win32base.GetTickCount64()
# 文本扫描注册 CTypedef:AST 字符串字段(attr/id)token 零拷贝指向 src_buf,跨阶段内容不稳定(悬空),读 attr 会 0xC0000005;
# 故改用源码文本逐行匹配 "NAME: t.CTypedef = TYPE",在 src_buf 有效期内注册。
GenApi.RegisterCTypedefFromSource(mb, src_buf)
# 生成 .vpi 存根文件(直接遍历 AST,不依赖 PythonToStubConverter)
# 1b-pre 已为所有待翻译文件生成 vpi,此处仅兜底:vpi 缺失才生成
# (消除 1b-pre 与 1b 对同一文件的重复 _GeneratePyiFromAst 工作)
vpi_path_1b: str = StubMergerCollect._sliced_path(temp_dir, dir_len, sha1, "vpi")
if vpi_path_1b is not None:
vf_1b: rawptr[fileio.File] = fileio.File(vpi_path_1b, fileio.MODE.R)
vpi_exists_1b: int = 0
if not vf_1b.closed:
vf_1b.close()
vpi_exists_1b = 1
if vpi_exists_1b == 0:
pyi_buf: bytes = stdlib.malloc(Phase1.PYI_BUF_SIZE)
if pyi_buf is not None:
pyi_pos: sizet = StubConverter._GeneratePyiFromAst(
mb, tree, entry.Path, pyi_buf, Phase1.PYI_BUF_SIZE)
if pyi_pos > 0:
pf_1b: rawptr[fileio.File] = fileio.File(vpi_path_1b, fileio.MODE.W)
if not pf_1b.closed:
pf_1b.write(pyi_buf, pyi_pos)
pf_1b.close()
stdlib.free(pyi_buf)
stdlib.free(vpi_path_1b)
# 翻译 AST → LLVM IR
tr: rawptr[HandlesTranslator.Translator] = HandlesTranslator.Translator()
if tr is None:
stdlib.free(src_buf)
failed += 1
continue
tr.ModuleSha1 = sha1
tr.CurrentPackage = GenApi.compute_package_from_relpath(mb, entry.RelPath)
GenApi.set_current_file_type(file_path)
# --traceback: mirror the DWARF switch into the Gen-layer emitter (the
# same BuildPipeline._traceback_mode that gates -frame-pointer=all /
# dbghelp / the Install() ctor). include/*.vp modules are compiled here,
# so this is what gives their frames DWARF line info.
DebugInfoGen.SetEnabled(BuildPipeline._traceback_mode)
GenApi.set_current_file_imports(file_path)
GenApi.set_current_module_sha1(sha1)
GenApi.clear_cdefine_constants()
GenApi.clear_ctypedefs()
GenApi.reset_visible_structs(mb, 0)
# 翻译前哨兵:AST 解析(lexer/tokenize/parse_tokens)同样在 pool 分配,
# 若解析器越界写破坏了存活块 header,此处立即定位到当前文件
# (崩溃点漂移排查:把"翻译过程中随机崩"变成确定性定位)
pre_rc: int = mb.check_alive_blocks(mark1)
if pre_rc != 0:
VLogger.error(f"内存损坏(翻译前存活块, rc={pre_rc}): {file_path}", "Phase1")
stdlib.exit(1)
ret: int = tr.translate(tree)
# 计时点 2:translate 返回
t_f2: uint64 = w32.win32base.GetTickCount64()
# 诊断:translate 正常返回(若挂起在 translate 内部,此行不会打印,
# 用于区分"translate 内部挂起"与"后续 dump 挂起")
if ret == 0:
VLogger.info(f"翻译完成(translate返回): {file_path}", "Phase1")
if ret != 0:
VLogger.error(f"翻译失败: {file_path}", "Phase1") # error 终止
stdlib.free(src_buf)
if tr._global_names is not None:
stdlib.free(tr._global_names)
if tr._nonlocal_names is not None:
stdlib.free(tr._nonlocal_names)
failed += 1
continue
# --traceback: inject the startup Install() ctor when translating traceback.vp
# (its stub/text.ll below are what Phase2 reuses to build the .obj). The
# injector lives in the Gen layer (already depends on llvmlite), so this
# module gains no cross-module llvmlite symbol dependency.
if BuildPipeline._traceback_mode != 0 and BuildPipeline._tb_basename_is_traceback(file_path) != 0:
HandlesTranslator.InjectTracebackInstallCtor(tr)
# 诊断:pre_rc 哨兵在 translate 前、覆盖不到 translate 期间的 AST/IR 写穿,故 dump 前立即检查存活块 header/canary
# (rc=6 块尾 canary 损坏 = 越界写穿数据区),把「dump 时链表成环卡死」变成「定位到 translate 期间写穿发生的文件」。
post_rc: int = mb.check_alive_blocks(mark1)
if post_rc != 0:
VLogger.error(f"内存损坏(translate后存活块, rc={post_rc}): {file_path}", "Phase1")
# dump stub IR(declarations only)
# 1MB 不够:ast/parser.vp 的 text.ll 超 1MB 被截断 → 合并 IR 尾部丢字
# (opt: "end of file in global variable name"),提至 8MB。
PHASE1_IR_SIZE: sizet = 8388608
stub_buf: bytes = stdlib.malloc(PHASE1_IR_SIZE)
if stub_buf is None:
stdlib.free(src_buf)
if tr._global_names is not None:
stdlib.free(tr._global_names)
if tr._nonlocal_names is not None:
stdlib.free(tr._nonlocal_names)
failed += 1
continue
tr.dump_ir(stub_buf, PHASE1_IR_SIZE, llvmlite.OUTPUT_STUB)
stub_len: sizet = string.strlen(stub_buf)
# save stub.ll
dir_len_p1: sizet = string.strlen(temp_dir)
stub_path_p1: str = StubMergerCollect._sliced_path(temp_dir, dir_len_p1, sha1, "stub.ll")
if stub_path_p1 is not None:
sf_p1: rawptr[fileio.File] = fileio.File(stub_path_p1, fileio.MODE.W)
if not sf_p1.closed:
sf_p1.write(stub_buf, stub_len)
sf_p1.close()
stdlib.free(stub_path_p1)
stdlib.free(stub_buf)
# dump text IR(definitions only)
text_buf: bytes = stdlib.malloc(PHASE1_IR_SIZE)
if text_buf is None:
stdlib.free(src_buf)
if tr._global_names is not None:
stdlib.free(tr._global_names)
if tr._nonlocal_names is not None:
stdlib.free(tr._nonlocal_names)
failed += 1
continue
tr.dump_ir(text_buf, PHASE1_IR_SIZE, llvmlite.OUTPUT_TEXT)
text_len: sizet = string.strlen(text_buf)
# save text.ll
text_path_p1: str = StubMergerCollect._sliced_path(temp_dir, dir_len_p1, sha1, "text.ll")
if text_path_p1 is not None:
tf_p1: rawptr[fileio.File] = fileio.File(text_path_p1, fileio.MODE.W)
if not tf_p1.closed:
tf_p1.write(text_buf, text_len)
tf_p1.close()
stdlib.free(text_path_p1)
stdlib.free(text_buf)
# 计时点 3:dump(stub+text)完成;输出单文件三段耗时
t_f3: uint64 = w32.win32base.GetTickCount64()
VLogger.info(f"[{i+1}/{loop_n_b}] 翻译: {file_path} (parse={t_f1-t_f0} ms, translate={t_f2-t_f1} ms, dump={t_f3-t_f2} ms, total={t_f3-t_f0} ms)", "Phase1")
# save dependencies(_imported_modules) for Phase B
deps_path_p1: str = StubMergerCollect._sliced_path(temp_dir, dir_len_p1, sha1, "deps.txt")
if deps_path_p1 is not None:
df_p1: rawptr[fileio.File] = fileio.File(deps_path_p1, fileio.MODE.W)
if not df_p1.closed:
if tr._imported_modules is not None:
dl: sizet = string.strlen(tr._imported_modules)
df_p1.write(tr._imported_modules, dl)
df_p1.close()
stdlib.free(deps_path_p1)
# 释放 Translator 的 C malloc 资源
if tr._global_names is not None:
stdlib.free(tr._global_names)
if tr._nonlocal_names is not None:
stdlib.free(tr._nonlocal_names)
# 所有 dump_ir 完成,src_buf 不再需要
stdlib.free(src_buf)
translated += 1
# OOM 回收:回收本文件翻译产生的瞬时数据(AST/tokens/lexer/IR/Handler/dump 临时);持久表(struct/enum/REnum/spec 缓存/全局函数名等)已迁出 reset 区间(stdlib),
# reset_to(mark1) 只回收 mark1 之后分配、未被单独 free 的块。
#
# 内存哨兵(崩溃点漂移排查):reset 前检查存活块 header(抓越界写),reset 后检查 free list 完整性(抓链表损坏);任一损坏立即终止,把「随机崩溃漂移」变成「确定性定位到当前文件」。
# rc 语义(check_alive_blocks):1=块地址低于 usable,2=高于 usable+size,3=order 为负,4=order 超 max_order,5=块未对齐到 order 大小,6=块尾 canary 损坏(越界写穿数据区)
# rc 语义(self_check):1/2=越界,3=对齐错,4=free list 成环/超步数
chk_rc: int = mb.check_alive_blocks(mark1)
if chk_rc != 0:
VLogger.error(f"内存损坏(存活块 header, rc={chk_rc}): {file_path}", "Phase1")
# 越界写是崩溃根因(与回收/UAF 无关,已由 A/B 实验确认)——恢复 reset_to,改用块尾 canary(check_alive_blocks 校验)把越界写事发地定位到具体文件。
mb.reset_to(mark1)
sc_rc: int = mb.self_check()
if sc_rc != 0:
VLogger.error(f"内存损坏(free list, rc={sc_rc}): {file_path}", "Phase1")
# 1b 阶段总耗时(全量翻译 + dump)
t_b1: uint64 = w32.win32base.GetTickCount64()
VLogger.info(f"1b 总耗时: {t_b1-t_b0} ms (全量翻译, trans={translated} skip={skipped} fail={failed})", "Phase1")
# 释放 Phase 1b 使用的拓扑排序资源
if topo_order is not None:
mb.free(topo_order)
if topo_count_box is not None:
mb.free(topo_count_box)
VLogger.info(f"完成: 翻译={translated} 跳过={skipped} 失败={failed}", "Phase1")
if failed > 0:
VLogger.error(f"Phase1 有 {failed} 个文件翻译失败", "Phase1")
return 1
# 释放可达 SHA1 集合(如果分配了)
if reachable_set is not None:
stdlib.free(reachable_set)
return 0