docs(tpv): R11 注释与 docstring 英文化并压缩到 2 行内
按 wiki/新规范：注释只讲作用、不掺杂历史原因、避免超长、尽可能英文（含三引号 __doc__）。全文 79 个 .vp：# 注释 16269 行（中文 13808）与 docstring（中文 1381 处）全部改英文；超 2 行的注释块（2093）与 docstring（397）全部压到 2 行。运行时可见文本未动（fatal_error/VLogger 的中文报错文案、stdio.printf 的横幅格式串）。校验：屏蔽注释与三引号后逐行比对各文件代码骨架，0 漂移。验证：自举重建 153.48s 通过（跨模块内联完成）；TPV 顺序回归 77/77 全通过。
This commit is contained in:
TermiNexus committed
1 parent
a2874a14e6
commit
0f4aeeceff
79 files changed
+10090
-20352
No files matched your search
+142
-217
@@ -30,21 +30,17 @@ import lib.core.Gen.GenApi as GenApi
|
||||
import lib.core.Gen.DebugInfoGen as DebugInfoGen
|
||||
|
||||
|
||||
# ============================================================
|
||||
# _InjectedCreatesCycle - 容器模块自动注入前的破环检查
|
||||
#
|
||||
# 注入 _list/_dict 前检查被注入模块自身的依赖是否已含当前模块，命中（如 memhub↔_list）则跳过注入——
|
||||
# 否则 Phase A 多遍收敛死锁（"无法完成翻译"）；去掉环边后 _list 仍因当前模块被其依赖而处于闭包，list 类仍由 Phase1b 注册。
|
||||
# ============================================================
|
||||
# _InjectedCreatesCycle - cycle check before auto-injecting a container module
|
||||
# (_list/_dict); a hit (e.g. memhub<->_list) skips injection to avoid a Phase A deadlock.
|
||||
def _InjectedCreatesCycle(entry: rawptr[IncludesScanner.FileEntry], mod_name: str) -> int:
|
||||
"""注入 mod_name 到当前模块前检查是否成环，返回 1=成环需跳过注入"""
|
||||
"""Return 1 if injecting mod_name into the current module would create a cycle"""
|
||||
if entry is None or entry.RelPath is None or mod_name is None: return 0
|
||||
inj_sha1: str = StubMergerStore.LookupSha1ByModName(mod_name)
|
||||
if inj_sha1 is None: return 0
|
||||
inj_deps: str = StubMergerStore.LookupIncludesDeps(inj_sha1)
|
||||
if inj_deps is None: return 0
|
||||
rp: str = entry.RelPath
|
||||
# 当前模块名 = RelPath 基名（最后 '/' 或 '\\' 之后）去扩展名
|
||||
# Current module name = basename of RelPath (after the last '/' or '\'), without extension
|
||||
base: str = string.strrchr(rp, '/')
|
||||
if base is None: base = string.strrchr(rp, '\\')
|
||||
if base is not None: base = base + 1
|
||||
@@ -76,7 +72,7 @@ def _InjectedCreatesCycle(entry: rawptr[IncludesScanner.FileEntry], mod_name: st
|
||||
|
||||
|
||||
def _DepsContainsName(deps: str, name: str) -> int:
|
||||
"""deps 字符串（空格分隔模块名）是否含 name token（词边界精确匹配）"""
|
||||
"""Return 1 if the space-separated deps string contains the name token exactly"""
|
||||
if deps is None or name is None: return 0
|
||||
nl: sizet = string.strlen(name)
|
||||
dl: sizet = string.strlen(deps)
|
||||
@@ -91,7 +87,7 @@ def _DepsContainsName(deps: str, name: str) -> int:
|
||||
|
||||
|
||||
def _DepsRemoveName(deps: str, name: str) -> int:
|
||||
"""从 deps 字符串移除 name token（原地重建，返回 1=有改动）"""
|
||||
"""Remove the name token from the deps string in place; returns 1 if changed"""
|
||||
if deps is None or name is None: return 0
|
||||
nl: sizet = string.strlen(name)
|
||||
dl: sizet = string.strlen(deps)
|
||||
@@ -119,7 +115,7 @@ def _DepsRemoveName(deps: str, name: str) -> int:
|
||||
|
||||
|
||||
def _RewriteDepsTxt(temp_dir: str, sha1: str, deps_str: str) -> None:
|
||||
"""重写 {temp_dir}/{sha1前2位}/{sha1}.deps.txt（内存破环后同步磁盘）"""
|
||||
"""Rewrite the on-disk deps.txt file to match the in-memory deps after breaking a cycle"""
|
||||
if temp_dir is None or sha1 is None or deps_str is None: return
|
||||
td_len: sizet = string.strlen(temp_dir)
|
||||
dpath: str = StubMergerCollect._sliced_path(temp_dir, td_len, sha1, "deps.txt")
|
||||
@@ -133,17 +129,7 @@ def _RewriteDepsTxt(temp_dir: str, sha1: str, deps_str: str) -> None:
|
||||
|
||||
|
||||
def _BreakContainerCyclesInDeps(temp_dir: str) -> int:
|
||||
"""扫描后统一清理 _list/_dict 注入环边。
|
||||
|
||||
Phase1a-pre 全量扫描用 FindFirstFileA（文件系统序），注入 _list/_dict 时
|
||||
被注入模块（_list/_dict）自身的 deps 未必已注册 → _InjectedCreatesCycle
|
||||
内联检查可能漏判（如 _list 显式 import memhub，而 memhub 的 list[str]
|
||||
注入 _list 未被拦截）→ memhub↔_list 2-环 → Phase A 永不收敛。此处扫描
|
||||
结束后全量检查：凡某模块 M 的 deps 含 _list/_dict，且 _list/_dict 的 deps
|
||||
含 M → 从 M 的 deps 移除注入边（list 类仍由 Phase1b 经闭包注册）。
|
||||
内存破环后同步重写 {temp_dir}/deps.txt，保证磁盘与内存一致（Phase2 依赖
|
||||
图诊断回退读 deps.txt）。
|
||||
"""
|
||||
"""Remove _list/_dict injection cycle edges after scanning and sync deps.txt"""
|
||||
if StubMerger._includes_deps_sha1 is None or StubMerger._includes_deps_str is None: return 0
|
||||
cnt: sizet = StubMerger._includes_deps_count
|
||||
ci: sizet
|
||||
@@ -152,7 +138,7 @@ def _BreakContainerCyclesInDeps(temp_dir: str) -> int:
|
||||
deps_m: str = StubMerger._includes_deps_str + sizet(ci) * StubMerger.MAX_DEPS_STR_LEN
|
||||
if sha1_m is None or deps_m is None: continue
|
||||
if string.strstr(deps_m, "_list") is None and string.strstr(deps_m, "_dict") is None: continue
|
||||
# 该模块名（RelPath 基名去扩展名）
|
||||
# Module name (basename of RelPath without extension)
|
||||
rel_m: str = StubMergerStore.LookupSha1RelPath(sha1_m)
|
||||
if rel_m is None: continue
|
||||
base_m: str = string.strrchr(rel_m, '/')
|
||||
@@ -171,16 +157,16 @@ def _BreakContainerCyclesInDeps(temp_dir: str) -> int:
|
||||
if nm_m is None: continue
|
||||
string.strncpy(nm_m, base_m, 128)
|
||||
nm_m[128] = '\0'
|
||||
# _list ↔ M 环
|
||||
# _list <-> M cycle
|
||||
if string.strstr(deps_m, "_list") is not None:
|
||||
lst_sha1: str = StubMergerStore.LookupSha1ByModName("_list")
|
||||
if lst_sha1 is not None:
|
||||
lst_deps: str = StubMergerStore.LookupIncludesDeps(lst_sha1)
|
||||
if lst_deps is not None and _DepsContainsName(lst_deps, nm_m) != 0:
|
||||
if _DepsRemoveName(deps_m, "_list") != 0:
|
||||
# 内存破环后同步重写 deps.txt（诊断回退路径读文件）
|
||||
# Rewrite deps.txt after breaking the cycle in memory (diagnostic fallback reads the file)
|
||||
_RewriteDepsTxt(temp_dir, sha1_m, deps_m)
|
||||
# _dict ↔ M 环
|
||||
# _dict <-> M cycle
|
||||
if string.strstr(deps_m, "_dict") is not None:
|
||||
dct_sha1: str = StubMergerStore.LookupSha1ByModName("_dict")
|
||||
if dct_sha1 is not None:
|
||||
@@ -191,18 +177,10 @@ def _BreakContainerCyclesInDeps(temp_dir: str) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
# ============================================================
|
||||
# _BuildReachableResult - 从 Sha1Store 构建可达 FileEntry 列表
|
||||
#
|
||||
# 依赖图按需扫描（StubMergerBuild._BuildReachableSha1Set）只把「可达 includes + App」登记到全局 Sha1Store；
|
||||
# 本函数把 incdir 非空条目（includes）重建为 Phase1 1a/1b-pre/1b 需要的 FileEntry 列表。
|
||||
# App 条目（incdir 空）不进入 result——Phase B 已编译其 .obj，加入会导致 Phase B+ 重复编译 → 重复符号。
|
||||
#
|
||||
# Args: mb 内存池；includes_dir includes 根目录（仅用于失败诊断，路径以 store 为准）
|
||||
# Returns: ScanResult 指针（stdlib 分配，调用方负责释放）；失败返回 None
|
||||
# ============================================================
|
||||
# _BuildReachableResult - rebuild the reachable FileEntry list from the Sha1Store.
|
||||
# Only incdir-nonempty (includes) entries enter result; App entries would cause duplicate symbols.
|
||||
def _BuildReachableResult(mb: memhub.MemBuddy*, includes_dir: str) -> rawptr[IncludesScanner.ScanResult]:
|
||||
"""从 Sha1Store（仅可达 includes + App）构建可达 FileEntry 列表"""
|
||||
"""Build the reachable FileEntry list from the Sha1Store (reachable includes only)"""
|
||||
if mb is None: return None
|
||||
store_count: sizet = sizet(StubMerger._sha1_store_count)
|
||||
if store_count <= 0 or StubMerger._sha1_store_arr is None: return None
|
||||
@@ -213,18 +191,18 @@ def _BuildReachableResult(mb: memhub.MemBuddy*, includes_dir: str) -> rawptr[Inc
|
||||
for i in range(store_count):
|
||||
sidx: sizet = sizet(i) * 17
|
||||
if StubMerger._sha1_store_arr[sidx] == '\0': continue
|
||||
# 仅 includes 条目（incdir 非空）；App 条目不进入 result
|
||||
# Only includes entries (non-empty incdir); App entries do not enter result
|
||||
iidx: sizet = sizet(i) * StubMerger.MAX_INC_DIR_LEN
|
||||
if StubMerger._sha1_store_incdir[iidx] == '\0': continue
|
||||
eaddr: uint64 = uint64(result.Entries) + uint64(result.Count) * entry_size
|
||||
entry: rawptr[IncludesScanner.FileEntry] = rawptr[IncludesScanner.FileEntry](eaddr)
|
||||
if entry is None: continue
|
||||
# SHA1 深拷贝（store 槽 17 字节）
|
||||
# Deep-copy SHA1 (store slot is 17 bytes)
|
||||
sha1_buf: bytes = stdlib.malloc(17)
|
||||
if sha1_buf is None: continue
|
||||
string.strcpy(sha1_buf, StubMerger._sha1_store_arr + sidx)
|
||||
entry.Sha1 = sha1_buf
|
||||
# RelPath / IncDir 深拷贝
|
||||
# Deep-copy RelPath / IncDir
|
||||
ridx: sizet = sizet(i) * StubMerger.MAX_REL_PATH_LEN
|
||||
rel_len: sizet = string.strlen(StubMerger._sha1_store_rel + ridx)
|
||||
inc_len: sizet = string.strlen(StubMerger._sha1_store_incdir + iidx)
|
||||
@@ -239,7 +217,7 @@ def _BuildReachableResult(mb: memhub.MemBuddy*, includes_dir: str) -> rawptr[Inc
|
||||
string.strcpy(inc_buf, StubMerger._sha1_store_incdir + iidx)
|
||||
entry.RelPath = rel_buf
|
||||
entry.IncDir = inc_buf
|
||||
# 绝对路径 = inc_dir + '/' + rel_path（与 _TryResolveModuleFile 拼接一致）
|
||||
# Absolute path = inc_dir + '/' + rel_path (matches _TryResolveModuleFile)
|
||||
abs_buf: bytes = stdlib.malloc(inc_len + 1 + rel_len + 1)
|
||||
if abs_buf is None:
|
||||
stdlib.free(sha1_buf)
|
||||
@@ -250,7 +228,7 @@ def _BuildReachableResult(mb: memhub.MemBuddy*, includes_dir: str) -> rawptr[Inc
|
||||
abs_buf[inc_len] = '/'
|
||||
string.strcpy(abs_buf + inc_len + 1, rel_buf)
|
||||
entry.Path = abs_buf
|
||||
# 模块名（store 登记时保存的绝对模块名）
|
||||
# Module name (absolute module name saved when registering in the store)
|
||||
midx: sizet = sizet(i) * 64
|
||||
mod_len: sizet = string.strlen(StubMerger._sha1_store_mod + midx)
|
||||
if mod_len > 0:
|
||||
@@ -262,43 +240,33 @@ def _BuildReachableResult(mb: memhub.MemBuddy*, includes_dir: str) -> rawptr[Inc
|
||||
return result
|
||||
|
||||
# ============================================================
|
||||
# Phase1Main - Phase1 主流程（扫描 → 预注册 → 1a/1b 翻译 → 生成 stub）
|
||||
#
|
||||
# 函数实现直接位于本模块，调用方经本模块前缀访问；全局符号经 Phase1._xxx 跨模块访问。
|
||||
# ============================================================
|
||||
# Phase1Main - Phase1 main flow (scan -> pre-register -> 1a/1b translate -> generate stub)
|
||||
|
||||
# ============================================================
|
||||
# RunPhase1 - Phase1: 扫描 includes 目录，按需翻译并生成 stub
|
||||
#
|
||||
# 对每个 .vp/.py 文件检查 stub 是否已存在，不存在才翻译并分离 stub（避免重复工作）。
|
||||
#
|
||||
# Args: mb 内存池；includes_dir includes 目录路径；temp_dir 临时目录（保存 stub）；log 日志器
|
||||
# Returns: 0 成功，非 0 失败
|
||||
# ============================================================
|
||||
# RunPhase1 - scan the includes dir; translate and split stubs only when missing.
|
||||
def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawptr[VLogger.Logger]) -> int:
|
||||
"""Phase1: 扫描 includes 目录，按需翻译并生成 stub"""
|
||||
"""Phase1: scan the includes directory and translate/generate stubs on demand"""
|
||||
if includes_dir is None or temp_dir is None:
|
||||
VLogger.error("includes_dir 或 temp_dir 为空，无法运行 Phase1", "Phase1")
|
||||
return 1
|
||||
|
||||
# 设置全局 temp_dir（供跨模块 CDefine 查找使用）
|
||||
# Set the global temp_dir (used for cross-module CDefine lookup)
|
||||
GenApi.set_temp_dir(temp_dir)
|
||||
# 同步设置 HandlesExprCall 的 temp_dir（供 vpi stub 返回注解缓存按需加载）。
|
||||
# 必须在 includes 翻译前设置：Phase1 翻译 includes 时「同模块方法调用（定义在后）」需读 {temp_dir}/{sha1[0:2]}/{sha1}.vpi 兜底，Phase2 的 SetTempDir 太晚。
|
||||
# Also set HandlesExprCall's temp_dir (for the on-demand vpi return-annotation cache);
|
||||
# must run before includes translation since Phase2's SetTempDir is too late.
|
||||
GenApi.SetTempDir(temp_dir)
|
||||
|
||||
# 确保 temp 目录存在（build_dir/temp 可能尚未创建）
|
||||
# Ensure the temp dir exists (build_dir/temp may not exist yet)
|
||||
BuildPipeline.ensure_dir(temp_dir)
|
||||
|
||||
log.banner("Phase1: 扫描 includes（按需翻译）")
|
||||
|
||||
# 初始化 AST 表（只需一次）：必须在依赖图 BFS 扫描之前调用。
|
||||
# _init_tables 懒分配的关键字/操作符表（全局 _kw_head 等）若首次触发在 BFS 的文件级 mark 之后，reset_to 会回收表节点 → 全局悬空 → 后续 tokenize 崩溃。
|
||||
# Init the AST tables once, before the dependency-graph BFS: if _init_tables first runs
|
||||
# after the BFS file-level mark, reset_to frees the tables and later tokenize crashes.
|
||||
ast._init_tables(mb)
|
||||
|
||||
# 依赖图按需扫描（App 优先模型）：先扫 App 源文件（Config.SourceDir）的 import 依赖作为初始队列，
|
||||
# BFS 按文件系统解析 includes 文件（只对可达者读文件+算 SHA1+解析 import）并登记 Sha1Store/deps/写 deps.txt；
|
||||
# 未引用的 includes 从不读取。
|
||||
# On-demand dependency scan (App-first): seed the queue from App imports, then BFS-resolve
|
||||
# reachable includes only (read + SHA1 + import parse); unreferenced ones are never read.
|
||||
t_scan0: uint64 = w32.win32base.GetTickCount64()
|
||||
reachable_set: str = None
|
||||
reachable_count: int = 0
|
||||
@@ -326,58 +294,48 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
t_scan1: uint64 = w32.win32base.GetTickCount64()
|
||||
VLogger.info(f"依赖图扫描总耗时: {t_scan1 - t_scan0} ms（可达 {reachable_count} 个 includes）", "Phase1")
|
||||
|
||||
# 从 Sha1Store（仅可达 includes + App）构造可达 FileEntry 列表：
|
||||
# 后续 1a/1b-pre/1b 直接遍历 result（incdir 非空条目=includes），
|
||||
# use_reachable 过滤幂等保留（result 即可达子集）
|
||||
# Build the reachable FileEntry list from the Sha1Store; 1a/1b-pre/1b iterate result
|
||||
# (non-empty incdir = includes). use_reachable filtering is idempotent.
|
||||
result: rawptr[IncludesScanner.ScanResult] = _BuildReachableResult(mb, includes_dir)
|
||||
if result is None:
|
||||
VLogger.error("可达 FileEntry 列表构建失败，终止编译", "Phase1")
|
||||
return 1
|
||||
VLogger.info(f"可达 includes 条目: {result.Count}", "Phase1")
|
||||
|
||||
# 扫描后统一清理 _list/_dict 注入环边（_list↔memhub 等 2-环）。
|
||||
# 依赖图按需扫描顺序不可靠，内联检查可能漏判，此处全量清理保证
|
||||
# 依赖图无注入环，Phase A 多遍可收敛。破环后同步重写 deps.txt。
|
||||
# After scanning, clear _list/_dict injection cycle edges (e.g. _list<->memhub) so the
|
||||
# graph has none and Phase A converges; deps.txt is rewritten in sync.
|
||||
_BreakContainerCyclesInDeps(temp_dir)
|
||||
|
||||
# 写 _sha1_map.txt（仅可达 includes，人类可读输出；机器分析使用内存存储器）
|
||||
# Write _sha1_map.txt (reachable includes only, human-readable; analysis uses the in-memory store)
|
||||
StubMergerCollect.WriteIncludesSha1Map(mb, temp_dir, result, reachable_set, reachable_count)
|
||||
|
||||
# 兼容旧引用：sha1_set/set_count 对齐可达集合（use_reachable=1 时不被使用）
|
||||
# Legacy compat: sha1_set/set_count alias the reachable set (unused when use_reachable=1)
|
||||
sha1_set: str = reachable_set
|
||||
set_count: int = reachable_count
|
||||
# 模块 SHA1 映射已由依赖图扫描登记到 StubMerger 全局存储器（仅可达 includes + App），
|
||||
# HandlesExprCall._lookup_module_sha1 直接查全局存储器。
|
||||
# Module SHA1 map is registered into StubMerger's global store by the dependency scan;
|
||||
# HandlesExprCall._lookup_module_sha1 queries that store directly.
|
||||
|
||||
# FileEntry 结构体大小（Phase 1a 和 1b 共用）
|
||||
# FileEntry struct size (shared by Phase 1a and 1b)
|
||||
entry_size: sizet = IncludesScanner.FileEntry.__sizeof__()
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Phase 1a: 注册可达 includes 文件的 struct/enum/union
|
||||
#
|
||||
# 解决字母序依赖问题：1a 先把所有 struct/enum/union 注册到全局表，1b 全量翻译时 struct 已注册、走 existing 路径只翻译方法体；
|
||||
# 只处理可达文件；处理前按类继承关系拓扑排序，确保父类所在文件先于子类所在文件。
|
||||
# ============================================================
|
||||
# Phase 1a: register struct/enum/union for reachable includes (fixes alphabetical-order deps).
|
||||
p1a_reg: int = 0
|
||||
p1a_skp: int = 0
|
||||
p1a_fl: int = 0
|
||||
|
||||
# ============================================================
|
||||
# [临时取证打点] Phase1 早期扁平缓冲分配失败现场快照（定位后清理）；仅只读 mb 状态 + 打印，不改变任何分配行为与逻辑。
|
||||
# 三维度判定：① OOM（dump_stats 显示 used≈total、free≈0）；② mb 损坏（usable 基址/大小荒谬、self_check != 0）；③ 本次请求 size 算出的 order > max_order。
|
||||
# 用 VLogger.info（非终止；VLogger.error 会 sys.exit(1) 掩盖后续打点）。
|
||||
# ============================================================
|
||||
# [temp diagnostic] Snapshot mb state if the early Phase1 flat buffer allocation fails.
|
||||
if mb is not None:
|
||||
# 先做带环检测的 self_check：free list 被写坏成环时 dump_stats 的
|
||||
# free_bytes/free_count 无步数上限会死循环（打点自身卡死），故
|
||||
# self_check 返回 4（成环）时跳过 dump_stats。
|
||||
# Run a cycle-detecting self_check first: a corrupted free list would loop forever in
|
||||
# dump_stats, so skip dump_stats when self_check returns 4 (cycle).
|
||||
p1_sc: int = mb.self_check()
|
||||
if p1_sc != 4: mb.dump_stats("phase1-pre")
|
||||
else: VLogger.info("[取证] self_check=4（free list 成环），跳过 dump_stats 防死循环", "Phase1")
|
||||
p1_mo: int = mb.max_order
|
||||
if p1_mo < 0 or p1_mo > 32:
|
||||
p1_mo = 32 # 钳制：max_order 被写坏时防本打点自身越界读 free_lists
|
||||
p1_mo = 32 # Clamp: prevents this probe from out-of-bounds reading free_lists if max_order is corrupted
|
||||
p1_fl_head: uint64 = uint64(mb.free_lists[p1_mo])
|
||||
p1_mb_addr: uint64 = uint64(mb)
|
||||
p1_base_addr: uint64 = uint64(mb.base)
|
||||
@@ -397,7 +355,7 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
p1_fl_ds: uint64 = uint64(mb.free_lists[p1_o_ds])
|
||||
VLogger.info(f"[取证] order(class_names {p1_sz_cn}B+16)={p1_o_cn} fl={p1_fl_cn} " f"order(def_sha1s {p1_sz_ds}B+16)={p1_o_ds} fl={p1_fl_ds} " f"order(topo {p1_sz_tp}B+16)={p1_o_tp} order(box {p1_sz_bx}B+16)={p1_o_bx} " f"(fl==0 只说明该阶空、仍可由更高阶拆分；若 alloc 失败而 dump_stats 的 free>0 才是元数据损坏铁证，" f"任一 order>max_order={mb.max_order} 则是 size 异常)", "Phase1")
|
||||
|
||||
# 预扫描类继承关系，构建拓扑顺序（用扁平字节数组，避免结构体 str 指针 NULL 崩溃）
|
||||
# Pre-scan class inheritance and build the topo order (flat byte arrays avoid str-pointer NULL crashes)
|
||||
class_names_buf: bytes = mb.alloc(Phase1.MAX_CLASSES * Phase1.CLASS_NAME_LEN)
|
||||
parent_names_buf: bytes = mb.alloc(Phase1.MAX_CLASSES * Phase1.CLASS_NAME_LEN)
|
||||
def_sha1s_buf: bytes = mb.alloc(Phase1.MAX_CLASSES * Phase1.SHA1_LEN)
|
||||
@@ -409,9 +367,8 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
if generic_file_count_box is not None: string.memset(generic_file_count_box, 0, 4)
|
||||
topo_count: int = 0
|
||||
|
||||
# 三块扁平缓冲是 Phase1Scan._ScanClassInheritance 的契约前置缓冲：该函数入口对 NULL fail-fast（静默跳过整段会退化为字母序并掩盖 OOM 等内存故障），
|
||||
# 故分配失败时逐块显式报错终止（VLogger.error 内部 sys.exit(1)），绝不静默回退。
|
||||
# 注：topo_order/topo_count_box 仅供 Phase1Topo._TopoSortFiles 使用，有效性由其入口自身校验（NULL 即返回 -1 并终止），不并入此处终止逻辑。
|
||||
# These three flat buffers are Phase1Scan._ScanClassInheritance's contract pre-buffers; it
|
||||
# fails fast on NULL, so report and abort per buffer on allocation failure (never fall back).
|
||||
if class_names_buf is None: VLogger.error("类继承预扫描扁平缓冲分配失败：class_names_buf 为 NULL，终止编译", "Phase1")
|
||||
if parent_names_buf is None: VLogger.error("类继承预扫描扁平缓冲分配失败：parent_names_buf 为 NULL，终止编译", "Phase1")
|
||||
if def_sha1s_buf is None: VLogger.error("类继承预扫描扁平缓冲分配失败：def_sha1s_buf 为 NULL，终止编译", "Phase1")
|
||||
@@ -441,32 +398,23 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
topo_count = 0
|
||||
|
||||
# ============================================================
|
||||
# Phase 1a-pre-generic-all: mark1 前全量预注册 includes 泛型模板
|
||||
#
|
||||
# 非可达泛型文件（_list/_dict/linkedlist/mlirlite 等）的模板若在 Phase2（mark2 之后）才注册会被 reset 回收 → list[int]/dict[...] 特化失败。
|
||||
# 故对全部 includes 轻量预注册，使模板表与节点都落在 mark1 之前（_PreRegisterAllGenericTemplates 内有 generic marker 文本预筛，不 parse 全部 AST，OOM 风险可控）。
|
||||
# Phase 1a-pre-generic-all: pre-register all includes' generic templates before mark1.
|
||||
full_scan_g: rawptr[IncludesScanner.ScanResult] = IncludesScanner.scan_includes(mb, includes_dir)
|
||||
if full_scan_g is not None: Phase1Scan._PreRegisterAllGenericTemplates(mb, full_scan_g)
|
||||
else: Phase1Scan._PreRegisterAllGenericTemplates(mb, result)
|
||||
|
||||
# ============================================================
|
||||
# Phase 1a: 注册 struct/enum/union（mark1 前，全量 includes）
|
||||
#
|
||||
# OOM 治理：1a 对全部 includes parse 完整 AST，若驻留到 mark1 会累积至 1GB 主池耗尽。故：
|
||||
# 1) StdlibAlloc=1：1a 注册的跨文件持久数据（StructTable 表、Fields/Name/ModuleSha1、struct_ty 深拷贝、枚举/Renum/泛型模板等）全部落 perm_pool（永不 reset）；
|
||||
# 2) 每文件翻译完 reset_to(文件级 mark)：只回收该文件的 AST/lexer/token/IR 等瞬时数据，注册数据不受影响。
|
||||
# mark1 仍在 1a 结束后打，1b 每文件 reset_to(mark1) 语义不变。
|
||||
# ============================================================
|
||||
# Phase 1a: register struct/enum/union (before mark1, all includes) with OOM control.
|
||||
GenApi.set_stdlib_alloc(1)
|
||||
|
||||
# 遍历文件：优先使用拓扑顺序，回退到字母序
|
||||
# Iterate files: topo order preferred, falling back to alphabetical
|
||||
iter_count: int = topo_count if topo_count > 0 else result.Count
|
||||
# 1a 阶段总耗时起点（struct/enum/union 注册）
|
||||
# 1a total-time start (struct/enum/union registration)
|
||||
t_a0: uint64 = w32.win32base.GetTickCount64()
|
||||
for i in range(iter_count):
|
||||
# 每文件级 mark：翻译完回收该文件 AST（注册数据在 perm_pool）
|
||||
# Per-file mark: free this file's AST after translating (registered data is in perm_pool)
|
||||
p1a_mark: sizet = mb.mark()
|
||||
# 获取文件索引：拓扑顺序或字母序
|
||||
# Get the file index: topo order or alphabetical
|
||||
file_idx: int = i
|
||||
if topo_count > 0:
|
||||
od_addr: uint64 = uint64(topo_order) + i * 4
|
||||
@@ -484,7 +432,7 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
p1a_fl += 1
|
||||
continue
|
||||
|
||||
# 按需翻译过滤（拓扑排序已过滤，但字母序回退需要检查）
|
||||
# On-demand filter (topo already filtered; the alphabetical fallback must check)
|
||||
if topo_count == 0:
|
||||
if use_reachable != 0:
|
||||
if StubMergerCollect._is_in_sha1_set(sha1_r, reachable_set, reachable_count) == 0:
|
||||
@@ -495,17 +443,17 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
p1a_skp += 1
|
||||
continue
|
||||
|
||||
# 单文件计时起点（读文件 + parse 段，定位 1a 卡点）
|
||||
# Per-file timer start (read + parse, to locate 1a stalls)
|
||||
t_fa0: uint64 = w32.win32base.GetTickCount64()
|
||||
|
||||
# 读取文件内容
|
||||
# Read file contents
|
||||
file_path_r: str = entry_r.Path
|
||||
f_r: rawptr[fileio.File] = fileio.File(file_path_r, fileio.MODE.R)
|
||||
if f_r.closed:
|
||||
p1a_fl += 1
|
||||
continue
|
||||
|
||||
# read_all 的容量参数含终止符(最多读 cap-1 字节), 文件更长会被静默截断
|
||||
# read_all's capacity includes the terminator (reads at most cap-1 bytes); longer files are truncated
|
||||
if f_r.size() >= Phase1.SRC_BUF_SIZE:
|
||||
VLogger.error(f"源文件超过 {Phase1.SRC_BUF_SIZE} 字节上限, 拒绝读取: {file_path_r}", "Phase1")
|
||||
f_r.close()
|
||||
@@ -526,7 +474,7 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
continue
|
||||
if bytes_read_r < Phase1.SRC_BUF_SIZE: src_buf_r[bytes_read_r] = 0
|
||||
|
||||
# 解析 AST
|
||||
# Parse AST
|
||||
lx_r: ast.Lexer* = ast.new_lexer(mb)
|
||||
if lx_r is None:
|
||||
stdlib.free(src_buf_r)
|
||||
@@ -540,10 +488,10 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
p1a_fl += 1
|
||||
continue
|
||||
|
||||
# 计时点 1：读文件 + parse 完成
|
||||
# Timer 1: read + parse done
|
||||
t_fa1: uint64 = w32.win32base.GetTickCount64()
|
||||
|
||||
# 创建 Translator，设置 _declare_only=1（注册 struct/enum/union）
|
||||
# Create Translator with _declare_only=1 (register struct/enum/union)
|
||||
tr_r: rawptr[HandlesTranslator.Translator] = HandlesTranslator.Translator()
|
||||
if tr_r is None:
|
||||
stdlib.free(src_buf_r)
|
||||
@@ -559,34 +507,32 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
GenApi.clear_ctypedefs()
|
||||
GenApi.reset_visible_structs(mb, 0)
|
||||
ret_r: int = tr_r.translate(tree_r)
|
||||
# 计时点 2：translate(declare=1) 返回
|
||||
# Timer 2: translate(declare=1) returned
|
||||
t_fa2: uint64 = w32.win32base.GetTickCount64()
|
||||
if ret_r != 0: p1a_fl += 1
|
||||
else: p1a_reg += 1
|
||||
|
||||
# 逐文件进度日志（parse/translate 两段，定位 1a 卡点）
|
||||
# Per-file progress log (parse/translate segments, to locate 1a stalls)
|
||||
VLogger.info(f"[{i+1}/{iter_count}] 1a: {file_path_r} (parse={t_fa1-t_fa0} ms, translate={t_fa2-t_fa1} ms, total={t_fa2-t_fa0} ms)", "Phase1")
|
||||
|
||||
# 释放 Translator 的 C malloc 资源
|
||||
# Release the Translator's C malloc resources
|
||||
if tr_r._global_names is not None: stdlib.free(tr_r._global_names)
|
||||
if tr_r._nonlocal_names is not None: stdlib.free(tr_r._nonlocal_names)
|
||||
stdlib.free(src_buf_r)
|
||||
|
||||
# [TEMP-DIAG] 1a 每文件哨兵：校验本文件窗口 [p1a_mark, chk_depth) 内所有存活块的块尾 canary。
|
||||
# 位置必须在 reset_to 之前：reset_to 会把 chk_depth 截断回 p1a_mark，之后再查同一 mark 时 check_alive_blocks 直接 return 0（窗口为空、必然漏报）。
|
||||
# p1a_mark 在循环体首行捕获（本文件 L597），故窗口恰为「本文件 parse + translate(declare=1)」期间分配的块。
|
||||
# 仅 --detection-deep（mb.deep_detect != 0）时执行（每文件 O(n) 遍历活块）；读法同本文件既有 mb.detection：直接读 MemBuddy 实例的公开 int 字段。
|
||||
# [TEMP-DIAG] Per-file sentinel: check the tail canary of live blocks in [p1a_mark,
|
||||
# chk_depth). Must run before reset_to; only under --detection-deep (mb.deep_detect != 0).
|
||||
if mb.deep_detect != 0:
|
||||
chk_rc_1a: int = mb.check_alive_blocks(p1a_mark)
|
||||
if chk_rc_1a != 0:
|
||||
VLogger.error(f"内存损坏(1a 每文件后续, rc={chk_rc_1a}): {file_path_r}", "Phase1")
|
||||
stdlib.exit(1)
|
||||
|
||||
# 回收本文件 AST/lexer/token/IR 等瞬时数据（注册数据已在 perm_pool，
|
||||
# reset 不影响；IncludesScanner result 等在 p1a_mark 前分配同样保留）
|
||||
# Reclaim this file's transient AST/lexer/token/IR (registered data is in perm_pool, so
|
||||
# reset does not affect it; IncludesScanner result predates p1a_mark and is kept too).
|
||||
mb.reset_to(p1a_mark)
|
||||
|
||||
# 1a 阶段总耗时（struct/enum/union 注册，含每文件读+parse+翻译声明）
|
||||
# 1a total time (struct/enum/union registration, incl. per-file read+parse+declaration)
|
||||
t_a1: uint64 = w32.win32base.GetTickCount64()
|
||||
VLogger.info(f"1a 总耗时: {t_a1-t_a0} ms (注册 struct/enum, reg={p1a_reg} skp={p1a_skp})", "Phase1")
|
||||
|
||||
@@ -594,33 +540,26 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
VLogger.error(f"Phase 1a 有 {p1a_fl} 个文件 struct 注册失败", "Phase1")
|
||||
return 1
|
||||
|
||||
# 释放拓扑排序资源（topo_order/topo_count_box 保留到 Phase 1b 结束，
|
||||
# 1b 按拓扑序翻译需要它们；class/def 数组 1b 不再使用）
|
||||
# Release topo-sort resources (topo_order/topo_count_box stay until Phase 1b ends;
|
||||
# the class/def arrays are no longer used by 1b).
|
||||
if class_names_buf is not None: mb.free(class_names_buf)
|
||||
if parent_names_buf is not None: mb.free(parent_names_buf)
|
||||
if def_sha1s_buf is not None: mb.free(def_sha1s_buf)
|
||||
if generic_file_indices is not None: mb.free(generic_file_indices)
|
||||
if generic_file_count_box is not None: mb.free(generic_file_count_box)
|
||||
|
||||
# ============================================================
|
||||
# Phase 1b: 全量翻译（struct 已注册，走 existing 路径翻译方法体）
|
||||
# ============================================================
|
||||
# OOM 回收：mark1 在 1a-pre/泛型预注册/1a 全部结束后打，所有跨文件持久表（_struct_table/_enum_table/_generic_class_nodes/_renum_*/关键字表等）都落在 mark1 之前；
|
||||
# 1b 每文件翻译完后 reset_to(mark1) 回收该文件的 AST/IR/Handler/dump 临时，避免 90 文件累积导致 1GB 池 OOM。
|
||||
# Phase 1b: full translation (structs already registered; translate method bodies).
|
||||
# mark1 precedes all cross-file persistent tables; reset per file to avoid pool exhaustion.
|
||||
mark1: sizet = mb.mark()
|
||||
|
||||
# 1b 阶段新注册的 struct/enum 条目改 stdlib 分配（跨文件永久存活，不被 reset 回收）
|
||||
# Struct/enum entries registered in 1b use stdlib allocation (permanent, not reset)
|
||||
GenApi.set_stdlib_alloc(1)
|
||||
|
||||
# ============================================================
|
||||
# Phase 1b-pre: 预生成所有待翻译文件的 .vpi stub
|
||||
#
|
||||
# 依赖模块的 .vpi 必须在翻译前可用：1b 主循环按扫描顺序（非拓扑）翻译，若依赖模块后翻译则 LoadDepVpiIntoModule 读不到 .vpi
|
||||
# → declare 未注入 → 跨模块调用的返回类型推断错误（如误推为 Ptr 而 define 返回 i8*）→ 运行时返回值截断崩溃。
|
||||
# ============================================================
|
||||
# Phase 1b-pre: pre-generate .vpi stubs for all files to translate (deps must exist first).
|
||||
p1bpre_failed: int = 0
|
||||
dir_len_pv: sizet = string.strlen(temp_dir)
|
||||
# 1b-pre 阶段总耗时起点（vpi 预生成）
|
||||
# 1b-pre total-time start (vpi pre-generation)
|
||||
t_pv0: uint64 = w32.win32base.GetTickCount64()
|
||||
for i_pv in range(result.Count):
|
||||
entry_addr_pv: uint64 = uint64(result.Entries) + i_pv * entry_size
|
||||
@@ -633,7 +572,7 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
if use_reachable != 0: in_set_pv = StubMergerCollect._is_in_sha1_set(sha1_pv, reachable_set, reachable_count)
|
||||
else: in_set_pv = StubMergerCollect._is_in_sha1_set(sha1_pv, sha1_set, set_count)
|
||||
if in_set_pv == 0: continue
|
||||
# vpi 已存在则跳过
|
||||
# Skip if the vpi already exists
|
||||
vpi_path_pv: str = StubMergerCollect._sliced_path(temp_dir, dir_len_pv, sha1_pv, "vpi")
|
||||
if vpi_path_pv is None:
|
||||
p1bpre_failed += 1
|
||||
@@ -645,14 +584,14 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
stdlib.free(vpi_path_pv)
|
||||
continue
|
||||
stdlib.free(vpi_path_pv)
|
||||
# 读取文件、解析 AST、生成 vpi
|
||||
# Read the file, parse the AST, generate the vpi
|
||||
file_path_pv: str = entry_pv.Path
|
||||
f_pv: rawptr[fileio.File] = fileio.File(file_path_pv, fileio.MODE.R)
|
||||
if f_pv.closed:
|
||||
p1bpre_failed += 1
|
||||
VLogger.error(f"[Phase 1b-pre] {file_path_pv} | 无法打开源文件", "Phase1")
|
||||
continue
|
||||
# read_all 的容量参数含终止符(最多读 cap-1 字节), 文件更长会被静默截断
|
||||
# read_all's capacity includes the terminator (reads at most cap-1 bytes); longer files are truncated
|
||||
if f_pv.size() >= Phase1.SRC_BUF_SIZE:
|
||||
p1bpre_failed += 1
|
||||
VLogger.error(f"[Phase 1b-pre] {file_path_pv} | 源文件超过 {Phase1.SRC_BUF_SIZE} 字节上限", "Phase1")
|
||||
@@ -684,10 +623,8 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
if tree_pv is None:
|
||||
stdlib.free(src_buf_pv)
|
||||
p1bpre_failed += 1
|
||||
# 解析失败：**日志行内联 行/列 + 原因**，并附完整 Python 风格 traceback
|
||||
# （Traceback + File/行 + 源码行 + caret + SyntaxError: 消息；消息含 token
|
||||
# 类型与前/后 token 附注，见 includes/ast/parser.vp 的 _record_parse_err_ctx
|
||||
# / parse_tokens 兜底补记），与单文件管线 BuildPipeline 同一条上报路径。
|
||||
# Parse failure: the log line inlines line/col + reason, plus a full Python-style
|
||||
# traceback (same reporting path as the single-file BuildPipeline).
|
||||
VLogger.error(f"[Phase 1b-pre] {file_path_pv} | AST 解析失败（第 {ast.get_parse_err_line()} 行第 {ast.get_parse_err_col()} 列: {ast.get_parse_err_msg()}）", "Phase1")
|
||||
GenApi.dump_parse_traceback(file_path_pv)
|
||||
continue
|
||||
@@ -707,19 +644,18 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
p1bpre_failed += 1
|
||||
VLogger.error(f"[Phase 1b-pre] {file_path_pv} | pyi 缓冲分配失败（OOM）", "Phase1")
|
||||
stdlib.free(src_buf_pv)
|
||||
# [TEMP-DIAG] 1b-pre 每文件哨兵（同上）：校验本文件窗口 [mark1, chk_depth) 内所有存活块的块尾 canary。
|
||||
# 同样必须在 reset_to(mark1) 之前（否则窗口被截断为空）。mark1 在 1b 段前捕获（本文件 L756），是 1b/1b-pre 的统一窗口起点。
|
||||
# 仅 --detection-deep（mb.deep_detect != 0）时执行，读法同上方 1a 哨兵。
|
||||
# [TEMP-DIAG] 1b-pre per-file sentinel (as above): check the tail canary of live blocks
|
||||
# in [mark1, chk_depth). Must run before reset_to(mark1); only under --detection-deep.
|
||||
if mb.deep_detect != 0:
|
||||
chk_rc_pv: int = mb.check_alive_blocks(mark1)
|
||||
if chk_rc_pv != 0:
|
||||
VLogger.error(f"内存损坏(1b-pre 每文件后续, rc={chk_rc_pv}): {file_path_pv}", "Phase1")
|
||||
stdlib.exit(1)
|
||||
|
||||
# 回收本文件 AST（vpi 已写盘，AST 不再需要；mark1 后无其他持久数据）
|
||||
# Reclaim this file's AST (the vpi is written; nothing else persists after mark1)
|
||||
mb.reset_to(mark1)
|
||||
|
||||
# 1b-pre 阶段总耗时（vpi 预生成）
|
||||
# 1b-pre total time (vpi pre-generation)
|
||||
t_pv1: uint64 = w32.win32base.GetTickCount64()
|
||||
VLogger.info(f"1b-pre 总耗时: {t_pv1-t_pv0} ms (vpi 预生成)", "Phase1")
|
||||
|
||||
@@ -727,18 +663,18 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
VLogger.error(f"Phase 1b-pre 有 {p1bpre_failed} 个文件 vpi 预生成失败（失败文件与原因见上方逐条 [Phase 1b-pre] 日志）", "Phase1")
|
||||
return 1
|
||||
|
||||
# 遍历每个文件（拓扑序优先：父类所在文件先翻译，确保跨模块 vtable
|
||||
# 生成时依赖模块的函数 TypeStr 已注册；topo_count==0 回退扫描顺序）
|
||||
# Iterate each file (topo order preferred so base-class files translate first and their
|
||||
# TypeStr is registered for cross-module vtables; fall back to scan order when topo_count==0).
|
||||
translated: int = 0
|
||||
skipped: int = 0
|
||||
failed: int = 0
|
||||
# 1b 阶段总耗时起点（全量翻译）
|
||||
# 1b total-time start (full translation)
|
||||
t_b0: uint64 = w32.win32base.GetTickCount64()
|
||||
|
||||
loop_n_b: int = result.Count
|
||||
if topo_count > 0: loop_n_b = topo_count
|
||||
for i in range(loop_n_b):
|
||||
# 获取 entry（拓扑序：file_idx 从 topo_order 取，否则用扫描索引）
|
||||
# Get the entry (topo: file_idx from topo_order, else the scan index)
|
||||
file_idx_b: int = i
|
||||
if topo_count > 0:
|
||||
od_addr_b: uint64 = uint64(topo_order) + sizet(i) * 4
|
||||
@@ -755,7 +691,7 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
failed += 1
|
||||
continue
|
||||
|
||||
# 检查 SHA1 是否在翻译集合中（拓扑序已过滤可达；仅回退扫描顺序时检查）
|
||||
# Check whether the SHA1 is in the translate set (topo already filtered; only the fallback checks)
|
||||
in_set: int = 1
|
||||
if topo_count == 0:
|
||||
in_set = 0
|
||||
@@ -765,7 +701,7 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
# 构造 stub 路径: {temp_dir}/{sha1}.stub.ll
|
||||
# Build the stub path: {temp_dir}/{sha1}.stub.ll
|
||||
dir_len: sizet = string.strlen(temp_dir)
|
||||
sha1_len: sizet = string.strlen(sha1)
|
||||
stub_path: str = StubMergerCollect._sliced_path(temp_dir, dir_len, sha1, "stub.ll")
|
||||
@@ -773,9 +709,8 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
failed += 1
|
||||
continue
|
||||
|
||||
# 检查 stub 是否已存在（按需翻译：跳过已存在的）
|
||||
# 例外：含 class 定义的模块必须翻译——类注册只在翻译时发生，增量编译跳过会导致跨模块类构造 find_struct_by_module 落空 → 返回类型 fail-fast。
|
||||
# 判据：扫描 1b-pre 生成的 vpi 是否有 "class " 行（vpi 为解析后存根，精确无注释/字符串误判）。
|
||||
# Skip already-existing stubs, except modules with class definitions: class registration
|
||||
# only happens at translation, so skipping breaks cross-module find_struct_by_module.
|
||||
force_translate: int = 0
|
||||
vpi_path_1b: str = StubMergerCollect._sliced_path(temp_dir, dir_len, sha1, "vpi")
|
||||
if vpi_path_1b is not None:
|
||||
@@ -803,22 +738,22 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
else: vf_1b.close()
|
||||
stdlib.free(vpi_path_1b)
|
||||
if force_translate == 1:
|
||||
# 含 class：强制翻译（覆盖 stub/text，重新注册类）
|
||||
# Has class: force translation (overwrite stub/text, re-register classes)
|
||||
rp: str = entry.RelPath
|
||||
VLogger.info(f"含 class 定义，强制翻译: {rp}(sha1={sha1})", "Phase1")
|
||||
else:
|
||||
# 检查 stub 是否已存在（按需翻译：跳过已存在的）
|
||||
# Skip already-existing stubs (on-demand translation)
|
||||
sf: rawptr[fileio.File] = fileio.File(stub_path, fileio.MODE.R)
|
||||
if not sf.closed:
|
||||
sf.close()
|
||||
# 检查 text.ll 是否也存在（虚表扫描需要 text.ll）
|
||||
# Check that text.ll exists too (vtable scan needs it)
|
||||
text_path: str = StubMergerCollect._sliced_path(temp_dir, dir_len, sha1, "text.ll")
|
||||
if text_path is not None:
|
||||
tf: rawptr[fileio.File] = fileio.File(text_path, fileio.MODE.R)
|
||||
if not tf.closed:
|
||||
tf.close()
|
||||
# sidecar（{sha1}.meta）与在线 FULL 产物（{sha1}.full.ll）也必须存在：
|
||||
# 缺失则重新翻译以补写（否则 Phase B+/C 的声明判定与编译拿不到数据）。
|
||||
# The sidecar ({sha1}.meta) and online FULL artifact ({sha1}.full.ll) must also exist;
|
||||
# otherwise re-translate to regenerate them for Phase B+/C.
|
||||
meta_path_1b: str = StubMergerCollect._sliced_path(temp_dir, dir_len, sha1, "meta")
|
||||
meta_ok_1b: int = 0
|
||||
if meta_path_1b is not None:
|
||||
@@ -840,20 +775,20 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
stdlib.free(stub_path)
|
||||
continue
|
||||
stdlib.free(text_path)
|
||||
# text.ll 不存在，需要重新翻译
|
||||
# text.ll missing, re-translate
|
||||
rp: str = entry.RelPath
|
||||
VLogger.info(f"text.ll 不存在，重新翻译: {rp}(sha1={sha1})", "Phase1")
|
||||
else:
|
||||
# stub 不存在，需要翻译
|
||||
# stub missing, translate
|
||||
rp: str = entry.RelPath
|
||||
VLogger.info(f"翻译: {rp}(sha1={sha1})", "Phase1")
|
||||
|
||||
stdlib.free(stub_path)
|
||||
|
||||
# 单文件计时起点：读文件（parse/translate/dump 三段，对齐 Phase2 插桩）
|
||||
# Per-file timer start: read (parse/translate/dump segments, matching Phase2 instrumentation)
|
||||
t_f0: uint64 = w32.win32base.GetTickCount64()
|
||||
|
||||
# 读取文件内容
|
||||
# Read file contents
|
||||
file_path: str = entry.Path
|
||||
f: rawptr[fileio.File] = fileio.File(file_path, fileio.MODE.R)
|
||||
if f is None:
|
||||
@@ -865,7 +800,7 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
failed += 1
|
||||
continue
|
||||
|
||||
# read_all 的容量参数含终止符(最多读 cap-1 字节), 文件更长会被静默截断
|
||||
# read_all's capacity includes the terminator (reads at most cap-1 bytes); longer files are truncated
|
||||
if f.size() >= Phase1.SRC_BUF_SIZE:
|
||||
VLogger.error(f"源文件超过 {Phase1.SRC_BUF_SIZE} 字节上限, 拒绝读取: {file_path}", "Phase1")
|
||||
f.close()
|
||||
@@ -887,7 +822,7 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
continue
|
||||
if bytes_read < Phase1.SRC_BUF_SIZE: src_buf[bytes_read] = 0
|
||||
|
||||
# 解析 AST
|
||||
# Parse AST
|
||||
lx: ast.Lexer* = ast.new_lexer(mb)
|
||||
if lx is None:
|
||||
stdlib.free(src_buf)
|
||||
@@ -897,23 +832,22 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
tokens: ast.Token* = ast.tokenize(lx)
|
||||
tree: ast.AST* = ast.parse_tokens(mb, tokens)
|
||||
if tree is None:
|
||||
# 解析失败：行/列 + 原因一并进日志行（原因由 parser.vp 的 _record_* /
|
||||
# parse_tokens 兜底补记提供）
|
||||
# Parse failure: line/col + reason go into the log line (reason supplied by parser.vp's
|
||||
# _record_* / parse_tokens fallback).
|
||||
VLogger.error(f"AST 解析失败: {file_path}（第 {ast.get_parse_err_line()} 行第 {ast.get_parse_err_col()} 列: {ast.get_parse_err_msg()}）", "Phase1")
|
||||
stdlib.free(src_buf)
|
||||
failed += 1
|
||||
continue
|
||||
|
||||
# 计时点 1：读文件 + lexer/tokenize/parse 完成
|
||||
# Timer 1: read + lexer/tokenize/parse done
|
||||
t_f1: uint64 = w32.win32base.GetTickCount64()
|
||||
|
||||
# 文本扫描注册 CTypedef：AST 字符串字段（attr/id）token 零拷贝指向 src_buf，跨阶段内容不稳定（悬空），读 attr 会 0xC0000005；
|
||||
# 故改用源码文本逐行匹配 "NAME: t.CTypedef = TYPE"，在 src_buf 有效期内注册。
|
||||
# Register CTypedef by scanning source text: AST string fields (attr/id) point into src_buf
|
||||
# and dangle across phases (reading attr faults), so match "NAME: t.CTypedef = TYPE" lines.
|
||||
GenApi.RegisterCTypedefFromSource(mb, src_buf)
|
||||
|
||||
# 生成 .vpi 存根文件（直接遍历 AST，不依赖 PythonToStubConverter）
|
||||
# 1b-pre 已为所有待翻译文件生成 vpi，此处仅兜底：vpi 缺失才生成
|
||||
# （消除 1b-pre 与 1b 对同一文件的重复 _GeneratePyiFromAst 工作）
|
||||
# Generate the .vpi stub (walk the AST directly). 1b-pre already generated one, so this
|
||||
# is only a fallback when the vpi is missing.
|
||||
vpi_path_1b: str = StubMergerCollect._sliced_path(temp_dir, dir_len, sha1, "vpi")
|
||||
if vpi_path_1b is not None:
|
||||
vf_1b: rawptr[fileio.File] = fileio.File(vpi_path_1b, fileio.MODE.R)
|
||||
@@ -933,7 +867,7 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
stdlib.free(pyi_buf)
|
||||
stdlib.free(vpi_path_1b)
|
||||
|
||||
# 翻译 AST → LLVM IR
|
||||
# Translate AST -> LLVM IR
|
||||
tr: rawptr[HandlesTranslator.Translator] = HandlesTranslator.Translator()
|
||||
if tr is None:
|
||||
stdlib.free(src_buf)
|
||||
@@ -942,10 +876,8 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
tr.ModuleSha1 = sha1
|
||||
tr.CurrentPackage = GenApi.compute_package_from_relpath(mb, entry.RelPath)
|
||||
GenApi.set_current_file_type(file_path)
|
||||
# --traceback: mirror the DWARF switch into the Gen-layer emitter (the
|
||||
# same BuildPipeline._traceback_mode that gates -frame-pointer=all /
|
||||
# dbghelp / the Install() ctor). include/*.vp modules are compiled here,
|
||||
# so this is what gives their frames DWARF line info.
|
||||
# --traceback: mirror the DWARF switch into the Gen-layer emitter (same
|
||||
# BuildPipeline._traceback_mode that gates -frame-pointer=all / dbghelp / Install()).
|
||||
DebugInfoGen.SetEnabled(BuildPipeline._traceback_mode)
|
||||
GenApi.set_current_file_imports(file_path)
|
||||
GenApi.set_current_module_sha1(sha1)
|
||||
@@ -953,44 +885,40 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
GenApi.clear_ctypedefs()
|
||||
GenApi.reset_visible_structs(mb, 0)
|
||||
|
||||
# 翻译前哨兵：AST 解析（lexer/tokenize/parse_tokens）同样在 pool 分配，
|
||||
# 若解析器越界写破坏了存活块 header，此处立即定位到当前文件
|
||||
# （崩溃点漂移排查：把"翻译过程中随机崩"变成确定性定位）
|
||||
# Pre-translate sentinel: parsing also allocates from the pool, so a parser overrun that
|
||||
# corrupts a live block header is pinned to the current file (deterministic crash location).
|
||||
pre_rc: int = mb.check_alive_blocks(mark1)
|
||||
if pre_rc != 0:
|
||||
VLogger.error(f"内存损坏(翻译前存活块, rc={pre_rc}): {file_path}", "Phase1")
|
||||
stdlib.exit(1)
|
||||
|
||||
ret: int = tr.translate(tree)
|
||||
# 计时点 2：translate 返回
|
||||
# Timer 2: translate returned
|
||||
t_f2: uint64 = w32.win32base.GetTickCount64()
|
||||
# 诊断：translate 正常返回（若挂起在 translate 内部，此行不会打印，
|
||||
# 用于区分"translate 内部挂起"与"后续 dump 挂起"）
|
||||
# Diagnostic: translate returned normally (if it hangs inside, this line is not printed,
|
||||
# distinguishing an internal hang from a later dump hang).
|
||||
if ret == 0: VLogger.info(f"翻译完成(translate返回): {file_path}", "Phase1")
|
||||
if ret != 0:
|
||||
VLogger.error(f"翻译失败: {file_path}", "Phase1") # error 终止
|
||||
VLogger.error(f"翻译失败: {file_path}", "Phase1") # error terminates
|
||||
stdlib.free(src_buf)
|
||||
if tr._global_names is not None: stdlib.free(tr._global_names)
|
||||
if tr._nonlocal_names is not None: stdlib.free(tr._nonlocal_names)
|
||||
failed += 1
|
||||
continue
|
||||
|
||||
# --traceback: inject the startup Install() ctor when translating traceback.vp
|
||||
# (its stub/text.ll below are what Phase2 reuses to build the .obj). The
|
||||
# injector lives in the Gen layer (already depends on mlirlite), so this
|
||||
# module gains no cross-module mlirlite symbol dependency.
|
||||
# --traceback: inject the startup Install() ctor when translating traceback.vp;
|
||||
# the injector lives in the Gen layer, adding no cross-module mlirlite dependency.
|
||||
if BuildPipeline._traceback_mode != 0 and BuildPipeline._tb_basename_is_traceback(file_path) != 0: HandlesTranslator.InjectTracebackInstallCtor(tr)
|
||||
|
||||
# 诊断：pre_rc 哨兵在 translate 前、覆盖不到 translate 期间的 AST/IR 写穿，故 dump 前立即检查存活块 header/canary
|
||||
# （rc=6 块尾 canary 损坏 = 越界写穿数据区），把「dump 时链表成环卡死」变成「定位到 translate 期间写穿发生的文件」。
|
||||
# Diagnostic: the pre-translate sentinel misses write-throughs during translate, so check
|
||||
# live block headers/canaries before dump (rc=6 means a tail canary was overwritten).
|
||||
post_rc: int = mb.check_alive_blocks(mark1)
|
||||
if post_rc != 0: VLogger.error(f"内存损坏(translate后存活块, rc={post_rc}): {file_path}", "Phase1")
|
||||
|
||||
# dump stub IR(declarations only)
|
||||
# 1MB 不够：ast/parser.vp 的 text.ll 超 1MB 被截断 → 合并 IR 尾部丢字
|
||||
# （opt: "end of file in global variable name"），提至 8MB。
|
||||
# Dump stub IR (declarations only). 1MB is not enough: ast/parser.vp's text.ll exceeds it
|
||||
# and is truncated (opt: "end of file in global variable name"), so use 8MB.
|
||||
PHASE1_IR_SIZE: sizet = 8388608
|
||||
# FULL dump = stub + text（同一模块），留 2× 余量
|
||||
# FULL dump = stub + text (same module), with 2x headroom
|
||||
PHASE1_FULL_SIZE: sizet = 16777216
|
||||
stub_buf: bytes = stdlib.malloc(PHASE1_IR_SIZE)
|
||||
if stub_buf is None:
|
||||
@@ -1034,11 +962,11 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
stdlib.free(text_path_p1)
|
||||
stdlib.free(text_buf)
|
||||
|
||||
# save {sha1}.meta（结构化 sidecar：函数清单/结构体布局/虚表，替代从 IR 文本反推）
|
||||
# save {sha1}.meta (structured sidecar: function list/struct layout/vtables, no IR-text reverse-engineering)
|
||||
StubMergerIR.WriteModuleMeta(tr.Module, temp_dir, dir_len_p1, sha1)
|
||||
|
||||
# save {sha1}.full.ll（在线 FULL 产物：模型在线时的自足单模块 IR，供 Phase B 直接编译，
|
||||
# 取代 stub + 依赖 stub/text 文本合并）
|
||||
# save {sha1}.full.ll (the online FULL artifact: a self-contained single-module IR for
|
||||
# Phase B to compile directly, replacing the stub + dependency text merge).
|
||||
full_buf_p1: bytes = stdlib.malloc(PHASE1_FULL_SIZE)
|
||||
if full_buf_p1 is None: VLogger.error("full.ll 缓冲区分配失败", "Phase1")
|
||||
tr.dump_ir(full_buf_p1, PHASE1_FULL_SIZE, mlirlite.OUTPUT_FULL)
|
||||
@@ -1052,15 +980,15 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
stdlib.free(full_path_p1)
|
||||
stdlib.free(full_buf_p1)
|
||||
|
||||
# --traceback: 取出本模块的调试调用点记录（mlirlite 累积）并写 <sha1>.vpcall，
|
||||
# 供链接后 WriteStackMap 结合 objdump 还原调用点行号。关闭时不产生任何旁文件。
|
||||
# --traceback: take this module's debug call-site records (accumulated in mlirlite) and
|
||||
# write <sha1>.vpcall for WriteStackMap to reconstruct call-site lines. No file when off.
|
||||
if BuildPipeline._traceback_mode != 0:
|
||||
vpcall_text_p1: str = mlirlite.DebugTakeText()
|
||||
if vpcall_text_p1 is not None:
|
||||
BuildPipeline.WriteCallSidecar(temp_dir, sha1, vpcall_text_p1)
|
||||
stdlib.free(vpcall_text_p1)
|
||||
|
||||
# 计时点 3：dump（stub+text）完成；输出单文件三段耗时
|
||||
# Timer 3: dump (stub+text) done; print the per-file three-segment timing
|
||||
t_f3: uint64 = w32.win32base.GetTickCount64()
|
||||
VLogger.info(f"[{i+1}/{loop_n_b}] 翻译: {file_path} (parse={t_f1-t_f0} ms, translate={t_f2-t_f1} ms, dump={t_f3-t_f2} ms, total={t_f3-t_f0} ms)", "Phase1")
|
||||
|
||||
@@ -1075,33 +1003,30 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
df_p1.close()
|
||||
stdlib.free(deps_path_p1)
|
||||
|
||||
# 释放 Translator 的 C malloc 资源
|
||||
# Release the Translator's C malloc resources
|
||||
if tr._global_names is not None: stdlib.free(tr._global_names)
|
||||
if tr._nonlocal_names is not None: stdlib.free(tr._nonlocal_names)
|
||||
|
||||
# 所有 dump_ir 完成，src_buf 不再需要
|
||||
# All dump_ir calls done, src_buf no longer needed
|
||||
stdlib.free(src_buf)
|
||||
|
||||
translated += 1
|
||||
|
||||
# OOM 回收：回收本文件翻译产生的瞬时数据（AST/tokens/lexer/IR/Handler/dump 临时）；持久表（struct/enum/REnum/spec 缓存/全局函数名等）已迁出 reset 区间（stdlib），
|
||||
# reset_to(mark1) 只回收 mark1 之后分配、未被单独 free 的块。
|
||||
#
|
||||
# 内存哨兵（崩溃点漂移排查）：reset 前检查存活块 header（抓越界写），reset 后检查 free list 完整性（抓链表损坏）；任一损坏立即终止，把「随机崩溃漂移」变成「确定性定位到当前文件」。
|
||||
# rc 语义（check_alive_blocks）：1=块地址低于 usable，2=高于 usable+size，3=order 为负，4=order 超 max_order，5=块未对齐到 order 大小，6=块尾 canary 损坏（越界写穿数据区）
|
||||
# rc 语义（self_check）：1/2=越界，3=对齐错，4=free list 成环/超步数
|
||||
# OOM reclamation: free this file's transient AST/tokens/lexer/IR/dump data (persistent
|
||||
# tables moved to stdlib). Memory sentinels locate overruns deterministically.
|
||||
chk_rc: int = mb.check_alive_blocks(mark1)
|
||||
if chk_rc != 0: VLogger.error(f"内存损坏(存活块 header, rc={chk_rc}): {file_path}", "Phase1")
|
||||
# 越界写是崩溃根因（与回收/UAF 无关，已由 A/B 实验确认）——恢复 reset_to，改用块尾 canary（check_alive_blocks 校验）把越界写事发地定位到具体文件。
|
||||
# Out-of-bounds writes are the crash root cause (unrelated to reclamation/UAF, confirmed
|
||||
# by A/B tests); reset_to plus the tail canary pins the offending file.
|
||||
mb.reset_to(mark1)
|
||||
sc_rc: int = mb.self_check()
|
||||
if sc_rc != 0: VLogger.error(f"内存损坏(free list, rc={sc_rc}): {file_path}", "Phase1")
|
||||
|
||||
# 1b 阶段总耗时（全量翻译 + dump）
|
||||
# 1b total time (full translation + dump)
|
||||
t_b1: uint64 = w32.win32base.GetTickCount64()
|
||||
VLogger.info(f"1b 总耗时: {t_b1-t_b0} ms (全量翻译, trans={translated} skip={skipped} fail={failed})", "Phase1")
|
||||
|
||||
# 释放 Phase 1b 使用的拓扑排序资源
|
||||
# Release the topo-sort resources used by Phase 1b
|
||||
if topo_order is not None: mb.free(topo_order)
|
||||
if topo_count_box is not None: mb.free(topo_count_box)
|
||||
|
||||
@@ -1111,7 +1036,7 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
VLogger.error(f"Phase1 有 {failed} 个文件翻译失败", "Phase1")
|
||||
return 1
|
||||
|
||||
# 释放可达 SHA1 集合（如果分配了）
|
||||
# Release the reachable SHA1 set (if allocated)
|
||||
if reachable_set is not None: stdlib.free(reachable_set)
|
||||
|
||||
return 0
|
||||
Reference in new issue
Block a user