docs(tpv): R11 注释与 docstring 英文化并压缩到 2 行内
按 wiki/新规范：注释只讲作用、不掺杂历史原因、避免超长、尽可能英文（含三引号 __doc__）。全文 79 个 .vp：# 注释 16269 行（中文 13808）与 docstring（中文 1381 处）全部改英文；超 2 行的注释块（2093）与 docstring（397）全部压到 2 行。运行时可见文本未动（fatal_error/VLogger 的中文报错文案、stdio.printf 的横幅格式串）。校验：屏蔽注释与三引号后逐行比对各文件代码骨架，0 漂移。验证：自举重建 153.48s 通过（跨模块内联完成）；TPV 顺序回归 77/77 全通过。
This commit is contained in:
TermiNexus committed
1 parent
a2874a14e6
commit
0f4aeeceff
79 files changed
+10090
-20352
No files matched your search
+211
-493
File diff suppressed because it is too large.
Load diff
+142
-217
@@ -30,21 +30,17 @@ import lib.core.Gen.GenApi as GenApi
|
||||
import lib.core.Gen.DebugInfoGen as DebugInfoGen
|
||||
|
||||
|
||||
# ============================================================
|
||||
# _InjectedCreatesCycle - 容器模块自动注入前的破环检查
|
||||
#
|
||||
# 注入 _list/_dict 前检查被注入模块自身的依赖是否已含当前模块，命中（如 memhub↔_list）则跳过注入——
|
||||
# 否则 Phase A 多遍收敛死锁（"无法完成翻译"）；去掉环边后 _list 仍因当前模块被其依赖而处于闭包，list 类仍由 Phase1b 注册。
|
||||
# ============================================================
|
||||
# _InjectedCreatesCycle - cycle check before auto-injecting a container module
|
||||
# (_list/_dict); a hit (e.g. memhub<->_list) skips injection to avoid a Phase A deadlock.
|
||||
def _InjectedCreatesCycle(entry: rawptr[IncludesScanner.FileEntry], mod_name: str) -> int:
|
||||
"""注入 mod_name 到当前模块前检查是否成环，返回 1=成环需跳过注入"""
|
||||
"""Return 1 if injecting mod_name into the current module would create a cycle"""
|
||||
if entry is None or entry.RelPath is None or mod_name is None: return 0
|
||||
inj_sha1: str = StubMergerStore.LookupSha1ByModName(mod_name)
|
||||
if inj_sha1 is None: return 0
|
||||
inj_deps: str = StubMergerStore.LookupIncludesDeps(inj_sha1)
|
||||
if inj_deps is None: return 0
|
||||
rp: str = entry.RelPath
|
||||
# 当前模块名 = RelPath 基名（最后 '/' 或 '\\' 之后）去扩展名
|
||||
# Current module name = basename of RelPath (after the last '/' or '\'), without extension
|
||||
base: str = string.strrchr(rp, '/')
|
||||
if base is None: base = string.strrchr(rp, '\\')
|
||||
if base is not None: base = base + 1
|
||||
@@ -76,7 +72,7 @@ def _InjectedCreatesCycle(entry: rawptr[IncludesScanner.FileEntry], mod_name: st
|
||||
|
||||
|
||||
def _DepsContainsName(deps: str, name: str) -> int:
|
||||
"""deps 字符串（空格分隔模块名）是否含 name token（词边界精确匹配）"""
|
||||
"""Return 1 if the space-separated deps string contains the name token exactly"""
|
||||
if deps is None or name is None: return 0
|
||||
nl: sizet = string.strlen(name)
|
||||
dl: sizet = string.strlen(deps)
|
||||
@@ -91,7 +87,7 @@ def _DepsContainsName(deps: str, name: str) -> int:
|
||||
|
||||
|
||||
def _DepsRemoveName(deps: str, name: str) -> int:
|
||||
"""从 deps 字符串移除 name token（原地重建，返回 1=有改动）"""
|
||||
"""Remove the name token from the deps string in place; returns 1 if changed"""
|
||||
if deps is None or name is None: return 0
|
||||
nl: sizet = string.strlen(name)
|
||||
dl: sizet = string.strlen(deps)
|
||||
@@ -119,7 +115,7 @@ def _DepsRemoveName(deps: str, name: str) -> int:
|
||||
|
||||
|
||||
def _RewriteDepsTxt(temp_dir: str, sha1: str, deps_str: str) -> None:
|
||||
"""重写 {temp_dir}/{sha1前2位}/{sha1}.deps.txt（内存破环后同步磁盘）"""
|
||||
"""Rewrite the on-disk deps.txt file to match the in-memory deps after breaking a cycle"""
|
||||
if temp_dir is None or sha1 is None or deps_str is None: return
|
||||
td_len: sizet = string.strlen(temp_dir)
|
||||
dpath: str = StubMergerCollect._sliced_path(temp_dir, td_len, sha1, "deps.txt")
|
||||
@@ -133,17 +129,7 @@ def _RewriteDepsTxt(temp_dir: str, sha1: str, deps_str: str) -> None:
|
||||
|
||||
|
||||
def _BreakContainerCyclesInDeps(temp_dir: str) -> int:
|
||||
"""扫描后统一清理 _list/_dict 注入环边。
|
||||
|
||||
Phase1a-pre 全量扫描用 FindFirstFileA（文件系统序），注入 _list/_dict 时
|
||||
被注入模块（_list/_dict）自身的 deps 未必已注册 → _InjectedCreatesCycle
|
||||
内联检查可能漏判（如 _list 显式 import memhub，而 memhub 的 list[str]
|
||||
注入 _list 未被拦截）→ memhub↔_list 2-环 → Phase A 永不收敛。此处扫描
|
||||
结束后全量检查：凡某模块 M 的 deps 含 _list/_dict，且 _list/_dict 的 deps
|
||||
含 M → 从 M 的 deps 移除注入边（list 类仍由 Phase1b 经闭包注册）。
|
||||
内存破环后同步重写 {temp_dir}/deps.txt，保证磁盘与内存一致（Phase2 依赖
|
||||
图诊断回退读 deps.txt）。
|
||||
"""
|
||||
"""Remove _list/_dict injection cycle edges after scanning and sync deps.txt"""
|
||||
if StubMerger._includes_deps_sha1 is None or StubMerger._includes_deps_str is None: return 0
|
||||
cnt: sizet = StubMerger._includes_deps_count
|
||||
ci: sizet
|
||||
@@ -152,7 +138,7 @@ def _BreakContainerCyclesInDeps(temp_dir: str) -> int:
|
||||
deps_m: str = StubMerger._includes_deps_str + sizet(ci) * StubMerger.MAX_DEPS_STR_LEN
|
||||
if sha1_m is None or deps_m is None: continue
|
||||
if string.strstr(deps_m, "_list") is None and string.strstr(deps_m, "_dict") is None: continue
|
||||
# 该模块名（RelPath 基名去扩展名）
|
||||
# Module name (basename of RelPath without extension)
|
||||
rel_m: str = StubMergerStore.LookupSha1RelPath(sha1_m)
|
||||
if rel_m is None: continue
|
||||
base_m: str = string.strrchr(rel_m, '/')
|
||||
@@ -171,16 +157,16 @@ def _BreakContainerCyclesInDeps(temp_dir: str) -> int:
|
||||
if nm_m is None: continue
|
||||
string.strncpy(nm_m, base_m, 128)
|
||||
nm_m[128] = '\0'
|
||||
# _list ↔ M 环
|
||||
# _list <-> M cycle
|
||||
if string.strstr(deps_m, "_list") is not None:
|
||||
lst_sha1: str = StubMergerStore.LookupSha1ByModName("_list")
|
||||
if lst_sha1 is not None:
|
||||
lst_deps: str = StubMergerStore.LookupIncludesDeps(lst_sha1)
|
||||
if lst_deps is not None and _DepsContainsName(lst_deps, nm_m) != 0:
|
||||
if _DepsRemoveName(deps_m, "_list") != 0:
|
||||
# 内存破环后同步重写 deps.txt（诊断回退路径读文件）
|
||||
# Rewrite deps.txt after breaking the cycle in memory (diagnostic fallback reads the file)
|
||||
_RewriteDepsTxt(temp_dir, sha1_m, deps_m)
|
||||
# _dict ↔ M 环
|
||||
# _dict <-> M cycle
|
||||
if string.strstr(deps_m, "_dict") is not None:
|
||||
dct_sha1: str = StubMergerStore.LookupSha1ByModName("_dict")
|
||||
if dct_sha1 is not None:
|
||||
@@ -191,18 +177,10 @@ def _BreakContainerCyclesInDeps(temp_dir: str) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
# ============================================================
|
||||
# _BuildReachableResult - 从 Sha1Store 构建可达 FileEntry 列表
|
||||
#
|
||||
# 依赖图按需扫描（StubMergerBuild._BuildReachableSha1Set）只把「可达 includes + App」登记到全局 Sha1Store；
|
||||
# 本函数把 incdir 非空条目（includes）重建为 Phase1 1a/1b-pre/1b 需要的 FileEntry 列表。
|
||||
# App 条目（incdir 空）不进入 result——Phase B 已编译其 .obj，加入会导致 Phase B+ 重复编译 → 重复符号。
|
||||
#
|
||||
# Args: mb 内存池；includes_dir includes 根目录（仅用于失败诊断，路径以 store 为准）
|
||||
# Returns: ScanResult 指针（stdlib 分配，调用方负责释放）；失败返回 None
|
||||
# ============================================================
|
||||
# _BuildReachableResult - rebuild the reachable FileEntry list from the Sha1Store.
|
||||
# Only incdir-nonempty (includes) entries enter result; App entries would cause duplicate symbols.
|
||||
def _BuildReachableResult(mb: memhub.MemBuddy*, includes_dir: str) -> rawptr[IncludesScanner.ScanResult]:
|
||||
"""从 Sha1Store（仅可达 includes + App）构建可达 FileEntry 列表"""
|
||||
"""Build the reachable FileEntry list from the Sha1Store (reachable includes only)"""
|
||||
if mb is None: return None
|
||||
store_count: sizet = sizet(StubMerger._sha1_store_count)
|
||||
if store_count <= 0 or StubMerger._sha1_store_arr is None: return None
|
||||
@@ -213,18 +191,18 @@ def _BuildReachableResult(mb: memhub.MemBuddy*, includes_dir: str) -> rawptr[Inc
|
||||
for i in range(store_count):
|
||||
sidx: sizet = sizet(i) * 17
|
||||
if StubMerger._sha1_store_arr[sidx] == '\0': continue
|
||||
# 仅 includes 条目（incdir 非空）；App 条目不进入 result
|
||||
# Only includes entries (non-empty incdir); App entries do not enter result
|
||||
iidx: sizet = sizet(i) * StubMerger.MAX_INC_DIR_LEN
|
||||
if StubMerger._sha1_store_incdir[iidx] == '\0': continue
|
||||
eaddr: uint64 = uint64(result.Entries) + uint64(result.Count) * entry_size
|
||||
entry: rawptr[IncludesScanner.FileEntry] = rawptr[IncludesScanner.FileEntry](eaddr)
|
||||
if entry is None: continue
|
||||
# SHA1 深拷贝（store 槽 17 字节）
|
||||
# Deep-copy SHA1 (store slot is 17 bytes)
|
||||
sha1_buf: bytes = stdlib.malloc(17)
|
||||
if sha1_buf is None: continue
|
||||
string.strcpy(sha1_buf, StubMerger._sha1_store_arr + sidx)
|
||||
entry.Sha1 = sha1_buf
|
||||
# RelPath / IncDir 深拷贝
|
||||
# Deep-copy RelPath / IncDir
|
||||
ridx: sizet = sizet(i) * StubMerger.MAX_REL_PATH_LEN
|
||||
rel_len: sizet = string.strlen(StubMerger._sha1_store_rel + ridx)
|
||||
inc_len: sizet = string.strlen(StubMerger._sha1_store_incdir + iidx)
|
||||
@@ -239,7 +217,7 @@ def _BuildReachableResult(mb: memhub.MemBuddy*, includes_dir: str) -> rawptr[Inc
|
||||
string.strcpy(inc_buf, StubMerger._sha1_store_incdir + iidx)
|
||||
entry.RelPath = rel_buf
|
||||
entry.IncDir = inc_buf
|
||||
# 绝对路径 = inc_dir + '/' + rel_path（与 _TryResolveModuleFile 拼接一致）
|
||||
# Absolute path = inc_dir + '/' + rel_path (matches _TryResolveModuleFile)
|
||||
abs_buf: bytes = stdlib.malloc(inc_len + 1 + rel_len + 1)
|
||||
if abs_buf is None:
|
||||
stdlib.free(sha1_buf)
|
||||
@@ -250,7 +228,7 @@ def _BuildReachableResult(mb: memhub.MemBuddy*, includes_dir: str) -> rawptr[Inc
|
||||
abs_buf[inc_len] = '/'
|
||||
string.strcpy(abs_buf + inc_len + 1, rel_buf)
|
||||
entry.Path = abs_buf
|
||||
# 模块名（store 登记时保存的绝对模块名）
|
||||
# Module name (absolute module name saved when registering in the store)
|
||||
midx: sizet = sizet(i) * 64
|
||||
mod_len: sizet = string.strlen(StubMerger._sha1_store_mod + midx)
|
||||
if mod_len > 0:
|
||||
@@ -262,43 +240,33 @@ def _BuildReachableResult(mb: memhub.MemBuddy*, includes_dir: str) -> rawptr[Inc
|
||||
return result
|
||||
|
||||
# ============================================================
|
||||
# Phase1Main - Phase1 主流程（扫描 → 预注册 → 1a/1b 翻译 → 生成 stub）
|
||||
#
|
||||
# 函数实现直接位于本模块，调用方经本模块前缀访问；全局符号经 Phase1._xxx 跨模块访问。
|
||||
# ============================================================
|
||||
# Phase1Main - Phase1 main flow (scan -> pre-register -> 1a/1b translate -> generate stub)
|
||||
|
||||
# ============================================================
|
||||
# RunPhase1 - Phase1: 扫描 includes 目录，按需翻译并生成 stub
|
||||
#
|
||||
# 对每个 .vp/.py 文件检查 stub 是否已存在，不存在才翻译并分离 stub（避免重复工作）。
|
||||
#
|
||||
# Args: mb 内存池；includes_dir includes 目录路径；temp_dir 临时目录（保存 stub）；log 日志器
|
||||
# Returns: 0 成功，非 0 失败
|
||||
# ============================================================
|
||||
# RunPhase1 - scan the includes dir; translate and split stubs only when missing.
|
||||
def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawptr[VLogger.Logger]) -> int:
|
||||
"""Phase1: 扫描 includes 目录，按需翻译并生成 stub"""
|
||||
"""Phase1: scan the includes directory and translate/generate stubs on demand"""
|
||||
if includes_dir is None or temp_dir is None:
|
||||
VLogger.error("includes_dir 或 temp_dir 为空，无法运行 Phase1", "Phase1")
|
||||
return 1
|
||||
|
||||
# 设置全局 temp_dir（供跨模块 CDefine 查找使用）
|
||||
# Set the global temp_dir (used for cross-module CDefine lookup)
|
||||
GenApi.set_temp_dir(temp_dir)
|
||||
# 同步设置 HandlesExprCall 的 temp_dir（供 vpi stub 返回注解缓存按需加载）。
|
||||
# 必须在 includes 翻译前设置：Phase1 翻译 includes 时「同模块方法调用（定义在后）」需读 {temp_dir}/{sha1[0:2]}/{sha1}.vpi 兜底，Phase2 的 SetTempDir 太晚。
|
||||
# Also set HandlesExprCall's temp_dir (for the on-demand vpi return-annotation cache);
|
||||
# must run before includes translation since Phase2's SetTempDir is too late.
|
||||
GenApi.SetTempDir(temp_dir)
|
||||
|
||||
# 确保 temp 目录存在（build_dir/temp 可能尚未创建）
|
||||
# Ensure the temp dir exists (build_dir/temp may not exist yet)
|
||||
BuildPipeline.ensure_dir(temp_dir)
|
||||
|
||||
log.banner("Phase1: 扫描 includes（按需翻译）")
|
||||
|
||||
# 初始化 AST 表（只需一次）：必须在依赖图 BFS 扫描之前调用。
|
||||
# _init_tables 懒分配的关键字/操作符表（全局 _kw_head 等）若首次触发在 BFS 的文件级 mark 之后，reset_to 会回收表节点 → 全局悬空 → 后续 tokenize 崩溃。
|
||||
# Init the AST tables once, before the dependency-graph BFS: if _init_tables first runs
|
||||
# after the BFS file-level mark, reset_to frees the tables and later tokenize crashes.
|
||||
ast._init_tables(mb)
|
||||
|
||||
# 依赖图按需扫描（App 优先模型）：先扫 App 源文件（Config.SourceDir）的 import 依赖作为初始队列，
|
||||
# BFS 按文件系统解析 includes 文件（只对可达者读文件+算 SHA1+解析 import）并登记 Sha1Store/deps/写 deps.txt；
|
||||
# 未引用的 includes 从不读取。
|
||||
# On-demand dependency scan (App-first): seed the queue from App imports, then BFS-resolve
|
||||
# reachable includes only (read + SHA1 + import parse); unreferenced ones are never read.
|
||||
t_scan0: uint64 = w32.win32base.GetTickCount64()
|
||||
reachable_set: str = None
|
||||
reachable_count: int = 0
|
||||
@@ -326,58 +294,48 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
t_scan1: uint64 = w32.win32base.GetTickCount64()
|
||||
VLogger.info(f"依赖图扫描总耗时: {t_scan1 - t_scan0} ms（可达 {reachable_count} 个 includes）", "Phase1")
|
||||
|
||||
# 从 Sha1Store（仅可达 includes + App）构造可达 FileEntry 列表：
|
||||
# 后续 1a/1b-pre/1b 直接遍历 result（incdir 非空条目=includes），
|
||||
# use_reachable 过滤幂等保留（result 即可达子集）
|
||||
# Build the reachable FileEntry list from the Sha1Store; 1a/1b-pre/1b iterate result
|
||||
# (non-empty incdir = includes). use_reachable filtering is idempotent.
|
||||
result: rawptr[IncludesScanner.ScanResult] = _BuildReachableResult(mb, includes_dir)
|
||||
if result is None:
|
||||
VLogger.error("可达 FileEntry 列表构建失败，终止编译", "Phase1")
|
||||
return 1
|
||||
VLogger.info(f"可达 includes 条目: {result.Count}", "Phase1")
|
||||
|
||||
# 扫描后统一清理 _list/_dict 注入环边（_list↔memhub 等 2-环）。
|
||||
# 依赖图按需扫描顺序不可靠，内联检查可能漏判，此处全量清理保证
|
||||
# 依赖图无注入环，Phase A 多遍可收敛。破环后同步重写 deps.txt。
|
||||
# After scanning, clear _list/_dict injection cycle edges (e.g. _list<->memhub) so the
|
||||
# graph has none and Phase A converges; deps.txt is rewritten in sync.
|
||||
_BreakContainerCyclesInDeps(temp_dir)
|
||||
|
||||
# 写 _sha1_map.txt（仅可达 includes，人类可读输出；机器分析使用内存存储器）
|
||||
# Write _sha1_map.txt (reachable includes only, human-readable; analysis uses the in-memory store)
|
||||
StubMergerCollect.WriteIncludesSha1Map(mb, temp_dir, result, reachable_set, reachable_count)
|
||||
|
||||
# 兼容旧引用：sha1_set/set_count 对齐可达集合（use_reachable=1 时不被使用）
|
||||
# Legacy compat: sha1_set/set_count alias the reachable set (unused when use_reachable=1)
|
||||
sha1_set: str = reachable_set
|
||||
set_count: int = reachable_count
|
||||
# 模块 SHA1 映射已由依赖图扫描登记到 StubMerger 全局存储器（仅可达 includes + App），
|
||||
# HandlesExprCall._lookup_module_sha1 直接查全局存储器。
|
||||
# Module SHA1 map is registered into StubMerger's global store by the dependency scan;
|
||||
# HandlesExprCall._lookup_module_sha1 queries that store directly.
|
||||
|
||||
# FileEntry 结构体大小（Phase 1a 和 1b 共用）
|
||||
# FileEntry struct size (shared by Phase 1a and 1b)
|
||||
entry_size: sizet = IncludesScanner.FileEntry.__sizeof__()
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Phase 1a: 注册可达 includes 文件的 struct/enum/union
|
||||
#
|
||||
# 解决字母序依赖问题：1a 先把所有 struct/enum/union 注册到全局表，1b 全量翻译时 struct 已注册、走 existing 路径只翻译方法体；
|
||||
# 只处理可达文件；处理前按类继承关系拓扑排序，确保父类所在文件先于子类所在文件。
|
||||
# ============================================================
|
||||
# Phase 1a: register struct/enum/union for reachable includes (fixes alphabetical-order deps).
|
||||
p1a_reg: int = 0
|
||||
p1a_skp: int = 0
|
||||
p1a_fl: int = 0
|
||||
|
||||
# ============================================================
|
||||
# [临时取证打点] Phase1 早期扁平缓冲分配失败现场快照（定位后清理）；仅只读 mb 状态 + 打印，不改变任何分配行为与逻辑。
|
||||
# 三维度判定：① OOM（dump_stats 显示 used≈total、free≈0）；② mb 损坏（usable 基址/大小荒谬、self_check != 0）；③ 本次请求 size 算出的 order > max_order。
|
||||
# 用 VLogger.info（非终止；VLogger.error 会 sys.exit(1) 掩盖后续打点）。
|
||||
# ============================================================
|
||||
# [temp diagnostic] Snapshot mb state if the early Phase1 flat buffer allocation fails.
|
||||
if mb is not None:
|
||||
# 先做带环检测的 self_check：free list 被写坏成环时 dump_stats 的
|
||||
# free_bytes/free_count 无步数上限会死循环（打点自身卡死），故
|
||||
# self_check 返回 4（成环）时跳过 dump_stats。
|
||||
# Run a cycle-detecting self_check first: a corrupted free list would loop forever in
|
||||
# dump_stats, so skip dump_stats when self_check returns 4 (cycle).
|
||||
p1_sc: int = mb.self_check()
|
||||
if p1_sc != 4: mb.dump_stats("phase1-pre")
|
||||
else: VLogger.info("[取证] self_check=4（free list 成环），跳过 dump_stats 防死循环", "Phase1")
|
||||
p1_mo: int = mb.max_order
|
||||
if p1_mo < 0 or p1_mo > 32:
|
||||
p1_mo = 32 # 钳制：max_order 被写坏时防本打点自身越界读 free_lists
|
||||
p1_mo = 32 # Clamp: prevents this probe from out-of-bounds reading free_lists if max_order is corrupted
|
||||
p1_fl_head: uint64 = uint64(mb.free_lists[p1_mo])
|
||||
p1_mb_addr: uint64 = uint64(mb)
|
||||
p1_base_addr: uint64 = uint64(mb.base)
|
||||
@@ -397,7 +355,7 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
p1_fl_ds: uint64 = uint64(mb.free_lists[p1_o_ds])
|
||||
VLogger.info(f"[取证] order(class_names {p1_sz_cn}B+16)={p1_o_cn} fl={p1_fl_cn} " f"order(def_sha1s {p1_sz_ds}B+16)={p1_o_ds} fl={p1_fl_ds} " f"order(topo {p1_sz_tp}B+16)={p1_o_tp} order(box {p1_sz_bx}B+16)={p1_o_bx} " f"(fl==0 只说明该阶空、仍可由更高阶拆分；若 alloc 失败而 dump_stats 的 free>0 才是元数据损坏铁证，" f"任一 order>max_order={mb.max_order} 则是 size 异常)", "Phase1")
|
||||
|
||||
# 预扫描类继承关系，构建拓扑顺序（用扁平字节数组，避免结构体 str 指针 NULL 崩溃）
|
||||
# Pre-scan class inheritance and build the topo order (flat byte arrays avoid str-pointer NULL crashes)
|
||||
class_names_buf: bytes = mb.alloc(Phase1.MAX_CLASSES * Phase1.CLASS_NAME_LEN)
|
||||
parent_names_buf: bytes = mb.alloc(Phase1.MAX_CLASSES * Phase1.CLASS_NAME_LEN)
|
||||
def_sha1s_buf: bytes = mb.alloc(Phase1.MAX_CLASSES * Phase1.SHA1_LEN)
|
||||
@@ -409,9 +367,8 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
if generic_file_count_box is not None: string.memset(generic_file_count_box, 0, 4)
|
||||
topo_count: int = 0
|
||||
|
||||
# 三块扁平缓冲是 Phase1Scan._ScanClassInheritance 的契约前置缓冲：该函数入口对 NULL fail-fast（静默跳过整段会退化为字母序并掩盖 OOM 等内存故障），
|
||||
# 故分配失败时逐块显式报错终止（VLogger.error 内部 sys.exit(1)），绝不静默回退。
|
||||
# 注：topo_order/topo_count_box 仅供 Phase1Topo._TopoSortFiles 使用，有效性由其入口自身校验（NULL 即返回 -1 并终止），不并入此处终止逻辑。
|
||||
# These three flat buffers are Phase1Scan._ScanClassInheritance's contract pre-buffers; it
|
||||
# fails fast on NULL, so report and abort per buffer on allocation failure (never fall back).
|
||||
if class_names_buf is None: VLogger.error("类继承预扫描扁平缓冲分配失败：class_names_buf 为 NULL，终止编译", "Phase1")
|
||||
if parent_names_buf is None: VLogger.error("类继承预扫描扁平缓冲分配失败：parent_names_buf 为 NULL，终止编译", "Phase1")
|
||||
if def_sha1s_buf is None: VLogger.error("类继承预扫描扁平缓冲分配失败：def_sha1s_buf 为 NULL，终止编译", "Phase1")
|
||||
@@ -441,32 +398,23 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
topo_count = 0
|
||||
|
||||
# ============================================================
|
||||
# Phase 1a-pre-generic-all: mark1 前全量预注册 includes 泛型模板
|
||||
#
|
||||
# 非可达泛型文件（_list/_dict/linkedlist/mlirlite 等）的模板若在 Phase2（mark2 之后）才注册会被 reset 回收 → list[int]/dict[...] 特化失败。
|
||||
# 故对全部 includes 轻量预注册，使模板表与节点都落在 mark1 之前（_PreRegisterAllGenericTemplates 内有 generic marker 文本预筛，不 parse 全部 AST，OOM 风险可控）。
|
||||
# Phase 1a-pre-generic-all: pre-register all includes' generic templates before mark1.
|
||||
full_scan_g: rawptr[IncludesScanner.ScanResult] = IncludesScanner.scan_includes(mb, includes_dir)
|
||||
if full_scan_g is not None: Phase1Scan._PreRegisterAllGenericTemplates(mb, full_scan_g)
|
||||
else: Phase1Scan._PreRegisterAllGenericTemplates(mb, result)
|
||||
|
||||
# ============================================================
|
||||
# Phase 1a: 注册 struct/enum/union（mark1 前，全量 includes）
|
||||
#
|
||||
# OOM 治理：1a 对全部 includes parse 完整 AST，若驻留到 mark1 会累积至 1GB 主池耗尽。故：
|
||||
# 1) StdlibAlloc=1：1a 注册的跨文件持久数据（StructTable 表、Fields/Name/ModuleSha1、struct_ty 深拷贝、枚举/Renum/泛型模板等）全部落 perm_pool（永不 reset）；
|
||||
# 2) 每文件翻译完 reset_to(文件级 mark)：只回收该文件的 AST/lexer/token/IR 等瞬时数据，注册数据不受影响。
|
||||
# mark1 仍在 1a 结束后打，1b 每文件 reset_to(mark1) 语义不变。
|
||||
# ============================================================
|
||||
# Phase 1a: register struct/enum/union (before mark1, all includes) with OOM control.
|
||||
GenApi.set_stdlib_alloc(1)
|
||||
|
||||
# 遍历文件：优先使用拓扑顺序，回退到字母序
|
||||
# Iterate files: topo order preferred, falling back to alphabetical
|
||||
iter_count: int = topo_count if topo_count > 0 else result.Count
|
||||
# 1a 阶段总耗时起点（struct/enum/union 注册）
|
||||
# 1a total-time start (struct/enum/union registration)
|
||||
t_a0: uint64 = w32.win32base.GetTickCount64()
|
||||
for i in range(iter_count):
|
||||
# 每文件级 mark：翻译完回收该文件 AST（注册数据在 perm_pool）
|
||||
# Per-file mark: free this file's AST after translating (registered data is in perm_pool)
|
||||
p1a_mark: sizet = mb.mark()
|
||||
# 获取文件索引：拓扑顺序或字母序
|
||||
# Get the file index: topo order or alphabetical
|
||||
file_idx: int = i
|
||||
if topo_count > 0:
|
||||
od_addr: uint64 = uint64(topo_order) + i * 4
|
||||
@@ -484,7 +432,7 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
p1a_fl += 1
|
||||
continue
|
||||
|
||||
# 按需翻译过滤（拓扑排序已过滤，但字母序回退需要检查）
|
||||
# On-demand filter (topo already filtered; the alphabetical fallback must check)
|
||||
if topo_count == 0:
|
||||
if use_reachable != 0:
|
||||
if StubMergerCollect._is_in_sha1_set(sha1_r, reachable_set, reachable_count) == 0:
|
||||
@@ -495,17 +443,17 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
p1a_skp += 1
|
||||
continue
|
||||
|
||||
# 单文件计时起点（读文件 + parse 段，定位 1a 卡点）
|
||||
# Per-file timer start (read + parse, to locate 1a stalls)
|
||||
t_fa0: uint64 = w32.win32base.GetTickCount64()
|
||||
|
||||
# 读取文件内容
|
||||
# Read file contents
|
||||
file_path_r: str = entry_r.Path
|
||||
f_r: rawptr[fileio.File] = fileio.File(file_path_r, fileio.MODE.R)
|
||||
if f_r.closed:
|
||||
p1a_fl += 1
|
||||
continue
|
||||
|
||||
# read_all 的容量参数含终止符(最多读 cap-1 字节), 文件更长会被静默截断
|
||||
# read_all's capacity includes the terminator (reads at most cap-1 bytes); longer files are truncated
|
||||
if f_r.size() >= Phase1.SRC_BUF_SIZE:
|
||||
VLogger.error(f"源文件超过 {Phase1.SRC_BUF_SIZE} 字节上限, 拒绝读取: {file_path_r}", "Phase1")
|
||||
f_r.close()
|
||||
@@ -526,7 +474,7 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
continue
|
||||
if bytes_read_r < Phase1.SRC_BUF_SIZE: src_buf_r[bytes_read_r] = 0
|
||||
|
||||
# 解析 AST
|
||||
# Parse AST
|
||||
lx_r: ast.Lexer* = ast.new_lexer(mb)
|
||||
if lx_r is None:
|
||||
stdlib.free(src_buf_r)
|
||||
@@ -540,10 +488,10 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
p1a_fl += 1
|
||||
continue
|
||||
|
||||
# 计时点 1：读文件 + parse 完成
|
||||
# Timer 1: read + parse done
|
||||
t_fa1: uint64 = w32.win32base.GetTickCount64()
|
||||
|
||||
# 创建 Translator，设置 _declare_only=1（注册 struct/enum/union）
|
||||
# Create Translator with _declare_only=1 (register struct/enum/union)
|
||||
tr_r: rawptr[HandlesTranslator.Translator] = HandlesTranslator.Translator()
|
||||
if tr_r is None:
|
||||
stdlib.free(src_buf_r)
|
||||
@@ -559,34 +507,32 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
GenApi.clear_ctypedefs()
|
||||
GenApi.reset_visible_structs(mb, 0)
|
||||
ret_r: int = tr_r.translate(tree_r)
|
||||
# 计时点 2：translate(declare=1) 返回
|
||||
# Timer 2: translate(declare=1) returned
|
||||
t_fa2: uint64 = w32.win32base.GetTickCount64()
|
||||
if ret_r != 0: p1a_fl += 1
|
||||
else: p1a_reg += 1
|
||||
|
||||
# 逐文件进度日志（parse/translate 两段，定位 1a 卡点）
|
||||
# Per-file progress log (parse/translate segments, to locate 1a stalls)
|
||||
VLogger.info(f"[{i+1}/{iter_count}] 1a: {file_path_r} (parse={t_fa1-t_fa0} ms, translate={t_fa2-t_fa1} ms, total={t_fa2-t_fa0} ms)", "Phase1")
|
||||
|
||||
# 释放 Translator 的 C malloc 资源
|
||||
# Release the Translator's C malloc resources
|
||||
if tr_r._global_names is not None: stdlib.free(tr_r._global_names)
|
||||
if tr_r._nonlocal_names is not None: stdlib.free(tr_r._nonlocal_names)
|
||||
stdlib.free(src_buf_r)
|
||||
|
||||
# [TEMP-DIAG] 1a 每文件哨兵：校验本文件窗口 [p1a_mark, chk_depth) 内所有存活块的块尾 canary。
|
||||
# 位置必须在 reset_to 之前：reset_to 会把 chk_depth 截断回 p1a_mark，之后再查同一 mark 时 check_alive_blocks 直接 return 0（窗口为空、必然漏报）。
|
||||
# p1a_mark 在循环体首行捕获（本文件 L597），故窗口恰为「本文件 parse + translate(declare=1)」期间分配的块。
|
||||
# 仅 --detection-deep（mb.deep_detect != 0）时执行（每文件 O(n) 遍历活块）；读法同本文件既有 mb.detection：直接读 MemBuddy 实例的公开 int 字段。
|
||||
# [TEMP-DIAG] Per-file sentinel: check the tail canary of live blocks in [p1a_mark,
|
||||
# chk_depth). Must run before reset_to; only under --detection-deep (mb.deep_detect != 0).
|
||||
if mb.deep_detect != 0:
|
||||
chk_rc_1a: int = mb.check_alive_blocks(p1a_mark)
|
||||
if chk_rc_1a != 0:
|
||||
VLogger.error(f"内存损坏(1a 每文件后续, rc={chk_rc_1a}): {file_path_r}", "Phase1")
|
||||
stdlib.exit(1)
|
||||
|
||||
# 回收本文件 AST/lexer/token/IR 等瞬时数据（注册数据已在 perm_pool，
|
||||
# reset 不影响；IncludesScanner result 等在 p1a_mark 前分配同样保留）
|
||||
# Reclaim this file's transient AST/lexer/token/IR (registered data is in perm_pool, so
|
||||
# reset does not affect it; IncludesScanner result predates p1a_mark and is kept too).
|
||||
mb.reset_to(p1a_mark)
|
||||
|
||||
# 1a 阶段总耗时（struct/enum/union 注册，含每文件读+parse+翻译声明）
|
||||
# 1a total time (struct/enum/union registration, incl. per-file read+parse+declaration)
|
||||
t_a1: uint64 = w32.win32base.GetTickCount64()
|
||||
VLogger.info(f"1a 总耗时: {t_a1-t_a0} ms (注册 struct/enum, reg={p1a_reg} skp={p1a_skp})", "Phase1")
|
||||
|
||||
@@ -594,33 +540,26 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
VLogger.error(f"Phase 1a 有 {p1a_fl} 个文件 struct 注册失败", "Phase1")
|
||||
return 1
|
||||
|
||||
# 释放拓扑排序资源（topo_order/topo_count_box 保留到 Phase 1b 结束，
|
||||
# 1b 按拓扑序翻译需要它们；class/def 数组 1b 不再使用）
|
||||
# Release topo-sort resources (topo_order/topo_count_box stay until Phase 1b ends;
|
||||
# the class/def arrays are no longer used by 1b).
|
||||
if class_names_buf is not None: mb.free(class_names_buf)
|
||||
if parent_names_buf is not None: mb.free(parent_names_buf)
|
||||
if def_sha1s_buf is not None: mb.free(def_sha1s_buf)
|
||||
if generic_file_indices is not None: mb.free(generic_file_indices)
|
||||
if generic_file_count_box is not None: mb.free(generic_file_count_box)
|
||||
|
||||
# ============================================================
|
||||
# Phase 1b: 全量翻译（struct 已注册，走 existing 路径翻译方法体）
|
||||
# ============================================================
|
||||
# OOM 回收：mark1 在 1a-pre/泛型预注册/1a 全部结束后打，所有跨文件持久表（_struct_table/_enum_table/_generic_class_nodes/_renum_*/关键字表等）都落在 mark1 之前；
|
||||
# 1b 每文件翻译完后 reset_to(mark1) 回收该文件的 AST/IR/Handler/dump 临时，避免 90 文件累积导致 1GB 池 OOM。
|
||||
# Phase 1b: full translation (structs already registered; translate method bodies).
|
||||
# mark1 precedes all cross-file persistent tables; reset per file to avoid pool exhaustion.
|
||||
mark1: sizet = mb.mark()
|
||||
|
||||
# 1b 阶段新注册的 struct/enum 条目改 stdlib 分配（跨文件永久存活，不被 reset 回收）
|
||||
# Struct/enum entries registered in 1b use stdlib allocation (permanent, not reset)
|
||||
GenApi.set_stdlib_alloc(1)
|
||||
|
||||
# ============================================================
|
||||
# Phase 1b-pre: 预生成所有待翻译文件的 .vpi stub
|
||||
#
|
||||
# 依赖模块的 .vpi 必须在翻译前可用：1b 主循环按扫描顺序（非拓扑）翻译，若依赖模块后翻译则 LoadDepVpiIntoModule 读不到 .vpi
|
||||
# → declare 未注入 → 跨模块调用的返回类型推断错误（如误推为 Ptr 而 define 返回 i8*）→ 运行时返回值截断崩溃。
|
||||
# ============================================================
|
||||
# Phase 1b-pre: pre-generate .vpi stubs for all files to translate (deps must exist first).
|
||||
p1bpre_failed: int = 0
|
||||
dir_len_pv: sizet = string.strlen(temp_dir)
|
||||
# 1b-pre 阶段总耗时起点（vpi 预生成）
|
||||
# 1b-pre total-time start (vpi pre-generation)
|
||||
t_pv0: uint64 = w32.win32base.GetTickCount64()
|
||||
for i_pv in range(result.Count):
|
||||
entry_addr_pv: uint64 = uint64(result.Entries) + i_pv * entry_size
|
||||
@@ -633,7 +572,7 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
if use_reachable != 0: in_set_pv = StubMergerCollect._is_in_sha1_set(sha1_pv, reachable_set, reachable_count)
|
||||
else: in_set_pv = StubMergerCollect._is_in_sha1_set(sha1_pv, sha1_set, set_count)
|
||||
if in_set_pv == 0: continue
|
||||
# vpi 已存在则跳过
|
||||
# Skip if the vpi already exists
|
||||
vpi_path_pv: str = StubMergerCollect._sliced_path(temp_dir, dir_len_pv, sha1_pv, "vpi")
|
||||
if vpi_path_pv is None:
|
||||
p1bpre_failed += 1
|
||||
@@ -645,14 +584,14 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
stdlib.free(vpi_path_pv)
|
||||
continue
|
||||
stdlib.free(vpi_path_pv)
|
||||
# 读取文件、解析 AST、生成 vpi
|
||||
# Read the file, parse the AST, generate the vpi
|
||||
file_path_pv: str = entry_pv.Path
|
||||
f_pv: rawptr[fileio.File] = fileio.File(file_path_pv, fileio.MODE.R)
|
||||
if f_pv.closed:
|
||||
p1bpre_failed += 1
|
||||
VLogger.error(f"[Phase 1b-pre] {file_path_pv} | 无法打开源文件", "Phase1")
|
||||
continue
|
||||
# read_all 的容量参数含终止符(最多读 cap-1 字节), 文件更长会被静默截断
|
||||
# read_all's capacity includes the terminator (reads at most cap-1 bytes); longer files are truncated
|
||||
if f_pv.size() >= Phase1.SRC_BUF_SIZE:
|
||||
p1bpre_failed += 1
|
||||
VLogger.error(f"[Phase 1b-pre] {file_path_pv} | 源文件超过 {Phase1.SRC_BUF_SIZE} 字节上限", "Phase1")
|
||||
@@ -684,10 +623,8 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
if tree_pv is None:
|
||||
stdlib.free(src_buf_pv)
|
||||
p1bpre_failed += 1
|
||||
# 解析失败：**日志行内联 行/列 + 原因**，并附完整 Python 风格 traceback
|
||||
# （Traceback + File/行 + 源码行 + caret + SyntaxError: 消息；消息含 token
|
||||
# 类型与前/后 token 附注，见 includes/ast/parser.vp 的 _record_parse_err_ctx
|
||||
# / parse_tokens 兜底补记），与单文件管线 BuildPipeline 同一条上报路径。
|
||||
# Parse failure: the log line inlines line/col + reason, plus a full Python-style
|
||||
# traceback (same reporting path as the single-file BuildPipeline).
|
||||
VLogger.error(f"[Phase 1b-pre] {file_path_pv} | AST 解析失败（第 {ast.get_parse_err_line()} 行第 {ast.get_parse_err_col()} 列: {ast.get_parse_err_msg()}）", "Phase1")
|
||||
GenApi.dump_parse_traceback(file_path_pv)
|
||||
continue
|
||||
@@ -707,19 +644,18 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
p1bpre_failed += 1
|
||||
VLogger.error(f"[Phase 1b-pre] {file_path_pv} | pyi 缓冲分配失败（OOM）", "Phase1")
|
||||
stdlib.free(src_buf_pv)
|
||||
# [TEMP-DIAG] 1b-pre 每文件哨兵（同上）：校验本文件窗口 [mark1, chk_depth) 内所有存活块的块尾 canary。
|
||||
# 同样必须在 reset_to(mark1) 之前（否则窗口被截断为空）。mark1 在 1b 段前捕获（本文件 L756），是 1b/1b-pre 的统一窗口起点。
|
||||
# 仅 --detection-deep（mb.deep_detect != 0）时执行，读法同上方 1a 哨兵。
|
||||
# [TEMP-DIAG] 1b-pre per-file sentinel (as above): check the tail canary of live blocks
|
||||
# in [mark1, chk_depth). Must run before reset_to(mark1); only under --detection-deep.
|
||||
if mb.deep_detect != 0:
|
||||
chk_rc_pv: int = mb.check_alive_blocks(mark1)
|
||||
if chk_rc_pv != 0:
|
||||
VLogger.error(f"内存损坏(1b-pre 每文件后续, rc={chk_rc_pv}): {file_path_pv}", "Phase1")
|
||||
stdlib.exit(1)
|
||||
|
||||
# 回收本文件 AST（vpi 已写盘，AST 不再需要；mark1 后无其他持久数据）
|
||||
# Reclaim this file's AST (the vpi is written; nothing else persists after mark1)
|
||||
mb.reset_to(mark1)
|
||||
|
||||
# 1b-pre 阶段总耗时（vpi 预生成）
|
||||
# 1b-pre total time (vpi pre-generation)
|
||||
t_pv1: uint64 = w32.win32base.GetTickCount64()
|
||||
VLogger.info(f"1b-pre 总耗时: {t_pv1-t_pv0} ms (vpi 预生成)", "Phase1")
|
||||
|
||||
@@ -727,18 +663,18 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
VLogger.error(f"Phase 1b-pre 有 {p1bpre_failed} 个文件 vpi 预生成失败（失败文件与原因见上方逐条 [Phase 1b-pre] 日志）", "Phase1")
|
||||
return 1
|
||||
|
||||
# 遍历每个文件（拓扑序优先：父类所在文件先翻译，确保跨模块 vtable
|
||||
# 生成时依赖模块的函数 TypeStr 已注册；topo_count==0 回退扫描顺序）
|
||||
# Iterate each file (topo order preferred so base-class files translate first and their
|
||||
# TypeStr is registered for cross-module vtables; fall back to scan order when topo_count==0).
|
||||
translated: int = 0
|
||||
skipped: int = 0
|
||||
failed: int = 0
|
||||
# 1b 阶段总耗时起点（全量翻译）
|
||||
# 1b total-time start (full translation)
|
||||
t_b0: uint64 = w32.win32base.GetTickCount64()
|
||||
|
||||
loop_n_b: int = result.Count
|
||||
if topo_count > 0: loop_n_b = topo_count
|
||||
for i in range(loop_n_b):
|
||||
# 获取 entry（拓扑序：file_idx 从 topo_order 取，否则用扫描索引）
|
||||
# Get the entry (topo: file_idx from topo_order, else the scan index)
|
||||
file_idx_b: int = i
|
||||
if topo_count > 0:
|
||||
od_addr_b: uint64 = uint64(topo_order) + sizet(i) * 4
|
||||
@@ -755,7 +691,7 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
failed += 1
|
||||
continue
|
||||
|
||||
# 检查 SHA1 是否在翻译集合中（拓扑序已过滤可达；仅回退扫描顺序时检查）
|
||||
# Check whether the SHA1 is in the translate set (topo already filtered; only the fallback checks)
|
||||
in_set: int = 1
|
||||
if topo_count == 0:
|
||||
in_set = 0
|
||||
@@ -765,7 +701,7 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
# 构造 stub 路径: {temp_dir}/{sha1}.stub.ll
|
||||
# Build the stub path: {temp_dir}/{sha1}.stub.ll
|
||||
dir_len: sizet = string.strlen(temp_dir)
|
||||
sha1_len: sizet = string.strlen(sha1)
|
||||
stub_path: str = StubMergerCollect._sliced_path(temp_dir, dir_len, sha1, "stub.ll")
|
||||
@@ -773,9 +709,8 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
failed += 1
|
||||
continue
|
||||
|
||||
# 检查 stub 是否已存在（按需翻译：跳过已存在的）
|
||||
# 例外：含 class 定义的模块必须翻译——类注册只在翻译时发生，增量编译跳过会导致跨模块类构造 find_struct_by_module 落空 → 返回类型 fail-fast。
|
||||
# 判据：扫描 1b-pre 生成的 vpi 是否有 "class " 行（vpi 为解析后存根，精确无注释/字符串误判）。
|
||||
# Skip already-existing stubs, except modules with class definitions: class registration
|
||||
# only happens at translation, so skipping breaks cross-module find_struct_by_module.
|
||||
force_translate: int = 0
|
||||
vpi_path_1b: str = StubMergerCollect._sliced_path(temp_dir, dir_len, sha1, "vpi")
|
||||
if vpi_path_1b is not None:
|
||||
@@ -803,22 +738,22 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
else: vf_1b.close()
|
||||
stdlib.free(vpi_path_1b)
|
||||
if force_translate == 1:
|
||||
# 含 class：强制翻译（覆盖 stub/text，重新注册类）
|
||||
# Has class: force translation (overwrite stub/text, re-register classes)
|
||||
rp: str = entry.RelPath
|
||||
VLogger.info(f"含 class 定义，强制翻译: {rp}(sha1={sha1})", "Phase1")
|
||||
else:
|
||||
# 检查 stub 是否已存在（按需翻译：跳过已存在的）
|
||||
# Skip already-existing stubs (on-demand translation)
|
||||
sf: rawptr[fileio.File] = fileio.File(stub_path, fileio.MODE.R)
|
||||
if not sf.closed:
|
||||
sf.close()
|
||||
# 检查 text.ll 是否也存在（虚表扫描需要 text.ll）
|
||||
# Check that text.ll exists too (vtable scan needs it)
|
||||
text_path: str = StubMergerCollect._sliced_path(temp_dir, dir_len, sha1, "text.ll")
|
||||
if text_path is not None:
|
||||
tf: rawptr[fileio.File] = fileio.File(text_path, fileio.MODE.R)
|
||||
if not tf.closed:
|
||||
tf.close()
|
||||
# sidecar（{sha1}.meta）与在线 FULL 产物（{sha1}.full.ll）也必须存在：
|
||||
# 缺失则重新翻译以补写（否则 Phase B+/C 的声明判定与编译拿不到数据）。
|
||||
# The sidecar ({sha1}.meta) and online FULL artifact ({sha1}.full.ll) must also exist;
|
||||
# otherwise re-translate to regenerate them for Phase B+/C.
|
||||
meta_path_1b: str = StubMergerCollect._sliced_path(temp_dir, dir_len, sha1, "meta")
|
||||
meta_ok_1b: int = 0
|
||||
if meta_path_1b is not None:
|
||||
@@ -840,20 +775,20 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
stdlib.free(stub_path)
|
||||
continue
|
||||
stdlib.free(text_path)
|
||||
# text.ll 不存在，需要重新翻译
|
||||
# text.ll missing, re-translate
|
||||
rp: str = entry.RelPath
|
||||
VLogger.info(f"text.ll 不存在，重新翻译: {rp}(sha1={sha1})", "Phase1")
|
||||
else:
|
||||
# stub 不存在，需要翻译
|
||||
# stub missing, translate
|
||||
rp: str = entry.RelPath
|
||||
VLogger.info(f"翻译: {rp}(sha1={sha1})", "Phase1")
|
||||
|
||||
stdlib.free(stub_path)
|
||||
|
||||
# 单文件计时起点：读文件（parse/translate/dump 三段，对齐 Phase2 插桩）
|
||||
# Per-file timer start: read (parse/translate/dump segments, matching Phase2 instrumentation)
|
||||
t_f0: uint64 = w32.win32base.GetTickCount64()
|
||||
|
||||
# 读取文件内容
|
||||
# Read file contents
|
||||
file_path: str = entry.Path
|
||||
f: rawptr[fileio.File] = fileio.File(file_path, fileio.MODE.R)
|
||||
if f is None:
|
||||
@@ -865,7 +800,7 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
failed += 1
|
||||
continue
|
||||
|
||||
# read_all 的容量参数含终止符(最多读 cap-1 字节), 文件更长会被静默截断
|
||||
# read_all's capacity includes the terminator (reads at most cap-1 bytes); longer files are truncated
|
||||
if f.size() >= Phase1.SRC_BUF_SIZE:
|
||||
VLogger.error(f"源文件超过 {Phase1.SRC_BUF_SIZE} 字节上限, 拒绝读取: {file_path}", "Phase1")
|
||||
f.close()
|
||||
@@ -887,7 +822,7 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
continue
|
||||
if bytes_read < Phase1.SRC_BUF_SIZE: src_buf[bytes_read] = 0
|
||||
|
||||
# 解析 AST
|
||||
# Parse AST
|
||||
lx: ast.Lexer* = ast.new_lexer(mb)
|
||||
if lx is None:
|
||||
stdlib.free(src_buf)
|
||||
@@ -897,23 +832,22 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
tokens: ast.Token* = ast.tokenize(lx)
|
||||
tree: ast.AST* = ast.parse_tokens(mb, tokens)
|
||||
if tree is None:
|
||||
# 解析失败：行/列 + 原因一并进日志行（原因由 parser.vp 的 _record_* /
|
||||
# parse_tokens 兜底补记提供）
|
||||
# Parse failure: line/col + reason go into the log line (reason supplied by parser.vp's
|
||||
# _record_* / parse_tokens fallback).
|
||||
VLogger.error(f"AST 解析失败: {file_path}（第 {ast.get_parse_err_line()} 行第 {ast.get_parse_err_col()} 列: {ast.get_parse_err_msg()}）", "Phase1")
|
||||
stdlib.free(src_buf)
|
||||
failed += 1
|
||||
continue
|
||||
|
||||
# 计时点 1：读文件 + lexer/tokenize/parse 完成
|
||||
# Timer 1: read + lexer/tokenize/parse done
|
||||
t_f1: uint64 = w32.win32base.GetTickCount64()
|
||||
|
||||
# 文本扫描注册 CTypedef：AST 字符串字段（attr/id）token 零拷贝指向 src_buf，跨阶段内容不稳定（悬空），读 attr 会 0xC0000005；
|
||||
# 故改用源码文本逐行匹配 "NAME: t.CTypedef = TYPE"，在 src_buf 有效期内注册。
|
||||
# Register CTypedef by scanning source text: AST string fields (attr/id) point into src_buf
|
||||
# and dangle across phases (reading attr faults), so match "NAME: t.CTypedef = TYPE" lines.
|
||||
GenApi.RegisterCTypedefFromSource(mb, src_buf)
|
||||
|
||||
# 生成 .vpi 存根文件（直接遍历 AST，不依赖 PythonToStubConverter）
|
||||
# 1b-pre 已为所有待翻译文件生成 vpi，此处仅兜底：vpi 缺失才生成
|
||||
# （消除 1b-pre 与 1b 对同一文件的重复 _GeneratePyiFromAst 工作）
|
||||
# Generate the .vpi stub (walk the AST directly). 1b-pre already generated one, so this
|
||||
# is only a fallback when the vpi is missing.
|
||||
vpi_path_1b: str = StubMergerCollect._sliced_path(temp_dir, dir_len, sha1, "vpi")
|
||||
if vpi_path_1b is not None:
|
||||
vf_1b: rawptr[fileio.File] = fileio.File(vpi_path_1b, fileio.MODE.R)
|
||||
@@ -933,7 +867,7 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
stdlib.free(pyi_buf)
|
||||
stdlib.free(vpi_path_1b)
|
||||
|
||||
# 翻译 AST → LLVM IR
|
||||
# Translate AST -> LLVM IR
|
||||
tr: rawptr[HandlesTranslator.Translator] = HandlesTranslator.Translator()
|
||||
if tr is None:
|
||||
stdlib.free(src_buf)
|
||||
@@ -942,10 +876,8 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
tr.ModuleSha1 = sha1
|
||||
tr.CurrentPackage = GenApi.compute_package_from_relpath(mb, entry.RelPath)
|
||||
GenApi.set_current_file_type(file_path)
|
||||
# --traceback: mirror the DWARF switch into the Gen-layer emitter (the
|
||||
# same BuildPipeline._traceback_mode that gates -frame-pointer=all /
|
||||
# dbghelp / the Install() ctor). include/*.vp modules are compiled here,
|
||||
# so this is what gives their frames DWARF line info.
|
||||
# --traceback: mirror the DWARF switch into the Gen-layer emitter (same
|
||||
# BuildPipeline._traceback_mode that gates -frame-pointer=all / dbghelp / Install()).
|
||||
DebugInfoGen.SetEnabled(BuildPipeline._traceback_mode)
|
||||
GenApi.set_current_file_imports(file_path)
|
||||
GenApi.set_current_module_sha1(sha1)
|
||||
@@ -953,44 +885,40 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
GenApi.clear_ctypedefs()
|
||||
GenApi.reset_visible_structs(mb, 0)
|
||||
|
||||
# 翻译前哨兵：AST 解析（lexer/tokenize/parse_tokens）同样在 pool 分配，
|
||||
# 若解析器越界写破坏了存活块 header，此处立即定位到当前文件
|
||||
# （崩溃点漂移排查：把"翻译过程中随机崩"变成确定性定位）
|
||||
# Pre-translate sentinel: parsing also allocates from the pool, so a parser overrun that
|
||||
# corrupts a live block header is pinned to the current file (deterministic crash location).
|
||||
pre_rc: int = mb.check_alive_blocks(mark1)
|
||||
if pre_rc != 0:
|
||||
VLogger.error(f"内存损坏(翻译前存活块, rc={pre_rc}): {file_path}", "Phase1")
|
||||
stdlib.exit(1)
|
||||
|
||||
ret: int = tr.translate(tree)
|
||||
# 计时点 2：translate 返回
|
||||
# Timer 2: translate returned
|
||||
t_f2: uint64 = w32.win32base.GetTickCount64()
|
||||
# 诊断：translate 正常返回（若挂起在 translate 内部，此行不会打印，
|
||||
# 用于区分"translate 内部挂起"与"后续 dump 挂起"）
|
||||
# Diagnostic: translate returned normally (if it hangs inside, this line is not printed,
|
||||
# distinguishing an internal hang from a later dump hang).
|
||||
if ret == 0: VLogger.info(f"翻译完成(translate返回): {file_path}", "Phase1")
|
||||
if ret != 0:
|
||||
VLogger.error(f"翻译失败: {file_path}", "Phase1") # error 终止
|
||||
VLogger.error(f"翻译失败: {file_path}", "Phase1") # error terminates
|
||||
stdlib.free(src_buf)
|
||||
if tr._global_names is not None: stdlib.free(tr._global_names)
|
||||
if tr._nonlocal_names is not None: stdlib.free(tr._nonlocal_names)
|
||||
failed += 1
|
||||
continue
|
||||
|
||||
# --traceback: inject the startup Install() ctor when translating traceback.vp
|
||||
# (its stub/text.ll below are what Phase2 reuses to build the .obj). The
|
||||
# injector lives in the Gen layer (already depends on mlirlite), so this
|
||||
# module gains no cross-module mlirlite symbol dependency.
|
||||
# --traceback: inject the startup Install() ctor when translating traceback.vp;
|
||||
# the injector lives in the Gen layer, adding no cross-module mlirlite dependency.
|
||||
if BuildPipeline._traceback_mode != 0 and BuildPipeline._tb_basename_is_traceback(file_path) != 0: HandlesTranslator.InjectTracebackInstallCtor(tr)
|
||||
|
||||
# 诊断：pre_rc 哨兵在 translate 前、覆盖不到 translate 期间的 AST/IR 写穿，故 dump 前立即检查存活块 header/canary
|
||||
# （rc=6 块尾 canary 损坏 = 越界写穿数据区），把「dump 时链表成环卡死」变成「定位到 translate 期间写穿发生的文件」。
|
||||
# Diagnostic: the pre-translate sentinel misses write-throughs during translate, so check
|
||||
# live block headers/canaries before dump (rc=6 means a tail canary was overwritten).
|
||||
post_rc: int = mb.check_alive_blocks(mark1)
|
||||
if post_rc != 0: VLogger.error(f"内存损坏(translate后存活块, rc={post_rc}): {file_path}", "Phase1")
|
||||
|
||||
# dump stub IR(declarations only)
|
||||
# 1MB 不够：ast/parser.vp 的 text.ll 超 1MB 被截断 → 合并 IR 尾部丢字
|
||||
# （opt: "end of file in global variable name"），提至 8MB。
|
||||
# Dump stub IR (declarations only). 1MB is not enough: ast/parser.vp's text.ll exceeds it
|
||||
# and is truncated (opt: "end of file in global variable name"), so use 8MB.
|
||||
PHASE1_IR_SIZE: sizet = 8388608
|
||||
# FULL dump = stub + text（同一模块），留 2× 余量
|
||||
# FULL dump = stub + text (same module), with 2x headroom
|
||||
PHASE1_FULL_SIZE: sizet = 16777216
|
||||
stub_buf: bytes = stdlib.malloc(PHASE1_IR_SIZE)
|
||||
if stub_buf is None:
|
||||
@@ -1034,11 +962,11 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
stdlib.free(text_path_p1)
|
||||
stdlib.free(text_buf)
|
||||
|
||||
# save {sha1}.meta（结构化 sidecar：函数清单/结构体布局/虚表，替代从 IR 文本反推）
|
||||
# save {sha1}.meta (structured sidecar: function list/struct layout/vtables, no IR-text reverse-engineering)
|
||||
StubMergerIR.WriteModuleMeta(tr.Module, temp_dir, dir_len_p1, sha1)
|
||||
|
||||
# save {sha1}.full.ll（在线 FULL 产物：模型在线时的自足单模块 IR，供 Phase B 直接编译，
|
||||
# 取代 stub + 依赖 stub/text 文本合并）
|
||||
# save {sha1}.full.ll (the online FULL artifact: a self-contained single-module IR for
|
||||
# Phase B to compile directly, replacing the stub + dependency text merge).
|
||||
full_buf_p1: bytes = stdlib.malloc(PHASE1_FULL_SIZE)
|
||||
if full_buf_p1 is None: VLogger.error("full.ll 缓冲区分配失败", "Phase1")
|
||||
tr.dump_ir(full_buf_p1, PHASE1_FULL_SIZE, mlirlite.OUTPUT_FULL)
|
||||
@@ -1052,15 +980,15 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
stdlib.free(full_path_p1)
|
||||
stdlib.free(full_buf_p1)
|
||||
|
||||
# --traceback: 取出本模块的调试调用点记录（mlirlite 累积）并写 <sha1>.vpcall，
|
||||
# 供链接后 WriteStackMap 结合 objdump 还原调用点行号。关闭时不产生任何旁文件。
|
||||
# --traceback: take this module's debug call-site records (accumulated in mlirlite) and
|
||||
# write <sha1>.vpcall for WriteStackMap to reconstruct call-site lines. No file when off.
|
||||
if BuildPipeline._traceback_mode != 0:
|
||||
vpcall_text_p1: str = mlirlite.DebugTakeText()
|
||||
if vpcall_text_p1 is not None:
|
||||
BuildPipeline.WriteCallSidecar(temp_dir, sha1, vpcall_text_p1)
|
||||
stdlib.free(vpcall_text_p1)
|
||||
|
||||
# 计时点 3：dump（stub+text）完成；输出单文件三段耗时
|
||||
# Timer 3: dump (stub+text) done; print the per-file three-segment timing
|
||||
t_f3: uint64 = w32.win32base.GetTickCount64()
|
||||
VLogger.info(f"[{i+1}/{loop_n_b}] 翻译: {file_path} (parse={t_f1-t_f0} ms, translate={t_f2-t_f1} ms, dump={t_f3-t_f2} ms, total={t_f3-t_f0} ms)", "Phase1")
|
||||
|
||||
@@ -1075,33 +1003,30 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
df_p1.close()
|
||||
stdlib.free(deps_path_p1)
|
||||
|
||||
# 释放 Translator 的 C malloc 资源
|
||||
# Release the Translator's C malloc resources
|
||||
if tr._global_names is not None: stdlib.free(tr._global_names)
|
||||
if tr._nonlocal_names is not None: stdlib.free(tr._nonlocal_names)
|
||||
|
||||
# 所有 dump_ir 完成，src_buf 不再需要
|
||||
# All dump_ir calls done, src_buf no longer needed
|
||||
stdlib.free(src_buf)
|
||||
|
||||
translated += 1
|
||||
|
||||
# OOM 回收：回收本文件翻译产生的瞬时数据（AST/tokens/lexer/IR/Handler/dump 临时）；持久表（struct/enum/REnum/spec 缓存/全局函数名等）已迁出 reset 区间（stdlib），
|
||||
# reset_to(mark1) 只回收 mark1 之后分配、未被单独 free 的块。
|
||||
#
|
||||
# 内存哨兵（崩溃点漂移排查）：reset 前检查存活块 header（抓越界写），reset 后检查 free list 完整性（抓链表损坏）；任一损坏立即终止，把「随机崩溃漂移」变成「确定性定位到当前文件」。
|
||||
# rc 语义（check_alive_blocks）：1=块地址低于 usable，2=高于 usable+size，3=order 为负，4=order 超 max_order，5=块未对齐到 order 大小，6=块尾 canary 损坏（越界写穿数据区）
|
||||
# rc 语义（self_check）：1/2=越界，3=对齐错，4=free list 成环/超步数
|
||||
# OOM reclamation: free this file's transient AST/tokens/lexer/IR/dump data (persistent
|
||||
# tables moved to stdlib). Memory sentinels locate overruns deterministically.
|
||||
chk_rc: int = mb.check_alive_blocks(mark1)
|
||||
if chk_rc != 0: VLogger.error(f"内存损坏(存活块 header, rc={chk_rc}): {file_path}", "Phase1")
|
||||
# 越界写是崩溃根因（与回收/UAF 无关，已由 A/B 实验确认）——恢复 reset_to，改用块尾 canary（check_alive_blocks 校验）把越界写事发地定位到具体文件。
|
||||
# Out-of-bounds writes are the crash root cause (unrelated to reclamation/UAF, confirmed
|
||||
# by A/B tests); reset_to plus the tail canary pins the offending file.
|
||||
mb.reset_to(mark1)
|
||||
sc_rc: int = mb.self_check()
|
||||
if sc_rc != 0: VLogger.error(f"内存损坏(free list, rc={sc_rc}): {file_path}", "Phase1")
|
||||
|
||||
# 1b 阶段总耗时（全量翻译 + dump）
|
||||
# 1b total time (full translation + dump)
|
||||
t_b1: uint64 = w32.win32base.GetTickCount64()
|
||||
VLogger.info(f"1b 总耗时: {t_b1-t_b0} ms (全量翻译, trans={translated} skip={skipped} fail={failed})", "Phase1")
|
||||
|
||||
# 释放 Phase 1b 使用的拓扑排序资源
|
||||
# Release the topo-sort resources used by Phase 1b
|
||||
if topo_order is not None: mb.free(topo_order)
|
||||
if topo_count_box is not None: mb.free(topo_count_box)
|
||||
|
||||
@@ -1111,7 +1036,7 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: rawpt
|
||||
VLogger.error(f"Phase1 有 {failed} 个文件翻译失败", "Phase1")
|
||||
return 1
|
||||
|
||||
# 释放可达 SHA1 集合（如果分配了）
|
||||
# Release the reachable SHA1 set (if allocated)
|
||||
if reachable_set is not None: stdlib.free(reachable_set)
|
||||
|
||||
return 0
|
||||
@@ -31,41 +31,28 @@ import lib.Phase.Phase1 as Phase1
|
||||
import lib.core.Gen.GenApi as GenApi
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Phase1Scan - 扫描类继承关系、泛型类标记与容器使用
|
||||
#
|
||||
# 函数实现直接位于本模块，调用方经本模块前缀访问；全局符号经 Phase1._xxx 跨模块访问。
|
||||
# ============================================================
|
||||
# Phase1Scan - scan class inheritance, generic-class markers and container usage.
|
||||
# Functions live in this module; globals are accessed cross-module via Phase1._xxx.
|
||||
|
||||
# 拓扑排序使用扁平字节数组存储类信息和文件依赖，
|
||||
# 避免结构体中的 str 指针字段在 mb.alloc 零填充时为 NULL 导致崩溃。
|
||||
# 类名/父类名存储在 class_names_buf/parent_names_buf（每个 Phase1.CLASS_NAME_LEN 字节），
|
||||
# 定义 SHA1 存储在 def_sha1s_buf（每个 Phase1.SHA1_LEN 字节）。
|
||||
# Topological sort stores class info/deps in flat byte arrays (names in
|
||||
# class_names_buf/parent_names_buf, SHA1s in def_sha1s_buf) to avoid NULL str fields.
|
||||
|
||||
|
||||
# ============================================================
|
||||
# _ScanClassInheritance - 预扫描 includes 文件，收集类继承关系
|
||||
#
|
||||
# 遍历所有可达 includes 文件解析 AST，收集 ClassDef 的类名与父类名存入扁平字节数组，返回收集到的类数量。
|
||||
#
|
||||
# Args: mb 内存池；result IncludesScanner 扫描结果；reachable_set 可达 SHA1 集合；reachable_count 可达 SHA1 数量；use_reachable 是否使用可达集合过滤；
|
||||
# sha1_set SHA1 集合（use_reachable=0 时使用）；set_count SHA1 集合数量；class_names_buf 类名扁平数组（每个 Phase1.CLASS_NAME_LEN 字节）；
|
||||
# parent_names_buf 父类名扁平数组（每个 Phase1.CLASS_NAME_LEN 字节，无父类填 '\0'）；def_sha1s_buf 定义 SHA1 扁平数组（每个 Phase1.SHA1_LEN 字节）
|
||||
# Returns: 收集到的类数量，-1 失败
|
||||
# ============================================================
|
||||
# _ScanClassInheritance - pre-scan includes files, parse ASTs and collect ClassDef
|
||||
# names/parent names into flat byte arrays; returns the class count (-1 on failure).
|
||||
def _ScanClassInheritance(mb: memhub.MemBuddy*, result: rawptr[IncludesScanner.ScanResult], reachable_set: str, reachable_count: int, use_reachable: int, sha1_set: str, set_count: int, class_names_buf: bytes, parent_names_buf: bytes, def_sha1s_buf: bytes, generic_file_indices: t.CPtr, generic_file_count_box: t.CPtr) -> int:
|
||||
"""预扫描 includes 文件，收集类继承关系"""
|
||||
"""Pre-scan includes files and collect class inheritance info."""
|
||||
if result is None: return -1
|
||||
# class_names_buf/parent_names_buf/def_sha1s_buf 由调用方 mb.alloc 分配（Phase1Main 预扫描前），本函数把其地址直接作为 strcpy 目的缓冲；
|
||||
# 分配失败返回 NULL 时目的指针为近零地址 → strcpy 写零页（ASan access-violation），故按 fail-fast 显式报错终止（VLogger.error 内部 sys.exit(1)），不静默返回 -1。
|
||||
# The three flat buffers are mb.alloc'ed by the caller; a NULL buffer would make
|
||||
# strcpy write the zero page, so fail-fast abort instead of returning -1 silently.
|
||||
if class_names_buf is None or parent_names_buf is None or def_sha1s_buf is None: VLogger.error("类继承预扫描扁平缓冲分配失败（class_names_buf/parent_names_buf/def_sha1s_buf 为 NULL）", "Phase1")
|
||||
|
||||
entry_size: sizet = IncludesScanner.FileEntry.__sizeof__()
|
||||
class_count: int = 0
|
||||
|
||||
for i in range(result.Count):
|
||||
# 文件级 mark：收集完类继承信息后立即回收本文件 AST（可达 includes 全量 parse 的 AST 若驻留到 mark1 会耗尽 1GB 主池 OOM）；
|
||||
# class_names_buf/parent_names_buf/def_sha1s_buf 与 generic_file_indices 在调用前分配、位于本 mark 之前，reset 不影响。
|
||||
# Per-file mark: reclaim this file's AST right after collecting inheritance
|
||||
# info (keeping all ASTs alive until mark1 would OOM the 1GB main pool).
|
||||
scan_mark: sizet = mb.mark()
|
||||
entry_addr: uint64 = uint64(result.Entries) + i * entry_size
|
||||
entry: rawptr[IncludesScanner.FileEntry] = rawptr[IncludesScanner.FileEntry](entry_addr)
|
||||
@@ -74,18 +61,18 @@ def _ScanClassInheritance(mb: memhub.MemBuddy*, result: rawptr[IncludesScanner.S
|
||||
sha1_e: str = entry.Sha1
|
||||
file_has_generic: int = 0
|
||||
|
||||
# 按需翻译过滤
|
||||
# on-demand translation filter
|
||||
in_set: int = 0
|
||||
if use_reachable != 0: in_set = StubMergerCollect._is_in_sha1_set(sha1_e, reachable_set, reachable_count)
|
||||
else: in_set = StubMergerCollect._is_in_sha1_set(sha1_e, sha1_set, set_count)
|
||||
if in_set == 0: continue
|
||||
|
||||
# 读取文件
|
||||
# read the file
|
||||
file_path: str = entry.Path
|
||||
f: rawptr[fileio.File] = fileio.File(file_path, fileio.MODE.R)
|
||||
if f.closed: continue
|
||||
|
||||
# read_all 的容量参数含终止符(最多读 cap-1 字节), 文件更长会被静默截断
|
||||
# read_all's capacity includes the terminator (reads at most cap-1 bytes); longer files are truncated
|
||||
if f.size() >= Phase1.SRC_BUF_SIZE:
|
||||
VLogger.error(f"源文件超过 {Phase1.SRC_BUF_SIZE} 字节上限, 拒绝读取: {file_path}", "Phase1")
|
||||
f.close()
|
||||
@@ -103,7 +90,7 @@ def _ScanClassInheritance(mb: memhub.MemBuddy*, result: rawptr[IncludesScanner.S
|
||||
continue
|
||||
if bytes_read < Phase1.SRC_BUF_SIZE: src_buf[bytes_read] = 0
|
||||
|
||||
# 解析 AST
|
||||
# parse the AST
|
||||
lx: ast.Lexer* = ast.new_lexer(mb)
|
||||
if lx is None:
|
||||
stdlib.free(src_buf)
|
||||
@@ -115,7 +102,7 @@ def _ScanClassInheritance(mb: memhub.MemBuddy*, result: rawptr[IncludesScanner.S
|
||||
stdlib.free(src_buf)
|
||||
continue
|
||||
|
||||
# 遍历 AST 顶层节点，收集 ClassDef
|
||||
# walk top-level AST nodes, collect ClassDef
|
||||
ch: list[ast.AST*]* = tree.children
|
||||
if ch is not None:
|
||||
cn: sizet = len(ch)
|
||||
@@ -128,7 +115,7 @@ def _ScanClassInheritance(mb: memhub.MemBuddy*, result: rawptr[IncludesScanner.S
|
||||
cd: ast.ClassDef* = ast.ClassDef*(child)
|
||||
if cd is None or cd.name is None: continue
|
||||
|
||||
# 检测泛型类（有 type_params），记录文件索引以供 Phase 1a 预注册
|
||||
# detect generic classes (type_params); record the file index for Phase 1a
|
||||
if file_has_generic == 0:
|
||||
tp_scan: list[str]* = cd.type_params
|
||||
if tp_scan is not None:
|
||||
@@ -145,7 +132,7 @@ def _ScanClassInheritance(mb: memhub.MemBuddy*, result: rawptr[IncludesScanner.S
|
||||
|
||||
if class_count >= Phase1.MAX_CLASSES: break
|
||||
|
||||
# 填充扁平数组：类名（偏移 = class_count * Phase1.CLASS_NAME_LEN）
|
||||
# fill flat array: class name (offset = class_count * Phase1.CLASS_NAME_LEN)
|
||||
name_dst: str = class_names_buf + class_count * Phase1.CLASS_NAME_LEN
|
||||
nm_len: sizet = string.strlen(cd.name)
|
||||
if nm_len >= Phase1.CLASS_NAME_LEN:
|
||||
@@ -153,11 +140,11 @@ def _ScanClassInheritance(mb: memhub.MemBuddy*, result: rawptr[IncludesScanner.S
|
||||
name_dst[Phase1.CLASS_NAME_LEN - 1] = '\0'
|
||||
else: string.strcpy(name_dst, cd.name)
|
||||
|
||||
# 填充扁平数组：父类名（无父类填 '\0'）
|
||||
# fill flat array: parent name (no parent -> '\0')
|
||||
parent_dst: str = parent_names_buf + class_count * Phase1.CLASS_NAME_LEN
|
||||
parent_dst[0] = '\0'
|
||||
parent_nm: str = GenApi._get_parent_class(cd, None)
|
||||
# parent_nm 是 AST 节点内部指针，不需要释放
|
||||
# parent_nm is an internal AST pointer; no need to free
|
||||
if parent_nm is not None:
|
||||
pnm_len: sizet = string.strlen(parent_nm)
|
||||
if pnm_len >= Phase1.CLASS_NAME_LEN:
|
||||
@@ -165,92 +152,75 @@ def _ScanClassInheritance(mb: memhub.MemBuddy*, result: rawptr[IncludesScanner.S
|
||||
parent_dst[Phase1.CLASS_NAME_LEN - 1] = '\0'
|
||||
else: string.strcpy(parent_dst, parent_nm)
|
||||
|
||||
# 填充扁平数组：定义 SHA1（偏移 = class_count * Phase1.SHA1_LEN）
|
||||
# fill flat array: defining SHA1 (offset = class_count * Phase1.SHA1_LEN)
|
||||
sha1_dst: str = def_sha1s_buf + class_count * Phase1.SHA1_LEN
|
||||
string.strcpy(sha1_dst, sha1_e)
|
||||
|
||||
class_count += 1
|
||||
|
||||
stdlib.free(src_buf)
|
||||
# 回收本文件 AST/lexer/token（类名/父类/SHA1 已拷入扁平数组）
|
||||
# reclaim this file's AST/lexer/token (names/SHA1 already copied into flat arrays)
|
||||
mb.reset_to(scan_mark)
|
||||
|
||||
return class_count
|
||||
# ============================================================
|
||||
# _BufHasGenericClassMarker - 文本扫描检测文件是否含泛型类标记
|
||||
#
|
||||
# 泛型类（PEP 695）语法: class Name[T]:（类名后紧跟 '['）。逐字节扫描找词边界 'class' 关键字，其后标识符后非空白是 '[' → 命中。
|
||||
# 用于 _PreRegisterAllGenericTemplates 预注册前筛选，避免 parse 全部 98 个 includes 致 AST 驻留 mark1 前把 pool 推到 OOM。
|
||||
# 误报（字符串/注释里的 "class X["）无害（多 parse 一个文件）；漏报有害（泛型模板未预注册 → 悬空复发），故保守只匹配单行形式。
|
||||
# ============================================================
|
||||
# _BufHasGenericClassMarker - text scan for a generic-class marker (PEP 695 "class
|
||||
# Name["). Used to pre-filter files before registering generic templates.
|
||||
def _BufHasGenericClassMarker(buf: bytes, blen: sizet) -> int:
|
||||
"""文本扫描：buf 中是否有 'class 标识符[' 泛型类标记，返回 1/0"""
|
||||
"""Return 1 if buf contains a 'class identifier[' generic-class marker, else 0."""
|
||||
if buf is None or blen <= 6: return 0
|
||||
pos: sizet = 0
|
||||
while pos + 5 < blen:
|
||||
if buf[pos] == 'c' and buf[pos + 1] == 'l' and buf[pos + 2] == 'a' and buf[pos + 3] == 's' and buf[pos + 4] == 's':
|
||||
# class 前是词边界（行首或非标识符字符）
|
||||
# 'class' preceded by a word boundary (line start or non-identifier)
|
||||
prev_ok: int = 1
|
||||
if pos > 0:
|
||||
pc: char = buf[pos - 1]
|
||||
if(pc >= 'a' and pc <= 'z') or(pc >= 'A' and pc <= 'Z') or(pc >= '0' and pc <= '9') or pc == '_': prev_ok = 0
|
||||
if prev_ok != 0:
|
||||
p: sizet = pos + 5
|
||||
# 跳过空白
|
||||
# skip whitespace
|
||||
while p < blen and(buf[p] == ' ' or buf[p] == '\t'): p += 1
|
||||
# 读类名标识符
|
||||
# read the class-name identifier
|
||||
if p < blen and((buf[p] >= 'a' and buf[p] <= 'z') or(buf[p] >= 'A' and buf[p] <= 'Z') or buf[p] == '_'):
|
||||
while p < blen and((buf[p] >= 'a' and buf[p] <= 'z') or(buf[p] >= 'A' and buf[p] <= 'Z') or(buf[p] >= '0' and buf[p] <= '9') or buf[p] == '_'):
|
||||
p += 1
|
||||
# 类名后非空白是 '[' → 泛型类
|
||||
# non-space after the name is '[' -> generic class
|
||||
while p < blen and(buf[p] == ' ' or buf[p] == '\t'): p += 1
|
||||
if p < blen and buf[p] == '[': return 1
|
||||
pos += 1
|
||||
else: pos += 1
|
||||
return 0
|
||||
# ============================================================
|
||||
# _BufHasGenericFuncMarker - 文本扫描检测文件是否含泛型函数标记
|
||||
#
|
||||
# 泛型函数（PEP 695）语法: def name[（函数名后紧跟 '['，如 def f[T](...):）。
|
||||
# 与 _BufHasGenericClassMarker 同款逐字节词边界扫描；用于泛型函数模板的
|
||||
# 轻量预筛（避免为无泛型函数的文件 parse AST）。误报无害、漏报有害（模板未
|
||||
# 预注册），故保守只匹配单行形式。
|
||||
# ============================================================
|
||||
# _BufHasGenericFuncMarker - text scan for a generic-function marker (PEP 695
|
||||
# "def name["); cheap pre-filter before parsing for generic function templates.
|
||||
def _BufHasGenericFuncMarker(buf: bytes, blen: sizet) -> int:
|
||||
"""文本扫描：buf 中是否有 'def 标识符[' 泛型函数标记，返回 1/0"""
|
||||
"""Return 1 if buf contains a 'def identifier[' generic-function marker, else 0."""
|
||||
if buf is None or blen <= 4: return 0
|
||||
pos: sizet = 0
|
||||
while pos + 3 < blen:
|
||||
if buf[pos] == 'd' and buf[pos + 1] == 'e' and buf[pos + 2] == 'f':
|
||||
# def 前是词边界（行首或非标识符字符）
|
||||
# 'def' preceded by a word boundary (line start or non-identifier)
|
||||
prev_ok: int = 1
|
||||
if pos > 0:
|
||||
pc: char = buf[pos - 1]
|
||||
if(pc >= 'a' and pc <= 'z') or(pc >= 'A' and pc <= 'Z') or(pc >= '0' and pc <= '9') or pc == '_': prev_ok = 0
|
||||
if prev_ok != 0:
|
||||
p: sizet = pos + 3
|
||||
# 跳过空白
|
||||
# skip whitespace
|
||||
while p < blen and(buf[p] == ' ' or buf[p] == '\t'): p += 1
|
||||
# 读函数名标识符
|
||||
# read the function-name identifier
|
||||
if p < blen and((buf[p] >= 'a' and buf[p] <= 'z') or(buf[p] >= 'A' and buf[p] <= 'Z') or buf[p] == '_'):
|
||||
while p < blen and((buf[p] >= 'a' and buf[p] <= 'z') or(buf[p] >= 'A' and buf[p] <= 'Z') or(buf[p] >= '0' and buf[p] <= '9') or buf[p] == '_'):
|
||||
p += 1
|
||||
# 函数名后非空白是 '[' → 泛型函数
|
||||
# non-space after the name is '[' -> generic function
|
||||
while p < blen and(buf[p] == ' ' or buf[p] == '\t'): p += 1
|
||||
if p < blen and buf[p] == '[': return 1
|
||||
pos += 1
|
||||
else: pos += 1
|
||||
return 0
|
||||
# ============================================================
|
||||
# _PreRegisterAllGenericTemplates - 预注册 includes 泛型类模板（mark1 前）
|
||||
#
|
||||
# 泛型模板表（GenericClassNames/GenericClassNodes/GenericClassSha1s/GenericClassFiles + 模板节点）是懒分配的跨文件持久表，
|
||||
# 首次注册必须落在 mark1 之前才能跨 Phase1 1b（reset_to(mark1)）与 Phase2（reset_to(mark2)）存活；否则表与节点被 reset 回收，
|
||||
# _find_generic_template 遍历悬空内存 strcmp 崩溃。本函数先文本扫描（_BufHasGenericClassMarker，不 parse）筛选出含泛型类的文件，
|
||||
# 仅对命中文件用**永久池（perm_pool）**解析（模板节点与表直接进 perm_pool，跨 mark 永久存活、不占主 pool）；重复注册由 _register_generic_template 内部查重跳过。
|
||||
# ============================================================
|
||||
# _PreRegisterAllGenericTemplates - pre-register includes generic templates before
|
||||
# mark1 so the lazy tables survive later resets; parses only marker-matched files.
|
||||
def _PreRegisterAllGenericTemplates(mb: memhub.MemBuddy*, result: rawptr[IncludesScanner.ScanResult]) -> int:
|
||||
"""预注册全部 includes 文件的泛型类模板，返回处理到的泛型类数量"""
|
||||
"""Pre-register generic templates from all includes files and return the count."""
|
||||
if result is None or mb is None: return 0
|
||||
perm: memhub.MemBuddy* = CompilerFactory.factory.PermPool
|
||||
if perm is None: return 0
|
||||
@@ -260,11 +230,11 @@ def _PreRegisterAllGenericTemplates(mb: memhub.MemBuddy*, result: rawptr[Include
|
||||
entry_addr_pg: uint64 = uint64(result.Entries) + i * p1e_size
|
||||
entry_pg: rawptr[IncludesScanner.FileEntry] = (rawptr[IncludesScanner.FileEntry])(entry_addr_pg)
|
||||
if entry_pg is None or entry_pg.Sha1 is None: continue
|
||||
# 读取文件
|
||||
# read the file
|
||||
file_path_pg: str = entry_pg.Path
|
||||
f_pg: rawptr[fileio.File] = fileio.File(file_path_pg, fileio.MODE.R)
|
||||
if f_pg.closed: continue
|
||||
# read_all 的容量参数含终止符(最多读 cap-1 字节), 文件更长会被静默截断
|
||||
# read_all's capacity includes the terminator (reads at most cap-1 bytes); longer files are truncated
|
||||
if f_pg.size() >= Phase1.SRC_BUF_SIZE:
|
||||
VLogger.error(f"源文件超过 {Phase1.SRC_BUF_SIZE} 字节上限, 拒绝读取: {file_path_pg}", "Phase1")
|
||||
f_pg.close()
|
||||
@@ -279,12 +249,12 @@ def _PreRegisterAllGenericTemplates(mb: memhub.MemBuddy*, result: rawptr[Include
|
||||
stdlib.free(src_buf_pg)
|
||||
continue
|
||||
if bytes_read_pg < Phase1.SRC_BUF_SIZE: src_buf_pg[bytes_read_pg] = 0
|
||||
# 文本扫描筛选：非泛型文件跳过（不 parse、不驻留，防 OOM）。
|
||||
# 泛型类（class X[）与泛型函数（def f[）任一命中即进入 parse。
|
||||
# Text-scan filter: skip non-generic files (no parse, no residency, avoids
|
||||
# OOM). A class or function marker hit selects the file for parsing.
|
||||
if _BufHasGenericClassMarker(src_buf_pg, bytes_read_pg) == 0 and _BufHasGenericFuncMarker(src_buf_pg, bytes_read_pg) == 0:
|
||||
stdlib.free(src_buf_pg)
|
||||
continue
|
||||
# 泛型文件：永久池解析 → 注册模板（节点/表进 perm_pool，跨 reset 存活）
|
||||
# generic file: parse in perm_pool and register templates (survives reset)
|
||||
lx_pg: ast.Lexer* = ast.new_lexer(perm)
|
||||
if lx_pg is None:
|
||||
stdlib.free(src_buf_pg)
|
||||
@@ -295,7 +265,7 @@ def _PreRegisterAllGenericTemplates(mb: memhub.MemBuddy*, result: rawptr[Include
|
||||
if tree_pg is None:
|
||||
stdlib.free(src_buf_pg)
|
||||
continue
|
||||
# 遍历顶层 ClassDef/FunctionDef，注册泛型模板（type_params 非 None）
|
||||
# walk top-level ClassDef/FunctionDef, register generic templates (type_params != None)
|
||||
ch_pg: list[ast.AST*]* = tree_pg.children
|
||||
if ch_pg is not None:
|
||||
cn_pg: sizet = len(ch_pg)
|
||||
@@ -323,33 +293,20 @@ def _PreRegisterAllGenericTemplates(mb: memhub.MemBuddy*, result: rawptr[Include
|
||||
if handled_count > 0: VLogger.info(f"mark1 前永久池预注册 {handled_count} 个 includes 泛型模板（类+函数）", "Phase1")
|
||||
else: VLogger.info(f"mark1 前永久池预注册 0 个泛型模板（扫描 {result.Count} 个文件，可能 marker 未命中或全量扫描失败）", "Phase1")
|
||||
return handled_count
|
||||
# ============================================================
|
||||
# _FindClassDefSha1 - 在扁平数组中查找类名对应的定义文件 SHA1
|
||||
#
|
||||
# Args: class_names_buf 类名扁平数组；def_sha1s_buf 定义 SHA1 扁平数组；class_count 类数量；class_name 要查找的类名
|
||||
# Returns: SHA1 字符串指针（指向 def_sha1s_buf 内部偏移），None 未找到
|
||||
# ============================================================
|
||||
# _FindClassDefSha1 - look up a class name's defining file SHA1 in the flat arrays;
|
||||
# returns a pointer into def_sha1s_buf, or None if not found.
|
||||
def _FindClassDefSha1(class_names_buf: bytes, def_sha1s_buf: bytes, class_count: int, class_name: str) -> str:
|
||||
"""查找类名对应的定义文件 SHA1"""
|
||||
"""Find a class name's defining file SHA1 in the flat arrays."""
|
||||
if class_names_buf is None or def_sha1s_buf is None or class_name is None: return None
|
||||
for i in range(class_count):
|
||||
name_ptr: str = class_names_buf + i * Phase1.CLASS_NAME_LEN
|
||||
if string.strcmp(name_ptr, class_name) == 0: return def_sha1s_buf + i * Phase1.SHA1_LEN
|
||||
return None
|
||||
# ============================================================
|
||||
# _ScanTreeForContainers - 递归扫描 AST 检测 list/dict 容器使用
|
||||
#
|
||||
# TPV 的 AST 中语句块挂在 children，而表达式/注解树全部以结构体字段挂载（value/slice/func/args/annotation/targets/returns/bases/left/right 等）；
|
||||
# 只递归 children 会漏掉 list[...]*、list[...](...)、字段注解等非语句块位置，故每个节点先递归表达式承载字段、再递归 children。
|
||||
# 命中即置 need_list/need_dict；误报无害（最多多编译两个空壳模块）。
|
||||
# ============================================================
|
||||
# _ScanTreeForContainers - recursively scan the AST for list/dict container usage:
|
||||
# recurse expression/annotation struct fields first, then statement-block children.
|
||||
|
||||
def _AnnIsContainer(ann: ast.AST*) -> int:
|
||||
"""注解是否为 list/dict 容器（Name 裸名或 Subscript list[...]/dict[...]）
|
||||
|
||||
数组初始化（t.CArray[...] / 裸数组注解）返回 0，避免 _ScanTreeForContainers
|
||||
把数组字面量误判为 list 容器使用（FIX 2026-08-30，联动 Translator.vp 同修）。
|
||||
"""
|
||||
"""Return 1 if the annotation is a list/dict container, else 0."""
|
||||
while ann is not None:
|
||||
ak_c: int = ann.kind()
|
||||
if ak_c == ast.ASTKind.Name:
|
||||
@@ -366,9 +323,9 @@ def _AnnIsContainer(ann: ast.AST*) -> int:
|
||||
|
||||
|
||||
def _ScanTreeForContainers(node: ast.AST*, need_list_box: t.CPtr, need_dict_box: t.CPtr, depth: int = 0):
|
||||
"""递归扫描 AST（字段 + 语句块），检测 list/dict 容器使用"""
|
||||
# 递归深度守卫：AST 损坏成自引用环（字段/children 回指祖先）时无限
|
||||
# 递归直至栈耗尽（每个 includes 文件 parse 后必执行）。fail-fast 终止。
|
||||
"""Recursively scan the AST for list/dict container usage."""
|
||||
# Recursion depth guard: a self-referential AST cycle would recurse until the
|
||||
# stack is exhausted; fail-fast abort.
|
||||
if depth > 128: VLogger.error("容器扫描递归过深（AST 环或嵌套超过 128 层）", "Phase1")
|
||||
if node is None: return
|
||||
k_s: int = node.kind()
|
||||
@@ -465,13 +422,8 @@ def _ScanTreeForContainers(node: ast.AST*, need_list_box: t.CPtr, need_dict_box:
|
||||
nv_s: sizet = len(bvs_s)
|
||||
i_v2: sizet
|
||||
for i_v2 in range(nv_s): _ScanTreeForContainers(bvs_s.get(i_v2), need_list_box, need_dict_box, depth + 1)
|
||||
# `[...]`/`{}` 字面量不触发 _list/_dict 导入：字面量广泛用于编译期常量列表（c.Asm 的 op/inp/out CLOBBER 参数）与数组数据初始化，并非容器使用；
|
||||
# 容器的真实使用由类型位置检测覆盖（Name/Subscript 注解、Call.func 构造名）。
|
||||
# elif k_s == ast.ASTKind.List:
|
||||
# need_list_box[0] = 1
|
||||
# elif k_s == ast.ASTKind.Dict:
|
||||
# need_dict_box[0] = 1
|
||||
# 递归语句块子节点
|
||||
# `[...]`/`{}` literals do not trigger _list/_dict imports (compile-time constant lists, array init);
|
||||
# recurse into statement-block children.
|
||||
if node.children is not None:
|
||||
ch_s: list[ast.AST*]* = node.children
|
||||
n_s: sizet = len(ch_s)
|
||||
|
||||
@@ -27,32 +27,19 @@ import lib.Phase.Phase1.Phase1Scan as Phase1Scan
|
||||
import lib.Phase.Phase1.Phase1Topo as Phase1Topo
|
||||
import lib.Phase.Phase1 as Phase1
|
||||
|
||||
# ============================================================
|
||||
# Phase1Topo - 按类继承与 import 依赖对源文件做拓扑排序
|
||||
#
|
||||
# 函数实现直接位于本模块，调用方经本模块前缀访问；全局符号经 Phase1._xxx 跨模块访问。
|
||||
# ============================================================
|
||||
# Phase1Topo - topologically sort source files by class inheritance and import dependencies.
|
||||
# Functions live in this module (accessed via its prefix); globals go through Phase1._xxx.
|
||||
|
||||
# ============================================================
|
||||
# _TopoSortFiles - 根据类继承关系对文件进行拓扑排序（Kahn）
|
||||
#
|
||||
# 步骤：1) 从扁平类信息数组构建「文件 SHA1 → 依赖文件 SHA1 集合」映射；2) 计算每个文件入度（DepCount，初始化 InDegree = DepCount）；
|
||||
# 3) 入度为 0 的先处理，处理后减少依赖它的文件入度；4) 重复直到全部处理完成。
|
||||
# 依赖方向是「父类所在文件 → 子类所在文件」，即父类文件必须先处理。
|
||||
#
|
||||
# Args: mb 内存池；result IncludesScanner 扫描结果；reachable_set 可达 SHA1 集合；reachable_count 可达 SHA1 数量；use_reachable 是否用可达集合过滤；
|
||||
# sha1_set SHA1 集合；set_count SHA1 集合数量；class_names_buf 类名扁平数组；parent_names_buf 父类名扁平数组；def_sha1s_buf 定义 SHA1 扁平数组；
|
||||
# class_count 类数量；out_order 输出拓扑顺序的文件索引数组（调用者分配，int 数组）；out_count 输出文件数量
|
||||
# Returns: 0 成功，-1 失败
|
||||
# ============================================================
|
||||
# _TopoSortFiles - topologically sort files by class inheritance (Kahn).
|
||||
# Builds a "file SHA1 -> dependency SHA1 set" map, computes in-degrees (parent-file first), then outputs the order; returns 0 or -1.
|
||||
def _TopoSortFiles(mb: memhub.MemBuddy*, result: rawptr[IncludesScanner.ScanResult], reachable_set: str, reachable_count: int, use_reachable: int, sha1_set: str, set_count: int, class_names_buf: bytes, parent_names_buf: bytes, def_sha1s_buf: bytes, class_count: int, out_order: t.CPtr, out_count: t.CPtr, temp_dir: str, td_len_ts: sizet) -> int:
|
||||
"""根据类继承关系对文件进行拓扑排序"""
|
||||
"""Topologically sort files by class inheritance and import dependencies."""
|
||||
if result is None or out_order is None or out_count is None: return -1
|
||||
|
||||
entry_size: sizet = IncludesScanner.FileEntry.__sizeof__()
|
||||
|
||||
# 1. 分配扁平数组：文件 SHA1、依赖 SHA1 列表、依赖数量、入度、处理状态
|
||||
# 用扁平数组代替 FileDepEntry 结构体，避免 str 指针字段 NULL 崩溃
|
||||
# 1. Allocate flat arrays: file SHA1, dependency SHA1 list, dep counts, in-degrees, processed flags.
|
||||
# Flat arrays replace a FileDepEntry struct to avoid NULL str-pointer fields crashing.
|
||||
file_sha1s_buf: bytes = mb.alloc(Phase1.MAX_FILE_DEPS * Phase1.SHA1_LEN)
|
||||
file_deps_buf: bytes = mb.alloc(Phase1.MAX_FILE_DEPS * Phase1.MAX_DEP_EDGES * Phase1.SHA1_LEN)
|
||||
file_dep_counts: t.CPtr = mb.alloc(Phase1.MAX_FILE_DEPS * 4)
|
||||
@@ -74,7 +61,7 @@ def _TopoSortFiles(mb: memhub.MemBuddy*, result: rawptr[IncludesScanner.ScanResu
|
||||
string.memset(file_processed, 0, Phase1.MAX_FILE_DEPS * 4)
|
||||
|
||||
file_count: int = 0
|
||||
# sha1→file_index 映射（简单线性查找，文件数不多）
|
||||
# sha1 -> file_index map (simple linear search; file count is small)
|
||||
for i in range(result.Count):
|
||||
entry_addr: uint64 = uint64(result.Entries) + i * entry_size
|
||||
entry: rawptr[IncludesScanner.FileEntry] = rawptr[IncludesScanner.FileEntry](entry_addr)
|
||||
@@ -88,42 +75,42 @@ def _TopoSortFiles(mb: memhub.MemBuddy*, result: rawptr[IncludesScanner.ScanResu
|
||||
|
||||
if file_count >= Phase1.MAX_FILE_DEPS: break
|
||||
|
||||
# 初始化扁平数组：文件 SHA1（偏移 = file_count * Phase1.SHA1_LEN）
|
||||
# DepCount/InDegree/Processed 已由 memset 清零
|
||||
# Initialize the flat array: file SHA1 (offset = file_count * Phase1.SHA1_LEN).
|
||||
# DepCount/InDegree/Processed are already zeroed by memset.
|
||||
sha1_dst: str = file_sha1s_buf + file_count * Phase1.SHA1_LEN
|
||||
string.strcpy(sha1_dst, sha1_e)
|
||||
|
||||
file_count += 1
|
||||
|
||||
# 2. 构建依赖边：对每个类，找到父类定义所在文件，建立 子文件→父文件 依赖
|
||||
# 2. Build dependency edges: for each class, find the parent's defining file and add a child->parent edge
|
||||
for ci in range(class_count):
|
||||
# 读取父类名（偏移 = ci * Phase1.CLASS_NAME_LEN）
|
||||
# Read parent class name (offset = ci * Phase1.CLASS_NAME_LEN)
|
||||
parent_ptr: str = parent_names_buf + ci * Phase1.CLASS_NAME_LEN
|
||||
if parent_ptr[0] == '\0': continue
|
||||
|
||||
# 读取子类定义 SHA1（偏移 = ci * Phase1.SHA1_LEN）
|
||||
# Read child class definition SHA1 (offset = ci * Phase1.SHA1_LEN)
|
||||
child_def_sha1: str = def_sha1s_buf + ci * Phase1.SHA1_LEN
|
||||
|
||||
# 查找父类定义所在文件的 SHA1
|
||||
# Find the SHA1 of the file defining the parent class
|
||||
parent_sha1: str = Phase1Scan._FindClassDefSha1(class_names_buf, def_sha1s_buf, class_count, parent_ptr)
|
||||
if parent_sha1 is None:
|
||||
continue # 父类不在 includes 中（如 Object、CEnum 等标记基类）
|
||||
continue # parent not in includes (e.g. Object, CEnum marker base classes)
|
||||
|
||||
# 父类文件 SHA1 不能等于子类文件 SHA1（同文件内继承不需要依赖）
|
||||
# The parent file SHA1 must differ from the child's (same-file inheritance needs no dependency)
|
||||
if string.strcmp(parent_sha1, child_def_sha1) == 0: continue
|
||||
|
||||
# 找到子类文件对应的索引，添加父文件 SHA1 到依赖列表
|
||||
# Find the child file's index and add the parent file SHA1 to its dependency list
|
||||
for fi in range(file_count):
|
||||
fsha1: str = file_sha1s_buf + fi * Phase1.SHA1_LEN
|
||||
if string.strcmp(fsha1, child_def_sha1) != 0: continue
|
||||
|
||||
# 读取当前依赖数量
|
||||
# Read the current dependency count
|
||||
cnt_addr: uint64 = uint64(file_dep_counts) + fi * 4
|
||||
cnt_ptr: t.CPtr = (int*)(cnt_addr)
|
||||
if cnt_ptr is None: break
|
||||
cur_cnt: int = cnt_ptr[0]
|
||||
|
||||
# 检查依赖是否已添加（去重）
|
||||
# Check whether the dependency is already present (dedupe)
|
||||
already: int = 0
|
||||
for di in range(cur_cnt):
|
||||
dep_slot: str = file_deps_buf + fi * Phase1.MAX_DEP_EDGES * Phase1.SHA1_LEN + di * Phase1.SHA1_LEN
|
||||
@@ -138,11 +125,8 @@ def _TopoSortFiles(mb: memhub.MemBuddy*, result: rawptr[IncludesScanner.ScanResu
|
||||
cnt_ptr[0] = cur_cnt + 1
|
||||
break
|
||||
|
||||
# 2.5 补充 import 依赖边：被 import 的模块先完整翻译。
|
||||
#
|
||||
# 仅按类继承建边不够：跨模块全局变量 stub、跨模块 vtable TypeStr、泛型模板都要求依赖模块先翻译，而它们之间可能无类继承关系。
|
||||
# deps（_includes_deps_*）由 Phase 1a-pre 填充（含容器按需注入的 _list/_dict）；用 _BuildIncludesSha1Map 复制局部 sha1→模块名 数组
|
||||
# （与 _BuildReachableSha1Set 的 BFS 一致），避免直接跨函数传全局 store 返回值导致地址异常。
|
||||
# 2.5 Add import dependency edges: imported modules must be fully translated first.
|
||||
# Class inheritance alone is insufficient (cross-module globals/vtable TypeStr/generics need deps); deps come from Phase 1a-pre, mapped via _BuildIncludesSha1Map.
|
||||
sha1_arr_ts: bytes = stdlib.malloc(StubMerger.MAX_INCLUDES * 17)
|
||||
mod_arr_ts: bytes = stdlib.malloc(StubMerger.MAX_INCLUDES * 64)
|
||||
map_count_ts: int = 0
|
||||
@@ -168,11 +152,11 @@ def _TopoSortFiles(mb: memhub.MemBuddy*, result: rawptr[IncludesScanner.ScanResu
|
||||
dsha1_imp: str = StubMergerIR._FindSha1ByModName(sha1_arr_ts, mod_arr_ts, map_count_ts, dname_imp)
|
||||
stdlib.free(dname_imp)
|
||||
if dsha1_imp is None: continue
|
||||
# 找依赖文件在列表中的索引（跳过自身）
|
||||
# Find the dependency file's index in the list (skip self)
|
||||
for fj_imp in range(file_count):
|
||||
if string.strcmp(file_sha1s_buf + fj_imp * Phase1.SHA1_LEN, dsha1_imp) != 0: continue
|
||||
if fj_imp == fi_imp: break
|
||||
# 去重 + 加边：fi_imp 依赖 fj_imp（InDegree[fi_imp]++）
|
||||
# Dedupe and add the edge: fi_imp depends on fj_imp (InDegree[fi_imp]++)
|
||||
cnt_addr_imp: uint64 = uint64(file_dep_counts) + fi_imp * 4
|
||||
cnt_ptr_imp: t.CPtr = (int*)(cnt_addr_imp)
|
||||
if cnt_ptr_imp is None: break
|
||||
@@ -189,12 +173,12 @@ def _TopoSortFiles(mb: memhub.MemBuddy*, result: rawptr[IncludesScanner.ScanResu
|
||||
cnt_ptr_imp[0] = cur_cnt_imp + 1
|
||||
break
|
||||
|
||||
# 释放 import 边用的局部映射数组
|
||||
# Free the local mapping arrays used for import edges
|
||||
if sha1_arr_ts is not None: stdlib.free(sha1_arr_ts)
|
||||
if mod_arr_ts is not None: stdlib.free(mod_arr_ts)
|
||||
|
||||
# 3. Kahn 算法：InDegree = DepCount（依赖多少个父文件）
|
||||
# 依赖为 0 的先处理，处理后减少依赖它的文件入度
|
||||
# 3. Kahn: InDegree = DepCount (number of parent files depended on).
|
||||
# Process zero-dependency files first, then decrement the in-degree of dependents.
|
||||
for fi in range(file_count):
|
||||
cnt_addr3: uint64 = uint64(file_dep_counts) + fi * 4
|
||||
cnt_ptr3: t.CPtr = (int*)(cnt_addr3)
|
||||
@@ -202,46 +186,46 @@ def _TopoSortFiles(mb: memhub.MemBuddy*, result: rawptr[IncludesScanner.ScanResu
|
||||
ind_ptr3: t.CPtr = (int*)(ind_addr3)
|
||||
if cnt_ptr3 is not None and ind_ptr3 is not None: ind_ptr3[0] = cnt_ptr3[0]
|
||||
|
||||
# 4. 拓扑排序输出
|
||||
# 4. Topological sort output
|
||||
out_idx: int = 0
|
||||
changed: int = 1
|
||||
while changed != 0:
|
||||
changed = 0
|
||||
for fi in range(file_count):
|
||||
# 检查处理状态
|
||||
# Check processed flag
|
||||
proc_addr4: uint64 = uint64(file_processed) + fi * 4
|
||||
proc_ptr4: t.CPtr = (int*)(proc_addr4)
|
||||
if proc_ptr4 is None or proc_ptr4[0] != 0: continue
|
||||
# 检查入度
|
||||
# Check in-degree
|
||||
ind_addr4: uint64 = uint64(file_indegrees) + fi * 4
|
||||
ind_ptr4: t.CPtr = (int*)(ind_addr4)
|
||||
if ind_ptr4 is None or ind_ptr4[0] > 0: continue
|
||||
|
||||
# 入度为 0，可以处理
|
||||
# 找到对应的 result.Entries 索引
|
||||
# In-degree 0: can be processed
|
||||
# Find the matching result.Entries index
|
||||
target_sha1: str = file_sha1s_buf + fi * Phase1.SHA1_LEN
|
||||
for ri in range(result.Count):
|
||||
ea_r: uint64 = uint64(result.Entries) + ri * entry_size
|
||||
ent_r: rawptr[IncludesScanner.FileEntry] = rawptr[IncludesScanner.FileEntry](ea_r)
|
||||
if ent_r is not None and ent_r.Sha1 is not None:
|
||||
if string.strcmp(ent_r.Sha1, target_sha1) == 0:
|
||||
# 写入输出索引数组
|
||||
# Write into the output index array
|
||||
out_idx_addr: uint64 = uint64(out_order) + out_idx * 4
|
||||
out_int_ptr: t.CPtr = (int*)(out_idx_addr)
|
||||
if out_int_ptr is not None: out_int_ptr[0] = ri
|
||||
out_idx += 1
|
||||
break
|
||||
|
||||
# 标记已处理
|
||||
# Mark as processed
|
||||
proc_ptr4[0] = 1
|
||||
changed = 1
|
||||
|
||||
# 减少依赖该文件的其他文件的入度
|
||||
# Decrement the in-degree of files that depend on this one
|
||||
for fj in range(file_count):
|
||||
proc_addr5: uint64 = uint64(file_processed) + fj * 4
|
||||
proc_ptr5: t.CPtr = (int*)(proc_addr5)
|
||||
if proc_ptr5 is None or proc_ptr5[0] != 0: continue
|
||||
# 读取 fj 的依赖数量
|
||||
# Read fj's dependency count
|
||||
cnt_addr5: uint64 = uint64(file_dep_counts) + fj * 4
|
||||
cnt_ptr5: t.CPtr = (int*)(cnt_addr5)
|
||||
if cnt_ptr5 is None: continue
|
||||
@@ -254,10 +238,8 @@ def _TopoSortFiles(mb: memhub.MemBuddy*, result: rawptr[IncludesScanner.ScanResu
|
||||
if ind_ptr5 is not None: ind_ptr5[0] = ind_ptr5[0] - 1
|
||||
break
|
||||
|
||||
# 处理循环依赖：环内优先输出被依赖者（被其余剩余文件依赖数量多者先）。
|
||||
#
|
||||
# 容器按需注入（string 用 list 泛型 → 依赖 _list）与真实 import 会形成伪环，Kahn 无法完全归零；若按扫描顺序追加，
|
||||
# 依赖者会先于被依赖者翻译 → 跨模块 vtable TypeStr 未注册 → 误杀。
|
||||
# Handle cycles: within a cycle emit the more-depended-upon files first.
|
||||
# On-demand container injection (e.g. string -> _list) plus real imports form pseudo-cycles Kahn cannot fully drain; scan order would translate dependents first -> unregistered cross-module vtable TypeStr.
|
||||
if out_idx < file_count:
|
||||
rem_idx: t.CPtr = stdlib.malloc(4 * file_count)
|
||||
rem_depcnt: t.CPtr = stdlib.malloc(4 * file_count)
|
||||
@@ -270,7 +252,7 @@ def _TopoSortFiles(mb: memhub.MemBuddy*, result: rawptr[IncludesScanner.ScanResu
|
||||
rem_idx[rem_count] = fi
|
||||
rem_depcnt[rem_count] = 0
|
||||
rem_count += 1
|
||||
# 计算每个剩余文件的被依赖数（被其余剩余文件依赖）
|
||||
# Compute each remaining file's dependents count (among the remaining files)
|
||||
for i_a in range(rem_count):
|
||||
fi_a: int = rem_idx[i_a]
|
||||
target_sha1_a: str = file_sha1s_buf + fi_a * Phase1.SHA1_LEN
|
||||
@@ -286,7 +268,7 @@ def _TopoSortFiles(mb: memhub.MemBuddy*, result: rawptr[IncludesScanner.ScanResu
|
||||
if string.strcmp(dep_slot_b, target_sha1_a) == 0:
|
||||
rem_depcnt[i_a] = rem_depcnt[i_a] + 1
|
||||
break
|
||||
# 冒泡降序（被依赖数多者在前）
|
||||
# Bubble sort descending (more dependents first)
|
||||
for i_a in range(rem_count):
|
||||
for j_a in range(rem_count - 1):
|
||||
if rem_depcnt[j_a] < rem_depcnt[j_a + 1]:
|
||||
@@ -296,7 +278,7 @@ def _TopoSortFiles(mb: memhub.MemBuddy*, result: rawptr[IncludesScanner.ScanResu
|
||||
tmp_d: int = rem_depcnt[j_a]
|
||||
rem_depcnt[j_a] = rem_depcnt[j_a + 1]
|
||||
rem_depcnt[j_a + 1] = tmp_d
|
||||
# 按序输出剩余文件
|
||||
# Emit the remaining files in order
|
||||
for i_a in range(rem_count):
|
||||
fi_a2: int = rem_idx[i_a]
|
||||
proc_addr8: uint64 = uint64(file_processed) + fi_a2 * 4
|
||||
@@ -317,11 +299,11 @@ def _TopoSortFiles(mb: memhub.MemBuddy*, result: rawptr[IncludesScanner.ScanResu
|
||||
stdlib.free(rem_idx)
|
||||
stdlib.free(rem_depcnt)
|
||||
|
||||
# 写入输出数量
|
||||
# Write the output count
|
||||
out_count_ptr: t.CPtr = (int*)(out_count)
|
||||
if out_count_ptr is not None: out_count_ptr[0] = out_idx
|
||||
|
||||
# 释放扁平数组
|
||||
# Free the flat arrays
|
||||
mb.free(file_sha1s_buf)
|
||||
mb.free(file_deps_buf)
|
||||
mb.free(file_dep_counts)
|
||||
|
||||
@@ -2,18 +2,17 @@ import t, c
|
||||
from stdint import *
|
||||
import memhub
|
||||
|
||||
# 全局 mbuddy 指针
|
||||
# Global mbuddy pointer
|
||||
_mbuddy: memhub.MemBuddy*
|
||||
|
||||
# 源代码缓冲区大小（1MB）
|
||||
# Source buffer size (1MB)
|
||||
define SRC_BUF_SIZE = 1048576
|
||||
|
||||
# vpi 缓冲区大小（256KB）
|
||||
# 编译期常量，用 define 而非模块级全局：模块级同名全局会在多个模块各发
|
||||
# 一份强定义符号（@PYI_BUF_SIZE），llvm-link 报 symbol multiply defined。
|
||||
# vpi buffer size (256KB).
|
||||
# Compile-time constant via define: a module-level global would emit a strong symbol per module, causing llvm-link errors.
|
||||
define PYI_BUF_SIZE = 262144
|
||||
|
||||
# 拓扑排序相关常量
|
||||
# Topological sort constants
|
||||
define MAX_CLASSES = 512
|
||||
define MAX_FILE_DEPS = 256
|
||||
define MAX_DEP_EDGES = 16
|
||||
|
||||
+221
-276
File diff suppressed because it is too large.
Load diff
Reference in new issue
Block a user