1150 lines
53 KiB
Plaintext
1150 lines
53 KiB
Plaintext
import t, c
|
||
from stdint import *
|
||
import stdio
|
||
import string
|
||
import stdlib
|
||
import memhub
|
||
import viperlib
|
||
import w32.fileio as fileio
|
||
import w32.win32base
|
||
import ast
|
||
import llvmlite
|
||
import lib.core.VLogger as VLogger
|
||
import lib.core.Gen.Translator as HandlesTranslator
|
||
import lib.core.IncludesScanner as IncludesScanner
|
||
import lib.Stub.StubMerger as StubMerger
|
||
import lib.core.Gen.Generator as GenGenerator
|
||
import lib.core.CompilerFactory as CompilerFactory
|
||
import lib.Phase.BuildPipeline as BuildPipeline
|
||
import lib.Projectrans.Config as Config
|
||
import lib.Stub.StubGen.Converter as StubConverter
|
||
import lib.Stub.StubMerger.StubMergerBuild as StubMergerBuild
|
||
import lib.Stub.StubMerger.StubMergerCollect as StubMergerCollect
|
||
import lib.Stub.StubMerger.StubMergerIR as StubMergerIR
|
||
import lib.Stub.StubMerger.StubMergerStore as StubMergerStore
|
||
|
||
import lib.Phase.Phase1.Phase1Scan as Phase1Scan
|
||
import lib.Phase.Phase1.Phase1Topo as Phase1Topo
|
||
import lib.Phase.Phase1 as Phase1
|
||
|
||
import lib.core.Gen.GenApi as GenApi
|
||
|
||
|
||
# ============================================================
|
||
# _InjectedCreatesCycle - 容器模块自动注入前的破环检查
|
||
#
|
||
# 注入 _list/_dict 到当前模块的 _imported_modules 前，检查被注入模块自身的
|
||
# 依赖是否已含当前模块。若命中（如 _list 显式 import memhub，而 memhub 因
|
||
# `__provides__: list[str]` 注解要注入 _list）→ memhub↔_list 循环依赖 →
|
||
# Phase A 多遍收敛死锁（gfx.vp "无法完成翻译" 根因）。命中则跳过注入：去掉
|
||
# 环边后，_list 已因当前模块被其依赖而处于闭包，list 类仍由 Phase1b 注册。
|
||
# ============================================================
|
||
def _InjectedCreatesCycle(entry: IncludesScanner.FileEntry | t.CPtr,
|
||
mod_name: str) -> int:
|
||
"""注入 mod_name 到当前模块前检查是否成环，返回 1=成环需跳过注入"""
|
||
if entry is None or entry.RelPath is None or mod_name is None:
|
||
return 0
|
||
inj_sha1: str = StubMergerStore.LookupSha1ByModName(mod_name)
|
||
if inj_sha1 is None:
|
||
return 0
|
||
inj_deps: str = StubMergerStore.LookupIncludesDeps(inj_sha1)
|
||
if inj_deps is None:
|
||
return 0
|
||
rp: str = entry.RelPath
|
||
# 当前模块名 = RelPath 基名（最后 '/' 或 '\\' 之后）去扩展名
|
||
base: str = string.strrchr(rp, '/')
|
||
if base is None:
|
||
base = string.strrchr(rp, '\\')
|
||
if base is not None:
|
||
base = base + 1
|
||
else:
|
||
base = rp
|
||
dot: str = string.strchr(base, '.')
|
||
nm_buf: bytes = stdlib.malloc(128)
|
||
if nm_buf is None:
|
||
return 0
|
||
if dot is not None:
|
||
nm_len: t.CSizeT = t.CSizeT(t.CUInt64T(dot) - t.CUInt64T(base))
|
||
if nm_len > 127:
|
||
nm_len = 127
|
||
string.strncpy(nm_buf, base, nm_len)
|
||
nm_buf[nm_len] = '\0'
|
||
else:
|
||
string.strncpy(nm_buf, base, 127)
|
||
nm_buf[127] = '\0'
|
||
nm_len2: t.CSizeT = string.strlen(nm_buf)
|
||
d_len: t.CSizeT = string.strlen(inj_deps)
|
||
dp: t.CSizeT = 0
|
||
while dp < d_len:
|
||
ts: t.CSizeT = dp
|
||
while dp < d_len and inj_deps[dp] != ' ':
|
||
dp += 1
|
||
tl: t.CSizeT = dp - ts
|
||
if dp < d_len:
|
||
dp += 1
|
||
if tl > 0 and tl == nm_len2 and string.strncmp(inj_deps + ts, nm_buf, tl) == 0:
|
||
stdlib.free(nm_buf)
|
||
return 1
|
||
stdlib.free(nm_buf)
|
||
return 0
|
||
|
||
|
||
def _DepsContainsName(deps: str, name: str) -> int:
|
||
"""deps 字符串（空格分隔模块名）是否含 name token（词边界精确匹配）"""
|
||
if deps is None or name is None:
|
||
return 0
|
||
nl: t.CSizeT = string.strlen(name)
|
||
dl: t.CSizeT = string.strlen(deps)
|
||
dp: t.CSizeT = 0
|
||
while dp < dl:
|
||
ts: t.CSizeT = dp
|
||
while dp < dl and deps[dp] != ' ':
|
||
dp += 1
|
||
tl: t.CSizeT = dp - ts
|
||
if dp < dl:
|
||
dp += 1
|
||
if tl == nl and string.strncmp(deps + ts, name, nl) == 0:
|
||
return 1
|
||
return 0
|
||
|
||
|
||
def _DepsRemoveName(deps: str, name: str) -> int:
|
||
"""从 deps 字符串移除 name token（原地重建，返回 1=有改动）"""
|
||
if deps is None or name is None:
|
||
return 0
|
||
nl: t.CSizeT = string.strlen(name)
|
||
dl: t.CSizeT = string.strlen(deps)
|
||
buf: bytes = stdlib.malloc(StubMerger.MAX_DEPS_STR_LEN)
|
||
if buf is None:
|
||
return 0
|
||
buf[0] = '\0'
|
||
bl: t.CSizeT = 0
|
||
dp: t.CSizeT = 0
|
||
while dp < dl:
|
||
ts: t.CSizeT = dp
|
||
while dp < dl and deps[dp] != ' ':
|
||
dp += 1
|
||
tl: t.CSizeT = dp - ts
|
||
if dp < dl:
|
||
dp += 1
|
||
if tl == 0:
|
||
continue
|
||
if tl == nl and string.strncmp(deps + ts, name, nl) == 0:
|
||
continue
|
||
if bl > 0:
|
||
buf[bl] = ' '
|
||
bl += 1
|
||
string.strncpy(buf + bl, deps + ts, tl)
|
||
bl += tl
|
||
buf[bl] = '\0'
|
||
string.strcpy(deps, buf)
|
||
stdlib.free(buf)
|
||
return 1
|
||
|
||
|
||
def _RewriteDepsTxt(temp_dir: str, sha1: str, deps_str: str) -> None:
|
||
"""重写 {temp_dir}/{sha1前2位}/{sha1}.deps.txt（内存破环后同步磁盘）"""
|
||
if temp_dir is None or sha1 is None or deps_str is None:
|
||
return
|
||
td_len: t.CSizeT = string.strlen(temp_dir)
|
||
dpath: str = StubMergerCollect._sliced_path(temp_dir, td_len, sha1, "deps.txt")
|
||
if dpath is None:
|
||
return
|
||
df: fileio.File | t.CPtr = fileio.File(dpath, fileio.MODE.W)
|
||
if not df.closed:
|
||
dl: t.CSizeT = string.strlen(deps_str)
|
||
if dl > 0:
|
||
df.write(deps_str, dl)
|
||
df.close()
|
||
stdlib.free(dpath)
|
||
|
||
|
||
def _BreakContainerCyclesInDeps(temp_dir: str) -> int:
|
||
"""扫描后统一清理 _list/_dict 注入环边。
|
||
|
||
Phase1a-pre 全量扫描用 FindFirstFileA（文件系统序），注入 _list/_dict 时
|
||
被注入模块（_list/_dict）自身的 deps 未必已注册 → _InjectedCreatesCycle
|
||
内联检查可能漏判（如 _list 显式 import memhub，而 memhub 的 list[str]
|
||
注入 _list 未被拦截）→ memhub↔_list 2-环 → Phase A 永不收敛。此处扫描
|
||
结束后全量检查：凡某模块 M 的 deps 含 _list/_dict，且 _list/_dict 的 deps
|
||
含 M → 从 M 的 deps 移除注入边（list 类仍由 Phase1b 经闭包注册）。
|
||
内存破环后同步重写 {temp_dir}/deps.txt，保证磁盘与内存一致（Phase2 依赖
|
||
图诊断回退读 deps.txt）。
|
||
"""
|
||
if StubMerger._includes_deps_sha1 is None or StubMerger._includes_deps_str is None:
|
||
return 0
|
||
cnt: t.CSizeT = StubMerger._includes_deps_count
|
||
# [CYCBREAK] 临时诊断：破环后处理是否执行、命中哪些模块（用后移除）
|
||
stdio.printf("[CYCBREAK] deps entries=%lu _list_sha1=%s\n",
|
||
t.CUInt64T(cnt),
|
||
StubMergerStore.LookupSha1ByModName("_list") if StubMergerStore.LookupSha1ByModName("_list") is not None else "none")
|
||
ci: t.CSizeT
|
||
for ci in range(cnt):
|
||
sha1_m: str = StubMerger._includes_deps_sha1 + t.CSizeT(ci) * 17
|
||
deps_m: str = StubMerger._includes_deps_str + t.CSizeT(ci) * StubMerger.MAX_DEPS_STR_LEN
|
||
if sha1_m is None or deps_m is None:
|
||
continue
|
||
if string.strstr(deps_m, "_list") is None and string.strstr(deps_m, "_dict") is None:
|
||
continue
|
||
# 该模块名（RelPath 基名去扩展名）
|
||
rel_m: str = StubMergerStore.LookupSha1RelPath(sha1_m)
|
||
if rel_m is None:
|
||
continue
|
||
base_m: str = string.strrchr(rel_m, '/')
|
||
if base_m is None:
|
||
base_m = string.strrchr(rel_m, '\\')
|
||
if base_m is not None:
|
||
base_m = base_m + 1
|
||
else:
|
||
base_m = rel_m
|
||
dot_m: str = string.strchr(base_m, '.')
|
||
if dot_m is not None:
|
||
nm_len_m: t.CSizeT = t.CSizeT(t.CUInt64T(dot_m) - t.CUInt64T(base_m))
|
||
nm_m: bytes = stdlib.malloc(nm_len_m + 1)
|
||
if nm_m is None:
|
||
continue
|
||
string.strncpy(nm_m, base_m, nm_len_m)
|
||
nm_m[nm_len_m] = '\0'
|
||
else:
|
||
nm_m: bytes = stdlib.malloc(129)
|
||
if nm_m is None:
|
||
continue
|
||
string.strncpy(nm_m, base_m, 128)
|
||
nm_m[128] = '\0'
|
||
# _list ↔ M 环
|
||
if string.strstr(deps_m, "_list") is not None:
|
||
lst_sha1: str = StubMergerStore.LookupSha1ByModName("_list")
|
||
if lst_sha1 is not None:
|
||
lst_deps: str = StubMergerStore.LookupIncludesDeps(lst_sha1)
|
||
if lst_deps is not None and _DepsContainsName(lst_deps, nm_m) != 0:
|
||
if _DepsRemoveName(deps_m, "_list") != 0:
|
||
# 内存破环后同步重写 deps.txt（诊断回退路径读文件）
|
||
_RewriteDepsTxt(temp_dir, sha1_m, deps_m)
|
||
# [CYCBREAK] 临时诊断：实际移除的环边（用后移除）
|
||
stdio.printf("[CYCBREAK] removed _list from %s deps\n", nm_m)
|
||
else:
|
||
stdio.printf("[CYCBREAK] %s has _list but _list deps no-match (lst_deps=%s)\n",
|
||
nm_m, lst_deps if lst_deps is not None else "none")
|
||
# _dict ↔ M 环
|
||
if string.strstr(deps_m, "_dict") is not None:
|
||
dct_sha1: str = StubMergerStore.LookupSha1ByModName("_dict")
|
||
if dct_sha1 is not None:
|
||
dct_deps: str = StubMergerStore.LookupIncludesDeps(dct_sha1)
|
||
if dct_deps is not None and _DepsContainsName(dct_deps, nm_m) != 0:
|
||
if _DepsRemoveName(deps_m, "_dict") != 0:
|
||
_RewriteDepsTxt(temp_dir, sha1_m, deps_m)
|
||
stdlib.free(nm_m)
|
||
return 0
|
||
|
||
|
||
# ============================================================
|
||
# _BuildReachableResult - 从 Sha1Store 构建可达 FileEntry 列表
|
||
#
|
||
# 依赖图按需扫描（StubMergerBuild._BuildReachableSha1Set）只登记
|
||
# "可达 includes + App"到全局 Sha1Store；本函数把 incdir 非空条目
|
||
# （includes）重建为 Phase1 1a/1b-pre/1b 需要的 FileEntry 列表。
|
||
# App 条目（incdir 空，仅登记不做可达文件处理）不进入 result——
|
||
# Phase B 已编译其 .obj，加入会导致 Phase B+ 重复编译 → 重复符号。
|
||
#
|
||
# Args:
|
||
# mb: 内存池
|
||
# includes_dir: includes 根目录（仅用于失败诊断，路径以 store 为准）
|
||
#
|
||
# Returns:
|
||
# ScanResult 指针（stdlib 分配，调用方负责释放）；失败返回 None
|
||
# ============================================================
|
||
def _BuildReachableResult(mb: memhub.MemBuddy | t.CPtr,
|
||
includes_dir: str) -> IncludesScanner.ScanResult | t.CPtr:
|
||
"""从 Sha1Store（仅可达 includes + App）构建可达 FileEntry 列表"""
|
||
if mb is None:
|
||
return None
|
||
store_count: t.CSizeT = t.CSizeT(StubMerger._sha1_store_count)
|
||
if store_count <= 0 or StubMerger._sha1_store_arr is None:
|
||
return None
|
||
result: IncludesScanner.ScanResult | t.CPtr = IncludesScanner.create_scan_result(mb)
|
||
if result is None:
|
||
return None
|
||
entry_size: t.CSizeT = IncludesScanner.FileEntry.__sizeof__()
|
||
i: t.CSizeT
|
||
for i in range(store_count):
|
||
sidx: t.CSizeT = t.CSizeT(i) * 17
|
||
if StubMerger._sha1_store_arr[sidx] == '\0':
|
||
continue
|
||
# 仅 includes 条目（incdir 非空）；App 条目不进入 result
|
||
iidx: t.CSizeT = t.CSizeT(i) * StubMerger.MAX_INC_DIR_LEN
|
||
if StubMerger._sha1_store_incdir[iidx] == '\0':
|
||
continue
|
||
eaddr: t.CUInt64T = t.CUInt64T(result.Entries) + t.CUInt64T(result.Count) * entry_size
|
||
entry: IncludesScanner.FileEntry | t.CPtr = (IncludesScanner.FileEntry | t.CPtr)(t.CVoid(eaddr, t.CPtr))
|
||
if entry is None:
|
||
continue
|
||
# SHA1 深拷贝（store 槽 17 字节）
|
||
sha1_buf: bytes = stdlib.malloc(17)
|
||
if sha1_buf is None:
|
||
continue
|
||
string.strcpy(sha1_buf, StubMerger._sha1_store_arr + sidx)
|
||
entry.Sha1 = sha1_buf
|
||
# RelPath / IncDir 深拷贝
|
||
ridx: t.CSizeT = t.CSizeT(i) * StubMerger.MAX_REL_PATH_LEN
|
||
rel_len: t.CSizeT = string.strlen(StubMerger._sha1_store_rel + ridx)
|
||
inc_len: t.CSizeT = string.strlen(StubMerger._sha1_store_incdir + iidx)
|
||
rel_buf: bytes = stdlib.malloc(rel_len + 1)
|
||
inc_buf: bytes = stdlib.malloc(inc_len + 1)
|
||
if rel_buf is None or inc_buf is None:
|
||
if rel_buf is not None:
|
||
stdlib.free(rel_buf)
|
||
if inc_buf is not None:
|
||
stdlib.free(inc_buf)
|
||
stdlib.free(sha1_buf)
|
||
continue
|
||
string.strcpy(rel_buf, StubMerger._sha1_store_rel + ridx)
|
||
string.strcpy(inc_buf, StubMerger._sha1_store_incdir + iidx)
|
||
entry.RelPath = rel_buf
|
||
entry.IncDir = inc_buf
|
||
# 绝对路径 = inc_dir + '/' + rel_path（与 _TryResolveModuleFile 拼接一致）
|
||
abs_buf: bytes = stdlib.malloc(inc_len + 1 + rel_len + 1)
|
||
if abs_buf is None:
|
||
stdlib.free(sha1_buf)
|
||
stdlib.free(rel_buf)
|
||
stdlib.free(inc_buf)
|
||
continue
|
||
string.strcpy(abs_buf, inc_buf)
|
||
abs_buf[inc_len] = '/'
|
||
string.strcpy(abs_buf + inc_len + 1, rel_buf)
|
||
entry.Path = abs_buf
|
||
# 模块名（store 登记时保存的绝对模块名）
|
||
midx: t.CSizeT = t.CSizeT(i) * 64
|
||
mod_len: t.CSizeT = string.strlen(StubMerger._sha1_store_mod + midx)
|
||
if mod_len > 0:
|
||
mod_buf: bytes = stdlib.malloc(mod_len + 1)
|
||
if mod_buf is not None:
|
||
string.strcpy(mod_buf, StubMerger._sha1_store_mod + midx)
|
||
entry.ModuleName = mod_buf
|
||
result.Count += 1
|
||
return result
|
||
|
||
# ============================================================
|
||
# Phase1Main - Phase1 拆分（阶段 E）：Phase1Main
|
||
#
|
||
# 直接移动模式：函数迁至此模块，原 Phase1.vp 仅保留常量/_mbuddy；
|
||
# 调用方经本模块前缀访问。全局符号经 Phase1._xxx 跨模块访问。
|
||
# ============================================================
|
||
|
||
# ============================================================
|
||
# RunPhase1 - Phase1: 扫描 includes 目录，按需翻译并生成 stub
|
||
#
|
||
# 对每个 .vp/.py 文件检查 stub 是否已存在，若不存在则翻译并分离 stub。
|
||
# "按需翻译"：仅对 stub 不存在的文件执行翻译，避免重复工作。
|
||
#
|
||
# Args:
|
||
# mb: 内存池
|
||
# includes_dir: includes 目录路径
|
||
# temp_dir: 临时目录（保存 stub）
|
||
# log: 日志器
|
||
#
|
||
# Returns:
|
||
# 0 成功，非 0 失败
|
||
# ============================================================
|
||
def RunPhase1(mb: memhub.MemBuddy | t.CPtr, includes_dir: str, temp_dir: str,
|
||
log: VLogger.Logger | t.CPtr) -> int:
|
||
"""Phase1: 扫描 includes 目录，按需翻译并生成 stub"""
|
||
if includes_dir is None or temp_dir is None:
|
||
VLogger.error("includes_dir 或 temp_dir 为空，无法运行 Phase1", "Phase1")
|
||
return 1
|
||
|
||
# 设置全局 temp_dir（供跨模块 CDefine 查找使用）
|
||
GenApi.set_temp_dir(temp_dir)
|
||
# 同步设置 HandlesExprCall 的 temp_dir（供 vpi stub 返回注解缓存按需加载）。
|
||
# 必须在 includes 翻译前设置：Phase1 翻译 json 等 includes 时，"同模块方法
|
||
# 调用（定义在后）"（如 json.__setitem__ 调 _pool_alloc）需要读
|
||
# {temp_dir}/{sha1[0:2]}/{sha1}.vpi 兜底，而 Phase2 的 SetTempDir 太晚。
|
||
GenApi.SetTempDir(temp_dir)
|
||
|
||
# 确保 temp 目录存在（build_dir/temp 可能尚未创建）
|
||
BuildPipeline.ensure_dir(temp_dir)
|
||
|
||
log.banner("Phase1: 扫描 includes（按需翻译）")
|
||
|
||
# 初始化 AST 表（只需一次）：必须在依赖图 BFS 扫描之前调用。
|
||
# _init_tables 懒分配关键字/操作符表（全局 _kw_head 等），若首次触发
|
||
# 发生在 BFS 的某个文件级 mark 之后，reset_to 会回收表节点 → 全局
|
||
# 悬空 → 后续 tokenize 的 _kw_lookup strcmp 访问 0x4 崩溃（1.log ASan）。
|
||
ast._init_tables(mb)
|
||
|
||
# 依赖图按需扫描（App 优先模型）：
|
||
# 1) 先扫 App 源文件（Config.SourceDir）的 import 依赖 → 初始队列
|
||
# 2) BFS 沿队列按文件系统解析 includes 文件，只对可达 includes
|
||
# 读文件+算 SHA1+解析 import，补充依赖图并登记 Sha1Store/
|
||
# deps 存储器/写 deps.txt
|
||
# 3) 未引用的 includes 从不读取（根本不扫描）
|
||
t_scan0: t.CUInt64T = w32.win32base.GetTickCount64()
|
||
reachable_set: str = None
|
||
reachable_count: int = 0
|
||
use_reachable: int = 0
|
||
if Config.SourceDir is not None:
|
||
reachable_set = stdlib.malloc(StubMerger.MAX_INCLUDES_SHA1 * 17)
|
||
if reachable_set is not None:
|
||
string.memset(reachable_set, 0, StubMerger.MAX_INCLUDES_SHA1 * 17)
|
||
reachable_count = StubMergerBuild._BuildReachableSha1Set(
|
||
mb, Config.SourceDir, temp_dir, reachable_set, None, includes_dir)
|
||
if reachable_count < 0:
|
||
VLogger.error("可达集合构建失败（依赖图扫描失败），终止编译", "Phase1")
|
||
return 1
|
||
if reachable_count <= 0:
|
||
VLogger.warning("App 无可达 includes 依赖，跳过 Phase1", "Phase1")
|
||
stdlib.free(reachable_set)
|
||
reachable_set = None
|
||
return 0
|
||
use_reachable = 1
|
||
else:
|
||
VLogger.error("reachable_set 分配失败，终止编译", "Phase1")
|
||
return 1
|
||
else:
|
||
VLogger.warning("SourceDir 不可用，无法构建依赖图，跳过 Phase1", "Phase1")
|
||
return 0
|
||
t_scan1: t.CUInt64T = w32.win32base.GetTickCount64()
|
||
VLogger.info(f"依赖图扫描总耗时: {t_scan1 - t_scan0} ms（可达 {reachable_count} 个 includes）", "Phase1")
|
||
|
||
# 从 Sha1Store（仅可达 includes + App）构造可达 FileEntry 列表：
|
||
# 后续 1a/1b-pre/1b 直接遍历 result（incdir 非空条目=includes），
|
||
# use_reachable 过滤幂等保留（result 即可达子集）
|
||
result: IncludesScanner.ScanResult | t.CPtr = _BuildReachableResult(mb, includes_dir)
|
||
if result is None:
|
||
VLogger.error("可达 FileEntry 列表构建失败，终止编译", "Phase1")
|
||
return 1
|
||
VLogger.info(f"可达 includes 条目: {result.Count}", "Phase1")
|
||
|
||
# 扫描后统一清理 _list/_dict 注入环边（_list↔memhub 等 2-环）。
|
||
# 依赖图按需扫描顺序不可靠，内联检查可能漏判，此处全量清理保证
|
||
# 依赖图无注入环，Phase A 多遍可收敛。破环后同步重写 deps.txt。
|
||
_BreakContainerCyclesInDeps(temp_dir)
|
||
|
||
# 写 _sha1_map.txt（仅可达 includes，人类可读输出；机器分析使用内存存储器）
|
||
StubMergerCollect.WriteIncludesSha1Map(mb, temp_dir, result, reachable_set, reachable_count)
|
||
|
||
# 兼容旧引用：sha1_set/set_count 对齐可达集合（use_reachable=1 时不被使用）
|
||
sha1_set: str = reachable_set
|
||
set_count: int = reachable_count
|
||
# 模块 SHA1 映射已由依赖图扫描登记到 StubMerger 全局存储器（仅可达 includes + App），
|
||
# HandlesExprCall._lookup_module_sha1 直接查全局存储器。
|
||
|
||
# FileEntry 结构体大小（Phase 1a 和 1b 共用）
|
||
entry_size: t.CSizeT = IncludesScanner.FileEntry.__sizeof__()
|
||
|
||
|
||
# ============================================================
|
||
# Phase 1a: 注册可达 includes 文件的 struct/enum/union
|
||
#
|
||
# 解决字母序依赖问题：按字母顺序翻译时，后面的 struct 未注册
|
||
# 前面就需要用。Phase 1a 先注册所有 struct/enum/union 到全局表，
|
||
# Phase 1b 全量翻译时 struct 已注册，走 existing 路径只翻译方法体。
|
||
# 只处理可达文件，避免翻译不需要的 includes（如 Test 不依赖 ast 模块）。
|
||
#
|
||
# 拓扑排序：预扫描所有文件收集类继承关系，按依赖顺序处理文件，
|
||
# 确保父类所在文件先于子类所在文件处理（如 base.vp 先于 astaux.vp）。
|
||
# ============================================================
|
||
p1a_reg: int = 0
|
||
p1a_skp: int = 0
|
||
p1a_fl: int = 0
|
||
|
||
# 预扫描类继承关系，构建拓扑顺序（用扁平字节数组，避免结构体 str 指针 NULL 崩溃）
|
||
class_names_buf: bytes = mb.alloc(Phase1.MAX_CLASSES * Phase1.CLASS_NAME_LEN)
|
||
parent_names_buf: bytes = mb.alloc(Phase1.MAX_CLASSES * Phase1.CLASS_NAME_LEN)
|
||
def_sha1s_buf: bytes = mb.alloc(Phase1.MAX_CLASSES * Phase1.SHA1_LEN)
|
||
topo_order: t.CPtr = mb.alloc(4 * Phase1.MAX_FILE_DEPS)
|
||
topo_count_box: t.CPtr = mb.alloc(4)
|
||
generic_file_indices: t.CPtr = mb.alloc(4 * Phase1.MAX_FILE_DEPS)
|
||
generic_file_count_box: t.CPtr = mb.alloc(4)
|
||
if generic_file_indices is not None:
|
||
string.memset(generic_file_indices, 0, 4 * Phase1.MAX_FILE_DEPS)
|
||
if generic_file_count_box is not None:
|
||
string.memset(generic_file_count_box, 0, 4)
|
||
topo_count: int = 0
|
||
|
||
if class_names_buf is not None and parent_names_buf is not None and def_sha1s_buf is not None and topo_order is not None and topo_count_box is not None:
|
||
string.memset(class_names_buf, 0, Phase1.MAX_CLASSES * Phase1.CLASS_NAME_LEN)
|
||
string.memset(parent_names_buf, 0, Phase1.MAX_CLASSES * Phase1.CLASS_NAME_LEN)
|
||
string.memset(def_sha1s_buf, 0, Phase1.MAX_CLASSES * Phase1.SHA1_LEN)
|
||
string.memset(topo_order, 0, 4 * Phase1.MAX_FILE_DEPS)
|
||
string.memset(topo_count_box, 0, 4)
|
||
if generic_file_indices is not None:
|
||
string.memset(generic_file_indices, 0, 4 * Phase1.MAX_FILE_DEPS)
|
||
if generic_file_count_box is not None:
|
||
string.memset(generic_file_count_box, 0, 4)
|
||
|
||
class_count_scanned: int = Phase1Scan._ScanClassInheritance(
|
||
mb, result, reachable_set, reachable_count,
|
||
use_reachable, sha1_set, set_count,
|
||
class_names_buf, parent_names_buf, def_sha1s_buf,
|
||
generic_file_indices, generic_file_count_box)
|
||
|
||
if class_count_scanned > 0:
|
||
td_len_ts_call: t.CSizeT = string.strlen(temp_dir)
|
||
ret_topo: int = Phase1Topo._TopoSortFiles(
|
||
mb, result, reachable_set, reachable_count,
|
||
use_reachable, sha1_set, set_count,
|
||
class_names_buf, parent_names_buf, def_sha1s_buf, class_count_scanned,
|
||
topo_order, topo_count_box,
|
||
temp_dir, td_len_ts_call)
|
||
if ret_topo == 0:
|
||
tc_ptr: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(topo_count_box, t.CPtr))
|
||
if tc_ptr is not None:
|
||
topo_count = tc_ptr[0]
|
||
VLogger.info(f"拓扑排序: {class_count_scanned} 个类, {topo_count} 个文件按依赖顺序处理", "Phase1")
|
||
else:
|
||
VLogger.error("拓扑排序失败（类继承依赖无法解析），终止编译", "Phase1")
|
||
return 1
|
||
else:
|
||
VLogger.warning("类继承预扫描无结果，回退到字母序", "Phase1")
|
||
topo_count = 0
|
||
|
||
# ============================================================
|
||
# Phase 1a-pre-generic-all: mark1 前全量预注册 includes 泛型模板
|
||
#
|
||
# 1a-pre-generic 只覆盖"可达"泛型文件；非可达泛型文件（如 FuncTest
|
||
# 场景的 linkedlist/llvmlite/_list）的模板要等 Phase2 Phase A-pre-inc
|
||
# （mark2 之后）才注册 → reset 回收表/节点 → _find_generic_template
|
||
# strcmp 悬空崩溃。此处对全部 includes 轻量预注册，使模板表与节点
|
||
# 都落在 mark1 之前（mark2 > mark1，reset_to(mark2) 不回收）。
|
||
# ============================================================
|
||
Phase1Scan._PreRegisterAllGenericTemplates(mb, result)
|
||
|
||
# ============================================================
|
||
# Phase 1a: 注册 struct/enum/union（mark1 前，全量 includes）
|
||
#
|
||
# OOM 治理：1a 对全部 includes parse 完整 AST（主 pool 分配），若
|
||
# 驻留到 mark1（循环后）会累积 98 文件 × 每文件 AST 树 → 1GB 主池
|
||
# 耗尽（NewAstTest alloc 失败 0xBE 根因）。现改为：
|
||
# 1) StdlibAlloc=1：1a 注册的跨文件持久数据（StructTable 表、
|
||
# Fields/Name/ModuleSha1、struct_ty 深拷贝、枚举/Renum/泛型
|
||
# 模板等）全部落在 perm_pool（永不 reset），不占主 pool；
|
||
# 2) 每文件翻译完 reset_to(文件级 mark)：仅回收该文件的 AST/
|
||
# lexer/token/IR 等瞬时数据，注册数据不受影响。
|
||
# mark1 仍在 1a 结束后打，1b 每文件 reset_to(mark1) 语义不变。
|
||
# ============================================================
|
||
GenApi.set_stdlib_alloc(1)
|
||
|
||
# 遍历文件：优先使用拓扑顺序，回退到字母序
|
||
iter_count: int = topo_count if topo_count > 0 else result.Count
|
||
# 1a 阶段总耗时起点（struct/enum/union 注册）
|
||
t_a0: t.CUInt64T = w32.win32base.GetTickCount64()
|
||
for i in range(iter_count):
|
||
# 每文件级 mark：翻译完回收该文件 AST（注册数据在 perm_pool）
|
||
p1a_mark: t.CSizeT = mb.mark()
|
||
# 获取文件索引：拓扑顺序或字母序
|
||
file_idx: int = i
|
||
if topo_count > 0:
|
||
od_addr: t.CUInt64T = t.CUInt64T(topo_order) + i * 4
|
||
od_ptr: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(od_addr, t.CPtr))
|
||
if od_ptr is not None:
|
||
file_idx = od_ptr[0]
|
||
|
||
entry_addr_r: t.CUInt64T = t.CUInt64T(result.Entries) + file_idx * entry_size
|
||
entry_r: IncludesScanner.FileEntry | t.CPtr = (IncludesScanner.FileEntry | t.CPtr)(t.CVoid(entry_addr_r, t.CPtr))
|
||
if entry_r is None:
|
||
p1a_fl += 1
|
||
continue
|
||
|
||
sha1_r: str = entry_r.Sha1
|
||
if sha1_r is None:
|
||
p1a_fl += 1
|
||
continue
|
||
|
||
# 按需翻译过滤（拓扑排序已过滤，但字母序回退需要检查）
|
||
if topo_count == 0:
|
||
if use_reachable != 0:
|
||
if StubMergerCollect._is_in_sha1_set(sha1_r, reachable_set, reachable_count) == 0:
|
||
p1a_skp += 1
|
||
continue
|
||
else:
|
||
if StubMergerCollect._is_in_sha1_set(sha1_r, sha1_set, set_count) == 0:
|
||
p1a_skp += 1
|
||
continue
|
||
|
||
# 单文件计时起点（读文件 + parse 段，定位 1a 卡点）
|
||
t_fa0: t.CUInt64T = w32.win32base.GetTickCount64()
|
||
|
||
# 读取文件内容
|
||
file_path_r: str = entry_r.Path
|
||
f_r: fileio.File | t.CPtr = fileio.File(file_path_r, fileio.MODE.R)
|
||
if f_r.closed:
|
||
p1a_fl += 1
|
||
continue
|
||
|
||
src_buf_r: bytes = stdlib.malloc(Phase1.SRC_BUF_SIZE)
|
||
if src_buf_r is None:
|
||
f_r.close()
|
||
p1a_fl += 1
|
||
continue
|
||
|
||
bytes_read_r: LONG = f_r.read_all(src_buf_r, Phase1.SRC_BUF_SIZE)
|
||
f_r.close()
|
||
if bytes_read_r <= 0:
|
||
stdlib.free(src_buf_r)
|
||
p1a_fl += 1
|
||
continue
|
||
if bytes_read_r < Phase1.SRC_BUF_SIZE:
|
||
src_buf_r[bytes_read_r] = 0
|
||
else:
|
||
src_buf_r[Phase1.SRC_BUF_SIZE - 1] = 0
|
||
|
||
# 解析 AST
|
||
lx_r: ast.Lexer | t.CPtr = ast.new_lexer(mb)
|
||
if lx_r is None:
|
||
stdlib.free(src_buf_r)
|
||
p1a_fl += 1
|
||
continue
|
||
ast._lexer_init(lx_r, src_buf_r, mb)
|
||
tokens_r: ast.Token | t.CPtr = ast.tokenize(lx_r)
|
||
tree_r: ast.AST | t.CPtr = ast.parse_tokens(mb, tokens_r)
|
||
if tree_r is None:
|
||
stdlib.free(src_buf_r)
|
||
p1a_fl += 1
|
||
continue
|
||
|
||
# 计时点 1：读文件 + parse 完成
|
||
t_fa1: t.CUInt64T = w32.win32base.GetTickCount64()
|
||
|
||
# 创建 Translator，设置 _declare_only=1（注册 struct/enum/union）
|
||
tr_r: HandlesTranslator.Translator | t.CPtr = HandlesTranslator.Translator()
|
||
if tr_r is None:
|
||
stdlib.free(src_buf_r)
|
||
p1a_fl += 1
|
||
continue
|
||
tr_r.ModuleSha1 = sha1_r
|
||
tr_r._declare_only = 1
|
||
tr_r.CurrentPackage = GenApi.compute_package_from_relpath(mb, entry_r.RelPath)
|
||
GenApi.set_current_file_type(file_path_r)
|
||
GenApi.set_current_file_imports(file_path_r)
|
||
GenApi.set_current_module_sha1(sha1_r)
|
||
GenApi.clear_cdefine_constants()
|
||
GenApi.clear_ctypedefs()
|
||
GenApi.reset_visible_structs(mb, 0)
|
||
ret_r: int = tr_r.translate(tree_r)
|
||
# 计时点 2：translate(declare=1) 返回
|
||
t_fa2: t.CUInt64T = w32.win32base.GetTickCount64()
|
||
if ret_r != 0:
|
||
p1a_fl += 1
|
||
else:
|
||
p1a_reg += 1
|
||
|
||
# 逐文件进度日志（parse/translate 两段，定位 1a 卡点）
|
||
VLogger.info(f"[{i+1}/{iter_count}] 1a: {file_path_r} (parse={t_fa1-t_fa0} ms, translate={t_fa2-t_fa1} ms, total={t_fa2-t_fa0} ms)", "Phase1")
|
||
|
||
# 释放 Translator 的 C malloc 资源
|
||
if tr_r._global_names is not None:
|
||
stdlib.free(tr_r._global_names)
|
||
if tr_r._nonlocal_names is not None:
|
||
stdlib.free(tr_r._nonlocal_names)
|
||
stdlib.free(src_buf_r)
|
||
|
||
# 回收本文件 AST/lexer/token/IR 等瞬时数据（注册数据已在 perm_pool，
|
||
# reset 不影响；IncludesScanner result 等在 p1a_mark 前分配同样保留）
|
||
mb.reset_to(p1a_mark)
|
||
|
||
# 1a 阶段总耗时（struct/enum/union 注册，含每文件读+parse+翻译声明）
|
||
t_a1: t.CUInt64T = w32.win32base.GetTickCount64()
|
||
VLogger.info(f"1a 总耗时: {t_a1-t_a0} ms (注册 struct/enum, reg={p1a_reg} skp={p1a_skp})", "Phase1")
|
||
|
||
if p1a_fl > 0:
|
||
VLogger.error(f"Phase 1a 有 {p1a_fl} 个文件 struct 注册失败", "Phase1")
|
||
return 1
|
||
|
||
# 释放拓扑排序资源（topo_order/topo_count_box 保留到 Phase 1b 结束，
|
||
# 1b 按拓扑序翻译需要它们；class/def 数组 1b 不再使用）
|
||
if class_names_buf is not None:
|
||
mb.free(class_names_buf)
|
||
if parent_names_buf is not None:
|
||
mb.free(parent_names_buf)
|
||
if def_sha1s_buf is not None:
|
||
mb.free(def_sha1s_buf)
|
||
if generic_file_indices is not None:
|
||
mb.free(generic_file_indices)
|
||
if generic_file_count_box is not None:
|
||
mb.free(generic_file_count_box)
|
||
|
||
# ============================================================
|
||
# Phase 1b: 全量翻译（struct 已注册，走 existing 路径翻译方法体）
|
||
# ============================================================
|
||
# OOM 回收：1a-pre/泛型预注册/1a 全部结束后打 mark1，
|
||
# 所有跨文件持久表（_struct_table/_enum_table/_generic_class_nodes/
|
||
# _renum_*/关键字表等）都落在 mark1 之前。
|
||
# 1b 每文件翻译完后 reset_to(mark1) 回收该文件的 AST/IR/Handler/dump 临时，
|
||
# 避免 90 文件累积导致 1GB 池 OOM。
|
||
mark1: t.CSizeT = mb.mark()
|
||
|
||
# 1b 阶段新注册的 struct/enum 条目改 stdlib 分配（跨文件永久存活，不被 reset 回收）
|
||
GenApi.set_stdlib_alloc(1)
|
||
|
||
# ============================================================
|
||
# Phase 1b-pre: 预生成所有待翻译文件的 .vpi stub
|
||
#
|
||
# 依赖模块的 .vpi 必须在翻译前可用：1b 主循环按扫描顺序翻译（非拓扑），
|
||
# 若 argparse 先于 hashtable 翻译，hashtable.vpi 尚未生成，
|
||
# LoadDepVpiIntoModule 读不到 → declare 未注入 → 跨模块调用
|
||
# 返回类型推断错误（如 HashTable.__getitem__ 被推断为 Ptr(HashTable)
|
||
# 而 define 返回 i8*）→ 运行时返回值截断崩溃。
|
||
# ============================================================
|
||
p1bpre_failed: int = 0
|
||
dir_len_pv: t.CSizeT = string.strlen(temp_dir)
|
||
# 1b-pre 阶段总耗时起点（vpi 预生成）
|
||
t_pv0: t.CUInt64T = w32.win32base.GetTickCount64()
|
||
for i_pv in range(result.Count):
|
||
entry_addr_pv: t.CUInt64T = t.CUInt64T(result.Entries) + i_pv * entry_size
|
||
entry_pv: IncludesScanner.FileEntry | t.CPtr = (IncludesScanner.FileEntry | t.CPtr)(t.CVoid(entry_addr_pv, t.CPtr))
|
||
if entry_pv is None or entry_pv.Sha1 is None:
|
||
p1bpre_failed += 1
|
||
continue
|
||
sha1_pv: str = entry_pv.Sha1
|
||
in_set_pv: int = 0
|
||
if use_reachable != 0:
|
||
in_set_pv = StubMergerCollect._is_in_sha1_set(sha1_pv, reachable_set, reachable_count)
|
||
else:
|
||
in_set_pv = StubMergerCollect._is_in_sha1_set(sha1_pv, sha1_set, set_count)
|
||
if in_set_pv == 0:
|
||
continue
|
||
# vpi 已存在则跳过
|
||
vpi_path_pv: str = StubMergerCollect._sliced_path(temp_dir, dir_len_pv, sha1_pv, "vpi")
|
||
if vpi_path_pv is None:
|
||
p1bpre_failed += 1
|
||
VLogger.error(f"[Phase 1b-pre] {entry_pv.Path} | vpi 路径生成失败", "Phase1")
|
||
continue
|
||
vf_pv: fileio.File | t.CPtr = fileio.File(vpi_path_pv, fileio.MODE.R)
|
||
if not vf_pv.closed:
|
||
vf_pv.close()
|
||
stdlib.free(vpi_path_pv)
|
||
continue
|
||
stdlib.free(vpi_path_pv)
|
||
# 读取文件、解析 AST、生成 vpi
|
||
file_path_pv: str = entry_pv.Path
|
||
f_pv: fileio.File | t.CPtr = fileio.File(file_path_pv, fileio.MODE.R)
|
||
if f_pv.closed:
|
||
p1bpre_failed += 1
|
||
VLogger.error(f"[Phase 1b-pre] {file_path_pv} | 无法打开源文件", "Phase1")
|
||
continue
|
||
src_buf_pv: bytes = stdlib.malloc(Phase1.SRC_BUF_SIZE)
|
||
if src_buf_pv is None:
|
||
f_pv.close()
|
||
p1bpre_failed += 1
|
||
VLogger.error(f"[Phase 1b-pre] {file_path_pv} | 源文件缓冲分配失败（OOM）", "Phase1")
|
||
continue
|
||
bytes_read_pv: LONG = f_pv.read_all(src_buf_pv, Phase1.SRC_BUF_SIZE)
|
||
f_pv.close()
|
||
if bytes_read_pv <= 0:
|
||
stdlib.free(src_buf_pv)
|
||
p1bpre_failed += 1
|
||
VLogger.error(f"[Phase 1b-pre] {file_path_pv} | 文件读取失败或为空", "Phase1")
|
||
continue
|
||
if bytes_read_pv < Phase1.SRC_BUF_SIZE:
|
||
src_buf_pv[bytes_read_pv] = 0
|
||
else:
|
||
src_buf_pv[Phase1.SRC_BUF_SIZE - 1] = 0
|
||
lx_pv: ast.Lexer | t.CPtr = ast.new_lexer(mb)
|
||
if lx_pv is None:
|
||
stdlib.free(src_buf_pv)
|
||
p1bpre_failed += 1
|
||
VLogger.error(f"[Phase 1b-pre] {file_path_pv} | 词法器分配失败（OOM）", "Phase1")
|
||
continue
|
||
ast._lexer_init(lx_pv, src_buf_pv, mb)
|
||
tokens_pv: ast.Token | t.CPtr = ast.tokenize(lx_pv)
|
||
tree_pv: ast.AST | t.CPtr = ast.parse_tokens(mb, tokens_pv)
|
||
if tree_pv is None:
|
||
stdlib.free(src_buf_pv)
|
||
p1bpre_failed += 1
|
||
VLogger.error(
|
||
f"[Phase 1b-pre] {file_path_pv} | AST 解析失败"
|
||
f"（第 {ast.get_parse_err_line()} 行第 {ast.get_parse_err_col()} 列: "
|
||
f"{ast.get_parse_err_msg()}）", "Phase1")
|
||
continue
|
||
pyi_buf_pv: bytes = stdlib.malloc(Phase1.PYI_BUF_SIZE)
|
||
if pyi_buf_pv is not None:
|
||
pyi_pos_pv: t.CSizeT = StubConverter._GeneratePyiFromAst(mb, tree_pv, entry_pv.RelPath, pyi_buf_pv, Phase1.PYI_BUF_SIZE)
|
||
if pyi_pos_pv > 0:
|
||
pyi_path_pv: str = StubMergerCollect._sliced_path(temp_dir, dir_len_pv, sha1_pv, "vpi")
|
||
if pyi_path_pv is not None:
|
||
pf_pv: fileio.File | t.CPtr = fileio.File(pyi_path_pv, fileio.MODE.W)
|
||
if not pf_pv.closed:
|
||
pf_pv.write(pyi_buf_pv, pyi_pos_pv)
|
||
pf_pv.close()
|
||
stdlib.free(pyi_path_pv)
|
||
stdlib.free(pyi_buf_pv)
|
||
else:
|
||
p1bpre_failed += 1
|
||
VLogger.error(f"[Phase 1b-pre] {file_path_pv} | pyi 缓冲分配失败（OOM）", "Phase1")
|
||
stdlib.free(src_buf_pv)
|
||
# 回收本文件 AST（vpi 已写盘，AST 不再需要；mark1 后无其他持久数据）
|
||
mb.reset_to(mark1)
|
||
|
||
# 1b-pre 阶段总耗时（vpi 预生成）
|
||
t_pv1: t.CUInt64T = w32.win32base.GetTickCount64()
|
||
VLogger.info(f"1b-pre 总耗时: {t_pv1-t_pv0} ms (vpi 预生成)", "Phase1")
|
||
|
||
if p1bpre_failed > 0:
|
||
VLogger.error(f"Phase 1b-pre 有 {p1bpre_failed} 个文件 vpi 预生成失败（失败文件与原因见上方逐条 [Phase 1b-pre] 日志）", "Phase1")
|
||
return 1
|
||
|
||
# 遍历每个文件（拓扑序优先：父类所在文件先翻译，确保跨模块 vtable
|
||
# 生成时依赖模块的函数 TypeStr 已注册；topo_count==0 回退扫描顺序）
|
||
translated: int = 0
|
||
skipped: int = 0
|
||
failed: int = 0
|
||
# 1b 阶段总耗时起点（全量翻译）
|
||
t_b0: t.CUInt64T = w32.win32base.GetTickCount64()
|
||
|
||
loop_n_b: int = result.Count
|
||
if topo_count > 0:
|
||
loop_n_b = topo_count
|
||
for i in range(loop_n_b):
|
||
# 获取 entry（拓扑序：file_idx 从 topo_order 取，否则用扫描索引）
|
||
file_idx_b: int = i
|
||
if topo_count > 0:
|
||
od_addr_b: t.CUInt64T = t.CUInt64T(topo_order) + t.CSizeT(i) * 4
|
||
od_ptr_b: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(od_addr_b, t.CPtr))
|
||
if od_ptr_b is not None:
|
||
file_idx_b = od_ptr_b[0]
|
||
entry_addr: t.CUInt64T = t.CUInt64T(result.Entries) + t.CSizeT(file_idx_b) * entry_size
|
||
entry: IncludesScanner.FileEntry | t.CPtr = (IncludesScanner.FileEntry | t.CPtr)(t.CVoid(entry_addr, t.CPtr))
|
||
if entry is None:
|
||
failed += 1
|
||
continue
|
||
|
||
sha1: str = entry.Sha1
|
||
if sha1 is None:
|
||
failed += 1
|
||
continue
|
||
|
||
# 检查 SHA1 是否在翻译集合中（拓扑序已过滤可达；仅回退扫描顺序时检查）
|
||
in_set: int = 1
|
||
if topo_count == 0:
|
||
in_set = 0
|
||
if use_reachable != 0:
|
||
in_set = StubMergerCollect._is_in_sha1_set(sha1, reachable_set, reachable_count)
|
||
else:
|
||
in_set = StubMergerCollect._is_in_sha1_set(sha1, sha1_set, set_count)
|
||
if in_set == 0:
|
||
skipped += 1
|
||
continue
|
||
|
||
# 构造 stub 路径: {temp_dir}/{sha1}.stub.ll
|
||
dir_len: t.CSizeT = string.strlen(temp_dir)
|
||
sha1_len: t.CSizeT = string.strlen(sha1)
|
||
stub_path: str = StubMergerCollect._sliced_path(temp_dir, dir_len, sha1, "stub.ll")
|
||
if stub_path is None:
|
||
failed += 1
|
||
continue
|
||
|
||
# 检查 stub 是否已存在（按需翻译：跳过已存在的）
|
||
# 例外：含 class 定义的模块必须翻译——类注册（register_struct）
|
||
# 只在翻译时发生；增量编译时 stub/text 已存在被跳过 → 跨模块类构造
|
||
# find_struct_by_module 落空 → 返回类型 fail-fast
|
||
# （fltk.Fl_Window(...) → "4049e14d73c7e091.Fl_Window 无法精确获取"，
|
||
# 仅 --clean 全量翻译时可用）。扫描 1b-pre 生成的 vpi 是否有
|
||
# "class " 行（vpi 为解析后存根，精确无注释/字符串误判）。
|
||
force_translate: int = 0
|
||
vpi_path_1b: str = StubMergerCollect._sliced_path(temp_dir, dir_len, sha1, "vpi")
|
||
if vpi_path_1b is not None:
|
||
vf_1b: fileio.File | t.CPtr = fileio.File(vpi_path_1b, fileio.MODE.R)
|
||
if not vf_1b.closed:
|
||
vbuf_1b: bytes = stdlib.malloc(1048576)
|
||
if vbuf_1b is not None:
|
||
vrd_1b: LONG = vf_1b.read_all(vbuf_1b, 1048576)
|
||
vf_1b.close()
|
||
if vrd_1b > 0:
|
||
if vrd_1b < 1048576:
|
||
vbuf_1b[vrd_1b] = 0
|
||
else:
|
||
vbuf_1b[1048576 - 1] = 0
|
||
vtotal_1b: t.CSizeT = string.strlen(vbuf_1b)
|
||
vpos_1b: t.CSizeT = 0
|
||
while vpos_1b < vtotal_1b:
|
||
vls_1b: t.CSizeT = vpos_1b
|
||
while vpos_1b < vtotal_1b and vbuf_1b[vpos_1b] != '\n':
|
||
vpos_1b += 1
|
||
vll_1b: t.CSizeT = vpos_1b - vls_1b
|
||
if vpos_1b < vtotal_1b:
|
||
vpos_1b += 1
|
||
vi2_1b: t.CSizeT = vls_1b
|
||
while vi2_1b < vls_1b + vll_1b and vbuf_1b[vi2_1b] == ' ':
|
||
vi2_1b += 1
|
||
if vll_1b >= 6 and string.strncmp(vbuf_1b + vi2_1b, "class ", 6) == 0:
|
||
force_translate = 1
|
||
break
|
||
stdlib.free(vbuf_1b)
|
||
else:
|
||
vf_1b.close()
|
||
stdlib.free(vpi_path_1b)
|
||
if force_translate == 1:
|
||
# 含 class：强制翻译（覆盖 stub/text，重新注册类）
|
||
rp: str = entry.RelPath
|
||
VLogger.info(f"含 class 定义，强制翻译: {rp}(sha1={sha1})", "Phase1")
|
||
else:
|
||
# 检查 stub 是否已存在（按需翻译：跳过已存在的）
|
||
sf: fileio.File | t.CPtr = fileio.File(stub_path, fileio.MODE.R)
|
||
if not sf.closed:
|
||
sf.close()
|
||
# 检查 text.ll 是否也存在（虚表扫描需要 text.ll）
|
||
text_path: str = StubMergerCollect._sliced_path(temp_dir, dir_len, sha1, "text.ll")
|
||
if text_path is not None:
|
||
tf: fileio.File | t.CPtr = fileio.File(text_path, fileio.MODE.R)
|
||
if not tf.closed:
|
||
tf.close()
|
||
stdlib.free(text_path)
|
||
stdlib.free(stub_path)
|
||
continue
|
||
stdlib.free(text_path)
|
||
# text.ll 不存在，需要重新翻译
|
||
rp: str = entry.RelPath
|
||
VLogger.info(f"text.ll 不存在，重新翻译: {rp}(sha1={sha1})", "Phase1")
|
||
else:
|
||
# stub 不存在，需要翻译
|
||
rp: str = entry.RelPath
|
||
VLogger.info(f"翻译: {rp}(sha1={sha1})", "Phase1")
|
||
|
||
stdlib.free(stub_path)
|
||
|
||
# 单文件计时起点：读文件（parse/translate/dump 三段，对齐 Phase2 插桩）
|
||
t_f0: t.CUInt64T = w32.win32base.GetTickCount64()
|
||
|
||
# 读取文件内容
|
||
file_path: str = entry.Path
|
||
f: fileio.File | t.CPtr = fileio.File(file_path, fileio.MODE.R)
|
||
if f is None:
|
||
VLogger.error(f"无法打开(f=None): {file_path}", "Phase1")
|
||
failed += 1
|
||
continue
|
||
if f.closed:
|
||
VLogger.error(f"无法打开: {file_path}", "Phase1")
|
||
failed += 1
|
||
continue
|
||
|
||
src_buf: bytes = stdlib.malloc(Phase1.SRC_BUF_SIZE)
|
||
if src_buf is None:
|
||
f.close()
|
||
failed += 1
|
||
continue
|
||
|
||
bytes_read: LONG = f.read_all(src_buf, Phase1.SRC_BUF_SIZE)
|
||
f.close()
|
||
if bytes_read <= 0:
|
||
VLogger.error(f"读取失败: {file_path}", "Phase1")
|
||
stdlib.free(src_buf)
|
||
failed += 1
|
||
continue
|
||
if bytes_read < Phase1.SRC_BUF_SIZE:
|
||
src_buf[bytes_read] = 0
|
||
else:
|
||
src_buf[Phase1.SRC_BUF_SIZE - 1] = 0
|
||
|
||
# 解析 AST
|
||
lx: ast.Lexer | t.CPtr = ast.new_lexer(mb)
|
||
if lx is None:
|
||
stdlib.free(src_buf)
|
||
failed += 1
|
||
continue
|
||
ast._lexer_init(lx, src_buf, mb)
|
||
tokens: ast.Token | t.CPtr = ast.tokenize(lx)
|
||
tree: ast.AST | t.CPtr = ast.parse_tokens(mb, tokens)
|
||
if tree is None:
|
||
VLogger.error(f"AST 解析失败: {file_path}", "Phase1")
|
||
stdlib.free(src_buf)
|
||
failed += 1
|
||
continue
|
||
|
||
# 计时点 1：读文件 + lexer/tokenize/parse 完成
|
||
t_f1: t.CUInt64T = w32.win32base.GetTickCount64()
|
||
|
||
# 文本扫描注册 CTypedef（stdint.vp 崩溃根因修复）：AST 字符串字段
|
||
# （attr/id）token 零拷贝指向 src_buf，跨阶段内容不稳定（悬空），
|
||
# ScanCTypedefDeclarations 读 attr 会 0xC0000005。改用源码文本逐行
|
||
# 匹配 "NAME: t.CTypedef = TYPE"，在 src_buf 有效期内注册。
|
||
GenApi.RegisterCTypedefFromSource(mb, src_buf)
|
||
|
||
# 生成 .vpi 存根文件（直接遍历 AST，不依赖 PythonToStubConverter）
|
||
# 1b-pre 已为所有待翻译文件生成 vpi，此处仅兜底：vpi 缺失才生成
|
||
# （消除 1b-pre 与 1b 对同一文件的重复 _GeneratePyiFromAst 工作）
|
||
vpi_path_1b: str = StubMergerCollect._sliced_path(temp_dir, dir_len, sha1, "vpi")
|
||
if vpi_path_1b is not None:
|
||
vf_1b: fileio.File | t.CPtr = fileio.File(vpi_path_1b, fileio.MODE.R)
|
||
vpi_exists_1b: int = 0
|
||
if not vf_1b.closed:
|
||
vf_1b.close()
|
||
vpi_exists_1b = 1
|
||
if vpi_exists_1b == 0:
|
||
pyi_buf: bytes = stdlib.malloc(Phase1.PYI_BUF_SIZE)
|
||
if pyi_buf is not None:
|
||
pyi_pos: t.CSizeT = StubConverter._GeneratePyiFromAst(
|
||
mb, tree, entry.RelPath, pyi_buf, Phase1.PYI_BUF_SIZE)
|
||
if pyi_pos > 0:
|
||
pf_1b: fileio.File | t.CPtr = fileio.File(vpi_path_1b, fileio.MODE.W)
|
||
if not pf_1b.closed:
|
||
pf_1b.write(pyi_buf, pyi_pos)
|
||
pf_1b.close()
|
||
stdlib.free(pyi_buf)
|
||
stdlib.free(vpi_path_1b)
|
||
|
||
# 翻译 AST → LLVM IR
|
||
tr: HandlesTranslator.Translator | t.CPtr = HandlesTranslator.Translator()
|
||
if tr is None:
|
||
stdlib.free(src_buf)
|
||
failed += 1
|
||
continue
|
||
tr.ModuleSha1 = sha1
|
||
tr.CurrentPackage = GenApi.compute_package_from_relpath(mb, entry.RelPath)
|
||
GenApi.set_current_file_type(file_path)
|
||
GenApi.set_current_file_imports(file_path)
|
||
GenApi.set_current_module_sha1(sha1)
|
||
GenApi.clear_cdefine_constants()
|
||
GenApi.clear_ctypedefs()
|
||
GenApi.reset_visible_structs(mb, 0)
|
||
|
||
# 翻译前哨兵：AST 解析（lexer/tokenize/parse_tokens）同样在 pool 分配，
|
||
# 若解析器越界写破坏了存活块 header，此处立即定位到当前文件
|
||
# （崩溃点漂移排查：把"翻译过程中随机崩"变成确定性定位）
|
||
pre_rc: t.CInt = mb.check_alive_blocks(mark1)
|
||
if pre_rc != 0:
|
||
VLogger.error(f"内存损坏(翻译前存活块, rc={pre_rc}): {file_path}", "Phase1")
|
||
stdlib.exit(1)
|
||
|
||
ret: int = tr.translate(tree)
|
||
# 计时点 2：translate 返回
|
||
t_f2: t.CUInt64T = w32.win32base.GetTickCount64()
|
||
# 诊断：translate 正常返回（若挂起在 translate 内部，此行不会打印，
|
||
# 用于区分"translate 内部挂起"与"后续 dump 挂起"）
|
||
if ret == 0:
|
||
VLogger.info(f"翻译完成(translate返回): {file_path}", "Phase1")
|
||
if ret != 0:
|
||
VLogger.error(f"翻译失败: {file_path}", "Phase1") # error 终止
|
||
stdlib.free(src_buf)
|
||
if tr._global_names is not None:
|
||
stdlib.free(tr._global_names)
|
||
if tr._nonlocal_names is not None:
|
||
stdlib.free(tr._nonlocal_names)
|
||
failed += 1
|
||
continue
|
||
|
||
# 诊断（builtins.vp dump 卡死根因排查）：pre_rc 哨兵在 translate 前，
|
||
# 覆盖不到 translate 期间的 AST/IR 写穿。dump 前立即检查存活块
|
||
# header/canary（rc=6 块尾 canary 损坏=越界写穿数据区），把"dump
|
||
# 时链表成环卡死"变成"定位到 translate 期间写穿发生的文件"。
|
||
post_rc: t.CInt = mb.check_alive_blocks(mark1)
|
||
if post_rc != 0:
|
||
VLogger.error(f"内存损坏(translate后存活块, rc={post_rc}): {file_path}", "Phase1")
|
||
|
||
# dump stub IR(declarations only)
|
||
PHASE1_IR_SIZE: t.CSizeT = 1048576
|
||
stub_buf: bytes = stdlib.malloc(PHASE1_IR_SIZE)
|
||
if stub_buf is None:
|
||
stdlib.free(src_buf)
|
||
if tr._global_names is not None:
|
||
stdlib.free(tr._global_names)
|
||
if tr._nonlocal_names is not None:
|
||
stdlib.free(tr._nonlocal_names)
|
||
failed += 1
|
||
continue
|
||
tr.dump_ir(stub_buf, PHASE1_IR_SIZE, llvmlite.OUTPUT_STUB)
|
||
stub_len: t.CSizeT = string.strlen(stub_buf)
|
||
|
||
# save stub.ll
|
||
dir_len_p1: t.CSizeT = string.strlen(temp_dir)
|
||
stub_path_p1: str = StubMergerCollect._sliced_path(temp_dir, dir_len_p1, sha1, "stub.ll")
|
||
if stub_path_p1 is not None:
|
||
sf_p1: fileio.File | t.CPtr = fileio.File(stub_path_p1, fileio.MODE.W)
|
||
if not sf_p1.closed:
|
||
sf_p1.write(stub_buf, stub_len)
|
||
sf_p1.close()
|
||
stdlib.free(stub_path_p1)
|
||
stdlib.free(stub_buf)
|
||
|
||
# dump text IR(definitions only)
|
||
text_buf: bytes = stdlib.malloc(PHASE1_IR_SIZE)
|
||
if text_buf is None:
|
||
stdlib.free(src_buf)
|
||
if tr._global_names is not None:
|
||
stdlib.free(tr._global_names)
|
||
if tr._nonlocal_names is not None:
|
||
stdlib.free(tr._nonlocal_names)
|
||
failed += 1
|
||
continue
|
||
tr.dump_ir(text_buf, PHASE1_IR_SIZE, llvmlite.OUTPUT_TEXT)
|
||
text_len: t.CSizeT = string.strlen(text_buf)
|
||
|
||
# save text.ll
|
||
text_path_p1: str = StubMergerCollect._sliced_path(temp_dir, dir_len_p1, sha1, "text.ll")
|
||
if text_path_p1 is not None:
|
||
tf_p1: fileio.File | t.CPtr = fileio.File(text_path_p1, fileio.MODE.W)
|
||
if not tf_p1.closed:
|
||
tf_p1.write(text_buf, text_len)
|
||
tf_p1.close()
|
||
stdlib.free(text_path_p1)
|
||
stdlib.free(text_buf)
|
||
|
||
# 计时点 3：dump（stub+text）完成；输出单文件三段耗时
|
||
t_f3: t.CUInt64T = w32.win32base.GetTickCount64()
|
||
VLogger.info(f"[{i+1}/{loop_n_b}] 翻译: {file_path} (parse={t_f1-t_f0} ms, translate={t_f2-t_f1} ms, dump={t_f3-t_f2} ms, total={t_f3-t_f0} ms)", "Phase1")
|
||
|
||
# save dependencies(_imported_modules) for Phase B
|
||
deps_path_p1: str = StubMergerCollect._sliced_path(temp_dir, dir_len_p1, sha1, "deps.txt")
|
||
if deps_path_p1 is not None:
|
||
df_p1: fileio.File | t.CPtr = fileio.File(deps_path_p1, fileio.MODE.W)
|
||
if not df_p1.closed:
|
||
if tr._imported_modules is not None:
|
||
dl: t.CSizeT = string.strlen(tr._imported_modules)
|
||
df_p1.write(tr._imported_modules, dl)
|
||
df_p1.close()
|
||
stdlib.free(deps_path_p1)
|
||
|
||
# 释放 Translator 的 C malloc 资源
|
||
if tr._global_names is not None:
|
||
stdlib.free(tr._global_names)
|
||
if tr._nonlocal_names is not None:
|
||
stdlib.free(tr._nonlocal_names)
|
||
|
||
# 所有 dump_ir 完成，src_buf 不再需要
|
||
stdlib.free(src_buf)
|
||
|
||
translated += 1
|
||
|
||
# OOM 回收：回收本文件翻译产生的瞬时数据（AST/tokens/lexer/IR/Handler/dump 临时）
|
||
# 持久表（struct/enum/REnum/spec 缓存/全局函数名等）已迁出 reset 区间（stdlib），
|
||
# reset_to(mark1) 只回收 mark1 之后分配、未被单独 free 的块。
|
||
#
|
||
# 内存哨兵（崩溃点漂移排查）：reset 前检查存活块 header（抓越界写），
|
||
# reset 后检查 free list 完整性（抓链表损坏）。任一损坏立即终止，
|
||
# 把"随机崩溃漂移"变成"确定性定位到当前文件"。
|
||
# rc 语义（check_alive_blocks）：1=块地址低于 usable，2=高于 usable+size，
|
||
# 3=order 为负，4=order 超 max_order，5=块未对齐到 order 大小，
|
||
# 6=块尾 canary 损坏（越界写穿数据区——事发地定位）
|
||
# rc 语义（self_check）：1/2=越界，3=对齐错，4=free list 成环/超步数
|
||
chk_rc: t.CInt = mb.check_alive_blocks(mark1)
|
||
if chk_rc != 0:
|
||
VLogger.error(f"内存损坏(存活块 header, rc={chk_rc}): {file_path}", "Phase1")
|
||
# A/B 实验结论（已确认）：禁用 reset_to 后 stdarg 仍崩 → 越界写是根因，
|
||
# 与回收/UAF 无关。恢复 reset_to，改用块尾 canary（check_alive_blocks 校验）
|
||
# 定位越界写事发地到具体文件。
|
||
mb.reset_to(mark1)
|
||
sc_rc: t.CInt = mb.self_check()
|
||
if sc_rc != 0:
|
||
VLogger.error(f"内存损坏(free list, rc={sc_rc}): {file_path}", "Phase1")
|
||
|
||
# 1b 阶段总耗时（全量翻译 + dump）
|
||
t_b1: t.CUInt64T = w32.win32base.GetTickCount64()
|
||
VLogger.info(f"1b 总耗时: {t_b1-t_b0} ms (全量翻译, trans={translated} skip={skipped} fail={failed})", "Phase1")
|
||
|
||
# 释放 Phase 1b 使用的拓扑排序资源
|
||
if topo_order is not None:
|
||
mb.free(topo_order)
|
||
if topo_count_box is not None:
|
||
mb.free(topo_count_box)
|
||
|
||
VLogger.info(f"完成: 翻译={translated} 跳过={skipped} 失败={failed}", "Phase1")
|
||
|
||
if failed > 0:
|
||
VLogger.error(f"Phase1 有 {failed} 个文件翻译失败", "Phase1")
|
||
return 1
|
||
|
||
# 释放可达 SHA1 集合（如果分配了）
|
||
if reachable_set is not None:
|
||
stdlib.free(reachable_set)
|
||
|
||
return 0
|