四处解析失败上报点(Phase2 的 [Phase A-pre-vpi]、Phase1 的 [Phase 1b-pre] 与单文件、main.vp 单文件)此前只打 AST 解析失败,看不到定位与原因;现内联 第 N 行第 M 列: <原因>,与 dump_parse_traceback 的完整 traceback 同源(原因由 includes/ast/parser.vp 的 _record_* 与 parse_tokens 出口兜底补记提供)。另:App/main.vp 同时带上你工作区既有的 define 迁移/版本号改动。
1194 lines
58 KiB
Plaintext
1194 lines
58 KiB
Plaintext
import t, c
|
||
from stdint import *
|
||
import string
|
||
import stdlib
|
||
import memhub
|
||
import viperlib
|
||
import w32.fileio as fileio
|
||
import w32.win32base
|
||
import ast
|
||
import llvmlite
|
||
import lib.core.VLogger as VLogger
|
||
import lib.core.Gen.Translator as HandlesTranslator
|
||
import lib.core.IncludesScanner as IncludesScanner
|
||
import lib.Stub.StubMerger as StubMerger
|
||
import lib.core.Gen.Generator as GenGenerator
|
||
import lib.core.CompilerFactory as CompilerFactory
|
||
import lib.Phase.BuildPipeline as BuildPipeline
|
||
import lib.Projectrans.Config as Config
|
||
import lib.Stub.StubGen.Converter as StubConverter
|
||
import lib.Stub.StubMerger.StubMergerBuild as StubMergerBuild
|
||
import lib.Stub.StubMerger.StubMergerCollect as StubMergerCollect
|
||
import lib.Stub.StubMerger.StubMergerIR as StubMergerIR
|
||
import lib.Stub.StubMerger.StubMergerStore as StubMergerStore
|
||
|
||
import lib.Phase.Phase1.Phase1Scan as Phase1Scan
|
||
import lib.Phase.Phase1.Phase1Topo as Phase1Topo
|
||
import lib.Phase.Phase1 as Phase1
|
||
|
||
import lib.core.Gen.GenApi as GenApi
|
||
import lib.core.Gen.DebugInfoGen as DebugInfoGen
|
||
|
||
|
||
# ============================================================
|
||
# _InjectedCreatesCycle - 容器模块自动注入前的破环检查
|
||
#
|
||
# 注入 _list/_dict 前检查被注入模块自身的依赖是否已含当前模块,命中(如 memhub↔_list)则跳过注入——
|
||
# 否则 Phase A 多遍收敛死锁("无法完成翻译");去掉环边后 _list 仍因当前模块被其依赖而处于闭包,list 类仍由 Phase1b 注册。
|
||
# ============================================================
|
||
def _InjectedCreatesCycle(entry: rawptr[IncludesScanner.FileEntry],
|
||
mod_name: str) -> int:
|
||
"""注入 mod_name 到当前模块前检查是否成环,返回 1=成环需跳过注入"""
|
||
if entry is None or entry.RelPath is None or mod_name is None:
|
||
return 0
|
||
inj_sha1: str = StubMergerStore.LookupSha1ByModName(mod_name)
|
||
if inj_sha1 is None:
|
||
return 0
|
||
inj_deps: str = StubMergerStore.LookupIncludesDeps(inj_sha1)
|
||
if inj_deps is None:
|
||
return 0
|
||
rp: str = entry.RelPath
|
||
# 当前模块名 = RelPath 基名(最后 '/' 或 '\\' 之后)去扩展名
|
||
base: str = string.strrchr(rp, '/')
|
||
if base is None:
|
||
base = string.strrchr(rp, '\\')
|
||
if base is not None:
|
||
base = base + 1
|
||
else:
|
||
base = rp
|
||
dot: str = string.strchr(base, '.')
|
||
nm_buf: bytes = stdlib.malloc(128)
|
||
if nm_buf is None:
|
||
return 0
|
||
if dot is not None:
|
||
nm_len: sizet = sizet(uint64(dot) - uint64(base))
|
||
if nm_len > 127:
|
||
nm_len = 127
|
||
string.strncpy(nm_buf, base, nm_len)
|
||
nm_buf[nm_len] = '\0'
|
||
else:
|
||
string.strncpy(nm_buf, base, 127)
|
||
nm_buf[127] = '\0'
|
||
nm_len2: sizet = string.strlen(nm_buf)
|
||
d_len: sizet = string.strlen(inj_deps)
|
||
dp: sizet = 0
|
||
while dp < d_len:
|
||
ts: sizet = dp
|
||
while dp < d_len and inj_deps[dp] != ' ':
|
||
dp += 1
|
||
tl: sizet = dp - ts
|
||
if dp < d_len:
|
||
dp += 1
|
||
if tl > 0 and tl == nm_len2 and string.strncmp(inj_deps + ts, nm_buf, tl) == 0:
|
||
stdlib.free(nm_buf)
|
||
return 1
|
||
stdlib.free(nm_buf)
|
||
return 0
|
||
|
||
|
||
def _DepsContainsName(deps: str, name: str) -> int:
|
||
"""deps 字符串(空格分隔模块名)是否含 name token(词边界精确匹配)"""
|
||
if deps is None or name is None:
|
||
return 0
|
||
nl: sizet = string.strlen(name)
|
||
dl: sizet = string.strlen(deps)
|
||
dp: sizet = 0
|
||
while dp < dl:
|
||
ts: sizet = dp
|
||
while dp < dl and deps[dp] != ' ':
|
||
dp += 1
|
||
tl: sizet = dp - ts
|
||
if dp < dl:
|
||
dp += 1
|
||
if tl == nl and string.strncmp(deps + ts, name, nl) == 0:
|
||
return 1
|
||
return 0
|
||
|
||
|
||
def _DepsRemoveName(deps: str, name: str) -> int:
|
||
"""从 deps 字符串移除 name token(原地重建,返回 1=有改动)"""
|
||
if deps is None or name is None:
|
||
return 0
|
||
nl: sizet = string.strlen(name)
|
||
dl: sizet = string.strlen(deps)
|
||
buf: bytes = stdlib.malloc(StubMerger.MAX_DEPS_STR_LEN)
|
||
if buf is None:
|
||
return 0
|
||
buf[0] = '\0'
|
||
bl: sizet = 0
|
||
dp: sizet = 0
|
||
while dp < dl:
|
||
ts: sizet = dp
|
||
while dp < dl and deps[dp] != ' ':
|
||
dp += 1
|
||
tl: sizet = dp - ts
|
||
if dp < dl:
|
||
dp += 1
|
||
if tl == 0:
|
||
continue
|
||
if tl == nl and string.strncmp(deps + ts, name, nl) == 0:
|
||
continue
|
||
if bl > 0:
|
||
buf[bl] = ' '
|
||
bl += 1
|
||
string.strncpy(buf + bl, deps + ts, tl)
|
||
bl += tl
|
||
buf[bl] = '\0'
|
||
string.strcpy(deps, buf)
|
||
stdlib.free(buf)
|
||
return 1
|
||
|
||
|
||
def _RewriteDepsTxt(temp_dir: str, sha1: str, deps_str: str) -> None:
|
||
"""重写 {temp_dir}/{sha1前2位}/{sha1}.deps.txt(内存破环后同步磁盘)"""
|
||
if temp_dir is None or sha1 is None or deps_str is None:
|
||
return
|
||
td_len: sizet = string.strlen(temp_dir)
|
||
dpath: str = StubMergerCollect._sliced_path(temp_dir, td_len, sha1, "deps.txt")
|
||
if dpath is None:
|
||
return
|
||
df: rawptr[fileio.File] = fileio.File(dpath, fileio.MODE.W)
|
||
if not df.closed:
|
||
dl: sizet = string.strlen(deps_str)
|
||
if dl > 0:
|
||
df.write(deps_str, dl)
|
||
df.close()
|
||
stdlib.free(dpath)
|
||
|
||
|
||
def _BreakContainerCyclesInDeps(temp_dir: str) -> int:
|
||
"""扫描后统一清理 _list/_dict 注入环边。
|
||
|
||
Phase1a-pre 全量扫描用 FindFirstFileA(文件系统序),注入 _list/_dict 时
|
||
被注入模块(_list/_dict)自身的 deps 未必已注册 → _InjectedCreatesCycle
|
||
内联检查可能漏判(如 _list 显式 import memhub,而 memhub 的 list[str]
|
||
注入 _list 未被拦截)→ memhub↔_list 2-环 → Phase A 永不收敛。此处扫描
|
||
结束后全量检查:凡某模块 M 的 deps 含 _list/_dict,且 _list/_dict 的 deps
|
||
含 M → 从 M 的 deps 移除注入边(list 类仍由 Phase1b 经闭包注册)。
|
||
内存破环后同步重写 {temp_dir}/deps.txt,保证磁盘与内存一致(Phase2 依赖
|
||
图诊断回退读 deps.txt)。
|
||
"""
|
||
if StubMerger._includes_deps_sha1 is None or StubMerger._includes_deps_str is None:
|
||
return 0
|
||
cnt: sizet = StubMerger._includes_deps_count
|
||
ci: sizet
|
||
for ci in range(cnt):
|
||
sha1_m: str = StubMerger._includes_deps_sha1 + sizet(ci) * 17
|
||
deps_m: str = StubMerger._includes_deps_str + sizet(ci) * StubMerger.MAX_DEPS_STR_LEN
|
||
if sha1_m is None or deps_m is None:
|
||
continue
|
||
if string.strstr(deps_m, "_list") is None and string.strstr(deps_m, "_dict") is None:
|
||
continue
|
||
# 该模块名(RelPath 基名去扩展名)
|
||
rel_m: str = StubMergerStore.LookupSha1RelPath(sha1_m)
|
||
if rel_m is None:
|
||
continue
|
||
base_m: str = string.strrchr(rel_m, '/')
|
||
if base_m is None:
|
||
base_m = string.strrchr(rel_m, '\\')
|
||
if base_m is not None:
|
||
base_m = base_m + 1
|
||
else:
|
||
base_m = rel_m
|
||
dot_m: str = string.strchr(base_m, '.')
|
||
if dot_m is not None:
|
||
nm_len_m: sizet = sizet(uint64(dot_m) - uint64(base_m))
|
||
nm_m: bytes = stdlib.malloc(nm_len_m + 1)
|
||
if nm_m is None:
|
||
continue
|
||
string.strncpy(nm_m, base_m, nm_len_m)
|
||
nm_m[nm_len_m] = '\0'
|
||
else:
|
||
nm_m: bytes = stdlib.malloc(129)
|
||
if nm_m is None:
|
||
continue
|
||
string.strncpy(nm_m, base_m, 128)
|
||
nm_m[128] = '\0'
|
||
# _list ↔ M 环
|
||
if string.strstr(deps_m, "_list") is not None:
|
||
lst_sha1: str = StubMergerStore.LookupSha1ByModName("_list")
|
||
if lst_sha1 is not None:
|
||
lst_deps: str = StubMergerStore.LookupIncludesDeps(lst_sha1)
|
||
if lst_deps is not None and _DepsContainsName(lst_deps, nm_m) != 0:
|
||
if _DepsRemoveName(deps_m, "_list") != 0:
|
||
# 内存破环后同步重写 deps.txt(诊断回退路径读文件)
|
||
_RewriteDepsTxt(temp_dir, sha1_m, deps_m)
|
||
# _dict ↔ M 环
|
||
if string.strstr(deps_m, "_dict") is not None:
|
||
dct_sha1: str = StubMergerStore.LookupSha1ByModName("_dict")
|
||
if dct_sha1 is not None:
|
||
dct_deps: str = StubMergerStore.LookupIncludesDeps(dct_sha1)
|
||
if dct_deps is not None and _DepsContainsName(dct_deps, nm_m) != 0:
|
||
if _DepsRemoveName(deps_m, "_dict") != 0:
|
||
_RewriteDepsTxt(temp_dir, sha1_m, deps_m)
|
||
stdlib.free(nm_m)
|
||
return 0
|
||
|
||
|
||
# ============================================================
|
||
# _BuildReachableResult - 从 Sha1Store 构建可达 FileEntry 列表
|
||
#
|
||
# 依赖图按需扫描(StubMergerBuild._BuildReachableSha1Set)只把「可达 includes + App」登记到全局 Sha1Store;
|
||
# 本函数把 incdir 非空条目(includes)重建为 Phase1 1a/1b-pre/1b 需要的 FileEntry 列表。
|
||
# App 条目(incdir 空)不进入 result——Phase B 已编译其 .obj,加入会导致 Phase B+ 重复编译 → 重复符号。
|
||
#
|
||
# Args: mb 内存池;includes_dir includes 根目录(仅用于失败诊断,路径以 store 为准)
|
||
# Returns: ScanResult 指针(stdlib 分配,调用方负责释放);失败返回 None
|
||
# ============================================================
|
||
def _BuildReachableResult(mb: memhub.MemBuddy*,
|
||
includes_dir: str) -> rawptr[IncludesScanner.ScanResult]:
|
||
"""从 Sha1Store(仅可达 includes + App)构建可达 FileEntry 列表"""
|
||
if mb is None:
|
||
return None
|
||
store_count: sizet = sizet(StubMerger._sha1_store_count)
|
||
if store_count <= 0 or StubMerger._sha1_store_arr is None:
|
||
return None
|
||
result: rawptr[IncludesScanner.ScanResult] = IncludesScanner.create_scan_result(mb)
|
||
if result is None:
|
||
return None
|
||
entry_size: sizet = IncludesScanner.FileEntry.__sizeof__()
|
||
i: sizet
|
||
for i in range(store_count):
|
||
sidx: sizet = sizet(i) * 17
|
||
if StubMerger._sha1_store_arr[sidx] == '\0':
|
||
continue
|
||
# 仅 includes 条目(incdir 非空);App 条目不进入 result
|
||
iidx: sizet = sizet(i) * StubMerger.MAX_INC_DIR_LEN
|
||
if StubMerger._sha1_store_incdir[iidx] == '\0':
|
||
continue
|
||
eaddr: uint64 = uint64(result.Entries) + uint64(result.Count) * entry_size
|
||
entry: rawptr[IncludesScanner.FileEntry] = rawptr[IncludesScanner.FileEntry](eaddr)
|
||
if entry is None:
|
||
continue
|
||
# SHA1 深拷贝(store 槽 17 字节)
|
||
sha1_buf: bytes = stdlib.malloc(17)
|
||
if sha1_buf is None:
|
||
continue
|
||
string.strcpy(sha1_buf, StubMerger._sha1_store_arr + sidx)
|
||
entry.Sha1 = sha1_buf
|
||
# RelPath / IncDir 深拷贝
|
||
ridx: sizet = sizet(i) * StubMerger.MAX_REL_PATH_LEN
|
||
rel_len: sizet = string.strlen(StubMerger._sha1_store_rel + ridx)
|
||
inc_len: sizet = string.strlen(StubMerger._sha1_store_incdir + iidx)
|
||
rel_buf: bytes = stdlib.malloc(rel_len + 1)
|
||
inc_buf: bytes = stdlib.malloc(inc_len + 1)
|
||
if rel_buf is None or inc_buf is None:
|
||
if rel_buf is not None:
|
||
stdlib.free(rel_buf)
|
||
if inc_buf is not None:
|
||
stdlib.free(inc_buf)
|
||
stdlib.free(sha1_buf)
|
||
continue
|
||
string.strcpy(rel_buf, StubMerger._sha1_store_rel + ridx)
|
||
string.strcpy(inc_buf, StubMerger._sha1_store_incdir + iidx)
|
||
entry.RelPath = rel_buf
|
||
entry.IncDir = inc_buf
|
||
# 绝对路径 = inc_dir + '/' + rel_path(与 _TryResolveModuleFile 拼接一致)
|
||
abs_buf: bytes = stdlib.malloc(inc_len + 1 + rel_len + 1)
|
||
if abs_buf is None:
|
||
stdlib.free(sha1_buf)
|
||
stdlib.free(rel_buf)
|
||
stdlib.free(inc_buf)
|
||
continue
|
||
string.strcpy(abs_buf, inc_buf)
|
||
abs_buf[inc_len] = '/'
|
||
string.strcpy(abs_buf + inc_len + 1, rel_buf)
|
||
entry.Path = abs_buf
|
||
# 模块名(store 登记时保存的绝对模块名)
|
||
midx: sizet = sizet(i) * 64
|
||
mod_len: sizet = string.strlen(StubMerger._sha1_store_mod + midx)
|
||
if mod_len > 0:
|
||
mod_buf: bytes = stdlib.malloc(mod_len + 1)
|
||
if mod_buf is not None:
|
||
string.strcpy(mod_buf, StubMerger._sha1_store_mod + midx)
|
||
entry.ModuleName = mod_buf
|
||
result.Count += 1
|
||
return result
|
||
|
||
# ============================================================
|
||
# Phase1Main - Phase1 主流程(扫描 → 预注册 → 1a/1b 翻译 → 生成 stub)
|
||
#
|
||
# 函数实现直接位于本模块,调用方经本模块前缀访问;全局符号经 Phase1._xxx 跨模块访问。
|
||
# ============================================================
|
||
|
||
# ============================================================
|
||
# RunPhase1 - Phase1: 扫描 includes 目录,按需翻译并生成 stub
|
||
#
|
||
# 对每个 .vp/.py 文件检查 stub 是否已存在,不存在才翻译并分离 stub(避免重复工作)。
|
||
#
|
||
# Args: mb 内存池;includes_dir includes 目录路径;temp_dir 临时目录(保存 stub);log 日志器
|
||
# Returns: 0 成功,非 0 失败
|
||
# ============================================================
|
||
def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str,
|
||
log: rawptr[VLogger.Logger]) -> int:
|
||
"""Phase1: 扫描 includes 目录,按需翻译并生成 stub"""
|
||
if includes_dir is None or temp_dir is None:
|
||
VLogger.error("includes_dir 或 temp_dir 为空,无法运行 Phase1", "Phase1")
|
||
return 1
|
||
|
||
# 设置全局 temp_dir(供跨模块 CDefine 查找使用)
|
||
GenApi.set_temp_dir(temp_dir)
|
||
# 同步设置 HandlesExprCall 的 temp_dir(供 vpi stub 返回注解缓存按需加载)。
|
||
# 必须在 includes 翻译前设置:Phase1 翻译 includes 时「同模块方法调用(定义在后)」需读 {temp_dir}/{sha1[0:2]}/{sha1}.vpi 兜底,Phase2 的 SetTempDir 太晚。
|
||
GenApi.SetTempDir(temp_dir)
|
||
|
||
# 确保 temp 目录存在(build_dir/temp 可能尚未创建)
|
||
BuildPipeline.ensure_dir(temp_dir)
|
||
|
||
log.banner("Phase1: 扫描 includes(按需翻译)")
|
||
|
||
# 初始化 AST 表(只需一次):必须在依赖图 BFS 扫描之前调用。
|
||
# _init_tables 懒分配的关键字/操作符表(全局 _kw_head 等)若首次触发在 BFS 的文件级 mark 之后,reset_to 会回收表节点 → 全局悬空 → 后续 tokenize 崩溃。
|
||
ast._init_tables(mb)
|
||
|
||
# 依赖图按需扫描(App 优先模型):先扫 App 源文件(Config.SourceDir)的 import 依赖作为初始队列,
|
||
# BFS 按文件系统解析 includes 文件(只对可达者读文件+算 SHA1+解析 import)并登记 Sha1Store/deps/写 deps.txt;
|
||
# 未引用的 includes 从不读取。
|
||
t_scan0: uint64 = w32.win32base.GetTickCount64()
|
||
reachable_set: str = None
|
||
reachable_count: int = 0
|
||
use_reachable: int = 0
|
||
if Config.SourceDir is not None:
|
||
reachable_set = stdlib.malloc(StubMerger.MAX_INCLUDES_SHA1 * 17)
|
||
if reachable_set is not None:
|
||
string.memset(reachable_set, 0, StubMerger.MAX_INCLUDES_SHA1 * 17)
|
||
reachable_count = StubMergerBuild._BuildReachableSha1Set(
|
||
mb, Config.SourceDir, temp_dir, reachable_set, None, includes_dir)
|
||
if reachable_count < 0:
|
||
VLogger.error("可达集合构建失败(依赖图扫描失败),终止编译", "Phase1")
|
||
return 1
|
||
if reachable_count <= 0:
|
||
VLogger.warning("App 无可达 includes 依赖,跳过 Phase1", "Phase1")
|
||
stdlib.free(reachable_set)
|
||
reachable_set = None
|
||
return 0
|
||
use_reachable = 1
|
||
else:
|
||
VLogger.error("reachable_set 分配失败,终止编译", "Phase1")
|
||
return 1
|
||
else:
|
||
VLogger.warning("SourceDir 不可用,无法构建依赖图,跳过 Phase1", "Phase1")
|
||
return 0
|
||
t_scan1: uint64 = w32.win32base.GetTickCount64()
|
||
VLogger.info(f"依赖图扫描总耗时: {t_scan1 - t_scan0} ms(可达 {reachable_count} 个 includes)", "Phase1")
|
||
|
||
# 从 Sha1Store(仅可达 includes + App)构造可达 FileEntry 列表:
|
||
# 后续 1a/1b-pre/1b 直接遍历 result(incdir 非空条目=includes),
|
||
# use_reachable 过滤幂等保留(result 即可达子集)
|
||
result: rawptr[IncludesScanner.ScanResult] = _BuildReachableResult(mb, includes_dir)
|
||
if result is None:
|
||
VLogger.error("可达 FileEntry 列表构建失败,终止编译", "Phase1")
|
||
return 1
|
||
VLogger.info(f"可达 includes 条目: {result.Count}", "Phase1")
|
||
|
||
# 扫描后统一清理 _list/_dict 注入环边(_list↔memhub 等 2-环)。
|
||
# 依赖图按需扫描顺序不可靠,内联检查可能漏判,此处全量清理保证
|
||
# 依赖图无注入环,Phase A 多遍可收敛。破环后同步重写 deps.txt。
|
||
_BreakContainerCyclesInDeps(temp_dir)
|
||
|
||
# 写 _sha1_map.txt(仅可达 includes,人类可读输出;机器分析使用内存存储器)
|
||
StubMergerCollect.WriteIncludesSha1Map(mb, temp_dir, result, reachable_set, reachable_count)
|
||
|
||
# 兼容旧引用:sha1_set/set_count 对齐可达集合(use_reachable=1 时不被使用)
|
||
sha1_set: str = reachable_set
|
||
set_count: int = reachable_count
|
||
# 模块 SHA1 映射已由依赖图扫描登记到 StubMerger 全局存储器(仅可达 includes + App),
|
||
# HandlesExprCall._lookup_module_sha1 直接查全局存储器。
|
||
|
||
# FileEntry 结构体大小(Phase 1a 和 1b 共用)
|
||
entry_size: sizet = IncludesScanner.FileEntry.__sizeof__()
|
||
|
||
|
||
# ============================================================
|
||
# Phase 1a: 注册可达 includes 文件的 struct/enum/union
|
||
#
|
||
# 解决字母序依赖问题:1a 先把所有 struct/enum/union 注册到全局表,1b 全量翻译时 struct 已注册、走 existing 路径只翻译方法体;
|
||
# 只处理可达文件;处理前按类继承关系拓扑排序,确保父类所在文件先于子类所在文件。
|
||
# ============================================================
|
||
p1a_reg: int = 0
|
||
p1a_skp: int = 0
|
||
p1a_fl: int = 0
|
||
|
||
# ============================================================
|
||
# [临时取证打点] Phase1 早期扁平缓冲分配失败现场快照(定位后清理);仅只读 mb 状态 + 打印,不改变任何分配行为与逻辑。
|
||
# 三维度判定:① OOM(dump_stats 显示 used≈total、free≈0);② mb 损坏(usable 基址/大小荒谬、self_check != 0);③ 本次请求 size 算出的 order > max_order。
|
||
# 用 VLogger.info(非终止;VLogger.error 会 sys.exit(1) 掩盖后续打点)。
|
||
# ============================================================
|
||
if mb is not None:
|
||
# 先做带环检测的 self_check:free list 被写坏成环时 dump_stats 的
|
||
# free_bytes/free_count 无步数上限会死循环(打点自身卡死),故
|
||
# self_check 返回 4(成环)时跳过 dump_stats。
|
||
p1_sc: int = mb.self_check()
|
||
if p1_sc != 4:
|
||
mb.dump_stats("phase1-pre")
|
||
else:
|
||
VLogger.info("[取证] self_check=4(free list 成环),跳过 dump_stats 防死循环", "Phase1")
|
||
p1_mo: int = mb.max_order
|
||
if p1_mo < 0 or p1_mo > 32:
|
||
p1_mo = 32 # 钳制:max_order 被写坏时防本打点自身越界读 free_lists
|
||
p1_fl_head: uint64 = uint64(mb.free_lists[p1_mo])
|
||
p1_mb_addr: uint64 = uint64(mb)
|
||
p1_base_addr: uint64 = uint64(mb.base)
|
||
p1_usable_addr: uint64 = uint64(mb.usable)
|
||
p1_chkbase_addr: uint64 = uint64(mb.chk_base)
|
||
VLogger.info(
|
||
f"[取证] mb={p1_mb_addr} base={p1_base_addr} usable={p1_usable_addr} "
|
||
f"usable_size={mb.usable_size} max_order={mb.max_order} self_check={p1_sc} "
|
||
f"detection={mb.detection}", "Phase1")
|
||
VLogger.info(
|
||
f"[取证] chk_base={p1_chkbase_addr} chk_depth={mb.chk_depth} "
|
||
f"chk_cap={mb.chk_cap} free_lists[clamp={p1_mo}]={p1_fl_head}",
|
||
"Phase1")
|
||
p1_sz_cn: sizet = sizet(Phase1.MAX_CLASSES * Phase1.CLASS_NAME_LEN)
|
||
p1_sz_ds: sizet = sizet(Phase1.MAX_CLASSES * Phase1.SHA1_LEN)
|
||
p1_sz_tp: sizet = sizet(4 * Phase1.MAX_FILE_DEPS)
|
||
p1_sz_bx: sizet = sizet(4)
|
||
p1_o_cn: int = mb._order_for_size(p1_sz_cn + 16)
|
||
p1_o_ds: int = mb._order_for_size(p1_sz_ds + 16)
|
||
p1_o_tp: int = mb._order_for_size(p1_sz_tp + 16)
|
||
p1_o_bx: int = mb._order_for_size(p1_sz_bx + 16)
|
||
p1_fl_cn: uint64 = uint64(mb.free_lists[p1_o_cn])
|
||
p1_fl_ds: uint64 = uint64(mb.free_lists[p1_o_ds])
|
||
VLogger.info(
|
||
f"[取证] order(class_names {p1_sz_cn}B+16)={p1_o_cn} fl={p1_fl_cn} "
|
||
f"order(def_sha1s {p1_sz_ds}B+16)={p1_o_ds} fl={p1_fl_ds} "
|
||
f"order(topo {p1_sz_tp}B+16)={p1_o_tp} order(box {p1_sz_bx}B+16)={p1_o_bx} "
|
||
f"(fl==0 只说明该阶空、仍可由更高阶拆分;若 alloc 失败而 dump_stats 的 free>0 才是元数据损坏铁证,"
|
||
f"任一 order>max_order={mb.max_order} 则是 size 异常)",
|
||
"Phase1")
|
||
|
||
# 预扫描类继承关系,构建拓扑顺序(用扁平字节数组,避免结构体 str 指针 NULL 崩溃)
|
||
class_names_buf: bytes = mb.alloc(Phase1.MAX_CLASSES * Phase1.CLASS_NAME_LEN)
|
||
parent_names_buf: bytes = mb.alloc(Phase1.MAX_CLASSES * Phase1.CLASS_NAME_LEN)
|
||
def_sha1s_buf: bytes = mb.alloc(Phase1.MAX_CLASSES * Phase1.SHA1_LEN)
|
||
topo_order: t.CPtr = mb.alloc(4 * Phase1.MAX_FILE_DEPS)
|
||
topo_count_box: t.CPtr = mb.alloc(4)
|
||
generic_file_indices: t.CPtr = mb.alloc(4 * Phase1.MAX_FILE_DEPS)
|
||
generic_file_count_box: t.CPtr = mb.alloc(4)
|
||
if generic_file_indices is not None:
|
||
string.memset(generic_file_indices, 0, 4 * Phase1.MAX_FILE_DEPS)
|
||
if generic_file_count_box is not None:
|
||
string.memset(generic_file_count_box, 0, 4)
|
||
topo_count: int = 0
|
||
|
||
# 三块扁平缓冲是 Phase1Scan._ScanClassInheritance 的契约前置缓冲:该函数入口对 NULL fail-fast(静默跳过整段会退化为字母序并掩盖 OOM 等内存故障),
|
||
# 故分配失败时逐块显式报错终止(VLogger.error 内部 sys.exit(1)),绝不静默回退。
|
||
# 注:topo_order/topo_count_box 仅供 Phase1Topo._TopoSortFiles 使用,有效性由其入口自身校验(NULL 即返回 -1 并终止),不并入此处终止逻辑。
|
||
if class_names_buf is None:
|
||
VLogger.error("类继承预扫描扁平缓冲分配失败:class_names_buf 为 NULL,终止编译", "Phase1")
|
||
if parent_names_buf is None:
|
||
VLogger.error("类继承预扫描扁平缓冲分配失败:parent_names_buf 为 NULL,终止编译", "Phase1")
|
||
if def_sha1s_buf is None:
|
||
VLogger.error("类继承预扫描扁平缓冲分配失败:def_sha1s_buf 为 NULL,终止编译", "Phase1")
|
||
|
||
string.memset(class_names_buf, 0, Phase1.MAX_CLASSES * Phase1.CLASS_NAME_LEN)
|
||
string.memset(parent_names_buf, 0, Phase1.MAX_CLASSES * Phase1.CLASS_NAME_LEN)
|
||
string.memset(def_sha1s_buf, 0, Phase1.MAX_CLASSES * Phase1.SHA1_LEN)
|
||
if topo_order is not None:
|
||
string.memset(topo_order, 0, 4 * Phase1.MAX_FILE_DEPS)
|
||
if topo_count_box is not None:
|
||
string.memset(topo_count_box, 0, 4)
|
||
if generic_file_indices is not None:
|
||
string.memset(generic_file_indices, 0, 4 * Phase1.MAX_FILE_DEPS)
|
||
if generic_file_count_box is not None:
|
||
string.memset(generic_file_count_box, 0, 4)
|
||
|
||
class_count_scanned: int = Phase1Scan._ScanClassInheritance(
|
||
mb, result, reachable_set, reachable_count,
|
||
use_reachable, sha1_set, set_count,
|
||
class_names_buf, parent_names_buf, def_sha1s_buf,
|
||
generic_file_indices, generic_file_count_box)
|
||
|
||
if class_count_scanned > 0:
|
||
td_len_ts_call: sizet = string.strlen(temp_dir)
|
||
ret_topo: int = Phase1Topo._TopoSortFiles(
|
||
mb, result, reachable_set, reachable_count,
|
||
use_reachable, sha1_set, set_count,
|
||
class_names_buf, parent_names_buf, def_sha1s_buf, class_count_scanned,
|
||
topo_order, topo_count_box,
|
||
temp_dir, td_len_ts_call)
|
||
if ret_topo == 0:
|
||
tc_ptr: t.CPtr = (int | t.CPtr)(topo_count_box)
|
||
if tc_ptr is not None:
|
||
topo_count = tc_ptr[0]
|
||
VLogger.info(f"拓扑排序: {class_count_scanned} 个类, {topo_count} 个文件按依赖顺序处理", "Phase1")
|
||
else:
|
||
VLogger.error("拓扑排序失败(类继承依赖无法解析),终止编译", "Phase1")
|
||
return 1
|
||
else:
|
||
VLogger.warning("类继承预扫描无结果,回退到字母序", "Phase1")
|
||
topo_count = 0
|
||
|
||
# ============================================================
|
||
# Phase 1a-pre-generic-all: mark1 前全量预注册 includes 泛型模板
|
||
#
|
||
# 非可达泛型文件(_list/_dict/linkedlist/llvmlite 等)的模板若在 Phase2(mark2 之后)才注册会被 reset 回收 → list[int]/dict[...] 特化失败。
|
||
# 故对全部 includes 轻量预注册,使模板表与节点都落在 mark1 之前(_PreRegisterAllGenericTemplates 内有 generic marker 文本预筛,不 parse 全部 AST,OOM 风险可控)。
|
||
full_scan_g: rawptr[IncludesScanner.ScanResult] = IncludesScanner.scan_includes(mb, includes_dir)
|
||
if full_scan_g is not None:
|
||
Phase1Scan._PreRegisterAllGenericTemplates(mb, full_scan_g)
|
||
else:
|
||
Phase1Scan._PreRegisterAllGenericTemplates(mb, result)
|
||
|
||
# ============================================================
|
||
# Phase 1a: 注册 struct/enum/union(mark1 前,全量 includes)
|
||
#
|
||
# OOM 治理:1a 对全部 includes parse 完整 AST,若驻留到 mark1 会累积至 1GB 主池耗尽。故:
|
||
# 1) StdlibAlloc=1:1a 注册的跨文件持久数据(StructTable 表、Fields/Name/ModuleSha1、struct_ty 深拷贝、枚举/Renum/泛型模板等)全部落 perm_pool(永不 reset);
|
||
# 2) 每文件翻译完 reset_to(文件级 mark):只回收该文件的 AST/lexer/token/IR 等瞬时数据,注册数据不受影响。
|
||
# mark1 仍在 1a 结束后打,1b 每文件 reset_to(mark1) 语义不变。
|
||
# ============================================================
|
||
GenApi.set_stdlib_alloc(1)
|
||
|
||
# 遍历文件:优先使用拓扑顺序,回退到字母序
|
||
iter_count: int = topo_count if topo_count > 0 else result.Count
|
||
# 1a 阶段总耗时起点(struct/enum/union 注册)
|
||
t_a0: uint64 = w32.win32base.GetTickCount64()
|
||
for i in range(iter_count):
|
||
# 每文件级 mark:翻译完回收该文件 AST(注册数据在 perm_pool)
|
||
p1a_mark: sizet = mb.mark()
|
||
# 获取文件索引:拓扑顺序或字母序
|
||
file_idx: int = i
|
||
if topo_count > 0:
|
||
od_addr: uint64 = uint64(topo_order) + i * 4
|
||
od_ptr: t.CPtr = (int | t.CPtr)(t.CPtr(od_addr))
|
||
if od_ptr is not None:
|
||
file_idx = od_ptr[0]
|
||
|
||
entry_addr_r: uint64 = uint64(result.Entries) + file_idx * entry_size
|
||
entry_r: rawptr[IncludesScanner.FileEntry] = rawptr[IncludesScanner.FileEntry](entry_addr_r)
|
||
if entry_r is None:
|
||
p1a_fl += 1
|
||
continue
|
||
|
||
sha1_r: str = entry_r.Sha1
|
||
if sha1_r is None:
|
||
p1a_fl += 1
|
||
continue
|
||
|
||
# 按需翻译过滤(拓扑排序已过滤,但字母序回退需要检查)
|
||
if topo_count == 0:
|
||
if use_reachable != 0:
|
||
if StubMergerCollect._is_in_sha1_set(sha1_r, reachable_set, reachable_count) == 0:
|
||
p1a_skp += 1
|
||
continue
|
||
else:
|
||
if StubMergerCollect._is_in_sha1_set(sha1_r, sha1_set, set_count) == 0:
|
||
p1a_skp += 1
|
||
continue
|
||
|
||
# 单文件计时起点(读文件 + parse 段,定位 1a 卡点)
|
||
t_fa0: uint64 = w32.win32base.GetTickCount64()
|
||
|
||
# 读取文件内容
|
||
file_path_r: str = entry_r.Path
|
||
f_r: rawptr[fileio.File] = fileio.File(file_path_r, fileio.MODE.R)
|
||
if f_r.closed:
|
||
p1a_fl += 1
|
||
continue
|
||
|
||
# read_all 的容量参数含终止符(最多读 cap-1 字节), 文件更长会被静默截断
|
||
if f_r.size() >= Phase1.SRC_BUF_SIZE:
|
||
VLogger.error(f"源文件超过 {Phase1.SRC_BUF_SIZE} 字节上限, 拒绝读取: {file_path_r}", "Phase1")
|
||
f_r.close()
|
||
p1a_fl += 1
|
||
continue
|
||
|
||
src_buf_r: bytes = stdlib.malloc(Phase1.SRC_BUF_SIZE)
|
||
if src_buf_r is None:
|
||
f_r.close()
|
||
p1a_fl += 1
|
||
continue
|
||
|
||
bytes_read_r: LONG = f_r.read_all(src_buf_r, Phase1.SRC_BUF_SIZE)
|
||
f_r.close()
|
||
if bytes_read_r <= 0:
|
||
stdlib.free(src_buf_r)
|
||
p1a_fl += 1
|
||
continue
|
||
if bytes_read_r < Phase1.SRC_BUF_SIZE:
|
||
src_buf_r[bytes_read_r] = 0
|
||
|
||
# 解析 AST
|
||
lx_r: ast.Lexer* = ast.new_lexer(mb)
|
||
if lx_r is None:
|
||
stdlib.free(src_buf_r)
|
||
p1a_fl += 1
|
||
continue
|
||
ast._lexer_init(lx_r, src_buf_r, mb)
|
||
tokens_r: ast.Token* = ast.tokenize(lx_r)
|
||
tree_r: ast.AST* = ast.parse_tokens(mb, tokens_r)
|
||
if tree_r is None:
|
||
stdlib.free(src_buf_r)
|
||
p1a_fl += 1
|
||
continue
|
||
|
||
# 计时点 1:读文件 + parse 完成
|
||
t_fa1: uint64 = w32.win32base.GetTickCount64()
|
||
|
||
# 创建 Translator,设置 _declare_only=1(注册 struct/enum/union)
|
||
tr_r: rawptr[HandlesTranslator.Translator] = HandlesTranslator.Translator()
|
||
if tr_r is None:
|
||
stdlib.free(src_buf_r)
|
||
p1a_fl += 1
|
||
continue
|
||
tr_r.ModuleSha1 = sha1_r
|
||
tr_r._declare_only = 1
|
||
tr_r.CurrentPackage = GenApi.compute_package_from_relpath(mb, entry_r.RelPath)
|
||
GenApi.set_current_file_type(file_path_r)
|
||
GenApi.set_current_file_imports(file_path_r)
|
||
GenApi.set_current_module_sha1(sha1_r)
|
||
GenApi.clear_cdefine_constants()
|
||
GenApi.clear_ctypedefs()
|
||
GenApi.reset_visible_structs(mb, 0)
|
||
ret_r: int = tr_r.translate(tree_r)
|
||
# 计时点 2:translate(declare=1) 返回
|
||
t_fa2: uint64 = w32.win32base.GetTickCount64()
|
||
if ret_r != 0:
|
||
p1a_fl += 1
|
||
else:
|
||
p1a_reg += 1
|
||
|
||
# 逐文件进度日志(parse/translate 两段,定位 1a 卡点)
|
||
VLogger.info(f"[{i+1}/{iter_count}] 1a: {file_path_r} (parse={t_fa1-t_fa0} ms, translate={t_fa2-t_fa1} ms, total={t_fa2-t_fa0} ms)", "Phase1")
|
||
|
||
# 释放 Translator 的 C malloc 资源
|
||
if tr_r._global_names is not None:
|
||
stdlib.free(tr_r._global_names)
|
||
if tr_r._nonlocal_names is not None:
|
||
stdlib.free(tr_r._nonlocal_names)
|
||
stdlib.free(src_buf_r)
|
||
|
||
# [TEMP-DIAG] 1a 每文件哨兵:校验本文件窗口 [p1a_mark, chk_depth) 内所有存活块的块尾 canary。
|
||
# 位置必须在 reset_to 之前:reset_to 会把 chk_depth 截断回 p1a_mark,之后再查同一 mark 时 check_alive_blocks 直接 return 0(窗口为空、必然漏报)。
|
||
# p1a_mark 在循环体首行捕获(本文件 L597),故窗口恰为「本文件 parse + translate(declare=1)」期间分配的块。
|
||
# 仅 --detection-deep(mb.deep_detect != 0)时执行(每文件 O(n) 遍历活块);读法同本文件既有 mb.detection:直接读 MemBuddy 实例的公开 int 字段。
|
||
if mb.deep_detect != 0:
|
||
chk_rc_1a: int = mb.check_alive_blocks(p1a_mark)
|
||
if chk_rc_1a != 0:
|
||
VLogger.error(f"内存损坏(1a 每文件后续, rc={chk_rc_1a}): {file_path_r}", "Phase1")
|
||
stdlib.exit(1)
|
||
|
||
# 回收本文件 AST/lexer/token/IR 等瞬时数据(注册数据已在 perm_pool,
|
||
# reset 不影响;IncludesScanner result 等在 p1a_mark 前分配同样保留)
|
||
mb.reset_to(p1a_mark)
|
||
|
||
# 1a 阶段总耗时(struct/enum/union 注册,含每文件读+parse+翻译声明)
|
||
t_a1: uint64 = w32.win32base.GetTickCount64()
|
||
VLogger.info(f"1a 总耗时: {t_a1-t_a0} ms (注册 struct/enum, reg={p1a_reg} skp={p1a_skp})", "Phase1")
|
||
|
||
if p1a_fl > 0:
|
||
VLogger.error(f"Phase 1a 有 {p1a_fl} 个文件 struct 注册失败", "Phase1")
|
||
return 1
|
||
|
||
# 释放拓扑排序资源(topo_order/topo_count_box 保留到 Phase 1b 结束,
|
||
# 1b 按拓扑序翻译需要它们;class/def 数组 1b 不再使用)
|
||
if class_names_buf is not None:
|
||
mb.free(class_names_buf)
|
||
if parent_names_buf is not None:
|
||
mb.free(parent_names_buf)
|
||
if def_sha1s_buf is not None:
|
||
mb.free(def_sha1s_buf)
|
||
if generic_file_indices is not None:
|
||
mb.free(generic_file_indices)
|
||
if generic_file_count_box is not None:
|
||
mb.free(generic_file_count_box)
|
||
|
||
# ============================================================
|
||
# Phase 1b: 全量翻译(struct 已注册,走 existing 路径翻译方法体)
|
||
# ============================================================
|
||
# OOM 回收:mark1 在 1a-pre/泛型预注册/1a 全部结束后打,所有跨文件持久表(_struct_table/_enum_table/_generic_class_nodes/_renum_*/关键字表等)都落在 mark1 之前;
|
||
# 1b 每文件翻译完后 reset_to(mark1) 回收该文件的 AST/IR/Handler/dump 临时,避免 90 文件累积导致 1GB 池 OOM。
|
||
mark1: sizet = mb.mark()
|
||
|
||
# 1b 阶段新注册的 struct/enum 条目改 stdlib 分配(跨文件永久存活,不被 reset 回收)
|
||
GenApi.set_stdlib_alloc(1)
|
||
|
||
# ============================================================
|
||
# Phase 1b-pre: 预生成所有待翻译文件的 .vpi stub
|
||
#
|
||
# 依赖模块的 .vpi 必须在翻译前可用:1b 主循环按扫描顺序(非拓扑)翻译,若依赖模块后翻译则 LoadDepVpiIntoModule 读不到 .vpi
|
||
# → declare 未注入 → 跨模块调用的返回类型推断错误(如误推为 Ptr 而 define 返回 i8*)→ 运行时返回值截断崩溃。
|
||
# ============================================================
|
||
p1bpre_failed: int = 0
|
||
dir_len_pv: sizet = string.strlen(temp_dir)
|
||
# 1b-pre 阶段总耗时起点(vpi 预生成)
|
||
t_pv0: uint64 = w32.win32base.GetTickCount64()
|
||
for i_pv in range(result.Count):
|
||
entry_addr_pv: uint64 = uint64(result.Entries) + i_pv * entry_size
|
||
entry_pv: rawptr[IncludesScanner.FileEntry] = rawptr[IncludesScanner.FileEntry](entry_addr_pv)
|
||
if entry_pv is None or entry_pv.Sha1 is None:
|
||
p1bpre_failed += 1
|
||
continue
|
||
sha1_pv: str = entry_pv.Sha1
|
||
in_set_pv: int = 0
|
||
if use_reachable != 0:
|
||
in_set_pv = StubMergerCollect._is_in_sha1_set(sha1_pv, reachable_set, reachable_count)
|
||
else:
|
||
in_set_pv = StubMergerCollect._is_in_sha1_set(sha1_pv, sha1_set, set_count)
|
||
if in_set_pv == 0:
|
||
continue
|
||
# vpi 已存在则跳过
|
||
vpi_path_pv: str = StubMergerCollect._sliced_path(temp_dir, dir_len_pv, sha1_pv, "vpi")
|
||
if vpi_path_pv is None:
|
||
p1bpre_failed += 1
|
||
VLogger.error(f"[Phase 1b-pre] {entry_pv.Path} | vpi 路径生成失败", "Phase1")
|
||
continue
|
||
vf_pv: rawptr[fileio.File] = fileio.File(vpi_path_pv, fileio.MODE.R)
|
||
if not vf_pv.closed:
|
||
vf_pv.close()
|
||
stdlib.free(vpi_path_pv)
|
||
continue
|
||
stdlib.free(vpi_path_pv)
|
||
# 读取文件、解析 AST、生成 vpi
|
||
file_path_pv: str = entry_pv.Path
|
||
f_pv: rawptr[fileio.File] = fileio.File(file_path_pv, fileio.MODE.R)
|
||
if f_pv.closed:
|
||
p1bpre_failed += 1
|
||
VLogger.error(f"[Phase 1b-pre] {file_path_pv} | 无法打开源文件", "Phase1")
|
||
continue
|
||
# read_all 的容量参数含终止符(最多读 cap-1 字节), 文件更长会被静默截断
|
||
if f_pv.size() >= Phase1.SRC_BUF_SIZE:
|
||
p1bpre_failed += 1
|
||
VLogger.error(f"[Phase 1b-pre] {file_path_pv} | 源文件超过 {Phase1.SRC_BUF_SIZE} 字节上限", "Phase1")
|
||
f_pv.close()
|
||
continue
|
||
src_buf_pv: bytes = stdlib.malloc(Phase1.SRC_BUF_SIZE)
|
||
if src_buf_pv is None:
|
||
f_pv.close()
|
||
p1bpre_failed += 1
|
||
VLogger.error(f"[Phase 1b-pre] {file_path_pv} | 源文件缓冲分配失败(OOM)", "Phase1")
|
||
continue
|
||
bytes_read_pv: LONG = f_pv.read_all(src_buf_pv, Phase1.SRC_BUF_SIZE)
|
||
f_pv.close()
|
||
if bytes_read_pv <= 0:
|
||
stdlib.free(src_buf_pv)
|
||
p1bpre_failed += 1
|
||
VLogger.error(f"[Phase 1b-pre] {file_path_pv} | 文件读取失败或为空", "Phase1")
|
||
continue
|
||
if bytes_read_pv < Phase1.SRC_BUF_SIZE:
|
||
src_buf_pv[bytes_read_pv] = 0
|
||
lx_pv: ast.Lexer* = ast.new_lexer(mb)
|
||
if lx_pv is None:
|
||
stdlib.free(src_buf_pv)
|
||
p1bpre_failed += 1
|
||
VLogger.error(f"[Phase 1b-pre] {file_path_pv} | 词法器分配失败(OOM)", "Phase1")
|
||
continue
|
||
ast._lexer_init(lx_pv, src_buf_pv, mb)
|
||
tokens_pv: ast.Token* = ast.tokenize(lx_pv)
|
||
tree_pv: ast.AST* = ast.parse_tokens(mb, tokens_pv)
|
||
if tree_pv is None:
|
||
stdlib.free(src_buf_pv)
|
||
p1bpre_failed += 1
|
||
# 解析失败:**日志行内联 行/列 + 原因**,并附完整 Python 风格 traceback
|
||
# (Traceback + File/行 + 源码行 + caret + SyntaxError: 消息;消息含 token
|
||
# 类型与前/后 token 附注,见 includes/ast/parser.vp 的 _record_parse_err_ctx
|
||
# / parse_tokens 兜底补记),与单文件管线 BuildPipeline 同一条上报路径。
|
||
VLogger.error(f"[Phase 1b-pre] {file_path_pv} | AST 解析失败(第 {ast.get_parse_err_line()} 行第 {ast.get_parse_err_col()} 列: {ast.get_parse_err_msg()})", "Phase1")
|
||
GenApi.dump_parse_traceback(file_path_pv)
|
||
continue
|
||
pyi_buf_pv: bytes = stdlib.malloc(Phase1.PYI_BUF_SIZE)
|
||
if pyi_buf_pv is not None:
|
||
pyi_pos_pv: sizet = StubConverter._GeneratePyiFromAst(mb, tree_pv, entry_pv.RelPath, pyi_buf_pv, Phase1.PYI_BUF_SIZE)
|
||
if pyi_pos_pv > 0:
|
||
pyi_path_pv: str = StubMergerCollect._sliced_path(temp_dir, dir_len_pv, sha1_pv, "vpi")
|
||
if pyi_path_pv is not None:
|
||
pf_pv: rawptr[fileio.File] = fileio.File(pyi_path_pv, fileio.MODE.W)
|
||
if not pf_pv.closed:
|
||
pf_pv.write(pyi_buf_pv, pyi_pos_pv)
|
||
pf_pv.close()
|
||
stdlib.free(pyi_path_pv)
|
||
stdlib.free(pyi_buf_pv)
|
||
else:
|
||
p1bpre_failed += 1
|
||
VLogger.error(f"[Phase 1b-pre] {file_path_pv} | pyi 缓冲分配失败(OOM)", "Phase1")
|
||
stdlib.free(src_buf_pv)
|
||
# [TEMP-DIAG] 1b-pre 每文件哨兵(同上):校验本文件窗口 [mark1, chk_depth) 内所有存活块的块尾 canary。
|
||
# 同样必须在 reset_to(mark1) 之前(否则窗口被截断为空)。mark1 在 1b 段前捕获(本文件 L756),是 1b/1b-pre 的统一窗口起点。
|
||
# 仅 --detection-deep(mb.deep_detect != 0)时执行,读法同上方 1a 哨兵。
|
||
if mb.deep_detect != 0:
|
||
chk_rc_pv: int = mb.check_alive_blocks(mark1)
|
||
if chk_rc_pv != 0:
|
||
VLogger.error(f"内存损坏(1b-pre 每文件后续, rc={chk_rc_pv}): {file_path_pv}", "Phase1")
|
||
stdlib.exit(1)
|
||
|
||
# 回收本文件 AST(vpi 已写盘,AST 不再需要;mark1 后无其他持久数据)
|
||
mb.reset_to(mark1)
|
||
|
||
# 1b-pre 阶段总耗时(vpi 预生成)
|
||
t_pv1: uint64 = w32.win32base.GetTickCount64()
|
||
VLogger.info(f"1b-pre 总耗时: {t_pv1-t_pv0} ms (vpi 预生成)", "Phase1")
|
||
|
||
if p1bpre_failed > 0:
|
||
VLogger.error(f"Phase 1b-pre 有 {p1bpre_failed} 个文件 vpi 预生成失败(失败文件与原因见上方逐条 [Phase 1b-pre] 日志)", "Phase1")
|
||
return 1
|
||
|
||
# 遍历每个文件(拓扑序优先:父类所在文件先翻译,确保跨模块 vtable
|
||
# 生成时依赖模块的函数 TypeStr 已注册;topo_count==0 回退扫描顺序)
|
||
translated: int = 0
|
||
skipped: int = 0
|
||
failed: int = 0
|
||
# 1b 阶段总耗时起点(全量翻译)
|
||
t_b0: uint64 = w32.win32base.GetTickCount64()
|
||
|
||
loop_n_b: int = result.Count
|
||
if topo_count > 0:
|
||
loop_n_b = topo_count
|
||
for i in range(loop_n_b):
|
||
# 获取 entry(拓扑序:file_idx 从 topo_order 取,否则用扫描索引)
|
||
file_idx_b: int = i
|
||
if topo_count > 0:
|
||
od_addr_b: uint64 = uint64(topo_order) + sizet(i) * 4
|
||
od_ptr_b: t.CPtr = (int | t.CPtr)(od_addr_b)
|
||
if od_ptr_b is not None:
|
||
file_idx_b = od_ptr_b[0]
|
||
entry_addr: uint64 = uint64(result.Entries) + sizet(file_idx_b) * entry_size
|
||
entry: rawptr[IncludesScanner.FileEntry] = rawptr[IncludesScanner.FileEntry](entry_addr)
|
||
if entry is None:
|
||
failed += 1
|
||
continue
|
||
|
||
sha1: str = entry.Sha1
|
||
if sha1 is None:
|
||
failed += 1
|
||
continue
|
||
|
||
# 检查 SHA1 是否在翻译集合中(拓扑序已过滤可达;仅回退扫描顺序时检查)
|
||
in_set: int = 1
|
||
if topo_count == 0:
|
||
in_set = 0
|
||
if use_reachable != 0:
|
||
in_set = StubMergerCollect._is_in_sha1_set(sha1, reachable_set, reachable_count)
|
||
else:
|
||
in_set = StubMergerCollect._is_in_sha1_set(sha1, sha1_set, set_count)
|
||
if in_set == 0:
|
||
skipped += 1
|
||
continue
|
||
|
||
# 构造 stub 路径: {temp_dir}/{sha1}.stub.ll
|
||
dir_len: sizet = string.strlen(temp_dir)
|
||
sha1_len: sizet = string.strlen(sha1)
|
||
stub_path: str = StubMergerCollect._sliced_path(temp_dir, dir_len, sha1, "stub.ll")
|
||
if stub_path is None:
|
||
failed += 1
|
||
continue
|
||
|
||
# 检查 stub 是否已存在(按需翻译:跳过已存在的)
|
||
# 例外:含 class 定义的模块必须翻译——类注册只在翻译时发生,增量编译跳过会导致跨模块类构造 find_struct_by_module 落空 → 返回类型 fail-fast。
|
||
# 判据:扫描 1b-pre 生成的 vpi 是否有 "class " 行(vpi 为解析后存根,精确无注释/字符串误判)。
|
||
force_translate: int = 0
|
||
vpi_path_1b: str = StubMergerCollect._sliced_path(temp_dir, dir_len, sha1, "vpi")
|
||
if vpi_path_1b is not None:
|
||
vf_1b: rawptr[fileio.File] = fileio.File(vpi_path_1b, fileio.MODE.R)
|
||
if not vf_1b.closed:
|
||
vbuf_1b: bytes = stdlib.malloc(1048576)
|
||
if vbuf_1b is not None:
|
||
vrd_1b: LONG = vf_1b.read_all(vbuf_1b, 1048576)
|
||
vf_1b.close()
|
||
if vrd_1b > 0:
|
||
if vrd_1b < 1048576:
|
||
vbuf_1b[vrd_1b] = 0
|
||
vtotal_1b: sizet = string.strlen(vbuf_1b)
|
||
vpos_1b: sizet = 0
|
||
while vpos_1b < vtotal_1b:
|
||
vls_1b: sizet = vpos_1b
|
||
while vpos_1b < vtotal_1b and vbuf_1b[vpos_1b] != '\n':
|
||
vpos_1b += 1
|
||
vll_1b: sizet = vpos_1b - vls_1b
|
||
if vpos_1b < vtotal_1b:
|
||
vpos_1b += 1
|
||
vi2_1b: sizet = vls_1b
|
||
while vi2_1b < vls_1b + vll_1b and vbuf_1b[vi2_1b] == ' ':
|
||
vi2_1b += 1
|
||
if vll_1b >= 6 and string.strncmp(vbuf_1b + vi2_1b, "class ", 6) == 0:
|
||
force_translate = 1
|
||
break
|
||
stdlib.free(vbuf_1b)
|
||
else:
|
||
vf_1b.close()
|
||
stdlib.free(vpi_path_1b)
|
||
if force_translate == 1:
|
||
# 含 class:强制翻译(覆盖 stub/text,重新注册类)
|
||
rp: str = entry.RelPath
|
||
VLogger.info(f"含 class 定义,强制翻译: {rp}(sha1={sha1})", "Phase1")
|
||
else:
|
||
# 检查 stub 是否已存在(按需翻译:跳过已存在的)
|
||
sf: rawptr[fileio.File] = fileio.File(stub_path, fileio.MODE.R)
|
||
if not sf.closed:
|
||
sf.close()
|
||
# 检查 text.ll 是否也存在(虚表扫描需要 text.ll)
|
||
text_path: str = StubMergerCollect._sliced_path(temp_dir, dir_len, sha1, "text.ll")
|
||
if text_path is not None:
|
||
tf: rawptr[fileio.File] = fileio.File(text_path, fileio.MODE.R)
|
||
if not tf.closed:
|
||
tf.close()
|
||
stdlib.free(text_path)
|
||
stdlib.free(stub_path)
|
||
continue
|
||
stdlib.free(text_path)
|
||
# text.ll 不存在,需要重新翻译
|
||
rp: str = entry.RelPath
|
||
VLogger.info(f"text.ll 不存在,重新翻译: {rp}(sha1={sha1})", "Phase1")
|
||
else:
|
||
# stub 不存在,需要翻译
|
||
rp: str = entry.RelPath
|
||
VLogger.info(f"翻译: {rp}(sha1={sha1})", "Phase1")
|
||
|
||
stdlib.free(stub_path)
|
||
|
||
# 单文件计时起点:读文件(parse/translate/dump 三段,对齐 Phase2 插桩)
|
||
t_f0: uint64 = w32.win32base.GetTickCount64()
|
||
|
||
# 读取文件内容
|
||
file_path: str = entry.Path
|
||
f: rawptr[fileio.File] = fileio.File(file_path, fileio.MODE.R)
|
||
if f is None:
|
||
VLogger.error(f"无法打开(f=None): {file_path}", "Phase1")
|
||
failed += 1
|
||
continue
|
||
if f.closed:
|
||
VLogger.error(f"无法打开: {file_path}", "Phase1")
|
||
failed += 1
|
||
continue
|
||
|
||
# read_all 的容量参数含终止符(最多读 cap-1 字节), 文件更长会被静默截断
|
||
if f.size() >= Phase1.SRC_BUF_SIZE:
|
||
VLogger.error(f"源文件超过 {Phase1.SRC_BUF_SIZE} 字节上限, 拒绝读取: {file_path}", "Phase1")
|
||
f.close()
|
||
failed += 1
|
||
continue
|
||
|
||
src_buf: bytes = stdlib.malloc(Phase1.SRC_BUF_SIZE)
|
||
if src_buf is None:
|
||
f.close()
|
||
failed += 1
|
||
continue
|
||
|
||
bytes_read: LONG = f.read_all(src_buf, Phase1.SRC_BUF_SIZE)
|
||
f.close()
|
||
if bytes_read <= 0:
|
||
VLogger.error(f"读取失败: {file_path}", "Phase1")
|
||
stdlib.free(src_buf)
|
||
failed += 1
|
||
continue
|
||
if bytes_read < Phase1.SRC_BUF_SIZE:
|
||
src_buf[bytes_read] = 0
|
||
|
||
# 解析 AST
|
||
lx: ast.Lexer* = ast.new_lexer(mb)
|
||
if lx is None:
|
||
stdlib.free(src_buf)
|
||
failed += 1
|
||
continue
|
||
ast._lexer_init(lx, src_buf, mb)
|
||
tokens: ast.Token* = ast.tokenize(lx)
|
||
tree: ast.AST* = ast.parse_tokens(mb, tokens)
|
||
if tree is None:
|
||
# 解析失败:行/列 + 原因一并进日志行(原因由 parser.vp 的 _record_* /
|
||
# parse_tokens 兜底补记提供)
|
||
VLogger.error(f"AST 解析失败: {file_path}(第 {ast.get_parse_err_line()} 行第 {ast.get_parse_err_col()} 列: {ast.get_parse_err_msg()})", "Phase1")
|
||
stdlib.free(src_buf)
|
||
failed += 1
|
||
continue
|
||
|
||
# 计时点 1:读文件 + lexer/tokenize/parse 完成
|
||
t_f1: uint64 = w32.win32base.GetTickCount64()
|
||
|
||
# 文本扫描注册 CTypedef:AST 字符串字段(attr/id)token 零拷贝指向 src_buf,跨阶段内容不稳定(悬空),读 attr 会 0xC0000005;
|
||
# 故改用源码文本逐行匹配 "NAME: t.CTypedef = TYPE",在 src_buf 有效期内注册。
|
||
GenApi.RegisterCTypedefFromSource(mb, src_buf)
|
||
|
||
# 生成 .vpi 存根文件(直接遍历 AST,不依赖 PythonToStubConverter)
|
||
# 1b-pre 已为所有待翻译文件生成 vpi,此处仅兜底:vpi 缺失才生成
|
||
# (消除 1b-pre 与 1b 对同一文件的重复 _GeneratePyiFromAst 工作)
|
||
vpi_path_1b: str = StubMergerCollect._sliced_path(temp_dir, dir_len, sha1, "vpi")
|
||
if vpi_path_1b is not None:
|
||
vf_1b: rawptr[fileio.File] = fileio.File(vpi_path_1b, fileio.MODE.R)
|
||
vpi_exists_1b: int = 0
|
||
if not vf_1b.closed:
|
||
vf_1b.close()
|
||
vpi_exists_1b = 1
|
||
if vpi_exists_1b == 0:
|
||
pyi_buf: bytes = stdlib.malloc(Phase1.PYI_BUF_SIZE)
|
||
if pyi_buf is not None:
|
||
pyi_pos: sizet = StubConverter._GeneratePyiFromAst(
|
||
mb, tree, entry.Path, pyi_buf, Phase1.PYI_BUF_SIZE)
|
||
if pyi_pos > 0:
|
||
pf_1b: rawptr[fileio.File] = fileio.File(vpi_path_1b, fileio.MODE.W)
|
||
if not pf_1b.closed:
|
||
pf_1b.write(pyi_buf, pyi_pos)
|
||
pf_1b.close()
|
||
stdlib.free(pyi_buf)
|
||
stdlib.free(vpi_path_1b)
|
||
|
||
# 翻译 AST → LLVM IR
|
||
tr: rawptr[HandlesTranslator.Translator] = HandlesTranslator.Translator()
|
||
if tr is None:
|
||
stdlib.free(src_buf)
|
||
failed += 1
|
||
continue
|
||
tr.ModuleSha1 = sha1
|
||
tr.CurrentPackage = GenApi.compute_package_from_relpath(mb, entry.RelPath)
|
||
GenApi.set_current_file_type(file_path)
|
||
# --traceback: mirror the DWARF switch into the Gen-layer emitter (the
|
||
# same BuildPipeline._traceback_mode that gates -frame-pointer=all /
|
||
# dbghelp / the Install() ctor). include/*.vp modules are compiled here,
|
||
# so this is what gives their frames DWARF line info.
|
||
DebugInfoGen.SetEnabled(BuildPipeline._traceback_mode)
|
||
GenApi.set_current_file_imports(file_path)
|
||
GenApi.set_current_module_sha1(sha1)
|
||
GenApi.clear_cdefine_constants()
|
||
GenApi.clear_ctypedefs()
|
||
GenApi.reset_visible_structs(mb, 0)
|
||
|
||
# 翻译前哨兵:AST 解析(lexer/tokenize/parse_tokens)同样在 pool 分配,
|
||
# 若解析器越界写破坏了存活块 header,此处立即定位到当前文件
|
||
# (崩溃点漂移排查:把"翻译过程中随机崩"变成确定性定位)
|
||
pre_rc: int = mb.check_alive_blocks(mark1)
|
||
if pre_rc != 0:
|
||
VLogger.error(f"内存损坏(翻译前存活块, rc={pre_rc}): {file_path}", "Phase1")
|
||
stdlib.exit(1)
|
||
|
||
ret: int = tr.translate(tree)
|
||
# 计时点 2:translate 返回
|
||
t_f2: uint64 = w32.win32base.GetTickCount64()
|
||
# 诊断:translate 正常返回(若挂起在 translate 内部,此行不会打印,
|
||
# 用于区分"translate 内部挂起"与"后续 dump 挂起")
|
||
if ret == 0:
|
||
VLogger.info(f"翻译完成(translate返回): {file_path}", "Phase1")
|
||
if ret != 0:
|
||
VLogger.error(f"翻译失败: {file_path}", "Phase1") # error 终止
|
||
stdlib.free(src_buf)
|
||
if tr._global_names is not None:
|
||
stdlib.free(tr._global_names)
|
||
if tr._nonlocal_names is not None:
|
||
stdlib.free(tr._nonlocal_names)
|
||
failed += 1
|
||
continue
|
||
|
||
# --traceback: inject the startup Install() ctor when translating traceback.vp
|
||
# (its stub/text.ll below are what Phase2 reuses to build the .obj). The
|
||
# injector lives in the Gen layer (already depends on llvmlite), so this
|
||
# module gains no cross-module llvmlite symbol dependency.
|
||
if BuildPipeline._traceback_mode != 0 and BuildPipeline._tb_basename_is_traceback(file_path) != 0:
|
||
HandlesTranslator.InjectTracebackInstallCtor(tr)
|
||
|
||
# 诊断:pre_rc 哨兵在 translate 前、覆盖不到 translate 期间的 AST/IR 写穿,故 dump 前立即检查存活块 header/canary
|
||
# (rc=6 块尾 canary 损坏 = 越界写穿数据区),把「dump 时链表成环卡死」变成「定位到 translate 期间写穿发生的文件」。
|
||
post_rc: int = mb.check_alive_blocks(mark1)
|
||
if post_rc != 0:
|
||
VLogger.error(f"内存损坏(translate后存活块, rc={post_rc}): {file_path}", "Phase1")
|
||
|
||
# dump stub IR(declarations only)
|
||
# 1MB 不够:ast/parser.vp 的 text.ll 超 1MB 被截断 → 合并 IR 尾部丢字
|
||
# (opt: "end of file in global variable name"),提至 8MB。
|
||
PHASE1_IR_SIZE: sizet = 8388608
|
||
stub_buf: bytes = stdlib.malloc(PHASE1_IR_SIZE)
|
||
if stub_buf is None:
|
||
stdlib.free(src_buf)
|
||
if tr._global_names is not None:
|
||
stdlib.free(tr._global_names)
|
||
if tr._nonlocal_names is not None:
|
||
stdlib.free(tr._nonlocal_names)
|
||
failed += 1
|
||
continue
|
||
tr.dump_ir(stub_buf, PHASE1_IR_SIZE, llvmlite.OUTPUT_STUB)
|
||
stub_len: sizet = string.strlen(stub_buf)
|
||
|
||
# save stub.ll
|
||
dir_len_p1: sizet = string.strlen(temp_dir)
|
||
stub_path_p1: str = StubMergerCollect._sliced_path(temp_dir, dir_len_p1, sha1, "stub.ll")
|
||
if stub_path_p1 is not None:
|
||
sf_p1: rawptr[fileio.File] = fileio.File(stub_path_p1, fileio.MODE.W)
|
||
if not sf_p1.closed:
|
||
sf_p1.write(stub_buf, stub_len)
|
||
sf_p1.close()
|
||
stdlib.free(stub_path_p1)
|
||
stdlib.free(stub_buf)
|
||
|
||
# dump text IR(definitions only)
|
||
text_buf: bytes = stdlib.malloc(PHASE1_IR_SIZE)
|
||
if text_buf is None:
|
||
stdlib.free(src_buf)
|
||
if tr._global_names is not None:
|
||
stdlib.free(tr._global_names)
|
||
if tr._nonlocal_names is not None:
|
||
stdlib.free(tr._nonlocal_names)
|
||
failed += 1
|
||
continue
|
||
tr.dump_ir(text_buf, PHASE1_IR_SIZE, llvmlite.OUTPUT_TEXT)
|
||
text_len: sizet = string.strlen(text_buf)
|
||
|
||
# save text.ll
|
||
text_path_p1: str = StubMergerCollect._sliced_path(temp_dir, dir_len_p1, sha1, "text.ll")
|
||
if text_path_p1 is not None:
|
||
tf_p1: rawptr[fileio.File] = fileio.File(text_path_p1, fileio.MODE.W)
|
||
if not tf_p1.closed:
|
||
tf_p1.write(text_buf, text_len)
|
||
tf_p1.close()
|
||
stdlib.free(text_path_p1)
|
||
stdlib.free(text_buf)
|
||
|
||
# 计时点 3:dump(stub+text)完成;输出单文件三段耗时
|
||
t_f3: uint64 = w32.win32base.GetTickCount64()
|
||
VLogger.info(f"[{i+1}/{loop_n_b}] 翻译: {file_path} (parse={t_f1-t_f0} ms, translate={t_f2-t_f1} ms, dump={t_f3-t_f2} ms, total={t_f3-t_f0} ms)", "Phase1")
|
||
|
||
# save dependencies(_imported_modules) for Phase B
|
||
deps_path_p1: str = StubMergerCollect._sliced_path(temp_dir, dir_len_p1, sha1, "deps.txt")
|
||
if deps_path_p1 is not None:
|
||
df_p1: rawptr[fileio.File] = fileio.File(deps_path_p1, fileio.MODE.W)
|
||
if not df_p1.closed:
|
||
if tr._imported_modules is not None:
|
||
dl: sizet = string.strlen(tr._imported_modules)
|
||
df_p1.write(tr._imported_modules, dl)
|
||
df_p1.close()
|
||
stdlib.free(deps_path_p1)
|
||
|
||
# 释放 Translator 的 C malloc 资源
|
||
if tr._global_names is not None:
|
||
stdlib.free(tr._global_names)
|
||
if tr._nonlocal_names is not None:
|
||
stdlib.free(tr._nonlocal_names)
|
||
|
||
# 所有 dump_ir 完成,src_buf 不再需要
|
||
stdlib.free(src_buf)
|
||
|
||
translated += 1
|
||
|
||
# OOM 回收:回收本文件翻译产生的瞬时数据(AST/tokens/lexer/IR/Handler/dump 临时);持久表(struct/enum/REnum/spec 缓存/全局函数名等)已迁出 reset 区间(stdlib),
|
||
# reset_to(mark1) 只回收 mark1 之后分配、未被单独 free 的块。
|
||
#
|
||
# 内存哨兵(崩溃点漂移排查):reset 前检查存活块 header(抓越界写),reset 后检查 free list 完整性(抓链表损坏);任一损坏立即终止,把「随机崩溃漂移」变成「确定性定位到当前文件」。
|
||
# rc 语义(check_alive_blocks):1=块地址低于 usable,2=高于 usable+size,3=order 为负,4=order 超 max_order,5=块未对齐到 order 大小,6=块尾 canary 损坏(越界写穿数据区)
|
||
# rc 语义(self_check):1/2=越界,3=对齐错,4=free list 成环/超步数
|
||
chk_rc: int = mb.check_alive_blocks(mark1)
|
||
if chk_rc != 0:
|
||
VLogger.error(f"内存损坏(存活块 header, rc={chk_rc}): {file_path}", "Phase1")
|
||
# 越界写是崩溃根因(与回收/UAF 无关,已由 A/B 实验确认)——恢复 reset_to,改用块尾 canary(check_alive_blocks 校验)把越界写事发地定位到具体文件。
|
||
mb.reset_to(mark1)
|
||
sc_rc: int = mb.self_check()
|
||
if sc_rc != 0:
|
||
VLogger.error(f"内存损坏(free list, rc={sc_rc}): {file_path}", "Phase1")
|
||
|
||
# 1b 阶段总耗时(全量翻译 + dump)
|
||
t_b1: uint64 = w32.win32base.GetTickCount64()
|
||
VLogger.info(f"1b 总耗时: {t_b1-t_b0} ms (全量翻译, trans={translated} skip={skipped} fail={failed})", "Phase1")
|
||
|
||
# 释放 Phase 1b 使用的拓扑排序资源
|
||
if topo_order is not None:
|
||
mb.free(topo_order)
|
||
if topo_count_box is not None:
|
||
mb.free(topo_count_box)
|
||
|
||
VLogger.info(f"完成: 翻译={translated} 跳过={skipped} 失败={failed}", "Phase1")
|
||
|
||
if failed > 0:
|
||
VLogger.error(f"Phase1 有 {failed} 个文件翻译失败", "Phase1")
|
||
return 1
|
||
|
||
# 释放可达 SHA1 集合(如果分配了)
|
||
if reachable_set is not None:
|
||
stdlib.free(reachable_set)
|
||
|
||
return 0
|