feat(tpv): App 与 TPVAppBK 新代版本对齐（rowptr 配套实现全套）
把 TransPyV/App 覆盖为 TPVAppBK 内容（39 文件）：6 参注册式 _BuildReachableSha1Set 全套（StubMergerBuild/Store/Collect/__init__/IncludesScanner/Phase1Scan 等）、CallGen 与 TypeGenResolve 等新代实现。配套根 includes 为 rowptr 工作区（另有 includes_rowptr_bk 备份）。
This commit is contained in:
TermiNexus committed
1 parent
c35246f426
commit
6b368b46a7
39 files changed
+2813
-505
No files matched your search
@@ -702,87 +702,139 @@ def _LoadAndAppendTextDeclares(temp_dir: str, td_len: t.CSizeT, dep_sha1: str,
|
||||
# 跳过其他行
|
||||
return out_pos
|
||||
# ============================================================
|
||||
# _BuildReachableSha1Set - 构建可达 SHA1 集合（依赖图按需翻译）
|
||||
# _IsAppModule - 模块名是否为 App 源文件（store 条目 incdir 为空）
|
||||
#
|
||||
# 从 source_dir 的源文件开始，解析 import 语句，递归收集可达的
|
||||
# includes 文件 SHA1。Phase 1b 只翻译可达集合中的文件，避免
|
||||
# 翻译不需要的 includes（如 Test 不依赖 llvmlite，则不翻译）。
|
||||
# 依赖图 BFS 中识别 App 中间节点：App 文件不入 reachable_set（Phase B
|
||||
# 已编译其 .obj），但其 includes 依赖必须继续展开。App 条目由 App 扫描
|
||||
# 阶段用 AppendToSha1MapStore 登记（incdir 留空）；includes 条目用
|
||||
# AddIncludeToSha1MapStore 登记（incdir 非空）。
|
||||
# ============================================================
|
||||
def _IsAppModule(mod_name: str) -> int:
|
||||
"""mod_name 是否为 App 源文件（store 命中且 incdir 为空）"""
|
||||
if mod_name is None:
|
||||
return 0
|
||||
if StubMerger._sha1_store_arr is None or StubMerger._sha1_store_mod is None:
|
||||
return 0
|
||||
for i in range(StubMerger._sha1_store_count):
|
||||
midx: t.CSizeT = t.CSizeT(i) * 64
|
||||
if string.strcmp(StubMerger._sha1_store_mod + midx, mod_name) == 0:
|
||||
iidx: t.CSizeT = t.CSizeT(i) * StubMerger.MAX_INC_DIR_LEN
|
||||
if StubMerger._sha1_store_incdir[iidx] == '\0':
|
||||
return 1
|
||||
return 0
|
||||
return 0
|
||||
# ============================================================
|
||||
# _AppendDepsToWorklist - 追加空格分隔模块名到 worklist（逐模块去重）
|
||||
#
|
||||
# 算法（工作列表）：
|
||||
# 1. 扫描 source_dir 下的 .vp/.py 文件，解析 AST 获取 _imported_modules
|
||||
# 2. 构建 SHA1→module_name 映射（从全局内存存储器）
|
||||
# 3. 工作列表递归：模块名 → 查 SHA1 → 加入 reachable_set → 读 .deps.txt → 追加新模块名
|
||||
# worklist 是 BFS 队列（空格分隔）。deps_str 中的模块若已在 worklist/
|
||||
# processed 中不重复追加，否则 BFS 沿共享依赖（builtins/stdlib 等）
|
||||
# 反复累积到 REACH_WORKLIST_BUF 溢出（"依赖 worklist 溢出"根因）。
|
||||
#
|
||||
# Args:
|
||||
# worklist: 队列缓冲（原地修改）
|
||||
# wl_len: 当前长度（值）
|
||||
# deps_str: 要追加的模块名（空格分隔）
|
||||
# processed: 已处理集合（空格分隔，查重）
|
||||
#
|
||||
# Returns:
|
||||
# 追加后的新长度（>=0）；溢出/参数错误返回 -1（调用方 fail-fast）
|
||||
# ============================================================
|
||||
def _AppendDepsToWorklist(worklist: bytes, wl_len: t.CSizeT,
|
||||
deps_str: str, processed: str) -> int:
|
||||
if worklist is None or deps_str is None:
|
||||
return -1
|
||||
cur_len: t.CSizeT = wl_len
|
||||
dl: t.CSizeT = string.strlen(deps_str)
|
||||
dpos: t.CSizeT = 0
|
||||
while dpos < dl:
|
||||
dstart: t.CSizeT = dpos
|
||||
while dpos < dl and deps_str[dpos] != ' ':
|
||||
dpos += 1
|
||||
dlen: t.CSizeT = dpos - dstart
|
||||
if dpos < dl:
|
||||
dpos += 1
|
||||
if dlen == 0:
|
||||
continue
|
||||
dmod: bytes = stdlib.malloc(dlen + 1)
|
||||
if dmod is None:
|
||||
continue
|
||||
string.strncpy(dmod, deps_str + dstart, dlen)
|
||||
dmod[dlen] = '\0'
|
||||
in_wl: int = 0
|
||||
if cur_len > 0:
|
||||
if GenApi.is_module_imported(worklist, dmod) != 0:
|
||||
in_wl = 1
|
||||
if in_wl == 0 and processed is not None:
|
||||
if GenApi.is_module_imported(processed, dmod) != 0:
|
||||
in_wl = 1
|
||||
if in_wl == 0:
|
||||
if cur_len + dlen + 2 < StubMerger.REACH_WORKLIST_BUF:
|
||||
if cur_len > 0 and worklist[cur_len - 1] != ' ':
|
||||
worklist[cur_len] = ' '
|
||||
cur_len = cur_len + 1
|
||||
string.strcpy(worklist + cur_len, dmod)
|
||||
cur_len = cur_len + dlen
|
||||
worklist[cur_len] = ' '
|
||||
cur_len = cur_len + 1
|
||||
worklist[cur_len] = '\0'
|
||||
else:
|
||||
stdlib.free(dmod)
|
||||
VLogger.error("依赖 worklist 溢出", "StubMerger")
|
||||
return -1
|
||||
stdlib.free(dmod)
|
||||
return t.CInt(cur_len)
|
||||
# ============================================================
|
||||
# _BuildReachableSha1Set - 构建可达 SHA1 集合（依赖图按需扫描）
|
||||
#
|
||||
# App 优先模型：先扫 App 源文件的 import 依赖，从 app_deps 出发 BFS，
|
||||
# 对每个模块名按文件系统解析到 includes 文件（ResolveModuleToPath），
|
||||
# 只对可达 includes 读文件+算 SHA1+解析 import，补充依赖图并登记
|
||||
# Sha1Store/deps 存储器/写 deps.txt。未引用的 includes 从不读取。
|
||||
# Phase 1b/Phase B+ 只处理可达集合中的文件。
|
||||
#
|
||||
# 算法（工作列表 BFS）：
|
||||
# 1. App 扫描（app_deps 为空时）：递归扫 source_dir，declare_only=2 解析
|
||||
# import（含相对导入 resolve 成绝对模块名）+ 容器注入（_list/_dict）
|
||||
# → worklist 初始队列；App 文件登记 store（incdir 空）+ 填 deps
|
||||
# 2. builtins 预加入队列（内置函数模块始终编译）
|
||||
# 3. BFS：模块名 → ResolveModuleToPath 按文件系统解析 → 读+hash+parse
|
||||
# import → 登记 reachable_set/store/deps → 追加依赖到队列，直到闭包
|
||||
#
|
||||
# Args:
|
||||
# mb: 内存池
|
||||
# source_dir: 源文件目录（Config.SourceDir）
|
||||
# temp_dir: 临时目录（.deps.txt 所在位置；_sha1_map.txt 仅人类可读输出，不读取）
|
||||
# source_dir: App 源文件目录（Config.SourceDir）
|
||||
# temp_dir: 临时目录（deps.txt 所在位置）
|
||||
# reachable_set: 输出参数，可达 SHA1 集合缓冲区（MAX_INCLUDES_SHA1 * 17 字节）
|
||||
# app_deps: App 源文件的直接依赖模块名（空格分隔），非空则跳过 App 扫描
|
||||
# includes_dirs: includes 根目录（空格分隔），依赖图按需解析必须
|
||||
#
|
||||
# Returns:
|
||||
# 可达 SHA1 数量（>0 成功，<=0 失败）
|
||||
# 可达 SHA1 数量（>0 成功；0 无 includes 依赖；<0 失败）
|
||||
# ============================================================
|
||||
def _BuildReachableSha1Set(mb: memhub.MemBuddy | t.CPtr, source_dir: str,
|
||||
temp_dir: str,
|
||||
reachable_set: str,
|
||||
app_deps: str = None) -> int:
|
||||
"""构建可达 SHA1 集合（依赖图按需翻译）
|
||||
|
||||
app_deps: App 源文件的直接依赖模块名（空格分隔），如果非空则跳过自己的扫描步骤
|
||||
"""
|
||||
app_deps: str = None,
|
||||
includes_dirs: str = None) -> int:
|
||||
"""构建可达 SHA1 集合（依赖图按需扫描，App 优先）"""
|
||||
if source_dir is None or temp_dir is None or reachable_set is None:
|
||||
return -1
|
||||
if includes_dirs is None:
|
||||
VLogger.error("includes_dirs 缺失，无法按需解析依赖图", "Reachable")
|
||||
return -1
|
||||
|
||||
SRC_BUF_SIZE_R: t.CSizeT = 1048576
|
||||
|
||||
# 1. 构建 SHA1→module_name 映射
|
||||
td_len_r: t.CSizeT = string.strlen(temp_dir)
|
||||
sha1_arr: bytes = stdlib.malloc(StubMerger.MAX_INCLUDES * 17)
|
||||
mod_arr: bytes = stdlib.malloc(StubMerger.MAX_INCLUDES * 64)
|
||||
if sha1_arr is None or mod_arr is None:
|
||||
if sha1_arr is not None:
|
||||
stdlib.free(sha1_arr)
|
||||
if mod_arr is not None:
|
||||
stdlib.free(mod_arr)
|
||||
return -1
|
||||
map_count: int = StubMergerIR._BuildIncludesSha1Map(temp_dir, td_len_r, sha1_arr, mod_arr)
|
||||
if map_count <= 0:
|
||||
VLogger.error("无法构建 SHA1 映射", "Reachable")
|
||||
stdlib.free(sha1_arr)
|
||||
stdlib.free(mod_arr)
|
||||
return -1
|
||||
VLogger.debug(f"SHA1 映射: {map_count} 个", "Reachable")
|
||||
|
||||
# 工作列表（空格分隔的模块名）
|
||||
worklist: bytes = stdlib.malloc(StubMerger.REACH_WORKLIST_BUF)
|
||||
if worklist is None:
|
||||
stdlib.free(sha1_arr)
|
||||
stdlib.free(mod_arr)
|
||||
return -1
|
||||
worklist[0] = '\0'
|
||||
wl_len: t.CSizeT = 0
|
||||
|
||||
reachable_count: int = 0
|
||||
|
||||
# 2. 如果 app_deps 非空，直接使用它作为初始 worklist（跳过自己的非递归扫描）
|
||||
# 否则，回退到自己的非递归扫描（source_dir/*.py，只扫描顶层）
|
||||
if app_deps is not None and app_deps[0] != '\0':
|
||||
ad_len: t.CSizeT = string.strlen(app_deps)
|
||||
if ad_len > 0 and ad_len < StubMerger.REACH_WORKLIST_BUF:
|
||||
string.strcpy(worklist, app_deps)
|
||||
wl_len = ad_len
|
||||
else:
|
||||
VLogger.error("app_deps 过长，无法构建可达集合", "StubMerger")
|
||||
else:
|
||||
# app_deps 缺失：Phase1 场景（未翻译 App 源文件）为设计性回退，
|
||||
# 显式 warning 暴露回退，不再静默。
|
||||
# 原实现仅非递归扫描 source_dir 顶层（"%s/*"），漏掉 lib/ 等子目录的
|
||||
# App 文件 → 其依赖（如 lib/Projectrans/Config.vp 的 json）不进入
|
||||
# worklist → json 不可达 → Phase1 不 stub 不翻译 → Phase2 Phase A
|
||||
# 翻译 Config.vp 时 root.is_object() 跨模块调用无 stub 可查 → 静默
|
||||
# translate_value 返回 None → "if 条件表达式翻译失败"（追加82 根因）。
|
||||
# 改为递归扫描（复用 IncludesScanner.scan_includes），覆盖全部子目录。
|
||||
VLogger.warning("app_deps 缺失，递归扫描 source_dir", "Reachable")
|
||||
# 1. App 扫描（app_deps 非空时跳过，直接用调用方提供的初始队列）
|
||||
if app_deps is None or app_deps[0] == '\0':
|
||||
VLogger.info("App 扫描：解析 source_dir 的 import 依赖", "Reachable")
|
||||
scan_res_r: IncludesScanner.ScanResult | t.CPtr = IncludesScanner.scan_includes(mb, source_dir)
|
||||
if scan_res_r is not None:
|
||||
es_r: t.CSizeT = IncludesScanner.FileEntry.__sizeof__()
|
||||
@@ -791,119 +843,137 @@ def _BuildReachableSha1Set(mb: memhub.MemBuddy | t.CPtr, source_dir: str,
|
||||
t.CVoid(t.CUInt64T(scan_res_r.Entries) + t.CUInt64T(e_i_r) * es_r, t.CPtr))
|
||||
if e_r is None or e_r.Path is None:
|
||||
continue
|
||||
# 文件级 mark：parse 完立即回收 AST（App 全量 parse 驻留会 OOM）
|
||||
app_mark: t.CSizeT = mb.mark()
|
||||
sbuf: bytes = Utils.read_file_to_buf(e_r.Path, SRC_BUF_SIZE_R)
|
||||
if sbuf is None:
|
||||
mb.reset_to(app_mark)
|
||||
continue
|
||||
# 解析 AST 获取 _imported_modules（与原顶层扫描一致）
|
||||
lx: ast.Lexer | t.CPtr = ast.new_lexer(mb)
|
||||
tree: ast.AST | t.CPtr = None
|
||||
if lx is not None:
|
||||
ast._lexer_init(lx, sbuf, mb)
|
||||
tokens: ast.Token | t.CPtr = ast.tokenize(lx)
|
||||
tree: ast.AST | t.CPtr = ast.parse_tokens(mb, tokens)
|
||||
if tree is not None:
|
||||
tr: HandlesTranslator.Translator | t.CPtr = HandlesTranslator.Translator()
|
||||
if tr is not None:
|
||||
tr._declare_only = 2
|
||||
# 源文件包名：用 RelPath（相对 source_dir，支持子目录），
|
||||
# 保证 from .xxx import 相对导入解析正确
|
||||
tr.CurrentPackage = GenApi.compute_package_from_relpath(mb, e_r.RelPath)
|
||||
GenApi.reset_visible_structs(mb, 0)
|
||||
tr.translate(tree)
|
||||
# 容器使用检测：App 源文件用 list/dict（裸名注解/构造/[]
|
||||
# /{} 字面量）时注入 _list/_dict 依赖（与原实现一致）
|
||||
tree = ast.parse_tokens(mb, tokens)
|
||||
if tree is not None:
|
||||
tr: HandlesTranslator.Translator | t.CPtr = HandlesTranslator.Translator()
|
||||
if tr is not None:
|
||||
tr._declare_only = 2
|
||||
# 源文件包名：用 RelPath（相对 source_dir，支持子目录），
|
||||
# 保证 from .xxx import 相对导入解析正确
|
||||
tr.CurrentPackage = GenApi.compute_package_from_relpath(mb, e_r.RelPath)
|
||||
GenApi.set_current_file_type(e_r.Path)
|
||||
GenApi.set_current_file_imports(e_r.Path)
|
||||
GenApi.set_current_module_sha1(e_r.Sha1)
|
||||
GenApi.clear_cdefine_constants()
|
||||
GenApi.clear_ctypedefs()
|
||||
GenApi.reset_visible_structs(mb, 0)
|
||||
ret_a: int = tr.translate(tree)
|
||||
if ret_a == 0:
|
||||
# 容器使用检测：App 文件用 list/dict（裸名注解/构造）
|
||||
# 时注入 _list/_dict 依赖（对齐原实现）
|
||||
tr._inject_auto_container_visibility(mb, tree)
|
||||
# 获取 _imported_modules
|
||||
# App 文件登记 store（incdir 空）+ 填 deps：
|
||||
# BFS 展开 App 中间节点的 includes 依赖时经
|
||||
# LookupIncludesDeps(app_sha1) 读取
|
||||
if e_r.Sha1 is not None:
|
||||
StubMergerStore.AppendToSha1MapStore(e_r.Sha1, e_r.RelPath)
|
||||
StubMergerStore.PopulateIncludesDeps(e_r.Sha1, tr._imported_modules)
|
||||
# 收集 deps 到初始队列（逐模块去重）
|
||||
if tr._imported_modules is not None:
|
||||
# 逐模块去重追加（同 app_deps 拼接，防 worklist 膨胀溢出）
|
||||
im_len: t.CSizeT = string.strlen(tr._imported_modules)
|
||||
im_pos: t.CSizeT = 0
|
||||
while im_pos < im_len:
|
||||
im_start: t.CSizeT = im_pos
|
||||
while im_pos < im_len and tr._imported_modules[im_pos] != ' ':
|
||||
im_pos += 1
|
||||
im_mod_len: t.CSizeT = im_pos - im_start
|
||||
if im_pos < im_len:
|
||||
im_pos += 1
|
||||
im_len_a: t.CSizeT = string.strlen(tr._imported_modules)
|
||||
im_pos_a: t.CSizeT = 0
|
||||
while im_pos_a < im_len_a:
|
||||
im_start_a: t.CSizeT = im_pos_a
|
||||
while im_pos_a < im_len_a and tr._imported_modules[im_pos_a] != ' ':
|
||||
im_pos_a += 1
|
||||
im_mod_len: t.CSizeT = im_pos_a - im_start_a
|
||||
if im_pos_a < im_len_a:
|
||||
im_pos_a += 1
|
||||
if im_mod_len == 0:
|
||||
continue
|
||||
im_buf: bytes = stdlib.malloc(im_mod_len + 1)
|
||||
if im_buf is None:
|
||||
im_buf_a: bytes = stdlib.malloc(im_mod_len + 1)
|
||||
if im_buf_a is None:
|
||||
continue
|
||||
string.strncpy(im_buf, tr._imported_modules + im_start, im_mod_len)
|
||||
im_buf[im_mod_len] = '\0'
|
||||
if wl_len > 0 and GenApi.is_module_imported(worklist, im_buf) != 0:
|
||||
stdlib.free(im_buf)
|
||||
continue
|
||||
if wl_len + im_mod_len + 2 < StubMerger.REACH_WORKLIST_BUF:
|
||||
if wl_len > 0:
|
||||
string.strncpy(im_buf_a, tr._imported_modules + im_start_a, im_mod_len)
|
||||
im_buf_a[im_mod_len] = '\0'
|
||||
in_wl_a: int = 0
|
||||
if wl_len > 0 and GenApi.is_module_imported(worklist, im_buf_a) != 0:
|
||||
in_wl_a = 1
|
||||
if in_wl_a == 0:
|
||||
if wl_len + im_mod_len + 2 < StubMerger.REACH_WORKLIST_BUF:
|
||||
if wl_len > 0:
|
||||
worklist[wl_len] = ' '
|
||||
wl_len += 1
|
||||
string.strcpy(worklist + wl_len, im_buf_a)
|
||||
wl_len += im_mod_len
|
||||
worklist[wl_len] = ' '
|
||||
wl_len += 1
|
||||
string.strcpy(worklist + wl_len, im_buf)
|
||||
wl_len += im_mod_len
|
||||
worklist[wl_len] = ' '
|
||||
wl_len += 1
|
||||
worklist[wl_len] = '\0'
|
||||
stdlib.free(im_buf)
|
||||
# 计算 App 源文件 SHA1，填充依赖到内存存储器
|
||||
# worklist 遍历时，LookupIncludesDeps 能找到 App 源文件的依赖
|
||||
if e_r.Sha1 is not None:
|
||||
StubMergerStore.PopulateIncludesDeps(e_r.Sha1, tr._imported_modules)
|
||||
# 释放 Translator 资源
|
||||
if tr._global_names is not None:
|
||||
stdlib.free(tr._global_names)
|
||||
if tr._nonlocal_names is not None:
|
||||
stdlib.free(tr._nonlocal_names)
|
||||
worklist[wl_len] = '\0'
|
||||
stdlib.free(im_buf_a)
|
||||
# 释放 Translator 资源
|
||||
if tr._global_names is not None:
|
||||
stdlib.free(tr._global_names)
|
||||
if tr._nonlocal_names is not None:
|
||||
stdlib.free(tr._nonlocal_names)
|
||||
stdlib.free(sbuf)
|
||||
mb.reset_to(app_mark)
|
||||
else:
|
||||
# app_deps 已提供（Phase2 场景）：直接作为初始队列
|
||||
ad_len: t.CSizeT = string.strlen(app_deps)
|
||||
if ad_len > 0 and ad_len < StubMerger.REACH_WORKLIST_BUF:
|
||||
string.strcpy(worklist, app_deps)
|
||||
wl_len = ad_len
|
||||
else:
|
||||
VLogger.error("app_deps 过长，无法构建可达集合", "StubMerger")
|
||||
stdlib.free(worklist)
|
||||
return -1
|
||||
|
||||
# 去掉末尾多余空格
|
||||
if wl_len > 0 and worklist[wl_len - 1] == ' ':
|
||||
worklist[wl_len - 1] = '\0'
|
||||
wl_len -= 1
|
||||
# 去掉末尾多余空格
|
||||
if wl_len > 0 and worklist[wl_len - 1] == ' ':
|
||||
worklist[wl_len - 1] = '\0'
|
||||
wl_len -= 1
|
||||
|
||||
VLogger.debug(f"源文件直接依赖: '{worklist}'", "Reachable")
|
||||
|
||||
# 2.5. 预加入基础模块（builtins）
|
||||
#
|
||||
# builtins（内置函数模块：ord/chr 等）始终纳入编译链，调用时自动关联。
|
||||
# _list/_dict 不再无条件预加入：Phase A 翻译 App 源文件时
|
||||
# _inject_auto_container_visibility 已按需检测 list/dict 使用并注入
|
||||
# _imported_modules（含 []/{} 字面量），json 由 _dict 的真实 import 链带入。
|
||||
if GenApi.is_module_imported(worklist, "builtins") == 0:
|
||||
bsha1_b: str = StubMergerIR._FindSha1ByModName(sha1_arr, mod_arr, map_count, "builtins")
|
||||
if bsha1_b is not None:
|
||||
if wl_len + 9 < StubMerger.REACH_WORKLIST_BUF:
|
||||
if wl_len > 0:
|
||||
worklist[wl_len] = ' '
|
||||
wl_len += 1
|
||||
string.strcpy(worklist + wl_len, "builtins")
|
||||
wl_len += 8
|
||||
worklist[wl_len] = ' '
|
||||
wl_len += 1
|
||||
worklist[wl_len] = '\0'
|
||||
# 2. 预加入基础模块（builtins）
|
||||
# builtins（内置函数模块）始终纳入编译链。_list/_dict 仅在显式
|
||||
# list/dict 注解/构造时经 _inject_auto_container_visibility 注入
|
||||
# （list 字面量赋给 t.CArray 走数组初始化，不触发 _list）。
|
||||
# processed 尚未分配，此处传 None 仅按 worklist 查重。
|
||||
wl_len = _AppendDepsToWorklist(worklist, wl_len, "builtins", None)
|
||||
if wl_len < 0:
|
||||
stdlib.free(worklist)
|
||||
return -1
|
||||
|
||||
VLogger.debug(f"预加入容器模块后: '{worklist}'", "Reachable")
|
||||
|
||||
# 3. 工作列表算法：递归收集可达 SHA1
|
||||
# 3. 工作列表 BFS：模块名 → 文件系统解析 → 读+hash+parse import
|
||||
processed: bytes = stdlib.malloc(StubMerger.REACH_WORKLIST_BUF)
|
||||
if processed is None:
|
||||
stdlib.free(sha1_arr)
|
||||
stdlib.free(mod_arr)
|
||||
stdlib.free(worklist)
|
||||
return -1
|
||||
processed[0] = '\0'
|
||||
proc_len: t.CSizeT = 0
|
||||
reachable_count: int = 0
|
||||
wl_pos: t.CPtr = stdlib.malloc(8)
|
||||
if wl_pos is None:
|
||||
stdlib.free(worklist)
|
||||
stdlib.free(processed)
|
||||
return -1
|
||||
wl_pos_slot: t.CSizeT | t.CPtr = (t.CSizeT | t.CPtr)(t.CVoid(wl_pos, t.CPtr))
|
||||
wl_pos_slot[0] = 0
|
||||
|
||||
wl_pos: t.CSizeT = 0
|
||||
wl_iter: int = 0
|
||||
while wl_pos < wl_len:
|
||||
wl_iter += 1
|
||||
while wl_pos_slot[0] < wl_len:
|
||||
pos: t.CSizeT = wl_pos_slot[0]
|
||||
# 提取一个模块名（到空格或末尾）
|
||||
name_start: t.CSizeT = wl_pos
|
||||
while wl_pos < wl_len and worklist[wl_pos] != ' ':
|
||||
wl_pos += 1
|
||||
name_len: t.CSizeT = wl_pos - name_start
|
||||
if wl_pos < wl_len:
|
||||
wl_pos += 1
|
||||
name_start: t.CSizeT = pos
|
||||
while pos < wl_len and worklist[pos] != ' ':
|
||||
pos += 1
|
||||
name_len: t.CSizeT = pos - name_start
|
||||
if pos < wl_len:
|
||||
pos += 1
|
||||
wl_pos_slot[0] = pos
|
||||
if name_len == 0:
|
||||
continue
|
||||
|
||||
@@ -914,120 +984,257 @@ def _BuildReachableSha1Set(mb: memhub.MemBuddy | t.CPtr, source_dir: str,
|
||||
string.strncpy(mod_buf, worklist + name_start, name_len)
|
||||
mod_buf[name_len] = '\0'
|
||||
|
||||
# 检查是否已处理
|
||||
already: int = 0
|
||||
# 已处理去重
|
||||
if GenApi.is_module_imported(processed, mod_buf) != 0:
|
||||
already = 1
|
||||
if already == 0:
|
||||
# 加入 processed（嵌套 if 拆分 + 用 = 代替 += 绕过 TPC AUGASGN bug）
|
||||
if processed is not None:
|
||||
if proc_len + name_len + 1 < StubMerger.REACH_WORKLIST_BUF:
|
||||
string.strcpy(processed + proc_len, mod_buf)
|
||||
proc_len = proc_len + name_len
|
||||
processed[proc_len] = ' '
|
||||
proc_len = proc_len + 1
|
||||
processed[proc_len] = '\0'
|
||||
else:
|
||||
VLogger.error("依赖 worklist 溢出", "StubMerger")
|
||||
stdlib.free(mod_buf)
|
||||
continue
|
||||
# 加入 processed
|
||||
if proc_len + name_len + 1 < StubMerger.REACH_WORKLIST_BUF:
|
||||
string.strcpy(processed + proc_len, mod_buf)
|
||||
proc_len = proc_len + name_len
|
||||
processed[proc_len] = ' '
|
||||
proc_len = proc_len + 1
|
||||
processed[proc_len] = '\0'
|
||||
else:
|
||||
VLogger.error("依赖 processed 溢出", "StubMerger")
|
||||
stdlib.free(mod_buf)
|
||||
continue
|
||||
|
||||
# 查找 SHA1
|
||||
found_sha1: str = StubMergerIR._FindSha1ByModName(sha1_arr, mod_arr, map_count, mod_buf)
|
||||
if found_sha1 is not None:
|
||||
# App 源文件（Sha1Store 中 IncDir 为空，AppendToSha1MapStore 规则）
|
||||
# 不加入 reachable_set：Phase B 已编译其 .obj，若加入会导致 Phase B+
|
||||
# 把 App 文件重复编译进 includes.binary，Phase C 链接时与 Phase B 的
|
||||
# App obj 重复符号（SysLibTest ld.lld duplicate symbol: StubMerger._mbuddy 根因）。
|
||||
# 依赖仍继续展开（App 文件是依赖图中间节点，其 includes 依赖必须可达）。
|
||||
is_app_r: int = 0
|
||||
if StubMerger._sha1_store_incdir is not None:
|
||||
for ai_r in range(StubMerger._sha1_store_count):
|
||||
asi_r: t.CSizeT = t.CSizeT(ai_r) * 17
|
||||
if string.strcmp(StubMerger._sha1_store_arr + asi_r, found_sha1) == 0:
|
||||
aii_r: t.CSizeT = t.CSizeT(ai_r) * StubMerger.MAX_INC_DIR_LEN
|
||||
if StubMerger._sha1_store_incdir[aii_r] == '\0':
|
||||
is_app_r = 1
|
||||
break
|
||||
if is_app_r == 0:
|
||||
reachable_count = StubMergerIR._AddSha1ToSet(reachable_set, reachable_count, found_sha1)
|
||||
|
||||
# 从内存查找依赖（不读 deps.txt 过程文件）
|
||||
# Phase B+ 前用 declare_only 翻译所有 includes 文件，提取 _imported_modules 存入内存
|
||||
deps_str: str = StubMergerStore.LookupIncludesDeps(found_sha1)
|
||||
if deps_str is not None:
|
||||
dl: t.CSizeT = string.strlen(deps_str)
|
||||
if dl > 0:
|
||||
# 逐模块追加 + 去重：worklist 初始已含全部 App 直接依赖，
|
||||
# deps_str 中的模块若已在 worklist/processed 不重复追加，
|
||||
# 否则 BFS 沿共享依赖（builtins/stdlib 等）反复累积到
|
||||
# 64KB 溢出（"依赖 worklist 溢出"根因）。
|
||||
dpos: t.CSizeT = 0
|
||||
while dpos < dl:
|
||||
dstart: t.CSizeT = dpos
|
||||
while dpos < dl and deps_str[dpos] != ' ':
|
||||
dpos += 1
|
||||
dlen: t.CSizeT = dpos - dstart
|
||||
if dpos < dl:
|
||||
dpos += 1
|
||||
if dlen == 0:
|
||||
continue
|
||||
dmod: bytes = stdlib.malloc(dlen + 1)
|
||||
if dmod is None:
|
||||
continue
|
||||
string.strncpy(dmod, deps_str + dstart, dlen)
|
||||
dmod[dlen] = '\0'
|
||||
in_wl: int = 0
|
||||
if GenApi.is_module_imported(worklist, dmod) != 0:
|
||||
in_wl = 1
|
||||
if in_wl == 0 and processed is not None:
|
||||
if GenApi.is_module_imported(processed, dmod) != 0:
|
||||
in_wl = 1
|
||||
if in_wl == 0:
|
||||
if wl_len + dlen + 2 < StubMerger.REACH_WORKLIST_BUF:
|
||||
if wl_len > 0 and worklist[wl_len - 1] != ' ':
|
||||
worklist[wl_len] = ' '
|
||||
wl_len = wl_len + 1
|
||||
string.strcpy(worklist + wl_len, dmod)
|
||||
wl_len = wl_len + dlen
|
||||
worklist[wl_len] = ' '
|
||||
wl_len = wl_len + 1
|
||||
worklist[wl_len] = '\0'
|
||||
else:
|
||||
VLogger.error("依赖 worklist 溢出", "StubMerger")
|
||||
stdlib.free(dmod)
|
||||
# deps_str 为空字符串视为无依赖（合法情况，如叶子模块）
|
||||
else:
|
||||
# 依赖未在内存中找到
|
||||
# 按需过滤: 检查是否是 App 源文件（source_dir 下的 .vp/.py 文件）
|
||||
# App 源文件不需要加入 reachable_set（只用于 includes 过滤），跳过
|
||||
sd_len_ff: t.CSizeT = string.strlen(source_dir)
|
||||
mb_len_ff: t.CSizeT = string.strlen(mod_buf)
|
||||
app_mod_path: bytes = stdlib.malloc(sd_len_ff + mb_len_ff + 6)
|
||||
is_app_src: int = 0
|
||||
if app_mod_path is not None:
|
||||
# 仅识别 .vp 源文件（不再做 vp -> py 后缀识别回退）
|
||||
viperlib.snprintf(app_mod_path, sd_len_ff + mb_len_ff + 6, "%s/%s.vp", source_dir, mod_buf)
|
||||
af: fileio.File | t.CPtr = fileio.File(app_mod_path, fileio.MODE.R)
|
||||
if not af.closed:
|
||||
is_app_src = 1
|
||||
af.close()
|
||||
stdlib.free(app_mod_path)
|
||||
if is_app_src == 0:
|
||||
# fast-fail: includes 文件依赖未在内存中找到，Phase 1a-pre 未正确填充
|
||||
VLogger.error(f"依赖未找到 fast-fail: 模块 {mod_buf}(sha1={found_sha1}) 的依赖未在内存中注册", "Reachable")
|
||||
stdlib.free(sha1_arr)
|
||||
stdlib.free(mod_arr)
|
||||
# App 中间节点：只展开其 includes 依赖，不入 reachable_set
|
||||
# （Phase B 已编译 App 的 .obj，加入会导致 Phase B+ 重复编译 → 重复符号）
|
||||
if _IsAppModule(mod_buf) != 0:
|
||||
app_sha1: str = StubMergerStore.LookupSha1ByModName(mod_buf)
|
||||
if app_sha1 is not None:
|
||||
app_deps_str: str = StubMergerStore.LookupIncludesDeps(app_sha1)
|
||||
if app_deps_str is not None:
|
||||
ret_ap: int = _AppendDepsToWorklist(worklist, wl_len, app_deps_str, processed)
|
||||
if ret_ap < 0:
|
||||
stdlib.free(mod_buf)
|
||||
stdlib.free(worklist)
|
||||
stdlib.free(processed)
|
||||
stdlib.free(mod_buf)
|
||||
stdlib.free(wl_pos)
|
||||
return -1
|
||||
wl_len = t.CSizeT(ret_ap)
|
||||
stdlib.free(mod_buf)
|
||||
continue
|
||||
|
||||
# includes：按文件系统按需解析（未引用的 includes 根本不触碰）
|
||||
out_abs: t.CPtr = stdlib.malloc(8)
|
||||
out_rel: t.CPtr = stdlib.malloc(8)
|
||||
out_inc: t.CPtr = stdlib.malloc(8)
|
||||
if out_abs is None or out_rel is None or out_inc is None:
|
||||
if out_abs is not None:
|
||||
stdlib.free(out_abs)
|
||||
if out_rel is not None:
|
||||
stdlib.free(out_rel)
|
||||
if out_inc is not None:
|
||||
stdlib.free(out_inc)
|
||||
stdlib.free(mod_buf)
|
||||
continue
|
||||
# 输出槽防御性初始化（成功路径必全写；未找到/未命中时仅 free 槽本身）
|
||||
# 槽联合必须用 (t.CSizeT | t.CPtr)：t.CPtr|t.CPtr 会被折叠成 t.CPtr，
|
||||
# [0] 变成 1 字节访问 → 指针被截断（0x2e 崩溃根因）。CSizeT 成员为
|
||||
# 8 字节，经 t.CVoid 显式做 u64↔指针转换。
|
||||
abs_slot: t.CSizeT | t.CPtr = (t.CSizeT | t.CPtr)(t.CVoid(out_abs, t.CPtr))
|
||||
rel_slot: t.CSizeT | t.CPtr = (t.CSizeT | t.CPtr)(t.CVoid(out_rel, t.CPtr))
|
||||
inc_slot: t.CSizeT | t.CPtr = (t.CSizeT | t.CPtr)(t.CVoid(out_inc, t.CPtr))
|
||||
abs_slot[0] = 0
|
||||
rel_slot[0] = 0
|
||||
inc_slot[0] = 0
|
||||
rcode: int = IncludesScanner.ResolveModuleToPath(
|
||||
includes_dirs, mod_buf, out_abs, out_rel, out_inc)
|
||||
if rcode != 0:
|
||||
# 内建模块（t/c/builtins 等内部库）或未找到：静默跳过
|
||||
stdlib.free(out_abs)
|
||||
stdlib.free(out_rel)
|
||||
stdlib.free(out_inc)
|
||||
stdlib.free(mod_buf)
|
||||
continue
|
||||
abs_path: str = (str)(t.CVoid(abs_slot[0], t.CPtr))
|
||||
rel_path: str = (str)(t.CVoid(rel_slot[0], t.CPtr))
|
||||
inc_dir: str = (str)(t.CVoid(inc_slot[0], t.CPtr))
|
||||
|
||||
# 读文件 + 计算 SHA1（CRLF→LF，与 IncludesScanner 一致）
|
||||
sha1_r2: str = IncludesScanner.compute_file_sha1(mb, abs_path)
|
||||
if sha1_r2 is None:
|
||||
stdlib.free(abs_path)
|
||||
stdlib.free(rel_path)
|
||||
stdlib.free(inc_dir)
|
||||
stdlib.free(out_abs)
|
||||
stdlib.free(out_rel)
|
||||
stdlib.free(out_inc)
|
||||
stdlib.free(mod_buf)
|
||||
continue
|
||||
|
||||
# 已在可达集合（前序 BFS 已 parse 过）：只展开其 deps
|
||||
if StubMergerCollect._is_in_sha1_set(sha1_r2, reachable_set, reachable_count) != 0:
|
||||
deps_ex: str = StubMergerStore.LookupIncludesDeps(sha1_r2)
|
||||
if deps_ex is not None:
|
||||
ret_ex: int = _AppendDepsToWorklist(worklist, wl_len, deps_ex, processed)
|
||||
if ret_ex < 0:
|
||||
stdlib.free(abs_path)
|
||||
stdlib.free(rel_path)
|
||||
stdlib.free(inc_dir)
|
||||
stdlib.free(out_abs)
|
||||
stdlib.free(out_rel)
|
||||
stdlib.free(out_inc)
|
||||
stdlib.free(mod_buf)
|
||||
stdlib.free(worklist)
|
||||
stdlib.free(processed)
|
||||
stdlib.free(wl_pos)
|
||||
return -1
|
||||
wl_len = t.CSizeT(ret_ex)
|
||||
stdlib.free(abs_path)
|
||||
stdlib.free(rel_path)
|
||||
stdlib.free(inc_dir)
|
||||
stdlib.free(out_abs)
|
||||
stdlib.free(out_rel)
|
||||
stdlib.free(out_inc)
|
||||
stdlib.free(mod_buf)
|
||||
continue
|
||||
|
||||
# 加入可达集合 + 登记 Sha1Store（仅可达 includes）
|
||||
reachable_count = StubMergerIR._AddSha1ToSet(reachable_set, reachable_count, sha1_r2)
|
||||
StubMergerStore.AddIncludeToSha1MapStore(sha1_r2, rel_path, inc_dir, mod_buf)
|
||||
|
||||
# 文件级 mark：parse import 后立即回收 AST（可达 includes 全量
|
||||
# parse 的 AST 若驻留 → 主池 OOM）
|
||||
scan_mark_r: t.CSizeT = mb.mark()
|
||||
sbuf_r: bytes = Utils.read_file_to_buf(abs_path, SRC_BUF_SIZE_R)
|
||||
if sbuf_r is None:
|
||||
stdlib.free(abs_path)
|
||||
stdlib.free(rel_path)
|
||||
stdlib.free(inc_dir)
|
||||
stdlib.free(out_abs)
|
||||
stdlib.free(out_rel)
|
||||
stdlib.free(out_inc)
|
||||
stdlib.free(mod_buf)
|
||||
mb.reset_to(scan_mark_r)
|
||||
continue
|
||||
lx_r: ast.Lexer | t.CPtr = ast.new_lexer(mb)
|
||||
tree_r: ast.AST | t.CPtr = None
|
||||
if lx_r is not None:
|
||||
ast._lexer_init(lx_r, sbuf_r, mb)
|
||||
tokens_r: ast.Token | t.CPtr = ast.tokenize(lx_r)
|
||||
tree_r = ast.parse_tokens(mb, tokens_r)
|
||||
if tree_r is None:
|
||||
VLogger.error(f"可达 includes 解析失败: {abs_path}", "Reachable")
|
||||
stdlib.free(abs_path)
|
||||
stdlib.free(rel_path)
|
||||
stdlib.free(inc_dir)
|
||||
stdlib.free(out_abs)
|
||||
stdlib.free(out_rel)
|
||||
stdlib.free(out_inc)
|
||||
stdlib.free(sbuf_r)
|
||||
stdlib.free(mod_buf)
|
||||
stdlib.free(worklist)
|
||||
stdlib.free(processed)
|
||||
stdlib.free(wl_pos)
|
||||
mb.reset_to(scan_mark_r)
|
||||
return -1
|
||||
tr_r2: HandlesTranslator.Translator | t.CPtr = HandlesTranslator.Translator()
|
||||
if tr_r2 is None:
|
||||
stdlib.free(abs_path)
|
||||
stdlib.free(rel_path)
|
||||
stdlib.free(inc_dir)
|
||||
stdlib.free(out_abs)
|
||||
stdlib.free(out_rel)
|
||||
stdlib.free(out_inc)
|
||||
stdlib.free(sbuf_r)
|
||||
stdlib.free(mod_buf)
|
||||
mb.reset_to(scan_mark_r)
|
||||
continue
|
||||
tr_r2.ModuleSha1 = sha1_r2
|
||||
tr_r2._declare_only = 2
|
||||
# 包名：相对 includes 根目录的 RelPath（支持 from .xxx import 相对导入）
|
||||
tr_r2.CurrentPackage = GenApi.compute_package_from_relpath(mb, rel_path)
|
||||
GenApi.set_current_file_type(abs_path)
|
||||
GenApi.set_current_file_imports(abs_path)
|
||||
GenApi.set_current_module_sha1(sha1_r2)
|
||||
GenApi.clear_cdefine_constants()
|
||||
GenApi.clear_ctypedefs()
|
||||
GenApi.reset_visible_structs(mb, 0)
|
||||
ret_r2: int = tr_r2.translate(tree_r)
|
||||
if ret_r2 != 0:
|
||||
VLogger.error(f"import 依赖扫描失败（可达 includes）: {abs_path}", "Reachable")
|
||||
stdlib.free(abs_path)
|
||||
stdlib.free(rel_path)
|
||||
stdlib.free(inc_dir)
|
||||
stdlib.free(out_abs)
|
||||
stdlib.free(out_rel)
|
||||
stdlib.free(out_inc)
|
||||
stdlib.free(sbuf_r)
|
||||
if tr_r2._global_names is not None:
|
||||
stdlib.free(tr_r2._global_names)
|
||||
if tr_r2._nonlocal_names is not None:
|
||||
stdlib.free(tr_r2._nonlocal_names)
|
||||
stdlib.free(mod_buf)
|
||||
stdlib.free(worklist)
|
||||
stdlib.free(processed)
|
||||
stdlib.free(wl_pos)
|
||||
mb.reset_to(scan_mark_r)
|
||||
return -1
|
||||
# 容器按需注入（includes 用 list/dict 泛型时追加 _list/_dict 到依赖，
|
||||
# 对齐 App 层 _inject_auto_container_visibility；内联破环防 _list/_dict 环）
|
||||
tr_r2._inject_auto_container_visibility(mb, tree_r)
|
||||
|
||||
# 填内存依赖 + 写 deps.txt（供依赖图展开与 Phase B 按需加载）
|
||||
deps_r2: str = tr_r2._imported_modules
|
||||
StubMergerStore.PopulateIncludesDeps(sha1_r2, deps_r2)
|
||||
if deps_r2 is not None:
|
||||
td_len_w: t.CSizeT = string.strlen(temp_dir)
|
||||
deps_path_w: str = StubMergerCollect._sliced_path(temp_dir, td_len_w, sha1_r2, "deps.txt")
|
||||
if deps_path_w is not None:
|
||||
df_w: fileio.File | t.CPtr = fileio.File(deps_path_w, fileio.MODE.W)
|
||||
if not df_w.closed:
|
||||
dl_w: t.CSizeT = string.strlen(deps_r2)
|
||||
if dl_w > 0:
|
||||
df_w.write(deps_r2, dl_w)
|
||||
df_w.close()
|
||||
stdlib.free(deps_path_w)
|
||||
# 追加依赖到 worklist（去重防溢出）
|
||||
ret_w: int = _AppendDepsToWorklist(worklist, wl_len, deps_r2, processed)
|
||||
if ret_w < 0:
|
||||
stdlib.free(abs_path)
|
||||
stdlib.free(rel_path)
|
||||
stdlib.free(inc_dir)
|
||||
stdlib.free(out_abs)
|
||||
stdlib.free(out_rel)
|
||||
stdlib.free(out_inc)
|
||||
stdlib.free(sbuf_r)
|
||||
stdlib.free(mod_buf)
|
||||
stdlib.free(worklist)
|
||||
stdlib.free(processed)
|
||||
stdlib.free(wl_pos)
|
||||
mb.reset_to(scan_mark_r)
|
||||
return -1
|
||||
wl_len = t.CSizeT(ret_w)
|
||||
else:
|
||||
VLogger.debug(f"无依赖: {rel_path}", "Reachable")
|
||||
|
||||
# 释放 Translator 的 C malloc 资源 + 回收本文件 AST
|
||||
if tr_r2._global_names is not None:
|
||||
stdlib.free(tr_r2._global_names)
|
||||
if tr_r2._nonlocal_names is not None:
|
||||
stdlib.free(tr_r2._nonlocal_names)
|
||||
stdlib.free(sbuf_r)
|
||||
mb.reset_to(scan_mark_r)
|
||||
|
||||
stdlib.free(abs_path)
|
||||
stdlib.free(rel_path)
|
||||
stdlib.free(inc_dir)
|
||||
stdlib.free(out_abs)
|
||||
stdlib.free(out_rel)
|
||||
stdlib.free(out_inc)
|
||||
stdlib.free(mod_buf)
|
||||
stdlib.free(sha1_arr)
|
||||
stdlib.free(mod_arr)
|
||||
|
||||
stdlib.free(worklist)
|
||||
stdlib.free(processed)
|
||||
stdlib.free(wl_pos)
|
||||
|
||||
VLogger.debug(f"可达 SHA1: {reachable_count} 个", "Reachable")
|
||||
return reachable_count
|
||||
|
||||
# ============================================================
|
||||
# BuildCombinedIR - 组合本地 stub + 依赖 stubs + 本地 text → 完整 IR
|
||||
#
|
||||
@@ -1512,6 +1719,12 @@ def LoadDepVpiIntoModule(pool: memhub.MemBuddy | t.CPtr,
|
||||
if imported_modules is None:
|
||||
return 0
|
||||
|
||||
# 每个模块的 stub 注入前重置 typedef 递归注册的防环标记：
|
||||
# CTypedef 表每文件隔离（translate 前 clear），visited 若跨模块残留会
|
||||
# 导致 stdint 等 typedef 只注册一次、后续模块注入再次解析时表里为空
|
||||
# → 参数类型退化 i8_ptr_ty → 整数实参误报 CastError（FuncTest 根因）。
|
||||
VpiReader.ResetVpiTypedefVisited()
|
||||
|
||||
td_len: t.CSizeT = string.strlen(temp_dir)
|
||||
total_len: t.CSizeT = string.strlen(imported_modules)
|
||||
success_count: int = 0
|
||||
|
||||
Reference in new issue
Block a user