按 wiki/新规范: - T | t.CPtr → T*(无 | 标准) - t.CInt/t.CUInt64T/t.CChar/t.CDouble 等 → int/uint64/char/double 等短别名 - X: t.CDefine = Y → define X = Y - c.Asm → __asm__,c.AsmInp/c.AsmOut → asminp/asmout,t.ASM_DESCR. → ASM_DESCR. - c.Addr(x) → &x,c.Deref(p) → *p,c.DerefAs(p, v) → *p = v 种子自举通过(96s)。
695 lines
33 KiB
Plaintext
695 lines
33 KiB
Plaintext
import lib.core.Gen.GenApi as GenApi
|
||
import t, c
|
||
from stdint import *
|
||
import memhub
|
||
import string
|
||
import stdlib
|
||
import mlirlite
|
||
import w32.win32file as win32file
|
||
import w32.win32base as win32base
|
||
import w32.fileio as fileio
|
||
import viperlib
|
||
import sys
|
||
import ast
|
||
import lib.core.VLogger as VLogger
|
||
import lib.core.IncludesScanner as IncludesScanner
|
||
import lib.core.Gen.Translator as HandlesTranslator
|
||
import lib.Phase.BuildPipeline as BuildPipeline
|
||
import lib.Stub.StubGen.VpiReader as VpiReader
|
||
import lib.Projectrans.Utils as Utils
|
||
import lib.Projectrans.Config as Config
|
||
|
||
import lib.Stub.StubMerger.StubMergerCollect as StubMergerCollect
|
||
import lib.Stub.StubMerger.StubMergerStore as StubMergerStore
|
||
import lib.Stub.StubMerger.StubMergerTopo as StubMergerTopo
|
||
import lib.Stub.StubMerger.StubMergerIR as StubMergerIR
|
||
import lib.Stub.StubMerger as StubMerger
|
||
|
||
# ============================================================
|
||
# _IsAppModule - 模块名是否为 App 源文件(store 条目 incdir 为空)
|
||
#
|
||
# 依赖图 BFS 中识别 App 中间节点:App 文件不入 reachable_set(Phase B
|
||
# 已编译其 .obj),但其 includes 依赖必须继续展开。App 条目由 App 扫描
|
||
# 阶段用 AppendToSha1MapStore 登记(incdir 留空);includes 条目用
|
||
# AddIncludeToSha1MapStore 登记(incdir 非空)。
|
||
# ============================================================
|
||
def _IsAppModule(mod_name: str) -> int:
|
||
"""mod_name 是否为 App 源文件(store 命中且 incdir 为空)"""
|
||
if mod_name is None:
|
||
return 0
|
||
if StubMerger._sha1_store_arr is None or StubMerger._sha1_store_mod is None:
|
||
return 0
|
||
for i in range(StubMerger._sha1_store_count):
|
||
midx: sizet = sizet(i) * 64
|
||
if string.strcmp(StubMerger._sha1_store_mod + midx, mod_name) == 0:
|
||
iidx: sizet = sizet(i) * StubMerger.MAX_INC_DIR_LEN
|
||
if StubMerger._sha1_store_incdir[iidx] == '\0':
|
||
return 1
|
||
return 0
|
||
return 0
|
||
# ============================================================
|
||
# _AppendDepsToWorklist - 追加空格分隔模块名到 worklist(逐模块去重)
|
||
#
|
||
# worklist 是 BFS 队列(空格分隔);deps_str 中的模块若已在 worklist/processed 中则不重复追加,
|
||
# 否则 BFS 沿共享依赖(builtins/stdlib 等)反复累积会致 REACH_WORKLIST_BUF 溢出。
|
||
#
|
||
# Args: worklist 队列缓冲(原地修改);wl_len 当前长度(值);deps_str 要追加的模块名(空格分隔);processed 已处理集合(空格分隔,查重)
|
||
# Returns: 追加后的新长度(>=0);溢出/参数错误返回 -1(调用方 fail-fast)
|
||
# ============================================================
|
||
def _AppendDepsToWorklist(worklist: bytes, wl_len: sizet,
|
||
deps_str: str, processed: str) -> int:
|
||
if worklist is None or deps_str is None:
|
||
return -1
|
||
cur_len: sizet = wl_len
|
||
dl: sizet = string.strlen(deps_str)
|
||
dpos: sizet = 0
|
||
while dpos < dl:
|
||
dstart: sizet = dpos
|
||
while dpos < dl and deps_str[dpos] != ' ':
|
||
dpos += 1
|
||
dlen: sizet = dpos - dstart
|
||
if dpos < dl:
|
||
dpos += 1
|
||
if dlen == 0:
|
||
continue
|
||
dmod: bytes = stdlib.malloc(dlen + 1)
|
||
if dmod is None:
|
||
continue
|
||
string.strncpy(dmod, deps_str + dstart, dlen)
|
||
dmod[dlen] = '\0'
|
||
in_wl: int = 0
|
||
if cur_len > 0:
|
||
if GenApi.is_module_imported(worklist, dmod) != 0:
|
||
in_wl = 1
|
||
if in_wl == 0 and processed is not None:
|
||
if GenApi.is_module_imported(processed, dmod) != 0:
|
||
in_wl = 1
|
||
if in_wl == 0:
|
||
if cur_len + dlen + 2 < StubMerger.REACH_WORKLIST_BUF:
|
||
if cur_len > 0 and worklist[cur_len - 1] != ' ':
|
||
worklist[cur_len] = ' '
|
||
cur_len = cur_len + 1
|
||
string.strcpy(worklist + cur_len, dmod)
|
||
cur_len = cur_len + dlen
|
||
worklist[cur_len] = ' '
|
||
cur_len = cur_len + 1
|
||
worklist[cur_len] = '\0'
|
||
else:
|
||
stdlib.free(dmod)
|
||
VLogger.error("依赖 worklist 溢出", "StubMerger")
|
||
return -1
|
||
stdlib.free(dmod)
|
||
return int(cur_len)
|
||
|
||
# ============================================================
|
||
# _BuildReachableSha1Set - 构建可达 SHA1 集合(依赖图按需扫描;未引用的 includes 从不读取)
|
||
#
|
||
# App 优先模型:从 app_deps 出发 BFS,对每个模块名按文件系统解析到 includes 文件(ResolveModuleToPath),只对可达 includes
|
||
# 读文件 + 算 SHA1 + 解析 import,补充依赖图并登记 Sha1Store/deps 存储器/写 deps.txt;Phase 1b / Phase B+ 只处理可达集合中的文件。
|
||
#
|
||
# 算法(工作列表 BFS):1) App 扫描(app_deps 为空时)递归扫 source_dir,declare_only=2 解析 import(相对导入 resolve 成绝对模块名)
|
||
# + 容器注入(_list/_dict)→ worklist 初始队列,App 文件登记 store(incdir 空)+ 填 deps;2) builtins 预加入队列;3) BFS 直到闭包。
|
||
#
|
||
# Args: mb 内存池;source_dir App 源文件目录(Config.SourceDir);temp_dir 临时目录(deps.txt 所在位置);reachable_set 输出参数,可达 SHA1 集合缓冲区(MAX_INCLUDES_SHA1 * 17 字节);app_deps App 源文件直接依赖模块名(空格分隔),非空则跳过 App 扫描;includes_dirs includes 根目录(空格分隔)
|
||
# Returns: 可达 SHA1 数量(>0 成功;0 无 includes 依赖;<0 失败)
|
||
# ============================================================
|
||
def _BuildReachableSha1Set(mb: memhub.MemBuddy*, source_dir: str,
|
||
temp_dir: str,
|
||
reachable_set: str,
|
||
app_deps: str = None,
|
||
includes_dirs: str = None) -> int:
|
||
"""构建可达 SHA1 集合(依赖图按需扫描,App 优先)"""
|
||
if source_dir is None or temp_dir is None or reachable_set is None:
|
||
return -1
|
||
if includes_dirs is None:
|
||
VLogger.error("includes_dirs 缺失,无法按需解析依赖图", "Reachable")
|
||
return -1
|
||
|
||
SRC_BUF_SIZE_R: sizet = 1048576
|
||
|
||
# 工作列表(空格分隔的模块名)
|
||
worklist: bytes = stdlib.malloc(StubMerger.REACH_WORKLIST_BUF)
|
||
if worklist is None:
|
||
return -1
|
||
worklist[0] = '\0'
|
||
wl_len: sizet = 0
|
||
|
||
# 1. App 扫描(app_deps 非空时跳过,直接用调用方提供的初始队列)
|
||
if app_deps is None or app_deps[0] == '\0':
|
||
VLogger.info("App 扫描:解析 source_dir 的 import 依赖", "Reachable")
|
||
scan_res_r: rawptr[IncludesScanner.ScanResult] = IncludesScanner.scan_includes(mb, source_dir)
|
||
if scan_res_r is not None:
|
||
es_r: sizet = IncludesScanner.FileEntry.__sizeof__()
|
||
for e_i_r in range(scan_res_r.Count):
|
||
e_r: rawptr[IncludesScanner.FileEntry] = rawptr[IncludesScanner.FileEntry](uint64(scan_res_r.Entries) + uint64(e_i_r) * es_r)
|
||
if e_r is None or e_r.Path is None:
|
||
continue
|
||
# 文件级 mark:parse 完立即回收 AST(App 全量 parse 驻留会 OOM)
|
||
app_mark: sizet = mb.mark()
|
||
sbuf: bytes = Utils.read_file_to_buf(e_r.Path, SRC_BUF_SIZE_R)
|
||
if sbuf is None:
|
||
mb.reset_to(app_mark)
|
||
continue
|
||
lx: ast.Lexer* = ast.new_lexer(mb)
|
||
tree: ast.AST* = None
|
||
if lx is not None:
|
||
ast._lexer_init(lx, sbuf, mb)
|
||
tokens: ast.Token* = ast.tokenize(lx)
|
||
tree = ast.parse_tokens(mb, tokens)
|
||
if tree is not None:
|
||
tr: rawptr[HandlesTranslator.Translator] = HandlesTranslator.Translator()
|
||
if tr is not None:
|
||
tr._declare_only = 2
|
||
# 源文件包名:用 RelPath(相对 source_dir,支持子目录),
|
||
# 保证 from .xxx import 相对导入解析正确
|
||
tr.CurrentPackage = GenApi.compute_package_from_relpath(mb, e_r.RelPath)
|
||
GenApi.set_current_file_type(e_r.Path)
|
||
GenApi.set_current_file_imports(e_r.Path)
|
||
GenApi.set_current_module_sha1(e_r.Sha1)
|
||
GenApi.clear_cdefine_constants()
|
||
GenApi.clear_ctypedefs()
|
||
GenApi.reset_visible_structs(mb, 0)
|
||
ret_a: int = tr.translate(tree)
|
||
if ret_a == 0:
|
||
# 容器使用检测:App 文件用 list/dict(裸名注解/构造)
|
||
# 时注入 _list/_dict 依赖(对齐原实现)
|
||
tr._inject_auto_container_visibility(mb, tree)
|
||
# App 文件登记 store(incdir 空)+ 填 deps:
|
||
# BFS 展开 App 中间节点的 includes 依赖时经
|
||
# LookupIncludesDeps(app_sha1) 读取
|
||
if e_r.Sha1 is not None:
|
||
StubMergerStore.AppendToSha1MapStore(e_r.Sha1, e_r.RelPath)
|
||
StubMergerStore.PopulateIncludesDeps(e_r.Sha1, tr._imported_modules)
|
||
# 收集 deps 到初始队列(逐模块去重)
|
||
if tr._imported_modules is not None:
|
||
im_len_a: sizet = string.strlen(tr._imported_modules)
|
||
im_pos_a: sizet = 0
|
||
while im_pos_a < im_len_a:
|
||
im_start_a: sizet = im_pos_a
|
||
while im_pos_a < im_len_a and tr._imported_modules[im_pos_a] != ' ':
|
||
im_pos_a += 1
|
||
im_mod_len: sizet = im_pos_a - im_start_a
|
||
if im_pos_a < im_len_a:
|
||
im_pos_a += 1
|
||
if im_mod_len == 0:
|
||
continue
|
||
im_buf_a: bytes = stdlib.malloc(im_mod_len + 1)
|
||
if im_buf_a is None:
|
||
continue
|
||
string.strncpy(im_buf_a, tr._imported_modules + im_start_a, im_mod_len)
|
||
im_buf_a[im_mod_len] = '\0'
|
||
in_wl_a: int = 0
|
||
if wl_len > 0 and GenApi.is_module_imported(worklist, im_buf_a) != 0:
|
||
in_wl_a = 1
|
||
if in_wl_a == 0:
|
||
if wl_len + im_mod_len + 2 < StubMerger.REACH_WORKLIST_BUF:
|
||
if wl_len > 0:
|
||
worklist[wl_len] = ' '
|
||
wl_len += 1
|
||
string.strcpy(worklist + wl_len, im_buf_a)
|
||
wl_len += im_mod_len
|
||
worklist[wl_len] = ' '
|
||
wl_len += 1
|
||
worklist[wl_len] = '\0'
|
||
stdlib.free(im_buf_a)
|
||
# 释放 Translator 资源
|
||
if tr._global_names is not None:
|
||
stdlib.free(tr._global_names)
|
||
if tr._nonlocal_names is not None:
|
||
stdlib.free(tr._nonlocal_names)
|
||
stdlib.free(sbuf)
|
||
mb.reset_to(app_mark)
|
||
else:
|
||
# app_deps 已提供(Phase2 场景):直接作为初始队列
|
||
ad_len: sizet = string.strlen(app_deps)
|
||
if ad_len > 0 and ad_len < StubMerger.REACH_WORKLIST_BUF:
|
||
string.strcpy(worklist, app_deps)
|
||
wl_len = ad_len
|
||
else:
|
||
VLogger.error("app_deps 过长,无法构建可达集合", "StubMerger")
|
||
stdlib.free(worklist)
|
||
return -1
|
||
|
||
# 去掉末尾多余空格
|
||
if wl_len > 0 and worklist[wl_len - 1] == ' ':
|
||
worklist[wl_len - 1] = '\0'
|
||
wl_len -= 1
|
||
|
||
# 2. 预加入基础模块(builtins)
|
||
# builtins(内置函数模块)始终纳入编译链。_list/_dict 仅在显式
|
||
# list/dict 注解/构造时经 _inject_auto_container_visibility 注入
|
||
# (list 字面量赋给 t.CArray 走数组初始化,不触发 _list)。
|
||
# processed 尚未分配,此处传 None 仅按 worklist 查重。
|
||
wl_len = _AppendDepsToWorklist(worklist, wl_len, "builtins", None)
|
||
if wl_len < 0:
|
||
stdlib.free(worklist)
|
||
return -1
|
||
|
||
# 2b. --traceback: force the traceback runtime module into the reachable set
|
||
# (TPC: used_includes.add('traceback')). Its Install() symbol and the
|
||
# @llvm.global_ctors ctor are only produced by compiling this module, so a
|
||
# missing file is fail-fast rather than a silent "feature off" build.
|
||
if BuildPipeline._traceback_mode != 0:
|
||
tb_abs: t.CPtr = stdlib.malloc(8)
|
||
tb_rel: t.CPtr = stdlib.malloc(8)
|
||
tb_inc: t.CPtr = stdlib.malloc(8)
|
||
tb_found: int = -1
|
||
if tb_abs is not None and tb_rel is not None and tb_inc is not None:
|
||
tb_abs_slot: sizet* = (sizet*)(tb_abs)
|
||
tb_rel_slot: sizet* = (sizet*)(tb_rel)
|
||
tb_inc_slot: sizet* = (sizet*)(tb_inc)
|
||
tb_abs_slot[0] = 0
|
||
tb_rel_slot[0] = 0
|
||
tb_inc_slot[0] = 0
|
||
tb_found = IncludesScanner.ResolveModuleToPath(
|
||
includes_dirs, "traceback", tb_abs, tb_rel, tb_inc)
|
||
if tb_abs is not None:
|
||
stdlib.free(tb_abs)
|
||
if tb_rel is not None:
|
||
stdlib.free(tb_rel)
|
||
if tb_inc is not None:
|
||
stdlib.free(tb_inc)
|
||
if tb_found != 0:
|
||
stdlib.free(worklist)
|
||
VLogger.error(f"--traceback: 在 includes 目录中找不到 traceback.vp,无法编译并链接 Install()(请确认 project.vpj 的 includes 包含仓库 includes 目录;已扫描: {includes_dirs})", "Traceback")
|
||
return -1
|
||
tb_wl: int = _AppendDepsToWorklist(worklist, wl_len, "traceback", None)
|
||
if tb_wl < 0:
|
||
stdlib.free(worklist)
|
||
return -1
|
||
wl_len = sizet(tb_wl)
|
||
|
||
# 3. 工作列表 BFS:模块名 → 文件系统解析 → 读+hash+parse import
|
||
processed: bytes = stdlib.malloc(StubMerger.REACH_WORKLIST_BUF)
|
||
if processed is None:
|
||
stdlib.free(worklist)
|
||
return -1
|
||
processed[0] = '\0'
|
||
proc_len: sizet = 0
|
||
reachable_count: int = 0
|
||
wl_pos: t.CPtr = stdlib.malloc(8)
|
||
if wl_pos is None:
|
||
stdlib.free(worklist)
|
||
stdlib.free(processed)
|
||
return -1
|
||
wl_pos_slot: sizet* = (sizet*)(wl_pos)
|
||
wl_pos_slot[0] = 0
|
||
|
||
while wl_pos_slot[0] < wl_len:
|
||
pos: sizet = wl_pos_slot[0]
|
||
# 提取一个模块名(到空格或末尾)
|
||
name_start: sizet = pos
|
||
while pos < wl_len and worklist[pos] != ' ':
|
||
pos += 1
|
||
name_len: sizet = pos - name_start
|
||
if pos < wl_len:
|
||
pos += 1
|
||
wl_pos_slot[0] = pos
|
||
if name_len == 0:
|
||
continue
|
||
|
||
# 复制模块名
|
||
mod_buf: bytes = stdlib.malloc(name_len + 1)
|
||
if mod_buf is None:
|
||
continue
|
||
string.strncpy(mod_buf, worklist + name_start, name_len)
|
||
mod_buf[name_len] = '\0'
|
||
|
||
# 已处理去重
|
||
if GenApi.is_module_imported(processed, mod_buf) != 0:
|
||
stdlib.free(mod_buf)
|
||
continue
|
||
# 加入 processed
|
||
if proc_len + name_len + 1 < StubMerger.REACH_WORKLIST_BUF:
|
||
string.strcpy(processed + proc_len, mod_buf)
|
||
proc_len = proc_len + name_len
|
||
processed[proc_len] = ' '
|
||
proc_len = proc_len + 1
|
||
processed[proc_len] = '\0'
|
||
else:
|
||
VLogger.error("依赖 processed 溢出", "StubMerger")
|
||
stdlib.free(mod_buf)
|
||
continue
|
||
|
||
# App 中间节点:只展开其 includes 依赖,不入 reachable_set
|
||
# (Phase B 已编译 App 的 .obj,加入会导致 Phase B+ 重复编译 → 重复符号)
|
||
if _IsAppModule(mod_buf) != 0:
|
||
app_sha1: str = StubMergerStore.LookupSha1ByModName(mod_buf)
|
||
if app_sha1 is not None:
|
||
app_deps_str: str = StubMergerStore.LookupIncludesDeps(app_sha1)
|
||
if app_deps_str is not None:
|
||
ret_ap: int = _AppendDepsToWorklist(worklist, wl_len, app_deps_str, processed)
|
||
if ret_ap < 0:
|
||
stdlib.free(mod_buf)
|
||
stdlib.free(worklist)
|
||
stdlib.free(processed)
|
||
stdlib.free(wl_pos)
|
||
return -1
|
||
wl_len = sizet(ret_ap)
|
||
stdlib.free(mod_buf)
|
||
continue
|
||
|
||
# includes:按文件系统按需解析(未引用的 includes 根本不触碰)
|
||
out_abs: t.CPtr = stdlib.malloc(8)
|
||
out_rel: t.CPtr = stdlib.malloc(8)
|
||
out_inc: t.CPtr = stdlib.malloc(8)
|
||
if out_abs is None or out_rel is None or out_inc is None:
|
||
if out_abs is not None:
|
||
stdlib.free(out_abs)
|
||
if out_rel is not None:
|
||
stdlib.free(out_rel)
|
||
if out_inc is not None:
|
||
stdlib.free(out_inc)
|
||
stdlib.free(mod_buf)
|
||
continue
|
||
# 输出槽防御性初始化(成功路径必全写;未找到/未命中时仅 free 槽本身)
|
||
# 槽联合必须用 (sizet*):t.CPtr* 会被折叠成 t.CPtr,
|
||
# [0] 变成 1 字节访问 → 指针被截断(0x2e 崩溃根因)。CSizeT 成员为
|
||
# 8 字节:写侧 sizet(ptr),读侧必须用 rawptr(slot[0]) 做 u64→
|
||
# 指针转换;str(slot[0]) 是内置"数字转字符串"(snprintf 到栈缓冲并返回
|
||
# 该栈地址),拿到的是十进制文本+栈指针 → free 即 bad-free。
|
||
abs_slot: sizet* = (sizet*)(out_abs)
|
||
rel_slot: sizet* = (sizet*)(out_rel)
|
||
inc_slot: sizet* = (sizet*)(out_inc)
|
||
abs_slot[0] = 0
|
||
rel_slot[0] = 0
|
||
inc_slot[0] = 0
|
||
rcode: int = IncludesScanner.ResolveModuleToPath(
|
||
includes_dirs, mod_buf, out_abs, out_rel, out_inc)
|
||
if rcode != 0:
|
||
# 内建模块(t/c/builtins 等内部库)或未找到:静默跳过
|
||
stdlib.free(out_abs)
|
||
stdlib.free(out_rel)
|
||
stdlib.free(out_inc)
|
||
stdlib.free(mod_buf)
|
||
continue
|
||
abs_path: str = rawptr(abs_slot[0])
|
||
rel_path: str = rawptr(rel_slot[0])
|
||
inc_dir: str = rawptr(inc_slot[0])
|
||
|
||
# 读文件 + 计算 SHA1(CRLF→LF,与 IncludesScanner 一致)
|
||
sha1_r2: str = IncludesScanner.compute_file_sha1(mb, abs_path)
|
||
if sha1_r2 is None:
|
||
stdlib.free(abs_path)
|
||
stdlib.free(rel_path)
|
||
stdlib.free(inc_dir)
|
||
stdlib.free(out_abs)
|
||
stdlib.free(out_rel)
|
||
stdlib.free(out_inc)
|
||
stdlib.free(mod_buf)
|
||
continue
|
||
|
||
# 已在可达集合(前序 BFS 已 parse 过):只展开其 deps
|
||
if StubMergerCollect._is_in_sha1_set(sha1_r2, reachable_set, reachable_count) != 0:
|
||
deps_ex: str = StubMergerStore.LookupIncludesDeps(sha1_r2)
|
||
if deps_ex is not None:
|
||
ret_ex: int = _AppendDepsToWorklist(worklist, wl_len, deps_ex, processed)
|
||
if ret_ex < 0:
|
||
stdlib.free(abs_path)
|
||
stdlib.free(rel_path)
|
||
stdlib.free(inc_dir)
|
||
stdlib.free(out_abs)
|
||
stdlib.free(out_rel)
|
||
stdlib.free(out_inc)
|
||
stdlib.free(mod_buf)
|
||
stdlib.free(worklist)
|
||
stdlib.free(processed)
|
||
stdlib.free(wl_pos)
|
||
return -1
|
||
wl_len = sizet(ret_ex)
|
||
stdlib.free(abs_path)
|
||
stdlib.free(rel_path)
|
||
stdlib.free(inc_dir)
|
||
stdlib.free(out_abs)
|
||
stdlib.free(out_rel)
|
||
stdlib.free(out_inc)
|
||
stdlib.free(mod_buf)
|
||
continue
|
||
|
||
# 加入可达集合 + 登记 Sha1Store(仅可达 includes)
|
||
reachable_count = StubMergerIR._AddSha1ToSet(reachable_set, reachable_count, sha1_r2)
|
||
StubMergerStore.AddIncludeToSha1MapStore(sha1_r2, rel_path, inc_dir, mod_buf)
|
||
|
||
# 文件级 mark:parse import 后立即回收 AST(可达 includes 全量
|
||
# parse 的 AST 若驻留 → 主池 OOM)
|
||
scan_mark_r: sizet = mb.mark()
|
||
sbuf_r: bytes = Utils.read_file_to_buf(abs_path, SRC_BUF_SIZE_R)
|
||
if sbuf_r is None:
|
||
stdlib.free(abs_path)
|
||
stdlib.free(rel_path)
|
||
stdlib.free(inc_dir)
|
||
stdlib.free(out_abs)
|
||
stdlib.free(out_rel)
|
||
stdlib.free(out_inc)
|
||
stdlib.free(mod_buf)
|
||
mb.reset_to(scan_mark_r)
|
||
continue
|
||
lx_r: ast.Lexer* = ast.new_lexer(mb)
|
||
tree_r: ast.AST* = None
|
||
if lx_r is not None:
|
||
ast._lexer_init(lx_r, sbuf_r, mb)
|
||
tokens_r: ast.Token* = ast.tokenize(lx_r)
|
||
tree_r = ast.parse_tokens(mb, tokens_r)
|
||
if tree_r is None:
|
||
# parse_tokens keeps the error detail (line/col/message) in the global
|
||
# channel only, so print the unified parse traceback before the one-line
|
||
# log; without it a reachable-includes syntax error is undiagnosable.
|
||
GenApi.dump_parse_traceback(abs_path)
|
||
VLogger.error(f"可达 includes 解析失败: {abs_path}", "Reachable")
|
||
stdlib.free(abs_path)
|
||
stdlib.free(rel_path)
|
||
stdlib.free(inc_dir)
|
||
stdlib.free(out_abs)
|
||
stdlib.free(out_rel)
|
||
stdlib.free(out_inc)
|
||
stdlib.free(sbuf_r)
|
||
stdlib.free(mod_buf)
|
||
stdlib.free(worklist)
|
||
stdlib.free(processed)
|
||
stdlib.free(wl_pos)
|
||
mb.reset_to(scan_mark_r)
|
||
return -1
|
||
tr_r2: rawptr[HandlesTranslator.Translator] = HandlesTranslator.Translator()
|
||
if tr_r2 is None:
|
||
stdlib.free(abs_path)
|
||
stdlib.free(rel_path)
|
||
stdlib.free(inc_dir)
|
||
stdlib.free(out_abs)
|
||
stdlib.free(out_rel)
|
||
stdlib.free(out_inc)
|
||
stdlib.free(sbuf_r)
|
||
stdlib.free(mod_buf)
|
||
mb.reset_to(scan_mark_r)
|
||
continue
|
||
tr_r2.ModuleSha1 = sha1_r2
|
||
tr_r2._declare_only = 2
|
||
# 包名:相对 includes 根目录的 RelPath(支持 from .xxx import 相对导入)
|
||
tr_r2.CurrentPackage = GenApi.compute_package_from_relpath(mb, rel_path)
|
||
GenApi.set_current_file_type(abs_path)
|
||
GenApi.set_current_file_imports(abs_path)
|
||
GenApi.set_current_module_sha1(sha1_r2)
|
||
GenApi.clear_cdefine_constants()
|
||
GenApi.clear_ctypedefs()
|
||
GenApi.reset_visible_structs(mb, 0)
|
||
ret_r2: int = tr_r2.translate(tree_r)
|
||
if ret_r2 != 0:
|
||
VLogger.error(f"import 依赖扫描失败(可达 includes): {abs_path}", "Reachable")
|
||
stdlib.free(abs_path)
|
||
stdlib.free(rel_path)
|
||
stdlib.free(inc_dir)
|
||
stdlib.free(out_abs)
|
||
stdlib.free(out_rel)
|
||
stdlib.free(out_inc)
|
||
stdlib.free(sbuf_r)
|
||
if tr_r2._global_names is not None:
|
||
stdlib.free(tr_r2._global_names)
|
||
if tr_r2._nonlocal_names is not None:
|
||
stdlib.free(tr_r2._nonlocal_names)
|
||
stdlib.free(mod_buf)
|
||
stdlib.free(worklist)
|
||
stdlib.free(processed)
|
||
stdlib.free(wl_pos)
|
||
mb.reset_to(scan_mark_r)
|
||
return -1
|
||
# 容器按需注入(includes 用 list/dict 泛型时追加 _list/_dict 到依赖,
|
||
# 对齐 App 层 _inject_auto_container_visibility;内联破环防 _list/_dict 环)
|
||
tr_r2._inject_auto_container_visibility(mb, tree_r)
|
||
|
||
# 填内存依赖 + 写 deps.txt(供依赖图展开与 Phase B 按需加载)
|
||
deps_r2: str = tr_r2._imported_modules
|
||
StubMergerStore.PopulateIncludesDeps(sha1_r2, deps_r2)
|
||
if deps_r2 is not None:
|
||
td_len_w: sizet = string.strlen(temp_dir)
|
||
deps_path_w: str = StubMergerCollect._sliced_path(temp_dir, td_len_w, sha1_r2, "deps.txt")
|
||
if deps_path_w is not None:
|
||
df_w: rawptr[fileio.File] = fileio.File(deps_path_w, fileio.MODE.W)
|
||
if not df_w.closed:
|
||
dl_w: sizet = string.strlen(deps_r2)
|
||
if dl_w > 0:
|
||
df_w.write(deps_r2, dl_w)
|
||
df_w.close()
|
||
stdlib.free(deps_path_w)
|
||
# 追加依赖到 worklist(去重防溢出)
|
||
ret_w: int = _AppendDepsToWorklist(worklist, wl_len, deps_r2, processed)
|
||
if ret_w < 0:
|
||
stdlib.free(abs_path)
|
||
stdlib.free(rel_path)
|
||
stdlib.free(inc_dir)
|
||
stdlib.free(out_abs)
|
||
stdlib.free(out_rel)
|
||
stdlib.free(out_inc)
|
||
stdlib.free(sbuf_r)
|
||
stdlib.free(mod_buf)
|
||
stdlib.free(worklist)
|
||
stdlib.free(processed)
|
||
stdlib.free(wl_pos)
|
||
mb.reset_to(scan_mark_r)
|
||
return -1
|
||
wl_len = sizet(ret_w)
|
||
else:
|
||
VLogger.debug(f"无依赖: {rel_path}", "Reachable")
|
||
|
||
# 释放 Translator 的 C malloc 资源 + 回收本文件 AST
|
||
if tr_r2._global_names is not None:
|
||
stdlib.free(tr_r2._global_names)
|
||
if tr_r2._nonlocal_names is not None:
|
||
stdlib.free(tr_r2._nonlocal_names)
|
||
stdlib.free(sbuf_r)
|
||
mb.reset_to(scan_mark_r)
|
||
|
||
stdlib.free(abs_path)
|
||
stdlib.free(rel_path)
|
||
stdlib.free(inc_dir)
|
||
stdlib.free(out_abs)
|
||
stdlib.free(out_rel)
|
||
stdlib.free(out_inc)
|
||
stdlib.free(mod_buf)
|
||
|
||
stdlib.free(worklist)
|
||
stdlib.free(processed)
|
||
stdlib.free(wl_pos)
|
||
return reachable_count
|
||
|
||
# ============================================================
|
||
# LoadFullIR - 读 {temp_dir}/{sha1}.full.ll 到 out_buf(④-c:合并期改用在线 FULL 产物)
|
||
#
|
||
# 契约:返回写入长度(sizet;0=失败),out_buf 以 '\0' 结尾。
|
||
# full.ll 由翻译点在模型在线时 dump_ir(OUTPUT_FULL) 产出 —— 自足的单模块 IR
|
||
# (类型定义 + external 声明 + 定义 + DI 一次齐全,类型定义天然在声明之前),
|
||
# 无需再把 stub + 依赖 stub/text 文本合并。
|
||
# ============================================================
|
||
def LoadFullIR(temp_dir: str, sha1: str,
|
||
out_buf: bytes, out_size: sizet) -> sizet:
|
||
"""读取 {sha1}.full.ll 到 out_buf;返回长度(0=失败)"""
|
||
if temp_dir is None or sha1 is None or out_buf is None or out_size == 0:
|
||
return 0
|
||
td_len: sizet = string.strlen(temp_dir)
|
||
p: str = StubMergerCollect._sliced_path(temp_dir, td_len, sha1, "full.ll")
|
||
if p is None:
|
||
return 0
|
||
f: rawptr[fileio.File] = fileio.File(p, fileio.MODE.R)
|
||
stdlib.free(p)
|
||
out_buf[0] = '\0'
|
||
if f.closed:
|
||
return 0
|
||
if f.size() >= out_size:
|
||
f.close()
|
||
VLogger.error("full.ll 超过 IR 缓冲上限", "StubMerger")
|
||
return 0
|
||
rd: int64 = f.read_all(out_buf, out_size)
|
||
f.close()
|
||
if rd <= 0:
|
||
return 0
|
||
out_buf[rd] = '\0'
|
||
return sizet(rd)
|
||
|
||
# ============================================================
|
||
# LoadDepVpiIntoModule - 加载依赖模块的 .vpi 文件,注入 declare 到当前模块
|
||
#
|
||
# 遍历 imported_modules(空格分隔的模块名):LookupSha1ByModName 取 SHA1 → _sliced_path 构造 .vpi 路径 → VpiReader.ParseVpiIntoModule 解析并注入 declare。
|
||
#
|
||
# 必须在 _translate_module_level 中、Import 处理完成后、translate_children 之前调用,确保跨模块函数调用时 declare 已存在。
|
||
#
|
||
# Args: pool 内存池;mod 当前 LLVM 模块;imported_modules 已导入模块名字符串(空格分隔);from_imports from-import 映射字符串("name:module name:module" 格式);temp_dir temp 目录路径;trans 翻译器对象
|
||
# Returns: 成功加载的 .vpi 文件数
|
||
# ============================================================
|
||
def LoadDepVpiIntoModule(pool: memhub.MemBuddy*,
|
||
mod: mlirlite.LLVMModule*,
|
||
imported_modules: str,
|
||
from_imports: str,
|
||
temp_dir: str,
|
||
trans: rawptr[HandlesTranslator.Translator]) -> int:
|
||
"""加载依赖模块的 .vpi 文件,将函数声明注入到当前模块"""
|
||
if pool is None or mod is None or temp_dir is None or trans is None:
|
||
return 0
|
||
if imported_modules is None:
|
||
return 0
|
||
|
||
# 每个模块的 stub 注入前重置 typedef 递归注册的防环标记:
|
||
# CTypedef 表每文件隔离(translate 前 clear),visited 若跨模块残留会
|
||
# 导致 stdint 等 typedef 只注册一次、后续模块注入再次解析时表里为空
|
||
# → 参数类型退化 i8_ptr_ty → 整数实参误报 CastError(FuncTest 根因)。
|
||
VpiReader.ResetVpiTypedefVisited()
|
||
# 按需注入防重入表同周期重置:注入目标 mod 每模块不同,跨模块沿用会让
|
||
# 后续模块命中缓存、拿不到本模块需要的 declare(包重导出按需注入用)。
|
||
VpiReader.ResetVpiInjectVisited()
|
||
|
||
td_len: sizet = string.strlen(temp_dir)
|
||
total_len: sizet = string.strlen(imported_modules)
|
||
success_count: int = 0
|
||
|
||
# 遍历 imported_modules 中的每个模块名(空格分隔)
|
||
ci: sizet = 0
|
||
while ci < total_len:
|
||
# 跳过前导空格
|
||
while ci < total_len and imported_modules[ci] == ' ':
|
||
ci += 1
|
||
if ci >= total_len:
|
||
break
|
||
# 提取模块名(到下一个空格或字符串末尾)
|
||
word_start: sizet = ci
|
||
while ci < total_len and imported_modules[ci] != ' ':
|
||
ci += 1
|
||
word_len: sizet = ci - word_start
|
||
if word_len == 0:
|
||
continue
|
||
|
||
# 复制模块名到临时缓冲区并 null 终止
|
||
mod_nm_buf: str = stdlib.malloc(word_len + 1)
|
||
if mod_nm_buf is None:
|
||
continue
|
||
string.strncpy(mod_nm_buf, imported_modules + word_start, word_len)
|
||
mod_nm_buf[word_len] = '\0'
|
||
|
||
# 用 LookupSha1ByModName 查找模块的 SHA1
|
||
sha1_val: str = StubMergerStore.LookupSha1ByModName(mod_nm_buf)
|
||
|
||
# 改名别名(import X as Y,如 HT→lib.core.Handles.HandlesTranslator):
|
||
# 精确/包名回退均失败时从 from_imports 查别名映射再查 SHA1,
|
||
# 否则该类模块的 vpi/stub 不注入 → 跨模块符号解析失败
|
||
if sha1_val is None and from_imports is not None:
|
||
sha1_val = StubMergerStore.LookupSha1ByAlias(from_imports, mod_nm_buf)
|
||
|
||
# 如果精确查找失败,尝试包名回退(如 "ast" → "ast.__init__")
|
||
# LookupSha1ByModName 内部已处理包名回退,此处无需重复
|
||
|
||
if sha1_val is not None:
|
||
# 构造 .vpi 文件路径
|
||
vpi_path: str = StubMergerCollect._sliced_path(temp_dir, td_len, sha1_val, "vpi")
|
||
if vpi_path is not None:
|
||
# 检查 .vpi 文件是否存在(可读)
|
||
vf: rawptr[fileio.File] = fileio.File(vpi_path, fileio.MODE.R)
|
||
if not vf.closed:
|
||
vf.close()
|
||
# 调用 VpiReader 解析 .vpi 并注入 declare
|
||
# (错误上下文帧由调用方 HandlesTranslator 维护)
|
||
ret: int = VpiReader.ParseVpiIntoModule(pool, mod, vpi_path, trans)
|
||
if ret == 1:
|
||
success_count += 1
|
||
stdlib.free(vpi_path)
|
||
# sha1_val 指向内部存储,无需 free
|
||
|
||
stdlib.free(mod_nm_buf)
|
||
|
||
return success_count
|