feat(sidecar): 翻译点增写 {sha1}.full.ll,Phase B 改用在线 FULL 产物(④-b/④-c)
- 翻译点(Phase1Main / Phase2 PhaseA / Phase2 includes 重试)在 dump stub/text/meta 后
追加 dump_ir(OUTPUT_FULL) → {sha1}.full.ll(FULL 缓冲 16MB = stub+text 2× 余量)
- Phase1 增量跳转新增要求 full.ll 存在(缺失即重译补写)
- 新增 StubMergerBuild.LoadFullIR(与 BuildCombinedIR 同契约:读文件到 out_buf)
- Phase B 三处调用点 BuildCombinedIR → LoadFullIR;COMBINED_IR_SIZE 4MB → 16MB
- 诊断报文/注释同步(读 full.ll 失败)
full.ll 为模型在线时产出的自足单模块 IR(类型定义在声明之前,天然满足 llc 顺序);
BuildCombinedIR 暂保留(④-d 再删)。未编译验证。
This commit is contained in:
1 parent
ec9e950185
commit
cc8ea01476
3 files changed
+120
-19
No files matched your search
@@ -922,8 +922,8 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str,
|
||||
tf: rawptr[fileio.File] = fileio.File(text_path, fileio.MODE.R)
|
||||
if not tf.closed:
|
||||
tf.close()
|
||||
# sidecar({sha1}.meta)也必须存在:缺失则重新翻译以补写,
|
||||
# 否则 Phase B+/C 的「纯声明文件」判定拿不到结构化清单。
|
||||
# sidecar({sha1}.meta)与在线 FULL 产物({sha1}.full.ll)也必须存在:
|
||||
# 缺失则重新翻译以补写(否则 Phase B+/C 的声明判定与编译拿不到数据)。
|
||||
meta_path_1b: str = StubMergerCollect._sliced_path(temp_dir, dir_len, sha1, "meta")
|
||||
meta_ok_1b: int = 0
|
||||
if meta_path_1b is not None:
|
||||
@@ -932,7 +932,15 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str,
|
||||
mf_chk.close()
|
||||
meta_ok_1b = 1
|
||||
stdlib.free(meta_path_1b)
|
||||
if meta_ok_1b != 0:
|
||||
full_path_1b: str = StubMergerCollect._sliced_path(temp_dir, dir_len, sha1, "full.ll")
|
||||
full_ok_1b: int = 0
|
||||
if full_path_1b is not None:
|
||||
ff_chk: rawptr[fileio.File] = fileio.File(full_path_1b, fileio.MODE.R)
|
||||
if not ff_chk.closed:
|
||||
ff_chk.close()
|
||||
full_ok_1b = 1
|
||||
stdlib.free(full_path_1b)
|
||||
if meta_ok_1b != 0 and full_ok_1b != 0:
|
||||
stdlib.free(text_path)
|
||||
stdlib.free(stub_path)
|
||||
continue
|
||||
@@ -1093,7 +1101,9 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str,
|
||||
# dump stub IR(declarations only)
|
||||
# 1MB 不够:ast/parser.vp 的 text.ll 超 1MB 被截断 → 合并 IR 尾部丢字
|
||||
# (opt: "end of file in global variable name"),提至 8MB。
|
||||
PHASE1_IR_SIZE: sizet = 8388608
|
||||
PHASE1_IR_SIZE: sizet = 8388608
|
||||
# FULL dump = stub + text(同一模块),留 2× 余量
|
||||
PHASE1_FULL_SIZE: sizet = 16777216
|
||||
stub_buf: bytes = stdlib.malloc(PHASE1_IR_SIZE)
|
||||
if stub_buf is None:
|
||||
stdlib.free(src_buf)
|
||||
@@ -1143,6 +1153,22 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str,
|
||||
# save {sha1}.meta(结构化 sidecar:函数清单/结构体布局/虚表,替代从 IR 文本反推)
|
||||
StubMergerIR.WriteModuleMeta(tr.Module, temp_dir, dir_len_p1, sha1)
|
||||
|
||||
# save {sha1}.full.ll(在线 FULL 产物:模型在线时的自足单模块 IR,供 Phase B 直接编译,
|
||||
# 取代 stub + 依赖 stub/text 文本合并)
|
||||
full_buf_p1: bytes = stdlib.malloc(PHASE1_FULL_SIZE)
|
||||
if full_buf_p1 is None:
|
||||
VLogger.error("full.ll 缓冲区分配失败", "Phase1")
|
||||
tr.dump_ir(full_buf_p1, PHASE1_FULL_SIZE, llvmlite.OUTPUT_FULL)
|
||||
full_len_p1: sizet = string.strlen(full_buf_p1)
|
||||
full_path_p1: str = StubMergerCollect._sliced_path(temp_dir, dir_len_p1, sha1, "full.ll")
|
||||
if full_path_p1 is not None:
|
||||
ff_p1: rawptr[fileio.File] = fileio.File(full_path_p1, fileio.MODE.W)
|
||||
if not ff_p1.closed:
|
||||
ff_p1.write(full_buf_p1, full_len_p1)
|
||||
ff_p1.close()
|
||||
stdlib.free(full_path_p1)
|
||||
stdlib.free(full_buf_p1)
|
||||
|
||||
# 计时点 3:dump(stub+text)完成;输出单文件三段耗时
|
||||
t_f3: uint64 = w32.win32base.GetTickCount64()
|
||||
VLogger.info(f"[{i+1}/{loop_n_b}] 翻译: {file_path} (parse={t_f1-t_f0} ms, translate={t_f2-t_f1} ms, dump={t_f3-t_f2} ms, total={t_f3-t_f0} ms)", "Phase1")
|
||||
|
||||
@@ -965,6 +965,8 @@ def RunMultiFileProject(mb: memhub.MemBuddy*,
|
||||
# Phase A IR 缓冲:1MB 不够,ast/parser.vp 的 text.ll 超 1MB 被截断 →
|
||||
# 合并 IR 尾部丢字(opt: "end of file in global variable name")。
|
||||
PHASE_A_IR_SIZE: sizet = 8388608
|
||||
# FULL dump = stub + text(同一模块),留 2× 余量
|
||||
PHASE_A_FULL_SIZE: sizet = 16777216
|
||||
td_len_pa: sizet = string.strlen(temp_dir)
|
||||
|
||||
# 设置 temp 目录路径,使 HandlesExprCall 能读取依赖模块的 text.ll
|
||||
@@ -1088,6 +1090,21 @@ def RunMultiFileProject(mb: memhub.MemBuddy*,
|
||||
# save {sha1}.meta(结构化 sidecar:函数清单/结构体布局/虚表)
|
||||
StubMergerIR.WriteModuleMeta(tr_a.Module, temp_dir, td_len_pa, ent.Sha1)
|
||||
|
||||
# save {sha1}.full.ll(在线 FULL 产物:模型在线时的自足单模块 IR)
|
||||
full_buf_a: bytes = stdlib.malloc(PHASE_A_FULL_SIZE)
|
||||
if full_buf_a is None:
|
||||
VLogger.error("full.ll 缓冲区分配失败", "PhaseA")
|
||||
tr_a.dump_ir(full_buf_a, PHASE_A_FULL_SIZE, llvmlite.OUTPUT_FULL)
|
||||
full_len_a: sizet = string.strlen(full_buf_a)
|
||||
full_path_a: str = StubMergerCollect._sliced_path(temp_dir, td_len_pa, ent.Sha1, "full.ll")
|
||||
if full_path_a is not None:
|
||||
ffa: rawptr[fileio.File] = fileio.File(full_path_a, fileio.MODE.W)
|
||||
if not ffa.closed:
|
||||
ffa.write(full_buf_a, full_len_a)
|
||||
ffa.close()
|
||||
stdlib.free(full_path_a)
|
||||
stdlib.free(full_buf_a)
|
||||
|
||||
# save dependencies (_imported_modules) for Phase B (切片路径)
|
||||
deps_path_a: str = StubMergerCollect._sliced_path(temp_dir, td_len_pa, ent.Sha1, "deps.txt")
|
||||
if deps_path_a is not None:
|
||||
@@ -1165,8 +1182,8 @@ def RunMultiFileProject(mb: memhub.MemBuddy*,
|
||||
main_obj_path[0] = '\0'
|
||||
compiled_count: int = 0
|
||||
|
||||
# 组合 IR 缓冲区大小(4MB,足够容纳本地 stub + 所有依赖 stub + 本地 text)
|
||||
COMBINED_IR_SIZE: sizet = 4194304
|
||||
# IR 缓冲区大小(16MB,容纳在线 FULL 产物 {sha1}.full.ll;FULL = stub + text)
|
||||
COMBINED_IR_SIZE: sizet = 16777216
|
||||
|
||||
# llc 任务表(stdlib.malloc,跨线程存活):连续 LlcTask 记录数组,
|
||||
# 循环内只写 .ll + 构造任务,循环后 run_llc_parallel 分批并行跑 llc
|
||||
@@ -1183,17 +1200,17 @@ def RunMultiFileProject(mb: memhub.MemBuddy*,
|
||||
continue
|
||||
# 单文件三步计时起点(GetTickCount64,ms)
|
||||
t_pb0: uint64 = w32.win32base.GetTickCount64()
|
||||
# 组合本地 stub + 所有依赖 stub + 本地 text → 完整 IR
|
||||
# 读在线 FULL 产物 {sha1}.full.ll → 完整 IR(取代 stub + 依赖 stub/text 合并)
|
||||
combined_ir: bytes = stdlib.malloc(COMBINED_IR_SIZE)
|
||||
if combined_ir is None:
|
||||
VLogger.error(f"combined_ir 分配失败: {ent.Path}", "PhaseB")
|
||||
continue
|
||||
combined_len: sizet = StubMergerBuild.BuildCombinedIR(temp_dir, ent.Sha1, combined_ir, COMBINED_IR_SIZE)
|
||||
combined_len: sizet = StubMergerBuild.LoadFullIR(temp_dir, ent.Sha1, combined_ir, COMBINED_IR_SIZE)
|
||||
if combined_len == 0:
|
||||
VLogger.error(f"BuildCombinedIR 失败: {ent.Path}", "PhaseB")
|
||||
VLogger.error(f"读 full.ll 失败: {ent.Path}", "PhaseB")
|
||||
stdlib.free(combined_ir)
|
||||
continue
|
||||
# 计时点 1:BuildCombinedIR 完成
|
||||
# 计时点 1:读 full.ll 完成
|
||||
t_pb1: uint64 = w32.win32base.GetTickCount64()
|
||||
|
||||
# 写 .ll(llc 线程只读此文件);llc 本身改为循环后批量并行
|
||||
@@ -1433,13 +1450,13 @@ def RunMultiFileProject(mb: memhub.MemBuddy*,
|
||||
stdlib.free(src_fp_mi)
|
||||
continue
|
||||
|
||||
# 尝试 BuildCombinedIR(stub/text 应已由 Phase1 生成)
|
||||
# 尝试读在线 FULL 产物 {sha1}.full.ll(应已由 Phase1/PhaseA 生成)
|
||||
inc_combined_mi: bytes = stdlib.malloc(COMBINED_IR_SIZE)
|
||||
inc_combined_len_mi: sizet = 0
|
||||
if inc_combined_mi is not None:
|
||||
inc_combined_len_mi = StubMergerBuild.BuildCombinedIR(temp_dir, inc_sha1_mi, inc_combined_mi, COMBINED_IR_SIZE)
|
||||
inc_combined_len_mi = StubMergerBuild.LoadFullIR(temp_dir, inc_sha1_mi, inc_combined_mi, COMBINED_IR_SIZE)
|
||||
|
||||
# 如果 stub/text 不存在,翻译源文件并保存 stub + text,然后重试
|
||||
# 如果 full.ll 不存在,翻译源文件并保存 stub/text/meta/full,然后重试
|
||||
if inc_combined_len_mi == 0 and inc_combined_mi is not None:
|
||||
# 计算 includes 文件的包名(相对 includes_dir 的目录部分)
|
||||
inc_pkg_mi: str = None
|
||||
@@ -1450,6 +1467,8 @@ def RunMultiFileProject(mb: memhub.MemBuddy*,
|
||||
if tr_mi is not None:
|
||||
# PBP IR 缓冲:同 PHASE_A_IR_SIZE(parser.vp text.ll >1MB 截断根因)
|
||||
PBP_IR_SIZE: sizet = 8388608
|
||||
# FULL dump = stub + text,留 2× 余量
|
||||
PBP_FULL_SIZE: sizet = 16777216
|
||||
# 保存 stub.ll (切片路径)
|
||||
inc_stub_buf: bytes = stdlib.malloc(PBP_IR_SIZE)
|
||||
if inc_stub_buf is not None:
|
||||
@@ -1478,18 +1497,40 @@ def RunMultiFileProject(mb: memhub.MemBuddy*,
|
||||
stdlib.free(inc_text_buf)
|
||||
# save {sha1}.meta(结构化 sidecar:函数清单/结构体布局/虚表)
|
||||
StubMergerIR.WriteModuleMeta(tr_mi.Module, temp_dir, td_len_mi, inc_sha1_mi)
|
||||
# 重试 BuildCombinedIR
|
||||
inc_combined_len_mi = StubMergerBuild.BuildCombinedIR(temp_dir, inc_sha1_mi, inc_combined_mi, COMBINED_IR_SIZE)
|
||||
# OOM 清理:stub/text 已写盘,回收本 includes 文件翻译瞬时数据
|
||||
# (AST/tokens/lexer/IR/Handler);BuildCombinedIR 只读文件,不受影响
|
||||
# save {sha1}.full.ll(在线 FULL 产物)
|
||||
inc_full_buf: bytes = stdlib.malloc(PBP_FULL_SIZE)
|
||||
if inc_full_buf is None:
|
||||
VLogger.error("full.ll 缓冲区分配失败", "PhaseB+")
|
||||
tr_mi.dump_ir(inc_full_buf, PBP_FULL_SIZE, llvmlite.OUTPUT_FULL)
|
||||
inc_full_len: sizet = string.strlen(inc_full_buf)
|
||||
inc_full_path: str = StubMergerCollect._sliced_path(temp_dir, td_len_mi, inc_sha1_mi, "full.ll")
|
||||
if inc_full_path is not None:
|
||||
ifff: rawptr[fileio.File] = fileio.File(inc_full_path, fileio.MODE.W)
|
||||
if not ifff.closed:
|
||||
ifff.write(inc_full_buf, inc_full_len)
|
||||
ifff.close()
|
||||
stdlib.free(inc_full_path)
|
||||
stdlib.free(inc_full_buf)
|
||||
# 重试读 full.ll
|
||||
inc_combined_len_mi = StubMergerBuild.LoadFullIR(temp_dir, inc_sha1_mi, inc_combined_mi, COMBINED_IR_SIZE)
|
||||
# OOM 清理:stub/text/meta/full 已写盘,回收本 includes 文件翻译瞬时数据
|
||||
# (AST/tokens/lexer/IR/Handler);读取只碰文件,不受影响
|
||||
mb.reset_to(mark2)
|
||||
|
||||
if inc_combined_len_mi == 0:
|
||||
# 诊断:检查 stub.ll/text.ll 是否存在
|
||||
# 诊断:检查 full.ll/stub.ll/text.ll 是否存在
|
||||
diag_full: str = StubMergerCollect._sliced_path(temp_dir, td_len_mi, inc_sha1_mi, "full.ll")
|
||||
diag_stub: str = StubMergerCollect._sliced_path(temp_dir, td_len_mi, inc_sha1_mi, "stub.ll")
|
||||
diag_text: str = StubMergerCollect._sliced_path(temp_dir, td_len_mi, inc_sha1_mi, "text.ll")
|
||||
diag_full_ex: int = 0
|
||||
diag_stub_ex: int = 0
|
||||
diag_text_ex: int = 0
|
||||
if diag_full is not None:
|
||||
df_diag_f: rawptr[fileio.File] = fileio.File(diag_full, fileio.MODE.R)
|
||||
if not df_diag_f.closed:
|
||||
diag_full_ex = 1
|
||||
df_diag_f.close()
|
||||
stdlib.free(diag_full)
|
||||
if diag_stub is not None:
|
||||
df_diag_s: rawptr[fileio.File] = fileio.File(diag_stub, fileio.MODE.R)
|
||||
if not df_diag_s.closed:
|
||||
@@ -1502,7 +1543,7 @@ def RunMultiFileProject(mb: memhub.MemBuddy*,
|
||||
diag_text_ex = 1
|
||||
df_diag_t.close()
|
||||
stdlib.free(diag_text)
|
||||
VLogger.error(f"BuildCombinedIR 失败: {src_fp_mi} (stub={diag_stub_ex} text={diag_text_ex} is_decl={is_decl_mi})", "PhaseB+")
|
||||
VLogger.error(f"读 full.ll 失败: {src_fp_mi} (full={diag_full_ex} stub={diag_stub_ex} text={diag_text_ex} is_decl={is_decl_mi})", "PhaseB+")
|
||||
if inc_combined_mi is not None:
|
||||
stdlib.free(inc_combined_mi)
|
||||
stdlib.free(check_pat_mi)
|
||||
|
||||
@@ -1389,6 +1389,40 @@ def HoistTypeDefsBeforeDeclares(buf: bytes, length: sizet) -> sizet:
|
||||
stdlib.free(tmp)
|
||||
return new_len
|
||||
|
||||
# ============================================================
|
||||
# LoadFullIR - 读 {temp_dir}/{sha1}.full.ll 到 out_buf(④-c:合并期改用在线 FULL 产物)
|
||||
#
|
||||
# 与 BuildCombinedIR 同契约:返回写入长度(sizet;0=失败),out_buf 以 '\0' 结尾。
|
||||
# full.ll 由翻译点在模型在线时 dump_ir(OUTPUT_FULL) 产出 —— 自足的单模块 IR
|
||||
# (类型定义 + external 声明 + 定义 + DI 一次齐全,类型定义天然在声明之前),
|
||||
# 无需再把 stub + 依赖 stub/text 文本合并。
|
||||
# ============================================================
|
||||
def LoadFullIR(temp_dir: str, sha1: str,
|
||||
out_buf: bytes, out_size: sizet) -> sizet:
|
||||
"""读取 {sha1}.full.ll 到 out_buf;返回长度(0=失败)"""
|
||||
if temp_dir is None or sha1 is None or out_buf is None or out_size == 0:
|
||||
return 0
|
||||
td_len: sizet = string.strlen(temp_dir)
|
||||
p: str = StubMergerCollect._sliced_path(temp_dir, td_len, sha1, "full.ll")
|
||||
if p is None:
|
||||
return 0
|
||||
f: rawptr[fileio.File] = fileio.File(p, fileio.MODE.R)
|
||||
stdlib.free(p)
|
||||
out_buf[0] = '\0'
|
||||
if f.closed:
|
||||
return 0
|
||||
if f.size() >= out_size:
|
||||
f.close()
|
||||
VLogger.error("full.ll 超过 IR 缓冲上限", "StubMerger")
|
||||
return 0
|
||||
rd: int64 = f.read_all(out_buf, out_size)
|
||||
f.close()
|
||||
if rd <= 0:
|
||||
return 0
|
||||
out_buf[rd] = '\0'
|
||||
return sizet(rd)
|
||||
|
||||
|
||||
# ============================================================
|
||||
# BuildCombinedIR - 组合本地 stub + 依赖 stubs + 本地 text → 完整 IR
|
||||
#
|
||||
|
||||
Reference in new issue
Block a user