feat(tpc): 函数级 DWARF 调试元数据 + 合并 IR 元数据规整
DWARF: 新增 LLVMCG/DebugInfo.py(DebugInfoEmitter)——每模块一个 DIFile/DICompileUnit、每个 define 一个 DISubprogram 并在函数入口首条指令挂 DILocation,模块落盘前补 !llvm.dbg.cu 与 !llvm.module.flags(Debug Info Version 3)。开关:新增 --dwarf(默认关闭),dwarf_enabled = args.dwarf or args.asan,沿 Phase2Translator → _translate_file → Translator.DwarfEnabled → LlvmCodeGenerator.dwarf_enabled 下传。define 之外的 declare 不挂。include 库刻意不给 DI(其 .obj 进跨项目共享的 includes.binary 缓存,注入会污染缓存)
关键约束(踩过的坑,已在探针中实测):DICompileUnit 与 DISubprogram 必须 is_distinct=True;emissionKind 用 FullDebug 而非 Full;spFlags 用 DISPFlagDefinition;DISubprogram 必带 type=DISubroutineType,缺了 llc 会崩 0xC0000005;模块级必须有 !llvm.dbg.cu + Debug Info Version 3
IRMetadataFix: 新增 SanitizeIRMetadata,在 _llc_compile 入口统一规整 IR——BuildCombinedIR 是多模块 IR 文本直接拼接,--dwarf 后每个片段自带 !0..!N 与两份命名元数据,22383 行样本里 !0/!1 各重复 67 次导致 opt/llc 报 Metadata id is already used。实现:按块重编号 !N 并改写 !dbg/命名元数据引用、!llvm.dbg.cu 全量聚合、其它命名元数据全同去重;无元数据文件逐字节 no-op、幂等(1.7MB/0.30s)
顺带修 _AddWeakComdat:函数头正则跨行吞换行,--dwarf 把 !dbg 放到 define 行尾后 comdat 被插到 ! 附着之后,llc 报 expected { in function body(现场 52 个 define 中 36 个中招);改为插在附着之前,无 !dbg 时产物逐字节不变
已自证:llc/opt RC=0、行表含 .vp 文件名与行号、opt -passes=forceattrs,asan 与 llvm-link/always-inline 均不丢 !dbg。已知待办:llc 现报 inlinable function call ... must have a !dbg location ⇒ 需补语句/调用点级 !dbg(否则警告 ignoring invalid debug info 会丢弃整份 DI)
This commit is contained in:
1 parent
033c0a7771
commit
43e7cdd2cc
9 files changed
+480
-27
No files matched your search
@@ -21,14 +21,15 @@ from lib.Projectrans.Utils import compute_sha1, _check_annotation_for_export, fi
|
||||
from lib.Projectrans.Phase1Generator import _TryGlobalCache, _SaveToGlobalCache, _GLOBAL_CACHE_DIR
|
||||
import TransPyC
|
||||
from StubGen import PythonToStubConverter
|
||||
from lib.core.LLVMCG.IRMetadataFix import SanitizeIRMetadata
|
||||
|
||||
|
||||
# PROF 热点:nt._path_isfile 2.6 万次 16.4s(Windows 磁盘 stat ~0.6ms/次)。
|
||||
# 编译进程内文件状态稳定(obj/ll 一旦生成不删除、不被移动),用模块级缓存
|
||||
# 避免重复 stat。仅用于"存在性检查"类调用(不用于内容变化的判断)。
|
||||
_FILE_STAT_CACHE: dict[str, bool] = {}
|
||||
|
||||
|
||||
_FILE_STAT_CACHE: dict[str, bool] = {}
|
||||
|
||||
|
||||
def _CachedIsFile(path: str) -> bool:
|
||||
r: bool | None = _FILE_STAT_CACHE.get(path)
|
||||
if r is None:
|
||||
@@ -233,7 +234,8 @@ def _parallel_translate_batch(batch: list, src_root: str, temp_dir: str, output_
|
||||
shared_sym_pickle_path: str, struct_sha1_map: dict[str, str],
|
||||
hotspot_sites_file: str | None = None,
|
||||
hotspot_sites_all: bool = False,
|
||||
hotspot_callees: list[str] | None = None) -> tuple[list, list]:
|
||||
hotspot_callees: list[str] | None = None,
|
||||
dwarf_enabled: bool = False) -> tuple[list, list]:
|
||||
"""批次子任务:worker 内创建一个 Phase2Translator + pickle.load 一次,顺序翻译批内全部文件。
|
||||
|
||||
batch 每项: (i, src_path, out_path, sha1, prebuilt_ModuleSha1Map)
|
||||
@@ -292,7 +294,7 @@ def _parallel_translate_batch(batch: list, src_root: str, temp_dir: str, output_
|
||||
if symbol_table_bytes is not None:
|
||||
trans._shared_symbol_table = pickle.loads(symbol_table_bytes)
|
||||
trans._shared_symbol_table.translator = None
|
||||
trans._translate_file(src_path, out_path, prebuilt_ModuleSha1Map=msm)
|
||||
trans._translate_file(src_path, out_path, prebuilt_ModuleSha1Map=msm, dwarf_enabled=dwarf_enabled)
|
||||
per_file_times.append((src_path, time.time() - _f0, os.path.relpath(src_path, src_root)))
|
||||
except Exception as e:
|
||||
# fail-fast(2026-08-18):任意文件编译出错立即终止整个编译进程,
|
||||
@@ -555,7 +557,7 @@ def _AddWeakComdat(ll_content: str) -> str:
|
||||
|
||||
COFF 下无 COMDAT 的 weak/linkonce 定义会被 llc 展开成强符号
|
||||
`.weak.<name>.default`(weak external 的实现副本)。同一弱定义出现在
|
||||
多个模块时(泛型特化方法如 `e94ef9e139461915.list[rowptr[AST]].
|
||||
多个模块时(泛型特化方法如 `e94ef9e139461915.list[rawptr[AST]].
|
||||
__before_init__`、跨模块共享常量如 PYI_BUF_SIZE),副本同名 →
|
||||
ld.lld 报 duplicate symbol。补 COMDAT(any) 后 llc 输出单一 COMDAT
|
||||
成员,链接器自动去重(llc/ld.lld 端到端已实验验证)。
|
||||
@@ -569,7 +571,16 @@ def _AddWeakComdat(ll_content: str) -> str:
|
||||
# 1) 函数定义:define <linkage> [dso_local] ... @name(...) { → @name(...) comdat {
|
||||
def _fn_repl(m: re.Match) -> str:
|
||||
comdat_names.append(m.group(3))
|
||||
return m.group(1) + ' comdat ' + m.group(4)
|
||||
head: str = m.group(1)
|
||||
# comdat 是函数头关键字,必须写在 !dbg 等元数据附着**之前**:LLVM 解析器
|
||||
# 读完 `!...` 附着后就只接受 `{`,附着之后写 comdat 会报
|
||||
# "expected '{' in function body"(`[^{]*` 会跨行吞掉换行,--dwarf 给
|
||||
# 每个 define 行尾加了 !dbg 后插入点就落到了附着之后)。故插入点前移到
|
||||
# 第一个元数据附着之前。
|
||||
attach = re.search(r'\s![\w.]', head)
|
||||
if attach is None:
|
||||
return head + ' comdat ' + m.group(4)
|
||||
return head[:attach.start()] + ' comdat' + head[attach.start():] + m.group(4)
|
||||
ll_content = re.sub(
|
||||
r'^(define\s+(?:dso_local\s+)?(?:%s)\s+[^@]*@("?)([^"\s\(]+)\2[^{]*)(\{)' % weak_links,
|
||||
_fn_repl, ll_content, flags=re.MULTILINE)
|
||||
@@ -625,7 +636,7 @@ class Phase2Translator:
|
||||
_include_sha1_cache: dict[str, str] = {}
|
||||
_include_exist_cache: dict[str, bool] = {}
|
||||
|
||||
def __init__(self, src_root: str, temp_dir: str, output_dir: str, compile_cmd: str = 'llc', compile_flags: list[str] | None = None, linker_cmd: str | None = None, linker_flags: list[str] | None = None, linker_output: str | None = None, static_libs: list[str] | None = None, dynamic_libs: list[str] | None = None, include_dirs: list[str] | None = None, entry_files: list[str] | None = None, target_triple: str | None = None, target_datalayout: str | None = None, force_recompile_includes: bool = False, build_pyi_cache: bool = True, asan_mode: bool = False) -> None:
|
||||
def __init__(self, src_root: str, temp_dir: str, output_dir: str, compile_cmd: str = 'llc', compile_flags: list[str] | None = None, linker_cmd: str | None = None, linker_flags: list[str] | None = None, linker_output: str | None = None, static_libs: list[str] | None = None, dynamic_libs: list[str] | None = None, include_dirs: list[str] | None = None, entry_files: list[str] | None = None, target_triple: str | None = None, target_datalayout: str | None = None, force_recompile_includes: bool = False, build_pyi_cache: bool = True, asan_mode: bool = False, dwarf_enabled: bool = False) -> None:
|
||||
self.src_root = os.path.abspath(src_root)
|
||||
self.temp_dir = os.path.abspath(temp_dir)
|
||||
self.output_dir = os.path.abspath(output_dir)
|
||||
@@ -656,13 +667,18 @@ class Phase2Translator:
|
||||
os.environ['PATH'] = _clang64_bin + os.pathsep + os.environ.get('PATH', '')
|
||||
self.asan_rt_dir = _FindAsanRuntimeDir()
|
||||
self.slice_level = 3
|
||||
# 调用点热点统计(--hotspot-sites):None=未启用(零影响);str=调用点计数
|
||||
# 落盘文件路径(每函数计时落同目录 hotspot_time.txt)。各模块运行时自带
|
||||
# Ctrl+C 处理器、统一以追加方式写该路径;含 main 的模块在 main 入口清空一次
|
||||
# → 每次运行一份结果。
|
||||
self.hotspot_sites_file: str | None = None
|
||||
# 是否对全部直接 call 点插桩(忽略白名单);白名单被调函数基名列表(None=默认)
|
||||
self.hotspot_sites_all: bool = False
|
||||
# 函数级 DWARF 调试元数据(--dwarf;--asan 时自动开启):False=未启用
|
||||
# (零影响,不构造任何 DI 元数据);True=App 源模块的 .ll 携带
|
||||
# !llvm.dbg.cu + 每函数 DISubprogram + 入口 DILocation,供 ASan /
|
||||
# llvm-symbolizer 反查 ".vp:函数起始行"。
|
||||
self.dwarf_enabled: bool = dwarf_enabled
|
||||
# 调用点热点统计(--hotspot-sites):None=未启用(零影响);str=调用点计数
|
||||
# 落盘文件路径(每函数计时落同目录 hotspot_time.txt)。各模块运行时自带
|
||||
# Ctrl+C 处理器、统一以追加方式写该路径;含 main 的模块在 main 入口清空一次
|
||||
# → 每次运行一份结果。
|
||||
self.hotspot_sites_file: str | None = None
|
||||
# 是否对全部直接 call 点插桩(忽略白名单);白名单被调函数基名列表(None=默认)
|
||||
self.hotspot_sites_all: bool = False
|
||||
self.hotspot_callees: list[str] | None = None
|
||||
self.sha1_map: dict[str, str] = {}
|
||||
self.sig_files: dict[str, str] = {}
|
||||
@@ -871,7 +887,9 @@ class Phase2Translator:
|
||||
# 启用时一律重译,保证每个模块都嵌入 internal 计数器、计时器与
|
||||
# @llvm.global_ctors 自注册。未启用时 hotspot_on=False,缓存与重组分支
|
||||
# 行为与改动前完全一致(零影响)。
|
||||
hotspot_on: bool = bool(self.hotspot_sites_file)
|
||||
# --dwarf(函数级 DWARF)同理:缓存 .ll/.obj 不含 !llvm.dbg.cu,必须一并
|
||||
# 强制重译,否则开启开关后仍复用旧产物、看不到任何 .vp 行号。
|
||||
hotspot_on: bool = bool(self.hotspot_sites_file) or bool(self.dwarf_enabled)
|
||||
for i, src_path in enumerate(py_files, 1):
|
||||
rel = os.path.relpath(src_path, self.src_root)
|
||||
# 复用 parse_python_file 的 AST 缓存(get_file_dependencies 已 parse 过)
|
||||
@@ -1049,6 +1067,7 @@ class Phase2Translator:
|
||||
hotspot_sites_file=self.hotspot_sites_file,
|
||||
hotspot_sites_all=self.hotspot_sites_all,
|
||||
hotspot_callees=self.hotspot_callees,
|
||||
dwarf_enabled=self.dwarf_enabled,
|
||||
)
|
||||
futures[future] = (bidx, rel_first, len(batch))
|
||||
|
||||
@@ -1095,7 +1114,7 @@ class Phase2Translator:
|
||||
rel = os.path.relpath(src_path, self.src_root)
|
||||
t0 = time.time()
|
||||
try:
|
||||
self._translate_file(src_path, out_path, prebuilt_ModuleSha1Map=msm)
|
||||
self._translate_file(src_path, out_path, prebuilt_ModuleSha1Map=msm, dwarf_enabled=self.dwarf_enabled)
|
||||
t1 = time.time()
|
||||
_vlog().info(f" [{idx+1}/{len(need_translate)}] {rel} ({t1-t0:.1f}s)")
|
||||
except Exception as e:
|
||||
@@ -1539,8 +1558,13 @@ class Phase2Translator:
|
||||
return prefix + code
|
||||
return code
|
||||
|
||||
def _translate_file(self, src_path: str, out_path: str, prebuilt_ModuleSha1Map: dict[str, str] | None = None) -> None:
|
||||
"""翻译单个源文件,嵌入相关 .stub.ll 声明"""
|
||||
def _translate_file(self, src_path: str, out_path: str, prebuilt_ModuleSha1Map: dict[str, str] | None = None, dwarf_enabled: bool = False) -> None:
|
||||
"""翻译单个源文件,嵌入相关 .stub.ll 声明
|
||||
|
||||
dwarf_enabled: 函数级 DWARF 调试元数据(--dwarf),默认 False(零影响)。
|
||||
仅 App 源模块传 True;include 文件的 .obj 会落进跨项目共享的
|
||||
includes.binary 预编译缓存,注入 DI 会污染该缓存,故一律保持 False。
|
||||
"""
|
||||
_t0: float = time.time()
|
||||
with open(src_path, 'r', encoding='utf-8') as f:
|
||||
code = f.read()
|
||||
@@ -1567,6 +1591,10 @@ class Phase2Translator:
|
||||
|
||||
trans = TransPyC.TransPyC(code=code, triple=self.triple, datalayout=self.datalayout)
|
||||
trans.translator.CurrentFile = src_path
|
||||
# 函数级 DWARF(--dwarf):透传给 Translator,由其传给每个新 LlvmCodeGenerator;
|
||||
# 未启用时不设置 → 与改动前逐字节一致(零影响)
|
||||
if dwarf_enabled:
|
||||
trans.translator.DwarfEnabled = True
|
||||
trans.SliceLevel = self.slice_level
|
||||
trans.translator.SliceLevel = self.slice_level
|
||||
trans.translator.SliceCount = 0
|
||||
@@ -1831,6 +1859,24 @@ class Phase2Translator:
|
||||
except Exception as e:
|
||||
_vlog().warning(f" [警告] 解析包重导出失败 {init_pyi_path}: {e}")
|
||||
|
||||
def _SanitizeIRFile(self, ll_path: str) -> str:
|
||||
"""送入 opt/llc 之前,对该 .ll 做一次元数据规整(见 IRMetadataFix)。
|
||||
|
||||
App 模块的 IR 是「本模块 IR + 各依赖 .stub.ll 片段」直接拼接而成,而
|
||||
--dwarf(--asan 自动开启)给每份片段都加了从 !0 开始编号的 DI 元数据,
|
||||
合并文件里同一个 !N 被定义多次 → opt/llc 报 "Metadata id is already
|
||||
used"、命名元数据 "defined more than once"。规整后编号全局唯一、
|
||||
命名元数据只留一份。无元数据 / 已规整的文件逐字节不变(no-op),
|
||||
原地覆盖没有额外产物。
|
||||
"""
|
||||
with open(ll_path, 'r', encoding='utf-8', newline='') as f:
|
||||
text = f.read()
|
||||
fixed = SanitizeIRMetadata(text)
|
||||
if fixed != text:
|
||||
with open(ll_path, 'w', encoding='utf-8', newline='') as f:
|
||||
f.write(fixed)
|
||||
return ll_path
|
||||
|
||||
def _llc_compile(self, ll_path: str, obj_path: str,
|
||||
cwd: str | None = None) -> subprocess.CompletedProcess:
|
||||
"""编译单个 .ll 为 obj。
|
||||
@@ -1843,6 +1889,7 @@ class Phase2Translator:
|
||||
函数插桩(clang -fsanitize=address 自动加;手写 IR 没有)——
|
||||
先 forceattrs 批量加属性再 asan,否则只链接 runtime 不插桩。
|
||||
"""
|
||||
ll_path = self._SanitizeIRFile(ll_path)
|
||||
compile_input = ll_path
|
||||
if self.asan_mode:
|
||||
asan_ll = ll_path[:-3] + '_asan.ll'
|
||||
|
||||
Reference in new issue
Block a user