feat(tpc): 函数级 DWARF 调试元数据 + 合并 IR 元数据规整
DWARF: 新增 LLVMCG/DebugInfo.py(DebugInfoEmitter)——每模块一个 DIFile/DICompileUnit、每个 define 一个 DISubprogram 并在函数入口首条指令挂 DILocation,模块落盘前补 !llvm.dbg.cu 与 !llvm.module.flags(Debug Info Version 3)。开关:新增 --dwarf(默认关闭),dwarf_enabled = args.dwarf or args.asan,沿 Phase2Translator → _translate_file → Translator.DwarfEnabled → LlvmCodeGenerator.dwarf_enabled 下传。define 之外的 declare 不挂。include 库刻意不给 DI(其 .obj 进跨项目共享的 includes.binary 缓存,注入会污染缓存)
关键约束(踩过的坑,已在探针中实测):DICompileUnit 与 DISubprogram 必须 is_distinct=True;emissionKind 用 FullDebug 而非 Full;spFlags 用 DISPFlagDefinition;DISubprogram 必带 type=DISubroutineType,缺了 llc 会崩 0xC0000005;模块级必须有 !llvm.dbg.cu + Debug Info Version 3
IRMetadataFix: 新增 SanitizeIRMetadata,在 _llc_compile 入口统一规整 IR——BuildCombinedIR 是多模块 IR 文本直接拼接,--dwarf 后每个片段自带 !0..!N 与两份命名元数据,22383 行样本里 !0/!1 各重复 67 次导致 opt/llc 报 Metadata id is already used。实现:按块重编号 !N 并改写 !dbg/命名元数据引用、!llvm.dbg.cu 全量聚合、其它命名元数据全同去重;无元数据文件逐字节 no-op、幂等(1.7MB/0.30s)
顺带修 _AddWeakComdat:函数头正则跨行吞换行,--dwarf 把 !dbg 放到 define 行尾后 comdat 被插到 ! 附着之后,llc 报 expected { in function body(现场 52 个 define 中 36 个中招);改为插在附着之前,无 !dbg 时产物逐字节不变
已自证:llc/opt RC=0、行表含 .vp 文件名与行号、opt -passes=forceattrs,asan 与 llvm-link/always-inline 均不丢 !dbg。已知待办:llc 现报 inlinable function call ... must have a !dbg location ⇒ 需补语句/调用点级 !dbg(否则警告 ignoring invalid debug info 会丢弃整份 DI)
This commit is contained in:
1 parent
033c0a7771
commit
43e7cdd2cc
9 files changed
+480
-27
No files matched your search
@@ -148,6 +148,7 @@ def main() -> None:
|
||||
parser.add_argument('--rebuild-includes', action='store_true', help='删除 includes.binary 预编译缓存并重新编译所有 includes')
|
||||
parser.add_argument('--clear-cache', action='store_true', help='清除 .transpyc_cache 全局缓存(.vpi/.stub.ll/.doc.json)')
|
||||
parser.add_argument('--asan', action='store_true', help='ASan 模式:对全部 .ll 做 opt -passes=asan 插桩,并显式链接 clang64 的 ASan 运行库(mingw64 的 clang 没有 x86_64-w64-windows-gnu 目标的 ASan runtime,不能用 -fsanitize=address)')
|
||||
parser.add_argument('--dwarf', action='store_true', help='函数级 DWARF 调试元数据:每模块一份 DICompileUnit/DIFile,每个 define 一份 DISubprogram 并挂 !dbg,函数入口第一条指令挂 DILocation。使 ASan/llvm-symbolizer 能把崩溃地址反查为 "<文件>.vp:函数起始行"(默认关闭;--asan 时自动开启)。开启时强制重译 App 模块(不复用不含 !llvm.dbg.cu 的旧 .ll/.obj),.ll/.obj 体积与编译时间略增')
|
||||
parser.add_argument('--nowarn-register', action='store_true', help='抑制"外部函数返回类型未登记"类 WARN 输出(透传给自举编译的 TransPyV.exe)')
|
||||
parser.add_argument('--hotspot-sites', nargs='?', const='', default=None, help='调用点热点统计:为每个匹配白名单的直接 call 点插入计数器,运行时 Ctrl+C 各模块按真实次数降序写入 <path>(默认为 hotspot_sites.txt);同时为每个函数定义加计时器,按自身耗时(self)降序写入同目录 hotspot_time.txt,行为 `<函数名> <self> <inc> <进入次数>`(单位 tick,x86_64 上为序列化 lfence; rdtsc);self 已扣除被插桩子调用的 inc,但不含未插桩被调方(间接调用、strcmp/memcpy 等无定义外部函数),它们计入调用者的 self')
|
||||
parser.add_argument('--hotspot-callees', default=None, help='--hotspot-sites 的白名单被调函数基名(逗号分隔),默认内存/字符串族 strcmp,strncmp,strchr,strlen,strstr,memcpy,memset,memmove,strcpy,strcat')
|
||||
@@ -161,6 +162,16 @@ def main() -> None:
|
||||
if args.asan:
|
||||
_vlog().info('ASan 模式已启用(opt -passes=asan + clang64 libclang_rt.asan_dynamic*)', 'asan')
|
||||
|
||||
# 函数级 DWARF 调试元数据(--dwarf):默认关闭(硬要求——普通构建不承担
|
||||
# 任何风险,生成的 .ll 与改动前逐字节一致)。--asan 时自动开启:ASan 报告
|
||||
# 本身就需要行号才能定位,且 ASan 构建已是显式选择的非默认路径。
|
||||
dwarf_enabled: bool = bool(args.dwarf or args.asan)
|
||||
if dwarf_enabled:
|
||||
_vlog().info(
|
||||
'函数级 DWARF 调试元数据已启用:每个 .vp 模块一份 DICompileUnit + 每函数 DISubprogram/DILocation'
|
||||
+ ('(由 --asan 自动开启)' if not args.dwarf else ''),
|
||||
'dwarf')
|
||||
|
||||
project_file: str | None = args.project
|
||||
if not project_file:
|
||||
candidates: list[str] = ['project.json', './project.json']
|
||||
@@ -328,7 +339,7 @@ def main() -> None:
|
||||
except:
|
||||
pass
|
||||
_vlog().info(f"includes.binary: 已清空 {precompiled_dir} ({cleared} 个文件)")
|
||||
trans: Phase2Translator = Phase2Translator(src, temp_dir, output_dir, compile_cmd=cc_cmd, compile_flags=cc_flags, linker_cmd=linker_cmd, linker_flags=linker_flags, linker_output=linker_output, static_libs=static_libs, dynamic_libs=dynamic_libs, include_dirs=project_include_dirs, target_triple=target_triple, target_datalayout=target_datalayout, force_recompile_includes=force_recompile_includes, asan_mode=args.asan)
|
||||
trans: Phase2Translator = Phase2Translator(src, temp_dir, output_dir, compile_cmd=cc_cmd, compile_flags=cc_flags, linker_cmd=linker_cmd, linker_flags=linker_flags, linker_output=linker_output, static_libs=static_libs, dynamic_libs=dynamic_libs, include_dirs=project_include_dirs, target_triple=target_triple, target_datalayout=target_datalayout, force_recompile_includes=force_recompile_includes, asan_mode=args.asan, dwarf_enabled=dwarf_enabled)
|
||||
trans.sha1_map = Load_sha1_map(temp_dir)
|
||||
trans.slice_level = slice_level
|
||||
# 调用点热点统计与每函数计时(--hotspot-sites):None=未启用(零影响)
|
||||
|
||||
@@ -21,14 +21,15 @@ from lib.Projectrans.Utils import compute_sha1, _check_annotation_for_export, fi
|
||||
from lib.Projectrans.Phase1Generator import _TryGlobalCache, _SaveToGlobalCache, _GLOBAL_CACHE_DIR
|
||||
import TransPyC
|
||||
from StubGen import PythonToStubConverter
|
||||
from lib.core.LLVMCG.IRMetadataFix import SanitizeIRMetadata
|
||||
|
||||
|
||||
# PROF 热点:nt._path_isfile 2.6 万次 16.4s(Windows 磁盘 stat ~0.6ms/次)。
|
||||
# 编译进程内文件状态稳定(obj/ll 一旦生成不删除、不被移动),用模块级缓存
|
||||
# 避免重复 stat。仅用于"存在性检查"类调用(不用于内容变化的判断)。
|
||||
_FILE_STAT_CACHE: dict[str, bool] = {}
|
||||
|
||||
|
||||
_FILE_STAT_CACHE: dict[str, bool] = {}
|
||||
|
||||
|
||||
def _CachedIsFile(path: str) -> bool:
|
||||
r: bool | None = _FILE_STAT_CACHE.get(path)
|
||||
if r is None:
|
||||
@@ -233,7 +234,8 @@ def _parallel_translate_batch(batch: list, src_root: str, temp_dir: str, output_
|
||||
shared_sym_pickle_path: str, struct_sha1_map: dict[str, str],
|
||||
hotspot_sites_file: str | None = None,
|
||||
hotspot_sites_all: bool = False,
|
||||
hotspot_callees: list[str] | None = None) -> tuple[list, list]:
|
||||
hotspot_callees: list[str] | None = None,
|
||||
dwarf_enabled: bool = False) -> tuple[list, list]:
|
||||
"""批次子任务:worker 内创建一个 Phase2Translator + pickle.load 一次,顺序翻译批内全部文件。
|
||||
|
||||
batch 每项: (i, src_path, out_path, sha1, prebuilt_ModuleSha1Map)
|
||||
@@ -292,7 +294,7 @@ def _parallel_translate_batch(batch: list, src_root: str, temp_dir: str, output_
|
||||
if symbol_table_bytes is not None:
|
||||
trans._shared_symbol_table = pickle.loads(symbol_table_bytes)
|
||||
trans._shared_symbol_table.translator = None
|
||||
trans._translate_file(src_path, out_path, prebuilt_ModuleSha1Map=msm)
|
||||
trans._translate_file(src_path, out_path, prebuilt_ModuleSha1Map=msm, dwarf_enabled=dwarf_enabled)
|
||||
per_file_times.append((src_path, time.time() - _f0, os.path.relpath(src_path, src_root)))
|
||||
except Exception as e:
|
||||
# fail-fast(2026-08-18):任意文件编译出错立即终止整个编译进程,
|
||||
@@ -555,7 +557,7 @@ def _AddWeakComdat(ll_content: str) -> str:
|
||||
|
||||
COFF 下无 COMDAT 的 weak/linkonce 定义会被 llc 展开成强符号
|
||||
`.weak.<name>.default`(weak external 的实现副本)。同一弱定义出现在
|
||||
多个模块时(泛型特化方法如 `e94ef9e139461915.list[rowptr[AST]].
|
||||
多个模块时(泛型特化方法如 `e94ef9e139461915.list[rawptr[AST]].
|
||||
__before_init__`、跨模块共享常量如 PYI_BUF_SIZE),副本同名 →
|
||||
ld.lld 报 duplicate symbol。补 COMDAT(any) 后 llc 输出单一 COMDAT
|
||||
成员,链接器自动去重(llc/ld.lld 端到端已实验验证)。
|
||||
@@ -569,7 +571,16 @@ def _AddWeakComdat(ll_content: str) -> str:
|
||||
# 1) 函数定义:define <linkage> [dso_local] ... @name(...) { → @name(...) comdat {
|
||||
def _fn_repl(m: re.Match) -> str:
|
||||
comdat_names.append(m.group(3))
|
||||
return m.group(1) + ' comdat ' + m.group(4)
|
||||
head: str = m.group(1)
|
||||
# comdat 是函数头关键字,必须写在 !dbg 等元数据附着**之前**:LLVM 解析器
|
||||
# 读完 `!...` 附着后就只接受 `{`,附着之后写 comdat 会报
|
||||
# "expected '{' in function body"(`[^{]*` 会跨行吞掉换行,--dwarf 给
|
||||
# 每个 define 行尾加了 !dbg 后插入点就落到了附着之后)。故插入点前移到
|
||||
# 第一个元数据附着之前。
|
||||
attach = re.search(r'\s![\w.]', head)
|
||||
if attach is None:
|
||||
return head + ' comdat ' + m.group(4)
|
||||
return head[:attach.start()] + ' comdat' + head[attach.start():] + m.group(4)
|
||||
ll_content = re.sub(
|
||||
r'^(define\s+(?:dso_local\s+)?(?:%s)\s+[^@]*@("?)([^"\s\(]+)\2[^{]*)(\{)' % weak_links,
|
||||
_fn_repl, ll_content, flags=re.MULTILINE)
|
||||
@@ -625,7 +636,7 @@ class Phase2Translator:
|
||||
_include_sha1_cache: dict[str, str] = {}
|
||||
_include_exist_cache: dict[str, bool] = {}
|
||||
|
||||
def __init__(self, src_root: str, temp_dir: str, output_dir: str, compile_cmd: str = 'llc', compile_flags: list[str] | None = None, linker_cmd: str | None = None, linker_flags: list[str] | None = None, linker_output: str | None = None, static_libs: list[str] | None = None, dynamic_libs: list[str] | None = None, include_dirs: list[str] | None = None, entry_files: list[str] | None = None, target_triple: str | None = None, target_datalayout: str | None = None, force_recompile_includes: bool = False, build_pyi_cache: bool = True, asan_mode: bool = False) -> None:
|
||||
def __init__(self, src_root: str, temp_dir: str, output_dir: str, compile_cmd: str = 'llc', compile_flags: list[str] | None = None, linker_cmd: str | None = None, linker_flags: list[str] | None = None, linker_output: str | None = None, static_libs: list[str] | None = None, dynamic_libs: list[str] | None = None, include_dirs: list[str] | None = None, entry_files: list[str] | None = None, target_triple: str | None = None, target_datalayout: str | None = None, force_recompile_includes: bool = False, build_pyi_cache: bool = True, asan_mode: bool = False, dwarf_enabled: bool = False) -> None:
|
||||
self.src_root = os.path.abspath(src_root)
|
||||
self.temp_dir = os.path.abspath(temp_dir)
|
||||
self.output_dir = os.path.abspath(output_dir)
|
||||
@@ -656,13 +667,18 @@ class Phase2Translator:
|
||||
os.environ['PATH'] = _clang64_bin + os.pathsep + os.environ.get('PATH', '')
|
||||
self.asan_rt_dir = _FindAsanRuntimeDir()
|
||||
self.slice_level = 3
|
||||
# 调用点热点统计(--hotspot-sites):None=未启用(零影响);str=调用点计数
|
||||
# 落盘文件路径(每函数计时落同目录 hotspot_time.txt)。各模块运行时自带
|
||||
# Ctrl+C 处理器、统一以追加方式写该路径;含 main 的模块在 main 入口清空一次
|
||||
# → 每次运行一份结果。
|
||||
self.hotspot_sites_file: str | None = None
|
||||
# 是否对全部直接 call 点插桩(忽略白名单);白名单被调函数基名列表(None=默认)
|
||||
self.hotspot_sites_all: bool = False
|
||||
# 函数级 DWARF 调试元数据(--dwarf;--asan 时自动开启):False=未启用
|
||||
# (零影响,不构造任何 DI 元数据);True=App 源模块的 .ll 携带
|
||||
# !llvm.dbg.cu + 每函数 DISubprogram + 入口 DILocation,供 ASan /
|
||||
# llvm-symbolizer 反查 ".vp:函数起始行"。
|
||||
self.dwarf_enabled: bool = dwarf_enabled
|
||||
# 调用点热点统计(--hotspot-sites):None=未启用(零影响);str=调用点计数
|
||||
# 落盘文件路径(每函数计时落同目录 hotspot_time.txt)。各模块运行时自带
|
||||
# Ctrl+C 处理器、统一以追加方式写该路径;含 main 的模块在 main 入口清空一次
|
||||
# → 每次运行一份结果。
|
||||
self.hotspot_sites_file: str | None = None
|
||||
# 是否对全部直接 call 点插桩(忽略白名单);白名单被调函数基名列表(None=默认)
|
||||
self.hotspot_sites_all: bool = False
|
||||
self.hotspot_callees: list[str] | None = None
|
||||
self.sha1_map: dict[str, str] = {}
|
||||
self.sig_files: dict[str, str] = {}
|
||||
@@ -871,7 +887,9 @@ class Phase2Translator:
|
||||
# 启用时一律重译,保证每个模块都嵌入 internal 计数器、计时器与
|
||||
# @llvm.global_ctors 自注册。未启用时 hotspot_on=False,缓存与重组分支
|
||||
# 行为与改动前完全一致(零影响)。
|
||||
hotspot_on: bool = bool(self.hotspot_sites_file)
|
||||
# --dwarf(函数级 DWARF)同理:缓存 .ll/.obj 不含 !llvm.dbg.cu,必须一并
|
||||
# 强制重译,否则开启开关后仍复用旧产物、看不到任何 .vp 行号。
|
||||
hotspot_on: bool = bool(self.hotspot_sites_file) or bool(self.dwarf_enabled)
|
||||
for i, src_path in enumerate(py_files, 1):
|
||||
rel = os.path.relpath(src_path, self.src_root)
|
||||
# 复用 parse_python_file 的 AST 缓存(get_file_dependencies 已 parse 过)
|
||||
@@ -1049,6 +1067,7 @@ class Phase2Translator:
|
||||
hotspot_sites_file=self.hotspot_sites_file,
|
||||
hotspot_sites_all=self.hotspot_sites_all,
|
||||
hotspot_callees=self.hotspot_callees,
|
||||
dwarf_enabled=self.dwarf_enabled,
|
||||
)
|
||||
futures[future] = (bidx, rel_first, len(batch))
|
||||
|
||||
@@ -1095,7 +1114,7 @@ class Phase2Translator:
|
||||
rel = os.path.relpath(src_path, self.src_root)
|
||||
t0 = time.time()
|
||||
try:
|
||||
self._translate_file(src_path, out_path, prebuilt_ModuleSha1Map=msm)
|
||||
self._translate_file(src_path, out_path, prebuilt_ModuleSha1Map=msm, dwarf_enabled=self.dwarf_enabled)
|
||||
t1 = time.time()
|
||||
_vlog().info(f" [{idx+1}/{len(need_translate)}] {rel} ({t1-t0:.1f}s)")
|
||||
except Exception as e:
|
||||
@@ -1539,8 +1558,13 @@ class Phase2Translator:
|
||||
return prefix + code
|
||||
return code
|
||||
|
||||
def _translate_file(self, src_path: str, out_path: str, prebuilt_ModuleSha1Map: dict[str, str] | None = None) -> None:
|
||||
"""翻译单个源文件,嵌入相关 .stub.ll 声明"""
|
||||
def _translate_file(self, src_path: str, out_path: str, prebuilt_ModuleSha1Map: dict[str, str] | None = None, dwarf_enabled: bool = False) -> None:
|
||||
"""翻译单个源文件,嵌入相关 .stub.ll 声明
|
||||
|
||||
dwarf_enabled: 函数级 DWARF 调试元数据(--dwarf),默认 False(零影响)。
|
||||
仅 App 源模块传 True;include 文件的 .obj 会落进跨项目共享的
|
||||
includes.binary 预编译缓存,注入 DI 会污染该缓存,故一律保持 False。
|
||||
"""
|
||||
_t0: float = time.time()
|
||||
with open(src_path, 'r', encoding='utf-8') as f:
|
||||
code = f.read()
|
||||
@@ -1567,6 +1591,10 @@ class Phase2Translator:
|
||||
|
||||
trans = TransPyC.TransPyC(code=code, triple=self.triple, datalayout=self.datalayout)
|
||||
trans.translator.CurrentFile = src_path
|
||||
# 函数级 DWARF(--dwarf):透传给 Translator,由其传给每个新 LlvmCodeGenerator;
|
||||
# 未启用时不设置 → 与改动前逐字节一致(零影响)
|
||||
if dwarf_enabled:
|
||||
trans.translator.DwarfEnabled = True
|
||||
trans.SliceLevel = self.slice_level
|
||||
trans.translator.SliceLevel = self.slice_level
|
||||
trans.translator.SliceCount = 0
|
||||
@@ -1831,6 +1859,24 @@ class Phase2Translator:
|
||||
except Exception as e:
|
||||
_vlog().warning(f" [警告] 解析包重导出失败 {init_pyi_path}: {e}")
|
||||
|
||||
def _SanitizeIRFile(self, ll_path: str) -> str:
|
||||
"""送入 opt/llc 之前,对该 .ll 做一次元数据规整(见 IRMetadataFix)。
|
||||
|
||||
App 模块的 IR 是「本模块 IR + 各依赖 .stub.ll 片段」直接拼接而成,而
|
||||
--dwarf(--asan 自动开启)给每份片段都加了从 !0 开始编号的 DI 元数据,
|
||||
合并文件里同一个 !N 被定义多次 → opt/llc 报 "Metadata id is already
|
||||
used"、命名元数据 "defined more than once"。规整后编号全局唯一、
|
||||
命名元数据只留一份。无元数据 / 已规整的文件逐字节不变(no-op),
|
||||
原地覆盖没有额外产物。
|
||||
"""
|
||||
with open(ll_path, 'r', encoding='utf-8', newline='') as f:
|
||||
text = f.read()
|
||||
fixed = SanitizeIRMetadata(text)
|
||||
if fixed != text:
|
||||
with open(ll_path, 'w', encoding='utf-8', newline='') as f:
|
||||
f.write(fixed)
|
||||
return ll_path
|
||||
|
||||
def _llc_compile(self, ll_path: str, obj_path: str,
|
||||
cwd: str | None = None) -> subprocess.CompletedProcess:
|
||||
"""编译单个 .ll 为 obj。
|
||||
@@ -1843,6 +1889,7 @@ class Phase2Translator:
|
||||
函数插桩(clang -fsanitize=address 自动加;手写 IR 没有)——
|
||||
先 forceattrs 批量加属性再 asan,否则只链接 runtime 不插桩。
|
||||
"""
|
||||
ll_path = self._SanitizeIRFile(ll_path)
|
||||
compile_input = ll_path
|
||||
if self.asan_mode:
|
||||
asan_ll = ll_path[:-3] + '_asan.ll'
|
||||
|
||||
@@ -1782,22 +1782,22 @@ class FunctionHandle(BaseHandle):
|
||||
# 导致方法调用查找 ndarray.sum 而非 ndarray[double].sum。
|
||||
if param_name in Gen.var_struct_class:
|
||||
_existing_ps: str = Gen.var_struct_class[param_name]
|
||||
# 指针占位名(rowptr / {sha1}.rowptr / rowptr[...])是模块限定
|
||||
# 指针占位名(rawptr / {sha1}.rawptr / rawptr[...])是模块限定
|
||||
# struct 联合(llvmlite.Function | t.CPtr)被 resolver 退化
|
||||
# 的产物,必须允许 FindStructNameInAnnotation 短名修正
|
||||
# (否则 func.IsVarArg → "属性在类 'rowptr' 中找不到")。
|
||||
if not (_existing_ps == 'rowptr' or '.rowptr' in _existing_ps
|
||||
or _existing_ps.startswith('rowptr[')):
|
||||
# (否则 func.IsVarArg → "属性在类 'rawptr' 中找不到")。
|
||||
if not (_existing_ps == 'rawptr' or '.rawptr' in _existing_ps
|
||||
or _existing_ps.startswith('rawptr[')):
|
||||
continue
|
||||
found_struct: str | None = FindStructNameInAnnotation(arg.annotation, Gen.structs)
|
||||
if found_struct:
|
||||
Gen.var_struct_class[param_name] = found_struct
|
||||
# 普通类注解形参(非 BinOp 联合):rowptr[X] / t.CPtr[X] / X / t.X
|
||||
# 普通类注解形参(非 BinOp 联合):rawptr[X] / t.CPtr[X] / X / t.X
|
||||
# 一律按"声明类"登记 var_struct_class。避免按 param.type.pointee 匹配
|
||||
# 到泛型父特化(如 class Function(GSListNode[Function]) 的形参 func 被
|
||||
# 登记成 GSListNode[Function],其 struct 仅含 Next → func.Blocks 访问
|
||||
# fast-fail "属性 'Blocks' 在类 'GSListNode[Function]' 中找不到")。
|
||||
# 泛型占位(rowptr[T])的 T 不在 Gen.structs,自动跳过、保留 pointee 匹配。
|
||||
# 泛型占位(rawptr[T])的 T 不在 Gen.structs,自动跳过、保留 pointee 匹配。
|
||||
for i, arg in enumerate(Node.args.args):
|
||||
if arg.annotation is None:
|
||||
continue
|
||||
@@ -1815,7 +1815,7 @@ class FunctionHandle(BaseHandle):
|
||||
elif isinstance(ann, ast.Subscript):
|
||||
subv = ann.value
|
||||
subname = getattr(subv, 'id', None) or getattr(subv, 'attr', '')
|
||||
if subname in ('rowptr', 'CPtr', 'Ptr'):
|
||||
if subname in ('rawptr', 'CPtr', 'Ptr'):
|
||||
slc = ann.slice
|
||||
if isinstance(slc, ast.Name):
|
||||
target = slc.id
|
||||
@@ -1943,6 +1943,22 @@ class FunctionHandle(BaseHandle):
|
||||
except Exception:
|
||||
pass
|
||||
raise
|
||||
# 函数级 DWARF 调试元数据(--dwarf / --asan 自动开启):为每个 define 建
|
||||
# DISubprogram 挂 func.metadata["dbg"],并在入口块第一条指令挂 DILocation
|
||||
# (line/column = 该 def 语句的位置),使 DWARF 行表至少有一条覆盖函数地址
|
||||
# 区间的行。declare/extern 在此之前已 return,不会被挂 !dbg;未启用时
|
||||
# _GetDebugInfo() 返回 None,不产生任何 IR 差异(零影响)。
|
||||
_DbgInfo = Gen._GetDebugInfo()
|
||||
if _DbgInfo is not None:
|
||||
try:
|
||||
_DbgLoc = _DbgInfo.AttachFunction(
|
||||
func, getattr(Node, 'lineno', 0) or 0, getattr(Node, 'col_offset', 0) + 1)
|
||||
if EntryBlock.instructions:
|
||||
EntryBlock.instructions[0].metadata['dbg'] = _DbgLoc
|
||||
except Exception as _e:
|
||||
if _config_mode == "strict":
|
||||
raise
|
||||
_vlog().warning(f"发射调试元数据失败: {_e}", "Exception")
|
||||
if not Gen.builder.block.is_terminated:
|
||||
if 'naked' in fn_attrs:
|
||||
Gen.builder.unreachable()
|
||||
|
||||
@@ -212,6 +212,11 @@ class BaseGenMixin:
|
||||
# 注册策略:False=种子 CLI 单模块(在 main 入口注册);True=Projectrans
|
||||
# 多模块(每模块用 @llvm.global_ctors 自注册,不依赖 main 所在模块)
|
||||
self.hotspot_sites_module_local: bool = False
|
||||
# 函数级 DWARF 调试元数据(--dwarf / --asan 自动开启):
|
||||
# False=未启用(零影响,不构造任何 DI 元数据、不挂 !dbg);
|
||||
# True=每个 define 发 DISubprogram + 入口首条指令发 DILocation。
|
||||
self.dwarf_enabled: bool = False
|
||||
self.DebugInfo: Any = None
|
||||
pi: dict[str, bool | int] = self._platform_info
|
||||
# 根据目标平台设置宏(声明式,从三元组推导)
|
||||
if pi['is_windows']:
|
||||
@@ -584,6 +589,22 @@ class BaseGenMixin:
|
||||
except Exception: # 文件读取失败时回退为空行列表
|
||||
self._current_source_lines = []
|
||||
|
||||
def _GetDebugInfo(self) -> Any:
|
||||
"""返回本模块的 DebugInfoEmitter;未启用 --dwarf 或无源文件时返回 None。
|
||||
|
||||
未启用(dwarf_enabled=False)时不构造任何对象、不产生任何 IR 差异,
|
||||
行为与改动前逐字节一致。源文件取 _current_source_file(由 GenerateCCode
|
||||
在生成 IR 前设定,模块内恒定),每个模块一份 DICompileUnit + DIFile。
|
||||
"""
|
||||
if not self.dwarf_enabled:
|
||||
return None
|
||||
if not self._current_source_file:
|
||||
return None
|
||||
if self.DebugInfo is None:
|
||||
from lib.core.LLVMCG.DebugInfo import DebugInfoEmitter
|
||||
self.DebugInfo = DebugInfoEmitter(self.module, self._current_source_file)
|
||||
return self.DebugInfo
|
||||
|
||||
def _skip_mangle(self, name: str) -> bool:
|
||||
skip_prefixes: tuple[str, ...] = ('llvm.', 'str_const_', 'printf', 'memset_pattern')
|
||||
if any(name.startswith(p) for p in skip_prefixes):
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import llvmlite.ir as ir
|
||||
|
||||
|
||||
class DebugInfoEmitter:
|
||||
"""函数级 DWARF 调试元数据发射器(llvmlite 文本 IR)。
|
||||
|
||||
每个 Module 一份 DICompileUnit + DIFile;每个 define 一份 DISubprogram
|
||||
(is_distinct=True)挂在 func.metadata["dbg"] 上;函数入口第一条指令挂
|
||||
DILocation(line=该函数 def 行,column=该行 col),使 LLVM 行表至少有一条
|
||||
覆盖函数地址区间的行 —— ASan / llvm-symbolizer 反查函数内任意地址即可得到
|
||||
"<file>.vp:函数起始行"。
|
||||
|
||||
仅在 --dwarf(或 --asan 自动开启)时被实例化;把 Module 落成 .ll 文本之前
|
||||
必须调用 Finalize() 补模块级 !llvm.dbg.cu 与 !llvm.module.flags
|
||||
(Debug Info Version = 3),否则 llc 会丢弃调试信息或报无效 DI 警告。
|
||||
"""
|
||||
|
||||
def __init__(self, module: ir.Module, source_path: str) -> None:
|
||||
self.Module: ir.Module = module
|
||||
normalized: str = (source_path or '').replace('\\', '/')
|
||||
head: str
|
||||
sep: int = normalized.rfind('/')
|
||||
if sep < 0:
|
||||
head, self.FileName = '', normalized
|
||||
else:
|
||||
head, self.FileName = normalized[:sep], normalized[sep + 1:]
|
||||
self.Directory: str = head
|
||||
self._File: ir.DIValue | None = None
|
||||
self._CompileUnit: ir.DIValue | None = None
|
||||
self._SubroutineType: ir.DIValue | None = None
|
||||
self.Finalized: bool = False
|
||||
|
||||
def _GetFile(self) -> ir.DIValue:
|
||||
if self._File is None:
|
||||
self._File = self.Module.add_debug_info("DIFile", {
|
||||
"filename": self.FileName,
|
||||
"directory": self.Directory,
|
||||
})
|
||||
return self._File
|
||||
|
||||
def _GetCompileUnit(self) -> ir.DIValue:
|
||||
if self._CompileUnit is None:
|
||||
# emissionKind 必须是 DIToken("FullDebug")(写成 "Full" llc 报无效 DI)
|
||||
self._CompileUnit = self.Module.add_debug_info("DICompileUnit", {
|
||||
"language": ir.DIToken("DW_LANG_C99"),
|
||||
"file": self._GetFile(),
|
||||
"producer": "TPC",
|
||||
"isOptimized": False,
|
||||
"runtimeVersion": 0,
|
||||
"emissionKind": ir.DIToken("FullDebug"),
|
||||
}, is_distinct=True)
|
||||
return self._CompileUnit
|
||||
|
||||
def _GetSubroutineType(self) -> ir.DIValue:
|
||||
if self._SubroutineType is None:
|
||||
# DISubprogram 必须带 type,缺失会让 llc 崩溃(0xC0000005)
|
||||
self._SubroutineType = self.Module.add_debug_info("DISubroutineType", {
|
||||
"types": self.Module.add_metadata([None]),
|
||||
})
|
||||
return self._SubroutineType
|
||||
|
||||
def AttachFunction(self, func: ir.Function, Line: int, Column: int) -> ir.DIValue:
|
||||
"""为 define(有函数体)建 DISubprogram 并返回其入口 DILocation。
|
||||
|
||||
仅对 define 调用;declare/extern 没有函数体,不得挂 !dbg。
|
||||
"""
|
||||
sp: ir.DIValue = self.Module.add_debug_info("DISubprogram", {
|
||||
"name": func.name,
|
||||
"scope": self._GetFile(),
|
||||
"file": self._GetFile(),
|
||||
"type": self._GetSubroutineType(),
|
||||
"line": Line,
|
||||
"scopeLine": Line,
|
||||
"unit": self._GetCompileUnit(),
|
||||
"spFlags": ir.DIToken("DISPFlagDefinition"),
|
||||
}, is_distinct=True)
|
||||
func.metadata["dbg"] = sp
|
||||
return self.Module.add_debug_info("DILocation", {
|
||||
"line": Line,
|
||||
"column": Column,
|
||||
"scope": sp,
|
||||
})
|
||||
|
||||
def Finalize(self) -> None:
|
||||
"""落 .ll 文本之前补模块级 named metadata;无任何函数时不做(no-op)。"""
|
||||
if self.Finalized or self._CompileUnit is None:
|
||||
return
|
||||
self.Module.add_named_metadata("llvm.dbg.cu", self._CompileUnit)
|
||||
flags: ir.MDValue = self.Module.add_metadata([
|
||||
ir.Constant(ir.IntType(32), 2),
|
||||
ir.MetaDataString(self.Module, "Debug Info Version"),
|
||||
ir.Constant(ir.IntType(32), 3),
|
||||
])
|
||||
self.Module.add_named_metadata("llvm.module.flags", flags)
|
||||
self.Finalized = True
|
||||
@@ -0,0 +1,248 @@
|
||||
"""合并 IR 的元数据规整(sanitize)。
|
||||
|
||||
场景:多个模块的 .ll 文本被直接拼接成一个文件(每个模块都自带一小段
|
||||
DWARF 元数据:!llvm.dbg.cu / !llvm.module.flags / !0 = !DIFile / ...
|
||||
以及内联在 `define ... !dbg !N`、指令行上的 !N 引用)时,各模块的编号
|
||||
元数据都从 !0 重新开始 —— 合并文件里同一个 !N 被定义多次、命名元数据
|
||||
也被定义多次,LLVM 解析即报:
|
||||
|
||||
error: Metadata id is already used
|
||||
error: ... !llvm.dbg.cu ... defined more than once
|
||||
|
||||
本模块提供 SanitizeIRMetadata(text) -> text:把全文的 `!N = ` 定义按
|
||||
「元数据块」为单位重新编号成全局唯一 id,并同步改写全文引用;命名元数据
|
||||
按名字聚合/去重成一份。无元数据的文本逐字节返回原样(严格 no-op),对
|
||||
已规整的文本再跑一次结果一致(幂等)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
# `!N = ` 形式的编号元数据定义行
|
||||
_Definition = re.compile(r'^!(\d+) = ')
|
||||
# 编号元数据引用
|
||||
_Reference = re.compile(r'!(\d+)')
|
||||
# `!name = ...` 形式的命名元数据(首字符为字母,故不会命中 `!N = `)
|
||||
_NamedDefinition = re.compile(r'^!([A-Za-z][\w.]*)\s*=\s*(.*)$')
|
||||
# `!{ ... }` 列表形式
|
||||
_ListForm = re.compile(r'^!\{(.*)\}$', re.DOTALL)
|
||||
# 纯 id 引用 `!N`
|
||||
_PureId = re.compile(r'^!(\d+)$')
|
||||
|
||||
|
||||
def _SplitTopLevel(Content: str) -> list[str]:
|
||||
"""按顶层逗号切分 !{ ... } 的条目(跳过字符串字面量与嵌套括号内的逗号)。"""
|
||||
Items: list[str] = []
|
||||
Depth: int = 0
|
||||
Buf: list[str] = []
|
||||
i: int = 0
|
||||
n: int = len(Content)
|
||||
while i < n:
|
||||
Ch: str = Content[i]
|
||||
if Ch == '"':
|
||||
j: int = i + 1
|
||||
while j < n:
|
||||
if Content[j] == '\\':
|
||||
j += 2
|
||||
continue
|
||||
if Content[j] == '"':
|
||||
j += 1
|
||||
break
|
||||
j += 1
|
||||
Buf.append(Content[i:j])
|
||||
i = j
|
||||
continue
|
||||
if Ch in '{[(':
|
||||
Depth += 1
|
||||
elif Ch in '}])':
|
||||
Depth -= 1
|
||||
elif Ch == ',' and Depth == 0:
|
||||
Items.append(''.join(Buf).strip())
|
||||
Buf = []
|
||||
i += 1
|
||||
continue
|
||||
Buf.append(Ch)
|
||||
i += 1
|
||||
Tail: str = ''.join(Buf).strip()
|
||||
if Tail:
|
||||
Items.append(Tail)
|
||||
return Items
|
||||
|
||||
|
||||
def _ListItems(Rhs: str) -> list[str] | None:
|
||||
"""`!{ a, b }` -> ['a', 'b'];非列表形式返回 None。"""
|
||||
m = _ListForm.match(Rhs.strip())
|
||||
if not m:
|
||||
return None
|
||||
return _SplitTopLevel(m.group(1))
|
||||
|
||||
|
||||
def _RewriteReferences(Line: str, Mapping: dict[int, int]) -> str:
|
||||
"""改写一行里对编号元数据的引用,跳过字符串字面量与 `;` 注释。"""
|
||||
if '!' not in Line:
|
||||
return Line
|
||||
if '"' not in Line and ';' not in Line:
|
||||
# 元数据行绝大多数不含字符串/注释:整体正则替换,避免逐字符扫描
|
||||
return _Reference.sub(
|
||||
lambda m: '!' + str(Mapping.get(int(m.group(1)), int(m.group(1)))), Line)
|
||||
Out: list[str] = []
|
||||
i: int = 0
|
||||
n: int = len(Line)
|
||||
while i < n:
|
||||
Ch: str = Line[i]
|
||||
if Ch == '"':
|
||||
j: int = i + 1
|
||||
while j < n:
|
||||
if Line[j] == '\\':
|
||||
j += 2
|
||||
continue
|
||||
if Line[j] == '"':
|
||||
j += 1
|
||||
break
|
||||
j += 1
|
||||
Out.append(Line[i:j])
|
||||
i = j
|
||||
continue
|
||||
if Ch == ';':
|
||||
Out.append(Line[i:])
|
||||
break
|
||||
if Ch == '!':
|
||||
m = _Reference.match(Line, i)
|
||||
if m:
|
||||
New: int | None = Mapping.get(int(m.group(1)))
|
||||
Out.append('!' + str(New) if New is not None else m.group(0))
|
||||
i = m.end()
|
||||
continue
|
||||
Out.append(Ch)
|
||||
i += 1
|
||||
continue
|
||||
Out.append(Ch)
|
||||
i += 1
|
||||
return ''.join(Out)
|
||||
|
||||
|
||||
def SanitizeIRMetadata(text: str) -> str:
|
||||
"""让拼接后的 IR 元数据合法:编号全局唯一 + 命名元数据只留一份。
|
||||
|
||||
无元数据(没有任何以 `!` 开头的行)时原样返回 —— 严格 no-op。
|
||||
"""
|
||||
if '!' not in text:
|
||||
return text
|
||||
Segments: list[str] = text.split('\n')
|
||||
LineCount: int = len(Segments)
|
||||
|
||||
# 1) 元数据块 = 连续的以 '!' 开头的行。
|
||||
# 每个模块的元数据(命名行 + 编号定义)是连续的一段;IR 行把它断开,
|
||||
# 所以块边界即模块边界,块内共享同一套局部编号空间。
|
||||
Blocks: list[tuple[int, int]] = [] # [Start, End)
|
||||
i: int = 0
|
||||
while i < LineCount:
|
||||
if Segments[i].startswith('!'):
|
||||
j: int = i
|
||||
while j < LineCount and Segments[j].startswith('!'):
|
||||
j += 1
|
||||
Blocks.append((i, j))
|
||||
i = j
|
||||
else:
|
||||
i += 1
|
||||
if not Blocks:
|
||||
return text
|
||||
|
||||
# 2) 每行归属的块:块内行归本块;块之间的行(IR、命名行)归「其后第一个
|
||||
# 块」—— 因为每个模块的 IR 之后紧跟它自己的元数据块(含被引用的 !N)。
|
||||
Owner: list[int] = [-1] * LineCount
|
||||
PrevEnd: int = 0
|
||||
for Index, (Start, End) in enumerate(Blocks):
|
||||
for k in range(PrevEnd, End):
|
||||
Owner[k] = Index
|
||||
PrevEnd = End
|
||||
|
||||
# 3) 块内按定义出现顺序重新编号(局部 id -> 全局 id)
|
||||
Lines: list[str] = list(Segments)
|
||||
BlockMaps: list[dict[int, int]] = []
|
||||
BlockIsIdentity: list[bool] = []
|
||||
Counter: int = 0
|
||||
for Start, End in Blocks:
|
||||
Mapping: dict[int, int] = {}
|
||||
Identity: bool = True
|
||||
for k in range(Start, End):
|
||||
m = _Definition.match(Lines[k])
|
||||
if not m:
|
||||
continue
|
||||
Local: int = int(m.group(1))
|
||||
Mapping[Local] = Counter
|
||||
if Local != Counter:
|
||||
Identity = False
|
||||
Lines[k] = '!' + str(Counter) + ' = ' + Lines[k][m.end():]
|
||||
Counter += 1
|
||||
BlockMaps.append(Mapping)
|
||||
BlockIsIdentity.append(Identity)
|
||||
|
||||
# 4) 改写全文引用(定义体右值、define/指令行的 !dbg、命名元数据列表)
|
||||
for k in range(LineCount):
|
||||
Index: int = Owner[k]
|
||||
if Index < 0 or BlockIsIdentity[Index] or not BlockMaps[Index]:
|
||||
continue
|
||||
Fixed: str = _RewriteReferences(Lines[k], BlockMaps[Index])
|
||||
if Fixed != Lines[k]:
|
||||
Lines[k] = Fixed
|
||||
|
||||
# 5) 定义体索引(新 id -> 右值文本),供命名元数据判重
|
||||
BodyOf: dict[int, str] = {}
|
||||
for k in range(LineCount):
|
||||
m = _Definition.match(Lines[k])
|
||||
if m:
|
||||
BodyOf[int(m.group(1))] = Lines[k][m.end():].strip()
|
||||
|
||||
# 6) 命名元数据:同名多份合并成一份。
|
||||
# - !llvm.dbg.cu:全部条目聚合(每个 DICompileUnit 都必须被列出,不能去重)
|
||||
# - 其它(!llvm.module.flags、!llvm.ident ...):条目内容(id 引用解析成
|
||||
# 节点正文)完全相同的多份只留一份,不同则聚合。
|
||||
NamedLines: dict[str, list[int]] = {}
|
||||
for k in range(LineCount):
|
||||
m = _NamedDefinition.match(Lines[k])
|
||||
if m:
|
||||
NamedLines.setdefault(m.group(1), []).append(k)
|
||||
|
||||
Dropped: set[int] = set()
|
||||
for Name, Indices in NamedLines.items():
|
||||
if len(Indices) == 1 and Name != 'llvm.dbg.cu':
|
||||
continue
|
||||
First: int = Indices[0]
|
||||
Cr: str = Segments[First][len(Segments[First].rstrip('\r')):]
|
||||
Items: list[str] = []
|
||||
SeenItems: set[str] = set()
|
||||
SeenBodies: set[tuple[str, ...]] = set()
|
||||
Single: bool = True
|
||||
for k in Indices:
|
||||
Rhs: str = _NamedDefinition.match(Lines[k]).group(2).strip()
|
||||
ListItems: list[str] | None = _ListItems(Rhs)
|
||||
if ListItems is None:
|
||||
Single = False
|
||||
break
|
||||
Bodies: tuple[str, ...] = tuple(
|
||||
BodyOf.get(int(_PureId.match(Item).group(1)), Item) if _PureId.match(Item) else Item
|
||||
for Item in ListItems
|
||||
)
|
||||
if Name != 'llvm.dbg.cu':
|
||||
if Bodies in SeenBodies:
|
||||
continue
|
||||
SeenBodies.add(Bodies)
|
||||
for Item in ListItems:
|
||||
if Item not in SeenItems:
|
||||
SeenItems.add(Item)
|
||||
Items.append(Item)
|
||||
if not Single:
|
||||
# 非 `!{ ... }` 列表形式(实际 IR 里不出现):保留首份,其余丢弃,
|
||||
# 避免出现同名命名元数据被定义多次。
|
||||
Dropped.update(Indices[1:])
|
||||
continue
|
||||
Rebuilt: str = '!%s = !{ %s }%s' % (Name, ', '.join(Items), Cr)
|
||||
if Rebuilt != Segments[First]:
|
||||
Lines[First] = Rebuilt
|
||||
Dropped.update(Indices[1:])
|
||||
|
||||
if not Dropped and Lines == Segments:
|
||||
return text
|
||||
return '\n'.join(Line for k, Line in enumerate(Lines) if k not in Dropped)
|
||||
@@ -134,6 +134,12 @@ class StructGenMixin:
|
||||
def finalize(self) -> str:
|
||||
self._set_Vtable_initializers()
|
||||
self._fix_global_var_init()
|
||||
# 函数级 DWARF(--dwarf / --asan):把 Module 落成 .ll 文本之前补模块级
|
||||
# !llvm.dbg.cu 与 !llvm.module.flags(Debug Info Version=3);未启用时
|
||||
# _GetDebugInfo() 返回 None,不做任何事(零影响)。
|
||||
_DbgInfo = self._GetDebugInfo()
|
||||
if _DbgInfo is not None:
|
||||
_DbgInfo.Finalize()
|
||||
ir_text: str
|
||||
try:
|
||||
ir_text = str(self.module)
|
||||
|
||||
@@ -85,6 +85,9 @@ class BaseTranslatorMixin:
|
||||
# 注册策略:False=种子 CLI 单模块(在 main 入口注册);True=Projectrans
|
||||
# 多模块(每模块 @llvm.global_ctors 自注册,见 BaseGen._InstallHotspotSitesModuleLocal)
|
||||
self.HotspotSitesModuleLocal: bool = False
|
||||
# 函数级 DWARF 调试元数据(--dwarf / --asan 自动开启):False=未启用
|
||||
# (零影响),True=生成的 .ll 携带每函数 !dbg(见 LlvmGenerator.GenerateCCode)
|
||||
self.DwarfEnabled: bool = False
|
||||
|
||||
# 导出表
|
||||
self.Exportable = Exportable(filename='')
|
||||
|
||||
@@ -665,6 +665,10 @@ class LlvmGeneratorMixin:
|
||||
self.LlvmGen.setup_from_symbol_table(self.SymbolTable)
|
||||
if hasattr(self, 'CurrentFile') and self.CurrentFile:
|
||||
self.LlvmGen._set_source_info(self.CurrentFile)
|
||||
# 函数级 DWARF 调试元数据(--dwarf / --asan):透传开关给本模块的
|
||||
# LlvmCodeGenerator;未启用时 dwarf_enabled=False → 不构造 DI、不挂 !dbg,
|
||||
# 生成的 .ll 与改动前逐字节一致(零影响)
|
||||
self.LlvmGen.dwarf_enabled = bool(getattr(self, 'DwarfEnabled', False))
|
||||
|
||||
# --- 修复:共享 vtable 状态跨模块 ---
|
||||
# 根因:LlvmGen 每模块重新创建,class_vtable/class_methods 等是实例级属性,
|
||||
|
||||
Reference in new issue
Block a user