feat(tpc): 新增 --func-hotspot 函数热点统计，Ctrl+C 时落盘
启用后为每个函数定义生成 internal i64 计数器并在入口递增；各模块 Ctrl+C 处理器以追加方式写最终文件（含 main 的模块启动时清空一次），经 llvm.global_ctors 自注册；未传参数时零影响。
This commit is contained in:
TermiNexus committed
1 parent
355811d64e
commit
5680592f37
7 files changed
+339
-14
No files matched your search
@@ -26,9 +26,9 @@ from StubGen import PythonToStubConverter
|
||||
# PROF 热点：nt._path_isfile 2.6 万次 16.4s（Windows 磁盘 stat ~0.6ms/次）。
|
||||
# 编译进程内文件状态稳定（obj/ll 一旦生成不删除、不被移动），用模块级缓存
|
||||
# 避免重复 stat。仅用于"存在性检查"类调用（不用于内容变化的判断）。
|
||||
_FILE_STAT_CACHE: dict[str, bool] = {}
|
||||
|
||||
|
||||
_FILE_STAT_CACHE: dict[str, bool] = {}
|
||||
|
||||
|
||||
def _CachedIsFile(path: str) -> bool:
|
||||
r: bool | None = _FILE_STAT_CACHE.get(path)
|
||||
if r is None:
|
||||
@@ -161,7 +161,8 @@ def _parallel_translate_worker(src_path: str, out_path: str, src_root: str, temp
|
||||
sha1_map: dict[str, str], sig_files: dict[str, str],
|
||||
stub_files: dict[str, str], include_py_map: dict[str, str], slice_level: int,
|
||||
shared_sym_pickle_path: str, struct_sha1_map: dict[str, str],
|
||||
prebuilt_ModuleSha1Map: dict[str, str] | None = None) -> tuple[str, str, str]:
|
||||
prebuilt_ModuleSha1Map: dict[str, str] | None = None,
|
||||
func_hotspot_file: str | None = None) -> tuple[str, str, str]:
|
||||
# 注意：不可跨任务复用 Phase2Translator/共享符号表！_translate_file 编译过程中
|
||||
# 会向共享 SymbolTable 写入符号（import insert/类注册等），复用导致前文件的
|
||||
# 符号污染后文件 → 跨模块 struct 类型前缀错位（如 7cadfe07e983fd90.CompilerFactory
|
||||
@@ -179,6 +180,9 @@ def _parallel_translate_worker(src_path: str, out_path: str, src_root: str, temp
|
||||
trans.include_py_map = include_py_map
|
||||
trans.slice_level = slice_level
|
||||
trans.struct_sha1_map = struct_sha1_map
|
||||
# 函数热点统计（--func-hotspot）：worker 子进程无父进程属性，须显式透传
|
||||
# （None=未启用，零影响）；生成 IR 时各模块处理器统一追加写最终 <path>
|
||||
trans.func_hotspot_file = func_hotspot_file
|
||||
|
||||
if shared_sym_pickle_path and os.path.exists(shared_sym_pickle_path):
|
||||
# 使用 pickle.load(f) 流式加载，避免 pickle.loads(f.read()) 一次性
|
||||
@@ -222,7 +226,8 @@ def _parallel_translate_batch(batch: list, src_root: str, temp_dir: str, output_
|
||||
include_dirs: list[str], triple: str | None, datalayout: str | None,
|
||||
sha1_map: dict[str, str], sig_files: dict[str, str],
|
||||
stub_files: dict[str, str], include_py_map: dict[str, str], slice_level: int,
|
||||
shared_sym_pickle_path: str, struct_sha1_map: dict[str, str]) -> tuple[list, list]:
|
||||
shared_sym_pickle_path: str, struct_sha1_map: dict[str, str],
|
||||
func_hotspot_file: str | None = None) -> tuple[list, list]:
|
||||
"""批次子任务：worker 内创建一个 Phase2Translator + pickle.load 一次，顺序翻译批内全部文件。
|
||||
|
||||
batch 每项: (i, src_path, out_path, sha1, prebuilt_ModuleSha1Map)
|
||||
@@ -243,6 +248,8 @@ def _parallel_translate_batch(batch: list, src_root: str, temp_dir: str, output_
|
||||
trans.include_py_map = include_py_map
|
||||
trans.slice_level = slice_level
|
||||
trans.struct_sha1_map = struct_sha1_map
|
||||
# 函数热点统计（--func-hotspot）：批内每文件的处理器统一追加写最终 <path>
|
||||
trans.func_hotspot_file = func_hotspot_file
|
||||
|
||||
symbol_table_bytes: bytes | None = None
|
||||
if shared_sym_pickle_path and os.path.exists(shared_sym_pickle_path):
|
||||
@@ -640,7 +647,11 @@ class Phase2Translator:
|
||||
if os.path.isdir(_clang64_bin):
|
||||
os.environ['PATH'] = _clang64_bin + os.pathsep + os.environ.get('PATH', '')
|
||||
self.asan_rt_dir = _FindAsanRuntimeDir()
|
||||
self.slice_level = 3
|
||||
self.slice_level = 3
|
||||
# 函数热点统计（--func-hotspot）：None=未启用（零影响）；str=计数器输出
|
||||
# 文件路径。各模块运行时自带 Ctrl+C 处理器、统一以追加方式写该路径；
|
||||
# 含 main 的模块在 main 入口清空一次 → 每次运行一份结果（无构建后合并步骤）。
|
||||
self.func_hotspot_file: str | None = None
|
||||
self.sha1_map: dict[str, str] = {}
|
||||
self.sig_files: dict[str, str] = {}
|
||||
self.stub_files: dict[str, str] = {}
|
||||
@@ -844,6 +855,10 @@ class Phase2Translator:
|
||||
self.struct_sha1_map = self._build_struct_sha1_map()
|
||||
|
||||
need_translate = []
|
||||
# --func-hotspot：缓存 .ll / .stub.ll+.text.ll 可能不含热点计数 IR，启用时
|
||||
# 一律重译，保证每个模块都嵌入 internal 计数器与 @llvm.global_ctors 自注册。
|
||||
# 未启用时 hotspot_on=False，缓存与重组分支行为与改动前完全一致（零影响）。
|
||||
hotspot_on: bool = bool(self.func_hotspot_file)
|
||||
for i, src_path in enumerate(py_files, 1):
|
||||
rel = os.path.relpath(src_path, self.src_root)
|
||||
# 复用 parse_python_file 的 AST 缓存（get_file_dependencies 已 parse 过）
|
||||
@@ -889,7 +904,7 @@ class Phase2Translator:
|
||||
raise
|
||||
_vlog().warning(f"收集源文件导入的 include 模块失败: {_e}", "Exception")
|
||||
|
||||
if _CachedIsFile(out_path):
|
||||
if not hotspot_on and _CachedIsFile(out_path):
|
||||
deps_changed = self._check_deps_changed(sha1, current_ModuleSha1Map)
|
||||
if not deps_changed:
|
||||
_vlog().info(f"[{i}/{len(py_files)}] 跳过(缓存): {rel} -> {sha1}.ll")
|
||||
@@ -900,7 +915,7 @@ class Phase2Translator:
|
||||
continue
|
||||
_vlog().info(f"[{i}/{len(py_files)}] 重译(依赖变化): {rel} -> {sha1}.ll")
|
||||
|
||||
else:
|
||||
elif not hotspot_on:
|
||||
stub_cache = os.path.join(self.output_dir, f"{sha1}.stub.ll")
|
||||
text_cache = os.path.join(self.output_dir, f"{sha1}.text.ll")
|
||||
if _CachedIsFile(stub_cache) and _CachedIsFile(text_cache):
|
||||
@@ -1002,6 +1017,7 @@ class Phase2Translator:
|
||||
self.sha1_map, self.sig_files, self.stub_files,
|
||||
self.include_py_map, self.slice_level,
|
||||
shared_pickle_path, self.struct_sha1_map,
|
||||
func_hotspot_file=self.func_hotspot_file,
|
||||
)
|
||||
futures[future] = (bidx, rel_first, len(batch))
|
||||
|
||||
@@ -1276,6 +1292,11 @@ class Phase2Translator:
|
||||
|
||||
if stripped.startswith('@') and ' global ' in stripped and 'external global' not in stripped:
|
||||
text_lines.append(line)
|
||||
# LLVM 保留全局（如 --func-hotspot 的 @llvm.global_ctors；名字含点故
|
||||
# 最终 IR 保留引号）是 appending 定义，禁止改写为 "external global"
|
||||
# 存根（否则 .ll 重组后 llc 报错）
|
||||
if re.match(r'@("?)llvm\.', stripped):
|
||||
continue
|
||||
if ' internal ' not in stripped and ' private ' not in stripped:
|
||||
name_match = re.match(r'(@\S+)', stripped)
|
||||
if name_match:
|
||||
@@ -1537,6 +1558,14 @@ class Phase2Translator:
|
||||
trans.translator._shared_class_methods = self._shared_class_methods
|
||||
trans.translator._shared_cross_module_novtable = self._shared_cross_module_novtable
|
||||
|
||||
# 函数热点统计（--func-hotspot，Projectrans 多模块）：
|
||||
# 各模块处理器统一以追加方式写最终 <path>；注册策略为本模块
|
||||
# @llvm.global_ctors 自注册（见 BaseGen._InstallFuncHotspotModuleLocal）。
|
||||
# self.func_hotspot_file 为 None 时不设置任何属性 → 与未启用时逐字节一致（零影响）。
|
||||
if self.func_hotspot_file:
|
||||
trans.translator.FuncHotspotFile = self.func_hotspot_file
|
||||
trans.translator.FuncHotspotModuleLocal = True
|
||||
|
||||
if self._shared_symbol_table is not None:
|
||||
trans.translator.SymbolTable = self._shared_symbol_table
|
||||
# 确保共享符号表的 translator 指向当前 translator，以便 import 别名解析正确
|
||||
@@ -1913,7 +1942,7 @@ class Phase2Translator:
|
||||
opt_content = re.sub(r'^(define|declare)\s+(?!private|internal|available_externally|linkonce|weak|common|appending|extern_weak|linkonce_odr|weak_odr|external|dso_local|@)', r'\1 dso_local ', opt_content, flags=re.MULTILINE)
|
||||
# LLVM 22+: external 不允许在带初值的全局定义上显式写出，改为 linkonce 保留多模块合并语义
|
||||
opt_content = re.sub(r'(@[\w.]+)\s*=\s*external\s+global\s+(\w[\w*]*)\s+(-?\d+|null|zeroinitializer|true|false)$', r'\1 = linkonce global \2 \3', opt_content, flags=re.MULTILINE)
|
||||
opt_content = re.sub(r'(@[\w.]+)\s*=\s*(private|internal|available_externally|linkonce|weak|common|appending|extern_weak|linkonce_odr|weak_odr)\s+(global|constant)', r'\1 = \2 dso_local \3', opt_content)
|
||||
opt_content = re.sub(r'(@(?!llvm\.)[\w.]+)\s*=\s*(private|internal|available_externally|linkonce|weak|common|appending|extern_weak|linkonce_odr|weak_odr)\s+(global|constant)', r'\1 = \2 dso_local \3', opt_content)
|
||||
opt_content = re.sub(r'(@[\w.]+)\s*=\s+(?!private|internal|available_externally|linkonce|weak|common|appending|extern_weak|linkonce_odr|weak_odr|dso_local)(global|constant)', r'\1 = dso_local \2', opt_content)
|
||||
opt_content = _AddWeakComdat(opt_content)
|
||||
with open(optimized_ll, 'w', encoding='utf-8') as f:
|
||||
@@ -2147,7 +2176,7 @@ class Phase2Translator:
|
||||
ll_content = re.sub(r'^(define|declare)\s+(?!private|internal|available_externally|linkonce|weak|common|appending|extern_weak|linkonce_odr|weak_odr|external|dso_local|@)', r'\1 dso_local ', ll_content, flags=re.MULTILINE)
|
||||
# LLVM 22+: external 不允许在带初值的全局定义上显式写出，改为 linkonce 保留多模块合并语义
|
||||
ll_content = re.sub(r'^(@[\w.]+)\s*=\s*external\s+global\s+(\w[\w*]*)\s+(-?\d+|null|zeroinitializer|true|false)$', r'\1 = linkonce global \2 \3', ll_content, flags=re.MULTILINE)
|
||||
ll_content = re.sub(r'^(@[\w.]+)\s*=\s*(private|internal|available_externally|linkonce|weak|common|appending|extern_weak|linkonce_odr|weak_odr)\s+(global|constant)', r'\1 = \2 dso_local \3', ll_content, flags=re.MULTILINE)
|
||||
ll_content = re.sub(r'(@(?!llvm\.)[\w.]+)\s*=\s*(private|internal|available_externally|linkonce|weak|common|appending|extern_weak|linkonce_odr|weak_odr)\s+(global|constant)', r'\1 = \2 dso_local \3', ll_content, flags=re.MULTILINE)
|
||||
ll_content = re.sub(r'^(@[\w.]+)\s*=\s+(?!private|internal|available_externally|linkonce|weak|common|appending|extern_weak|linkonce_odr|weak_odr|dso_local)(global|constant)', r'\1 = dso_local \2', ll_content, flags=re.MULTILINE)
|
||||
ll_content = _AddWeakComdat(ll_content)
|
||||
# LLVM 22+: 移除同一模块中已有 define 的 declare 语句（LLVM 22 将 declare+define 视为重复定义）
|
||||
@@ -2815,7 +2844,8 @@ class Phase2Translator:
|
||||
self.sha1_map, self.sig_files, self.stub_files,
|
||||
self.include_py_map, self.slice_level,
|
||||
shared_pickle_path, self.struct_sha1_map,
|
||||
include_ModuleSha1Map
|
||||
include_ModuleSha1Map,
|
||||
func_hotspot_file=self.func_hotspot_file
|
||||
)
|
||||
in_progress[future] = item
|
||||
|
||||
@@ -2831,7 +2861,8 @@ class Phase2Translator:
|
||||
self.sha1_map, self.sig_files, self.stub_files,
|
||||
self.include_py_map, self.slice_level,
|
||||
shared_pickle_path, self.struct_sha1_map,
|
||||
include_ModuleSha1Map
|
||||
include_ModuleSha1Map,
|
||||
func_hotspot_file=self.func_hotspot_file
|
||||
)
|
||||
in_progress[future] = item
|
||||
remaining.clear()
|
||||
@@ -2898,7 +2929,7 @@ class Phase2Translator:
|
||||
ll_content = re.sub(r'^(define|declare)\s+(?!private|internal|available_externally|linkonce|weak|common|appending|extern_weak|linkonce_odr|weak_odr|external|dso_local)', r'\1 dso_local ', ll_content, flags=re.MULTILINE)
|
||||
# LLVM 22+: external 不允许在带初值的全局定义上显式写出，改为 linkonce 保留多模块合并语义
|
||||
ll_content = re.sub(r'^(@[\w.]+)\s*=\s*external\s+global\s+(\w[\w*]*)\s+(-?\d+|null|zeroinitializer|true|false)$', r'\1 = linkonce global \2 \3', ll_content, flags=re.MULTILINE)
|
||||
ll_content = re.sub(r'^(@[\w.]+)\s*=\s*(private|internal|available_externally|linkonce|weak|common|appending|extern_weak|linkonce_odr|weak_odr)\s+(global|constant)', r'\1 = \2 dso_local \3', ll_content, flags=re.MULTILINE)
|
||||
ll_content = re.sub(r'(@(?!llvm\.)[\w.]+)\s*=\s*(private|internal|available_externally|linkonce|weak|common|appending|extern_weak|linkonce_odr|weak_odr)\s+(global|constant)', r'\1 = \2 dso_local \3', ll_content, flags=re.MULTILINE)
|
||||
ll_content = re.sub(r'^(@[\w.]+)\s*=\s+(?!private|internal|available_externally|linkonce|weak|common|appending|extern_weak|linkonce_odr|weak_odr|dso_local)(global|constant)', r'\1 = dso_local \2', ll_content, flags=re.MULTILINE)
|
||||
ll_content = _AddWeakComdat(ll_content)
|
||||
|
||||
|
||||
Reference in new issue
Block a user