feat(tpc): 新增 --func-hotspot 函数热点统计,Ctrl+C 时落盘
启用后为每个函数定义生成 internal i64 计数器并在入口递增;各模块 Ctrl+C 处理器以追加方式写最终文件(含 main 的模块启动时清空一次),经 llvm.global_ctors 自注册;未传参数时零影响。
This commit is contained in:
1 parent
355811d64e
commit
5680592f37
7 files changed
+339
-14
No files matched your search
@@ -149,6 +149,7 @@ def main() -> None:
|
||||
parser.add_argument('--clear-cache', action='store_true', help='清除 .transpyc_cache 全局缓存(.vpi/.stub.ll/.doc.json)')
|
||||
parser.add_argument('--asan', action='store_true', help='ASan 模式:对全部 .ll 做 opt -passes=asan 插桩,并显式链接 clang64 的 ASan 运行库(mingw64 的 clang 没有 x86_64-w64-windows-gnu 目标的 ASan runtime,不能用 -fsanitize=address)')
|
||||
parser.add_argument('--nowarn-register', action='store_true', help='抑制"外部函数返回类型未登记"类 WARN 输出(透传给自举编译的 TransPyV.exe)')
|
||||
parser.add_argument('--func-hotspot', help='函数热点统计:为每个模块的函数定义插入计数器,运行时 Ctrl+C 各模块以追加方式把计数写入 <path>(含 main 的模块在启动时清空一次,保证每次运行一份结果)(可用 python Projectrans.py --project ... --func-hotspot <path>)')
|
||||
|
||||
args: argparse.Namespace = parser.parse_args()
|
||||
|
||||
@@ -193,6 +194,11 @@ def main() -> None:
|
||||
pass
|
||||
target_triple: str | None = resolved.get('target', {}).get('triple')
|
||||
target_datalayout: str | None = resolved.get('target', {}).get('datalayout')
|
||||
# 函数热点统计(--func-hotspot):解析为绝对路径,保证运行时(exe 工作目录为
|
||||
# output_dir)各模块处理器追加写入的路径与预期完全一致。
|
||||
func_hotspot_file: str | None = args.func_hotspot
|
||||
if func_hotspot_file:
|
||||
func_hotspot_file = os.path.abspath(func_hotspot_file)
|
||||
|
||||
if isinstance(src, list):
|
||||
if len(src) == 1 and os.path.isdir(src[0]):
|
||||
@@ -209,6 +215,8 @@ def main() -> None:
|
||||
_vlog().info(f"目标三元组: {target_triple}")
|
||||
if target_datalayout:
|
||||
_vlog().info(f"数据布局: {target_datalayout}")
|
||||
if func_hotspot_file:
|
||||
_vlog().info(f"函数热点统计: {func_hotspot_file}(各模块运行时以追加方式写入,含 main 的模块启动时清空一次)")
|
||||
|
||||
# 每次编译自动清理子进程性能剖析输出(PROF 目录 *.prof,硬编码路径)
|
||||
from lib.Projectrans.Phase2Translator import _CleanProfFiles, PROF_DIR
|
||||
@@ -314,6 +322,8 @@ def main() -> None:
|
||||
trans: Phase2Translator = Phase2Translator(src, temp_dir, output_dir, compile_cmd=cc_cmd, compile_flags=cc_flags, linker_cmd=linker_cmd, linker_flags=linker_flags, linker_output=linker_output, static_libs=static_libs, dynamic_libs=dynamic_libs, include_dirs=project_include_dirs, target_triple=target_triple, target_datalayout=target_datalayout, force_recompile_includes=force_recompile_includes, asan_mode=args.asan)
|
||||
trans.sha1_map = Load_sha1_map(temp_dir)
|
||||
trans.slice_level = slice_level
|
||||
# 函数热点统计(--func-hotspot):None=未启用(零影响)
|
||||
trans.func_hotspot_file = func_hotspot_file
|
||||
trans.run()
|
||||
|
||||
if args.run and linker_output and not linker_output.endswith('.bin'):
|
||||
|
||||
@@ -26,9 +26,9 @@ from StubGen import PythonToStubConverter
|
||||
# PROF 热点:nt._path_isfile 2.6 万次 16.4s(Windows 磁盘 stat ~0.6ms/次)。
|
||||
# 编译进程内文件状态稳定(obj/ll 一旦生成不删除、不被移动),用模块级缓存
|
||||
# 避免重复 stat。仅用于"存在性检查"类调用(不用于内容变化的判断)。
|
||||
_FILE_STAT_CACHE: dict[str, bool] = {}
|
||||
|
||||
|
||||
_FILE_STAT_CACHE: dict[str, bool] = {}
|
||||
|
||||
|
||||
def _CachedIsFile(path: str) -> bool:
|
||||
r: bool | None = _FILE_STAT_CACHE.get(path)
|
||||
if r is None:
|
||||
@@ -161,7 +161,8 @@ def _parallel_translate_worker(src_path: str, out_path: str, src_root: str, temp
|
||||
sha1_map: dict[str, str], sig_files: dict[str, str],
|
||||
stub_files: dict[str, str], include_py_map: dict[str, str], slice_level: int,
|
||||
shared_sym_pickle_path: str, struct_sha1_map: dict[str, str],
|
||||
prebuilt_ModuleSha1Map: dict[str, str] | None = None) -> tuple[str, str, str]:
|
||||
prebuilt_ModuleSha1Map: dict[str, str] | None = None,
|
||||
func_hotspot_file: str | None = None) -> tuple[str, str, str]:
|
||||
# 注意:不可跨任务复用 Phase2Translator/共享符号表!_translate_file 编译过程中
|
||||
# 会向共享 SymbolTable 写入符号(import insert/类注册等),复用导致前文件的
|
||||
# 符号污染后文件 → 跨模块 struct 类型前缀错位(如 7cadfe07e983fd90.CompilerFactory
|
||||
@@ -179,6 +180,9 @@ def _parallel_translate_worker(src_path: str, out_path: str, src_root: str, temp
|
||||
trans.include_py_map = include_py_map
|
||||
trans.slice_level = slice_level
|
||||
trans.struct_sha1_map = struct_sha1_map
|
||||
# 函数热点统计(--func-hotspot):worker 子进程无父进程属性,须显式透传
|
||||
# (None=未启用,零影响);生成 IR 时各模块处理器统一追加写最终 <path>
|
||||
trans.func_hotspot_file = func_hotspot_file
|
||||
|
||||
if shared_sym_pickle_path and os.path.exists(shared_sym_pickle_path):
|
||||
# 使用 pickle.load(f) 流式加载,避免 pickle.loads(f.read()) 一次性
|
||||
@@ -222,7 +226,8 @@ def _parallel_translate_batch(batch: list, src_root: str, temp_dir: str, output_
|
||||
include_dirs: list[str], triple: str | None, datalayout: str | None,
|
||||
sha1_map: dict[str, str], sig_files: dict[str, str],
|
||||
stub_files: dict[str, str], include_py_map: dict[str, str], slice_level: int,
|
||||
shared_sym_pickle_path: str, struct_sha1_map: dict[str, str]) -> tuple[list, list]:
|
||||
shared_sym_pickle_path: str, struct_sha1_map: dict[str, str],
|
||||
func_hotspot_file: str | None = None) -> tuple[list, list]:
|
||||
"""批次子任务:worker 内创建一个 Phase2Translator + pickle.load 一次,顺序翻译批内全部文件。
|
||||
|
||||
batch 每项: (i, src_path, out_path, sha1, prebuilt_ModuleSha1Map)
|
||||
@@ -243,6 +248,8 @@ def _parallel_translate_batch(batch: list, src_root: str, temp_dir: str, output_
|
||||
trans.include_py_map = include_py_map
|
||||
trans.slice_level = slice_level
|
||||
trans.struct_sha1_map = struct_sha1_map
|
||||
# 函数热点统计(--func-hotspot):批内每文件的处理器统一追加写最终 <path>
|
||||
trans.func_hotspot_file = func_hotspot_file
|
||||
|
||||
symbol_table_bytes: bytes | None = None
|
||||
if shared_sym_pickle_path and os.path.exists(shared_sym_pickle_path):
|
||||
@@ -640,7 +647,11 @@ class Phase2Translator:
|
||||
if os.path.isdir(_clang64_bin):
|
||||
os.environ['PATH'] = _clang64_bin + os.pathsep + os.environ.get('PATH', '')
|
||||
self.asan_rt_dir = _FindAsanRuntimeDir()
|
||||
self.slice_level = 3
|
||||
self.slice_level = 3
|
||||
# 函数热点统计(--func-hotspot):None=未启用(零影响);str=计数器输出
|
||||
# 文件路径。各模块运行时自带 Ctrl+C 处理器、统一以追加方式写该路径;
|
||||
# 含 main 的模块在 main 入口清空一次 → 每次运行一份结果(无构建后合并步骤)。
|
||||
self.func_hotspot_file: str | None = None
|
||||
self.sha1_map: dict[str, str] = {}
|
||||
self.sig_files: dict[str, str] = {}
|
||||
self.stub_files: dict[str, str] = {}
|
||||
@@ -844,6 +855,10 @@ class Phase2Translator:
|
||||
self.struct_sha1_map = self._build_struct_sha1_map()
|
||||
|
||||
need_translate = []
|
||||
# --func-hotspot:缓存 .ll / .stub.ll+.text.ll 可能不含热点计数 IR,启用时
|
||||
# 一律重译,保证每个模块都嵌入 internal 计数器与 @llvm.global_ctors 自注册。
|
||||
# 未启用时 hotspot_on=False,缓存与重组分支行为与改动前完全一致(零影响)。
|
||||
hotspot_on: bool = bool(self.func_hotspot_file)
|
||||
for i, src_path in enumerate(py_files, 1):
|
||||
rel = os.path.relpath(src_path, self.src_root)
|
||||
# 复用 parse_python_file 的 AST 缓存(get_file_dependencies 已 parse 过)
|
||||
@@ -889,7 +904,7 @@ class Phase2Translator:
|
||||
raise
|
||||
_vlog().warning(f"收集源文件导入的 include 模块失败: {_e}", "Exception")
|
||||
|
||||
if _CachedIsFile(out_path):
|
||||
if not hotspot_on and _CachedIsFile(out_path):
|
||||
deps_changed = self._check_deps_changed(sha1, current_ModuleSha1Map)
|
||||
if not deps_changed:
|
||||
_vlog().info(f"[{i}/{len(py_files)}] 跳过(缓存): {rel} -> {sha1}.ll")
|
||||
@@ -900,7 +915,7 @@ class Phase2Translator:
|
||||
continue
|
||||
_vlog().info(f"[{i}/{len(py_files)}] 重译(依赖变化): {rel} -> {sha1}.ll")
|
||||
|
||||
else:
|
||||
elif not hotspot_on:
|
||||
stub_cache = os.path.join(self.output_dir, f"{sha1}.stub.ll")
|
||||
text_cache = os.path.join(self.output_dir, f"{sha1}.text.ll")
|
||||
if _CachedIsFile(stub_cache) and _CachedIsFile(text_cache):
|
||||
@@ -1002,6 +1017,7 @@ class Phase2Translator:
|
||||
self.sha1_map, self.sig_files, self.stub_files,
|
||||
self.include_py_map, self.slice_level,
|
||||
shared_pickle_path, self.struct_sha1_map,
|
||||
func_hotspot_file=self.func_hotspot_file,
|
||||
)
|
||||
futures[future] = (bidx, rel_first, len(batch))
|
||||
|
||||
@@ -1276,6 +1292,11 @@ class Phase2Translator:
|
||||
|
||||
if stripped.startswith('@') and ' global ' in stripped and 'external global' not in stripped:
|
||||
text_lines.append(line)
|
||||
# LLVM 保留全局(如 --func-hotspot 的 @llvm.global_ctors;名字含点故
|
||||
# 最终 IR 保留引号)是 appending 定义,禁止改写为 "external global"
|
||||
# 存根(否则 .ll 重组后 llc 报错)
|
||||
if re.match(r'@("?)llvm\.', stripped):
|
||||
continue
|
||||
if ' internal ' not in stripped and ' private ' not in stripped:
|
||||
name_match = re.match(r'(@\S+)', stripped)
|
||||
if name_match:
|
||||
@@ -1537,6 +1558,14 @@ class Phase2Translator:
|
||||
trans.translator._shared_class_methods = self._shared_class_methods
|
||||
trans.translator._shared_cross_module_novtable = self._shared_cross_module_novtable
|
||||
|
||||
# 函数热点统计(--func-hotspot,Projectrans 多模块):
|
||||
# 各模块处理器统一以追加方式写最终 <path>;注册策略为本模块
|
||||
# @llvm.global_ctors 自注册(见 BaseGen._InstallFuncHotspotModuleLocal)。
|
||||
# self.func_hotspot_file 为 None 时不设置任何属性 → 与未启用时逐字节一致(零影响)。
|
||||
if self.func_hotspot_file:
|
||||
trans.translator.FuncHotspotFile = self.func_hotspot_file
|
||||
trans.translator.FuncHotspotModuleLocal = True
|
||||
|
||||
if self._shared_symbol_table is not None:
|
||||
trans.translator.SymbolTable = self._shared_symbol_table
|
||||
# 确保共享符号表的 translator 指向当前 translator,以便 import 别名解析正确
|
||||
@@ -1913,7 +1942,7 @@ class Phase2Translator:
|
||||
opt_content = re.sub(r'^(define|declare)\s+(?!private|internal|available_externally|linkonce|weak|common|appending|extern_weak|linkonce_odr|weak_odr|external|dso_local|@)', r'\1 dso_local ', opt_content, flags=re.MULTILINE)
|
||||
# LLVM 22+: external 不允许在带初值的全局定义上显式写出,改为 linkonce 保留多模块合并语义
|
||||
opt_content = re.sub(r'(@[\w.]+)\s*=\s*external\s+global\s+(\w[\w*]*)\s+(-?\d+|null|zeroinitializer|true|false)$', r'\1 = linkonce global \2 \3', opt_content, flags=re.MULTILINE)
|
||||
opt_content = re.sub(r'(@[\w.]+)\s*=\s*(private|internal|available_externally|linkonce|weak|common|appending|extern_weak|linkonce_odr|weak_odr)\s+(global|constant)', r'\1 = \2 dso_local \3', opt_content)
|
||||
opt_content = re.sub(r'(@(?!llvm\.)[\w.]+)\s*=\s*(private|internal|available_externally|linkonce|weak|common|appending|extern_weak|linkonce_odr|weak_odr)\s+(global|constant)', r'\1 = \2 dso_local \3', opt_content)
|
||||
opt_content = re.sub(r'(@[\w.]+)\s*=\s+(?!private|internal|available_externally|linkonce|weak|common|appending|extern_weak|linkonce_odr|weak_odr|dso_local)(global|constant)', r'\1 = dso_local \2', opt_content)
|
||||
opt_content = _AddWeakComdat(opt_content)
|
||||
with open(optimized_ll, 'w', encoding='utf-8') as f:
|
||||
@@ -2147,7 +2176,7 @@ class Phase2Translator:
|
||||
ll_content = re.sub(r'^(define|declare)\s+(?!private|internal|available_externally|linkonce|weak|common|appending|extern_weak|linkonce_odr|weak_odr|external|dso_local|@)', r'\1 dso_local ', ll_content, flags=re.MULTILINE)
|
||||
# LLVM 22+: external 不允许在带初值的全局定义上显式写出,改为 linkonce 保留多模块合并语义
|
||||
ll_content = re.sub(r'^(@[\w.]+)\s*=\s*external\s+global\s+(\w[\w*]*)\s+(-?\d+|null|zeroinitializer|true|false)$', r'\1 = linkonce global \2 \3', ll_content, flags=re.MULTILINE)
|
||||
ll_content = re.sub(r'^(@[\w.]+)\s*=\s*(private|internal|available_externally|linkonce|weak|common|appending|extern_weak|linkonce_odr|weak_odr)\s+(global|constant)', r'\1 = \2 dso_local \3', ll_content, flags=re.MULTILINE)
|
||||
ll_content = re.sub(r'(@(?!llvm\.)[\w.]+)\s*=\s*(private|internal|available_externally|linkonce|weak|common|appending|extern_weak|linkonce_odr|weak_odr)\s+(global|constant)', r'\1 = \2 dso_local \3', ll_content, flags=re.MULTILINE)
|
||||
ll_content = re.sub(r'^(@[\w.]+)\s*=\s+(?!private|internal|available_externally|linkonce|weak|common|appending|extern_weak|linkonce_odr|weak_odr|dso_local)(global|constant)', r'\1 = dso_local \2', ll_content, flags=re.MULTILINE)
|
||||
ll_content = _AddWeakComdat(ll_content)
|
||||
# LLVM 22+: 移除同一模块中已有 define 的 declare 语句(LLVM 22 将 declare+define 视为重复定义)
|
||||
@@ -2815,7 +2844,8 @@ class Phase2Translator:
|
||||
self.sha1_map, self.sig_files, self.stub_files,
|
||||
self.include_py_map, self.slice_level,
|
||||
shared_pickle_path, self.struct_sha1_map,
|
||||
include_ModuleSha1Map
|
||||
include_ModuleSha1Map,
|
||||
func_hotspot_file=self.func_hotspot_file
|
||||
)
|
||||
in_progress[future] = item
|
||||
|
||||
@@ -2831,7 +2861,8 @@ class Phase2Translator:
|
||||
self.sha1_map, self.sig_files, self.stub_files,
|
||||
self.include_py_map, self.slice_level,
|
||||
shared_pickle_path, self.struct_sha1_map,
|
||||
include_ModuleSha1Map
|
||||
include_ModuleSha1Map,
|
||||
func_hotspot_file=self.func_hotspot_file
|
||||
)
|
||||
in_progress[future] = item
|
||||
remaining.clear()
|
||||
@@ -2898,7 +2929,7 @@ class Phase2Translator:
|
||||
ll_content = re.sub(r'^(define|declare)\s+(?!private|internal|available_externally|linkonce|weak|common|appending|extern_weak|linkonce_odr|weak_odr|external|dso_local)', r'\1 dso_local ', ll_content, flags=re.MULTILINE)
|
||||
# LLVM 22+: external 不允许在带初值的全局定义上显式写出,改为 linkonce 保留多模块合并语义
|
||||
ll_content = re.sub(r'^(@[\w.]+)\s*=\s*external\s+global\s+(\w[\w*]*)\s+(-?\d+|null|zeroinitializer|true|false)$', r'\1 = linkonce global \2 \3', ll_content, flags=re.MULTILINE)
|
||||
ll_content = re.sub(r'^(@[\w.]+)\s*=\s*(private|internal|available_externally|linkonce|weak|common|appending|extern_weak|linkonce_odr|weak_odr)\s+(global|constant)', r'\1 = \2 dso_local \3', ll_content, flags=re.MULTILINE)
|
||||
ll_content = re.sub(r'(@(?!llvm\.)[\w.]+)\s*=\s*(private|internal|available_externally|linkonce|weak|common|appending|extern_weak|linkonce_odr|weak_odr)\s+(global|constant)', r'\1 = \2 dso_local \3', ll_content, flags=re.MULTILINE)
|
||||
ll_content = re.sub(r'^(@[\w.]+)\s*=\s+(?!private|internal|available_externally|linkonce|weak|common|appending|extern_weak|linkonce_odr|weak_odr|dso_local)(global|constant)', r'\1 = dso_local \2', ll_content, flags=re.MULTILINE)
|
||||
ll_content = _AddWeakComdat(ll_content)
|
||||
|
||||
|
||||
Reference in new issue
Block a user