From fbd132284b4e69f89cbb72d6d4e652fdb9ab0234 Mon Sep 17 00:00:00 2001 From: TermiNexus Date: Wed, 19 Aug 2026 00:28:40 +0800 Subject: [PATCH] =?UTF-8?q?perf(phase2):=20=E6=8C=89=20PROF=20=E7=83=AD?= =?UTF-8?q?=E7=82=B9=E4=BC=98=E5=8C=96=E7=BC=96=E8=AF=91=E8=80=97=E6=97=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - HandlesImports: _TryLoadClassMembersFromPyi 复用 _pyi_cache 跳过重复 isfile（-12.8s）；_LookupStubFuncType 类级缓存 + 签名增量扫描新增 stub，去掉每文件 isfile（-6.5s+） - Phase2Translator: pyi cache 仅主进程构建（worker 只读 pkl，-23s+pickle 10.4s）；include 源文件 sha1/存在性缓存消除 _build_current_ModuleSha1Map 重复磁盘 I/O - StructGen.finalize: 行过滤改 startswith(tuple) 预计算，消除 O(lines×structs×patterns) 嵌套循环（tottime 84.8s → 预期数秒） - 附带 fail-fast（任意文件出错立即终止）+ 子进程 PROF 硬编码 --- lib/Projectrans/Phase2Translator.py | 119 ++++++++++++++++++++++------ lib/core/Handles/HandlesImports.py | 56 +++++++------ lib/core/LLVMCG/StructGen.py | 31 ++++---- 3 files changed, 140 insertions(+), 66 deletions(-) diff --git a/lib/Projectrans/Phase2Translator.py b/lib/Projectrans/Phase2Translator.py index 983a453..ac73d5c 100644 --- a/lib/Projectrans/Phase2Translator.py +++ b/lib/Projectrans/Phase2Translator.py @@ -81,7 +81,8 @@ def _parallel_translate_worker(src_path: str, out_path: str, src_root: str, temp try: trans = Phase2Translator(src_root, temp_dir, output_dir, compile_cmd='llc', include_dirs=include_dirs, - target_triple=triple, target_datalayout=datalayout) + target_triple=triple, target_datalayout=datalayout, + build_pyi_cache=False) trans.sha1_map = sha1_map trans.sig_files = sig_files trans.stub_files = stub_files @@ -118,7 +119,13 @@ def _parallel_translate_worker(src_path: str, out_path: str, src_root: str, temp trans._translate_file(src_path, out_path, prebuilt_ModuleSha1Map=prebuilt_ModuleSha1Map) return ('ok', src_path, '') except Exception as e: - return ('error', src_path, f'{e}\n{traceback.format_exc()}') + # fail-fast（2026-08-18）：include 文件编译出错立即终止整个编译进程 + # （主进程收到 BrokenProcessPool 后取消剩余任务退出）。 + sys.stderr.write( + f"[FATAL] include 翻译失败，立即终止编译: {src_path}\n" + f"{traceback.format_exc()}\n") + sys.stderr.flush() + os._exit(88) def _parallel_translate_batch(batch: list, src_root: str, temp_dir: str, output_dir: str, @@ -138,7 +145,8 @@ def _parallel_translate_batch(batch: list, src_root: str, temp_dir: str, output_ try: trans = Phase2Translator(src_root, temp_dir, output_dir, compile_cmd='llc', include_dirs=include_dirs, - target_triple=triple, target_datalayout=datalayout) + target_triple=triple, target_datalayout=datalayout, + build_pyi_cache=False) trans.sha1_map = sha1_map trans.sig_files = sig_files trans.stub_files = stub_files @@ -179,7 +187,14 @@ def _parallel_translate_batch(batch: list, src_root: str, temp_dir: str, output_ trans._shared_symbol_table.translator = None trans._translate_file(src_path, out_path, prebuilt_ModuleSha1Map=msm) except Exception as e: - batch_errors.append((src_path, f'{e}\n{traceback.format_exc()}')) + # fail-fast（2026-08-18）：任意文件编译出错立即终止整个编译进程， + # 不再收集错误继续翻译批内剩余文件（避免浪费大量时间）。主进程 + # 收到 BrokenProcessPool/future 异常后取消全部剩余批次退出。 + sys.stderr.write( + f"[FATAL] 文件翻译失败，立即终止编译: {src_path}\n" + f"{traceback.format_exc()}\n") + sys.stderr.flush() + os._exit(88) return (batch_errors, batch) except Exception as e: return ([(item[1], f'{e}\n{traceback.format_exc()}') for item in batch], batch) @@ -188,18 +203,31 @@ def _parallel_translate_batch(batch: list, src_root: str, temp_dir: str, output_ # ================================================================== # 子进程性能剖析：主进程 cProfile 只能看到进程池等待（WaitForMultipleObjects/ # as_completed），看不到 worker 进程内部 CPU 分布（vast parse 等热点）。 -# 设置环境变量 TPC_WORKER_PROF=<输出目录> 时，worker 进程内每个任务 -# 导出 cProfile dump（worker_prof_{pid}_{seq}.prof），可用 pstats 合并分析。 -# 未设置时装饰器零开销（原样返回函数）。 +# 固定启用（不设环境变量开关），worker 每个任务导出 cProfile dump +# （worker_prof_{pid}_{seq}.prof）到 PROF_DIR（D:\Users\TermiNexus\Desktop\TransPyC\PROF）， +# 可用 pstats 合并分析。每次编译开始时由 _CleanProfFiles 清理该目录 *.prof。 # ================================================================== import functools as _functools +# 硬编码 PROF 输出目录（2026-08-18 起不再用 TPC_WORKER_PROF 环境变量控制） +PROF_DIR: str = r'D:\Users\TermiNexus\Desktop\TransPyC\PROF' + + +def _CleanProfFiles() -> None: + """清理 PROF_DIR 下全部 *.prof（每次编译开始调用）。""" + try: + if os.path.isdir(PROF_DIR): + for _n in os.listdir(PROF_DIR): + if _n.endswith('.prof'): + try: + os.remove(os.path.join(PROF_DIR, _n)) + except OSError: + pass + except OSError: + pass + def _ProfileWorker(_fn): - _prof_dir: str = os.environ.get('TPC_WORKER_PROF') or '' - if not _prof_dir: - return _fn - @_functools.wraps(_fn) def _wrapper(*args, **kwargs): _prof: cProfile.Profile = cProfile.Profile() @@ -211,9 +239,9 @@ def _ProfileWorker(_fn): _seq: int = getattr(_wrapper, '_seq', 0) + 1 _wrapper._seq = _seq try: - if not os.path.isdir(_prof_dir): - os.makedirs(_prof_dir, exist_ok=True) - _out: str = os.path.join(_prof_dir, f'worker_prof_{os.getpid()}_{_seq}.prof') + if not os.path.isdir(PROF_DIR): + os.makedirs(PROF_DIR, exist_ok=True) + _out: str = os.path.join(PROF_DIR, f'worker_prof_{os.getpid()}_{_seq}.prof') _prof.dump_stats(_out) except Exception: pass @@ -410,8 +438,14 @@ class Phase2Translator: INCLUDE_DIRS: list[str] = [ os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), 'includes'), ] + # include 源文件 sha1/存在性缓存（_build_current_ModuleSha1Map）： + # 编译期间 include .vp/.py 源文件不变，避免每文件重复 isfile + open + + # compute_sha1（PROF 显示 9306 isfile + 4653 open + TextIOWrapper.read + # 35.9s 的大头来源，99 次调用 × 全量 include 重算）。 + _include_sha1_cache: dict[str, str] = {} + _include_exist_cache: dict[str, bool] = {} - def __init__(self, src_root: str, temp_dir: str, output_dir: str, compile_cmd: str = 'llc', compile_flags: list[str] | None = None, linker_cmd: str | None = None, linker_flags: list[str] | None = None, linker_output: str | None = None, include_dirs: list[str] | None = None, entry_files: list[str] | None = None, target_triple: str | None = None, target_datalayout: str | None = None, force_recompile_includes: bool = False) -> None: + def __init__(self, src_root: str, temp_dir: str, output_dir: str, compile_cmd: str = 'llc', compile_flags: list[str] | None = None, linker_cmd: str | None = None, linker_flags: list[str] | None = None, linker_output: str | None = None, include_dirs: list[str] | None = None, entry_files: list[str] | None = None, target_triple: str | None = None, target_datalayout: str | None = None, force_recompile_includes: bool = False, build_pyi_cache: bool = True) -> None: self.src_root = os.path.abspath(src_root) self.temp_dir = os.path.abspath(temp_dir) self.output_dir = os.path.abspath(output_dir) @@ -456,6 +490,11 @@ class Phase2Translator: self._executor_pool: ProcessPoolExecutor | None = None self.entry_files: list[str] | None = entry_files self.force_recompile_includes: bool = force_recompile_includes + # pyi cache 构建开关：仅主进程编译启动时构建一次（_LoadSha1Map 内）， + # worker 子进程只读主进程落盘的 temp/_pyi_cache_v1.pkl（_PyiCacheWorkerInitializer）， + # 不重复构建——worker 每任务重建 pkl 的 pickle.load/dump（43 次 × ~0.5s ≈ 23s + # + pickle.loads 10.4s）纯属浪费；worker 缺 includes vpi 时运行期按需 parse 回退。 + self._build_pyi_cache: bool = build_pyi_cache self._shared_symbol_table: object | None = None self._shared_source_module_sig_files: dict[str, str] = {} self._shared_all_dc: dict[str, dict] = {} @@ -538,7 +577,8 @@ class Phase2Translator: # 构建 pyi AST pickle 缓存（temp/_pyi_cache_v1.pkl）供 worker 预热： # Phase1 已生成全部 .vpi，此处把 vast 树落盘，避免每个 worker 首次任务 # 重新 vast.parse（4479 次 parse / 245s）。签名未变时零开销跳过。 - if os.environ.get('TPC_WORKER_WARMUP', '1') == '1': + # 仅主进程构建（build_pyi_cache=True）；worker 只读 pkl，不重复构建。 + if self._build_pyi_cache and os.environ.get('TPC_WORKER_WARMUP', '1') == '1': _BuildPyiCacheFile(self.temp_dir) def _build_struct_sha1_map(self) -> dict[str, str]: @@ -794,8 +834,17 @@ class Phase2Translator: errors.append((rel, err)) _vlog().info(f" [{done_files}/{total_files}] 批次完成({bcount} 文件, 首个 {rel_first})") except Exception as e: - _vlog().error(f" 批次 {bidx} 异常: {e}") - errors.append((rel_first, str(e))) + # fail-fast：worker 已 os._exit（文件编译出错）或进程池 + # 已 broken → 立即取消全部剩余批次并终止编译，不空等。 + _vlog().error(f" 批次 {bidx} 异常（worker 已终止）: {e}") + for fut in futures: + if not fut.done(): + fut.cancel() + if self._executor_pool is not None: + self._executor_pool.shutdown(wait=False, cancel_futures=True) + self._executor_pool = None + raise RuntimeError( + f"批次 {bidx} worker 异常终止，编译立即停止: {e}") except Exception: # 异常时立即清理，正常流程不 shutdown（留给 includes 阶段复用） if self._executor_pool is not None: @@ -852,12 +901,24 @@ class Phase2Translator: if os.path.isdir(inc_dir): py_path = os.path.join(inc_dir, rel_path[len('includes/'):].replace(os.sep, '/')) py_path = os.path.splitext(py_path)[0] + '.vp' - if not os.path.isfile(py_path): + # 存在性/内容缓存：编译期 include 源文件不变，命中跳过磁盘 I/O + exists = Phase2Translator._include_exist_cache.get(py_path) + if exists is None: + exists = os.path.isfile(py_path) + Phase2Translator._include_exist_cache[py_path] = exists + if not exists: # 回退到 .py（向后兼容） py_path = os.path.splitext(py_path)[0] + '.py' - if os.path.isfile(py_path): - with open(py_path, 'r', encoding='utf-8') as f: - py_sha1 = compute_sha1(f.read()) + exists = Phase2Translator._include_exist_cache.get(py_path) + if exists is None: + exists = os.path.isfile(py_path) + Phase2Translator._include_exist_cache[py_path] = exists + if exists: + py_sha1 = Phase2Translator._include_sha1_cache.get(py_path) + if py_sha1 is None: + with open(py_path, 'r', encoding='utf-8') as f: + py_sha1 = compute_sha1(f.read()) + Phase2Translator._include_sha1_cache[py_path] = py_sha1 actual_sha1 = py_sha1 break ModuleSha1Map[inc_mod_name] = actual_sha1 @@ -2562,11 +2623,19 @@ class Phase2Translator: if status == 'ok': _vlog().info(f" [{done_count}/{len(need_translate)}] 翻译完成: {module_name}.py") else: - _vlog().error(f" {module_name}.py: {err}") - errors.append((module_name, err)) + # fail-fast：文件编译出错立即终止（worker 已 os._exit， + # 此处兜底处理残留的 error 返回） + raise RuntimeError( + f"include 翻译失败，立即终止: {module_name}.py: {err}") except Exception as e: _vlog().error(f" {module_name}.py: {e}") - errors.append((module_name, str(e))) + for f2 in in_progress: + f2.cancel() + if self._executor_pool is not None: + self._executor_pool.shutdown(wait=False, cancel_futures=True) + self._executor_pool = None + raise RuntimeError( + f"include worker 异常终止，编译立即停止: {e}") except Exception: if self._executor_pool is not None: self._executor_pool.shutdown(wait=False, cancel_futures=True) diff --git a/lib/core/Handles/HandlesImports.py b/lib/core/Handles/HandlesImports.py index 7c293c6..9e250a7 100644 --- a/lib/core/Handles/HandlesImports.py +++ b/lib/core/Handles/HandlesImports.py @@ -38,6 +38,13 @@ class ImportHandle(BaseHandle): # 保证进程内唯一），避免 _TryLoadClassMembersFromPyi 跨任务反复线性扫描 body。 _extern_class_classify_cache: dict[tuple[str, str], tuple] = {} _extern_class_decl_cache: dict[tuple[str, str], ast.ClassDef] = {} + # 类级 stub 函数签名缓存（_LookupStubFuncType）：跨 worker 任务共享。 + # 与 _stub_index/_stub_index_scanned 同模式：签名变化只增量解析新增 stub + # 文件（编译进程内 stub 只增不改），避免每次任务全量重扫 + 每文件 + # os.path.isfile（Windows stat ~0.5ms/次，原 5840 次 ≈ 6.5s）。 + _stub_func_cache: dict[str, tuple[str, str, bool, str]] = {} + _stub_func_cache_signature: tuple[int, int] | None = None + _stub_func_scanned: set[str] = set() def __init__(self, translator: "Translator") -> None: super().__init__(translator) @@ -2290,12 +2297,11 @@ class ImportHandle(BaseHandle): try: _pyi_b0: str = stub_filename.replace('.stub.ll', '.vpi') _path0: str = os.path.join(temp_dir, _pyi_b0) - if os.path.isfile(_path0): - _tree0: ast.AST | None = getattr(self, '_pyi_cache', {}).get(_path0) - if _tree0 is None: - with open(_path0, 'r', encoding='utf-8') as _fh0: - _tree0 = ast.parse(_fh0.read()) - getattr(self, '_pyi_cache', {})[_path0] = _tree0 + _tree0: ast.AST | None = getattr(self, '_pyi_cache', {}).get(_path0) + if _tree0 is None and os.path.isfile(_path0): + with open(_path0, 'r', encoding='utf-8') as _fh0: + _tree0 = ast.parse(_fh0.read()) + getattr(self, '_pyi_cache', {})[_path0] = _tree0 if _tree0 is not None: for _n0 in _tree0.body: if isinstance(_n0, ast.ClassDef) and _n0.name == class_name: @@ -2314,7 +2320,10 @@ class ImportHandle(BaseHandle): return False pyi_basename = stub_filename.replace('.stub.ll', '.vpi') pyi_path = os.path.join(temp_dir, pyi_basename) - if not os.path.isfile(pyi_path): + # 已探测过的路径（_pyi_cache 含键，值为树或 None=不存在）不再重复 + # os.path.isfile（Windows stat ~0.5ms/次；本函数 28543 次调用 × 探测 + # ≈ 12.8s，占 _path_isfile 总耗时 63%）。 + if pyi_path not in self._pyi_cache and not os.path.isfile(pyi_path): # 回退读取旧缓存 .pyi pyi_basename = stub_filename.replace('.stub.ll', '.pyi') pyi_path = os.path.join(temp_dir, pyi_basename) @@ -2399,29 +2408,23 @@ class ImportHandle(BaseHandle): 将无法被解析，导致跨模块调用使用默认 i32 返回类型 → 64 位指针截断。 """ - if not getattr(self, '_stub_func_cache', None): - self._stub_func_cache = {} - ProjectRoot = self._find_project_root() temp_dir = os.path.join(ProjectRoot, 'temp') output_dir = os.path.join(ProjectRoot, 'output') - # 文件数签名：检测 stub 新增（增量编译时 output 逐步填充完整签名） - output_count = sum(1 for f in os.listdir(output_dir) if f.endswith('.stub.ll')) if os.path.isdir(output_dir) else 0 - temp_count = sum(1 for f in os.listdir(temp_dir) if f.endswith('.stub.ll')) if os.path.isdir(temp_dir) else 0 - signature = (output_count, temp_count) - if getattr(self, '_stub_func_cache_signature', None) != signature or not self._stub_func_cache: - # 签名变化或缓存为空，重建缓存 - self._stub_func_cache = {} - self._stub_func_cache_signature = signature - if not os.path.isdir(temp_dir) and not os.path.isdir(output_dir): return None - if func_name not in self._stub_func_cache: - # 先扫描 temp/（Phase 1，不完整签名），再扫描 output/（Phase 2，完整签名）覆盖之 - # 这样 output 中存在的继承包装方法会覆盖 temp 中的同名旧声明（若有）， - # 也填补 temp 中缺失的函数声明 + # 文件数签名：检测 stub 新增（增量编译时 output 逐步填充完整签名）。 + # 类级缓存（ImportHandle._stub_func_cache）跨 worker 任务共享；签名变化 + # 只增量解析新增 stub 文件（编译进程内 stub 只增不改，旧文件不重扫， + # 省略每文件 os.path.isfile ——Windows stat ~0.5ms/次，原 5840 次 ≈ 6.5s）。 + # 覆盖语义不变：先扫 temp/ 后扫 output/，output 同名声明覆盖 temp。 + output_count = sum(1 for f in os.listdir(output_dir) if f.endswith('.stub.ll')) if os.path.isdir(output_dir) else 0 + temp_count = sum(1 for f in os.listdir(temp_dir) if f.endswith('.stub.ll')) if os.path.isdir(temp_dir) else 0 + signature = (output_count, temp_count) + if ImportHandle._stub_func_cache_signature != signature: + ImportHandle._stub_func_cache_signature = signature for scan_dir in [temp_dir, output_dir]: if not os.path.isdir(scan_dir): continue @@ -2430,9 +2433,9 @@ class ImportHandle(BaseHandle): continue stub_path = os.path.join(scan_dir, filename) - # 预检查文件存在性：os.listdir 与 open 之间文件可能被并发写入删除 - if not os.path.isfile(stub_path): + if stub_path in ImportHandle._stub_func_scanned: continue + ImportHandle._stub_func_scanned.add(stub_path) source_sha1 = filename.replace('.stub.ll', '') try: with open(stub_path, 'r', encoding='utf-8') as f: @@ -2453,8 +2456,9 @@ class ImportHandle(BaseHandle): if match: stub_ret = match.group(1).strip() stub_params = match.group(3).strip() - self._stub_func_cache[stub_func_name] = (stub_ret, stub_params, '...' in stub_params, source_sha1) + ImportHandle._stub_func_cache[stub_func_name] = (stub_ret, stub_params, '...' in stub_params, source_sha1) except FileNotFoundError: + # listdir 与 open 之间文件被并发删除：跳过（等效旧 isfile 预检查） continue except Exception as _e: if _config_mode == "strict": diff --git a/lib/core/LLVMCG/StructGen.py b/lib/core/LLVMCG/StructGen.py index cb5cb40..865045f 100644 --- a/lib/core/LLVMCG/StructGen.py +++ b/lib/core/LLVMCG/StructGen.py @@ -170,23 +170,24 @@ class StructGenMixin: struct_names.add(sname) if struct_defs: lines: list[str] = ir_text.split('\n') + # 预计算全部 "= type" 定义行前缀（一次性），用 str.startswith(tuple) + # 在 C 层批量比较，替代原 O(lines × structs × patterns) 的 Python 嵌套 + # 循环（PROF: finalize tottime 84.8s + startswith 4114 万次 46.9s）。 + name_prefixes: list[str] = [] + for name in struct_names: + if name[0].isdigit() or '.' in name or not name.replace('_', '').replace('.', '').isalnum(): + name_prefixes.append(f'%"{name}" = type') + name_prefixes.append(f'%"{name}"=type') + name_prefixes.append(f'%{name} = type') + name_prefixes.append(f'%{name}=type') + else: + name_prefixes.append(f'%{name} = type') + name_prefixes.append(f'%{name}= type') + name_prefixes.append(f'%{name}=type') + prefixes_tuple: tuple[str, ...] = tuple(name_prefixes) filtered: list[str] = [] for line in lines: - stripped: str = line.strip() - skip: bool = False - for name in struct_names: - patterns: list[str] - if name[0].isdigit() or '.' in name or not name.replace('_', '').replace('.', '').isalnum(): - patterns = [f'%"{name}" = type', f'%"{name}"=type', f'%{name} = type', f'%{name}=type'] - else: - patterns = [f'%{name} = type', f'%{name}= type', f'%{name}=type'] - for p in patterns: - if stripped.startswith(p): - skip = True - break - if skip: - break - if not skip: + if not line.strip().startswith(prefixes_tuple): filtered.append(line) ir_text = '\n'.join(filtered) header_end: int = ir_text.find('\ndefine')