perf(phase2): 按 PROF 热点优化编译耗时
- HandlesImports: _TryLoadClassMembersFromPyi 复用 _pyi_cache 跳过重复 isfile（-12.8s）；_LookupStubFuncType 类级缓存 + 签名增量扫描新增 stub，去掉每文件 isfile（-6.5s+） - Phase2Translator: pyi cache 仅主进程构建（worker 只读 pkl，-23s+pickle 10.4s）；include 源文件 sha1/存在性缓存消除 _build_current_ModuleSha1Map 重复磁盘 I/O - StructGen.finalize: 行过滤改 startswith(tuple) 预计算，消除 O(lines×structs×patterns) 嵌套循环（tottime 84.8s → 预期数秒） - 附带 fail-fast（任意文件出错立即终止）+ 子进程 PROF 硬编码
This commit is contained in:
TermiNexus committed
1 parent
1808e90fd3
commit
fbd132284b
3 files changed
+140
-66
No files matched your search
@@ -81,7 +81,8 @@ def _parallel_translate_worker(src_path: str, out_path: str, src_root: str, temp
|
||||
try:
|
||||
trans = Phase2Translator(src_root, temp_dir, output_dir,
|
||||
compile_cmd='llc', include_dirs=include_dirs,
|
||||
target_triple=triple, target_datalayout=datalayout)
|
||||
target_triple=triple, target_datalayout=datalayout,
|
||||
build_pyi_cache=False)
|
||||
trans.sha1_map = sha1_map
|
||||
trans.sig_files = sig_files
|
||||
trans.stub_files = stub_files
|
||||
@@ -118,7 +119,13 @@ def _parallel_translate_worker(src_path: str, out_path: str, src_root: str, temp
|
||||
trans._translate_file(src_path, out_path, prebuilt_ModuleSha1Map=prebuilt_ModuleSha1Map)
|
||||
return ('ok', src_path, '')
|
||||
except Exception as e:
|
||||
return ('error', src_path, f'{e}\n{traceback.format_exc()}')
|
||||
# fail-fast（2026-08-18）：include 文件编译出错立即终止整个编译进程
|
||||
# （主进程收到 BrokenProcessPool 后取消剩余任务退出）。
|
||||
sys.stderr.write(
|
||||
f"[FATAL] include 翻译失败，立即终止编译: {src_path}\n"
|
||||
f"{traceback.format_exc()}\n")
|
||||
sys.stderr.flush()
|
||||
os._exit(88)
|
||||
|
||||
|
||||
def _parallel_translate_batch(batch: list, src_root: str, temp_dir: str, output_dir: str,
|
||||
@@ -138,7 +145,8 @@ def _parallel_translate_batch(batch: list, src_root: str, temp_dir: str, output_
|
||||
try:
|
||||
trans = Phase2Translator(src_root, temp_dir, output_dir,
|
||||
compile_cmd='llc', include_dirs=include_dirs,
|
||||
target_triple=triple, target_datalayout=datalayout)
|
||||
target_triple=triple, target_datalayout=datalayout,
|
||||
build_pyi_cache=False)
|
||||
trans.sha1_map = sha1_map
|
||||
trans.sig_files = sig_files
|
||||
trans.stub_files = stub_files
|
||||
@@ -179,7 +187,14 @@ def _parallel_translate_batch(batch: list, src_root: str, temp_dir: str, output_
|
||||
trans._shared_symbol_table.translator = None
|
||||
trans._translate_file(src_path, out_path, prebuilt_ModuleSha1Map=msm)
|
||||
except Exception as e:
|
||||
batch_errors.append((src_path, f'{e}\n{traceback.format_exc()}'))
|
||||
# fail-fast（2026-08-18）：任意文件编译出错立即终止整个编译进程，
|
||||
# 不再收集错误继续翻译批内剩余文件（避免浪费大量时间）。主进程
|
||||
# 收到 BrokenProcessPool/future 异常后取消全部剩余批次退出。
|
||||
sys.stderr.write(
|
||||
f"[FATAL] 文件翻译失败，立即终止编译: {src_path}\n"
|
||||
f"{traceback.format_exc()}\n")
|
||||
sys.stderr.flush()
|
||||
os._exit(88)
|
||||
return (batch_errors, batch)
|
||||
except Exception as e:
|
||||
return ([(item[1], f'{e}\n{traceback.format_exc()}') for item in batch], batch)
|
||||
@@ -188,18 +203,31 @@ def _parallel_translate_batch(batch: list, src_root: str, temp_dir: str, output_
|
||||
# ==================================================================
|
||||
# 子进程性能剖析：主进程 cProfile 只能看到进程池等待（WaitForMultipleObjects/
|
||||
# as_completed），看不到 worker 进程内部 CPU 分布（vast parse 等热点）。
|
||||
# 设置环境变量 TPC_WORKER_PROF=<输出目录> 时，worker 进程内每个任务
|
||||
# 导出 cProfile dump（worker_prof_{pid}_{seq}.prof），可用 pstats 合并分析。
|
||||
# 未设置时装饰器零开销（原样返回函数）。
|
||||
# 固定启用（不设环境变量开关），worker 每个任务导出 cProfile dump
|
||||
# （worker_prof_{pid}_{seq}.prof）到 PROF_DIR（D:\Users\TermiNexus\Desktop\TransPyC\PROF），
|
||||
# 可用 pstats 合并分析。每次编译开始时由 _CleanProfFiles 清理该目录 *.prof。
|
||||
# ==================================================================
|
||||
import functools as _functools
|
||||
|
||||
# 硬编码 PROF 输出目录（2026-08-18 起不再用 TPC_WORKER_PROF 环境变量控制）
|
||||
PROF_DIR: str = r'D:\Users\TermiNexus\Desktop\TransPyC\PROF'
|
||||
|
||||
|
||||
def _CleanProfFiles() -> None:
|
||||
"""清理 PROF_DIR 下全部 *.prof（每次编译开始调用）。"""
|
||||
try:
|
||||
if os.path.isdir(PROF_DIR):
|
||||
for _n in os.listdir(PROF_DIR):
|
||||
if _n.endswith('.prof'):
|
||||
try:
|
||||
os.remove(os.path.join(PROF_DIR, _n))
|
||||
except OSError:
|
||||
pass
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _ProfileWorker(_fn):
|
||||
_prof_dir: str = os.environ.get('TPC_WORKER_PROF') or ''
|
||||
if not _prof_dir:
|
||||
return _fn
|
||||
|
||||
@_functools.wraps(_fn)
|
||||
def _wrapper(*args, **kwargs):
|
||||
_prof: cProfile.Profile = cProfile.Profile()
|
||||
@@ -211,9 +239,9 @@ def _ProfileWorker(_fn):
|
||||
_seq: int = getattr(_wrapper, '_seq', 0) + 1
|
||||
_wrapper._seq = _seq
|
||||
try:
|
||||
if not os.path.isdir(_prof_dir):
|
||||
os.makedirs(_prof_dir, exist_ok=True)
|
||||
_out: str = os.path.join(_prof_dir, f'worker_prof_{os.getpid()}_{_seq}.prof')
|
||||
if not os.path.isdir(PROF_DIR):
|
||||
os.makedirs(PROF_DIR, exist_ok=True)
|
||||
_out: str = os.path.join(PROF_DIR, f'worker_prof_{os.getpid()}_{_seq}.prof')
|
||||
_prof.dump_stats(_out)
|
||||
except Exception:
|
||||
pass
|
||||
@@ -410,8 +438,14 @@ class Phase2Translator:
|
||||
INCLUDE_DIRS: list[str] = [
|
||||
os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), 'includes'),
|
||||
]
|
||||
# include 源文件 sha1/存在性缓存（_build_current_ModuleSha1Map）：
|
||||
# 编译期间 include .vp/.py 源文件不变，避免每文件重复 isfile + open +
|
||||
# compute_sha1（PROF 显示 9306 isfile + 4653 open + TextIOWrapper.read
|
||||
# 35.9s 的大头来源，99 次调用 × 全量 include 重算）。
|
||||
_include_sha1_cache: dict[str, str] = {}
|
||||
_include_exist_cache: dict[str, bool] = {}
|
||||
|
||||
def __init__(self, src_root: str, temp_dir: str, output_dir: str, compile_cmd: str = 'llc', compile_flags: list[str] | None = None, linker_cmd: str | None = None, linker_flags: list[str] | None = None, linker_output: str | None = None, include_dirs: list[str] | None = None, entry_files: list[str] | None = None, target_triple: str | None = None, target_datalayout: str | None = None, force_recompile_includes: bool = False) -> None:
|
||||
def __init__(self, src_root: str, temp_dir: str, output_dir: str, compile_cmd: str = 'llc', compile_flags: list[str] | None = None, linker_cmd: str | None = None, linker_flags: list[str] | None = None, linker_output: str | None = None, include_dirs: list[str] | None = None, entry_files: list[str] | None = None, target_triple: str | None = None, target_datalayout: str | None = None, force_recompile_includes: bool = False, build_pyi_cache: bool = True) -> None:
|
||||
self.src_root = os.path.abspath(src_root)
|
||||
self.temp_dir = os.path.abspath(temp_dir)
|
||||
self.output_dir = os.path.abspath(output_dir)
|
||||
@@ -456,6 +490,11 @@ class Phase2Translator:
|
||||
self._executor_pool: ProcessPoolExecutor | None = None
|
||||
self.entry_files: list[str] | None = entry_files
|
||||
self.force_recompile_includes: bool = force_recompile_includes
|
||||
# pyi cache 构建开关：仅主进程编译启动时构建一次（_LoadSha1Map 内），
|
||||
# worker 子进程只读主进程落盘的 temp/_pyi_cache_v1.pkl（_PyiCacheWorkerInitializer），
|
||||
# 不重复构建——worker 每任务重建 pkl 的 pickle.load/dump（43 次 × ~0.5s ≈ 23s
|
||||
# + pickle.loads 10.4s）纯属浪费；worker 缺 includes vpi 时运行期按需 parse 回退。
|
||||
self._build_pyi_cache: bool = build_pyi_cache
|
||||
self._shared_symbol_table: object | None = None
|
||||
self._shared_source_module_sig_files: dict[str, str] = {}
|
||||
self._shared_all_dc: dict[str, dict] = {}
|
||||
@@ -538,7 +577,8 @@ class Phase2Translator:
|
||||
# 构建 pyi AST pickle 缓存（temp/_pyi_cache_v1.pkl）供 worker 预热：
|
||||
# Phase1 已生成全部 .vpi，此处把 vast 树落盘，避免每个 worker 首次任务
|
||||
# 重新 vast.parse（4479 次 parse / 245s）。签名未变时零开销跳过。
|
||||
if os.environ.get('TPC_WORKER_WARMUP', '1') == '1':
|
||||
# 仅主进程构建（build_pyi_cache=True）；worker 只读 pkl，不重复构建。
|
||||
if self._build_pyi_cache and os.environ.get('TPC_WORKER_WARMUP', '1') == '1':
|
||||
_BuildPyiCacheFile(self.temp_dir)
|
||||
|
||||
def _build_struct_sha1_map(self) -> dict[str, str]:
|
||||
@@ -794,8 +834,17 @@ class Phase2Translator:
|
||||
errors.append((rel, err))
|
||||
_vlog().info(f" [{done_files}/{total_files}] 批次完成({bcount} 文件, 首个 {rel_first})")
|
||||
except Exception as e:
|
||||
_vlog().error(f" 批次 {bidx} 异常: {e}")
|
||||
errors.append((rel_first, str(e)))
|
||||
# fail-fast：worker 已 os._exit（文件编译出错）或进程池
|
||||
# 已 broken → 立即取消全部剩余批次并终止编译，不空等。
|
||||
_vlog().error(f" 批次 {bidx} 异常（worker 已终止）: {e}")
|
||||
for fut in futures:
|
||||
if not fut.done():
|
||||
fut.cancel()
|
||||
if self._executor_pool is not None:
|
||||
self._executor_pool.shutdown(wait=False, cancel_futures=True)
|
||||
self._executor_pool = None
|
||||
raise RuntimeError(
|
||||
f"批次 {bidx} worker 异常终止，编译立即停止: {e}")
|
||||
except Exception:
|
||||
# 异常时立即清理，正常流程不 shutdown（留给 includes 阶段复用）
|
||||
if self._executor_pool is not None:
|
||||
@@ -852,12 +901,24 @@ class Phase2Translator:
|
||||
if os.path.isdir(inc_dir):
|
||||
py_path = os.path.join(inc_dir, rel_path[len('includes/'):].replace(os.sep, '/'))
|
||||
py_path = os.path.splitext(py_path)[0] + '.vp'
|
||||
if not os.path.isfile(py_path):
|
||||
# 存在性/内容缓存：编译期 include 源文件不变，命中跳过磁盘 I/O
|
||||
exists = Phase2Translator._include_exist_cache.get(py_path)
|
||||
if exists is None:
|
||||
exists = os.path.isfile(py_path)
|
||||
Phase2Translator._include_exist_cache[py_path] = exists
|
||||
if not exists:
|
||||
# 回退到 .py（向后兼容）
|
||||
py_path = os.path.splitext(py_path)[0] + '.py'
|
||||
if os.path.isfile(py_path):
|
||||
with open(py_path, 'r', encoding='utf-8') as f:
|
||||
py_sha1 = compute_sha1(f.read())
|
||||
exists = Phase2Translator._include_exist_cache.get(py_path)
|
||||
if exists is None:
|
||||
exists = os.path.isfile(py_path)
|
||||
Phase2Translator._include_exist_cache[py_path] = exists
|
||||
if exists:
|
||||
py_sha1 = Phase2Translator._include_sha1_cache.get(py_path)
|
||||
if py_sha1 is None:
|
||||
with open(py_path, 'r', encoding='utf-8') as f:
|
||||
py_sha1 = compute_sha1(f.read())
|
||||
Phase2Translator._include_sha1_cache[py_path] = py_sha1
|
||||
actual_sha1 = py_sha1
|
||||
break
|
||||
ModuleSha1Map[inc_mod_name] = actual_sha1
|
||||
@@ -2562,11 +2623,19 @@ class Phase2Translator:
|
||||
if status == 'ok':
|
||||
_vlog().info(f" [{done_count}/{len(need_translate)}] 翻译完成: {module_name}.py")
|
||||
else:
|
||||
_vlog().error(f" {module_name}.py: {err}")
|
||||
errors.append((module_name, err))
|
||||
# fail-fast：文件编译出错立即终止（worker 已 os._exit，
|
||||
# 此处兜底处理残留的 error 返回）
|
||||
raise RuntimeError(
|
||||
f"include 翻译失败，立即终止: {module_name}.py: {err}")
|
||||
except Exception as e:
|
||||
_vlog().error(f" {module_name}.py: {e}")
|
||||
errors.append((module_name, str(e)))
|
||||
for f2 in in_progress:
|
||||
f2.cancel()
|
||||
if self._executor_pool is not None:
|
||||
self._executor_pool.shutdown(wait=False, cancel_futures=True)
|
||||
self._executor_pool = None
|
||||
raise RuntimeError(
|
||||
f"include worker 异常终止，编译立即停止: {e}")
|
||||
except Exception:
|
||||
if self._executor_pool is not None:
|
||||
self._executor_pool.shutdown(wait=False, cancel_futures=True)
|
||||
|
||||
Reference in new issue
Block a user