From 50c0703afd4a72a87130c19cf5b8ed4cf25d3edf Mon Sep 17 00:00:00 2001 From: TermiNexus Date: Mon, 17 Aug 2026 22:42:14 +0800 Subject: [PATCH] =?UTF-8?q?feat(profiling):=20worker=20=E8=BF=9B=E7=A8=8B?= =?UTF-8?q?=E5=AF=BC=E5=87=BA=20cProfile=E2=80=94=E2=80=94TPC=5FWORKER=5FP?= =?UTF-8?q?ROF=20=E7=8E=AF=E5=A2=83=E5=8F=98=E9=87=8F=E5=BC=80=E5=85=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 主进程 cProfile 只能看到进程池等待(WaitForMultipleObjects/as_completed 约 280s),看不到 worker 进程内部 CPU 分布(vast parse 等真实热点)。 新增 _ProfileWorker 装饰器包裹 _parallel_translate_worker 与 _parallel_translate_batch:设置环境变量 TPC_WORKER_PROF 为输出目录时, 每个任务导出 worker_prof_{pid}_{seq}.prof(可用 pstats 合并分析); 未设置时零开销(原样返回函数)。 --- lib/Projectrans/Phase2Translator.py | 41 +++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/lib/Projectrans/Phase2Translator.py b/lib/Projectrans/Phase2Translator.py index 6f08132..e41e5b6 100644 --- a/lib/Projectrans/Phase2Translator.py +++ b/lib/Projectrans/Phase2Translator.py @@ -10,6 +10,7 @@ import traceback import json import pickle import time +import cProfile from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed, wait, FIRST_COMPLETED from lib.core.VLogger import get_logger as _vlog from lib.constants.config import mode as _config_mode @@ -181,6 +182,46 @@ def _parallel_translate_batch(batch: list, src_root: str, temp_dir: str, output_ return ([(item[1], f'{e}\n{traceback.format_exc()}') for item in batch], batch) +# ================================================================== +# 子进程性能剖析:主进程 cProfile 只能看到进程池等待(WaitForMultipleObjects/ +# as_completed),看不到 worker 进程内部 CPU 分布(vast parse 等热点)。 +# 设置环境变量 TPC_WORKER_PROF=<输出目录> 时,worker 进程内每个任务 +# 导出 cProfile dump(worker_prof_{pid}_{seq}.prof),可用 pstats 合并分析。 +# 未设置时装饰器零开销(原样返回函数)。 +# ================================================================== +import functools as _functools + + +def _ProfileWorker(_fn): + _prof_dir: str = os.environ.get('TPC_WORKER_PROF') or '' + if not _prof_dir: + return _fn + + @_functools.wraps(_fn) + def _wrapper(*args, **kwargs): + _prof: cProfile.Profile = cProfile.Profile() + _prof.enable() + try: + return _fn(*args, **kwargs) + finally: + _prof.disable() + _seq: int = getattr(_wrapper, '_seq', 0) + 1 + _wrapper._seq = _seq + try: + if not os.path.isdir(_prof_dir): + os.makedirs(_prof_dir, exist_ok=True) + _out: str = os.path.join(_prof_dir, f'worker_prof_{os.getpid()}_{_seq}.prof') + _prof.dump_stats(_out) + except Exception: + pass + + return _wrapper + + +_parallel_translate_worker = _ProfileWorker(_parallel_translate_worker) +_parallel_translate_batch = _ProfileWorker(_parallel_translate_batch) + + def _count_braces_outside_strings(text: str) -> int: """统计花括号差值,跳过 c"..." 字符串字面量内部的括号""" count = 0