feat(profiling): worker 进程导出 cProfile——TPC_WORKER_PROF 环境变量开关
主进程 cProfile 只能看到进程池等待(WaitForMultipleObjects/as_completed
约 280s),看不到 worker 进程内部 CPU 分布(vast parse 等真实热点)。
新增 _ProfileWorker 装饰器包裹 _parallel_translate_worker 与
_parallel_translate_batch:设置环境变量 TPC_WORKER_PROF 为输出目录时,
每个任务导出 worker_prof_{pid}_{seq}.prof(可用 pstats 合并分析);
未设置时零开销(原样返回函数)。
This commit is contained in:
1 parent
3326c41702
commit
50c0703afd
1 file changed
+41
@@ -10,6 +10,7 @@ import traceback
|
||||
import json
|
||||
import pickle
|
||||
import time
|
||||
import cProfile
|
||||
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed, wait, FIRST_COMPLETED
|
||||
from lib.core.VLogger import get_logger as _vlog
|
||||
from lib.constants.config import mode as _config_mode
|
||||
@@ -181,6 +182,46 @@ def _parallel_translate_batch(batch: list, src_root: str, temp_dir: str, output_
|
||||
return ([(item[1], f'{e}\n{traceback.format_exc()}') for item in batch], batch)
|
||||
|
||||
|
||||
# ==================================================================
|
||||
# 子进程性能剖析:主进程 cProfile 只能看到进程池等待(WaitForMultipleObjects/
|
||||
# as_completed),看不到 worker 进程内部 CPU 分布(vast parse 等热点)。
|
||||
# 设置环境变量 TPC_WORKER_PROF=<输出目录> 时,worker 进程内每个任务
|
||||
# 导出 cProfile dump(worker_prof_{pid}_{seq}.prof),可用 pstats 合并分析。
|
||||
# 未设置时装饰器零开销(原样返回函数)。
|
||||
# ==================================================================
|
||||
import functools as _functools
|
||||
|
||||
|
||||
def _ProfileWorker(_fn):
|
||||
_prof_dir: str = os.environ.get('TPC_WORKER_PROF') or ''
|
||||
if not _prof_dir:
|
||||
return _fn
|
||||
|
||||
@_functools.wraps(_fn)
|
||||
def _wrapper(*args, **kwargs):
|
||||
_prof: cProfile.Profile = cProfile.Profile()
|
||||
_prof.enable()
|
||||
try:
|
||||
return _fn(*args, **kwargs)
|
||||
finally:
|
||||
_prof.disable()
|
||||
_seq: int = getattr(_wrapper, '_seq', 0) + 1
|
||||
_wrapper._seq = _seq
|
||||
try:
|
||||
if not os.path.isdir(_prof_dir):
|
||||
os.makedirs(_prof_dir, exist_ok=True)
|
||||
_out: str = os.path.join(_prof_dir, f'worker_prof_{os.getpid()}_{_seq}.prof')
|
||||
_prof.dump_stats(_out)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return _wrapper
|
||||
|
||||
|
||||
_parallel_translate_worker = _ProfileWorker(_parallel_translate_worker)
|
||||
_parallel_translate_batch = _ProfileWorker(_parallel_translate_batch)
|
||||
|
||||
|
||||
def _count_braces_outside_strings(text: str) -> int:
|
||||
"""统计花括号差值,跳过 c"..." 字符串字面量内部的括号"""
|
||||
count = 0
|
||||
|
||||
Reference in new issue
Block a user