fix(tpc): worker cProfile 与 viztracer 互斥——外部 profiler 活跃时跳过
- 冲突机制:viztracer --log_multiprocess 经 sys.setprofile 注入 profiler 到 worker,TPC _ProfileWorker 的 cProfile.enable() 检测到已激活 profiler 抛 'Another profiling tool is already active' -> worker 崩溃 -> batch 0 异常 -> 编译终止 - 修复:_ProfileWorker 先查 sys.getprofile(),非 None(viztracer 等外部 profiler 活跃)时跳过 cProfile 让 viztracer 独占;正常编译(无外部 profiler)仍保留 cProfile(PROF 数据不受影响)
This commit is contained in:
1 parent
d092b4d31f
commit
c2f07e10cd
1 file changed
+20
-12
@@ -286,21 +286,29 @@ def _CleanProfFiles() -> None:
|
||||
def _ProfileWorker(_fn):
|
||||
@_functools.wraps(_fn)
|
||||
def _wrapper(*args, **kwargs):
|
||||
_prof: cProfile.Profile = cProfile.Profile()
|
||||
_prof.enable()
|
||||
_prof = None
|
||||
try:
|
||||
# 互斥:viztracer 等外部 profiler 已激活(sys.getprofile() 非 None)时
|
||||
# 跳过 cProfile——Python 只允许一个 profiling tool 同时活跃,
|
||||
# cProfile.enable() 会抛 "Another profiling tool is already active"
|
||||
# → worker 崩溃 → batch 异常 → 编译终止(viztracer --log_multiprocess 场景)。
|
||||
# 此时让 viztracer 独占记录,正常编译(无外部 profiler)仍保留 cProfile。
|
||||
if sys.getprofile() is None:
|
||||
_prof = cProfile.Profile()
|
||||
_prof.enable()
|
||||
return _fn(*args, **kwargs)
|
||||
finally:
|
||||
_prof.disable()
|
||||
_seq: int = getattr(_wrapper, '_seq', 0) + 1
|
||||
_wrapper._seq = _seq
|
||||
try:
|
||||
if not os.path.isdir(PROF_DIR):
|
||||
os.makedirs(PROF_DIR, exist_ok=True)
|
||||
_out: str = os.path.join(PROF_DIR, f'worker_prof_{os.getpid()}_{_seq}.prof')
|
||||
_prof.dump_stats(_out)
|
||||
except Exception:
|
||||
pass
|
||||
if _prof is not None:
|
||||
_prof.disable()
|
||||
_seq: int = getattr(_wrapper, '_seq', 0) + 1
|
||||
_wrapper._seq = _seq
|
||||
try:
|
||||
if not os.path.isdir(PROF_DIR):
|
||||
os.makedirs(PROF_DIR, exist_ok=True)
|
||||
_out: str = os.path.join(PROF_DIR, f'worker_prof_{os.getpid()}_{_seq}.prof')
|
||||
_prof.dump_stats(_out)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return _wrapper
|
||||
|
||||
|
||||
Reference in new issue
Block a user