GVSDS/TransPyC
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf(compile): worker 进程级复用 Phase2Translator，消除每文件重复 pickle.load

Browse Source 
cProfile 显示 ~163s 墙钟在 ProcessPool 等待（非计算）。App 阶段 200+ 文件分散到 worker，原实现每个任务都新建 Phase2Translator 并 pickle.load 2.5MB 共享符号表（_shared_sym.json）→ 每 worker 12+ 次重复反序列化。

- _parallel_translate_worker 增加进程级缓存：以 shared_sym_pickle_path 为 key，首次遇到才创建 trans + 加载 pickle，后续任务直接复用 trans._translate_file
- _translate_file 每次新建内部 TransPyC 且对 self 的写均为覆盖型（translator 引用/错误栈），复用安全
- App/includes 两阶段 pickle 路径不同，key 变化时每 worker 各自重建一次
This commit is contained in:
TermiNexus committed 2026-08-17 19:04:22 +08:00
1 parent 58ce673f01
commit 3aebf402bb
1 file changed
+48 -33
Download Patch File Download Diff File Expand all files Collapse all files
lib/Projectrans/Phase2Translator.py
+48 -33
View File
@@ -65,47 +65,62 @@ def _ExtractCDefineConstantsFromPyi(pyi_path: str, all_dc: dict[str, object]) ->
_vlog().warning(f"Phase2Translator: 忽略异常 {e}", exc_info=e)
# worker 进程内缓存 Phase2Translator：同一 worker 进程处理多个文件时复用，
# 避免每个任务重复创建 Phase2Translator + pickle.load 2.5MB 共享符号表
# （App 阶段 200+ 文件分散到 16 worker，每 worker 处理 12+ 文件；原实现每
# 文件都重建 trans + 重载 pickle → 大量重复反序列化，构成墙钟等待大户）。
# key = shared_sym_pickle_path：App/includes 两阶段 pickle 不同，首次遇到
# 新 key 才重建（每 worker 每阶段至多一次）。
_worker_trans_cache: Phase2Translator | None = None
_worker_trans_cache_key: str = ''
def _parallel_translate_worker(src_path: str, out_path: str, src_root: str, temp_dir: str, output_dir: str,
include_dirs: list[str], triple: str | None, datalayout: str | None,
sha1_map: dict[str, str], sig_files: dict[str, str],
stub_files: dict[str, str], include_py_map: dict[str, str], slice_level: int,
shared_sym_pickle_path: str, struct_sha1_map: dict[str, str],
prebuilt_ModuleSha1Map: dict[str, str] | None = None) -> tuple[str, str, str]:
global _worker_trans_cache, _worker_trans_cache_key
try:
trans = Phase2Translator(src_root, temp_dir, output_dir,
compile_cmd='llc', include_dirs=include_dirs,
target_triple=triple, target_datalayout=datalayout)
trans.sha1_map = sha1_map
trans.sig_files = sig_files
trans.stub_files = stub_files
trans.include_py_map = include_py_map
trans.slice_level = slice_level
trans.struct_sha1_map = struct_sha1_map
if shared_sym_pickle_path and os.path.exists(shared_sym_pickle_path):
# 使用 pickle.load(f) 流式加载，避免 pickle.loads(f.read()) 一次性
# 将整个 pickle 文件读入内存（磁盘副本 + 反序列化对象同时占用内存）导致 MemoryError。
# 同时提高递归限制，防止深层嵌套对象反序列化失败。
sys.setrecursionlimit(max(sys.getrecursionlimit(), 100000))
with open(shared_sym_pickle_path, 'rb') as f:
shared_data = pickle.load(f)
trans._shared_symbol_table = shared_data['symbol_table']
trans._shared_source_module_sig_files = shared_data['source_module_sig_files']
trans._shared_all_dc = shared_data['all_dc']
trans._shared_export_extern_funcs = shared_data['export_extern_funcs']
trans.inline_func_symbols = shared_data['inline_func_symbols']
trans.function_default_args = shared_data['function_default_args']
trans.function_param_names = shared_data.get('function_param_names', {})
trans._shared_generic_class_templates = shared_data.get('generic_class_templates', {})
trans._shared_doc_meta = shared_data.get('shared_doc_meta', {})
trans._shared_class_vtable = shared_data.get('class_vtable', set())
trans._shared_cross_module_vtable = shared_data.get('cross_module_vtable', set())
trans._shared_class_methods = shared_data.get('class_methods', {})
trans._shared_cross_module_novtable = shared_data.get('cross_module_novtable', set())
else:
trans._precollect_inline_symbols()
trans._build_shared_symbol_data()
if (_worker_trans_cache is None
or _worker_trans_cache_key != shared_sym_pickle_path):
trans = Phase2Translator(src_root, temp_dir, output_dir,
compile_cmd='llc', include_dirs=include_dirs,
target_triple=triple, target_datalayout=datalayout)
trans.sha1_map = sha1_map
trans.sig_files = sig_files
trans.stub_files = stub_files
trans.include_py_map = include_py_map
trans.slice_level = slice_level
trans.struct_sha1_map = struct_sha1_map
if shared_sym_pickle_path and os.path.exists(shared_sym_pickle_path):
# 使用 pickle.load(f) 流式加载，避免 pickle.loads(f.read()) 一次性
# 将整个 pickle 文件读入内存（磁盘副本 + 反序列化对象同时占用内存）导致 MemoryError。
# 同时提高递归限制，防止深层嵌套对象反序列化失败。
sys.setrecursionlimit(max(sys.getrecursionlimit(), 100000))
with open(shared_sym_pickle_path, 'rb') as f:
shared_data = pickle.load(f)
trans._shared_symbol_table = shared_data['symbol_table']
trans._shared_source_module_sig_files = shared_data['source_module_sig_files']
trans._shared_all_dc = shared_data['all_dc']
trans._shared_export_extern_funcs = shared_data['export_extern_funcs']
trans.inline_func_symbols = shared_data['inline_func_symbols']
trans.function_default_args = shared_data['function_default_args']
trans.function_param_names = shared_data.get('function_param_names', {})
trans._shared_generic_class_templates = shared_data.get('generic_class_templates', {})
trans._shared_doc_meta = shared_data.get('shared_doc_meta', {})
trans._shared_class_vtable = shared_data.get('class_vtable', set())
trans._shared_cross_module_vtable = shared_data.get('cross_module_vtable', set())
trans._shared_class_methods = shared_data.get('class_methods', {})
trans._shared_cross_module_novtable = shared_data.get('cross_module_novtable', set())
else:
trans._precollect_inline_symbols()
trans._build_shared_symbol_data()
_worker_trans_cache = trans
_worker_trans_cache_key = shared_sym_pickle_path
trans = _worker_trans_cache
trans._translate_file(src_path, out_path, prebuilt_ModuleSha1Map=prebuilt_ModuleSha1Map)
return ('ok', src_path, '')
except Exception as e: