From 3aebf402bbbc345908acb4cb890af0c482a7522a Mon Sep 17 00:00:00 2001 From: TermiNexus Date: Mon, 17 Aug 2026 19:04:22 +0800 Subject: [PATCH] =?UTF-8?q?perf(compile):=20worker=20=E8=BF=9B=E7=A8=8B?= =?UTF-8?q?=E7=BA=A7=E5=A4=8D=E7=94=A8=20Phase2Translator=EF=BC=8C?= =?UTF-8?q?=E6=B6=88=E9=99=A4=E6=AF=8F=E6=96=87=E4=BB=B6=E9=87=8D=E5=A4=8D?= =?UTF-8?q?=20pickle.load?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cProfile 显示 ~163s 墙钟在 ProcessPool 等待（非计算）。App 阶段 200+ 文件分散到 worker，原实现每个任务都新建 Phase2Translator 并 pickle.load 2.5MB 共享符号表（_shared_sym.json）→ 每 worker 12+ 次重复反序列化。 - _parallel_translate_worker 增加进程级缓存：以 shared_sym_pickle_path 为 key，首次遇到才创建 trans + 加载 pickle，后续任务直接复用 trans._translate_file - _translate_file 每次新建内部 TransPyC 且对 self 的写均为覆盖型（translator 引用/错误栈），复用安全 - App/includes 两阶段 pickle 路径不同，key 变化时每 worker 各自重建一次 --- lib/Projectrans/Phase2Translator.py | 81 +++++++++++++++++------------ 1 file changed, 48 insertions(+), 33 deletions(-) diff --git a/lib/Projectrans/Phase2Translator.py b/lib/Projectrans/Phase2Translator.py index 72e2d1d..f9f6bfc 100644 --- a/lib/Projectrans/Phase2Translator.py +++ b/lib/Projectrans/Phase2Translator.py @@ -65,47 +65,62 @@ def _ExtractCDefineConstantsFromPyi(pyi_path: str, all_dc: dict[str, object]) -> _vlog().warning(f"Phase2Translator: 忽略异常 {e}", exc_info=e) +# worker 进程内缓存 Phase2Translator：同一 worker 进程处理多个文件时复用， +# 避免每个任务重复创建 Phase2Translator + pickle.load 2.5MB 共享符号表 +# （App 阶段 200+ 文件分散到 16 worker，每 worker 处理 12+ 文件；原实现每 +# 文件都重建 trans + 重载 pickle → 大量重复反序列化，构成墙钟等待大户）。 +# key = shared_sym_pickle_path：App/includes 两阶段 pickle 不同，首次遇到 +# 新 key 才重建（每 worker 每阶段至多一次）。 +_worker_trans_cache: Phase2Translator | None = None +_worker_trans_cache_key: str = '' + + def _parallel_translate_worker(src_path: str, out_path: str, src_root: str, temp_dir: str, output_dir: str, include_dirs: list[str], triple: str | None, datalayout: str | None, sha1_map: dict[str, str], sig_files: dict[str, str], stub_files: dict[str, str], include_py_map: dict[str, str], slice_level: int, shared_sym_pickle_path: str, struct_sha1_map: dict[str, str], prebuilt_ModuleSha1Map: dict[str, str] | None = None) -> tuple[str, str, str]: + global _worker_trans_cache, _worker_trans_cache_key try: - trans = Phase2Translator(src_root, temp_dir, output_dir, - compile_cmd='llc', include_dirs=include_dirs, - target_triple=triple, target_datalayout=datalayout) - trans.sha1_map = sha1_map - trans.sig_files = sig_files - trans.stub_files = stub_files - trans.include_py_map = include_py_map - trans.slice_level = slice_level - trans.struct_sha1_map = struct_sha1_map - - if shared_sym_pickle_path and os.path.exists(shared_sym_pickle_path): - # 使用 pickle.load(f) 流式加载，避免 pickle.loads(f.read()) 一次性 - # 将整个 pickle 文件读入内存（磁盘副本 + 反序列化对象同时占用内存）导致 MemoryError。 - # 同时提高递归限制，防止深层嵌套对象反序列化失败。 - sys.setrecursionlimit(max(sys.getrecursionlimit(), 100000)) - with open(shared_sym_pickle_path, 'rb') as f: - shared_data = pickle.load(f) - trans._shared_symbol_table = shared_data['symbol_table'] - trans._shared_source_module_sig_files = shared_data['source_module_sig_files'] - trans._shared_all_dc = shared_data['all_dc'] - trans._shared_export_extern_funcs = shared_data['export_extern_funcs'] - trans.inline_func_symbols = shared_data['inline_func_symbols'] - trans.function_default_args = shared_data['function_default_args'] - trans.function_param_names = shared_data.get('function_param_names', {}) - trans._shared_generic_class_templates = shared_data.get('generic_class_templates', {}) - trans._shared_doc_meta = shared_data.get('shared_doc_meta', {}) - trans._shared_class_vtable = shared_data.get('class_vtable', set()) - trans._shared_cross_module_vtable = shared_data.get('cross_module_vtable', set()) - trans._shared_class_methods = shared_data.get('class_methods', {}) - trans._shared_cross_module_novtable = shared_data.get('cross_module_novtable', set()) - else: - trans._precollect_inline_symbols() - trans._build_shared_symbol_data() + if (_worker_trans_cache is None + or _worker_trans_cache_key != shared_sym_pickle_path): + trans = Phase2Translator(src_root, temp_dir, output_dir, + compile_cmd='llc', include_dirs=include_dirs, + target_triple=triple, target_datalayout=datalayout) + trans.sha1_map = sha1_map + trans.sig_files = sig_files + trans.stub_files = stub_files + trans.include_py_map = include_py_map + trans.slice_level = slice_level + trans.struct_sha1_map = struct_sha1_map + if shared_sym_pickle_path and os.path.exists(shared_sym_pickle_path): + # 使用 pickle.load(f) 流式加载，避免 pickle.loads(f.read()) 一次性 + # 将整个 pickle 文件读入内存（磁盘副本 + 反序列化对象同时占用内存）导致 MemoryError。 + # 同时提高递归限制，防止深层嵌套对象反序列化失败。 + sys.setrecursionlimit(max(sys.getrecursionlimit(), 100000)) + with open(shared_sym_pickle_path, 'rb') as f: + shared_data = pickle.load(f) + trans._shared_symbol_table = shared_data['symbol_table'] + trans._shared_source_module_sig_files = shared_data['source_module_sig_files'] + trans._shared_all_dc = shared_data['all_dc'] + trans._shared_export_extern_funcs = shared_data['export_extern_funcs'] + trans.inline_func_symbols = shared_data['inline_func_symbols'] + trans.function_default_args = shared_data['function_default_args'] + trans.function_param_names = shared_data.get('function_param_names', {}) + trans._shared_generic_class_templates = shared_data.get('generic_class_templates', {}) + trans._shared_doc_meta = shared_data.get('shared_doc_meta', {}) + trans._shared_class_vtable = shared_data.get('class_vtable', set()) + trans._shared_cross_module_vtable = shared_data.get('cross_module_vtable', set()) + trans._shared_class_methods = shared_data.get('class_methods', {}) + trans._shared_cross_module_novtable = shared_data.get('cross_module_novtable', set()) + else: + trans._precollect_inline_symbols() + trans._build_shared_symbol_data() + _worker_trans_cache = trans + _worker_trans_cache_key = shared_sym_pickle_path + trans = _worker_trans_cache trans._translate_file(src_path, out_path, prebuilt_ModuleSha1Map=prebuilt_ModuleSha1Map) return ('ok', src_path, '') except Exception as e: