From a28aacda0b863f41578789de1c711d0a36c02499 Mon Sep 17 00:00:00 2001 From: TermiNexus Date: Mon, 17 Aug 2026 19:58:28 +0800 Subject: [PATCH] =?UTF-8?q?fix(compile):=20=E5=9B=9E=E9=80=80=20worker=20?= =?UTF-8?q?=E8=BF=9B=E7=A8=8B=E7=BA=A7=E5=A4=8D=E7=94=A8=EF=BC=8C=E6=B6=88?= =?UTF-8?q?=E9=99=A4=E5=85=B1=E4=BA=AB=20SymbolTable=20=E8=B7=A8=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E6=B1=A1=E6=9F=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 复用后同一 worker 的 trans 处理多文件，_translate_file 编译过程会向共享 SymbolTable 写入符号（import insert/类注册），前文件符号污染后文件 → 跨模块 struct 类型前缀错位（7cadfe07e983fd90 即 _list 前缀混入 CompilerFactory/Translator/File，Type of #1 arg mismatch）。每任务从 pickle 反序列化独立副本（原实现）。 收益评估：worker 复用仅省 pickle.load ~0.5s/次 × 每 worker 12 次 ≈ 6s 并行墙钟，远小于 worker 内翻译计算（百秒级），不值得以正确性为代价。n_workers 满核优化保留。 --- lib/Projectrans/Phase2Translator.py | 87 +++++++++++++---------------- 1 file changed, 39 insertions(+), 48 deletions(-) diff --git a/lib/Projectrans/Phase2Translator.py b/lib/Projectrans/Phase2Translator.py index f9f6bfc..1cd0ca7 100644 --- a/lib/Projectrans/Phase2Translator.py +++ b/lib/Projectrans/Phase2Translator.py @@ -65,62 +65,53 @@ def _ExtractCDefineConstantsFromPyi(pyi_path: str, all_dc: dict[str, object]) -> _vlog().warning(f"Phase2Translator: 忽略异常 {e}", exc_info=e) -# worker 进程内缓存 Phase2Translator：同一 worker 进程处理多个文件时复用， -# 避免每个任务重复创建 Phase2Translator + pickle.load 2.5MB 共享符号表 -# （App 阶段 200+ 文件分散到 16 worker，每 worker 处理 12+ 文件；原实现每 -# 文件都重建 trans + 重载 pickle → 大量重复反序列化，构成墙钟等待大户）。 -# key = shared_sym_pickle_path：App/includes 两阶段 pickle 不同，首次遇到 -# 新 key 才重建（每 worker 每阶段至多一次）。 -_worker_trans_cache: Phase2Translator | None = None -_worker_trans_cache_key: str = '' - - def _parallel_translate_worker(src_path: str, out_path: str, src_root: str, temp_dir: str, output_dir: str, include_dirs: list[str], triple: str | None, datalayout: str | None, sha1_map: dict[str, str], sig_files: dict[str, str], stub_files: dict[str, str], include_py_map: dict[str, str], slice_level: int, shared_sym_pickle_path: str, struct_sha1_map: dict[str, str], prebuilt_ModuleSha1Map: dict[str, str] | None = None) -> tuple[str, str, str]: - global _worker_trans_cache, _worker_trans_cache_key + # 注意：不可跨任务复用 Phase2Translator/共享符号表！_translate_file 编译过程中 + # 会向共享 SymbolTable 写入符号（import insert/类注册等），复用导致前文件的 + # 符号污染后文件 → 跨模块 struct 类型前缀错位（如 7cadfe07e983fd90.CompilerFactory + # 混入 _list 前缀，Type of #1 arg mismatch）。每任务必须从 pickle 反序列化出 + # 独立副本（原实现；worker 复用省掉的 pickle.load 时间仅 ~0.5s/次，远小于 + # worker 内翻译计算，不值得以正确性为代价）。 try: - if (_worker_trans_cache is None - or _worker_trans_cache_key != shared_sym_pickle_path): - trans = Phase2Translator(src_root, temp_dir, output_dir, - compile_cmd='llc', include_dirs=include_dirs, - target_triple=triple, target_datalayout=datalayout) - trans.sha1_map = sha1_map - trans.sig_files = sig_files - trans.stub_files = stub_files - trans.include_py_map = include_py_map - trans.slice_level = slice_level - trans.struct_sha1_map = struct_sha1_map + trans = Phase2Translator(src_root, temp_dir, output_dir, + compile_cmd='llc', include_dirs=include_dirs, + target_triple=triple, target_datalayout=datalayout) + trans.sha1_map = sha1_map + trans.sig_files = sig_files + trans.stub_files = stub_files + trans.include_py_map = include_py_map + trans.slice_level = slice_level + trans.struct_sha1_map = struct_sha1_map + + if shared_sym_pickle_path and os.path.exists(shared_sym_pickle_path): + # 使用 pickle.load(f) 流式加载，避免 pickle.loads(f.read()) 一次性 + # 将整个 pickle 文件读入内存（磁盘副本 + 反序列化对象同时占用内存）导致 MemoryError。 + # 同时提高递归限制，防止深层嵌套对象反序列化失败。 + sys.setrecursionlimit(max(sys.getrecursionlimit(), 100000)) + with open(shared_sym_pickle_path, 'rb') as f: + shared_data = pickle.load(f) + trans._shared_symbol_table = shared_data['symbol_table'] + trans._shared_source_module_sig_files = shared_data['source_module_sig_files'] + trans._shared_all_dc = shared_data['all_dc'] + trans._shared_export_extern_funcs = shared_data['export_extern_funcs'] + trans.inline_func_symbols = shared_data['inline_func_symbols'] + trans.function_default_args = shared_data['function_default_args'] + trans.function_param_names = shared_data.get('function_param_names', {}) + trans._shared_generic_class_templates = shared_data.get('generic_class_templates', {}) + trans._shared_doc_meta = shared_data.get('shared_doc_meta', {}) + trans._shared_class_vtable = shared_data.get('class_vtable', set()) + trans._shared_cross_module_vtable = shared_data.get('cross_module_vtable', set()) + trans._shared_class_methods = shared_data.get('class_methods', {}) + trans._shared_cross_module_novtable = shared_data.get('cross_module_novtable', set()) + else: + trans._precollect_inline_symbols() + trans._build_shared_symbol_data() - if shared_sym_pickle_path and os.path.exists(shared_sym_pickle_path): - # 使用 pickle.load(f) 流式加载，避免 pickle.loads(f.read()) 一次性 - # 将整个 pickle 文件读入内存（磁盘副本 + 反序列化对象同时占用内存）导致 MemoryError。 - # 同时提高递归限制，防止深层嵌套对象反序列化失败。 - sys.setrecursionlimit(max(sys.getrecursionlimit(), 100000)) - with open(shared_sym_pickle_path, 'rb') as f: - shared_data = pickle.load(f) - trans._shared_symbol_table = shared_data['symbol_table'] - trans._shared_source_module_sig_files = shared_data['source_module_sig_files'] - trans._shared_all_dc = shared_data['all_dc'] - trans._shared_export_extern_funcs = shared_data['export_extern_funcs'] - trans.inline_func_symbols = shared_data['inline_func_symbols'] - trans.function_default_args = shared_data['function_default_args'] - trans.function_param_names = shared_data.get('function_param_names', {}) - trans._shared_generic_class_templates = shared_data.get('generic_class_templates', {}) - trans._shared_doc_meta = shared_data.get('shared_doc_meta', {}) - trans._shared_class_vtable = shared_data.get('class_vtable', set()) - trans._shared_cross_module_vtable = shared_data.get('cross_module_vtable', set()) - trans._shared_class_methods = shared_data.get('class_methods', {}) - trans._shared_cross_module_novtable = shared_data.get('cross_module_novtable', set()) - else: - trans._precollect_inline_symbols() - trans._build_shared_symbol_data() - _worker_trans_cache = trans - _worker_trans_cache_key = shared_sym_pickle_path - trans = _worker_trans_cache trans._translate_file(src_path, out_path, prebuilt_ModuleSha1Map=prebuilt_ModuleSha1Map) return ('ok', src_path, '') except Exception as e: