From f66e119d1f2cb3d33d55c83d7917889bc1341023 Mon Sep 17 00:00:00 2001 From: TermiNexus Date: Mon, 17 Aug 2026 20:01:33 +0800 Subject: [PATCH] =?UTF-8?q?perf(compile):=20App=20=E7=BF=BB=E8=AF=91?= =?UTF-8?q?=E6=94=B9=E6=89=B9=E6=AC=A1=E5=AD=90=E4=BB=BB=E5=8A=A1=E6=A8=A1?= =?UTF-8?q?=E5=9E=8B=EF=BC=88=E5=A4=9A=E6=96=87=E4=BB=B6/=E4=BB=BB?= =?UTF-8?q?=E5=8A=A1=20+=20=E6=AF=8F=E6=96=87=E4=BB=B6=E9=87=8D=E7=BD=AE?= =?UTF-8?q?=20SymbolTable=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 原"一文件一任务"：200+ 文件每个都经 ProcessPool 调度 + worker 内创建 Phase2Translator + pickle.load 全部共享数据。改为批次模型：need_translate 按 worker 数分块，每 worker 一个批次子任务——Phase2Translator 创建 + pickle.load 仅一次，批内顺序翻译。 防污染：SymbolTable 是编译期唯一被写的共享对象，dump 时额外存 symbol_table_bytes（独立 pickle），批内每文件前 pickle.loads 出干净副本，前文件 import 写入不残留后文件（避免 7cadfe07e983fd90 类前缀错位）。includes 阶段保持拓扑单文件任务不变。 --- lib/Projectrans/Phase2Translator.py | 108 ++++++++++++++++++++++++---- 1 file changed, 93 insertions(+), 15 deletions(-) diff --git a/lib/Projectrans/Phase2Translator.py b/lib/Projectrans/Phase2Translator.py index 1cd0ca7..086edba 100644 --- a/lib/Projectrans/Phase2Translator.py +++ b/lib/Projectrans/Phase2Translator.py @@ -118,6 +118,69 @@ def _parallel_translate_worker(src_path: str, out_path: str, src_root: str, temp return ('error', src_path, f'{e}\n{traceback.format_exc()}') +def _parallel_translate_batch(batch: list, src_root: str, temp_dir: str, output_dir: str, + include_dirs: list[str], triple: str | None, datalayout: str | None, + sha1_map: dict[str, str], sig_files: dict[str, str], + stub_files: dict[str, str], include_py_map: dict[str, str], slice_level: int, + shared_sym_pickle_path: str, struct_sha1_map: dict[str, str]) -> tuple[list, list]: + """批次子任务：worker 内创建一个 Phase2Translator + pickle.load 一次，顺序翻译批内全部文件。 + + batch 每项: (i, src_path, out_path, sha1, prebuilt_ModuleSha1Map) + + 防污染：SymbolTable 是编译期唯一被写的共享对象（import insert/类注册）， + 批内每个文件翻译前从 symbol_table_bytes 反序列化出干净副本，前文件的 + 符号写入不会残留到后文件（否则跨模块 struct 类型前缀错位）。 + 返回 (错误列表[(src_path, err)], batch)。 + """ + try: + trans = Phase2Translator(src_root, temp_dir, output_dir, + compile_cmd='llc', include_dirs=include_dirs, + target_triple=triple, target_datalayout=datalayout) + trans.sha1_map = sha1_map + trans.sig_files = sig_files + trans.stub_files = stub_files + trans.include_py_map = include_py_map + trans.slice_level = slice_level + trans.struct_sha1_map = struct_sha1_map + + symbol_table_bytes: bytes | None = None + if shared_sym_pickle_path and os.path.exists(shared_sym_pickle_path): + sys.setrecursionlimit(max(sys.getrecursionlimit(), 100000)) + with open(shared_sym_pickle_path, 'rb') as f: + shared_data = pickle.load(f) + symbol_table_bytes = shared_data.get('symbol_table_bytes') + trans._shared_symbol_table = shared_data['symbol_table'] + trans._shared_source_module_sig_files = shared_data['source_module_sig_files'] + trans._shared_all_dc = shared_data['all_dc'] + trans._shared_export_extern_funcs = shared_data['export_extern_funcs'] + trans.inline_func_symbols = shared_data['inline_func_symbols'] + trans.function_default_args = shared_data['function_default_args'] + trans.function_param_names = shared_data.get('function_param_names', {}) + trans._shared_generic_class_templates = shared_data.get('generic_class_templates', {}) + trans._shared_doc_meta = shared_data.get('shared_doc_meta', {}) + trans._shared_class_vtable = shared_data.get('class_vtable', set()) + trans._shared_cross_module_vtable = shared_data.get('cross_module_vtable', set()) + trans._shared_class_methods = shared_data.get('class_methods', {}) + trans._shared_cross_module_novtable = shared_data.get('cross_module_novtable', set()) + else: + trans._precollect_inline_symbols() + trans._build_shared_symbol_data() + + batch_errors: list = [] + for _i, src_path, out_path, _sha1, msm in batch: + try: + # 每文件前重置 SymbolTable 为干净副本（防跨文件污染） + if symbol_table_bytes is not None: + trans._shared_symbol_table = pickle.loads(symbol_table_bytes) + trans._shared_symbol_table.translator = None + trans._translate_file(src_path, out_path, prebuilt_ModuleSha1Map=msm) + except Exception as e: + batch_errors.append((src_path, f'{e}\n{traceback.format_exc()}')) + return (batch_errors, batch) + except Exception as e: + return ([(item[1], f'{e}\n{traceback.format_exc()}') for item in batch], batch) + + def _count_braces_outside_strings(text: str) -> int: """统计花括号差值，跳过 c"..." 字符串字面量内部的括号""" count = 0 @@ -430,8 +493,14 @@ class Phase2Translator: shared_pickle_path = os.path.join(self.temp_dir, '_shared_sym.json') try: + # symbol_table_bytes：SymbolTable 独立 pickle 字节，供批次 worker + # 每文件前反序列化出干净副本（SymbolTable 是编译期唯一被写的共享 + # 对象，批次内复用 trans 时必须在每个文件前重置，防止前文件 import + # 写入的符号污染后文件 → 跨模块 struct 类型前缀错位）。 + symbol_table_bytes: bytes = pickle.dumps(self._shared_symbol_table) shared_data = { 'symbol_table': self._shared_symbol_table, + 'symbol_table_bytes': symbol_table_bytes, 'source_module_sig_files': self._shared_source_module_sig_files, 'all_dc': self._shared_all_dc, 'export_extern_funcs': self._shared_export_extern_funcs, @@ -459,35 +528,44 @@ class Phase2Translator: self._executor_pool = ProcessPoolExecutor(max_workers=n_workers) executor = self._executor_pool try: + # 批次子任务模型：多个文件包装为一个子任务（batch）， + # 每 worker 处理一批——Phase2Translator 创建 + pickle.load 只做 + # 一次，批内文件顺序翻译（每文件前重置 SymbolTable 防污染）。 + # 相比"一文件一任务"：减少进程池调度/通信/构造开销 + # （200+ 文件 → n_workers 个批次任务）。 + batches: list[list] = [] + batch_size: int = max(1, (len(need_translate) + n_workers - 1) // n_workers) + for bi in range(0, len(need_translate), batch_size): + batches.append(need_translate[bi:bi + batch_size]) futures = {} - for i, src_path, out_path, sha1, msm in need_translate: - rel = os.path.relpath(src_path, self.src_root) + for bidx, batch in enumerate(batches): + rel_first = os.path.relpath(batch[0][1], self.src_root) future = executor.submit( - _parallel_translate_worker, - src_path, out_path, + _parallel_translate_batch, + batch, self.src_root, self.temp_dir, self.output_dir, self.include_dirs, self.triple, self.datalayout, self.sha1_map, self.sig_files, self.stub_files, self.include_py_map, self.slice_level, shared_pickle_path, self.struct_sha1_map, - prebuilt_ModuleSha1Map=msm ) - futures[future] = (i, rel) + futures[future] = (bidx, rel_first, len(batch)) - done_count = 0 + done_files = 0 + total_files = len(need_translate) for future in as_completed(futures): - i, rel = futures[future] - done_count += 1 + bidx, rel_first, bcount = futures[future] try: - status, _, err = future.result() - if status == 'ok': - _vlog().info(f" [{done_count}/{len(need_translate)}] 完成: {rel}") - else: + batch_errors, batch = future.result() + done_files += len(batch) + for src, err in batch_errors: + rel = os.path.relpath(src, self.src_root) _vlog().error(f" {rel}: {err}") errors.append((rel, err)) + _vlog().info(f" [{done_files}/{total_files}] 批次完成({bcount} 文件, 首个 {rel_first})") except Exception as e: - _vlog().error(f" {rel}: {e}") - errors.append((rel, str(e))) + _vlog().error(f" 批次 {bidx} 异常: {e}") + errors.append((rel_first, str(e))) except Exception: # 异常时立即清理，正常流程不 shutdown（留给 includes 阶段复用） if self._executor_pool is not None: