From 15e8bc7c7dd32d29c63886b55ce1174b11751074 Mon Sep 17 00:00:00 2001 From: TermiNexus Date: Mon, 17 Aug 2026 20:22:45 +0800 Subject: [PATCH] =?UTF-8?q?refactor(compile):=20=E5=AE=8C=E5=85=A8?= =?UTF-8?q?=E5=8E=BB=E9=99=A4=E5=A4=9A=E8=BF=9B=E7=A8=8B=EF=BC=8C=E7=BF=BB?= =?UTF-8?q?=E8=AF=91/=E7=BC=96=E8=AF=91=E5=85=A8=E9=83=A8=E6=94=B9?= =?UTF-8?q?=E9=A1=BA=E5=BA=8F=E9=98=BB=E5=A1=9E=E6=89=A7=E8=A1=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ProcessPoolExecutor 批次/波次子任务模型导致跨模块 struct 类型前缀错位 (7cadfe07e983fd90.Translator != 6ccfee1e5bd522ed.Translator,见 1.log), 且 cProfile 墙钟被进程池等待支配(223s 中 173s 耗在 WaitForMultipleObjects)。 删除 _parallel_translate_worker/_parallel_translate_batch、_executor_pool 及两处 llc ThreadPoolExecutor,App/includes 翻译按依赖拓扑顺序阻塞翻译, llc 编译逐文件阻塞执行,消除 pickle 序列化共享符号表与进程池等待开销。 --- lib/Projectrans/Phase2Translator.py | 444 +++------------------------- lib/Projectrans/__init__.py | 4 +- 2 files changed, 48 insertions(+), 400 deletions(-) diff --git a/lib/Projectrans/Phase2Translator.py b/lib/Projectrans/Phase2Translator.py index 086edba..2d4454c 100644 --- a/lib/Projectrans/Phase2Translator.py +++ b/lib/Projectrans/Phase2Translator.py @@ -8,9 +8,7 @@ import subprocess import vast as ast import traceback import json -import pickle import time -from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed, wait, FIRST_COMPLETED from lib.core.VLogger import get_logger as _vlog from lib.constants.config import mode as _config_mode from lib.core.Handles.HandlesBase import CTypeInfo @@ -65,122 +63,6 @@ def _ExtractCDefineConstantsFromPyi(pyi_path: str, all_dc: dict[str, object]) -> _vlog().warning(f"Phase2Translator: 忽略异常 {e}", exc_info=e) -def _parallel_translate_worker(src_path: str, out_path: str, src_root: str, temp_dir: str, output_dir: str, - include_dirs: list[str], triple: str | None, datalayout: str | None, - sha1_map: dict[str, str], sig_files: dict[str, str], - stub_files: dict[str, str], include_py_map: dict[str, str], slice_level: int, - shared_sym_pickle_path: str, struct_sha1_map: dict[str, str], - prebuilt_ModuleSha1Map: dict[str, str] | None = None) -> tuple[str, str, str]: - # 注意:不可跨任务复用 Phase2Translator/共享符号表!_translate_file 编译过程中 - # 会向共享 SymbolTable 写入符号(import insert/类注册等),复用导致前文件的 - # 符号污染后文件 → 跨模块 struct 类型前缀错位(如 7cadfe07e983fd90.CompilerFactory - # 混入 _list 前缀,Type of #1 arg mismatch)。每任务必须从 pickle 反序列化出 - # 独立副本(原实现;worker 复用省掉的 pickle.load 时间仅 ~0.5s/次,远小于 - # worker 内翻译计算,不值得以正确性为代价)。 - try: - trans = Phase2Translator(src_root, temp_dir, output_dir, - compile_cmd='llc', include_dirs=include_dirs, - target_triple=triple, target_datalayout=datalayout) - trans.sha1_map = sha1_map - trans.sig_files = sig_files - trans.stub_files = stub_files - trans.include_py_map = include_py_map - trans.slice_level = slice_level - trans.struct_sha1_map = struct_sha1_map - - if shared_sym_pickle_path and os.path.exists(shared_sym_pickle_path): - # 使用 pickle.load(f) 流式加载,避免 pickle.loads(f.read()) 一次性 - # 将整个 pickle 文件读入内存(磁盘副本 + 反序列化对象同时占用内存)导致 MemoryError。 - # 同时提高递归限制,防止深层嵌套对象反序列化失败。 - sys.setrecursionlimit(max(sys.getrecursionlimit(), 100000)) - with open(shared_sym_pickle_path, 'rb') as f: - shared_data = pickle.load(f) - trans._shared_symbol_table = shared_data['symbol_table'] - trans._shared_source_module_sig_files = shared_data['source_module_sig_files'] - trans._shared_all_dc = shared_data['all_dc'] - trans._shared_export_extern_funcs = shared_data['export_extern_funcs'] - trans.inline_func_symbols = shared_data['inline_func_symbols'] - trans.function_default_args = shared_data['function_default_args'] - trans.function_param_names = shared_data.get('function_param_names', {}) - trans._shared_generic_class_templates = shared_data.get('generic_class_templates', {}) - trans._shared_doc_meta = shared_data.get('shared_doc_meta', {}) - trans._shared_class_vtable = shared_data.get('class_vtable', set()) - trans._shared_cross_module_vtable = shared_data.get('cross_module_vtable', set()) - trans._shared_class_methods = shared_data.get('class_methods', {}) - trans._shared_cross_module_novtable = shared_data.get('cross_module_novtable', set()) - else: - trans._precollect_inline_symbols() - trans._build_shared_symbol_data() - - trans._translate_file(src_path, out_path, prebuilt_ModuleSha1Map=prebuilt_ModuleSha1Map) - return ('ok', src_path, '') - except Exception as e: - return ('error', src_path, f'{e}\n{traceback.format_exc()}') - - -def _parallel_translate_batch(batch: list, src_root: str, temp_dir: str, output_dir: str, - include_dirs: list[str], triple: str | None, datalayout: str | None, - sha1_map: dict[str, str], sig_files: dict[str, str], - stub_files: dict[str, str], include_py_map: dict[str, str], slice_level: int, - shared_sym_pickle_path: str, struct_sha1_map: dict[str, str]) -> tuple[list, list]: - """批次子任务:worker 内创建一个 Phase2Translator + pickle.load 一次,顺序翻译批内全部文件。 - - batch 每项: (i, src_path, out_path, sha1, prebuilt_ModuleSha1Map) - - 防污染:SymbolTable 是编译期唯一被写的共享对象(import insert/类注册), - 批内每个文件翻译前从 symbol_table_bytes 反序列化出干净副本,前文件的 - 符号写入不会残留到后文件(否则跨模块 struct 类型前缀错位)。 - 返回 (错误列表[(src_path, err)], batch)。 - """ - try: - trans = Phase2Translator(src_root, temp_dir, output_dir, - compile_cmd='llc', include_dirs=include_dirs, - target_triple=triple, target_datalayout=datalayout) - trans.sha1_map = sha1_map - trans.sig_files = sig_files - trans.stub_files = stub_files - trans.include_py_map = include_py_map - trans.slice_level = slice_level - trans.struct_sha1_map = struct_sha1_map - - symbol_table_bytes: bytes | None = None - if shared_sym_pickle_path and os.path.exists(shared_sym_pickle_path): - sys.setrecursionlimit(max(sys.getrecursionlimit(), 100000)) - with open(shared_sym_pickle_path, 'rb') as f: - shared_data = pickle.load(f) - symbol_table_bytes = shared_data.get('symbol_table_bytes') - trans._shared_symbol_table = shared_data['symbol_table'] - trans._shared_source_module_sig_files = shared_data['source_module_sig_files'] - trans._shared_all_dc = shared_data['all_dc'] - trans._shared_export_extern_funcs = shared_data['export_extern_funcs'] - trans.inline_func_symbols = shared_data['inline_func_symbols'] - trans.function_default_args = shared_data['function_default_args'] - trans.function_param_names = shared_data.get('function_param_names', {}) - trans._shared_generic_class_templates = shared_data.get('generic_class_templates', {}) - trans._shared_doc_meta = shared_data.get('shared_doc_meta', {}) - trans._shared_class_vtable = shared_data.get('class_vtable', set()) - trans._shared_cross_module_vtable = shared_data.get('cross_module_vtable', set()) - trans._shared_class_methods = shared_data.get('class_methods', {}) - trans._shared_cross_module_novtable = shared_data.get('cross_module_novtable', set()) - else: - trans._precollect_inline_symbols() - trans._build_shared_symbol_data() - - batch_errors: list = [] - for _i, src_path, out_path, _sha1, msm in batch: - try: - # 每文件前重置 SymbolTable 为干净副本(防跨文件污染) - if symbol_table_bytes is not None: - trans._shared_symbol_table = pickle.loads(symbol_table_bytes) - trans._shared_symbol_table.translator = None - trans._translate_file(src_path, out_path, prebuilt_ModuleSha1Map=msm) - except Exception as e: - batch_errors.append((src_path, f'{e}\n{traceback.format_exc()}')) - return (batch_errors, batch) - except Exception as e: - return ([(item[1], f'{e}\n{traceback.format_exc()}') for item in batch], batch) - - def _count_braces_outside_strings(text: str) -> int: """统计花括号差值,跳过 c"..." 字符串字面量内部的括号""" count = 0 @@ -252,9 +134,6 @@ class Phase2Translator: self.extra_compile_files: list[str] = [] self.extra_py_files: list[str] = [] self._include_sha1s: set[str] = set() - # 持有 ProcessPoolExecutor 引用,避免局部变量 GC 时 finalizer 阻塞 ~99s - # (join_executor_internals 在 GC 时同步等待 worker 退出队列) - self._executor_pool: ProcessPoolExecutor | None = None self.entry_files: list[str] | None = entry_files self.force_recompile_includes: bool = force_recompile_includes self._shared_symbol_table: object | None = None @@ -480,116 +359,26 @@ class Phase2Translator: # 提前从 include 文件源码中提取函数参数名,供主项目编译时 keyword 参数匹配使用 self._extract_include_param_names_from_dirs() # 提前从 App 源文件中提取函数默认参数,供跨模块调用参数检查使用 - # (避免并行编译时,被调用函数的文件尚未编译完,导致默认参数信息缺失) self._extract_app_default_args(need_translate) _vlog().info(f"\n需要翻译 {len(need_translate)} 个文件") t_start = time.time() - n_workers = min(os.cpu_count() or 4, len(need_translate)) - - if n_workers > 1 and len(need_translate) > 1: - _vlog().info(f" 并行翻译 (workers={n_workers})") - - shared_pickle_path = os.path.join(self.temp_dir, '_shared_sym.json') + # 顺序阻塞翻译(完全去除多进程):依赖排序保证被依赖文件先翻译, + # 单进程内共享符号表状态与 _TryLoadStructFromStub 的 output 目录扫描 + # 天然一致,杜绝批次子任务模型跨模块 struct 类型前缀错位 + # (7cadfe07e983fd90.Translator != 6ccfee1e5bd522ed.Translator)。 + for idx, (i, src_path, out_path, sha1, msm) in enumerate(need_translate): + rel = os.path.relpath(src_path, self.src_root) + t0 = time.time() try: - # symbol_table_bytes:SymbolTable 独立 pickle 字节,供批次 worker - # 每文件前反序列化出干净副本(SymbolTable 是编译期唯一被写的共享 - # 对象,批次内复用 trans 时必须在每个文件前重置,防止前文件 import - # 写入的符号污染后文件 → 跨模块 struct 类型前缀错位)。 - symbol_table_bytes: bytes = pickle.dumps(self._shared_symbol_table) - shared_data = { - 'symbol_table': self._shared_symbol_table, - 'symbol_table_bytes': symbol_table_bytes, - 'source_module_sig_files': self._shared_source_module_sig_files, - 'all_dc': self._shared_all_dc, - 'export_extern_funcs': self._shared_export_extern_funcs, - 'inline_func_symbols': self.inline_func_symbols, - 'function_default_args': self.function_default_args, - 'function_param_names': self.function_param_names, - 'generic_class_templates': self._shared_generic_class_templates, - 'shared_doc_meta': self._shared_doc_meta, - 'class_vtable': self._shared_class_vtable, - 'cross_module_vtable': self._shared_cross_module_vtable, - 'class_methods': self._shared_class_methods, - 'cross_module_novtable': self._shared_cross_module_novtable, - } - with open(shared_pickle_path, 'wb') as f: - f.write(pickle.dumps(shared_data)) - _vlog().info(f" 共享符号表已序列化 ({os.path.getsize(shared_pickle_path)//1024}KB)") + self._translate_file(src_path, out_path, prebuilt_ModuleSha1Map=msm) + t1 = time.time() + _vlog().info(f" [{idx+1}/{len(need_translate)}] {rel} ({t1-t0:.1f}s)") except Exception as e: - _vlog().warning(f"符号表序列化失败({e}),worker将自行构建") - shared_pickle_path = '' - - errors = [] - # 复用 self._executor_pool:避免局部变量 GC 时 finalizer 阻塞 ~99s - # (join_executor_internals 在 GC 时同步等待 worker 退出队列) - if self._executor_pool is None: - self._executor_pool = ProcessPoolExecutor(max_workers=n_workers) - executor = self._executor_pool - try: - # 批次子任务模型:多个文件包装为一个子任务(batch), - # 每 worker 处理一批——Phase2Translator 创建 + pickle.load 只做 - # 一次,批内文件顺序翻译(每文件前重置 SymbolTable 防污染)。 - # 相比"一文件一任务":减少进程池调度/通信/构造开销 - # (200+ 文件 → n_workers 个批次任务)。 - batches: list[list] = [] - batch_size: int = max(1, (len(need_translate) + n_workers - 1) // n_workers) - for bi in range(0, len(need_translate), batch_size): - batches.append(need_translate[bi:bi + batch_size]) - futures = {} - for bidx, batch in enumerate(batches): - rel_first = os.path.relpath(batch[0][1], self.src_root) - future = executor.submit( - _parallel_translate_batch, - batch, - self.src_root, self.temp_dir, self.output_dir, - self.include_dirs, self.triple, self.datalayout, - self.sha1_map, self.sig_files, self.stub_files, - self.include_py_map, self.slice_level, - shared_pickle_path, self.struct_sha1_map, - ) - futures[future] = (bidx, rel_first, len(batch)) - - done_files = 0 - total_files = len(need_translate) - for future in as_completed(futures): - bidx, rel_first, bcount = futures[future] - try: - batch_errors, batch = future.result() - done_files += len(batch) - for src, err in batch_errors: - rel = os.path.relpath(src, self.src_root) - _vlog().error(f" {rel}: {err}") - errors.append((rel, err)) - _vlog().info(f" [{done_files}/{total_files}] 批次完成({bcount} 文件, 首个 {rel_first})") - except Exception as e: - _vlog().error(f" 批次 {bidx} 异常: {e}") - errors.append((rel_first, str(e))) - except Exception: - # 异常时立即清理,正常流程不 shutdown(留给 includes 阶段复用) - if self._executor_pool is not None: - self._executor_pool.shutdown(wait=False, cancel_futures=True) - self._executor_pool = None - raise - - if errors: - _vlog().error(f"\n{len(errors)} 个文件翻译失败,立即终止编译") - for rel, err in errors: - _vlog().error(f" {rel}: {err}") - raise RuntimeError(f"{len(errors)} 个文件翻译失败,已终止编译") - else: - for idx, (i, src_path, out_path, sha1, msm) in enumerate(need_translate): - rel = os.path.relpath(src_path, self.src_root) - t0 = time.time() - try: - self._translate_file(src_path, out_path, prebuilt_ModuleSha1Map=msm) - t1 = time.time() - _vlog().info(f" [{idx+1}/{len(need_translate)}] {rel} ({t1-t0:.1f}s)") - except Exception as e: - _vlog().error(f" {rel}: {e}") - traceback.print_exc() - raise RuntimeError(f"翻译 {rel} 失败,已终止编译: {e}") from e + _vlog().error(f" {rel}: {e}") + traceback.print_exc() + raise RuntimeError(f"翻译 {rel} 失败,已终止编译: {e}") from e t_end = time.time() _vlog().info(f" 翻译总耗时: {t_end-t_start:.1f}s") @@ -600,11 +389,6 @@ class Phase2Translator: self._compile_include_sources() if self.linker_cmd: self._link_obj_files(active_sha1s) - # 清理 ProcessPoolExecutor:所有翻译已完成,shutdown(wait=False) 让 worker - # 后台退出。避免 GC finalizer 在程序末尾阻塞 ~99s(join_executor_internals) - if self._executor_pool is not None: - self._executor_pool.shutdown(wait=False, cancel_futures=True) - self._executor_pool = None def _build_current_ModuleSha1Map(self, self_sha1: str | None) -> dict[str, str]: """构建当前模块的依赖 SHA1 映射(与 _translate_file 中逻辑一致)""" @@ -1727,20 +1511,17 @@ class Phase2Translator: need_compile.append(ll_path) if need_compile: - n_workers = min(os.cpu_count() or 4, len(need_compile)) - _vlog().info(f" 并行编译 {len(need_compile)} 个文件 (workers={n_workers})") + _vlog().info(f" 编译 {len(need_compile)} 个 .ll 文件") errors = [] - with ThreadPoolExecutor(max_workers=n_workers) as executor: - futures = {executor.submit(_compile_one, ll_path): ll_path for ll_path in need_compile} - for future in as_completed(futures): - rel, status, err = future.result() - if status == 'ok': - _vlog().success(f" {rel}") - elif status == 'error': - _vlog().error(f" {rel}: {err}") - errors.append((rel, err)) - elif status == 'cached': - pass + for ll_path in need_compile: + rel, status, err = _compile_one(ll_path) + if status == 'ok': + _vlog().success(f" {rel}") + elif status == 'error': + _vlog().error(f" {rel}: {err}") + errors.append((rel, err)) + elif status == 'cached': + pass if errors: _vlog().error(f"\n{len(errors)} 个文件编译失败") sys.exit(1) @@ -1748,8 +1529,8 @@ class Phase2Translator: def _precollect_vtable_state(self) -> None: """预扫描 includes .py 文件,收集 @t.CVTable 类的虚表信息。 - 并行编译(ProcessPoolExecutor)创建独立内存的 worker 进程, - vtable 状态必须预收集并通过 pickle 共享。 + 顺序阻塞编译单进程内共享符号表,虚表状态必须在全部文件翻译前收集, + 保证跨模块 CVTable 继承方法分派一致(pool.alloc 虚表大小不漂移)。 复用 parse_python_file 的 AST 缓存。 """ for includes_dir in self.include_dirs: @@ -2134,7 +1915,7 @@ class Phase2Translator: return # 按依赖关系排序 include 文件,确保被依赖的文件先编译 - sorted_files, file_deps = self._sort_include_files_by_deps(self.extra_py_files) + sorted_files, _ = self._sort_include_files_by_deps(self.extra_py_files) _vlog().info(f"\n[includes 编译] 找到 {len(sorted_files)} 个 Python 源文件") # 构建 include 文件的模块 SHA1 映射(预计算所有文件,构建完整映射) @@ -2199,7 +1980,7 @@ class Phase2Translator: return # 预注册所有待翻译文件的符号(从 Phase1 缓存加载 .pyi / .stub.ll) - # 这样并行翻译时所有跨模块符号都已可用 + # 翻译前全部注册,保证顺序翻译时所有跨模块符号都已可用 for src_path, sha1, module_name, ll_path, obj_path, precompiled_obj in need_translate: self._register_compiled_include(src_path, sha1, None, include_ModuleSha1Map) @@ -2223,129 +2004,12 @@ class Phase2Translator: if tname: all_type_defs[tname] = sdecl - # ========== 阶段1:并行翻译 ========== + # ========== 阶段1:顺序阻塞翻译 ========== + # 完全去除多进程:need_translate 已按依赖拓扑排序(_sort_include_files_by_deps), + # 顺序翻译保证被依赖文件先完成,_TryLoadStructFromStub 扫描 output 目录 + # 总能取到完整的跨模块 struct 定义,且单进程共享符号表状态天然一致。 _vlog().info(f" 翻译 {len(need_translate)} 个文件") - if len(need_translate) > 1: - # 序列化共享符号表供 worker 使用 - shared_pickle_path = os.path.join(self.output_dir, '_shared_sym_includes.pickle') - try: - shared_data = { - 'symbol_table': self._shared_symbol_table, - 'source_module_sig_files': self._shared_source_module_sig_files, - 'all_dc': self._shared_all_dc, - 'export_extern_funcs': self._shared_export_extern_funcs, - 'inline_func_symbols': self.inline_func_symbols, - 'function_default_args': self.function_default_args, - 'function_param_names': self.function_param_names, - 'generic_class_templates': self._shared_generic_class_templates, - 'shared_doc_meta': self._shared_doc_meta, - 'class_vtable': self._shared_class_vtable, - 'cross_module_vtable': self._shared_cross_module_vtable, - 'class_methods': self._shared_class_methods, - 'cross_module_novtable': self._shared_cross_module_novtable, - } - with open(shared_pickle_path, 'wb') as f: - f.write(pickle.dumps(shared_data)) - except Exception as e: - _vlog().warning(f"符号表序列化失败({e}),include将串行翻译") - shared_pickle_path = '' - - if shared_pickle_path and os.path.exists(shared_pickle_path): - n_workers = min(os.cpu_count() or 4, len(need_translate)) - errors = [] - # 波次并行翻译:按依赖关系分批提交,确保被依赖的文件先翻译完成。 - # 根因:_TryLoadStructFromStub 扫描 output 目录加载跨模块 struct 定义, - # 若依赖文件尚未完成翻译(output stub 未写入),只能取到 Phase1 temp - # 中的不完整 stub(缺少跨模块继承字段),导致运行时崩溃。 - item_by_path = {item[0]: item for item in need_translate} - translate_paths = set(item_by_path.keys()) - # 每个文件的依赖(仅限本轮待翻译集合内) - translate_deps = { - p: (file_deps.get(p, set()) & translate_paths) - for p in translate_paths - } - # 复用 self._executor_pool(与 App 源文件阶段共享),避免 GC 阻塞 - if self._executor_pool is None: - self._executor_pool = ProcessPoolExecutor(max_workers=n_workers) - executor = self._executor_pool - try: - completed_paths: set[str] = set() - remaining = list(need_translate) # 保持拓扑序 - in_progress: dict = {} # future -> item - done_count = 0 - - while remaining or in_progress: - # 提交所有依赖已完成的文件 - ready = [ - item for item in remaining - if translate_deps[item[0]] <= completed_paths - ] - for item in ready: - remaining.remove(item) - src_path, sha1, module_name, ll_path, obj_path, precompiled_obj = item - future = executor.submit( - _parallel_translate_worker, - src_path, ll_path, - self.src_root, self.temp_dir, self.output_dir, - self.include_dirs, self.triple, self.datalayout, - self.sha1_map, self.sig_files, self.stub_files, - self.include_py_map, self.slice_level, - shared_pickle_path, self.struct_sha1_map, - include_ModuleSha1Map - ) - in_progress[future] = item - - if not in_progress: - # 无可提交且无进行中:循环依赖 fallback - for item in remaining: - src_path, sha1, module_name, ll_path, obj_path, precompiled_obj = item - future = executor.submit( - _parallel_translate_worker, - src_path, ll_path, - self.src_root, self.temp_dir, self.output_dir, - self.include_dirs, self.triple, self.datalayout, - self.sha1_map, self.sig_files, self.stub_files, - self.include_py_map, self.slice_level, - shared_pickle_path, self.struct_sha1_map, - include_ModuleSha1Map - ) - in_progress[future] = item - remaining.clear() - - # 等待至少一个完成 - done_set, _ = wait(in_progress, return_when=FIRST_COMPLETED) - for future in done_set: - item = in_progress.pop(future) - src_path, sha1, module_name, ll_path, obj_path, precompiled_obj = item - completed_paths.add(src_path) - done_count += 1 - try: - status, _, err = future.result() - if status == 'ok': - _vlog().info(f" [{done_count}/{len(need_translate)}] 翻译完成: {module_name}.py") - else: - _vlog().error(f" {module_name}.py: {err}") - errors.append((module_name, err)) - except Exception as e: - _vlog().error(f" {module_name}.py: {e}") - errors.append((module_name, str(e))) - except Exception: - if self._executor_pool is not None: - self._executor_pool.shutdown(wait=False, cancel_futures=True) - self._executor_pool = None - raise - - if errors: - _vlog().error(f"\n{len(errors)} 个 include 文件翻译失败") - sys.exit(1) - else: - # 串行回退 - for item in need_translate: - src_path, sha1, module_name, ll_path, obj_path, precompiled_obj = item - _vlog().info(f" 翻译: {module_name}.py -> {sha1}.ll") - self._translate_file(src_path, ll_path, prebuilt_ModuleSha1Map=include_ModuleSha1Map) - else: - item = need_translate[0] + for item in need_translate: src_path, sha1, module_name, ll_path, obj_path, precompiled_obj = item _vlog().info(f" 翻译: {module_name}.py -> {sha1}.ll") self._translate_file(src_path, ll_path, prebuilt_ModuleSha1Map=include_ModuleSha1Map) @@ -2466,7 +2130,7 @@ class Phase2Translator: if not compile_tasks: return - # ========== 阶段3:并行编译(llc)========== + # ========== 阶段3:顺序编译(llc)========== _vlog().info(f" 编译 {len(compile_tasks)} 个 .ll 文件") compile_results = {} # task -> result @@ -2475,35 +2139,19 @@ class Phase2Translator: result = self._llc_compile(dso_ll_path, obj_path, self.output_dir) return (task, result) - if len(compile_tasks) > 1: - n_workers = min(os.cpu_count() or 4, len(compile_tasks)) - with ThreadPoolExecutor(max_workers=n_workers) as executor: - futures = {executor.submit(_compile_one_include, task): task for task in compile_tasks} - done_count = 0 - for future in as_completed(futures): - done_count += 1 - try: - task, result = future.result() - module_name = task[4] - if result.returncode == 0: - _vlog().success(f" [{done_count}/{len(compile_tasks)}] 编译成功: {module_name}.py") - else: - _vlog().warning(f" [{done_count}/{len(compile_tasks)}] 编译失败: {module_name}.py: {result.stderr.strip()}") - compile_results[task] = result - except Exception as e: - task = futures[future] - module_name = task[4] - _vlog().error(f" [{done_count}/{len(compile_tasks)}] 编译异常: {module_name}.py: {e}") - compile_results[task] = None - else: - task = compile_tasks[0] - task_result, result = _compile_one_include(task) - module_name = task[4] - if result.returncode == 0: - _vlog().success(f" 编译成功: {module_name}.py") - else: - _vlog().warning(f" 编译失败: {module_name}.py: {result.stderr.strip()}") - compile_results[task] = result + for done_count, task in enumerate(compile_tasks, 1): + try: + task_result, result = _compile_one_include(task) + module_name = task[4] + if result.returncode == 0: + _vlog().success(f" [{done_count}/{len(compile_tasks)}] 编译成功: {module_name}.py") + else: + _vlog().warning(f" [{done_count}/{len(compile_tasks)}] 编译失败: {module_name}.py: {result.stderr.strip()}") + compile_results[task] = result + except Exception as e: + module_name = task[4] + _vlog().error(f" [{done_count}/{len(compile_tasks)}] 编译异常: {module_name}.py: {e}") + compile_results[task] = None # ========== 阶段4:注册 + 缓存 ========== has_error = False diff --git a/lib/Projectrans/__init__.py b/lib/Projectrans/__init__.py index 727be8e..f7373a3 100644 --- a/lib/Projectrans/__init__.py +++ b/lib/Projectrans/__init__.py @@ -18,14 +18,14 @@ from lib.Projectrans.Utils import ( ) from lib.Projectrans.Phase1Generator import Phase1Generator from lib.Projectrans.DeclarationGenerator import DeclarationGenerator -from lib.Projectrans.Phase2Translator import Phase2Translator, _parallel_translate_worker +from lib.Projectrans.Phase2Translator import Phase2Translator from lib.Projectrans.Config import save_sha1_map, Load_sha1_map, get_sha1_map_store, clear_sha1_map_store, Load_project_config, resolve_paths, main __all__ = [ 'compute_sha1', '_check_annotation_for_export', 'sha1_file', 'get_file_dependencies', 'find_reachable_files_from_entries', 'topological_sort_files', 'Phase1Generator', 'DeclarationGenerator', - 'Phase2Translator', '_parallel_translate_worker', + 'Phase2Translator', 'save_sha1_map', 'Load_sha1_map', 'get_sha1_map_store', 'clear_sha1_map_store', 'Load_project_config', 'resolve_paths', 'main' ]