diff --git a/lib/Projectrans/Phase2Translator.py b/lib/Projectrans/Phase2Translator.py index 2d4454c..086edba 100644 --- a/lib/Projectrans/Phase2Translator.py +++ b/lib/Projectrans/Phase2Translator.py @@ -8,7 +8,9 @@ import subprocess import vast as ast import traceback import json +import pickle import time +from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed, wait, FIRST_COMPLETED from lib.core.VLogger import get_logger as _vlog from lib.constants.config import mode as _config_mode from lib.core.Handles.HandlesBase import CTypeInfo @@ -63,6 +65,122 @@ def _ExtractCDefineConstantsFromPyi(pyi_path: str, all_dc: dict[str, object]) -> _vlog().warning(f"Phase2Translator: 忽略异常 {e}", exc_info=e) +def _parallel_translate_worker(src_path: str, out_path: str, src_root: str, temp_dir: str, output_dir: str, + include_dirs: list[str], triple: str | None, datalayout: str | None, + sha1_map: dict[str, str], sig_files: dict[str, str], + stub_files: dict[str, str], include_py_map: dict[str, str], slice_level: int, + shared_sym_pickle_path: str, struct_sha1_map: dict[str, str], + prebuilt_ModuleSha1Map: dict[str, str] | None = None) -> tuple[str, str, str]: + # 注意:不可跨任务复用 Phase2Translator/共享符号表!_translate_file 编译过程中 + # 会向共享 SymbolTable 写入符号(import insert/类注册等),复用导致前文件的 + # 符号污染后文件 → 跨模块 struct 类型前缀错位(如 7cadfe07e983fd90.CompilerFactory + # 混入 _list 前缀,Type of #1 arg mismatch)。每任务必须从 pickle 反序列化出 + # 独立副本(原实现;worker 复用省掉的 pickle.load 时间仅 ~0.5s/次,远小于 + # worker 内翻译计算,不值得以正确性为代价)。 + try: + trans = Phase2Translator(src_root, temp_dir, output_dir, + compile_cmd='llc', include_dirs=include_dirs, + target_triple=triple, target_datalayout=datalayout) + trans.sha1_map = sha1_map + trans.sig_files = sig_files + trans.stub_files = stub_files + trans.include_py_map = include_py_map + trans.slice_level = slice_level + trans.struct_sha1_map = struct_sha1_map + + if shared_sym_pickle_path and os.path.exists(shared_sym_pickle_path): + # 使用 pickle.load(f) 流式加载,避免 pickle.loads(f.read()) 一次性 + # 将整个 pickle 文件读入内存(磁盘副本 + 反序列化对象同时占用内存)导致 MemoryError。 + # 同时提高递归限制,防止深层嵌套对象反序列化失败。 + sys.setrecursionlimit(max(sys.getrecursionlimit(), 100000)) + with open(shared_sym_pickle_path, 'rb') as f: + shared_data = pickle.load(f) + trans._shared_symbol_table = shared_data['symbol_table'] + trans._shared_source_module_sig_files = shared_data['source_module_sig_files'] + trans._shared_all_dc = shared_data['all_dc'] + trans._shared_export_extern_funcs = shared_data['export_extern_funcs'] + trans.inline_func_symbols = shared_data['inline_func_symbols'] + trans.function_default_args = shared_data['function_default_args'] + trans.function_param_names = shared_data.get('function_param_names', {}) + trans._shared_generic_class_templates = shared_data.get('generic_class_templates', {}) + trans._shared_doc_meta = shared_data.get('shared_doc_meta', {}) + trans._shared_class_vtable = shared_data.get('class_vtable', set()) + trans._shared_cross_module_vtable = shared_data.get('cross_module_vtable', set()) + trans._shared_class_methods = shared_data.get('class_methods', {}) + trans._shared_cross_module_novtable = shared_data.get('cross_module_novtable', set()) + else: + trans._precollect_inline_symbols() + trans._build_shared_symbol_data() + + trans._translate_file(src_path, out_path, prebuilt_ModuleSha1Map=prebuilt_ModuleSha1Map) + return ('ok', src_path, '') + except Exception as e: + return ('error', src_path, f'{e}\n{traceback.format_exc()}') + + +def _parallel_translate_batch(batch: list, src_root: str, temp_dir: str, output_dir: str, + include_dirs: list[str], triple: str | None, datalayout: str | None, + sha1_map: dict[str, str], sig_files: dict[str, str], + stub_files: dict[str, str], include_py_map: dict[str, str], slice_level: int, + shared_sym_pickle_path: str, struct_sha1_map: dict[str, str]) -> tuple[list, list]: + """批次子任务:worker 内创建一个 Phase2Translator + pickle.load 一次,顺序翻译批内全部文件。 + + batch 每项: (i, src_path, out_path, sha1, prebuilt_ModuleSha1Map) + + 防污染:SymbolTable 是编译期唯一被写的共享对象(import insert/类注册), + 批内每个文件翻译前从 symbol_table_bytes 反序列化出干净副本,前文件的 + 符号写入不会残留到后文件(否则跨模块 struct 类型前缀错位)。 + 返回 (错误列表[(src_path, err)], batch)。 + """ + try: + trans = Phase2Translator(src_root, temp_dir, output_dir, + compile_cmd='llc', include_dirs=include_dirs, + target_triple=triple, target_datalayout=datalayout) + trans.sha1_map = sha1_map + trans.sig_files = sig_files + trans.stub_files = stub_files + trans.include_py_map = include_py_map + trans.slice_level = slice_level + trans.struct_sha1_map = struct_sha1_map + + symbol_table_bytes: bytes | None = None + if shared_sym_pickle_path and os.path.exists(shared_sym_pickle_path): + sys.setrecursionlimit(max(sys.getrecursionlimit(), 100000)) + with open(shared_sym_pickle_path, 'rb') as f: + shared_data = pickle.load(f) + symbol_table_bytes = shared_data.get('symbol_table_bytes') + trans._shared_symbol_table = shared_data['symbol_table'] + trans._shared_source_module_sig_files = shared_data['source_module_sig_files'] + trans._shared_all_dc = shared_data['all_dc'] + trans._shared_export_extern_funcs = shared_data['export_extern_funcs'] + trans.inline_func_symbols = shared_data['inline_func_symbols'] + trans.function_default_args = shared_data['function_default_args'] + trans.function_param_names = shared_data.get('function_param_names', {}) + trans._shared_generic_class_templates = shared_data.get('generic_class_templates', {}) + trans._shared_doc_meta = shared_data.get('shared_doc_meta', {}) + trans._shared_class_vtable = shared_data.get('class_vtable', set()) + trans._shared_cross_module_vtable = shared_data.get('cross_module_vtable', set()) + trans._shared_class_methods = shared_data.get('class_methods', {}) + trans._shared_cross_module_novtable = shared_data.get('cross_module_novtable', set()) + else: + trans._precollect_inline_symbols() + trans._build_shared_symbol_data() + + batch_errors: list = [] + for _i, src_path, out_path, _sha1, msm in batch: + try: + # 每文件前重置 SymbolTable 为干净副本(防跨文件污染) + if symbol_table_bytes is not None: + trans._shared_symbol_table = pickle.loads(symbol_table_bytes) + trans._shared_symbol_table.translator = None + trans._translate_file(src_path, out_path, prebuilt_ModuleSha1Map=msm) + except Exception as e: + batch_errors.append((src_path, f'{e}\n{traceback.format_exc()}')) + return (batch_errors, batch) + except Exception as e: + return ([(item[1], f'{e}\n{traceback.format_exc()}') for item in batch], batch) + + def _count_braces_outside_strings(text: str) -> int: """统计花括号差值,跳过 c"..." 字符串字面量内部的括号""" count = 0 @@ -134,6 +252,9 @@ class Phase2Translator: self.extra_compile_files: list[str] = [] self.extra_py_files: list[str] = [] self._include_sha1s: set[str] = set() + # 持有 ProcessPoolExecutor 引用,避免局部变量 GC 时 finalizer 阻塞 ~99s + # (join_executor_internals 在 GC 时同步等待 worker 退出队列) + self._executor_pool: ProcessPoolExecutor | None = None self.entry_files: list[str] | None = entry_files self.force_recompile_includes: bool = force_recompile_includes self._shared_symbol_table: object | None = None @@ -359,26 +480,116 @@ class Phase2Translator: # 提前从 include 文件源码中提取函数参数名,供主项目编译时 keyword 参数匹配使用 self._extract_include_param_names_from_dirs() # 提前从 App 源文件中提取函数默认参数,供跨模块调用参数检查使用 + # (避免并行编译时,被调用函数的文件尚未编译完,导致默认参数信息缺失) self._extract_app_default_args(need_translate) _vlog().info(f"\n需要翻译 {len(need_translate)} 个文件") t_start = time.time() - # 顺序阻塞翻译(完全去除多进程):依赖排序保证被依赖文件先翻译, - # 单进程内共享符号表状态与 _TryLoadStructFromStub 的 output 目录扫描 - # 天然一致,杜绝批次子任务模型跨模块 struct 类型前缀错位 - # (7cadfe07e983fd90.Translator != 6ccfee1e5bd522ed.Translator)。 - for idx, (i, src_path, out_path, sha1, msm) in enumerate(need_translate): - rel = os.path.relpath(src_path, self.src_root) - t0 = time.time() + n_workers = min(os.cpu_count() or 4, len(need_translate)) + + if n_workers > 1 and len(need_translate) > 1: + _vlog().info(f" 并行翻译 (workers={n_workers})") + + shared_pickle_path = os.path.join(self.temp_dir, '_shared_sym.json') try: - self._translate_file(src_path, out_path, prebuilt_ModuleSha1Map=msm) - t1 = time.time() - _vlog().info(f" [{idx+1}/{len(need_translate)}] {rel} ({t1-t0:.1f}s)") + # symbol_table_bytes:SymbolTable 独立 pickle 字节,供批次 worker + # 每文件前反序列化出干净副本(SymbolTable 是编译期唯一被写的共享 + # 对象,批次内复用 trans 时必须在每个文件前重置,防止前文件 import + # 写入的符号污染后文件 → 跨模块 struct 类型前缀错位)。 + symbol_table_bytes: bytes = pickle.dumps(self._shared_symbol_table) + shared_data = { + 'symbol_table': self._shared_symbol_table, + 'symbol_table_bytes': symbol_table_bytes, + 'source_module_sig_files': self._shared_source_module_sig_files, + 'all_dc': self._shared_all_dc, + 'export_extern_funcs': self._shared_export_extern_funcs, + 'inline_func_symbols': self.inline_func_symbols, + 'function_default_args': self.function_default_args, + 'function_param_names': self.function_param_names, + 'generic_class_templates': self._shared_generic_class_templates, + 'shared_doc_meta': self._shared_doc_meta, + 'class_vtable': self._shared_class_vtable, + 'cross_module_vtable': self._shared_cross_module_vtable, + 'class_methods': self._shared_class_methods, + 'cross_module_novtable': self._shared_cross_module_novtable, + } + with open(shared_pickle_path, 'wb') as f: + f.write(pickle.dumps(shared_data)) + _vlog().info(f" 共享符号表已序列化 ({os.path.getsize(shared_pickle_path)//1024}KB)") except Exception as e: - _vlog().error(f" {rel}: {e}") - traceback.print_exc() - raise RuntimeError(f"翻译 {rel} 失败,已终止编译: {e}") from e + _vlog().warning(f"符号表序列化失败({e}),worker将自行构建") + shared_pickle_path = '' + + errors = [] + # 复用 self._executor_pool:避免局部变量 GC 时 finalizer 阻塞 ~99s + # (join_executor_internals 在 GC 时同步等待 worker 退出队列) + if self._executor_pool is None: + self._executor_pool = ProcessPoolExecutor(max_workers=n_workers) + executor = self._executor_pool + try: + # 批次子任务模型:多个文件包装为一个子任务(batch), + # 每 worker 处理一批——Phase2Translator 创建 + pickle.load 只做 + # 一次,批内文件顺序翻译(每文件前重置 SymbolTable 防污染)。 + # 相比"一文件一任务":减少进程池调度/通信/构造开销 + # (200+ 文件 → n_workers 个批次任务)。 + batches: list[list] = [] + batch_size: int = max(1, (len(need_translate) + n_workers - 1) // n_workers) + for bi in range(0, len(need_translate), batch_size): + batches.append(need_translate[bi:bi + batch_size]) + futures = {} + for bidx, batch in enumerate(batches): + rel_first = os.path.relpath(batch[0][1], self.src_root) + future = executor.submit( + _parallel_translate_batch, + batch, + self.src_root, self.temp_dir, self.output_dir, + self.include_dirs, self.triple, self.datalayout, + self.sha1_map, self.sig_files, self.stub_files, + self.include_py_map, self.slice_level, + shared_pickle_path, self.struct_sha1_map, + ) + futures[future] = (bidx, rel_first, len(batch)) + + done_files = 0 + total_files = len(need_translate) + for future in as_completed(futures): + bidx, rel_first, bcount = futures[future] + try: + batch_errors, batch = future.result() + done_files += len(batch) + for src, err in batch_errors: + rel = os.path.relpath(src, self.src_root) + _vlog().error(f" {rel}: {err}") + errors.append((rel, err)) + _vlog().info(f" [{done_files}/{total_files}] 批次完成({bcount} 文件, 首个 {rel_first})") + except Exception as e: + _vlog().error(f" 批次 {bidx} 异常: {e}") + errors.append((rel_first, str(e))) + except Exception: + # 异常时立即清理,正常流程不 shutdown(留给 includes 阶段复用) + if self._executor_pool is not None: + self._executor_pool.shutdown(wait=False, cancel_futures=True) + self._executor_pool = None + raise + + if errors: + _vlog().error(f"\n{len(errors)} 个文件翻译失败,立即终止编译") + for rel, err in errors: + _vlog().error(f" {rel}: {err}") + raise RuntimeError(f"{len(errors)} 个文件翻译失败,已终止编译") + else: + for idx, (i, src_path, out_path, sha1, msm) in enumerate(need_translate): + rel = os.path.relpath(src_path, self.src_root) + t0 = time.time() + try: + self._translate_file(src_path, out_path, prebuilt_ModuleSha1Map=msm) + t1 = time.time() + _vlog().info(f" [{idx+1}/{len(need_translate)}] {rel} ({t1-t0:.1f}s)") + except Exception as e: + _vlog().error(f" {rel}: {e}") + traceback.print_exc() + raise RuntimeError(f"翻译 {rel} 失败,已终止编译: {e}") from e t_end = time.time() _vlog().info(f" 翻译总耗时: {t_end-t_start:.1f}s") @@ -389,6 +600,11 @@ class Phase2Translator: self._compile_include_sources() if self.linker_cmd: self._link_obj_files(active_sha1s) + # 清理 ProcessPoolExecutor:所有翻译已完成,shutdown(wait=False) 让 worker + # 后台退出。避免 GC finalizer 在程序末尾阻塞 ~99s(join_executor_internals) + if self._executor_pool is not None: + self._executor_pool.shutdown(wait=False, cancel_futures=True) + self._executor_pool = None def _build_current_ModuleSha1Map(self, self_sha1: str | None) -> dict[str, str]: """构建当前模块的依赖 SHA1 映射(与 _translate_file 中逻辑一致)""" @@ -1511,17 +1727,20 @@ class Phase2Translator: need_compile.append(ll_path) if need_compile: - _vlog().info(f" 编译 {len(need_compile)} 个 .ll 文件") + n_workers = min(os.cpu_count() or 4, len(need_compile)) + _vlog().info(f" 并行编译 {len(need_compile)} 个文件 (workers={n_workers})") errors = [] - for ll_path in need_compile: - rel, status, err = _compile_one(ll_path) - if status == 'ok': - _vlog().success(f" {rel}") - elif status == 'error': - _vlog().error(f" {rel}: {err}") - errors.append((rel, err)) - elif status == 'cached': - pass + with ThreadPoolExecutor(max_workers=n_workers) as executor: + futures = {executor.submit(_compile_one, ll_path): ll_path for ll_path in need_compile} + for future in as_completed(futures): + rel, status, err = future.result() + if status == 'ok': + _vlog().success(f" {rel}") + elif status == 'error': + _vlog().error(f" {rel}: {err}") + errors.append((rel, err)) + elif status == 'cached': + pass if errors: _vlog().error(f"\n{len(errors)} 个文件编译失败") sys.exit(1) @@ -1529,8 +1748,8 @@ class Phase2Translator: def _precollect_vtable_state(self) -> None: """预扫描 includes .py 文件,收集 @t.CVTable 类的虚表信息。 - 顺序阻塞编译单进程内共享符号表,虚表状态必须在全部文件翻译前收集, - 保证跨模块 CVTable 继承方法分派一致(pool.alloc 虚表大小不漂移)。 + 并行编译(ProcessPoolExecutor)创建独立内存的 worker 进程, + vtable 状态必须预收集并通过 pickle 共享。 复用 parse_python_file 的 AST 缓存。 """ for includes_dir in self.include_dirs: @@ -1915,7 +2134,7 @@ class Phase2Translator: return # 按依赖关系排序 include 文件,确保被依赖的文件先编译 - sorted_files, _ = self._sort_include_files_by_deps(self.extra_py_files) + sorted_files, file_deps = self._sort_include_files_by_deps(self.extra_py_files) _vlog().info(f"\n[includes 编译] 找到 {len(sorted_files)} 个 Python 源文件") # 构建 include 文件的模块 SHA1 映射(预计算所有文件,构建完整映射) @@ -1980,7 +2199,7 @@ class Phase2Translator: return # 预注册所有待翻译文件的符号(从 Phase1 缓存加载 .pyi / .stub.ll) - # 翻译前全部注册,保证顺序翻译时所有跨模块符号都已可用 + # 这样并行翻译时所有跨模块符号都已可用 for src_path, sha1, module_name, ll_path, obj_path, precompiled_obj in need_translate: self._register_compiled_include(src_path, sha1, None, include_ModuleSha1Map) @@ -2004,12 +2223,129 @@ class Phase2Translator: if tname: all_type_defs[tname] = sdecl - # ========== 阶段1:顺序阻塞翻译 ========== - # 完全去除多进程:need_translate 已按依赖拓扑排序(_sort_include_files_by_deps), - # 顺序翻译保证被依赖文件先完成,_TryLoadStructFromStub 扫描 output 目录 - # 总能取到完整的跨模块 struct 定义,且单进程共享符号表状态天然一致。 + # ========== 阶段1:并行翻译 ========== _vlog().info(f" 翻译 {len(need_translate)} 个文件") - for item in need_translate: + if len(need_translate) > 1: + # 序列化共享符号表供 worker 使用 + shared_pickle_path = os.path.join(self.output_dir, '_shared_sym_includes.pickle') + try: + shared_data = { + 'symbol_table': self._shared_symbol_table, + 'source_module_sig_files': self._shared_source_module_sig_files, + 'all_dc': self._shared_all_dc, + 'export_extern_funcs': self._shared_export_extern_funcs, + 'inline_func_symbols': self.inline_func_symbols, + 'function_default_args': self.function_default_args, + 'function_param_names': self.function_param_names, + 'generic_class_templates': self._shared_generic_class_templates, + 'shared_doc_meta': self._shared_doc_meta, + 'class_vtable': self._shared_class_vtable, + 'cross_module_vtable': self._shared_cross_module_vtable, + 'class_methods': self._shared_class_methods, + 'cross_module_novtable': self._shared_cross_module_novtable, + } + with open(shared_pickle_path, 'wb') as f: + f.write(pickle.dumps(shared_data)) + except Exception as e: + _vlog().warning(f"符号表序列化失败({e}),include将串行翻译") + shared_pickle_path = '' + + if shared_pickle_path and os.path.exists(shared_pickle_path): + n_workers = min(os.cpu_count() or 4, len(need_translate)) + errors = [] + # 波次并行翻译:按依赖关系分批提交,确保被依赖的文件先翻译完成。 + # 根因:_TryLoadStructFromStub 扫描 output 目录加载跨模块 struct 定义, + # 若依赖文件尚未完成翻译(output stub 未写入),只能取到 Phase1 temp + # 中的不完整 stub(缺少跨模块继承字段),导致运行时崩溃。 + item_by_path = {item[0]: item for item in need_translate} + translate_paths = set(item_by_path.keys()) + # 每个文件的依赖(仅限本轮待翻译集合内) + translate_deps = { + p: (file_deps.get(p, set()) & translate_paths) + for p in translate_paths + } + # 复用 self._executor_pool(与 App 源文件阶段共享),避免 GC 阻塞 + if self._executor_pool is None: + self._executor_pool = ProcessPoolExecutor(max_workers=n_workers) + executor = self._executor_pool + try: + completed_paths: set[str] = set() + remaining = list(need_translate) # 保持拓扑序 + in_progress: dict = {} # future -> item + done_count = 0 + + while remaining or in_progress: + # 提交所有依赖已完成的文件 + ready = [ + item for item in remaining + if translate_deps[item[0]] <= completed_paths + ] + for item in ready: + remaining.remove(item) + src_path, sha1, module_name, ll_path, obj_path, precompiled_obj = item + future = executor.submit( + _parallel_translate_worker, + src_path, ll_path, + self.src_root, self.temp_dir, self.output_dir, + self.include_dirs, self.triple, self.datalayout, + self.sha1_map, self.sig_files, self.stub_files, + self.include_py_map, self.slice_level, + shared_pickle_path, self.struct_sha1_map, + include_ModuleSha1Map + ) + in_progress[future] = item + + if not in_progress: + # 无可提交且无进行中:循环依赖 fallback + for item in remaining: + src_path, sha1, module_name, ll_path, obj_path, precompiled_obj = item + future = executor.submit( + _parallel_translate_worker, + src_path, ll_path, + self.src_root, self.temp_dir, self.output_dir, + self.include_dirs, self.triple, self.datalayout, + self.sha1_map, self.sig_files, self.stub_files, + self.include_py_map, self.slice_level, + shared_pickle_path, self.struct_sha1_map, + include_ModuleSha1Map + ) + in_progress[future] = item + remaining.clear() + + # 等待至少一个完成 + done_set, _ = wait(in_progress, return_when=FIRST_COMPLETED) + for future in done_set: + item = in_progress.pop(future) + src_path, sha1, module_name, ll_path, obj_path, precompiled_obj = item + completed_paths.add(src_path) + done_count += 1 + try: + status, _, err = future.result() + if status == 'ok': + _vlog().info(f" [{done_count}/{len(need_translate)}] 翻译完成: {module_name}.py") + else: + _vlog().error(f" {module_name}.py: {err}") + errors.append((module_name, err)) + except Exception as e: + _vlog().error(f" {module_name}.py: {e}") + errors.append((module_name, str(e))) + except Exception: + if self._executor_pool is not None: + self._executor_pool.shutdown(wait=False, cancel_futures=True) + self._executor_pool = None + raise + + if errors: + _vlog().error(f"\n{len(errors)} 个 include 文件翻译失败") + sys.exit(1) + else: + # 串行回退 + for item in need_translate: + src_path, sha1, module_name, ll_path, obj_path, precompiled_obj = item + _vlog().info(f" 翻译: {module_name}.py -> {sha1}.ll") + self._translate_file(src_path, ll_path, prebuilt_ModuleSha1Map=include_ModuleSha1Map) + else: + item = need_translate[0] src_path, sha1, module_name, ll_path, obj_path, precompiled_obj = item _vlog().info(f" 翻译: {module_name}.py -> {sha1}.ll") self._translate_file(src_path, ll_path, prebuilt_ModuleSha1Map=include_ModuleSha1Map) @@ -2130,7 +2466,7 @@ class Phase2Translator: if not compile_tasks: return - # ========== 阶段3:顺序编译(llc)========== + # ========== 阶段3:并行编译(llc)========== _vlog().info(f" 编译 {len(compile_tasks)} 个 .ll 文件") compile_results = {} # task -> result @@ -2139,19 +2475,35 @@ class Phase2Translator: result = self._llc_compile(dso_ll_path, obj_path, self.output_dir) return (task, result) - for done_count, task in enumerate(compile_tasks, 1): - try: - task_result, result = _compile_one_include(task) - module_name = task[4] - if result.returncode == 0: - _vlog().success(f" [{done_count}/{len(compile_tasks)}] 编译成功: {module_name}.py") - else: - _vlog().warning(f" [{done_count}/{len(compile_tasks)}] 编译失败: {module_name}.py: {result.stderr.strip()}") - compile_results[task] = result - except Exception as e: - module_name = task[4] - _vlog().error(f" [{done_count}/{len(compile_tasks)}] 编译异常: {module_name}.py: {e}") - compile_results[task] = None + if len(compile_tasks) > 1: + n_workers = min(os.cpu_count() or 4, len(compile_tasks)) + with ThreadPoolExecutor(max_workers=n_workers) as executor: + futures = {executor.submit(_compile_one_include, task): task for task in compile_tasks} + done_count = 0 + for future in as_completed(futures): + done_count += 1 + try: + task, result = future.result() + module_name = task[4] + if result.returncode == 0: + _vlog().success(f" [{done_count}/{len(compile_tasks)}] 编译成功: {module_name}.py") + else: + _vlog().warning(f" [{done_count}/{len(compile_tasks)}] 编译失败: {module_name}.py: {result.stderr.strip()}") + compile_results[task] = result + except Exception as e: + task = futures[future] + module_name = task[4] + _vlog().error(f" [{done_count}/{len(compile_tasks)}] 编译异常: {module_name}.py: {e}") + compile_results[task] = None + else: + task = compile_tasks[0] + task_result, result = _compile_one_include(task) + module_name = task[4] + if result.returncode == 0: + _vlog().success(f" 编译成功: {module_name}.py") + else: + _vlog().warning(f" 编译失败: {module_name}.py: {result.stderr.strip()}") + compile_results[task] = result # ========== 阶段4:注册 + 缓存 ========== has_error = False diff --git a/lib/Projectrans/__init__.py b/lib/Projectrans/__init__.py index f7373a3..727be8e 100644 --- a/lib/Projectrans/__init__.py +++ b/lib/Projectrans/__init__.py @@ -18,14 +18,14 @@ from lib.Projectrans.Utils import ( ) from lib.Projectrans.Phase1Generator import Phase1Generator from lib.Projectrans.DeclarationGenerator import DeclarationGenerator -from lib.Projectrans.Phase2Translator import Phase2Translator +from lib.Projectrans.Phase2Translator import Phase2Translator, _parallel_translate_worker from lib.Projectrans.Config import save_sha1_map, Load_sha1_map, get_sha1_map_store, clear_sha1_map_store, Load_project_config, resolve_paths, main __all__ = [ 'compute_sha1', '_check_annotation_for_export', 'sha1_file', 'get_file_dependencies', 'find_reachable_files_from_entries', 'topological_sort_files', 'Phase1Generator', 'DeclarationGenerator', - 'Phase2Translator', + 'Phase2Translator', '_parallel_translate_worker', 'save_sha1_map', 'Load_sha1_map', 'get_sha1_map_store', 'clear_sha1_map_store', 'Load_project_config', 'resolve_paths', 'main' ]