diff --git a/lib/Projectrans/Phase1Generator.py b/lib/Projectrans/Phase1Generator.py index 5714366..f8fb5b0 100644 --- a/lib/Projectrans/Phase1Generator.py +++ b/lib/Projectrans/Phase1Generator.py @@ -357,9 +357,9 @@ class Phase1Generator: continue pyi_path: str = os.path.join(self.temp_dir, fname) try: - with open(pyi_path, 'r', encoding='utf-8') as f: - content: str = f.read() - tree: ast.Module = ast.parse(content) + content, tree = parse_python_file(pyi_path) + if tree is None: + continue for node in ast.iter_child_nodes(tree): if isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name): var_name: str = node.target.id @@ -676,9 +676,9 @@ class Phase1Generator: continue pyi_path: str = os.path.join(self.temp_dir, fname) try: - with open(pyi_path, 'r', encoding='utf-8') as f: - content: str = f.read() - tree: ast.Module = ast.parse(content) + content, tree = parse_python_file(pyi_path) + if tree is None: + continue for node in ast.iter_child_nodes(tree): if isinstance(node, ast.ClassDef): is_enum: bool = False diff --git a/lib/Projectrans/Phase2Translator.py b/lib/Projectrans/Phase2Translator.py index 5b796dd..87a1be5 100644 --- a/lib/Projectrans/Phase2Translator.py +++ b/lib/Projectrans/Phase2Translator.py @@ -96,7 +96,9 @@ def _parallel_translate_worker(src_path: str, out_path: str, src_root: str, temp sys.setrecursionlimit(max(sys.getrecursionlimit(), 100000)) with open(shared_sym_pickle_path, 'rb') as f: shared_data = pickle.load(f) - trans._shared_symbol_table = shared_data['symbol_table'] + _sym_bytes: bytes | None = shared_data.get('symbol_table_bytes') + if _sym_bytes is not None: + trans._shared_symbol_table = pickle.loads(_sym_bytes) trans._shared_source_module_sig_files = shared_data['source_module_sig_files'] trans._shared_all_dc = shared_data['all_dc'] trans._shared_export_extern_funcs = shared_data['export_extern_funcs'] @@ -150,7 +152,8 @@ def _parallel_translate_batch(batch: list, src_root: str, temp_dir: str, output_ with open(shared_sym_pickle_path, 'rb') as f: shared_data = pickle.load(f) symbol_table_bytes = shared_data.get('symbol_table_bytes') - trans._shared_symbol_table = shared_data['symbol_table'] + if symbol_table_bytes is not None: + trans._shared_symbol_table = pickle.loads(symbol_table_bytes) trans._shared_source_module_sig_files = shared_data['source_module_sig_files'] trans._shared_all_dc = shared_data['all_dc'] trans._shared_export_extern_funcs = shared_data['export_extern_funcs'] @@ -687,7 +690,6 @@ class Phase2Translator: # 写入的符号污染后文件 → 跨模块 struct 类型前缀错位)。 symbol_table_bytes: bytes = pickle.dumps(self._shared_symbol_table) shared_data = { - 'symbol_table': self._shared_symbol_table, 'symbol_table_bytes': symbol_table_bytes, 'source_module_sig_files': self._shared_source_module_sig_files, 'all_dc': self._shared_all_dc, @@ -726,9 +728,26 @@ class Phase2Translator: # 相比"一文件一任务":减少进程池调度/通信/构造开销 # (200+ 文件 → n_workers 个批次任务)。 batches: list[list] = [] - batch_size: int = max(1, (len(need_translate) + n_workers - 1) // n_workers) - for bi in range(0, len(need_translate), batch_size): - batches.append(need_translate[bi:bi + batch_size]) + # 按源文件大小加权分桶(LPT 贪心:最大文件优先放入当前总成本最小的桶), + # 使各批次总成本均匀。原按文件数均分(batch_size=ceil(N/n))导致大文件 + # 集中的批次拖长尾,其他 worker 提前完成空等(任务颗粒度不均匀)。 + if len(need_translate) <= n_workers: + batches = [[item] for item in need_translate] + else: + sized_items: list = [] + for item in need_translate: + try: + _size: int = os.path.getsize(item[1]) + except OSError: + _size = 0 + sized_items.append((item, _size)) + buckets: list[list] = [[] for _ in range(n_workers)] + bucket_cost: list[int] = [0] * n_workers + for item, _size in sorted(sized_items, key=lambda t: t[1], reverse=True): + target: int = min(range(n_workers), key=lambda w: bucket_cost[w]) + buckets[target].append(item) + bucket_cost[target] += _size + batches = [b for b in buckets if b] futures = {} for bidx, batch in enumerate(batches): rel_first = os.path.relpath(batch[0][1], self.src_root) @@ -2421,8 +2440,10 @@ class Phase2Translator: # 序列化共享符号表供 worker 使用 shared_pickle_path = os.path.join(self.output_dir, '_shared_sym_includes.pickle') try: + # symbol_table 只以 pickle 字节传递(symbol_table_bytes),不重复序列化 + # 对象本身(原双重序列化使文件体积/时间翻倍,4.65MB 中近半冗余)。 shared_data = { - 'symbol_table': self._shared_symbol_table, + 'symbol_table_bytes': pickle.dumps(self._shared_symbol_table), 'source_module_sig_files': self._shared_source_module_sig_files, 'all_dc': self._shared_all_dc, 'export_extern_funcs': self._shared_export_extern_funcs,