From e3886a244ddf36085c2c8ef6fca5b8acc09eabab Mon Sep 17 00:00:00 2001 From: TermiNexus Date: Tue, 18 Aug 2026 05:36:12 +0800 Subject: [PATCH] =?UTF-8?q?perf(compile):=20=E6=89=B9=E6=AC=A1=E5=8A=A0?= =?UTF-8?q?=E6=9D=83=E5=88=86=E6=A1=B6=E6=B6=88=E9=99=A4=E9=95=BF=E5=B0=BE?= =?UTF-8?q?=20+=20=E5=85=B1=E4=BA=AB=E7=AC=A6=E5=8F=B7=E8=A1=A8=E5=8E=BB?= =?UTF-8?q?=E9=87=8D=E5=A4=8D=E5=BA=8F=E5=88=97=E5=8C=96=20+=20typedef=20?= =?UTF-8?q?=E6=94=B6=E9=9B=86=E8=B5=B0=20AST=20=E7=BC=93=E5=AD=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Phase2 批次切分：原按文件数均分（batch_size=ceil(N/n)），大文件集中批次拖长尾、worker 空等；改为按源文件大小 LPT 贪心分桶，各批次总成本均匀（任务颗粒度不均匀） - 共享符号表序列化：shared_data 同时含 symbol_table 对象与 symbol_table_bytes 双重序列化（_shared_sym.json 4.65MB 近半冗余）；统一为仅传 symbol_table_bytes，worker/批次端从字节反序列化（includes 与 App 两阶段同步修改） - Phase1 _collect_typedef_map / _process_file_pyi：直读+ast.parse 改为 parse_python_file mtime AST 缓存，Phase1 内重复收集与 Phase2 主进程解析共享同一缓存 --- lib/Projectrans/Phase1Generator.py | 12 +++++----- lib/Projectrans/Phase2Translator.py | 35 +++++++++++++++++++++++------ 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/lib/Projectrans/Phase1Generator.py b/lib/Projectrans/Phase1Generator.py index 5714366..f8fb5b0 100644 --- a/lib/Projectrans/Phase1Generator.py +++ b/lib/Projectrans/Phase1Generator.py @@ -357,9 +357,9 @@ class Phase1Generator: continue pyi_path: str = os.path.join(self.temp_dir, fname) try: - with open(pyi_path, 'r', encoding='utf-8') as f: - content: str = f.read() - tree: ast.Module = ast.parse(content) + content, tree = parse_python_file(pyi_path) + if tree is None: + continue for node in ast.iter_child_nodes(tree): if isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name): var_name: str = node.target.id @@ -676,9 +676,9 @@ class Phase1Generator: continue pyi_path: str = os.path.join(self.temp_dir, fname) try: - with open(pyi_path, 'r', encoding='utf-8') as f: - content: str = f.read() - tree: ast.Module = ast.parse(content) + content, tree = parse_python_file(pyi_path) + if tree is None: + continue for node in ast.iter_child_nodes(tree): if isinstance(node, ast.ClassDef): is_enum: bool = False diff --git a/lib/Projectrans/Phase2Translator.py b/lib/Projectrans/Phase2Translator.py index 5b796dd..87a1be5 100644 --- a/lib/Projectrans/Phase2Translator.py +++ b/lib/Projectrans/Phase2Translator.py @@ -96,7 +96,9 @@ def _parallel_translate_worker(src_path: str, out_path: str, src_root: str, temp sys.setrecursionlimit(max(sys.getrecursionlimit(), 100000)) with open(shared_sym_pickle_path, 'rb') as f: shared_data = pickle.load(f) - trans._shared_symbol_table = shared_data['symbol_table'] + _sym_bytes: bytes | None = shared_data.get('symbol_table_bytes') + if _sym_bytes is not None: + trans._shared_symbol_table = pickle.loads(_sym_bytes) trans._shared_source_module_sig_files = shared_data['source_module_sig_files'] trans._shared_all_dc = shared_data['all_dc'] trans._shared_export_extern_funcs = shared_data['export_extern_funcs'] @@ -150,7 +152,8 @@ def _parallel_translate_batch(batch: list, src_root: str, temp_dir: str, output_ with open(shared_sym_pickle_path, 'rb') as f: shared_data = pickle.load(f) symbol_table_bytes = shared_data.get('symbol_table_bytes') - trans._shared_symbol_table = shared_data['symbol_table'] + if symbol_table_bytes is not None: + trans._shared_symbol_table = pickle.loads(symbol_table_bytes) trans._shared_source_module_sig_files = shared_data['source_module_sig_files'] trans._shared_all_dc = shared_data['all_dc'] trans._shared_export_extern_funcs = shared_data['export_extern_funcs'] @@ -687,7 +690,6 @@ class Phase2Translator: # 写入的符号污染后文件 → 跨模块 struct 类型前缀错位）。 symbol_table_bytes: bytes = pickle.dumps(self._shared_symbol_table) shared_data = { - 'symbol_table': self._shared_symbol_table, 'symbol_table_bytes': symbol_table_bytes, 'source_module_sig_files': self._shared_source_module_sig_files, 'all_dc': self._shared_all_dc, @@ -726,9 +728,26 @@ class Phase2Translator: # 相比"一文件一任务"：减少进程池调度/通信/构造开销 # （200+ 文件 → n_workers 个批次任务）。 batches: list[list] = [] - batch_size: int = max(1, (len(need_translate) + n_workers - 1) // n_workers) - for bi in range(0, len(need_translate), batch_size): - batches.append(need_translate[bi:bi + batch_size]) + # 按源文件大小加权分桶（LPT 贪心：最大文件优先放入当前总成本最小的桶）， + # 使各批次总成本均匀。原按文件数均分（batch_size=ceil(N/n)）导致大文件 + # 集中的批次拖长尾，其他 worker 提前完成空等（任务颗粒度不均匀）。 + if len(need_translate) <= n_workers: + batches = [[item] for item in need_translate] + else: + sized_items: list = [] + for item in need_translate: + try: + _size: int = os.path.getsize(item[1]) + except OSError: + _size = 0 + sized_items.append((item, _size)) + buckets: list[list] = [[] for _ in range(n_workers)] + bucket_cost: list[int] = [0] * n_workers + for item, _size in sorted(sized_items, key=lambda t: t[1], reverse=True): + target: int = min(range(n_workers), key=lambda w: bucket_cost[w]) + buckets[target].append(item) + bucket_cost[target] += _size + batches = [b for b in buckets if b] futures = {} for bidx, batch in enumerate(batches): rel_first = os.path.relpath(batch[0][1], self.src_root) @@ -2421,8 +2440,10 @@ class Phase2Translator: # 序列化共享符号表供 worker 使用 shared_pickle_path = os.path.join(self.output_dir, '_shared_sym_includes.pickle') try: + # symbol_table 只以 pickle 字节传递（symbol_table_bytes），不重复序列化 + # 对象本身（原双重序列化使文件体积/时间翻倍，4.65MB 中近半冗余）。 shared_data = { - 'symbol_table': self._shared_symbol_table, + 'symbol_table_bytes': pickle.dumps(self._shared_symbol_table), 'source_module_sig_files': self._shared_source_module_sig_files, 'all_dc': self._shared_all_dc, 'export_extern_funcs': self._shared_export_extern_funcs,