perf(compile): App 翻译改批次子任务模型（多文件/任务 + 每文件重置 SymbolTable）
原"一文件一任务"：200+ 文件每个都经 ProcessPool 调度 + worker 内创建 Phase2Translator + pickle.load 全部共享数据。改为批次模型：need_translate 按 worker 数分块，每 worker 一个批次子任务——Phase2Translator 创建 + pickle.load 仅一次，批内顺序翻译。 防污染：SymbolTable 是编译期唯一被写的共享对象，dump 时额外存 symbol_table_bytes（独立 pickle），批内每文件前 pickle.loads 出干净副本，前文件 import 写入不残留后文件（避免 7cadfe07e983fd90 类前缀错位）。includes 阶段保持拓扑单文件任务不变。
This commit is contained in:
TermiNexus committed
1 parent
a28aacda0b
commit
f66e119d1f
1 file changed
+93
-15
@@ -118,6 +118,69 @@ def _parallel_translate_worker(src_path: str, out_path: str, src_root: str, temp
|
||||
return ('error', src_path, f'{e}\n{traceback.format_exc()}')
|
||||
|
||||
|
||||
def _parallel_translate_batch(batch: list, src_root: str, temp_dir: str, output_dir: str,
|
||||
include_dirs: list[str], triple: str | None, datalayout: str | None,
|
||||
sha1_map: dict[str, str], sig_files: dict[str, str],
|
||||
stub_files: dict[str, str], include_py_map: dict[str, str], slice_level: int,
|
||||
shared_sym_pickle_path: str, struct_sha1_map: dict[str, str]) -> tuple[list, list]:
|
||||
"""批次子任务：worker 内创建一个 Phase2Translator + pickle.load 一次，顺序翻译批内全部文件。
|
||||
|
||||
batch 每项: (i, src_path, out_path, sha1, prebuilt_ModuleSha1Map)
|
||||
|
||||
防污染：SymbolTable 是编译期唯一被写的共享对象（import insert/类注册），
|
||||
批内每个文件翻译前从 symbol_table_bytes 反序列化出干净副本，前文件的
|
||||
符号写入不会残留到后文件（否则跨模块 struct 类型前缀错位）。
|
||||
返回 (错误列表[(src_path, err)], batch)。
|
||||
"""
|
||||
try:
|
||||
trans = Phase2Translator(src_root, temp_dir, output_dir,
|
||||
compile_cmd='llc', include_dirs=include_dirs,
|
||||
target_triple=triple, target_datalayout=datalayout)
|
||||
trans.sha1_map = sha1_map
|
||||
trans.sig_files = sig_files
|
||||
trans.stub_files = stub_files
|
||||
trans.include_py_map = include_py_map
|
||||
trans.slice_level = slice_level
|
||||
trans.struct_sha1_map = struct_sha1_map
|
||||
|
||||
symbol_table_bytes: bytes | None = None
|
||||
if shared_sym_pickle_path and os.path.exists(shared_sym_pickle_path):
|
||||
sys.setrecursionlimit(max(sys.getrecursionlimit(), 100000))
|
||||
with open(shared_sym_pickle_path, 'rb') as f:
|
||||
shared_data = pickle.load(f)
|
||||
symbol_table_bytes = shared_data.get('symbol_table_bytes')
|
||||
trans._shared_symbol_table = shared_data['symbol_table']
|
||||
trans._shared_source_module_sig_files = shared_data['source_module_sig_files']
|
||||
trans._shared_all_dc = shared_data['all_dc']
|
||||
trans._shared_export_extern_funcs = shared_data['export_extern_funcs']
|
||||
trans.inline_func_symbols = shared_data['inline_func_symbols']
|
||||
trans.function_default_args = shared_data['function_default_args']
|
||||
trans.function_param_names = shared_data.get('function_param_names', {})
|
||||
trans._shared_generic_class_templates = shared_data.get('generic_class_templates', {})
|
||||
trans._shared_doc_meta = shared_data.get('shared_doc_meta', {})
|
||||
trans._shared_class_vtable = shared_data.get('class_vtable', set())
|
||||
trans._shared_cross_module_vtable = shared_data.get('cross_module_vtable', set())
|
||||
trans._shared_class_methods = shared_data.get('class_methods', {})
|
||||
trans._shared_cross_module_novtable = shared_data.get('cross_module_novtable', set())
|
||||
else:
|
||||
trans._precollect_inline_symbols()
|
||||
trans._build_shared_symbol_data()
|
||||
|
||||
batch_errors: list = []
|
||||
for _i, src_path, out_path, _sha1, msm in batch:
|
||||
try:
|
||||
# 每文件前重置 SymbolTable 为干净副本（防跨文件污染）
|
||||
if symbol_table_bytes is not None:
|
||||
trans._shared_symbol_table = pickle.loads(symbol_table_bytes)
|
||||
trans._shared_symbol_table.translator = None
|
||||
trans._translate_file(src_path, out_path, prebuilt_ModuleSha1Map=msm)
|
||||
except Exception as e:
|
||||
batch_errors.append((src_path, f'{e}\n{traceback.format_exc()}'))
|
||||
return (batch_errors, batch)
|
||||
except Exception as e:
|
||||
return ([(item[1], f'{e}\n{traceback.format_exc()}') for item in batch], batch)
|
||||
|
||||
|
||||
def _count_braces_outside_strings(text: str) -> int:
|
||||
"""统计花括号差值，跳过 c"..." 字符串字面量内部的括号"""
|
||||
count = 0
|
||||
@@ -430,8 +493,14 @@ class Phase2Translator:
|
||||
|
||||
shared_pickle_path = os.path.join(self.temp_dir, '_shared_sym.json')
|
||||
try:
|
||||
# symbol_table_bytes：SymbolTable 独立 pickle 字节，供批次 worker
|
||||
# 每文件前反序列化出干净副本（SymbolTable 是编译期唯一被写的共享
|
||||
# 对象，批次内复用 trans 时必须在每个文件前重置，防止前文件 import
|
||||
# 写入的符号污染后文件 → 跨模块 struct 类型前缀错位）。
|
||||
symbol_table_bytes: bytes = pickle.dumps(self._shared_symbol_table)
|
||||
shared_data = {
|
||||
'symbol_table': self._shared_symbol_table,
|
||||
'symbol_table_bytes': symbol_table_bytes,
|
||||
'source_module_sig_files': self._shared_source_module_sig_files,
|
||||
'all_dc': self._shared_all_dc,
|
||||
'export_extern_funcs': self._shared_export_extern_funcs,
|
||||
@@ -459,35 +528,44 @@ class Phase2Translator:
|
||||
self._executor_pool = ProcessPoolExecutor(max_workers=n_workers)
|
||||
executor = self._executor_pool
|
||||
try:
|
||||
# 批次子任务模型：多个文件包装为一个子任务（batch），
|
||||
# 每 worker 处理一批——Phase2Translator 创建 + pickle.load 只做
|
||||
# 一次，批内文件顺序翻译（每文件前重置 SymbolTable 防污染）。
|
||||
# 相比"一文件一任务"：减少进程池调度/通信/构造开销
|
||||
# （200+ 文件 → n_workers 个批次任务）。
|
||||
batches: list[list] = []
|
||||
batch_size: int = max(1, (len(need_translate) + n_workers - 1) // n_workers)
|
||||
for bi in range(0, len(need_translate), batch_size):
|
||||
batches.append(need_translate[bi:bi + batch_size])
|
||||
futures = {}
|
||||
for i, src_path, out_path, sha1, msm in need_translate:
|
||||
rel = os.path.relpath(src_path, self.src_root)
|
||||
for bidx, batch in enumerate(batches):
|
||||
rel_first = os.path.relpath(batch[0][1], self.src_root)
|
||||
future = executor.submit(
|
||||
_parallel_translate_worker,
|
||||
src_path, out_path,
|
||||
_parallel_translate_batch,
|
||||
batch,
|
||||
self.src_root, self.temp_dir, self.output_dir,
|
||||
self.include_dirs, self.triple, self.datalayout,
|
||||
self.sha1_map, self.sig_files, self.stub_files,
|
||||
self.include_py_map, self.slice_level,
|
||||
shared_pickle_path, self.struct_sha1_map,
|
||||
prebuilt_ModuleSha1Map=msm
|
||||
)
|
||||
futures[future] = (i, rel)
|
||||
futures[future] = (bidx, rel_first, len(batch))
|
||||
|
||||
done_count = 0
|
||||
done_files = 0
|
||||
total_files = len(need_translate)
|
||||
for future in as_completed(futures):
|
||||
i, rel = futures[future]
|
||||
done_count += 1
|
||||
bidx, rel_first, bcount = futures[future]
|
||||
try:
|
||||
status, _, err = future.result()
|
||||
if status == 'ok':
|
||||
_vlog().info(f" [{done_count}/{len(need_translate)}] 完成: {rel}")
|
||||
else:
|
||||
batch_errors, batch = future.result()
|
||||
done_files += len(batch)
|
||||
for src, err in batch_errors:
|
||||
rel = os.path.relpath(src, self.src_root)
|
||||
_vlog().error(f" {rel}: {err}")
|
||||
errors.append((rel, err))
|
||||
_vlog().info(f" [{done_files}/{total_files}] 批次完成({bcount} 文件, 首个 {rel_first})")
|
||||
except Exception as e:
|
||||
_vlog().error(f" {rel}: {e}")
|
||||
errors.append((rel, str(e)))
|
||||
_vlog().error(f" 批次 {bidx} 异常: {e}")
|
||||
errors.append((rel_first, str(e)))
|
||||
except Exception:
|
||||
# 异常时立即清理，正常流程不 shutdown（留给 includes 阶段复用）
|
||||
if self._executor_pool is not None:
|
||||
|
||||
Reference in new issue
Block a user