fix(compile): 回退 worker 进程级复用，消除共享 SymbolTable 跨文件污染
复用后同一 worker 的 trans 处理多文件，_translate_file 编译过程会向共享 SymbolTable 写入符号（import insert/类注册），前文件符号污染后文件 → 跨模块 struct 类型前缀错位（7cadfe07e983fd90 即 _list 前缀混入 CompilerFactory/Translator/File，Type of #1 arg mismatch）。每任务从 pickle 反序列化独立副本（原实现）。 收益评估：worker 复用仅省 pickle.load ~0.5s/次 × 每 worker 12 次 ≈ 6s 并行墙钟，远小于 worker 内翻译计算（百秒级），不值得以正确性为代价。n_workers 满核优化保留。
This commit is contained in:
TermiNexus committed
1 parent
2dbf348329
commit
a28aacda0b
1 file changed
+39
-48
@@ -65,62 +65,53 @@ def _ExtractCDefineConstantsFromPyi(pyi_path: str, all_dc: dict[str, object]) ->
|
||||
_vlog().warning(f"Phase2Translator: 忽略异常 {e}", exc_info=e)
|
||||
|
||||
|
||||
# worker 进程内缓存 Phase2Translator：同一 worker 进程处理多个文件时复用，
|
||||
# 避免每个任务重复创建 Phase2Translator + pickle.load 2.5MB 共享符号表
|
||||
# （App 阶段 200+ 文件分散到 16 worker，每 worker 处理 12+ 文件；原实现每
|
||||
# 文件都重建 trans + 重载 pickle → 大量重复反序列化，构成墙钟等待大户）。
|
||||
# key = shared_sym_pickle_path：App/includes 两阶段 pickle 不同，首次遇到
|
||||
# 新 key 才重建（每 worker 每阶段至多一次）。
|
||||
_worker_trans_cache: Phase2Translator | None = None
|
||||
_worker_trans_cache_key: str = ''
|
||||
|
||||
|
||||
def _parallel_translate_worker(src_path: str, out_path: str, src_root: str, temp_dir: str, output_dir: str,
|
||||
include_dirs: list[str], triple: str | None, datalayout: str | None,
|
||||
sha1_map: dict[str, str], sig_files: dict[str, str],
|
||||
stub_files: dict[str, str], include_py_map: dict[str, str], slice_level: int,
|
||||
shared_sym_pickle_path: str, struct_sha1_map: dict[str, str],
|
||||
prebuilt_ModuleSha1Map: dict[str, str] | None = None) -> tuple[str, str, str]:
|
||||
global _worker_trans_cache, _worker_trans_cache_key
|
||||
# 注意：不可跨任务复用 Phase2Translator/共享符号表！_translate_file 编译过程中
|
||||
# 会向共享 SymbolTable 写入符号（import insert/类注册等），复用导致前文件的
|
||||
# 符号污染后文件 → 跨模块 struct 类型前缀错位（如 7cadfe07e983fd90.CompilerFactory
|
||||
# 混入 _list 前缀，Type of #1 arg mismatch）。每任务必须从 pickle 反序列化出
|
||||
# 独立副本（原实现；worker 复用省掉的 pickle.load 时间仅 ~0.5s/次，远小于
|
||||
# worker 内翻译计算，不值得以正确性为代价）。
|
||||
try:
|
||||
if (_worker_trans_cache is None
|
||||
or _worker_trans_cache_key != shared_sym_pickle_path):
|
||||
trans = Phase2Translator(src_root, temp_dir, output_dir,
|
||||
compile_cmd='llc', include_dirs=include_dirs,
|
||||
target_triple=triple, target_datalayout=datalayout)
|
||||
trans.sha1_map = sha1_map
|
||||
trans.sig_files = sig_files
|
||||
trans.stub_files = stub_files
|
||||
trans.include_py_map = include_py_map
|
||||
trans.slice_level = slice_level
|
||||
trans.struct_sha1_map = struct_sha1_map
|
||||
trans = Phase2Translator(src_root, temp_dir, output_dir,
|
||||
compile_cmd='llc', include_dirs=include_dirs,
|
||||
target_triple=triple, target_datalayout=datalayout)
|
||||
trans.sha1_map = sha1_map
|
||||
trans.sig_files = sig_files
|
||||
trans.stub_files = stub_files
|
||||
trans.include_py_map = include_py_map
|
||||
trans.slice_level = slice_level
|
||||
trans.struct_sha1_map = struct_sha1_map
|
||||
|
||||
if shared_sym_pickle_path and os.path.exists(shared_sym_pickle_path):
|
||||
# 使用 pickle.load(f) 流式加载，避免 pickle.loads(f.read()) 一次性
|
||||
# 将整个 pickle 文件读入内存（磁盘副本 + 反序列化对象同时占用内存）导致 MemoryError。
|
||||
# 同时提高递归限制，防止深层嵌套对象反序列化失败。
|
||||
sys.setrecursionlimit(max(sys.getrecursionlimit(), 100000))
|
||||
with open(shared_sym_pickle_path, 'rb') as f:
|
||||
shared_data = pickle.load(f)
|
||||
trans._shared_symbol_table = shared_data['symbol_table']
|
||||
trans._shared_source_module_sig_files = shared_data['source_module_sig_files']
|
||||
trans._shared_all_dc = shared_data['all_dc']
|
||||
trans._shared_export_extern_funcs = shared_data['export_extern_funcs']
|
||||
trans.inline_func_symbols = shared_data['inline_func_symbols']
|
||||
trans.function_default_args = shared_data['function_default_args']
|
||||
trans.function_param_names = shared_data.get('function_param_names', {})
|
||||
trans._shared_generic_class_templates = shared_data.get('generic_class_templates', {})
|
||||
trans._shared_doc_meta = shared_data.get('shared_doc_meta', {})
|
||||
trans._shared_class_vtable = shared_data.get('class_vtable', set())
|
||||
trans._shared_cross_module_vtable = shared_data.get('cross_module_vtable', set())
|
||||
trans._shared_class_methods = shared_data.get('class_methods', {})
|
||||
trans._shared_cross_module_novtable = shared_data.get('cross_module_novtable', set())
|
||||
else:
|
||||
trans._precollect_inline_symbols()
|
||||
trans._build_shared_symbol_data()
|
||||
|
||||
if shared_sym_pickle_path and os.path.exists(shared_sym_pickle_path):
|
||||
# 使用 pickle.load(f) 流式加载，避免 pickle.loads(f.read()) 一次性
|
||||
# 将整个 pickle 文件读入内存（磁盘副本 + 反序列化对象同时占用内存）导致 MemoryError。
|
||||
# 同时提高递归限制，防止深层嵌套对象反序列化失败。
|
||||
sys.setrecursionlimit(max(sys.getrecursionlimit(), 100000))
|
||||
with open(shared_sym_pickle_path, 'rb') as f:
|
||||
shared_data = pickle.load(f)
|
||||
trans._shared_symbol_table = shared_data['symbol_table']
|
||||
trans._shared_source_module_sig_files = shared_data['source_module_sig_files']
|
||||
trans._shared_all_dc = shared_data['all_dc']
|
||||
trans._shared_export_extern_funcs = shared_data['export_extern_funcs']
|
||||
trans.inline_func_symbols = shared_data['inline_func_symbols']
|
||||
trans.function_default_args = shared_data['function_default_args']
|
||||
trans.function_param_names = shared_data.get('function_param_names', {})
|
||||
trans._shared_generic_class_templates = shared_data.get('generic_class_templates', {})
|
||||
trans._shared_doc_meta = shared_data.get('shared_doc_meta', {})
|
||||
trans._shared_class_vtable = shared_data.get('class_vtable', set())
|
||||
trans._shared_cross_module_vtable = shared_data.get('cross_module_vtable', set())
|
||||
trans._shared_class_methods = shared_data.get('class_methods', {})
|
||||
trans._shared_cross_module_novtable = shared_data.get('cross_module_novtable', set())
|
||||
else:
|
||||
trans._precollect_inline_symbols()
|
||||
trans._build_shared_symbol_data()
|
||||
_worker_trans_cache = trans
|
||||
_worker_trans_cache_key = shared_sym_pickle_path
|
||||
trans = _worker_trans_cache
|
||||
trans._translate_file(src_path, out_path, prebuilt_ModuleSha1Map=prebuilt_ModuleSha1Map)
|
||||
return ('ok', src_path, '')
|
||||
except Exception as e:
|
||||
|
||||
Reference in new issue
Block a user