From 12207e91d84ba8bcbecdc165e20d9d201dcf7c54 Mon Sep 17 00:00:00 2001 From: TermiNexus Date: Fri, 21 Aug 2026 09:02:39 +0800 Subject: [PATCH] =?UTF-8?q?fix(phase2):=20=E5=B9=B6=E8=A1=8C=E7=BC=96?= =?UTF-8?q?=E8=AF=91=20worker=20=E6=95=B0=E6=8C=89=E5=8F=AF=E7=94=A8?= =?UTF-8?q?=E5=86=85=E5=AD=98=E5=8A=A8=E6=80=81=E9=99=90=E5=88=B6=EF=BC=8C?= =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=86=85=E5=AD=98=E4=B8=8D=E8=B6=B3=20OOM?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 系统可用内存低（Trae CN/Java 占用后仅剩 ~1GB）时，ProcessPoolExecutor 按 cpu_count 开满 worker，多进程编译内存叠加在 StructGen.finalize 拼接 module IR 时 MemoryError（编译 StubMergerTopo.vp 报错即此）。新增 _PickWorkerCount：GlobalMemoryStatusEx 读 ullAvailPhys，按 per_worker_mb 估算 worker 上限（翻译 700MB / llc 400MB），内存不足自动降为单 worker 串行，内存充足回落 cpu_count。StructGen.finalize 拼接改中间变量拆分， 降低单 worker 峰值副本。 --- lib/Projectrans/Phase2Translator.py | 45 ++++++++++++++++++++++++++--- lib/core/LLVMCG/StructGen.py | 4 ++- 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/lib/Projectrans/Phase2Translator.py b/lib/Projectrans/Phase2Translator.py index f2b8130..de0a325 100644 --- a/lib/Projectrans/Phase2Translator.py +++ b/lib/Projectrans/Phase2Translator.py @@ -23,6 +23,43 @@ import TransPyC from StubGen import PythonToStubConverter +def _PickWorkerCount(n_files: int, per_worker_mb: int = 700) -> int: + """按可用物理内存动态限制并行 worker 数，避免多进程编译内存峰值 OOM。 + + 单 worker 预估 per_worker_mb（Python 解释器 + llvmlite module + 共享 + 符号表快照）；系统可用内存不足时降为单 worker 串行，内存充足时回落到 + cpu_count 并行。Windows 用 GlobalMemoryStatusEx 读 ullAvailPhys。 + """ + base = min(os.cpu_count() or 4, n_files) + if base <= 1: + return base + try: + import ctypes + + class _MemStat(ctypes.Structure): + _fields_ = [ + ("dwLength", ctypes.c_ulong), + ("dwMemoryLoad", ctypes.c_ulong), + ("ullTotalPhys", ctypes.c_ulonglong), + ("ullAvailPhys", ctypes.c_ulonglong), + ("ullTotalPageFile", ctypes.c_ulonglong), + ("ullAvailPageFile", ctypes.c_ulonglong), + ("ullTotalVirtual", ctypes.c_ulonglong), + ("ullAvailVirtual", ctypes.c_ulonglong), + ("ullAvailExtendedVirtual", ctypes.c_ulonglong), + ] + + stat = _MemStat() + stat.dwLength = ctypes.sizeof(_MemStat) + if ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(stat)): + avail_mb = stat.ullAvailPhys // (1024 * 1024) + by_mem = max(1, avail_mb // per_worker_mb) + return max(1, min(base, by_mem)) + except Exception: + pass + return base + + def _ExtractCDefineConstantsFromPyi(pyi_path: str, all_dc: dict[str, object]) -> None: """从 .pyi 文件中提取 CDefine 常量到 all_dc。文件不存在时静默跳过。 @@ -737,7 +774,7 @@ class Phase2Translator: _vlog().info(f"\n需要翻译 {len(need_translate)} 个文件") t_start = time.time() - n_workers = min(os.cpu_count() or 4, len(need_translate)) + n_workers = _PickWorkerCount(len(need_translate)) if n_workers > 1 and len(need_translate) > 1: _vlog().info(f" 并行翻译 (workers={n_workers})") @@ -2019,7 +2056,7 @@ class Phase2Translator: need_compile.append(ll_path) if need_compile: - n_workers = min(os.cpu_count() or 4, len(need_compile)) + n_workers = _PickWorkerCount(len(need_compile), 400) _vlog().info(f" 并行编译 {len(need_compile)} 个文件 (workers={n_workers})") errors = [] with ThreadPoolExecutor(max_workers=n_workers) as executor: @@ -2545,7 +2582,7 @@ class Phase2Translator: shared_pickle_path = '' if shared_pickle_path and os.path.exists(shared_pickle_path): - n_workers = min(os.cpu_count() or 4, len(need_translate)) + n_workers = _PickWorkerCount(len(need_translate)) errors = [] # 波次并行翻译：按依赖关系分批提交，确保被依赖的文件先翻译完成。 # 根因：_TryLoadStructFromStub 扫描 output 目录加载跨模块 struct 定义， @@ -2784,7 +2821,7 @@ class Phase2Translator: return (task, result) if len(compile_tasks) > 1: - n_workers = min(os.cpu_count() or 4, len(compile_tasks)) + n_workers = _PickWorkerCount(len(compile_tasks), 400) with ThreadPoolExecutor(max_workers=n_workers) as executor: futures = {executor.submit(_compile_one_include, task): task for task in compile_tasks} done_count = 0 diff --git a/lib/core/LLVMCG/StructGen.py b/lib/core/LLVMCG/StructGen.py index 97e994a..f6eb280 100644 --- a/lib/core/LLVMCG/StructGen.py +++ b/lib/core/LLVMCG/StructGen.py @@ -201,7 +201,9 @@ class StructGenMixin: if insert_pos == -1: insert_pos = header_end struct_block: str = '\n'.join(struct_defs) + '\n' - ir_text = ir_text[:insert_pos + 1] + struct_block + ir_text[insert_pos + 1:] + _ir_head = ir_text[:insert_pos + 1] + _ir_tail = ir_text[insert_pos + 1:] + ir_text = _ir_head + struct_block + _ir_tail # 仅含特殊字符（数字开头或含点）的 struct 名需要加引号： # 合并为单个正则一次扫描全文本，替代逐 name 各自 re.sub（每个 name # 一次 O(text) 扫描）。lookahead 边界排除保证各备选互斥，语义等价。