fix(phase2): 并行编译 worker 数按可用内存动态限制,修复内存不足 OOM

系统可用内存低(Trae CN/Java 占用后仅剩 ~1GB)时,ProcessPoolExecutor
按 cpu_count 开满 worker,多进程编译内存叠加在 StructGen.finalize 拼接
module IR 时 MemoryError(编译 StubMergerTopo.vp 报错即此)。新增
_PickWorkerCount:GlobalMemoryStatusEx 读 ullAvailPhys,按 per_worker_mb
估算 worker 上限(翻译 700MB / llc 400MB),内存不足自动降为单 worker
串行,内存充足回落 cpu_count。StructGen.finalize 拼接改中间变量拆分,
降低单 worker 峰值副本。
This commit is contained in:
TermiNexus committed 2026-08-21 09:04:40 +08:00
1 parent e37cdf2eb0
commit 12207e91d8
2 files changed
+44 -5

No files matched your search

+41 -4
View File
@@ -23,6 +23,43 @@ import TransPyC
from StubGen import PythonToStubConverter
def _PickWorkerCount(n_files: int, per_worker_mb: int = 700) -> int:
"""按可用物理内存动态限制并行 worker 数,避免多进程编译内存峰值 OOM。
单 worker 预估 per_worker_mb(Python 解释器 + llvmlite module + 共享
符号表快照);系统可用内存不足时降为单 worker 串行,内存充足时回落到
cpu_count 并行。Windows 用 GlobalMemoryStatusEx 读 ullAvailPhys。
"""
base = min(os.cpu_count() or 4, n_files)
if base <= 1:
return base
try:
import ctypes
class _MemStat(ctypes.Structure):
_fields_ = [
("dwLength", ctypes.c_ulong),
("dwMemoryLoad", ctypes.c_ulong),
("ullTotalPhys", ctypes.c_ulonglong),
("ullAvailPhys", ctypes.c_ulonglong),
("ullTotalPageFile", ctypes.c_ulonglong),
("ullAvailPageFile", ctypes.c_ulonglong),
("ullTotalVirtual", ctypes.c_ulonglong),
("ullAvailVirtual", ctypes.c_ulonglong),
("ullAvailExtendedVirtual", ctypes.c_ulonglong),
]
stat = _MemStat()
stat.dwLength = ctypes.sizeof(_MemStat)
if ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(stat)):
avail_mb = stat.ullAvailPhys // (1024 * 1024)
by_mem = max(1, avail_mb // per_worker_mb)
return max(1, min(base, by_mem))
except Exception:
pass
return base
def _ExtractCDefineConstantsFromPyi(pyi_path: str, all_dc: dict[str, object]) -> None:
"""从 .pyi 文件中提取 CDefine 常量到 all_dc。文件不存在时静默跳过。
@@ -737,7 +774,7 @@ class Phase2Translator:
_vlog().info(f"\n需要翻译 {len(need_translate)} 个文件")
t_start = time.time()
n_workers = min(os.cpu_count() or 4, len(need_translate))
n_workers = _PickWorkerCount(len(need_translate))
if n_workers > 1 and len(need_translate) > 1:
_vlog().info(f" 并行翻译 (workers={n_workers})")
@@ -2019,7 +2056,7 @@ class Phase2Translator:
need_compile.append(ll_path)
if need_compile:
n_workers = min(os.cpu_count() or 4, len(need_compile))
n_workers = _PickWorkerCount(len(need_compile), 400)
_vlog().info(f" 并行编译 {len(need_compile)} 个文件 (workers={n_workers})")
errors = []
with ThreadPoolExecutor(max_workers=n_workers) as executor:
@@ -2545,7 +2582,7 @@ class Phase2Translator:
shared_pickle_path = ''
if shared_pickle_path and os.path.exists(shared_pickle_path):
n_workers = min(os.cpu_count() or 4, len(need_translate))
n_workers = _PickWorkerCount(len(need_translate))
errors = []
# 波次并行翻译:按依赖关系分批提交,确保被依赖的文件先翻译完成。
# 根因:_TryLoadStructFromStub 扫描 output 目录加载跨模块 struct 定义,
@@ -2784,7 +2821,7 @@ class Phase2Translator:
return (task, result)
if len(compile_tasks) > 1:
n_workers = min(os.cpu_count() or 4, len(compile_tasks))
n_workers = _PickWorkerCount(len(compile_tasks), 400)
with ThreadPoolExecutor(max_workers=n_workers) as executor:
futures = {executor.submit(_compile_one_include, task): task for task in compile_tasks}
done_count = 0
+3 -1
View File
@@ -201,7 +201,9 @@ class StructGenMixin:
if insert_pos == -1:
insert_pos = header_end
struct_block: str = '\n'.join(struct_defs) + '\n'
ir_text = ir_text[:insert_pos + 1] + struct_block + ir_text[insert_pos + 1:]
_ir_head = ir_text[:insert_pos + 1]
_ir_tail = ir_text[insert_pos + 1:]
ir_text = _ir_head + struct_block + _ir_tail
# 仅含特殊字符(数字开头或含点)的 struct 名需要加引号:
# 合并为单个正则一次扫描全文本,替代逐 name 各自 re.sub(每个 name
# 一次 O(text) 扫描)。lookahead 边界排除保证各备选互斥,语义等价。