fix(phase2): 并行编译 worker 数按可用内存动态限制,修复内存不足 OOM
系统可用内存低(Trae CN/Java 占用后仅剩 ~1GB)时,ProcessPoolExecutor 按 cpu_count 开满 worker,多进程编译内存叠加在 StructGen.finalize 拼接 module IR 时 MemoryError(编译 StubMergerTopo.vp 报错即此)。新增 _PickWorkerCount:GlobalMemoryStatusEx 读 ullAvailPhys,按 per_worker_mb 估算 worker 上限(翻译 700MB / llc 400MB),内存不足自动降为单 worker 串行,内存充足回落 cpu_count。StructGen.finalize 拼接改中间变量拆分, 降低单 worker 峰值副本。
This commit is contained in:
1 parent
e37cdf2eb0
commit
12207e91d8
2 files changed
+44
-5
No files matched your search
@@ -23,6 +23,43 @@ import TransPyC
|
||||
from StubGen import PythonToStubConverter
|
||||
|
||||
|
||||
def _PickWorkerCount(n_files: int, per_worker_mb: int = 700) -> int:
|
||||
"""按可用物理内存动态限制并行 worker 数,避免多进程编译内存峰值 OOM。
|
||||
|
||||
单 worker 预估 per_worker_mb(Python 解释器 + llvmlite module + 共享
|
||||
符号表快照);系统可用内存不足时降为单 worker 串行,内存充足时回落到
|
||||
cpu_count 并行。Windows 用 GlobalMemoryStatusEx 读 ullAvailPhys。
|
||||
"""
|
||||
base = min(os.cpu_count() or 4, n_files)
|
||||
if base <= 1:
|
||||
return base
|
||||
try:
|
||||
import ctypes
|
||||
|
||||
class _MemStat(ctypes.Structure):
|
||||
_fields_ = [
|
||||
("dwLength", ctypes.c_ulong),
|
||||
("dwMemoryLoad", ctypes.c_ulong),
|
||||
("ullTotalPhys", ctypes.c_ulonglong),
|
||||
("ullAvailPhys", ctypes.c_ulonglong),
|
||||
("ullTotalPageFile", ctypes.c_ulonglong),
|
||||
("ullAvailPageFile", ctypes.c_ulonglong),
|
||||
("ullTotalVirtual", ctypes.c_ulonglong),
|
||||
("ullAvailVirtual", ctypes.c_ulonglong),
|
||||
("ullAvailExtendedVirtual", ctypes.c_ulonglong),
|
||||
]
|
||||
|
||||
stat = _MemStat()
|
||||
stat.dwLength = ctypes.sizeof(_MemStat)
|
||||
if ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(stat)):
|
||||
avail_mb = stat.ullAvailPhys // (1024 * 1024)
|
||||
by_mem = max(1, avail_mb // per_worker_mb)
|
||||
return max(1, min(base, by_mem))
|
||||
except Exception:
|
||||
pass
|
||||
return base
|
||||
|
||||
|
||||
def _ExtractCDefineConstantsFromPyi(pyi_path: str, all_dc: dict[str, object]) -> None:
|
||||
"""从 .pyi 文件中提取 CDefine 常量到 all_dc。文件不存在时静默跳过。
|
||||
|
||||
@@ -737,7 +774,7 @@ class Phase2Translator:
|
||||
_vlog().info(f"\n需要翻译 {len(need_translate)} 个文件")
|
||||
t_start = time.time()
|
||||
|
||||
n_workers = min(os.cpu_count() or 4, len(need_translate))
|
||||
n_workers = _PickWorkerCount(len(need_translate))
|
||||
|
||||
if n_workers > 1 and len(need_translate) > 1:
|
||||
_vlog().info(f" 并行翻译 (workers={n_workers})")
|
||||
@@ -2019,7 +2056,7 @@ class Phase2Translator:
|
||||
need_compile.append(ll_path)
|
||||
|
||||
if need_compile:
|
||||
n_workers = min(os.cpu_count() or 4, len(need_compile))
|
||||
n_workers = _PickWorkerCount(len(need_compile), 400)
|
||||
_vlog().info(f" 并行编译 {len(need_compile)} 个文件 (workers={n_workers})")
|
||||
errors = []
|
||||
with ThreadPoolExecutor(max_workers=n_workers) as executor:
|
||||
@@ -2545,7 +2582,7 @@ class Phase2Translator:
|
||||
shared_pickle_path = ''
|
||||
|
||||
if shared_pickle_path and os.path.exists(shared_pickle_path):
|
||||
n_workers = min(os.cpu_count() or 4, len(need_translate))
|
||||
n_workers = _PickWorkerCount(len(need_translate))
|
||||
errors = []
|
||||
# 波次并行翻译:按依赖关系分批提交,确保被依赖的文件先翻译完成。
|
||||
# 根因:_TryLoadStructFromStub 扫描 output 目录加载跨模块 struct 定义,
|
||||
@@ -2784,7 +2821,7 @@ class Phase2Translator:
|
||||
return (task, result)
|
||||
|
||||
if len(compile_tasks) > 1:
|
||||
n_workers = min(os.cpu_count() or 4, len(compile_tasks))
|
||||
n_workers = _PickWorkerCount(len(compile_tasks), 400)
|
||||
with ThreadPoolExecutor(max_workers=n_workers) as executor:
|
||||
futures = {executor.submit(_compile_one_include, task): task for task in compile_tasks}
|
||||
done_count = 0
|
||||
|
||||
@@ -201,7 +201,9 @@ class StructGenMixin:
|
||||
if insert_pos == -1:
|
||||
insert_pos = header_end
|
||||
struct_block: str = '\n'.join(struct_defs) + '\n'
|
||||
ir_text = ir_text[:insert_pos + 1] + struct_block + ir_text[insert_pos + 1:]
|
||||
_ir_head = ir_text[:insert_pos + 1]
|
||||
_ir_tail = ir_text[insert_pos + 1:]
|
||||
ir_text = _ir_head + struct_block + _ir_tail
|
||||
# 仅含特殊字符(数字开头或含点)的 struct 名需要加引号:
|
||||
# 合并为单个正则一次扫描全文本,替代逐 name 各自 re.sub(每个 name
|
||||
# 一次 O(text) 扫描)。lookahead 边界排除保证各备选互斥,语义等价。
|
||||
|
||||
Reference in new issue
Block a user