From b08a0aff48ee4e701a502fb2fd450eacbcb5e11d Mon Sep 17 00:00:00 2001 From: TermiNexus Date: Thu, 20 Aug 2026 01:15:05 +0800 Subject: [PATCH] =?UTF-8?q?perf(phase2):=20=E7=BC=93=E5=AD=98=20BuildCombi?= =?UTF-8?q?nedIR=20=E5=85=A8=E7=9B=AE=E5=BD=95=E6=89=AB=E6=8F=8F=E7=BB=93?= =?UTF-8?q?=E6=9E=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase B 编译 .obj 慢得像同步编译的根因：BuildCombinedIR 对每个源文件 调用一次 _collect_stub_sha1s 递归扫描 temp_dir 全部切片子目录（90 文件 × 全目录枚举 O(N×S)），且目录扫描与 llc 并行无关（llc 并行已生效）。 按 temp_dir 缓存扫描结果（StubMerger._stub_scan_cache_*，stdlib 分配 跨 reset 存活），首次扫描后复用，temp_dir 变化时自动重建。BuildCombinedIR 的所有调用点（Phase B 主循环/Phase B+ includes 编译/stubgen）统一受益。 --- App/lib/core/StubMerger.vp | 8 +++ App/lib/core/StubMergerBuild.vp | 116 ++++++++++++++++++++------------ 2 files changed, 80 insertions(+), 44 deletions(-) diff --git a/App/lib/core/StubMerger.vp b/App/lib/core/StubMerger.vp index 14a84ad..11a25eb 100644 --- a/App/lib/core/StubMerger.vp +++ b/App/lib/core/StubMerger.vp @@ -63,6 +63,14 @@ _includes_deps_sha1: bytes # SHA1 数组（每个 17 字节，MAX_INCLUDES 个 _includes_deps_str: bytes # 依赖字符串数组（每个 512 字节，MAX_INCLUDES 个） _includes_deps_count: int # 条目数 +# BuildCombinedIR 全目录扫描缓存：Phase B 对每个源文件调用一次 BuildCombinedIR， +# 每次 _collect_stub_sha1s 递归扫描 temp_dir 全部切片子目录（O(N×S) 目录枚举， +# 90 文件 × 全目录是 Phase B 慢的根因之一）。按 temp_dir 缓存扫描结果， +# 首次扫描后复用，temp_dir 变化时自动重建。 +_stub_scan_cache_dir: str # 缓存对应的 temp_dir（None=未初始化） +_stub_scan_cache_arr: str # 扫描结果数组（每 17 字节一个 SHA1，MAX_INCLUDES_SHA1 个） +_stub_scan_cache_count: int # 收集数量 + # 单条依赖字符串最大长度 MAX_DEPS_STR_LEN: t.CDefine = 512 diff --git a/App/lib/core/StubMergerBuild.vp b/App/lib/core/StubMergerBuild.vp index 6eeca01..264bbd1 100644 --- a/App/lib/core/StubMergerBuild.vp +++ b/App/lib/core/StubMergerBuild.vp @@ -833,51 +833,79 @@ def BuildCombinedIR(temp_dir: str, local_sha1: str, VLogger.error("依赖文件 deps.txt 缺失，无法合并 includes stub", "StubMerger") # 2c. 递归扫描 temp_dir 中所有 .stub.ll，按需加载 - all_sha1_arr: str = stdlib.malloc(MAX_INCLUDES_SHA1 * 17) - all_count: int = 0 - if all_sha1_arr is not None: - string.memset(all_sha1_arr, 0, MAX_INCLUDES_SHA1 * 17) - all_count = StubMergerCollect._collect_stub_sha1s(temp_dir, td_len, Config.Sha1SliceLevel, all_sha1_arr, 0, MAX_INCLUDES_SHA1) - if all_count > 0: - dep_buf: bytes = stdlib.malloc(STUB_READ_BUF_SIZE) - if dep_buf is not None: - si: int = 0 - while si < all_count: - dep_sha1: str = all_sha1_arr + t.CSizeT(si) * 17 - if string.strcmp(dep_sha1, local_sha1) == 0: - si += 1 - continue - should_load: int = 0 - # 检查是否在 includes 映射中 - is_include: int = 0 - if inc_count > 0: - for ii in range(inc_count): - idx_ii: t.CSizeT = t.CSizeT(ii) * 17 - if string.strcmp(sha1_arr + idx_ii, dep_sha1) != 0: - continue - is_include = 1 - if deps_loaded != 0: - idx_mi: t.CSizeT = t.CSizeT(ii) * 64 - dep_mod_name: str = mod_arr + idx_mi - if HandlesImports.is_module_imported(deps_buf, dep_mod_name) != 0: - should_load = 1 - elif StubMergerIR._is_submodule_of_imported(deps_buf, dep_mod_name) != 0: - # 传递性依赖：deps.txt 只记录直接导入（如 ast）， - # 但包的 __init__.vp 内部 from .lexer import Lexer - # 会让 IR 引用 ast.lexer 子模块类型。 - # 检查 dep_mod_name 是否是已导入包的子模块 - # （如 ast.lexer 是 ast 的子模块） - should_load = 1 - break - if is_include == 0: - # 非 includes stub（用户文件），总是加载 - should_load = 1 - if should_load != 0: - out_pos = _LoadAndAppendStub(temp_dir, td_len, dep_sha1, dep_buf, out_buf, out_size, out_pos) - out_pos = _LoadAndAppendTextDeclares(temp_dir, td_len, dep_sha1, dep_buf, out_buf, out_size, out_pos) + # 全目录扫描结果按 temp_dir 缓存（Phase B 对每个源文件调用一次 + # BuildCombinedIR，每次递归扫盘 O(N×S) 是 Phase B 慢的根因之一； + # 首次扫描后复用缓存，temp_dir 变化时自动重建） + cache_valid_sc: int = 0 + if StubMerger._stub_scan_cache_arr is not None and StubMerger._stub_scan_cache_dir is not None: + if string.strcmp(StubMerger._stub_scan_cache_dir, temp_dir) == 0: + cache_valid_sc = 1 + if cache_valid_sc == 0: + new_scan_arr: str = stdlib.malloc(MAX_INCLUDES_SHA1 * 17) + if new_scan_arr is not None: + string.memset(new_scan_arr, 0, MAX_INCLUDES_SHA1 * 17) + new_scan_count: int = StubMergerCollect._collect_stub_sha1s( + temp_dir, td_len, Config.Sha1SliceLevel, new_scan_arr, 0, MAX_INCLUDES_SHA1) + if new_scan_count > 0: + # 缓存目录键（stdlib 深拷贝，跨调用存活） + dir_buf_sc: str = stdlib.malloc(td_len + 1) + if dir_buf_sc is not None: + string.strcpy(dir_buf_sc, temp_dir) + if StubMerger._stub_scan_cache_arr is not None: + stdlib.free(StubMerger._stub_scan_cache_arr) + StubMerger._stub_scan_cache_arr = None + if StubMerger._stub_scan_cache_dir is not None: + stdlib.free(StubMerger._stub_scan_cache_dir) + StubMerger._stub_scan_cache_dir = None + StubMerger._stub_scan_cache_arr = new_scan_arr + StubMerger._stub_scan_cache_dir = dir_buf_sc + StubMerger._stub_scan_cache_count = new_scan_count + else: + stdlib.free(new_scan_arr) + else: + stdlib.free(new_scan_arr) + all_sha1_arr: str = StubMerger._stub_scan_cache_arr + all_count: int = StubMerger._stub_scan_cache_count + if all_sha1_arr is not None and all_count > 0: + dep_buf: bytes = stdlib.malloc(STUB_READ_BUF_SIZE) + if dep_buf is not None: + si: int = 0 + while si < all_count: + dep_sha1: str = all_sha1_arr + t.CSizeT(si) * 17 + if string.strcmp(dep_sha1, local_sha1) == 0: si += 1 - stdlib.free(dep_buf) - stdlib.free(all_sha1_arr) + continue + should_load: int = 0 + # 检查是否在 includes 映射中 + is_include: int = 0 + if inc_count > 0: + for ii in range(inc_count): + idx_ii: t.CSizeT = t.CSizeT(ii) * 17 + if string.strcmp(sha1_arr + idx_ii, dep_sha1) != 0: + continue + is_include = 1 + if deps_loaded != 0: + idx_mi: t.CSizeT = t.CSizeT(ii) * 64 + dep_mod_name: str = mod_arr + idx_mi + if HandlesImports.is_module_imported(deps_buf, dep_mod_name) != 0: + should_load = 1 + elif StubMergerIR._is_submodule_of_imported(deps_buf, dep_mod_name) != 0: + # 传递性依赖：deps.txt 只记录直接导入（如 ast）， + # 但包的 __init__.vp 内部 from .lexer import Lexer + # 会让 IR 引用 ast.lexer 子模块类型。 + # 检查 dep_mod_name 是否是已导入包的子模块 + # （如 ast.lexer 是 ast 的子模块） + should_load = 1 + break + if is_include == 0: + # 非 includes stub（用户文件），总是加载 + should_load = 1 + if should_load != 0: + out_pos = _LoadAndAppendStub(temp_dir, td_len, dep_sha1, dep_buf, out_buf, out_size, out_pos) + out_pos = _LoadAndAppendTextDeclares(temp_dir, td_len, dep_sha1, dep_buf, out_buf, out_size, out_pos) + si += 1 + stdlib.free(dep_buf) + # all_sha1_arr 是模块级缓存（StubMerger._stub_scan_cache_arr），跨调用共享，不释放 # 2d. 验证 deps.txt 中所有依赖模块的 stub 文件都存在（fail-fast） # 避免到 llc 才报 undefined value 错误。