perf(phase2): 缓存 BuildCombinedIR 全目录扫描结果
Phase B 编译 .obj 慢得像同步编译的根因:BuildCombinedIR 对每个源文件 调用一次 _collect_stub_sha1s 递归扫描 temp_dir 全部切片子目录(90 文件 × 全目录枚举 O(N×S)),且目录扫描与 llc 并行无关(llc 并行已生效)。 按 temp_dir 缓存扫描结果(StubMerger._stub_scan_cache_*,stdlib 分配 跨 reset 存活),首次扫描后复用,temp_dir 变化时自动重建。BuildCombinedIR 的所有调用点(Phase B 主循环/Phase B+ includes 编译/stubgen)统一受益。
This commit is contained in:
1 parent
efc0e47058
commit
b08a0aff48
2 files changed
+43
-7
No files matched your search
@@ -63,6 +63,14 @@ _includes_deps_sha1: bytes # SHA1 数组(每个 17 字节,MAX_INCLUDES 个
|
||||
_includes_deps_str: bytes # 依赖字符串数组(每个 512 字节,MAX_INCLUDES 个)
|
||||
_includes_deps_count: int # 条目数
|
||||
|
||||
# BuildCombinedIR 全目录扫描缓存:Phase B 对每个源文件调用一次 BuildCombinedIR,
|
||||
# 每次 _collect_stub_sha1s 递归扫描 temp_dir 全部切片子目录(O(N×S) 目录枚举,
|
||||
# 90 文件 × 全目录是 Phase B 慢的根因之一)。按 temp_dir 缓存扫描结果,
|
||||
# 首次扫描后复用,temp_dir 变化时自动重建。
|
||||
_stub_scan_cache_dir: str # 缓存对应的 temp_dir(None=未初始化)
|
||||
_stub_scan_cache_arr: str # 扫描结果数组(每 17 字节一个 SHA1,MAX_INCLUDES_SHA1 个)
|
||||
_stub_scan_cache_count: int # 收集数量
|
||||
|
||||
# 单条依赖字符串最大长度
|
||||
MAX_DEPS_STR_LEN: t.CDefine = 512
|
||||
|
||||
|
||||
@@ -833,12 +833,40 @@ def BuildCombinedIR(temp_dir: str, local_sha1: str,
|
||||
VLogger.error("依赖文件 deps.txt 缺失,无法合并 includes stub", "StubMerger")
|
||||
|
||||
# 2c. 递归扫描 temp_dir 中所有 .stub.ll,按需加载
|
||||
all_sha1_arr: str = stdlib.malloc(MAX_INCLUDES_SHA1 * 17)
|
||||
all_count: int = 0
|
||||
if all_sha1_arr is not None:
|
||||
string.memset(all_sha1_arr, 0, MAX_INCLUDES_SHA1 * 17)
|
||||
all_count = StubMergerCollect._collect_stub_sha1s(temp_dir, td_len, Config.Sha1SliceLevel, all_sha1_arr, 0, MAX_INCLUDES_SHA1)
|
||||
if all_count > 0:
|
||||
# 全目录扫描结果按 temp_dir 缓存(Phase B 对每个源文件调用一次
|
||||
# BuildCombinedIR,每次递归扫盘 O(N×S) 是 Phase B 慢的根因之一;
|
||||
# 首次扫描后复用缓存,temp_dir 变化时自动重建)
|
||||
cache_valid_sc: int = 0
|
||||
if StubMerger._stub_scan_cache_arr is not None and StubMerger._stub_scan_cache_dir is not None:
|
||||
if string.strcmp(StubMerger._stub_scan_cache_dir, temp_dir) == 0:
|
||||
cache_valid_sc = 1
|
||||
if cache_valid_sc == 0:
|
||||
new_scan_arr: str = stdlib.malloc(MAX_INCLUDES_SHA1 * 17)
|
||||
if new_scan_arr is not None:
|
||||
string.memset(new_scan_arr, 0, MAX_INCLUDES_SHA1 * 17)
|
||||
new_scan_count: int = StubMergerCollect._collect_stub_sha1s(
|
||||
temp_dir, td_len, Config.Sha1SliceLevel, new_scan_arr, 0, MAX_INCLUDES_SHA1)
|
||||
if new_scan_count > 0:
|
||||
# 缓存目录键(stdlib 深拷贝,跨调用存活)
|
||||
dir_buf_sc: str = stdlib.malloc(td_len + 1)
|
||||
if dir_buf_sc is not None:
|
||||
string.strcpy(dir_buf_sc, temp_dir)
|
||||
if StubMerger._stub_scan_cache_arr is not None:
|
||||
stdlib.free(StubMerger._stub_scan_cache_arr)
|
||||
StubMerger._stub_scan_cache_arr = None
|
||||
if StubMerger._stub_scan_cache_dir is not None:
|
||||
stdlib.free(StubMerger._stub_scan_cache_dir)
|
||||
StubMerger._stub_scan_cache_dir = None
|
||||
StubMerger._stub_scan_cache_arr = new_scan_arr
|
||||
StubMerger._stub_scan_cache_dir = dir_buf_sc
|
||||
StubMerger._stub_scan_cache_count = new_scan_count
|
||||
else:
|
||||
stdlib.free(new_scan_arr)
|
||||
else:
|
||||
stdlib.free(new_scan_arr)
|
||||
all_sha1_arr: str = StubMerger._stub_scan_cache_arr
|
||||
all_count: int = StubMerger._stub_scan_cache_count
|
||||
if all_sha1_arr is not None and all_count > 0:
|
||||
dep_buf: bytes = stdlib.malloc(STUB_READ_BUF_SIZE)
|
||||
if dep_buf is not None:
|
||||
si: int = 0
|
||||
@@ -877,7 +905,7 @@ def BuildCombinedIR(temp_dir: str, local_sha1: str,
|
||||
out_pos = _LoadAndAppendTextDeclares(temp_dir, td_len, dep_sha1, dep_buf, out_buf, out_size, out_pos)
|
||||
si += 1
|
||||
stdlib.free(dep_buf)
|
||||
stdlib.free(all_sha1_arr)
|
||||
# all_sha1_arr 是模块级缓存(StubMerger._stub_scan_cache_arr),跨调用共享,不释放
|
||||
|
||||
# 2d. 验证 deps.txt 中所有依赖模块的 stub 文件都存在(fail-fast)
|
||||
# 避免到 llc 才报 undefined value 错误。
|
||||
|
||||
Reference in new issue
Block a user