perf(phase2): 按 PROF 热点优化编译耗时
- HandlesImports: _TryLoadClassMembersFromPyi 复用 _pyi_cache 跳过重复 isfile（-12.8s）；_LookupStubFuncType 类级缓存 + 签名增量扫描新增 stub，去掉每文件 isfile（-6.5s+） - Phase2Translator: pyi cache 仅主进程构建（worker 只读 pkl，-23s+pickle 10.4s）；include 源文件 sha1/存在性缓存消除 _build_current_ModuleSha1Map 重复磁盘 I/O - StructGen.finalize: 行过滤改 startswith(tuple) 预计算，消除 O(lines×structs×patterns) 嵌套循环（tottime 84.8s → 预期数秒） - 附带 fail-fast（任意文件出错立即终止）+ 子进程 PROF 硬编码
This commit is contained in:
TermiNexus committed
1 parent
1808e90fd3
commit
fbd132284b
3 files changed
+140
-66
No files matched your search
@@ -38,6 +38,13 @@ class ImportHandle(BaseHandle):
|
||||
# 保证进程内唯一），避免 _TryLoadClassMembersFromPyi 跨任务反复线性扫描 body。
|
||||
_extern_class_classify_cache: dict[tuple[str, str], tuple] = {}
|
||||
_extern_class_decl_cache: dict[tuple[str, str], ast.ClassDef] = {}
|
||||
# 类级 stub 函数签名缓存（_LookupStubFuncType）：跨 worker 任务共享。
|
||||
# 与 _stub_index/_stub_index_scanned 同模式：签名变化只增量解析新增 stub
|
||||
# 文件（编译进程内 stub 只增不改），避免每次任务全量重扫 + 每文件
|
||||
# os.path.isfile（Windows stat ~0.5ms/次，原 5840 次 ≈ 6.5s）。
|
||||
_stub_func_cache: dict[str, tuple[str, str, bool, str]] = {}
|
||||
_stub_func_cache_signature: tuple[int, int] | None = None
|
||||
_stub_func_scanned: set[str] = set()
|
||||
|
||||
def __init__(self, translator: "Translator") -> None:
|
||||
super().__init__(translator)
|
||||
@@ -2290,12 +2297,11 @@ class ImportHandle(BaseHandle):
|
||||
try:
|
||||
_pyi_b0: str = stub_filename.replace('.stub.ll', '.vpi')
|
||||
_path0: str = os.path.join(temp_dir, _pyi_b0)
|
||||
if os.path.isfile(_path0):
|
||||
_tree0: ast.AST | None = getattr(self, '_pyi_cache', {}).get(_path0)
|
||||
if _tree0 is None:
|
||||
with open(_path0, 'r', encoding='utf-8') as _fh0:
|
||||
_tree0 = ast.parse(_fh0.read())
|
||||
getattr(self, '_pyi_cache', {})[_path0] = _tree0
|
||||
_tree0: ast.AST | None = getattr(self, '_pyi_cache', {}).get(_path0)
|
||||
if _tree0 is None and os.path.isfile(_path0):
|
||||
with open(_path0, 'r', encoding='utf-8') as _fh0:
|
||||
_tree0 = ast.parse(_fh0.read())
|
||||
getattr(self, '_pyi_cache', {})[_path0] = _tree0
|
||||
if _tree0 is not None:
|
||||
for _n0 in _tree0.body:
|
||||
if isinstance(_n0, ast.ClassDef) and _n0.name == class_name:
|
||||
@@ -2314,7 +2320,10 @@ class ImportHandle(BaseHandle):
|
||||
return False
|
||||
pyi_basename = stub_filename.replace('.stub.ll', '.vpi')
|
||||
pyi_path = os.path.join(temp_dir, pyi_basename)
|
||||
if not os.path.isfile(pyi_path):
|
||||
# 已探测过的路径（_pyi_cache 含键，值为树或 None=不存在）不再重复
|
||||
# os.path.isfile（Windows stat ~0.5ms/次；本函数 28543 次调用 × 探测
|
||||
# ≈ 12.8s，占 _path_isfile 总耗时 63%）。
|
||||
if pyi_path not in self._pyi_cache and not os.path.isfile(pyi_path):
|
||||
# 回退读取旧缓存 .pyi
|
||||
pyi_basename = stub_filename.replace('.stub.ll', '.pyi')
|
||||
pyi_path = os.path.join(temp_dir, pyi_basename)
|
||||
@@ -2399,29 +2408,23 @@ class ImportHandle(BaseHandle):
|
||||
将无法被解析，导致跨模块调用使用默认 i32 返回类型 → 64 位指针截断。
|
||||
"""
|
||||
|
||||
if not getattr(self, '_stub_func_cache', None):
|
||||
self._stub_func_cache = {}
|
||||
|
||||
ProjectRoot = self._find_project_root()
|
||||
temp_dir = os.path.join(ProjectRoot, 'temp')
|
||||
output_dir = os.path.join(ProjectRoot, 'output')
|
||||
|
||||
# 文件数签名：检测 stub 新增（增量编译时 output 逐步填充完整签名）
|
||||
output_count = sum(1 for f in os.listdir(output_dir) if f.endswith('.stub.ll')) if os.path.isdir(output_dir) else 0
|
||||
temp_count = sum(1 for f in os.listdir(temp_dir) if f.endswith('.stub.ll')) if os.path.isdir(temp_dir) else 0
|
||||
signature = (output_count, temp_count)
|
||||
if getattr(self, '_stub_func_cache_signature', None) != signature or not self._stub_func_cache:
|
||||
# 签名变化或缓存为空，重建缓存
|
||||
self._stub_func_cache = {}
|
||||
self._stub_func_cache_signature = signature
|
||||
|
||||
if not os.path.isdir(temp_dir) and not os.path.isdir(output_dir):
|
||||
return None
|
||||
|
||||
if func_name not in self._stub_func_cache:
|
||||
# 先扫描 temp/（Phase 1，不完整签名），再扫描 output/（Phase 2，完整签名）覆盖之
|
||||
# 这样 output 中存在的继承包装方法会覆盖 temp 中的同名旧声明（若有），
|
||||
# 也填补 temp 中缺失的函数声明
|
||||
# 文件数签名：检测 stub 新增（增量编译时 output 逐步填充完整签名）。
|
||||
# 类级缓存（ImportHandle._stub_func_cache）跨 worker 任务共享；签名变化
|
||||
# 只增量解析新增 stub 文件（编译进程内 stub 只增不改，旧文件不重扫，
|
||||
# 省略每文件 os.path.isfile ——Windows stat ~0.5ms/次，原 5840 次 ≈ 6.5s）。
|
||||
# 覆盖语义不变：先扫 temp/ 后扫 output/，output 同名声明覆盖 temp。
|
||||
output_count = sum(1 for f in os.listdir(output_dir) if f.endswith('.stub.ll')) if os.path.isdir(output_dir) else 0
|
||||
temp_count = sum(1 for f in os.listdir(temp_dir) if f.endswith('.stub.ll')) if os.path.isdir(temp_dir) else 0
|
||||
signature = (output_count, temp_count)
|
||||
if ImportHandle._stub_func_cache_signature != signature:
|
||||
ImportHandle._stub_func_cache_signature = signature
|
||||
for scan_dir in [temp_dir, output_dir]:
|
||||
if not os.path.isdir(scan_dir):
|
||||
continue
|
||||
@@ -2430,9 +2433,9 @@ class ImportHandle(BaseHandle):
|
||||
continue
|
||||
|
||||
stub_path = os.path.join(scan_dir, filename)
|
||||
# 预检查文件存在性：os.listdir 与 open 之间文件可能被并发写入删除
|
||||
if not os.path.isfile(stub_path):
|
||||
if stub_path in ImportHandle._stub_func_scanned:
|
||||
continue
|
||||
ImportHandle._stub_func_scanned.add(stub_path)
|
||||
source_sha1 = filename.replace('.stub.ll', '')
|
||||
try:
|
||||
with open(stub_path, 'r', encoding='utf-8') as f:
|
||||
@@ -2453,8 +2456,9 @@ class ImportHandle(BaseHandle):
|
||||
if match:
|
||||
stub_ret = match.group(1).strip()
|
||||
stub_params = match.group(3).strip()
|
||||
self._stub_func_cache[stub_func_name] = (stub_ret, stub_params, '...' in stub_params, source_sha1)
|
||||
ImportHandle._stub_func_cache[stub_func_name] = (stub_ret, stub_params, '...' in stub_params, source_sha1)
|
||||
except FileNotFoundError:
|
||||
# listdir 与 open 之间文件被并发删除：跳过（等效旧 isfile 预检查）
|
||||
continue
|
||||
except Exception as _e:
|
||||
if _config_mode == "strict":
|
||||
|
||||
@@ -170,23 +170,24 @@ class StructGenMixin:
|
||||
struct_names.add(sname)
|
||||
if struct_defs:
|
||||
lines: list[str] = ir_text.split('\n')
|
||||
# 预计算全部 "= type" 定义行前缀（一次性），用 str.startswith(tuple)
|
||||
# 在 C 层批量比较，替代原 O(lines × structs × patterns) 的 Python 嵌套
|
||||
# 循环（PROF: finalize tottime 84.8s + startswith 4114 万次 46.9s）。
|
||||
name_prefixes: list[str] = []
|
||||
for name in struct_names:
|
||||
if name[0].isdigit() or '.' in name or not name.replace('_', '').replace('.', '').isalnum():
|
||||
name_prefixes.append(f'%"{name}" = type')
|
||||
name_prefixes.append(f'%"{name}"=type')
|
||||
name_prefixes.append(f'%{name} = type')
|
||||
name_prefixes.append(f'%{name}=type')
|
||||
else:
|
||||
name_prefixes.append(f'%{name} = type')
|
||||
name_prefixes.append(f'%{name}= type')
|
||||
name_prefixes.append(f'%{name}=type')
|
||||
prefixes_tuple: tuple[str, ...] = tuple(name_prefixes)
|
||||
filtered: list[str] = []
|
||||
for line in lines:
|
||||
stripped: str = line.strip()
|
||||
skip: bool = False
|
||||
for name in struct_names:
|
||||
patterns: list[str]
|
||||
if name[0].isdigit() or '.' in name or not name.replace('_', '').replace('.', '').isalnum():
|
||||
patterns = [f'%"{name}" = type', f'%"{name}"=type', f'%{name} = type', f'%{name}=type']
|
||||
else:
|
||||
patterns = [f'%{name} = type', f'%{name}= type', f'%{name}=type']
|
||||
for p in patterns:
|
||||
if stripped.startswith(p):
|
||||
skip = True
|
||||
break
|
||||
if skip:
|
||||
break
|
||||
if not skip:
|
||||
if not line.strip().startswith(prefixes_tuple):
|
||||
filtered.append(line)
|
||||
ir_text = '\n'.join(filtered)
|
||||
header_end: int = ir_text.find('\ndefine')
|
||||
|
||||
Reference in new issue
Block a user