perf(compile): TPC 编译提速——并行 worker 满核 + vast 解析器热点优化

cProfile 分析(206s 全量重编译 TPV):~163s 为 ProcessPool 等待(16 核机器仅用 8 worker),vast.parse 35.9s(1231 次)为最大 Python 计算热点。

- Phase2Translator:4 处 n_workers 上限 8 → os.cpu_count()(16 核满负荷),并行翻译/llc 编译利用率翻倍
- vast/Tokenizer._PosInfo:isascii 快速路径 + 行索引增量查找(~300 万次调用,免每次 bisect+encode)
- vast/Tokenizer._EmitOp:运算符按首字符分组,58 万次调用从 42 项全表 startswith 降到 2-3 项
- vast/Tokenizer._ScanName:内联 _IsIdentPart(70 万次循环免函数调用)
- Projectrans/Utils:find_reachable 用 deque + 后缀索引替代 pop(0)/全表扫描;topological_sort 用反向邻接表 + heapq 替代每轮 sort+全图扫描
This commit is contained in:
TermiNexus committed 2026-08-17 18:56:55 +08:00
1 parent 5d93692baa
commit 58ce673f01
4 files changed
+101 -44

No files matched your search

+40 -26
View File
@@ -2,6 +2,8 @@ from __future__ import annotations
import hashlib
import os
from collections import deque
from typing import Deque
import vast as ast
from lib.core.VLogger import get_logger as _vlog
@@ -150,33 +152,37 @@ def find_reachable_files_from_entries(src_root: str, entry_files: list[str]) ->
if top_module not in all_files:
all_files[top_module] = filepath
# 后缀索引:pkg.sub.mod → 注册 "pkg.sub.mod" / "sub.mod" / "mod",
# 使 from-import 的短依赖名(dep)O(1) 定位目标文件,替代原
# for key,val in all_files.items() 全表线性扫描(O(n²),Phase1 全量
# 重编译可达文件多时显著拖慢)。
_suffix_index: dict[str, str] = {}
for _key, _val in all_files.items():
_parts: list[str] = _key.split('.')
for _i in range(len(_parts)):
_suf: str = '.'.join(_parts[_i:])
if _suf not in _suffix_index:
_suffix_index[_suf] = _val
reachable: set[str] = set()
queue: list[str] = list(entry_files)
# BFS 队列用 deque:pop(0) 是 O(n) 出队(每次移动整表),
# popleft 为 O(1),可达文件数百时累计节省显著。
queue_dq: Deque[str] = deque(entry_files)
# BFS trace: record each file processed and unresolved deps
_bfs_trace: list[str] = []
_unresolved_deps: list[tuple[str, str]] = [] # (current_file, dep)
while queue:
current: str = queue.pop(0)
while queue_dq:
current: str = queue_dq.popleft()
if current in reachable:
continue
reachable.add(current)
deps: set[str] = get_file_dependencies(current, src_root)
_bfs_trace.append(f" {os.path.basename(current)} deps={len(deps)}")
for dep in deps:
if dep in all_files and all_files[dep] not in reachable:
queue.append(all_files[dep])
queue_dq.append(all_files[dep])
else:
_matched = False
for key, val in all_files.items():
if key.endswith('.' + dep) or key == dep:
if val not in reachable:
queue.append(val)
_matched = True
break
if not _matched:
_unresolved_deps.append((os.path.basename(current), dep))
_target: str | None = _suffix_index.get(dep)
if _target is not None and _target not in reachable:
queue_dq.append(_target)
return reachable
@@ -204,6 +210,14 @@ def topological_sort_files(py_files: list[str], src_root: str) -> list[str]:
if dep_file != filepath: # 避免自依赖
graph[filepath].add(dep_file)
# 反向邻接表:依赖当前文件的文件列表。Kahn 出队时只需遍历 reverse[cur],
# 替代原"每出队一个节点就全图扫描找依赖者"的 O(V²)(文件多时显著拖慢)。
reverse: dict[str, list[str]] = {f: [] for f in py_files}
for f, deps in graph.items():
for dep in deps:
if dep in reverse:
reverse[dep].append(f)
# 拓扑排序 (Kahn's algorithm)
in_degree: dict[str, int] = {f: 0 for f in py_files}
for f, deps in graph.items():
@@ -211,20 +225,20 @@ def topological_sort_files(py_files: list[str], src_root: str) -> list[str]:
if dep in in_degree:
in_degree[f] += 1
# 小根堆替代"每轮 queue.sort() + pop(0)"(O(n² log n)):heappop 取最小
# 保证同级别文件按字母序处理(确定性),单次 O(log n)。
import heapq
queue: list[str] = [f for f, degree in in_degree.items() if degree == 0]
heapq.heapify(queue)
sorted_files: list[str] = []
while queue:
# 按字母顺序处理同级别的文件,确保确定性
queue.sort()
current: str = queue.pop(0)
current: str = heapq.heappop(queue)
sorted_files.append(current)
for f, deps in graph.items():
if current in deps:
in_degree[f] -= 1
if in_degree[f] == 0:
queue.append(f)
for f in reverse[current]:
in_degree[f] -= 1
if in_degree[f] == 0:
heapq.heappush(queue, f)
# 如果有环,添加剩余文件
if len(sorted_files) < len(py_files):