perf(compile): TPC 编译提速——并行 worker 满核 + vast 解析器热点优化
cProfile 分析(206s 全量重编译 TPV):~163s 为 ProcessPool 等待(16 核机器仅用 8 worker),vast.parse 35.9s(1231 次)为最大 Python 计算热点。 - Phase2Translator:4 处 n_workers 上限 8 → os.cpu_count()(16 核满负荷),并行翻译/llc 编译利用率翻倍 - vast/Tokenizer._PosInfo:isascii 快速路径 + 行索引增量查找(~300 万次调用,免每次 bisect+encode) - vast/Tokenizer._EmitOp:运算符按首字符分组,58 万次调用从 42 项全表 startswith 降到 2-3 项 - vast/Tokenizer._ScanName:内联 _IsIdentPart(70 万次循环免函数调用) - Projectrans/Utils:find_reachable 用 deque + 后缀索引替代 pop(0)/全表扫描;topological_sort 用反向邻接表 + heapq 替代每轮 sort+全图扫描
This commit is contained in:
1 parent
5d93692baa
commit
58ce673f01
4 files changed
+101
-44
No files matched your search
+40
-26
@@ -2,6 +2,8 @@ from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
from collections import deque
|
||||
from typing import Deque
|
||||
import vast as ast
|
||||
|
||||
from lib.core.VLogger import get_logger as _vlog
|
||||
@@ -150,33 +152,37 @@ def find_reachable_files_from_entries(src_root: str, entry_files: list[str]) ->
|
||||
if top_module not in all_files:
|
||||
all_files[top_module] = filepath
|
||||
|
||||
# 后缀索引:pkg.sub.mod → 注册 "pkg.sub.mod" / "sub.mod" / "mod",
|
||||
# 使 from-import 的短依赖名(dep)O(1) 定位目标文件,替代原
|
||||
# for key,val in all_files.items() 全表线性扫描(O(n²),Phase1 全量
|
||||
# 重编译可达文件多时显著拖慢)。
|
||||
_suffix_index: dict[str, str] = {}
|
||||
for _key, _val in all_files.items():
|
||||
_parts: list[str] = _key.split('.')
|
||||
for _i in range(len(_parts)):
|
||||
_suf: str = '.'.join(_parts[_i:])
|
||||
if _suf not in _suffix_index:
|
||||
_suffix_index[_suf] = _val
|
||||
|
||||
reachable: set[str] = set()
|
||||
queue: list[str] = list(entry_files)
|
||||
# BFS 队列用 deque:pop(0) 是 O(n) 出队(每次移动整表),
|
||||
# popleft 为 O(1),可达文件数百时累计节省显著。
|
||||
queue_dq: Deque[str] = deque(entry_files)
|
||||
|
||||
# BFS trace: record each file processed and unresolved deps
|
||||
_bfs_trace: list[str] = []
|
||||
_unresolved_deps: list[tuple[str, str]] = [] # (current_file, dep)
|
||||
|
||||
while queue:
|
||||
current: str = queue.pop(0)
|
||||
while queue_dq:
|
||||
current: str = queue_dq.popleft()
|
||||
if current in reachable:
|
||||
continue
|
||||
reachable.add(current)
|
||||
deps: set[str] = get_file_dependencies(current, src_root)
|
||||
_bfs_trace.append(f" {os.path.basename(current)} deps={len(deps)}")
|
||||
for dep in deps:
|
||||
if dep in all_files and all_files[dep] not in reachable:
|
||||
queue.append(all_files[dep])
|
||||
queue_dq.append(all_files[dep])
|
||||
else:
|
||||
_matched = False
|
||||
for key, val in all_files.items():
|
||||
if key.endswith('.' + dep) or key == dep:
|
||||
if val not in reachable:
|
||||
queue.append(val)
|
||||
_matched = True
|
||||
break
|
||||
if not _matched:
|
||||
_unresolved_deps.append((os.path.basename(current), dep))
|
||||
_target: str | None = _suffix_index.get(dep)
|
||||
if _target is not None and _target not in reachable:
|
||||
queue_dq.append(_target)
|
||||
|
||||
return reachable
|
||||
|
||||
@@ -204,6 +210,14 @@ def topological_sort_files(py_files: list[str], src_root: str) -> list[str]:
|
||||
if dep_file != filepath: # 避免自依赖
|
||||
graph[filepath].add(dep_file)
|
||||
|
||||
# 反向邻接表:依赖当前文件的文件列表。Kahn 出队时只需遍历 reverse[cur],
|
||||
# 替代原"每出队一个节点就全图扫描找依赖者"的 O(V²)(文件多时显著拖慢)。
|
||||
reverse: dict[str, list[str]] = {f: [] for f in py_files}
|
||||
for f, deps in graph.items():
|
||||
for dep in deps:
|
||||
if dep in reverse:
|
||||
reverse[dep].append(f)
|
||||
|
||||
# 拓扑排序 (Kahn's algorithm)
|
||||
in_degree: dict[str, int] = {f: 0 for f in py_files}
|
||||
for f, deps in graph.items():
|
||||
@@ -211,20 +225,20 @@ def topological_sort_files(py_files: list[str], src_root: str) -> list[str]:
|
||||
if dep in in_degree:
|
||||
in_degree[f] += 1
|
||||
|
||||
# 小根堆替代"每轮 queue.sort() + pop(0)"(O(n² log n)):heappop 取最小
|
||||
# 保证同级别文件按字母序处理(确定性),单次 O(log n)。
|
||||
import heapq
|
||||
queue: list[str] = [f for f, degree in in_degree.items() if degree == 0]
|
||||
heapq.heapify(queue)
|
||||
sorted_files: list[str] = []
|
||||
|
||||
while queue:
|
||||
# 按字母顺序处理同级别的文件,确保确定性
|
||||
queue.sort()
|
||||
current: str = queue.pop(0)
|
||||
current: str = heapq.heappop(queue)
|
||||
sorted_files.append(current)
|
||||
|
||||
for f, deps in graph.items():
|
||||
if current in deps:
|
||||
in_degree[f] -= 1
|
||||
if in_degree[f] == 0:
|
||||
queue.append(f)
|
||||
for f in reverse[current]:
|
||||
in_degree[f] -= 1
|
||||
if in_degree[f] == 0:
|
||||
heapq.heappush(queue, f)
|
||||
|
||||
# 如果有环,添加剩余文件
|
||||
if len(sorted_files) < len(py_files):
|
||||
|
||||
Reference in new issue
Block a user