perf(compile): TPC 编译提速——并行 worker 满核 + vast 解析器热点优化

cProfile 分析(206s 全量重编译 TPV):~163s 为 ProcessPool 等待(16 核机器仅用 8 worker),vast.parse 35.9s(1231 次)为最大 Python 计算热点。

- Phase2Translator:4 处 n_workers 上限 8 → os.cpu_count()(16 核满负荷),并行翻译/llc 编译利用率翻倍
- vast/Tokenizer._PosInfo:isascii 快速路径 + 行索引增量查找(~300 万次调用,免每次 bisect+encode)
- vast/Tokenizer._EmitOp:运算符按首字符分组,58 万次调用从 42 项全表 startswith 降到 2-3 项
- vast/Tokenizer._ScanName:内联 _IsIdentPart(70 万次循环免函数调用)
- Projectrans/Utils:find_reachable 用 deque + 后缀索引替代 pop(0)/全表扫描;topological_sort 用反向邻接表 + heapq 替代每轮 sort+全图扫描
This commit is contained in:
TermiNexus committed 2026-08-17 18:56:55 +08:00
1 parent 5d93692baa
commit 58ce673f01
4 files changed
+101 -44

No files matched your search

+54 -12
View File
@@ -42,6 +42,17 @@ _OPERATORS: tuple[str, ...] = (
'@', '&', '|', '^', '~', '<', '>', '=', '!',
)
# 按首字符分组的运算符表(组内保持最长优先顺序)。_EmitOp 每 token 只遍历
# 同首字符的少数候选(平均 2-3 个),替代原 42 项全表 startswith 扫描
# (cProfile 热点:58 万次调用 tottime 3.6s,首字符分组后显著下降)。
_OP_BY_FIRST: dict[str, tuple[str, ...]] = {}
for _op in _OPERATORS:
_first: str = _op[0]
if _first in _OP_BY_FIRST:
_OP_BY_FIRST[_first] = _OP_BY_FIRST[_first] + (_op,)
else:
_OP_BY_FIRST[_first] = (_op,)
# 行首空白 + 关键字 + 空白 + 标识符 → 软关键字函数定义(Parser 层处理)
SOFT_KEYWORDS: frozenset[str] = frozenset({'fn', 'fun', 'func', 'function', 'extern'})
@@ -120,15 +131,36 @@ class Tokenizer:
self.Comments: list[Token] = []
# 行起始字符索引表(索引 0 对应第 1 行),用于行列换算
self._LineStarts: list[int] = [0]
# _PosInfo 增量缓存:token 顺序创建时 pos 单调递增,免 bisect
self._LastLineIdx: int = 0
self._LastPos: int = -1
# ------------------------------------------------------------------
# 位置换算
# ------------------------------------------------------------------
def _PosInfo(self, pos: int) -> tuple[int, int]:
"""将字符索引换算为 (1-based 行号, UTF-8 字节列偏移)。"""
idx: int = bisect.bisect_right(self._LineStarts, pos) - 1
"""将字符索引换算为 (1-based 行号, UTF-8 字节列偏移)。
优化(cProfile 热点:~300 万次调用):token 顺序创建时 pos 单调
递增,用上次行索引增量顺延替代 bisect(O(log n)→摊销 O(1));
列偏移对 ASCII 行用 len() 直接量取,避免每次 encode 分配字节串。
"""
ls: list[int] = self._LineStarts
if pos >= self._LastPos and self._LastLineIdx + 1 < len(ls):
idx: int = self._LastLineIdx
nls: int = len(ls)
while idx + 1 < nls and ls[idx + 1] <= pos:
idx += 1
else:
idx = bisect.bisect_right(ls, pos) - 1
self._LastLineIdx = idx
self._LastPos = pos
line: int = idx + 1
col_bytes: int = len(self.Source[self._LineStarts[idx]:pos].encode('utf-8'))
seg: str = self.Source[ls[idx]:pos]
if seg.isascii():
col_bytes: int = len(seg)
else:
col_bytes = len(seg.encode('utf-8'))
return line, col_bytes
def _Make(self, tok_type: str, start: int, end: int) -> Token:
@@ -283,22 +315,32 @@ class Tokenizer:
def _EmitOp(self) -> None:
"""运算符最长匹配(含 ... 与增强赋值运算符)。"""
start: int = self.Pos
for op in _OPERATORS:
if self.Source.startswith(op, self.Pos):
self.Pos += len(op)
self.Tokens.append(self._Make(OP, start, self.Pos))
self.AtLineStart = False
return
ops: tuple[str, ...] | None = _OP_BY_FIRST.get(self.Source[self.Pos])
if ops is not None:
for op in ops:
if self.Source.startswith(op, self.Pos):
self.Pos += len(op)
self.Tokens.append(self._Make(OP, start, self.Pos))
self.AtLineStart = False
return
# 未知字符:单字符 OP 兜底
self.Pos += 1
self.Tokens.append(self._Make(OP, start, self.Pos))
self.AtLineStart = False
def _ScanName(self) -> None:
"""扫描标识符。"""
"""扫描标识符(内联 _IsIdentPart 避免 70 万次函数调用开销)。"""
start: int = self.Pos
while self.Pos < self.Length and _IsIdentPart(self.Source[self.Pos]):
self.Pos += 1
src: str = self.Source
n: int = self.Length
pos: int = self.Pos
while pos < n:
ch: str = src[pos]
if ch == '_' or ch.isalnum() or ord(ch) > 127:
pos += 1
else:
break
self.Pos = pos
self.Tokens.append(self._Make(NAME, start, self.Pos))
self.AtLineStart = False