GVSDS/TransPyC
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf(compile): TPC 编译提速——并行 worker 满核 + vast 解析器热点优化

Browse Source 
cProfile 分析（206s 全量重编译 TPV）：~163s 为 ProcessPool 等待（16 核机器仅用 8 worker），vast.parse 35.9s（1231 次）为最大 Python 计算热点。

- Phase2Translator：4 处 n_workers 上限 8 → os.cpu_count()（16 核满负荷），并行翻译/llc 编译利用率翻倍
- vast/Tokenizer._PosInfo：isascii 快速路径 + 行索引增量查找（~300 万次调用，免每次 bisect+encode）
- vast/Tokenizer._EmitOp：运算符按首字符分组，58 万次调用从 42 项全表 startswith 降到 2-3 项
- vast/Tokenizer._ScanName：内联 _IsIdentPart（70 万次循环免函数调用）
- Projectrans/Utils：find_reachable 用 deque + 后缀索引替代 pop(0)/全表扫描；topological_sort 用反向邻接表 + heapq 替代每轮 sort+全图扫描
This commit is contained in:
TermiNexus committed 2026-08-17 18:56:55 +08:00
1 parent 5d93692baa
commit 58ce673f01
4 files changed
+101 -44
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

lib/Projectrans/Phase1Generator.py
+3 -2
View File
@@ -179,9 +179,10 @@ class Phase1Generator:
needed: set[str] = set()
dep_graph: dict[str, set[str]] = {}
queue: list[str] = list(imported_from_src)
from collections import deque
queue: deque[str] = deque(imported_from_src)
while queue:
mod_name: str = queue.pop(0)
mod_name: str = queue.popleft()
if mod_name in needed:
continue
if mod_name in include_file_map:
lib/Projectrans/Phase2Translator.py
+4 -4
View File
@@ -417,7 +417,7 @@ class Phase2Translator:
_vlog().info(f"\n需要翻译 {len(need_translate)} 个文件")
t_start = time.time()
n_workers = min(8, len(need_translate))
n_workers = min(os.cpu_count() or 4, len(need_translate))
if n_workers > 1 and len(need_translate) > 1:
_vlog().info(f" 并行翻译 (workers={n_workers})")
@@ -1643,7 +1643,7 @@ class Phase2Translator:
need_compile.append(ll_path)
if need_compile:
n_workers = min(os.cpu_count() or 4, len(need_compile), 8)
n_workers = min(os.cpu_count() or 4, len(need_compile))
_vlog().info(f" 并行编译 {len(need_compile)} 个文件 (workers={n_workers})")
errors = []
with ThreadPoolExecutor(max_workers=n_workers) as executor:
@@ -2167,7 +2167,7 @@ class Phase2Translator:
shared_pickle_path = ''
if shared_pickle_path and os.path.exists(shared_pickle_path):
n_workers = min(8, len(need_translate))
n_workers = min(os.cpu_count() or 4, len(need_translate))
errors = []
# 波次并行翻译：按依赖关系分批提交，确保被依赖的文件先翻译完成。
# 根因：_TryLoadStructFromStub 扫描 output 目录加载跨模块 struct 定义，
@@ -2392,7 +2392,7 @@ class Phase2Translator:
return (task, result)
if len(compile_tasks) > 1:
n_workers = min(8, len(compile_tasks))
n_workers = min(os.cpu_count() or 4, len(compile_tasks))
with ThreadPoolExecutor(max_workers=n_workers) as executor:
futures = {executor.submit(_compile_one_include, task): task for task in compile_tasks}
done_count = 0
lib/Projectrans/Utils.py
+40 -26
View File
@@ -2,6 +2,8 @@ from __future__ import annotations
import hashlib
import os
from collections import deque
from typing import Deque
import vast as ast
from lib.core.VLogger import get_logger as _vlog
@@ -150,33 +152,37 @@ def find_reachable_files_from_entries(src_root: str, entry_files: list[str]) ->
if top_module not in all_files:
all_files[top_module] = filepath
# 后缀索引：pkg.sub.mod → 注册 "pkg.sub.mod" / "sub.mod" / "mod"，
# 使 from-import 的短依赖名（dep）O(1) 定位目标文件，替代原
# for key,val in all_files.items() 全表线性扫描（O(n²)，Phase1 全量
# 重编译可达文件多时显著拖慢）。
_suffix_index: dict[str, str] = {}
for _key, _val in all_files.items():
_parts: list[str] = _key.split('.')
for _i in range(len(_parts)):
_suf: str = '.'.join(_parts[_i:])
if _suf not in _suffix_index:
_suffix_index[_suf] = _val
reachable: set[str] = set()
queue: list[str] = list(entry_files)
# BFS 队列用 deque：pop(0) 是 O(n) 出队（每次移动整表），
# popleft 为 O(1)，可达文件数百时累计节省显著。
queue_dq: Deque[str] = deque(entry_files)
# BFS trace: record each file processed and unresolved deps
_bfs_trace: list[str] = []
_unresolved_deps: list[tuple[str, str]] = [] # (current_file, dep)
while queue:
current: str = queue.pop(0)
while queue_dq:
current: str = queue_dq.popleft()
if current in reachable:
continue
reachable.add(current)
deps: set[str] = get_file_dependencies(current, src_root)
_bfs_trace.append(f" {os.path.basename(current)} deps={len(deps)}")
for dep in deps:
if dep in all_files and all_files[dep] not in reachable:
queue.append(all_files[dep])
queue_dq.append(all_files[dep])
else:
_matched = False
for key, val in all_files.items():
if key.endswith('.' + dep) or key == dep:
if val not in reachable:
queue.append(val)
_matched = True
break
if not _matched:
_unresolved_deps.append((os.path.basename(current), dep))
_target: str | None = _suffix_index.get(dep)
if _target is not None and _target not in reachable:
queue_dq.append(_target)
return reachable
@@ -204,6 +210,14 @@ def topological_sort_files(py_files: list[str], src_root: str) -> list[str]:
if dep_file != filepath: # 避免自依赖
graph[filepath].add(dep_file)
# 反向邻接表：依赖当前文件的文件列表。Kahn 出队时只需遍历 reverse[cur]，
# 替代原"每出队一个节点就全图扫描找依赖者"的 O(V²)（文件多时显著拖慢）。
reverse: dict[str, list[str]] = {f: [] for f in py_files}
for f, deps in graph.items():
for dep in deps:
if dep in reverse:
reverse[dep].append(f)
# 拓扑排序 (Kahn's algorithm)
in_degree: dict[str, int] = {f: 0 for f in py_files}
for f, deps in graph.items():
@@ -211,20 +225,20 @@ def topological_sort_files(py_files: list[str], src_root: str) -> list[str]:
if dep in in_degree:
in_degree[f] += 1
# 小根堆替代"每轮 queue.sort() + pop(0)"（O(n² log n)）：heappop 取最小
# 保证同级别文件按字母序处理（确定性），单次 O(log n)。
import heapq
queue: list[str] = [f for f, degree in in_degree.items() if degree == 0]
heapq.heapify(queue)
sorted_files: list[str] = []
while queue:
# 按字母顺序处理同级别的文件，确保确定性
queue.sort()
current: str = queue.pop(0)
current: str = heapq.heappop(queue)
sorted_files.append(current)
for f, deps in graph.items():
if current in deps:
in_degree[f] -= 1
if in_degree[f] == 0:
queue.append(f)
for f in reverse[current]:
in_degree[f] -= 1
if in_degree[f] == 0:
heapq.heappush(queue, f)
# 如果有环，添加剩余文件
if len(sorted_files) < len(py_files):
lib/vast/Tokenizer.py
+54 -12
View File
@@ -42,6 +42,17 @@ _OPERATORS: tuple[str, ...] = (
'@', '&', '|', '^', '~', '<', '>', '=', '!',
)
# 按首字符分组的运算符表（组内保持最长优先顺序）。_EmitOp 每 token 只遍历
# 同首字符的少数候选（平均 2-3 个），替代原 42 项全表 startswith 扫描
# （cProfile 热点：58 万次调用 tottime 3.6s，首字符分组后显著下降）。
_OP_BY_FIRST: dict[str, tuple[str, ...]] = {}
for _op in _OPERATORS:
_first: str = _op[0]
if _first in _OP_BY_FIRST:
_OP_BY_FIRST[_first] = _OP_BY_FIRST[_first] + (_op,)
else:
_OP_BY_FIRST[_first] = (_op,)
# 行首空白 + 关键字 + 空白 + 标识符 → 软关键字函数定义（Parser 层处理）
SOFT_KEYWORDS: frozenset[str] = frozenset({'fn', 'fun', 'func', 'function', 'extern'})
@@ -120,15 +131,36 @@ class Tokenizer:
self.Comments: list[Token] = []
# 行起始字符索引表（索引 0 对应第 1 行），用于行列换算
self._LineStarts: list[int] = [0]
# _PosInfo 增量缓存：token 顺序创建时 pos 单调递增，免 bisect
self._LastLineIdx: int = 0
self._LastPos: int = -1
# ------------------------------------------------------------------
# 位置换算
# ------------------------------------------------------------------
def _PosInfo(self, pos: int) -> tuple[int, int]:
"""将字符索引换算为 (1-based 行号, UTF-8 字节列偏移)。"""
idx: int = bisect.bisect_right(self._LineStarts, pos) - 1
"""将字符索引换算为 (1-based 行号, UTF-8 字节列偏移)。
优化（cProfile 热点：~300 万次调用）：token 顺序创建时 pos 单调
递增，用上次行索引增量顺延替代 bisect（O(log n)→摊销 O(1)）；
列偏移对 ASCII 行用 len() 直接量取，避免每次 encode 分配字节串。
"""
ls: list[int] = self._LineStarts
if pos >= self._LastPos and self._LastLineIdx + 1 < len(ls):
idx: int = self._LastLineIdx
nls: int = len(ls)
while idx + 1 < nls and ls[idx + 1] <= pos:
idx += 1
else:
idx = bisect.bisect_right(ls, pos) - 1
self._LastLineIdx = idx
self._LastPos = pos
line: int = idx + 1
col_bytes: int = len(self.Source[self._LineStarts[idx]:pos].encode('utf-8'))
seg: str = self.Source[ls[idx]:pos]
if seg.isascii():
col_bytes: int = len(seg)
else:
col_bytes = len(seg.encode('utf-8'))
return line, col_bytes
def _Make(self, tok_type: str, start: int, end: int) -> Token:
@@ -283,22 +315,32 @@ class Tokenizer:
def _EmitOp(self) -> None:
"""运算符最长匹配（含 ... 与增强赋值运算符）。"""
start: int = self.Pos
for op in _OPERATORS:
if self.Source.startswith(op, self.Pos):
self.Pos += len(op)
self.Tokens.append(self._Make(OP, start, self.Pos))
self.AtLineStart = False
return
ops: tuple[str, ...] | None = _OP_BY_FIRST.get(self.Source[self.Pos])
if ops is not None:
for op in ops:
if self.Source.startswith(op, self.Pos):
self.Pos += len(op)
self.Tokens.append(self._Make(OP, start, self.Pos))
self.AtLineStart = False
return
# 未知字符：单字符 OP 兜底
self.Pos += 1
self.Tokens.append(self._Make(OP, start, self.Pos))
self.AtLineStart = False
def _ScanName(self) -> None:
"""扫描标识符。"""
"""扫描标识符（内联 _IsIdentPart 避免 70 万次函数调用开销）。"""
start: int = self.Pos
while self.Pos < self.Length and _IsIdentPart(self.Source[self.Pos]):
self.Pos += 1
src: str = self.Source
n: int = self.Length
pos: int = self.Pos
while pos < n:
ch: str = src[pos]
if ch == '_' or ch.isalnum() or ord(ch) > 127:
pos += 1
else:
break
self.Pos = pos
self.Tokens.append(self._Make(NAME, start, self.Pos))
self.AtLineStart = False