diff --git a/lib/Projectrans/Phase1Generator.py b/lib/Projectrans/Phase1Generator.py index 8252959..118092b 100644 --- a/lib/Projectrans/Phase1Generator.py +++ b/lib/Projectrans/Phase1Generator.py @@ -179,9 +179,10 @@ class Phase1Generator: needed: set[str] = set() dep_graph: dict[str, set[str]] = {} - queue: list[str] = list(imported_from_src) + from collections import deque + queue: deque[str] = deque(imported_from_src) while queue: - mod_name: str = queue.pop(0) + mod_name: str = queue.popleft() if mod_name in needed: continue if mod_name in include_file_map: diff --git a/lib/Projectrans/Phase2Translator.py b/lib/Projectrans/Phase2Translator.py index 08b13cf..72e2d1d 100644 --- a/lib/Projectrans/Phase2Translator.py +++ b/lib/Projectrans/Phase2Translator.py @@ -417,7 +417,7 @@ class Phase2Translator: _vlog().info(f"\n需要翻译 {len(need_translate)} 个文件") t_start = time.time() - n_workers = min(8, len(need_translate)) + n_workers = min(os.cpu_count() or 4, len(need_translate)) if n_workers > 1 and len(need_translate) > 1: _vlog().info(f" 并行翻译 (workers={n_workers})") @@ -1643,7 +1643,7 @@ class Phase2Translator: need_compile.append(ll_path) if need_compile: - n_workers = min(os.cpu_count() or 4, len(need_compile), 8) + n_workers = min(os.cpu_count() or 4, len(need_compile)) _vlog().info(f" 并行编译 {len(need_compile)} 个文件 (workers={n_workers})") errors = [] with ThreadPoolExecutor(max_workers=n_workers) as executor: @@ -2167,7 +2167,7 @@ class Phase2Translator: shared_pickle_path = '' if shared_pickle_path and os.path.exists(shared_pickle_path): - n_workers = min(8, len(need_translate)) + n_workers = min(os.cpu_count() or 4, len(need_translate)) errors = [] # 波次并行翻译：按依赖关系分批提交，确保被依赖的文件先翻译完成。 # 根因：_TryLoadStructFromStub 扫描 output 目录加载跨模块 struct 定义， @@ -2392,7 +2392,7 @@ class Phase2Translator: return (task, result) if len(compile_tasks) > 1: - n_workers = min(8, len(compile_tasks)) + n_workers = min(os.cpu_count() or 4, len(compile_tasks)) with ThreadPoolExecutor(max_workers=n_workers) as executor: futures = {executor.submit(_compile_one_include, task): task for task in compile_tasks} done_count = 0 diff --git a/lib/Projectrans/Utils.py b/lib/Projectrans/Utils.py index 830453a..8790d0b 100644 --- a/lib/Projectrans/Utils.py +++ b/lib/Projectrans/Utils.py @@ -2,6 +2,8 @@ from __future__ import annotations import hashlib import os +from collections import deque +from typing import Deque import vast as ast from lib.core.VLogger import get_logger as _vlog @@ -150,33 +152,37 @@ def find_reachable_files_from_entries(src_root: str, entry_files: list[str]) -> if top_module not in all_files: all_files[top_module] = filepath + # 后缀索引：pkg.sub.mod → 注册 "pkg.sub.mod" / "sub.mod" / "mod"， + # 使 from-import 的短依赖名（dep）O(1) 定位目标文件，替代原 + # for key,val in all_files.items() 全表线性扫描（O(n²)，Phase1 全量 + # 重编译可达文件多时显著拖慢）。 + _suffix_index: dict[str, str] = {} + for _key, _val in all_files.items(): + _parts: list[str] = _key.split('.') + for _i in range(len(_parts)): + _suf: str = '.'.join(_parts[_i:]) + if _suf not in _suffix_index: + _suffix_index[_suf] = _val + reachable: set[str] = set() queue: list[str] = list(entry_files) + # BFS 队列用 deque：pop(0) 是 O(n) 出队（每次移动整表）， + # popleft 为 O(1)，可达文件数百时累计节省显著。 + queue_dq: Deque[str] = deque(entry_files) - # BFS trace: record each file processed and unresolved deps - _bfs_trace: list[str] = [] - _unresolved_deps: list[tuple[str, str]] = [] # (current_file, dep) - - while queue: - current: str = queue.pop(0) + while queue_dq: + current: str = queue_dq.popleft() if current in reachable: continue reachable.add(current) deps: set[str] = get_file_dependencies(current, src_root) - _bfs_trace.append(f" {os.path.basename(current)} deps={len(deps)}") for dep in deps: if dep in all_files and all_files[dep] not in reachable: - queue.append(all_files[dep]) + queue_dq.append(all_files[dep]) else: - _matched = False - for key, val in all_files.items(): - if key.endswith('.' + dep) or key == dep: - if val not in reachable: - queue.append(val) - _matched = True - break - if not _matched: - _unresolved_deps.append((os.path.basename(current), dep)) + _target: str | None = _suffix_index.get(dep) + if _target is not None and _target not in reachable: + queue_dq.append(_target) return reachable @@ -204,6 +210,14 @@ def topological_sort_files(py_files: list[str], src_root: str) -> list[str]: if dep_file != filepath: # 避免自依赖 graph[filepath].add(dep_file) + # 反向邻接表：依赖当前文件的文件列表。Kahn 出队时只需遍历 reverse[cur]， + # 替代原"每出队一个节点就全图扫描找依赖者"的 O(V²)（文件多时显著拖慢）。 + reverse: dict[str, list[str]] = {f: [] for f in py_files} + for f, deps in graph.items(): + for dep in deps: + if dep in reverse: + reverse[dep].append(f) + # 拓扑排序 (Kahn's algorithm) in_degree: dict[str, int] = {f: 0 for f in py_files} for f, deps in graph.items(): @@ -211,20 +225,20 @@ def topological_sort_files(py_files: list[str], src_root: str) -> list[str]: if dep in in_degree: in_degree[f] += 1 + # 小根堆替代"每轮 queue.sort() + pop(0)"（O(n² log n)）：heappop 取最小 + # 保证同级别文件按字母序处理（确定性），单次 O(log n)。 + import heapq queue: list[str] = [f for f, degree in in_degree.items() if degree == 0] + heapq.heapify(queue) sorted_files: list[str] = [] while queue: - # 按字母顺序处理同级别的文件，确保确定性 - queue.sort() - current: str = queue.pop(0) + current: str = heapq.heappop(queue) sorted_files.append(current) - - for f, deps in graph.items(): - if current in deps: - in_degree[f] -= 1 - if in_degree[f] == 0: - queue.append(f) + for f in reverse[current]: + in_degree[f] -= 1 + if in_degree[f] == 0: + heapq.heappush(queue, f) # 如果有环，添加剩余文件 if len(sorted_files) < len(py_files): diff --git a/lib/vast/Tokenizer.py b/lib/vast/Tokenizer.py index 73dab55..15b10db 100644 --- a/lib/vast/Tokenizer.py +++ b/lib/vast/Tokenizer.py @@ -42,6 +42,17 @@ _OPERATORS: tuple[str, ...] = ( '@', '&', '|', '^', '~', '<', '>', '=', '!', ) +# 按首字符分组的运算符表（组内保持最长优先顺序）。_EmitOp 每 token 只遍历 +# 同首字符的少数候选（平均 2-3 个），替代原 42 项全表 startswith 扫描 +# （cProfile 热点：58 万次调用 tottime 3.6s，首字符分组后显著下降）。 +_OP_BY_FIRST: dict[str, tuple[str, ...]] = {} +for _op in _OPERATORS: + _first: str = _op[0] + if _first in _OP_BY_FIRST: + _OP_BY_FIRST[_first] = _OP_BY_FIRST[_first] + (_op,) + else: + _OP_BY_FIRST[_first] = (_op,) + # 行首空白 + 关键字 + 空白 + 标识符 → 软关键字函数定义（Parser 层处理） SOFT_KEYWORDS: frozenset[str] = frozenset({'fn', 'fun', 'func', 'function', 'extern'}) @@ -120,15 +131,36 @@ class Tokenizer: self.Comments: list[Token] = [] # 行起始字符索引表（索引 0 对应第 1 行），用于行列换算 self._LineStarts: list[int] = [0] + # _PosInfo 增量缓存：token 顺序创建时 pos 单调递增，免 bisect + self._LastLineIdx: int = 0 + self._LastPos: int = -1 # ------------------------------------------------------------------ # 位置换算 # ------------------------------------------------------------------ def _PosInfo(self, pos: int) -> tuple[int, int]: - """将字符索引换算为 (1-based 行号, UTF-8 字节列偏移)。""" - idx: int = bisect.bisect_right(self._LineStarts, pos) - 1 + """将字符索引换算为 (1-based 行号, UTF-8 字节列偏移)。 + + 优化（cProfile 热点：~300 万次调用）：token 顺序创建时 pos 单调 + 递增，用上次行索引增量顺延替代 bisect（O(log n)→摊销 O(1)）； + 列偏移对 ASCII 行用 len() 直接量取，避免每次 encode 分配字节串。 + """ + ls: list[int] = self._LineStarts + if pos >= self._LastPos and self._LastLineIdx + 1 < len(ls): + idx: int = self._LastLineIdx + nls: int = len(ls) + while idx + 1 < nls and ls[idx + 1] <= pos: + idx += 1 + else: + idx = bisect.bisect_right(ls, pos) - 1 + self._LastLineIdx = idx + self._LastPos = pos line: int = idx + 1 - col_bytes: int = len(self.Source[self._LineStarts[idx]:pos].encode('utf-8')) + seg: str = self.Source[ls[idx]:pos] + if seg.isascii(): + col_bytes: int = len(seg) + else: + col_bytes = len(seg.encode('utf-8')) return line, col_bytes def _Make(self, tok_type: str, start: int, end: int) -> Token: @@ -283,22 +315,32 @@ class Tokenizer: def _EmitOp(self) -> None: """运算符最长匹配（含 ... 与增强赋值运算符）。""" start: int = self.Pos - for op in _OPERATORS: - if self.Source.startswith(op, self.Pos): - self.Pos += len(op) - self.Tokens.append(self._Make(OP, start, self.Pos)) - self.AtLineStart = False - return + ops: tuple[str, ...] | None = _OP_BY_FIRST.get(self.Source[self.Pos]) + if ops is not None: + for op in ops: + if self.Source.startswith(op, self.Pos): + self.Pos += len(op) + self.Tokens.append(self._Make(OP, start, self.Pos)) + self.AtLineStart = False + return # 未知字符：单字符 OP 兜底 self.Pos += 1 self.Tokens.append(self._Make(OP, start, self.Pos)) self.AtLineStart = False def _ScanName(self) -> None: - """扫描标识符。""" + """扫描标识符（内联 _IsIdentPart 避免 70 万次函数调用开销）。""" start: int = self.Pos - while self.Pos < self.Length and _IsIdentPart(self.Source[self.Pos]): - self.Pos += 1 + src: str = self.Source + n: int = self.Length + pos: int = self.Pos + while pos < n: + ch: str = src[pos] + if ch == '_' or ch.isalnum() or ord(ch) > 127: + pos += 1 + else: + break + self.Pos = pos self.Tokens.append(self._Make(NAME, start, self.Pos)) self.AtLineStart = False