From 567d925235871d2f07e5fc90753107d6ae5e4ee7 Mon Sep 17 00:00:00 2001 From: TermiNexus Date: Sat, 22 Aug 2026 03:08:29 +0800 Subject: [PATCH] =?UTF-8?q?perf(tpc):=20PROF=20=E7=83=AD=E7=82=B9=E4=BC=98?= =?UTF-8?q?=E5=8C=96=E2=80=94=E2=80=94vast=20=E8=8A=82=E7=82=B9=20=5FInit?= =?UTF-8?q?=20unroll=20=E7=89=B9=E5=8C=96=20+=20isfile=20=E7=BB=93?= =?UTF-8?q?=E6=9E=9C=E7=BC=93=E5=AD=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - vast/Nodes.py：_MakeNodeClass 动态生成 __init__，全位置参数路径按字段 索引直赋（unroll，去 zip 迭代器），语义与通用路径完全等价 （PROF：_Init 158 万次 29s 最大纯 Python 热点，实测 ~1.4x） - Phase2Translator.py：新增模块级 _CachedIsFile（编译进程内文件状态稳定）， 替换全部 os.path.isfile 存在性检查（PROF：2.6 万次 16.4s） --- lib/Projectrans/Phase2Translator.py | 84 +++++++++++++++++------------ lib/vast/Nodes.py | 40 ++++++++++---- 2 files changed, 79 insertions(+), 45 deletions(-) diff --git a/lib/Projectrans/Phase2Translator.py b/lib/Projectrans/Phase2Translator.py index de0a325..fd98003 100644 --- a/lib/Projectrans/Phase2Translator.py +++ b/lib/Projectrans/Phase2Translator.py @@ -23,6 +23,20 @@ import TransPyC from StubGen import PythonToStubConverter +# PROF 热点：nt._path_isfile 2.6 万次 16.4s（Windows 磁盘 stat ~0.6ms/次）。 +# 编译进程内文件状态稳定（obj/ll 一旦生成不删除、不被移动），用模块级缓存 +# 避免重复 stat。仅用于"存在性检查"类调用（不用于内容变化的判断）。 +_FILE_STAT_CACHE: dict[str, bool] = {} + + +def _CachedIsFile(path: str) -> bool: + r: bool | None = _FILE_STAT_CACHE.get(path) + if r is None: + r = os.path.isfile(path) + _FILE_STAT_CACHE[path] = r + return r + + def _PickWorkerCount(n_files: int, per_worker_mb: int = 700) -> int: """按可用物理内存动态限制并行 worker 数，避免多进程编译内存峰值 OOM。 @@ -364,7 +378,7 @@ def _BuildPyiCacheFile(temp_dir: str) -> str | None: pkl_path: str = os.path.join(temp_dir, _PYI_CACHE_PKL_NAME) sig_path: str = os.path.join(temp_dir, _PYI_CACHE_SIG_NAME) try: - if os.path.isfile(pkl_path) and os.path.isfile(sig_path): + if _CachedIsFile(pkl_path) and _CachedIsFile(sig_path): with open(sig_path, 'r', encoding='utf-8') as f: stored = json.load(f) # json 往返后 tuple→list，需规范化再比较 @@ -743,7 +757,7 @@ class Phase2Translator: raise _vlog().warning(f"收集源文件导入的 include 模块失败: {_e}", "Exception") - if os.path.isfile(out_path): + if _CachedIsFile(out_path): deps_changed = self._check_deps_changed(sha1, current_ModuleSha1Map) if not deps_changed: _vlog().info(f"[{i}/{len(py_files)}] 跳过(缓存): {rel} -> {sha1}.ll") @@ -757,7 +771,7 @@ class Phase2Translator: else: stub_cache = os.path.join(self.output_dir, f"{sha1}.stub.ll") text_cache = os.path.join(self.output_dir, f"{sha1}.text.ll") - if os.path.isfile(stub_cache) and os.path.isfile(text_cache): + if _CachedIsFile(stub_cache) and _CachedIsFile(text_cache): if self._recombine_ll(sha1, current_ModuleSha1Map): _vlog().info(f"[{i}/{len(py_files)}] 重组(缓存恢复): {rel} -> {sha1}.ll") continue @@ -942,14 +956,14 @@ class Phase2Translator: # 存在性/内容缓存：编译期 include 源文件不变，命中跳过磁盘 I/O exists = Phase2Translator._include_exist_cache.get(py_path) if exists is None: - exists = os.path.isfile(py_path) + exists = _CachedIsFile(py_path) Phase2Translator._include_exist_cache[py_path] = exists if not exists: # 回退到 .py（向后兼容） py_path = os.path.splitext(py_path)[0] + '.py' exists = Phase2Translator._include_exist_cache.get(py_path) if exists is None: - exists = os.path.isfile(py_path) + exists = _CachedIsFile(py_path) Phase2Translator._include_exist_cache[py_path] = exists if exists: py_sha1 = Phase2Translator._include_sha1_cache.get(py_path) @@ -1196,7 +1210,7 @@ class Phase2Translator: def _LoadDeps(self, sha1: str) -> dict[str, str] | None: """加载依赖指纹""" deps_path = os.path.join(self.output_dir, f"{sha1}.deps.json") - if os.path.isfile(deps_path): + if _CachedIsFile(deps_path): try: with open(deps_path, 'r', encoding='utf-8') as f: return json.load(f) @@ -1229,7 +1243,7 @@ class Phase2Translator: text_path = os.path.join(self.output_dir, f"{sha1}.text.ll") ll_path = os.path.join(self.output_dir, f"{sha1}.ll") - if not os.path.isfile(stub_path) or not os.path.isfile(text_path): + if not _CachedIsFile(stub_path) or not _CachedIsFile(text_path): return False with open(stub_path, 'r', encoding='utf-8') as f: @@ -1263,7 +1277,7 @@ class Phase2Translator: self._save_deps(sha1, current_ModuleSha1Map) obj_path = os.path.join(self.output_dir, f"{sha1}.obj") - if os.path.isfile(obj_path): + if _CachedIsFile(obj_path): os.remove(obj_path) return True @@ -1670,7 +1684,7 @@ class Phase2Translator: if sha1 in active_sha1s: fpath = os.path.join(root, file) obj_path = os.path.join(root, file[:-3] + '.obj') - if not os.path.isfile(obj_path): + if not _CachedIsFile(obj_path): ll_files.append(fpath) if not ll_files: _vlog().info("[编译] 无 .ll 文件需要重新编译") @@ -1763,7 +1777,7 @@ class Phase2Translator: for ll_path in ll_files: obj_path = ll_path[:-3] + '.obj' sha1_obj = os.path.join(self.output_dir, os.path.basename(ll_path)[:-3] + '.obj') - if os.path.isfile(sha1_obj): + if _CachedIsFile(sha1_obj): os.remove(sha1_obj) return else: @@ -1781,7 +1795,7 @@ class Phase2Translator: def _compile_one(ll_path: str) -> tuple[str, str, str]: rel = os.path.relpath(ll_path, self.output_dir) obj_path = ll_path[:-3] + '.obj' - if os.path.isfile(obj_path): + if _CachedIsFile(obj_path): return (rel, 'cached', '') try: with open(ll_path, 'r', encoding='utf-8') as f: @@ -1862,9 +1876,9 @@ class Phase2Translator: continue seen_stub_sha1.add(dep_sha1) stub_path = os.path.join(self.output_dir, f"{dep_sha1}.stub.ll") - if not os.path.isfile(stub_path): + if not _CachedIsFile(stub_path): stub_path = os.path.join(self.temp_dir, f"{dep_sha1}.stub.ll") - if os.path.isfile(stub_path): + if _CachedIsFile(stub_path): with open(stub_path, 'r', encoding='utf-8') as sf: stub_content = sf.read() for line in stub_content.splitlines(): @@ -2052,7 +2066,7 @@ class Phase2Translator: need_compile = [] for ll_path in ll_files: obj_path = ll_path[:-3] + '.obj' - if not os.path.isfile(obj_path): + if not _CachedIsFile(obj_path): need_compile.append(ll_path) if need_compile: @@ -2174,13 +2188,13 @@ class Phase2Translator: for ext in self.INCLUDE_LIB_EXTENSIONS: lib_path = os.path.join(includes_dir, module_name + ext) - if os.path.isfile(lib_path): + if _CachedIsFile(lib_path): self.extra_link_files.append(lib_path) _vlog().info(f" [includes] 发现库文件: {os.path.relpath(lib_path, includes_dir)}") for ext in self.INCLUDE_SRC_EXTENSIONS: src_path = os.path.join(includes_dir, module_name + ext) - if os.path.isfile(src_path): + if _CachedIsFile(src_path): self.extra_compile_files.append(src_path) _vlog().info(f" [includes] 发现源文件: {os.path.relpath(src_path, includes_dir)}") @@ -2264,7 +2278,7 @@ class Phase2Translator: if py_index is not None: exists = os.path.normcase(py_path) in py_index else: - exists = os.path.isfile(py_path) + exists = _CachedIsFile(py_path) if exists and py_path not in self.extra_py_files: self.extra_py_files.append(py_path) _vlog().info(f" [includes] 发现间接依赖: {os.path.relpath(py_path, includes_dir)}") @@ -2505,7 +2519,7 @@ class Phase2Translator: continue # 优先检查 includes.binary 预编译缓存（跨项目共享） - if not self.force_recompile_includes and precompiled_obj and os.path.isfile(precompiled_obj): + if not self.force_recompile_includes and precompiled_obj and _CachedIsFile(precompiled_obj): _vlog().info(f" 跳过(预编译): {module_name}.py -> {os.path.basename(precompiled_obj)}") # 预编译跳过的文件仍需注册 .pyi/.stub.ll 声明和符号， # 否则依赖它的 include 文件（如 vpui.py import vpsdk.window） @@ -2515,7 +2529,7 @@ class Phase2Translator: self.extra_link_files.append(precompiled_obj) continue - if not self.force_recompile_includes and os.path.isfile(obj_path): + if not self.force_recompile_includes and _CachedIsFile(obj_path): _vlog().info(f" 跳过(缓存): {module_name}.py -> {sha1}.obj") self._register_compiled_include(src_path, sha1, None, include_ModuleSha1Map) self._collect_inline_symbols(src_path) @@ -2699,7 +2713,7 @@ class Phase2Translator: compile_tasks = [] # [(dso_ll_path, obj_path, src_path, sha1, module_name, ll_path, precompiled_obj, ll_content)] for item in need_translate: src_path, sha1, module_name, ll_path, obj_path, precompiled_obj = item - if not os.path.isfile(ll_path): + if not _CachedIsFile(ll_path): _vlog().error(f" 翻译未生成 .ll 文件: {module_name}.py") continue @@ -2873,7 +2887,7 @@ class Phase2Translator: # DEBUG: 保留 .ll 文件以便静态分析 # for ext in ['.ll', '_dso.ll']: # p = ll_path[:-3] + ext - # if os.path.isfile(p): + # if _CachedIsFile(p): # os.remove(p) else: has_error = True @@ -3007,10 +3021,10 @@ class Phase2Translator: # 1. 生成 .pyi 签名文件（如果还没有的话） sig_path = os.path.join(self.temp_dir, f"{sha1}.vpi") - if not os.path.isfile(sig_path): + if not _CachedIsFile(sig_path): # 全局缓存检查（不被 --clean 清除） _TryGlobalCache(sha1, 'pyi', sig_path) - if not os.path.isfile(sig_path): + if not _CachedIsFile(sig_path): try: with open(src_path, 'r', encoding='utf-8') as f: py_content = f.read() @@ -3025,7 +3039,7 @@ class Phase2Translator: _vlog().warning(f"签名生成失败不阻塞编译: {e}", "Exception") # 注册到 sig_files 和 _source_module_sig_files - if os.path.isfile(sig_path): + if _CachedIsFile(sig_path): self.sig_files[sha1] = sig_path self.sha1_map[sha1] = f"includes/{mod_full.replace('.', '/')}.vp" if self._shared_source_module_sig_files is not None: @@ -3043,10 +3057,10 @@ class Phase2Translator: # 2. 生成 .stub.ll 文件 stub_path = os.path.join(self.output_dir, f"{sha1}.stub.ll") - if not os.path.isfile(stub_path): + if not _CachedIsFile(stub_path): # 全局缓存检查（不被 --clean 清除） _TryGlobalCache(sha1, 'stub.ll', stub_path) - if not os.path.isfile(stub_path) and ll_content: + if not _CachedIsFile(stub_path) and ll_content: try: stub_content, _ = self._split_ll(ll_content) with open(stub_path, 'w', encoding='utf-8', newline='\n') as f: @@ -3058,17 +3072,17 @@ class Phase2Translator: _vlog().warning(f"生成 .stub.ll 文件失败: {_e}", "Exception") # 注册到 stub_files - if os.path.isfile(stub_path): + if _CachedIsFile(stub_path): self.stub_files[sha1] = stub_path - elif os.path.isfile(sig_path): + elif _CachedIsFile(sig_path): # 如果 stub 文件不存在，也尝试从 temp 目录查找 temp_stub = os.path.join(self.temp_dir, f"{sha1}.stub.ll") - if os.path.isfile(temp_stub): + if _CachedIsFile(temp_stub): self.stub_files[sha1] = temp_stub else: # 最后尝试从全局缓存直接读取 _TryGlobalCache(sha1, 'stub.ll', temp_stub) - if os.path.isfile(temp_stub): + if _CachedIsFile(temp_stub): self.stub_files[sha1] = temp_stub def _precollect_inline_symbols(self) -> None: @@ -3088,10 +3102,10 @@ class Phase2Translator: src_path = os.path.join(root, fname) self._collect_inline_symbols(src_path) py_path = os.path.join(includes_dir, module_name + '.vp') - if not os.path.isfile(py_path): + if not _CachedIsFile(py_path): # 回退到 .py（向后兼容） py_path = os.path.join(includes_dir, module_name + '.py') - if os.path.isfile(py_path): + if _CachedIsFile(py_path): self._collect_inline_symbols(py_path) for ef in self.extra_py_files: self._collect_inline_symbols(ef) @@ -3404,7 +3418,7 @@ class Phase2Translator: """将 active_sha1s 对应的 .obj 文件和 includes 库文件链接为可执行文件""" obj_files = [] merged_obj = os.path.join(self.output_dir, '_merged.obj') - if os.path.isfile(merged_obj): + if _CachedIsFile(merged_obj): obj_files.append(merged_obj) else: for root, dirs, files in os.walk(self.output_dir): @@ -3446,10 +3460,10 @@ class Phase2Translator: # 自动复制或生成 linker.ld 到输出目录 linker_ld_src = os.path.join(os.path.dirname(self.output_dir), 'linker.ld') linker_ld_dst = os.path.join(self.output_dir, 'linker.ld') - if os.path.isfile(linker_ld_src) and not os.path.isfile(linker_ld_dst): + if _CachedIsFile(linker_ld_src) and not _CachedIsFile(linker_ld_dst): shutil.copy2(linker_ld_src, linker_ld_dst) _vlog().info(f" [链接] 使用链接脚本: linker.ld") - elif not os.path.isfile(linker_ld_dst) and any('-T' in f or '-T' in f.replace(',', ' ') for f in self.linker_flags): + elif not _CachedIsFile(linker_ld_dst) and any('-T' in f or '-T' in f.replace(',', ' ') for f in self.linker_flags): default_ld = """ENTRY(_start) SECTIONS { diff --git a/lib/vast/Nodes.py b/lib/vast/Nodes.py index 028b65f..5c7a49e 100644 --- a/lib/vast/Nodes.py +++ b/lib/vast/Nodes.py @@ -311,22 +311,42 @@ _BASE_CLASS: dict[str, str] = { def _MakeNodeClass(name: str, fields: tuple[str, ...], attrs: tuple[str, ...]) -> type: """按字段表生成节点类（继承对应抽象基类）。""" - def _Init(self: AST, *args: object, **kwargs: object) -> None: - if len(args) > len(fields): - raise TypeError( - f'{name} constructor takes at most {len(fields)} positional arguments' - ) - for fname, val in zip(fields, args): - object.__setattr__(self, fname, val) - for k, v in kwargs.items(): - object.__setattr__(self, k, v) + # 特化 __init__：常见路径（全位置参数，vast_c 回退 Vectorcall 的主形态）按 + # 字段索引直赋（unroll），避免 zip 迭代器 + 循环控制开销（PROF：_Init + # 158 万次 29s 是最大纯 Python 热点）。语义与通用路径完全等价： + # 位置参数按序设置前 n 个字段，kwargs 后设覆盖；越界同样抛 TypeError。 + def _MakeInit(name: str, fields: tuple[str, ...]) -> object: + n = len(fields) + lines: list[str] = [ + "def _Init(self: AST, *args: object, **kwargs: object) -> None:", + " _set = object.__setattr__", + " _n = len(args)", + f" if _n > {n}:", + f" raise TypeError(f'{name} constructor takes at most {n} positional arguments')", + ] + if n > 0: + lines.append(" if _n:") + for i in range(1, min(n, 8) + 1): + assigns = "; ".join( + f"_set(self, {fields[j]!r}, args[{j}])" for j in range(i)) + lines.append(f" if _n == {i}:") + lines.append(f" {assigns}") + if n > 8: + lines.append(" else:") + lines.append(" for _i in range(_n):") + lines.append(" _set(self, _F[_i], args[_i])") + lines.append(" for _k, _v in kwargs.items():") + lines.append(" _set(self, _k, _v)") + ns: dict[str, object] = {"AST": AST, "_F": fields} + exec("\n".join(lines), ns) + return ns["_Init"] base_name: str | None = _BASE_CLASS.get(name) base_cls: type = globals()[base_name] if base_name else AST # __slots__ = 位置字段 + 位置属性（实例的全部属性名） slots: tuple[str, ...] = tuple(dict.fromkeys(fields + attrs)) ns: dict[str, object] = { - '__init__': _Init, + '__init__': _MakeInit(name, fields), '__slots__': slots, '_fields': fields, '_attributes': attrs,