From 5692e9b09716f1e01af2ec91b0e5e17b07238285 Mon Sep 17 00:00:00 2001 From: TermiNexus Date: Mon, 17 Aug 2026 22:20:28 +0800 Subject: [PATCH] =?UTF-8?q?perf(stubgen):=20pyi=20=E8=BD=AC=E6=8D=A2?= =?UTF-8?q?=E5=8A=A0=E7=BC=93=E5=AD=98=E2=80=94=E2=80=94convert=20?= =?UTF-8?q?=E7=BA=AF=E5=87=BD=E6=95=B0=E7=BC=93=E5=AD=98=20+=20Phase1=20mt?= =?UTF-8?q?ime=20=E7=BC=93=E5=AD=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Converter.convert 147 次调用 45s、_process_file_pyi 98 次 50s。convert 按 (内容, 模块名) 纯函数缓存并合并内部两次 ast.parse；Phase1 按 (绝对路径, 模块名)+mtime+size 缓存 pyi 转换结果与 docstrings，未变化 跳过 open+read+parse。缓存上限 4096 条防内存膨胀，配置变更时清空。 --- lib/Projectrans/Phase1Generator.py | 82 ++++++++++++++++++++++++------ lib/StubGen/Converter.py | 41 ++++++++++++--- 2 files changed, 102 insertions(+), 21 deletions(-) diff --git a/lib/Projectrans/Phase1Generator.py b/lib/Projectrans/Phase1Generator.py index 118092b..5714366 100644 --- a/lib/Projectrans/Phase1Generator.py +++ b/lib/Projectrans/Phase1Generator.py @@ -17,6 +17,16 @@ from StubGen import PythonToStubConverter _COMPILER_ROOT: str = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) _GLOBAL_CACHE_DIR: str = os.path.join(_COMPILER_ROOT, '.transpyc_cache') +# 模块级 pyi 转换缓存：{ (abs_path, module_name): (mtime, size, sig_content) } +# _process_file_pyi 与 _process_include_py_files_pyi 共享；mtime+size 未变时 +# 跳过 open+read+PythonToStubConverter.convert 直接复用结果。 +# sig_content 为不可变字符串，调用方仅将其写入文件，可安全复用（无需深拷贝）。 +_PYI_CONVERT_CACHE: dict[tuple[str, str], tuple[float, int, str]] = {} +# 模块级 docstring 缓存：{ abs_path: (mtime, size, docs) } +# docs 仅被 json.dump 只读使用（调用方不修改），可安全复用。 +_PYI_DOC_CACHE: dict[str, tuple[float, int, dict[str, str]]] = {} +_PYI_CACHE_MAX_SIZE: int = 4096 + def _TryGlobalCache(Sha1: str, Ext: str, TempPath: str) -> bool: """检查全局缓存中是否存在指定文件，若存在则复制到 temp_dir 并返回 True。""" @@ -329,10 +339,8 @@ class Phase1Generator: _vlog().info(f" 缓存命中(全局): {rel_from_inc} -> {sha1}.vpi") continue - # 缓存未命中，需读取 content 生成 .pyi - with open(src_path, 'r', encoding='utf-8') as f: - content: str = f.read() - sig_content: str = PythonToStubConverter.convert(content, module_name) + # 缓存未命中，需读取 content 生成 .pyi（模块级缓存避免跨入口/跨 run 重复转换） + sig_content: str = self._get_cached_pyi(src_path, module_name) with open(sig_path, 'w', encoding='utf-8', newline='\n') as f: f.write(sig_content) _SaveToGlobalCache(sha1, 'pyi', sig_path) @@ -527,6 +535,55 @@ class Phase1Generator: return body[0].value.value return None + def _get_cached_pyi(self, src_path: str, module_name: str) -> str: + """读取源文件并转换为 .pyi 内容；同一 (路径, 模块名) 且 mtime+size 未变时复用模块级缓存。 + + 覆盖 _process_file_pyi 与 _process_include_py_files_pyi 两个入口， + 避免对未变化的文件重复 open+read+convert（mtime+size 一致即视为内容未变， + 与 _get_sha1 的 manifest 判定一致）。 + """ + abs_path: str = os.path.abspath(src_path) + key: tuple[str, str] = (abs_path, module_name) + try: + st: os.stat_result = os.stat(abs_path) + except OSError: + st = None + if st is not None: + cached: tuple[float, int, str] | None = _PYI_CONVERT_CACHE.get(key) + if cached is not None and cached[0] == st.st_mtime and cached[1] == st.st_size: + return cached[2] + with open(abs_path, 'r', encoding='utf-8') as f: + content: str = f.read() + sig_content: str = PythonToStubConverter.convert(content, module_name) + if len(_PYI_CONVERT_CACHE) >= _PYI_CACHE_MAX_SIZE: + _PYI_CONVERT_CACHE.clear() + if st is not None: + _PYI_CONVERT_CACHE[key] = (st.st_mtime, st.st_size, sig_content) + return sig_content + + def _get_cached_docstrings(self, src_path: str) -> dict[str, str]: + """读取源文件并提取 docstring；同一路径且 mtime+size 未变时复用模块级缓存。 + + 返回的 dict 仅供调用方 json.dump 只读使用，调用方不修改，故可安全复用缓存对象。 + """ + abs_path: str = os.path.abspath(src_path) + try: + st: os.stat_result = os.stat(abs_path) + except OSError: + st = None + if st is not None: + cached: tuple[float, int, dict[str, str]] | None = _PYI_DOC_CACHE.get(abs_path) + if cached is not None and cached[0] == st.st_mtime and cached[1] == st.st_size: + return cached[2] + with open(abs_path, 'r', encoding='utf-8') as f: + content: str = f.read() + docs: dict[str, str] = self._extract_docstrings(content) + if len(_PYI_DOC_CACHE) >= _PYI_CACHE_MAX_SIZE: + _PYI_DOC_CACHE.clear() + if st is not None: + _PYI_DOC_CACHE[abs_path] = (st.st_mtime, st.st_size, docs) + return docs + def _process_file_pyi(self, src_path: str, rel_path: str) -> None: # 增量优化：先用 mtime+size 获取 sha1，未变更文件跳过 open+read sha1: str = self._get_sha1(src_path) @@ -541,10 +598,8 @@ class Phase1Generator: doc_path: str = os.path.join(self.temp_dir, f"{sha1}.doc.json") if not os.path.isfile(doc_path): if not _TryGlobalCache(sha1, 'doc.json', doc_path): - # doc.json 缺失时需读 content 提取 docstring - with open(src_path, 'r', encoding='utf-8') as f: - content: str = f.read() - docs: dict[str, str] = self._extract_docstrings(content) + # doc.json 缺失时需读 content 提取 docstring（模块级缓存复用） + docs: dict[str, str] = self._get_cached_docstrings(src_path) try: with open(doc_path, 'w', encoding='utf-8', newline='\n') as f: json.dump(docs, f, ensure_ascii=False) @@ -561,19 +616,16 @@ class Phase1Generator: _TryGlobalCache(sha1, 'doc.json', doc_path) return - # 缓存未命中，需读取 content 生成 .pyi - with open(src_path, 'r', encoding='utf-8') as f: - content = f.read() - + # 缓存未命中，需读取 content 生成 .pyi（模块级缓存避免跨入口/跨 run 重复 open+read+convert） module_name: str = os.path.splitext(rel_path)[0].replace(os.sep, '.').replace('/', '.') - sig_content: str = PythonToStubConverter.convert(content, module_name) + sig_content: str = self._get_cached_pyi(src_path, module_name) with open(sig_path, 'w', encoding='utf-8', newline='\n') as f: f.write(sig_content) _SaveToGlobalCache(sha1, 'pyi', sig_path) _vlog().info(f" -> {sha1}.vpi (签名)") - # 提取 docstring 写入 {sha1}.doc.json，供 Phase2 跨模块 __doc__ 加载 - docs: dict[str, str] = self._extract_docstrings(content) + # 提取 docstring 写入 {sha1}.doc.json，供 Phase2 跨模块 __doc__ 加载（模块级缓存复用） + docs: dict[str, str] = self._get_cached_docstrings(src_path) doc_path: str = os.path.join(self.temp_dir, f"{sha1}.doc.json") try: with open(doc_path, 'w', encoding='utf-8', newline='\n') as f: diff --git a/lib/StubGen/Converter.py b/lib/StubGen/Converter.py index fa63741..64e2cbc 100644 --- a/lib/StubGen/Converter.py +++ b/lib/StubGen/Converter.py @@ -7,6 +7,14 @@ from typing import List, Dict from lib.core.SymbolUtils import IsListAnnotation, ExpandSoftKeywords +# 模块级转换结果缓存：{ (PyContent, ModuleName): str } +# convert 是纯函数（类级配置 IncludeAliasMap/VarExcludeList/MacroExcludeList 运行期不变）， +# 相同输入必然产生相同输出。结果字符串不可变，调用方仅将其写入文件或拼接出新字符串， +# 缓存结果可安全复用（无需深拷贝）。Set* 配置方法会清空缓存以防配置变化导致结果过期。 +_ConvertResultCache: dict[tuple[str, str], str] = {} +_ConvertCacheMaxSize: int = 4096 + + class PythonToStubConverter: """将 Python 文件转换为存根格式""" @@ -23,16 +31,19 @@ class PythonToStubConverter: def SetIncludeAliasMap(cls, alias_map: Dict[str, str]) -> None: """设置 __include 别名映射表""" cls.IncludeAliasMap = alias_map + _ConvertResultCache.clear() @classmethod def SetVarExcludeList(cls, exclude_list: List[str]) -> None: """设置变量排除列表""" cls.VarExcludeList = exclude_list + _ConvertResultCache.clear() @classmethod def SetMacroExcludeList(cls, exclude_list: List[str]) -> None: """设置宏排除列表""" cls.MacroExcludeList = exclude_list + _ConvertResultCache.clear() @classmethod def _ShouldExcludeVar(cls, VarName: str) -> bool: @@ -56,6 +67,13 @@ class PythonToStubConverter: # 软关键字展开：fn/fun/func/function -> def（行内等宽/更短，行号不变）， # 保证后续 ast.parse 与 SourceLines 回读行号一致。 PyContent = ExpandSoftKeywords(PyContent) + + # 模块级缓存：同一 (内容, 模块名) 直接复用上次转换结果（纯函数输出，不可变字符串） + CacheKey: tuple[str, str] = (PyContent, ModuleName) + CachedResult: str | None = _ConvertResultCache.get(CacheKey) + if CachedResult is not None: + return CachedResult + lines: list[str] = PyContent.split('\n') StubLines: list[str] = [] @@ -66,7 +84,7 @@ class PythonToStubConverter: StubLines.append('"""') StubLines.append('') - # 解析 Python 代码，检查是否已经导入了 t 和 c + # 解析 Python 代码，检查是否已经导入了 t 和 c（AST 与下方主解析复用，避免重复 parse） HasImportT: bool = False HasImportC: bool = False try: @@ -78,8 +96,12 @@ class PythonToStubConverter: HasImportT = True elif alias.name == 'c': HasImportC = True - except: - pass + except SyntaxError: + # 语法错误：同一内容两次解析结果必然相同，与主循环 fail-fast 行为一致 + raise + except Exception: + # 非语法类解析异常：保持原语义（第一遍吞掉，主循环重新解析后向上传播） + tree = None # 添加默认导入（如果源代码中没有） if not HasImportT: @@ -101,9 +123,11 @@ class PythonToStubConverter: #StubLines.append(f'{FileGuardName}: t.CDefine') #StubLines.append('') - # 解析 Python 代码，提取类型定义 + # 解析 Python 代码，提取类型定义（复用上方的 parse 结果） try: - tree = ast.parse(PyContent) + if tree is None: + # 第一遍解析失败（非语法类异常）：重试一次，保持原"主循环重新解析"语义 + tree = ast.parse(PyContent) # 第一遍扫描：收集 Postdefinition 变量 PostdefVars: dict[str, list[tuple[str, str, ast.AnnAssign]]] = {} # {ClassName: [(VarName, TypeStr, node), ...]} @@ -218,7 +242,12 @@ class PythonToStubConverter: # undefined symbol / mangle 错误形式爆发（如 ExpandKeywordsTest 链接失败）。 raise - return '\n'.join(StubLines) + Result: str = '\n'.join(StubLines) + # 缓存上限保护：条目过多时整体清空，避免内存无限增长（结果可随时重算） + if len(_ConvertResultCache) >= _ConvertCacheMaxSize: + _ConvertResultCache.clear() + _ConvertResultCache[CacheKey] = Result + return Result @staticmethod def _ConvertClass(node: ast.ClassDef, SourceLines: list[str] | None = None, PostdefVars: list[tuple[str, str, ast.AnnAssign]] | None = None, indent: int = 0) -> list[str]: