perf(stubgen): pyi 转换加缓存——convert 纯函数缓存 + Phase1 mtime 缓存
Converter.convert 147 次调用 45s、_process_file_pyi 98 次 50s。convert 按 (内容, 模块名) 纯函数缓存并合并内部两次 ast.parse；Phase1 按 (绝对路径, 模块名)+mtime+size 缓存 pyi 转换结果与 docstrings，未变化 跳过 open+read+parse。缓存上限 4096 条防内存膨胀，配置变更时清空。
This commit is contained in:
TermiNexus committed
1 parent
e25095da43
commit
5692e9b097
2 files changed
+102
-21
No files matched your search
@@ -17,6 +17,16 @@ from StubGen import PythonToStubConverter
|
||||
_COMPILER_ROOT: str = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
_GLOBAL_CACHE_DIR: str = os.path.join(_COMPILER_ROOT, '.transpyc_cache')
|
||||
|
||||
# 模块级 pyi 转换缓存：{ (abs_path, module_name): (mtime, size, sig_content) }
|
||||
# _process_file_pyi 与 _process_include_py_files_pyi 共享；mtime+size 未变时
|
||||
# 跳过 open+read+PythonToStubConverter.convert 直接复用结果。
|
||||
# sig_content 为不可变字符串，调用方仅将其写入文件，可安全复用（无需深拷贝）。
|
||||
_PYI_CONVERT_CACHE: dict[tuple[str, str], tuple[float, int, str]] = {}
|
||||
# 模块级 docstring 缓存：{ abs_path: (mtime, size, docs) }
|
||||
# docs 仅被 json.dump 只读使用（调用方不修改），可安全复用。
|
||||
_PYI_DOC_CACHE: dict[str, tuple[float, int, dict[str, str]]] = {}
|
||||
_PYI_CACHE_MAX_SIZE: int = 4096
|
||||
|
||||
|
||||
def _TryGlobalCache(Sha1: str, Ext: str, TempPath: str) -> bool:
|
||||
"""检查全局缓存中是否存在指定文件，若存在则复制到 temp_dir 并返回 True。"""
|
||||
@@ -329,10 +339,8 @@ class Phase1Generator:
|
||||
_vlog().info(f" 缓存命中(全局): {rel_from_inc} -> {sha1}.vpi")
|
||||
continue
|
||||
|
||||
# 缓存未命中，需读取 content 生成 .pyi
|
||||
with open(src_path, 'r', encoding='utf-8') as f:
|
||||
content: str = f.read()
|
||||
sig_content: str = PythonToStubConverter.convert(content, module_name)
|
||||
# 缓存未命中，需读取 content 生成 .pyi（模块级缓存避免跨入口/跨 run 重复转换）
|
||||
sig_content: str = self._get_cached_pyi(src_path, module_name)
|
||||
with open(sig_path, 'w', encoding='utf-8', newline='\n') as f:
|
||||
f.write(sig_content)
|
||||
_SaveToGlobalCache(sha1, 'pyi', sig_path)
|
||||
@@ -527,6 +535,55 @@ class Phase1Generator:
|
||||
return body[0].value.value
|
||||
return None
|
||||
|
||||
def _get_cached_pyi(self, src_path: str, module_name: str) -> str:
|
||||
"""读取源文件并转换为 .pyi 内容；同一 (路径, 模块名) 且 mtime+size 未变时复用模块级缓存。
|
||||
|
||||
覆盖 _process_file_pyi 与 _process_include_py_files_pyi 两个入口，
|
||||
避免对未变化的文件重复 open+read+convert（mtime+size 一致即视为内容未变，
|
||||
与 _get_sha1 的 manifest 判定一致）。
|
||||
"""
|
||||
abs_path: str = os.path.abspath(src_path)
|
||||
key: tuple[str, str] = (abs_path, module_name)
|
||||
try:
|
||||
st: os.stat_result = os.stat(abs_path)
|
||||
except OSError:
|
||||
st = None
|
||||
if st is not None:
|
||||
cached: tuple[float, int, str] | None = _PYI_CONVERT_CACHE.get(key)
|
||||
if cached is not None and cached[0] == st.st_mtime and cached[1] == st.st_size:
|
||||
return cached[2]
|
||||
with open(abs_path, 'r', encoding='utf-8') as f:
|
||||
content: str = f.read()
|
||||
sig_content: str = PythonToStubConverter.convert(content, module_name)
|
||||
if len(_PYI_CONVERT_CACHE) >= _PYI_CACHE_MAX_SIZE:
|
||||
_PYI_CONVERT_CACHE.clear()
|
||||
if st is not None:
|
||||
_PYI_CONVERT_CACHE[key] = (st.st_mtime, st.st_size, sig_content)
|
||||
return sig_content
|
||||
|
||||
def _get_cached_docstrings(self, src_path: str) -> dict[str, str]:
|
||||
"""读取源文件并提取 docstring；同一路径且 mtime+size 未变时复用模块级缓存。
|
||||
|
||||
返回的 dict 仅供调用方 json.dump 只读使用，调用方不修改，故可安全复用缓存对象。
|
||||
"""
|
||||
abs_path: str = os.path.abspath(src_path)
|
||||
try:
|
||||
st: os.stat_result = os.stat(abs_path)
|
||||
except OSError:
|
||||
st = None
|
||||
if st is not None:
|
||||
cached: tuple[float, int, dict[str, str]] | None = _PYI_DOC_CACHE.get(abs_path)
|
||||
if cached is not None and cached[0] == st.st_mtime and cached[1] == st.st_size:
|
||||
return cached[2]
|
||||
with open(abs_path, 'r', encoding='utf-8') as f:
|
||||
content: str = f.read()
|
||||
docs: dict[str, str] = self._extract_docstrings(content)
|
||||
if len(_PYI_DOC_CACHE) >= _PYI_CACHE_MAX_SIZE:
|
||||
_PYI_DOC_CACHE.clear()
|
||||
if st is not None:
|
||||
_PYI_DOC_CACHE[abs_path] = (st.st_mtime, st.st_size, docs)
|
||||
return docs
|
||||
|
||||
def _process_file_pyi(self, src_path: str, rel_path: str) -> None:
|
||||
# 增量优化：先用 mtime+size 获取 sha1，未变更文件跳过 open+read
|
||||
sha1: str = self._get_sha1(src_path)
|
||||
@@ -541,10 +598,8 @@ class Phase1Generator:
|
||||
doc_path: str = os.path.join(self.temp_dir, f"{sha1}.doc.json")
|
||||
if not os.path.isfile(doc_path):
|
||||
if not _TryGlobalCache(sha1, 'doc.json', doc_path):
|
||||
# doc.json 缺失时需读 content 提取 docstring
|
||||
with open(src_path, 'r', encoding='utf-8') as f:
|
||||
content: str = f.read()
|
||||
docs: dict[str, str] = self._extract_docstrings(content)
|
||||
# doc.json 缺失时需读 content 提取 docstring（模块级缓存复用）
|
||||
docs: dict[str, str] = self._get_cached_docstrings(src_path)
|
||||
try:
|
||||
with open(doc_path, 'w', encoding='utf-8', newline='\n') as f:
|
||||
json.dump(docs, f, ensure_ascii=False)
|
||||
@@ -561,19 +616,16 @@ class Phase1Generator:
|
||||
_TryGlobalCache(sha1, 'doc.json', doc_path)
|
||||
return
|
||||
|
||||
# 缓存未命中，需读取 content 生成 .pyi
|
||||
with open(src_path, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
|
||||
# 缓存未命中，需读取 content 生成 .pyi（模块级缓存避免跨入口/跨 run 重复 open+read+convert）
|
||||
module_name: str = os.path.splitext(rel_path)[0].replace(os.sep, '.').replace('/', '.')
|
||||
sig_content: str = PythonToStubConverter.convert(content, module_name)
|
||||
sig_content: str = self._get_cached_pyi(src_path, module_name)
|
||||
with open(sig_path, 'w', encoding='utf-8', newline='\n') as f:
|
||||
f.write(sig_content)
|
||||
_SaveToGlobalCache(sha1, 'pyi', sig_path)
|
||||
_vlog().info(f" -> {sha1}.vpi (签名)")
|
||||
|
||||
# 提取 docstring 写入 {sha1}.doc.json，供 Phase2 跨模块 __doc__ 加载
|
||||
docs: dict[str, str] = self._extract_docstrings(content)
|
||||
# 提取 docstring 写入 {sha1}.doc.json，供 Phase2 跨模块 __doc__ 加载（模块级缓存复用）
|
||||
docs: dict[str, str] = self._get_cached_docstrings(src_path)
|
||||
doc_path: str = os.path.join(self.temp_dir, f"{sha1}.doc.json")
|
||||
try:
|
||||
with open(doc_path, 'w', encoding='utf-8', newline='\n') as f:
|
||||
|
||||
@@ -7,6 +7,14 @@ from typing import List, Dict
|
||||
from lib.core.SymbolUtils import IsListAnnotation, ExpandSoftKeywords
|
||||
|
||||
|
||||
# 模块级转换结果缓存：{ (PyContent, ModuleName): str }
|
||||
# convert 是纯函数（类级配置 IncludeAliasMap/VarExcludeList/MacroExcludeList 运行期不变），
|
||||
# 相同输入必然产生相同输出。结果字符串不可变，调用方仅将其写入文件或拼接出新字符串，
|
||||
# 缓存结果可安全复用（无需深拷贝）。Set* 配置方法会清空缓存以防配置变化导致结果过期。
|
||||
_ConvertResultCache: dict[tuple[str, str], str] = {}
|
||||
_ConvertCacheMaxSize: int = 4096
|
||||
|
||||
|
||||
class PythonToStubConverter:
|
||||
"""将 Python 文件转换为存根格式"""
|
||||
|
||||
@@ -23,16 +31,19 @@ class PythonToStubConverter:
|
||||
def SetIncludeAliasMap(cls, alias_map: Dict[str, str]) -> None:
|
||||
"""设置 __include 别名映射表"""
|
||||
cls.IncludeAliasMap = alias_map
|
||||
_ConvertResultCache.clear()
|
||||
|
||||
@classmethod
|
||||
def SetVarExcludeList(cls, exclude_list: List[str]) -> None:
|
||||
"""设置变量排除列表"""
|
||||
cls.VarExcludeList = exclude_list
|
||||
_ConvertResultCache.clear()
|
||||
|
||||
@classmethod
|
||||
def SetMacroExcludeList(cls, exclude_list: List[str]) -> None:
|
||||
"""设置宏排除列表"""
|
||||
cls.MacroExcludeList = exclude_list
|
||||
_ConvertResultCache.clear()
|
||||
|
||||
@classmethod
|
||||
def _ShouldExcludeVar(cls, VarName: str) -> bool:
|
||||
@@ -56,6 +67,13 @@ class PythonToStubConverter:
|
||||
# 软关键字展开：fn/fun/func/function -> def（行内等宽/更短，行号不变），
|
||||
# 保证后续 ast.parse 与 SourceLines 回读行号一致。
|
||||
PyContent = ExpandSoftKeywords(PyContent)
|
||||
|
||||
# 模块级缓存：同一 (内容, 模块名) 直接复用上次转换结果（纯函数输出，不可变字符串）
|
||||
CacheKey: tuple[str, str] = (PyContent, ModuleName)
|
||||
CachedResult: str | None = _ConvertResultCache.get(CacheKey)
|
||||
if CachedResult is not None:
|
||||
return CachedResult
|
||||
|
||||
lines: list[str] = PyContent.split('\n')
|
||||
StubLines: list[str] = []
|
||||
|
||||
@@ -66,7 +84,7 @@ class PythonToStubConverter:
|
||||
StubLines.append('"""')
|
||||
StubLines.append('')
|
||||
|
||||
# 解析 Python 代码，检查是否已经导入了 t 和 c
|
||||
# 解析 Python 代码，检查是否已经导入了 t 和 c（AST 与下方主解析复用，避免重复 parse）
|
||||
HasImportT: bool = False
|
||||
HasImportC: bool = False
|
||||
try:
|
||||
@@ -78,8 +96,12 @@ class PythonToStubConverter:
|
||||
HasImportT = True
|
||||
elif alias.name == 'c':
|
||||
HasImportC = True
|
||||
except:
|
||||
pass
|
||||
except SyntaxError:
|
||||
# 语法错误：同一内容两次解析结果必然相同，与主循环 fail-fast 行为一致
|
||||
raise
|
||||
except Exception:
|
||||
# 非语法类解析异常：保持原语义（第一遍吞掉，主循环重新解析后向上传播）
|
||||
tree = None
|
||||
|
||||
# 添加默认导入（如果源代码中没有）
|
||||
if not HasImportT:
|
||||
@@ -101,9 +123,11 @@ class PythonToStubConverter:
|
||||
#StubLines.append(f'{FileGuardName}: t.CDefine')
|
||||
#StubLines.append('')
|
||||
|
||||
# 解析 Python 代码，提取类型定义
|
||||
# 解析 Python 代码，提取类型定义（复用上方的 parse 结果）
|
||||
try:
|
||||
tree = ast.parse(PyContent)
|
||||
if tree is None:
|
||||
# 第一遍解析失败（非语法类异常）：重试一次，保持原"主循环重新解析"语义
|
||||
tree = ast.parse(PyContent)
|
||||
|
||||
# 第一遍扫描：收集 Postdefinition 变量
|
||||
PostdefVars: dict[str, list[tuple[str, str, ast.AnnAssign]]] = {} # {ClassName: [(VarName, TypeStr, node), ...]}
|
||||
@@ -218,7 +242,12 @@ class PythonToStubConverter:
|
||||
# undefined symbol / mangle 错误形式爆发（如 ExpandKeywordsTest 链接失败）。
|
||||
raise
|
||||
|
||||
return '\n'.join(StubLines)
|
||||
Result: str = '\n'.join(StubLines)
|
||||
# 缓存上限保护：条目过多时整体清空，避免内存无限增长（结果可随时重算）
|
||||
if len(_ConvertResultCache) >= _ConvertCacheMaxSize:
|
||||
_ConvertResultCache.clear()
|
||||
_ConvertResultCache[CacheKey] = Result
|
||||
return Result
|
||||
|
||||
@staticmethod
|
||||
def _ConvertClass(node: ast.ClassDef, SourceLines: list[str] | None = None, PostdefVars: list[tuple[str, str, ast.AnnAssign]] | None = None, indent: int = 0) -> list[str]:
|
||||
|
||||
Reference in new issue
Block a user