perf(compile): 依赖扫描缓存键 abspath 归一化 + compute_sha1 缓存
parse_python_file/get_file_dependencies 缓存键改用 abspath 归一化： 同一文件以相对/绝对、正斜杠/反斜杠/含 .. 等形式传入会产生多条缓存 条目导致重复 open+read+ast.parse（439 次 get_file_dependencies 25.2s、 1496 次 parse 33.1s 的浪费来源）。compute_sha1 加有界结果缓存。 验证：同文件 4 种路径形式缓存合并为 1 条，find_reachable 二次调用 显著加速，相对导入解析结果不变。
This commit is contained in:
TermiNexus committed
1 parent
5692e9b097
commit
3326c41702
1 file changed
+35
-13
+35
-13
@@ -11,8 +11,22 @@ from lib.constants.config import mode as _ConfigMode
|
||||
from lib.core.SymbolUtils import AnnotationContainsName, ExpandSoftKeywords
|
||||
|
||||
|
||||
# compute_sha1 结果缓存：同一内容字符串（如 parse_python_file 缓存中的 content）会被
|
||||
# Phase1/Phase2 多处重复哈希（Phase2 对每个源文件多次 compute_sha1(content)），
|
||||
# 缓存避免重复 utf-8 编码与 sha1 计算。纯函数哈希，缓存不改变语义；有界防内存膨胀。
|
||||
_SHA1_CACHE_LIMIT: int = 512
|
||||
_SHA1_CACHE: dict[str, str] = {}
|
||||
|
||||
|
||||
def compute_sha1(content: str) -> str:
|
||||
return hashlib.sha1(content.encode('utf-8')).hexdigest()[:16]
|
||||
cached: str | None = _SHA1_CACHE.get(content)
|
||||
if cached is not None:
|
||||
return cached
|
||||
digest: str = hashlib.sha1(content.encode('utf-8')).hexdigest()[:16]
|
||||
if len(_SHA1_CACHE) >= _SHA1_CACHE_LIMIT:
|
||||
_SHA1_CACHE.clear()
|
||||
_SHA1_CACHE[content] = digest
|
||||
return digest
|
||||
|
||||
|
||||
# get_file_dependencies 缓存：键 (filepath, src_root)，值 set[str]
|
||||
@@ -45,18 +59,21 @@ def parse_python_file(filepath: str) -> tuple[str, ast.Module] | tuple[str, None
|
||||
"""读取并解析 Python 文件，带 mtime 校验的 AST 缓存
|
||||
|
||||
多处调用共享同一文件的 parse 结果，避免重复 open+read+ast.parse。
|
||||
缓存键用 os.path.abspath 归一化：同一文件无论以相对/绝对、正斜杠/反斜杠、
|
||||
含 ../ 等形式传入都命中同一条目（否则同一文件会被重复 open+parse）。
|
||||
返回 (content, tree)；文件读取或解析失败返回 (content, None)。
|
||||
"""
|
||||
norm_path: str = os.path.abspath(filepath)
|
||||
try:
|
||||
st: os.stat_result = os.stat(filepath)
|
||||
st: os.stat_result = os.stat(norm_path)
|
||||
except OSError:
|
||||
return ('', None)
|
||||
mtime: float = st.st_mtime
|
||||
cached: tuple[float, str, ast.Module] | None = _FILE_AST_CACHE.get(filepath)
|
||||
cached: tuple[float, str, ast.Module] | None = _FILE_AST_CACHE.get(norm_path)
|
||||
if cached is not None and cached[0] == mtime:
|
||||
return (cached[1], cached[2])
|
||||
try:
|
||||
with open(filepath, 'r', encoding='utf-8') as f:
|
||||
with open(norm_path, 'r', encoding='utf-8') as f:
|
||||
content: str = f.read()
|
||||
except Exception:
|
||||
return ('', None)
|
||||
@@ -67,22 +84,27 @@ def parse_python_file(filepath: str) -> tuple[str, ast.Module] | tuple[str, None
|
||||
tree: ast.Module = ast.parse(ExpandSoftKeywords(content))
|
||||
except SyntaxError:
|
||||
return (content, None)
|
||||
_FILE_AST_CACHE[filepath] = (mtime, content, tree)
|
||||
_FILE_AST_CACHE[norm_path] = (mtime, content, tree)
|
||||
return (content, tree)
|
||||
|
||||
|
||||
def get_file_dependencies(filepath: str, src_root: str) -> set[str]:
|
||||
"""解析 Python 文件的导入依赖（带缓存）
|
||||
|
||||
单次编译中 find_reachable_files_from_entries 和 topological_sort_files
|
||||
会对同一批文件重复调用，缓存避免重复 ast.parse（~8.6s→~4.3s）。
|
||||
缓存键 (abspath(filepath), abspath(src_root))：src_root 与文件路径都用
|
||||
os.path.abspath 归一化后，find_reachable_files_from_entries / topological_sort_files
|
||||
/ Phase1 各调用点即使传入不同形式（相对路径、正斜杠、含 ../）也命中同一条目，
|
||||
避免同一文件被重复 open+read+ast.parse。归一化不改变解析结果
|
||||
（os.path.relpath 内部本就按目录身份归一化，结果与 src_root 的书写形式无关）。
|
||||
"""
|
||||
cache_key: tuple[str, str] = (filepath, src_root)
|
||||
norm_path: str = os.path.abspath(filepath)
|
||||
norm_root: str = os.path.abspath(src_root)
|
||||
cache_key: tuple[str, str] = (norm_path, norm_root)
|
||||
cached: set[str] | None = _FILE_DEPS_CACHE.get(cache_key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
dependencies: set[str] = set()
|
||||
content, tree = parse_python_file(filepath)
|
||||
content, tree = parse_python_file(norm_path)
|
||||
if tree is not None:
|
||||
try:
|
||||
for node in ast.walk(tree):
|
||||
@@ -95,8 +117,8 @@ def get_file_dependencies(filepath: str, src_root: str) -> set[str]:
|
||||
module: str = node.module
|
||||
if node.level > 0:
|
||||
# from .module import name → 依赖 pkg.module（模块本身）
|
||||
pkg_dir: str = os.path.dirname(filepath)
|
||||
pkg_rel: str = os.path.relpath(pkg_dir, src_root).replace(os.sep, '.').replace('/', '.')
|
||||
pkg_dir: str = os.path.dirname(norm_path)
|
||||
pkg_rel: str = os.path.relpath(pkg_dir, norm_root).replace(os.sep, '.').replace('/', '.')
|
||||
if pkg_rel == '.':
|
||||
pkg_rel = ''
|
||||
dep_module: str
|
||||
@@ -111,8 +133,8 @@ def get_file_dependencies(filepath: str, src_root: str) -> set[str]:
|
||||
elif node.level > 0 and node.names:
|
||||
# from . import name → 依赖 pkg.name（子模块）
|
||||
for alias in node.names:
|
||||
pkg_dir: str = os.path.dirname(filepath)
|
||||
pkg_rel: str = os.path.relpath(pkg_dir, src_root).replace(os.sep, '.').replace('/', '.')
|
||||
pkg_dir: str = os.path.dirname(norm_path)
|
||||
pkg_rel: str = os.path.relpath(pkg_dir, norm_root).replace(os.sep, '.').replace('/', '.')
|
||||
if pkg_rel == '.':
|
||||
pkg_rel = ''
|
||||
dep_module: str = f'{pkg_rel}.{alias.name}' if pkg_rel else alias.name
|
||||
|
||||
Reference in new issue
Block a user