perf(tpc): PROF 热点优化——vast 节点 _Init unroll 特化 + isfile 结果缓存
- vast/Nodes.py：_MakeNodeClass 动态生成 __init__，全位置参数路径按字段 索引直赋（unroll，去 zip 迭代器），语义与通用路径完全等价 （PROF：_Init 158 万次 29s 最大纯 Python 热点，实测 ~1.4x） - Phase2Translator.py：新增模块级 _CachedIsFile（编译进程内文件状态稳定）， 替换全部 os.path.isfile 存在性检查（PROF：2.6 万次 16.4s）
This commit is contained in:
TermiNexus committed
1 parent
b7b17e0e7b
commit
567d925235
2 files changed
+79
-45
No files matched your search
@@ -23,6 +23,20 @@ import TransPyC
|
||||
from StubGen import PythonToStubConverter
|
||||
|
||||
|
||||
# PROF 热点：nt._path_isfile 2.6 万次 16.4s（Windows 磁盘 stat ~0.6ms/次）。
|
||||
# 编译进程内文件状态稳定（obj/ll 一旦生成不删除、不被移动），用模块级缓存
|
||||
# 避免重复 stat。仅用于"存在性检查"类调用（不用于内容变化的判断）。
|
||||
_FILE_STAT_CACHE: dict[str, bool] = {}
|
||||
|
||||
|
||||
def _CachedIsFile(path: str) -> bool:
|
||||
r: bool | None = _FILE_STAT_CACHE.get(path)
|
||||
if r is None:
|
||||
r = os.path.isfile(path)
|
||||
_FILE_STAT_CACHE[path] = r
|
||||
return r
|
||||
|
||||
|
||||
def _PickWorkerCount(n_files: int, per_worker_mb: int = 700) -> int:
|
||||
"""按可用物理内存动态限制并行 worker 数，避免多进程编译内存峰值 OOM。
|
||||
|
||||
@@ -364,7 +378,7 @@ def _BuildPyiCacheFile(temp_dir: str) -> str | None:
|
||||
pkl_path: str = os.path.join(temp_dir, _PYI_CACHE_PKL_NAME)
|
||||
sig_path: str = os.path.join(temp_dir, _PYI_CACHE_SIG_NAME)
|
||||
try:
|
||||
if os.path.isfile(pkl_path) and os.path.isfile(sig_path):
|
||||
if _CachedIsFile(pkl_path) and _CachedIsFile(sig_path):
|
||||
with open(sig_path, 'r', encoding='utf-8') as f:
|
||||
stored = json.load(f)
|
||||
# json 往返后 tuple→list，需规范化再比较
|
||||
@@ -743,7 +757,7 @@ class Phase2Translator:
|
||||
raise
|
||||
_vlog().warning(f"收集源文件导入的 include 模块失败: {_e}", "Exception")
|
||||
|
||||
if os.path.isfile(out_path):
|
||||
if _CachedIsFile(out_path):
|
||||
deps_changed = self._check_deps_changed(sha1, current_ModuleSha1Map)
|
||||
if not deps_changed:
|
||||
_vlog().info(f"[{i}/{len(py_files)}] 跳过(缓存): {rel} -> {sha1}.ll")
|
||||
@@ -757,7 +771,7 @@ class Phase2Translator:
|
||||
else:
|
||||
stub_cache = os.path.join(self.output_dir, f"{sha1}.stub.ll")
|
||||
text_cache = os.path.join(self.output_dir, f"{sha1}.text.ll")
|
||||
if os.path.isfile(stub_cache) and os.path.isfile(text_cache):
|
||||
if _CachedIsFile(stub_cache) and _CachedIsFile(text_cache):
|
||||
if self._recombine_ll(sha1, current_ModuleSha1Map):
|
||||
_vlog().info(f"[{i}/{len(py_files)}] 重组(缓存恢复): {rel} -> {sha1}.ll")
|
||||
continue
|
||||
@@ -942,14 +956,14 @@ class Phase2Translator:
|
||||
# 存在性/内容缓存：编译期 include 源文件不变，命中跳过磁盘 I/O
|
||||
exists = Phase2Translator._include_exist_cache.get(py_path)
|
||||
if exists is None:
|
||||
exists = os.path.isfile(py_path)
|
||||
exists = _CachedIsFile(py_path)
|
||||
Phase2Translator._include_exist_cache[py_path] = exists
|
||||
if not exists:
|
||||
# 回退到 .py（向后兼容）
|
||||
py_path = os.path.splitext(py_path)[0] + '.py'
|
||||
exists = Phase2Translator._include_exist_cache.get(py_path)
|
||||
if exists is None:
|
||||
exists = os.path.isfile(py_path)
|
||||
exists = _CachedIsFile(py_path)
|
||||
Phase2Translator._include_exist_cache[py_path] = exists
|
||||
if exists:
|
||||
py_sha1 = Phase2Translator._include_sha1_cache.get(py_path)
|
||||
@@ -1196,7 +1210,7 @@ class Phase2Translator:
|
||||
def _LoadDeps(self, sha1: str) -> dict[str, str] | None:
|
||||
"""加载依赖指纹"""
|
||||
deps_path = os.path.join(self.output_dir, f"{sha1}.deps.json")
|
||||
if os.path.isfile(deps_path):
|
||||
if _CachedIsFile(deps_path):
|
||||
try:
|
||||
with open(deps_path, 'r', encoding='utf-8') as f:
|
||||
return json.load(f)
|
||||
@@ -1229,7 +1243,7 @@ class Phase2Translator:
|
||||
text_path = os.path.join(self.output_dir, f"{sha1}.text.ll")
|
||||
ll_path = os.path.join(self.output_dir, f"{sha1}.ll")
|
||||
|
||||
if not os.path.isfile(stub_path) or not os.path.isfile(text_path):
|
||||
if not _CachedIsFile(stub_path) or not _CachedIsFile(text_path):
|
||||
return False
|
||||
|
||||
with open(stub_path, 'r', encoding='utf-8') as f:
|
||||
@@ -1263,7 +1277,7 @@ class Phase2Translator:
|
||||
self._save_deps(sha1, current_ModuleSha1Map)
|
||||
|
||||
obj_path = os.path.join(self.output_dir, f"{sha1}.obj")
|
||||
if os.path.isfile(obj_path):
|
||||
if _CachedIsFile(obj_path):
|
||||
os.remove(obj_path)
|
||||
|
||||
return True
|
||||
@@ -1670,7 +1684,7 @@ class Phase2Translator:
|
||||
if sha1 in active_sha1s:
|
||||
fpath = os.path.join(root, file)
|
||||
obj_path = os.path.join(root, file[:-3] + '.obj')
|
||||
if not os.path.isfile(obj_path):
|
||||
if not _CachedIsFile(obj_path):
|
||||
ll_files.append(fpath)
|
||||
if not ll_files:
|
||||
_vlog().info("[编译] 无 .ll 文件需要重新编译")
|
||||
@@ -1763,7 +1777,7 @@ class Phase2Translator:
|
||||
for ll_path in ll_files:
|
||||
obj_path = ll_path[:-3] + '.obj'
|
||||
sha1_obj = os.path.join(self.output_dir, os.path.basename(ll_path)[:-3] + '.obj')
|
||||
if os.path.isfile(sha1_obj):
|
||||
if _CachedIsFile(sha1_obj):
|
||||
os.remove(sha1_obj)
|
||||
return
|
||||
else:
|
||||
@@ -1781,7 +1795,7 @@ class Phase2Translator:
|
||||
def _compile_one(ll_path: str) -> tuple[str, str, str]:
|
||||
rel = os.path.relpath(ll_path, self.output_dir)
|
||||
obj_path = ll_path[:-3] + '.obj'
|
||||
if os.path.isfile(obj_path):
|
||||
if _CachedIsFile(obj_path):
|
||||
return (rel, 'cached', '')
|
||||
try:
|
||||
with open(ll_path, 'r', encoding='utf-8') as f:
|
||||
@@ -1862,9 +1876,9 @@ class Phase2Translator:
|
||||
continue
|
||||
seen_stub_sha1.add(dep_sha1)
|
||||
stub_path = os.path.join(self.output_dir, f"{dep_sha1}.stub.ll")
|
||||
if not os.path.isfile(stub_path):
|
||||
if not _CachedIsFile(stub_path):
|
||||
stub_path = os.path.join(self.temp_dir, f"{dep_sha1}.stub.ll")
|
||||
if os.path.isfile(stub_path):
|
||||
if _CachedIsFile(stub_path):
|
||||
with open(stub_path, 'r', encoding='utf-8') as sf:
|
||||
stub_content = sf.read()
|
||||
for line in stub_content.splitlines():
|
||||
@@ -2052,7 +2066,7 @@ class Phase2Translator:
|
||||
need_compile = []
|
||||
for ll_path in ll_files:
|
||||
obj_path = ll_path[:-3] + '.obj'
|
||||
if not os.path.isfile(obj_path):
|
||||
if not _CachedIsFile(obj_path):
|
||||
need_compile.append(ll_path)
|
||||
|
||||
if need_compile:
|
||||
@@ -2174,13 +2188,13 @@ class Phase2Translator:
|
||||
|
||||
for ext in self.INCLUDE_LIB_EXTENSIONS:
|
||||
lib_path = os.path.join(includes_dir, module_name + ext)
|
||||
if os.path.isfile(lib_path):
|
||||
if _CachedIsFile(lib_path):
|
||||
self.extra_link_files.append(lib_path)
|
||||
_vlog().info(f" [includes] 发现库文件: {os.path.relpath(lib_path, includes_dir)}")
|
||||
|
||||
for ext in self.INCLUDE_SRC_EXTENSIONS:
|
||||
src_path = os.path.join(includes_dir, module_name + ext)
|
||||
if os.path.isfile(src_path):
|
||||
if _CachedIsFile(src_path):
|
||||
self.extra_compile_files.append(src_path)
|
||||
_vlog().info(f" [includes] 发现源文件: {os.path.relpath(src_path, includes_dir)}")
|
||||
|
||||
@@ -2264,7 +2278,7 @@ class Phase2Translator:
|
||||
if py_index is not None:
|
||||
exists = os.path.normcase(py_path) in py_index
|
||||
else:
|
||||
exists = os.path.isfile(py_path)
|
||||
exists = _CachedIsFile(py_path)
|
||||
if exists and py_path not in self.extra_py_files:
|
||||
self.extra_py_files.append(py_path)
|
||||
_vlog().info(f" [includes] 发现间接依赖: {os.path.relpath(py_path, includes_dir)}")
|
||||
@@ -2505,7 +2519,7 @@ class Phase2Translator:
|
||||
continue
|
||||
|
||||
# 优先检查 includes.binary 预编译缓存（跨项目共享）
|
||||
if not self.force_recompile_includes and precompiled_obj and os.path.isfile(precompiled_obj):
|
||||
if not self.force_recompile_includes and precompiled_obj and _CachedIsFile(precompiled_obj):
|
||||
_vlog().info(f" 跳过(预编译): {module_name}.py -> {os.path.basename(precompiled_obj)}")
|
||||
# 预编译跳过的文件仍需注册 .pyi/.stub.ll 声明和符号，
|
||||
# 否则依赖它的 include 文件（如 vpui.py import vpsdk.window）
|
||||
@@ -2515,7 +2529,7 @@ class Phase2Translator:
|
||||
self.extra_link_files.append(precompiled_obj)
|
||||
continue
|
||||
|
||||
if not self.force_recompile_includes and os.path.isfile(obj_path):
|
||||
if not self.force_recompile_includes and _CachedIsFile(obj_path):
|
||||
_vlog().info(f" 跳过(缓存): {module_name}.py -> {sha1}.obj")
|
||||
self._register_compiled_include(src_path, sha1, None, include_ModuleSha1Map)
|
||||
self._collect_inline_symbols(src_path)
|
||||
@@ -2699,7 +2713,7 @@ class Phase2Translator:
|
||||
compile_tasks = [] # [(dso_ll_path, obj_path, src_path, sha1, module_name, ll_path, precompiled_obj, ll_content)]
|
||||
for item in need_translate:
|
||||
src_path, sha1, module_name, ll_path, obj_path, precompiled_obj = item
|
||||
if not os.path.isfile(ll_path):
|
||||
if not _CachedIsFile(ll_path):
|
||||
_vlog().error(f" 翻译未生成 .ll 文件: {module_name}.py")
|
||||
continue
|
||||
|
||||
@@ -2873,7 +2887,7 @@ class Phase2Translator:
|
||||
# DEBUG: 保留 .ll 文件以便静态分析
|
||||
# for ext in ['.ll', '_dso.ll']:
|
||||
# p = ll_path[:-3] + ext
|
||||
# if os.path.isfile(p):
|
||||
# if _CachedIsFile(p):
|
||||
# os.remove(p)
|
||||
else:
|
||||
has_error = True
|
||||
@@ -3007,10 +3021,10 @@ class Phase2Translator:
|
||||
|
||||
# 1. 生成 .pyi 签名文件（如果还没有的话）
|
||||
sig_path = os.path.join(self.temp_dir, f"{sha1}.vpi")
|
||||
if not os.path.isfile(sig_path):
|
||||
if not _CachedIsFile(sig_path):
|
||||
# 全局缓存检查（不被 --clean 清除）
|
||||
_TryGlobalCache(sha1, 'pyi', sig_path)
|
||||
if not os.path.isfile(sig_path):
|
||||
if not _CachedIsFile(sig_path):
|
||||
try:
|
||||
with open(src_path, 'r', encoding='utf-8') as f:
|
||||
py_content = f.read()
|
||||
@@ -3025,7 +3039,7 @@ class Phase2Translator:
|
||||
_vlog().warning(f"签名生成失败不阻塞编译: {e}", "Exception")
|
||||
|
||||
# 注册到 sig_files 和 _source_module_sig_files
|
||||
if os.path.isfile(sig_path):
|
||||
if _CachedIsFile(sig_path):
|
||||
self.sig_files[sha1] = sig_path
|
||||
self.sha1_map[sha1] = f"includes/{mod_full.replace('.', '/')}.vp"
|
||||
if self._shared_source_module_sig_files is not None:
|
||||
@@ -3043,10 +3057,10 @@ class Phase2Translator:
|
||||
|
||||
# 2. 生成 .stub.ll 文件
|
||||
stub_path = os.path.join(self.output_dir, f"{sha1}.stub.ll")
|
||||
if not os.path.isfile(stub_path):
|
||||
if not _CachedIsFile(stub_path):
|
||||
# 全局缓存检查（不被 --clean 清除）
|
||||
_TryGlobalCache(sha1, 'stub.ll', stub_path)
|
||||
if not os.path.isfile(stub_path) and ll_content:
|
||||
if not _CachedIsFile(stub_path) and ll_content:
|
||||
try:
|
||||
stub_content, _ = self._split_ll(ll_content)
|
||||
with open(stub_path, 'w', encoding='utf-8', newline='\n') as f:
|
||||
@@ -3058,17 +3072,17 @@ class Phase2Translator:
|
||||
_vlog().warning(f"生成 .stub.ll 文件失败: {_e}", "Exception")
|
||||
|
||||
# 注册到 stub_files
|
||||
if os.path.isfile(stub_path):
|
||||
if _CachedIsFile(stub_path):
|
||||
self.stub_files[sha1] = stub_path
|
||||
elif os.path.isfile(sig_path):
|
||||
elif _CachedIsFile(sig_path):
|
||||
# 如果 stub 文件不存在，也尝试从 temp 目录查找
|
||||
temp_stub = os.path.join(self.temp_dir, f"{sha1}.stub.ll")
|
||||
if os.path.isfile(temp_stub):
|
||||
if _CachedIsFile(temp_stub):
|
||||
self.stub_files[sha1] = temp_stub
|
||||
else:
|
||||
# 最后尝试从全局缓存直接读取
|
||||
_TryGlobalCache(sha1, 'stub.ll', temp_stub)
|
||||
if os.path.isfile(temp_stub):
|
||||
if _CachedIsFile(temp_stub):
|
||||
self.stub_files[sha1] = temp_stub
|
||||
|
||||
def _precollect_inline_symbols(self) -> None:
|
||||
@@ -3088,10 +3102,10 @@ class Phase2Translator:
|
||||
src_path = os.path.join(root, fname)
|
||||
self._collect_inline_symbols(src_path)
|
||||
py_path = os.path.join(includes_dir, module_name + '.vp')
|
||||
if not os.path.isfile(py_path):
|
||||
if not _CachedIsFile(py_path):
|
||||
# 回退到 .py（向后兼容）
|
||||
py_path = os.path.join(includes_dir, module_name + '.py')
|
||||
if os.path.isfile(py_path):
|
||||
if _CachedIsFile(py_path):
|
||||
self._collect_inline_symbols(py_path)
|
||||
for ef in self.extra_py_files:
|
||||
self._collect_inline_symbols(ef)
|
||||
@@ -3404,7 +3418,7 @@ class Phase2Translator:
|
||||
"""将 active_sha1s 对应的 .obj 文件和 includes 库文件链接为可执行文件"""
|
||||
obj_files = []
|
||||
merged_obj = os.path.join(self.output_dir, '_merged.obj')
|
||||
if os.path.isfile(merged_obj):
|
||||
if _CachedIsFile(merged_obj):
|
||||
obj_files.append(merged_obj)
|
||||
else:
|
||||
for root, dirs, files in os.walk(self.output_dir):
|
||||
@@ -3446,10 +3460,10 @@ class Phase2Translator:
|
||||
# 自动复制或生成 linker.ld 到输出目录
|
||||
linker_ld_src = os.path.join(os.path.dirname(self.output_dir), 'linker.ld')
|
||||
linker_ld_dst = os.path.join(self.output_dir, 'linker.ld')
|
||||
if os.path.isfile(linker_ld_src) and not os.path.isfile(linker_ld_dst):
|
||||
if _CachedIsFile(linker_ld_src) and not _CachedIsFile(linker_ld_dst):
|
||||
shutil.copy2(linker_ld_src, linker_ld_dst)
|
||||
_vlog().info(f" [链接] 使用链接脚本: linker.ld")
|
||||
elif not os.path.isfile(linker_ld_dst) and any('-T' in f or '-T' in f.replace(',', ' ') for f in self.linker_flags):
|
||||
elif not _CachedIsFile(linker_ld_dst) and any('-T' in f or '-T' in f.replace(',', ' ') for f in self.linker_flags):
|
||||
default_ld = """ENTRY(_start)
|
||||
|
||||
SECTIONS {
|
||||
|
||||
+30
-10
@@ -311,22 +311,42 @@ _BASE_CLASS: dict[str, str] = {
|
||||
def _MakeNodeClass(name: str, fields: tuple[str, ...], attrs: tuple[str, ...]) -> type:
|
||||
"""按字段表生成节点类（继承对应抽象基类）。"""
|
||||
|
||||
def _Init(self: AST, *args: object, **kwargs: object) -> None:
|
||||
if len(args) > len(fields):
|
||||
raise TypeError(
|
||||
f'{name} constructor takes at most {len(fields)} positional arguments'
|
||||
)
|
||||
for fname, val in zip(fields, args):
|
||||
object.__setattr__(self, fname, val)
|
||||
for k, v in kwargs.items():
|
||||
object.__setattr__(self, k, v)
|
||||
# 特化 __init__：常见路径（全位置参数，vast_c 回退 Vectorcall 的主形态）按
|
||||
# 字段索引直赋（unroll），避免 zip 迭代器 + 循环控制开销（PROF：_Init
|
||||
# 158 万次 29s 是最大纯 Python 热点）。语义与通用路径完全等价：
|
||||
# 位置参数按序设置前 n 个字段，kwargs 后设覆盖；越界同样抛 TypeError。
|
||||
def _MakeInit(name: str, fields: tuple[str, ...]) -> object:
|
||||
n = len(fields)
|
||||
lines: list[str] = [
|
||||
"def _Init(self: AST, *args: object, **kwargs: object) -> None:",
|
||||
" _set = object.__setattr__",
|
||||
" _n = len(args)",
|
||||
f" if _n > {n}:",
|
||||
f" raise TypeError(f'{name} constructor takes at most {n} positional arguments')",
|
||||
]
|
||||
if n > 0:
|
||||
lines.append(" if _n:")
|
||||
for i in range(1, min(n, 8) + 1):
|
||||
assigns = "; ".join(
|
||||
f"_set(self, {fields[j]!r}, args[{j}])" for j in range(i))
|
||||
lines.append(f" if _n == {i}:")
|
||||
lines.append(f" {assigns}")
|
||||
if n > 8:
|
||||
lines.append(" else:")
|
||||
lines.append(" for _i in range(_n):")
|
||||
lines.append(" _set(self, _F[_i], args[_i])")
|
||||
lines.append(" for _k, _v in kwargs.items():")
|
||||
lines.append(" _set(self, _k, _v)")
|
||||
ns: dict[str, object] = {"AST": AST, "_F": fields}
|
||||
exec("\n".join(lines), ns)
|
||||
return ns["_Init"]
|
||||
|
||||
base_name: str | None = _BASE_CLASS.get(name)
|
||||
base_cls: type = globals()[base_name] if base_name else AST
|
||||
# __slots__ = 位置字段 + 位置属性（实例的全部属性名）
|
||||
slots: tuple[str, ...] = tuple(dict.fromkeys(fields + attrs))
|
||||
ns: dict[str, object] = {
|
||||
'__init__': _Init,
|
||||
'__init__': _MakeInit(name, fields),
|
||||
'__slots__': slots,
|
||||
'_fields': fields,
|
||||
'_attributes': attrs,
|
||||
|
||||
Reference in new issue
Block a user