perf(handles): 跨模块类声明分类/节点查找缓存，去重 _EmitExternalClassDeclLlvm
新增类级 _extern_class_classify_cache（(sha1,class)->分类元组）与 _extern_class_decl_cache（(sha1,class)->pyi ClassDef 节点）：bases/ decorator 分类遍历提取为 _ClassifyExternalClass 纯函数并缓存， _TryLoadClassMembersFromPyi 节点查找改走索引避免跨任务重复线性扫描 pyi_tree.body（8214 次调用热点之一）。Gen 依赖部分（字段/方法/struct 注册）保持每 Gen 执行，不做整体跳过。
This commit is contained in:
TermiNexus committed
1 parent
b760082be8
commit
68d180e718
1 file changed
+124
-81
@@ -27,6 +27,14 @@ class ImportHandle(BaseHandle):
|
||||
# （原 609 次调用 × ~100 stub 文件 × ~50 行 = 5.5M 正则匹配 → 一次扫描 ~5000 匹配）
|
||||
_stub_index: dict[str, list[tuple[str, str, str, bool]]] = {}
|
||||
_stub_index_signature: tuple[int, int] | None = None # (output_stub_count, temp_stub_count)
|
||||
# 类级缓存：跨 worker 任务共享（同一进程内，worker 单线程无竞争）。
|
||||
# 键均为 (source_sha1, class_name)，不同文件同名类（不同 sha1）不冲突。
|
||||
# classify：_EmitExternalClassDeclLlvm 开头的 bases/decorator_list 分类遍历结果
|
||||
# （与 Gen 无关的纯函数计算，同一 (sha1, class) 跨任务重复调用时跳过遍历）。
|
||||
# decl：pyi 树中的 ClassDef 节点引用（pyi 树索引，pyi 树由 _pyi_cache 类级缓存
|
||||
# 保证进程内唯一），避免 _TryLoadClassMembersFromPyi 跨任务反复线性扫描 body。
|
||||
_extern_class_classify_cache: dict[tuple[str, str], tuple] = {}
|
||||
_extern_class_decl_cache: dict[tuple[str, str], ast.ClassDef] = {}
|
||||
|
||||
def __init__(self, translator: "Translator") -> None:
|
||||
super().__init__(translator)
|
||||
@@ -1370,78 +1378,111 @@ class ImportHandle(BaseHandle):
|
||||
if ReexportName2 not in Gen.functions:
|
||||
Gen.functions[ReexportName2] = func
|
||||
|
||||
def _ClassifyExternalClass(self, Node: ast.ClassDef) -> tuple:
|
||||
"""遍历 Node.bases/decorator_list 做类别判断（纯函数、与 Gen 无关）。
|
||||
|
||||
结果由 _EmitExternalClassDeclLlvm 按 (source_sha1, class_name) 类级缓存，
|
||||
同一 worker 进程内多个任务对同一跨模块类重复调用时跳过本遍历。
|
||||
返回 (IsGenericClass, IsCenum, IsRenum, IsCpythonObject, IsCVTable,
|
||||
IsNoVTable, IsPacked)。
|
||||
|
||||
PEP 695 泛型类（如 GSListNode[T]）特殊：跨模块继承展平需要从泛型基类
|
||||
获取字段（如 Next），否则子类（如 Value(GSListNode[Value])）的
|
||||
class_members 会缺少继承字段；但泛型类的方法声明必须跳过（方法参数
|
||||
类型 T 会被解析为 opaque struct，LLVM 不允许 opaque struct 作为值参数）。
|
||||
"""
|
||||
IsGenericClass: bool = hasattr(Node, 'type_params') and bool(Node.type_params)
|
||||
IsCenum = False
|
||||
IsRenum = False
|
||||
IsCpythonObject = False
|
||||
IsCVTable = False
|
||||
IsNoVTable = False
|
||||
if Node.bases:
|
||||
for base in Node.bases:
|
||||
if getattr(base, 'attr', None):
|
||||
if base.attr == 'REnum':
|
||||
IsRenum = True
|
||||
break
|
||||
elif base.attr == 'CEnum' or base.attr == 'Enum':
|
||||
IsCenum = True
|
||||
break
|
||||
elif getattr(base, 'id', None):
|
||||
if base.id == 'REnum':
|
||||
IsRenum = True
|
||||
break
|
||||
elif base.id == 'CEnum' or base.id == 'Enum':
|
||||
IsCenum = True
|
||||
break
|
||||
if not IsCenum and not IsRenum and getattr(Node, 'decorator_list', None):
|
||||
for decorator in Node.decorator_list:
|
||||
if isinstance(decorator, ast.Attribute):
|
||||
if getattr(decorator.value, 'id', None) == 't':
|
||||
if decorator.attr == 'Object':
|
||||
IsCpythonObject = True
|
||||
elif decorator.attr == 'CVTable':
|
||||
IsCVTable = True
|
||||
elif decorator.attr == 'NoVTable':
|
||||
IsNoVTable = True
|
||||
elif decorator.attr == 'REnum':
|
||||
IsRenum = True
|
||||
elif decorator.attr == 'CEnum' or decorator.attr == 'Enum':
|
||||
IsCenum = True
|
||||
elif isinstance(decorator, ast.Call):
|
||||
if isinstance(decorator.func, ast.Attribute):
|
||||
if getattr(decorator.func.value, 'id', None) == 't':
|
||||
if decorator.func.attr == 'Object':
|
||||
IsCpythonObject = True
|
||||
elif decorator.func.attr == 'CVTable':
|
||||
IsCVTable = True
|
||||
elif decorator.func.attr == 'NoVTable':
|
||||
IsNoVTable = True
|
||||
elif isinstance(decorator.func, ast.Name):
|
||||
if decorator.func.id == 'Object':
|
||||
IsCpythonObject = True
|
||||
elif decorator.func.id == 'CVTable':
|
||||
IsCVTable = True
|
||||
elif decorator.func.id == 'NoVTable':
|
||||
IsNoVTable = True
|
||||
IsPacked = False
|
||||
if getattr(Node, 'decorator_list', None):
|
||||
for decorator in Node.decorator_list:
|
||||
if isinstance(decorator, ast.Call):
|
||||
if isinstance(decorator.func, ast.Attribute):
|
||||
if getattr(decorator.func.value, 'id', None) == 'c' and decorator.func.attr == 'Attribute':
|
||||
for arg in decorator.args:
|
||||
if isinstance(arg, ast.Attribute):
|
||||
if isinstance(arg.value, ast.Attribute):
|
||||
if getattr(arg.value.value, 'id', None) == 't' and arg.value.attr == 'attr' and arg.attr == 'packed':
|
||||
IsPacked = True
|
||||
return (IsGenericClass, IsCenum, IsRenum, IsCpythonObject,
|
||||
IsCVTable, IsNoVTable, IsPacked)
|
||||
|
||||
def _EmitExternalClassDeclLlvm(self, Node: ast.ClassDef, Gen: LlvmGeneratorMixin, module_name: str | None = None, actual_module_name: str | None = None, source_sha1: str | None = None) -> None:
|
||||
ClassName = Node.name
|
||||
# PEP 695 泛型类（如 GSListNode[T]）不再完全跳过：跨模块继承展平需要从泛型基类
|
||||
# 获取字段（如 Next），否则子类（如 Value(GSListNode[Value])）的 class_members
|
||||
# 会缺少继承字段，导致 struct body 字段数不正确 → GEP 索引错位 → 崩溃。
|
||||
# 但泛型类的方法声明必须跳过：方法参数类型 T 会被解析为 opaque struct，
|
||||
# LLVM 不允许 opaque struct 作为值参数（invalid type for function argument）。
|
||||
IsGenericClass: bool = hasattr(Node, 'type_params') and bool(Node.type_params)
|
||||
IsCenum = False
|
||||
IsRenum = False
|
||||
IsCpythonObject = False
|
||||
IsCVTable = False
|
||||
IsNoVTable = False
|
||||
if Node.bases:
|
||||
for base in Node.bases:
|
||||
if getattr(base, 'attr', None):
|
||||
if base.attr == 'REnum':
|
||||
IsRenum = True
|
||||
break
|
||||
elif base.attr == 'CEnum' or base.attr == 'Enum':
|
||||
IsCenum = True
|
||||
break
|
||||
elif getattr(base, 'id', None):
|
||||
if base.id == 'REnum':
|
||||
IsRenum = True
|
||||
break
|
||||
elif base.id == 'CEnum' or base.id == 'Enum':
|
||||
IsCenum = True
|
||||
break
|
||||
if not IsCenum and not IsRenum and getattr(Node, 'decorator_list', None):
|
||||
for decorator in Node.decorator_list:
|
||||
if isinstance(decorator, ast.Attribute):
|
||||
if getattr(decorator.value, 'id', None) == 't':
|
||||
if decorator.attr == 'Object':
|
||||
IsCpythonObject = True
|
||||
elif decorator.attr == 'CVTable':
|
||||
IsCVTable = True
|
||||
elif decorator.attr == 'NoVTable':
|
||||
IsNoVTable = True
|
||||
elif decorator.attr == 'REnum':
|
||||
IsRenum = True
|
||||
elif decorator.attr == 'CEnum' or decorator.attr == 'Enum':
|
||||
IsCenum = True
|
||||
elif isinstance(decorator, ast.Call):
|
||||
if isinstance(decorator.func, ast.Attribute):
|
||||
if getattr(decorator.func.value, 'id', None) == 't':
|
||||
if decorator.func.attr == 'Object':
|
||||
IsCpythonObject = True
|
||||
elif decorator.func.attr == 'CVTable':
|
||||
IsCVTable = True
|
||||
elif decorator.func.attr == 'NoVTable':
|
||||
IsNoVTable = True
|
||||
elif isinstance(decorator.func, ast.Name):
|
||||
if decorator.func.id == 'Object':
|
||||
IsCpythonObject = True
|
||||
elif decorator.func.id == 'CVTable':
|
||||
IsCVTable = True
|
||||
elif decorator.func.id == 'NoVTable':
|
||||
IsNoVTable = True
|
||||
# Compute source_sha1 up front (moved from mid-function) so it can serve as
|
||||
# the key of the class-level classification cache. Prefer the explicit
|
||||
# source_sha1 passed by _TryLoadClassMembersFromPyi (module_name lookup may
|
||||
# fail when the current module does not import the defining module directly).
|
||||
if not source_sha1:
|
||||
if actual_module_name and hasattr(Gen, 'ModuleSha1Map') and actual_module_name in Gen.ModuleSha1Map:
|
||||
source_sha1 = Gen.ModuleSha1Map[actual_module_name]
|
||||
elif module_name and hasattr(Gen, 'ModuleSha1Map') and module_name in Gen.ModuleSha1Map:
|
||||
source_sha1 = Gen.ModuleSha1Map[module_name]
|
||||
# Class-level classification cache: the bases/decorator_list walk below is pure
|
||||
# Gen-independent computation. Repeated calls on the same (source_sha1, class_name)
|
||||
# across worker tasks (same process) skip the walk. When source_sha1 is None
|
||||
# (lookup failed) neither read nor write the cache, to avoid mismatching
|
||||
# same-named classes from different modules under a shared None key.
|
||||
_classify_key = (source_sha1, ClassName) if source_sha1 else None
|
||||
_cached_classify = self._extern_class_classify_cache.get(_classify_key) if _classify_key else None
|
||||
if _cached_classify is None:
|
||||
_cached_classify = self._ClassifyExternalClass(Node)
|
||||
if _classify_key is not None:
|
||||
self._extern_class_classify_cache[_classify_key] = _cached_classify
|
||||
(IsGenericClass, IsCenum, IsRenum, IsCpythonObject,
|
||||
IsCVTable, IsNoVTable, IsPacked) = _cached_classify
|
||||
if ClassName.startswith('__') and not IsCpythonObject and not IsCenum and not IsRenum:
|
||||
return
|
||||
IsPacked = False
|
||||
if getattr(Node, 'decorator_list', None):
|
||||
for decorator in Node.decorator_list:
|
||||
if isinstance(decorator, ast.Call):
|
||||
if isinstance(decorator.func, ast.Attribute):
|
||||
if getattr(decorator.func.value, 'id', None) == 'c' and decorator.func.attr == 'Attribute':
|
||||
for arg in decorator.args:
|
||||
if isinstance(arg, ast.Attribute):
|
||||
if isinstance(arg.value, ast.Attribute):
|
||||
if getattr(arg.value.value, 'id', None) == 't' and arg.value.attr == 'attr' and arg.attr == 'packed':
|
||||
IsPacked = True
|
||||
if IsPacked:
|
||||
Gen.class_packed.add(ClassName)
|
||||
if IsCpythonObject:
|
||||
@@ -1609,14 +1650,8 @@ class ImportHandle(BaseHandle):
|
||||
Gen.class_member_bitoffsets[ClassName] = {}
|
||||
if ClassName not in Gen.class_methods:
|
||||
Gen.class_methods[ClassName] = []
|
||||
# 先计算 source_sha1，确保首次 _TryLoadStructFromStub 调用就能使用正确的 sha1
|
||||
# 优先使用传入的 source_sha1（来自 _TryLoadClassMembersFromPyi 的直接调用，
|
||||
# 当前模块未直接导入定义模块时 module_name 查找会失败，需依赖传入的 source_sha1）
|
||||
if not source_sha1:
|
||||
if actual_module_name and hasattr(Gen, 'ModuleSha1Map') and actual_module_name in Gen.ModuleSha1Map:
|
||||
source_sha1 = Gen.ModuleSha1Map[actual_module_name]
|
||||
elif module_name and hasattr(Gen, 'ModuleSha1Map') and module_name in Gen.ModuleSha1Map:
|
||||
source_sha1 = Gen.ModuleSha1Map[module_name]
|
||||
# source_sha1 已在函数开头统一计算（作为分类缓存键），此处直接使用。
|
||||
# 注意：REnum 分支（上方）会按 actual_module_name 优先重新推导并已 return。
|
||||
# 提前注册 struct：使用导入的 source_sha1 创建 opaque struct，
|
||||
# 避免后续 _TryLoadStructFromStub 触发嵌套编译时，StructGen 用本地 sha1
|
||||
# 创建缺少基类字段（如 GSListNode.Next）的本地类型定义。
|
||||
@@ -2211,14 +2246,22 @@ class ImportHandle(BaseHandle):
|
||||
# module_name 反查失败时（当前模块未直接导入定义模块），
|
||||
# 仍需注册类成员和 vtable 信息。传 source_sha1 让 _EmitExternalClassDeclLlvm
|
||||
# 正确设置 class_sha1_map 和注册 vtable。
|
||||
# 类级 pyi 树节点索引：同一 (source_sha1, class_name) 的 ClassDef 节点在进程内
|
||||
# 唯一（_pyi_cache 以 pyi_path 为键只 parse 一次），避免跨 worker 任务反复
|
||||
# 线性扫描 pyi_tree.body 查找同一定义类。
|
||||
found = False
|
||||
for node in pyi_tree.body:
|
||||
if isinstance(node, ast.ClassDef) and node.name == class_name:
|
||||
self._EmitExternalClassDeclLlvm(node, Gen, module_name=module_name, source_sha1=source_sha1)
|
||||
found = True
|
||||
break
|
||||
node = self._extern_class_decl_cache.get((source_sha1, class_name))
|
||||
if node is not None:
|
||||
found = True
|
||||
else:
|
||||
for node in pyi_tree.body:
|
||||
if isinstance(node, ast.ClassDef) and node.name == class_name:
|
||||
self._extern_class_decl_cache[(source_sha1, class_name)] = node
|
||||
found = True
|
||||
break
|
||||
if not found:
|
||||
return
|
||||
self._EmitExternalClassDeclLlvm(node, Gen, module_name=module_name, source_sha1=source_sha1)
|
||||
if class_name in Gen.class_members and len(Gen.class_members[class_name]) > 0:
|
||||
return
|
||||
return
|
||||
|
||||
Reference in new issue
Block a user