snapshot before regression test
This commit is contained in:
t
452
lib/core/Translator/PythonParser.py
Normal file
452
lib/core/Translator/PythonParser.py
Normal file
@@ -0,0 +1,452 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import json
|
||||
import struct
|
||||
|
||||
from lib.utils.helpers import DetectFileType
|
||||
from lib.core.Handles.HandlesBase import CTypeInfo
|
||||
from lib.core.SymbolUtils import IsTModule, AnnotationContainsName
|
||||
from lib.includes import t
|
||||
from lib.constants.config import mode as _ConfigMode
|
||||
from lib.core.VLogger import get_logger as _vlog
|
||||
|
||||
|
||||
class PythonParserMixin:
|
||||
"""Python 文件解析 Mixin
|
||||
|
||||
提供辅助文件解析、symbin 文件加载与 Python 源文件解析能力。
|
||||
"""
|
||||
|
||||
def ParseHelperFiles(self, HelperFiles: list[str], encoding: str = 'utf-8') -> None:
|
||||
"""解析辅助文件，提取符号信息
|
||||
|
||||
支持 .py, .c, .h 和 .symbin 文件
|
||||
"""
|
||||
for FilePath in HelperFiles:
|
||||
ext: str = DetectFileType(FilePath)
|
||||
if ext == '.py':
|
||||
self.ParsePythonFile(FilePath, encoding, UpdateCurrentFile=False)
|
||||
elif ext == '.c':
|
||||
pass # .c 文件解析已移除，不再使用旧的正则解析路径
|
||||
elif FilePath.endswith('.symbin'):
|
||||
self.LoadSymbinFile(FilePath)
|
||||
|
||||
def LoadSymbinFile(self, FilePath: str) -> None:
|
||||
"""从.symbin文件加载符号表
|
||||
|
||||
Args:
|
||||
FilePath: .symbin文件路径
|
||||
"""
|
||||
try:
|
||||
with open(FilePath, 'rb') as f:
|
||||
data: bytes = f.read()
|
||||
|
||||
# 导入序列化函数（避免循环导入）
|
||||
# 解析二进制格式
|
||||
if len(data) < 4:
|
||||
_vlog().warning(f"无效的 symbin 文件 (太短): {FilePath}")
|
||||
return
|
||||
|
||||
# 解析4字节长度头（小端序）
|
||||
length: int = struct.unpack('<I', data[:4])[0]
|
||||
JsonData: bytes = data[4:4+length]
|
||||
symbols: dict = json.loads(JsonData.decode('utf-8'))
|
||||
|
||||
# 合并到当前符号表
|
||||
self.SymbolTable.update(symbols)
|
||||
self.DebugPrint(f"[SYMBIN] Loaded {len(symbols)} symbols from {FilePath}")
|
||||
|
||||
except Exception as e:
|
||||
_vlog().warning(f"加载 symbin 文件失败 {FilePath}: {e}")
|
||||
|
||||
def ParsePythonFile(self, FilePath: str, encoding: str = 'utf-8', UpdateCurrentFile: bool = True) -> None:
|
||||
"""解析Python文件，提取类、函数、变量信息
|
||||
|
||||
Args:
|
||||
FilePath: 要解析的文件路径
|
||||
encoding: 文件编码
|
||||
UpdateCurrentFile: 是否更新 CurrentFile（默认为 True，处理导入时设为 False）
|
||||
"""
|
||||
try:
|
||||
with open(FilePath, 'r', encoding=encoding) as f:
|
||||
content: str = f.read()
|
||||
|
||||
# 保存当前处理的文件路径（仅在主文件处理时更新）
|
||||
if UpdateCurrentFile:
|
||||
self.CurrentFile = FilePath
|
||||
|
||||
# 解析Python代码为AST
|
||||
tree: ast.Module = ast.parse(content)
|
||||
|
||||
# 保存 AST 树供后续使用
|
||||
self.tree = tree
|
||||
|
||||
# 第一遍：检测特殊赋值语句，收集跟在 class 后面的
|
||||
# 遍历 tree.body，找到所有 ClassDef，然后收集紧随其后的赋值语句
|
||||
i: int = 0
|
||||
while i < len(tree.body):
|
||||
node: ast.stmt = tree.body[i]
|
||||
|
||||
# 检测 ClassDef
|
||||
if isinstance(node, ast.ClassDef):
|
||||
ClassName: str = node.name
|
||||
# 使用类名_行号作为唯一 key
|
||||
ClassKey: str = f"{ClassName}_{node.lineno}"
|
||||
|
||||
# 检查后面的语句，收集特殊赋值
|
||||
j: int = i + 1
|
||||
while j < len(tree.body):
|
||||
NextNode: ast.stmt = tree.body[j]
|
||||
|
||||
if isinstance(NextNode, ast.Assign):
|
||||
# 检测 xxx = xxx 或 xxx = yyy
|
||||
if len(NextNode.targets) == 1 and isinstance(NextNode.targets[0], ast.Name):
|
||||
TargetName: str = NextNode.targets[0].id
|
||||
if isinstance(NextNode.value, ast.Name):
|
||||
ValueName: str = NextNode.value.id
|
||||
if ValueName == ClassName:
|
||||
# 这是 class 的实例化
|
||||
if ClassKey not in self.EmbeddedAssignments:
|
||||
self.EmbeddedAssignments[ClassKey] = []
|
||||
self.EmbeddedAssignments[ClassKey].append(TargetName)
|
||||
j += 1
|
||||
continue
|
||||
elif isinstance(NextNode, ast.AnnAssign):
|
||||
# 检测 xxx: t.Postdefinition(yyy) 或 xxx: t.CTypedef = xxx 或 xxx: t.CPtr = xxx
|
||||
if isinstance(NextNode.target, ast.Name):
|
||||
TargetName: str = NextNode.target.id
|
||||
# 先处理新格式：t.Postdefinition(...)
|
||||
annotation: ast.expr | None = NextNode.annotation
|
||||
PostdefNode: ast.Call | None = None
|
||||
|
||||
# 递归搜索整个注解树，找到 t.Postdefinition(...) 调用
|
||||
def find_PostdefNode(node: ast.AST) -> ast.Call | None:
|
||||
if isinstance(node, ast.Call):
|
||||
if isinstance(node.func, ast.Attribute):
|
||||
if hasattr(node.func.value, 'id') and IsTModule(node.func.value.id, self.SymbolTable) and node.func.attr == 'Postdefinition':
|
||||
return node
|
||||
elif isinstance(node, ast.BinOp) and isinstance(node.op, ast.BitOr):
|
||||
LeftResult: ast.Call | None = find_PostdefNode(node.left)
|
||||
if LeftResult:
|
||||
return LeftResult
|
||||
RightResult: ast.Call | None = find_PostdefNode(node.right)
|
||||
if RightResult:
|
||||
return RightResult
|
||||
return None
|
||||
|
||||
PostdefNode = find_PostdefNode(annotation)
|
||||
# 检查是否找到 t.Postdefinition(...) 调用
|
||||
if PostdefNode:
|
||||
if (isinstance(PostdefNode.func, ast.Attribute) and
|
||||
hasattr(PostdefNode.func, 'value') and
|
||||
hasattr(PostdefNode.func.value, 'id') and
|
||||
IsTModule(PostdefNode.func.value.id, self.SymbolTable) and
|
||||
PostdefNode.func.attr == 'Postdefinition' and
|
||||
PostdefNode.args):
|
||||
# 检查第一个参数是否是当前类名
|
||||
arg: ast.expr = PostdefNode.args[0]
|
||||
if isinstance(arg, ast.Name) and arg.id == ClassName:
|
||||
# 检查是否包含 CTypedef 或 CPtr
|
||||
HasCtypedef: bool = AnnotationContainsName(NextNode.annotation, 'CTypedef')
|
||||
IsPtr: bool = AnnotationContainsName(NextNode.annotation, 'CPtr')
|
||||
|
||||
# 解析维度列表（第二个参数）
|
||||
dimensions: list[ast.expr] = []
|
||||
if len(PostdefNode.args) > 1:
|
||||
# 第二个参数应该是一个列表
|
||||
dim_arg: ast.expr = PostdefNode.args[1]
|
||||
if isinstance(dim_arg, ast.List):
|
||||
for DimExpr in dim_arg.elts:
|
||||
dimensions.append(DimExpr)
|
||||
|
||||
# 如果没有从 Postdefinition 参数中获取维度，尝试从类型注解中的 Subscript 获取
|
||||
# 例如：memory_block_t[MAX_ORDER + 1] | t.CPtr
|
||||
if not dimensions:
|
||||
def extract_subscript_dims(node: ast.AST) -> list[ast.expr]:
|
||||
"""递归提取 Subscript 节点的维度"""
|
||||
dims: list[ast.expr] = []
|
||||
if isinstance(node, ast.Subscript):
|
||||
# 提取当前维度
|
||||
if isinstance(node.slice, ast.Constant):
|
||||
dims.append(ast.Constant(value=node.slice.value))
|
||||
elif isinstance(node.slice, ast.Name):
|
||||
dims.append(ast.Name(id=node.slice.id, ctx=ast.Load()))
|
||||
elif isinstance(node.slice, ast.BinOp):
|
||||
dims.append(node.slice)
|
||||
# 递归提取更外层的维度
|
||||
dims.extend(extract_subscript_dims(node.value))
|
||||
elif isinstance(node, ast.BinOp) and isinstance(node.op, ast.BitOr):
|
||||
# 处理 Type | t.CPtr 的情况，检查左侧
|
||||
dims.extend(extract_subscript_dims(node.left))
|
||||
return dims
|
||||
|
||||
# 从注解中提取维度
|
||||
subscript_dims: list[ast.expr] = extract_subscript_dims(annotation)
|
||||
if subscript_dims:
|
||||
# 反转维度顺序（从内到外）
|
||||
dimensions = list(reversed(subscript_dims))
|
||||
|
||||
# 获取赋值值
|
||||
value: ast.expr | None = None
|
||||
if hasattr(NextNode, 'value') and NextNode.value is not None:
|
||||
value = NextNode.value
|
||||
|
||||
# 添加到对应的 assignments
|
||||
# 对于 t.Postdefinition(...)，只有包含 CTypedef 或 CPtr 时才视为 typedef 类型
|
||||
if HasCtypedef or IsPtr:
|
||||
if ClassKey not in self.TypedefAssignments:
|
||||
self.TypedefAssignments[ClassKey] = []
|
||||
# 如果有维度信息，使用字典存储
|
||||
if dimensions:
|
||||
self.TypedefAssignments[ClassKey].append({
|
||||
'name': TargetName,
|
||||
'IsPtr': IsPtr,
|
||||
'dimensions': dimensions,
|
||||
'value': value
|
||||
})
|
||||
else:
|
||||
self.TypedefAssignments[ClassKey].append((TargetName, IsPtr))
|
||||
else:
|
||||
if ClassKey not in self.EmbeddedAssignments:
|
||||
self.EmbeddedAssignments[ClassKey] = []
|
||||
# 如果有维度信息，使用字典存储
|
||||
if dimensions:
|
||||
self.EmbeddedAssignments[ClassKey].append({
|
||||
'name': TargetName,
|
||||
'dimensions': dimensions,
|
||||
'value': value
|
||||
})
|
||||
else:
|
||||
self.EmbeddedAssignments[ClassKey].append(TargetName)
|
||||
|
||||
# 检查是否有 __set_default__ 初始化
|
||||
if AnnotationContainsName(NextNode.annotation, '__set_default__'):
|
||||
self._handle_set_default_in_parse(ClassKey, TargetName, ClassName, NextNode)
|
||||
j += 1
|
||||
continue
|
||||
# 再处理旧格式：xxx: t.CTypedef = xxx 或 xxx: t.CPtr = xxx
|
||||
# 注意：这种格式的 typedef 不应该被合并到结构体定义中，
|
||||
# 而应该生成独立的 typedef 语句，所以不添加到 TypedefAssignments
|
||||
elif NextNode.annotation and isinstance(NextNode.annotation, ast.Attribute):
|
||||
if isinstance(NextNode.annotation.value, ast.Name):
|
||||
annot_id: str = NextNode.annotation.value.id
|
||||
annot_attr: str = NextNode.annotation.attr
|
||||
if annot_id == 't' and NextNode.value:
|
||||
if isinstance(NextNode.value, ast.Name):
|
||||
ValueName: str = NextNode.value.id
|
||||
if ValueName == ClassName:
|
||||
# 检查是 CTypedef 还是 CPtr
|
||||
if annot_attr == t.CPtr.__name__:
|
||||
# t.CPtr 表示指针变量，加入 TypedefAssignments，设置 IsPtr=True
|
||||
if ClassKey not in self.TypedefAssignments:
|
||||
self.TypedefAssignments[ClassKey] = []
|
||||
self.TypedefAssignments[ClassKey].append((TargetName, True))
|
||||
j += 1
|
||||
continue
|
||||
# 注意：t.CTypedef 不在这里处理，由 HandlesAnnAssign 生成独立的 typedef 语句
|
||||
# 如果不是特殊赋值，停止收集
|
||||
break
|
||||
|
||||
i = j
|
||||
continue
|
||||
|
||||
i += 1
|
||||
|
||||
# 提取类定义（作为结构体）
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.ClassDef):
|
||||
ClassName: str = node.name
|
||||
# 先把类名加入 GeneratedTypes，这样 GetTypeName 可以正确识别
|
||||
self.GeneratedTypes.add(ClassName)
|
||||
|
||||
# 检测是否是 CEnum
|
||||
IsCenum: bool = False
|
||||
for base in node.bases:
|
||||
if isinstance(base, ast.Attribute):
|
||||
if base.attr in ('CEnum', 'REnum'):
|
||||
IsCenum = True
|
||||
break
|
||||
elif isinstance(base, ast.Name):
|
||||
if base.id in ('CEnum', 'REnum'):
|
||||
IsCenum = True
|
||||
break
|
||||
|
||||
# 检测是否有 @t.Object 装饰器或继承自 t.Object
|
||||
IsCpythonObject: bool = False
|
||||
if hasattr(node, 'decorator_list') and node.decorator_list:
|
||||
for decorator in node.decorator_list:
|
||||
if isinstance(decorator, ast.Attribute):
|
||||
if (hasattr(decorator.value, 'id') and
|
||||
IsTModule(decorator.value.id, self.SymbolTable) and
|
||||
decorator.attr == 'Object'):
|
||||
IsCpythonObject = True
|
||||
break
|
||||
elif isinstance(decorator, ast.Call):
|
||||
if isinstance(decorator.func, ast.Attribute):
|
||||
if (hasattr(decorator.func.value, 'id') and
|
||||
IsTModule(decorator.func.value.id, self.SymbolTable) and
|
||||
decorator.func.attr == 'Object'):
|
||||
IsCpythonObject = True
|
||||
break
|
||||
elif isinstance(decorator.func, ast.Name):
|
||||
if decorator.func.id == 'Object':
|
||||
IsCpythonObject = True
|
||||
break
|
||||
elif isinstance(decorator, ast.Name):
|
||||
if decorator.id == 'Object':
|
||||
IsCpythonObject = True
|
||||
break
|
||||
if not IsCpythonObject and hasattr(node, 'bases') and node.bases:
|
||||
for base in node.bases:
|
||||
if isinstance(base, ast.Attribute):
|
||||
if (hasattr(base.value, 'id') and
|
||||
IsTModule(base.value.id, self.SymbolTable) and
|
||||
base.attr == 'Object'):
|
||||
IsCpythonObject = True
|
||||
break
|
||||
elif isinstance(base, ast.Name):
|
||||
if base.id == 'Object':
|
||||
IsCpythonObject = True
|
||||
break
|
||||
|
||||
if IsCenum:
|
||||
# 注册为 enum 类型
|
||||
EnumNode: CTypeInfo = CTypeInfo()
|
||||
EnumNode.Name = ClassName
|
||||
EnumNode.IsEnum = True
|
||||
EnumNode.set('file', '<stdin>')
|
||||
self.SymbolTable.insert(ClassName, EnumNode)
|
||||
|
||||
# 注册 enum 成员（提取枚举值，与 SymbolExtractor 逻辑一致）
|
||||
next_enum_value: int = 0
|
||||
for item in node.body:
|
||||
MemberName: str | None = None
|
||||
ItemValue: object | None = None
|
||||
if isinstance(item, ast.AnnAssign) and isinstance(item.target, ast.Name):
|
||||
MemberName = item.target.id
|
||||
ItemValue = item.value
|
||||
elif isinstance(item, ast.Assign) and len(item.targets) == 1 and isinstance(item.targets[0], ast.Name):
|
||||
MemberName = item.targets[0].id
|
||||
ItemValue = item.value
|
||||
if MemberName is None:
|
||||
continue
|
||||
MemberNode: CTypeInfo = CTypeInfo()
|
||||
MemberNode.Name = MemberName
|
||||
MemberNode.BaseType = t.CEnum(ClassName)
|
||||
# 提取枚举值：优先用显式赋值，否则自动编号
|
||||
if ItemValue is not None and isinstance(ItemValue, ast.Constant) and isinstance(ItemValue.value, int):
|
||||
MemberNode.value = ItemValue.value
|
||||
next_enum_value = ItemValue.value + 1
|
||||
else:
|
||||
MemberNode.value = next_enum_value
|
||||
next_enum_value += 1
|
||||
MemberNode.EnumName = ClassName
|
||||
MemberNode.library_name = None
|
||||
MemberNode.IsEnumMember = True
|
||||
self.SymbolTable.insert(MemberName, MemberNode)
|
||||
self.SymbolTable.insert(f"{ClassName}.{MemberName}", MemberNode)
|
||||
self.SymbolTable.insert(f"{ClassName}_{MemberName}", MemberNode)
|
||||
|
||||
continue
|
||||
|
||||
# 检测是否是 Anonymous
|
||||
IsAnonymous: bool = False
|
||||
|
||||
# 提取结构体成员信息
|
||||
members: dict[str, CTypeInfo] = {}
|
||||
for item in node.body:
|
||||
if isinstance(item, ast.AnnAssign) and isinstance(item.target, ast.Name):
|
||||
VarName: str = item.target.id
|
||||
try:
|
||||
TypeInfo: CTypeInfo | None = self.TypeMergeHandler.GetCTypeInfo(item.annotation)
|
||||
if TypeInfo is None:
|
||||
TypeInfo = CTypeInfo()
|
||||
TypeInfo.BaseType = t.CInt()
|
||||
IsPtr: bool = TypeInfo.IsPtr
|
||||
if not IsPtr:
|
||||
if isinstance(item.annotation, ast.BinOp) and isinstance(item.annotation.op, ast.BitOr):
|
||||
if AnnotationContainsName(item.annotation, 'CPtr'):
|
||||
IsPtr = True
|
||||
dims: list[int] = []
|
||||
if hasattr(item.annotation, 'slice') and isinstance(item.annotation.slice, ast.Constant):
|
||||
if isinstance(item.annotation.value, ast.Name):
|
||||
if IsTModule(item.annotation.value.id, self.SymbolTable):
|
||||
if hasattr(item.annotation, 'slice'):
|
||||
try:
|
||||
dims.append(int(item.annotation.slice.value))
|
||||
except Exception as _e:
|
||||
if _ConfigMode == "strict":
|
||||
raise
|
||||
_vlog().warning(f"解析数组维度失败: {_e}", "Exception")
|
||||
MemberInfo: CTypeInfo = TypeInfo.Copy()
|
||||
if dims:
|
||||
MemberInfo.ArrayDims = [str(d) for d in dims]
|
||||
members[VarName] = MemberInfo
|
||||
except Exception as _e:
|
||||
if _ConfigMode == "strict":
|
||||
raise
|
||||
_vlog().warning(f"解析结构体成员失败: {_e}", "Exception")
|
||||
StructNode: CTypeInfo = CTypeInfo()
|
||||
StructNode.Name = ClassName
|
||||
StructNode.IsStruct = True
|
||||
StructNode.Members = members or {}
|
||||
StructNode.set('file', '<stdin>')
|
||||
StructNode.set('IsAnonymous', IsAnonymous)
|
||||
StructNode.set('IsCpythonObject', IsCpythonObject)
|
||||
self.SymbolTable.insert(ClassName, StructNode)
|
||||
elif isinstance(node, ast.FunctionDef):
|
||||
FuncName: str = node.name
|
||||
if not self.SymbolTable.has(FuncName):
|
||||
FuncNode: CTypeInfo = CTypeInfo()
|
||||
FuncNode.Name = FuncName
|
||||
FuncNode.IsFunction = True
|
||||
FuncNode.set('file', '<stdin>')
|
||||
self.SymbolTable.insert(FuncName, FuncNode)
|
||||
elif isinstance(node, ast.AnnAssign):
|
||||
if isinstance(node.target, ast.Name):
|
||||
VarName: str = node.target.id
|
||||
# 检查是否是 t.CTypedef 格式，如果是则跳过（由 HandleClassDef 处理）
|
||||
is_ctypedef: bool = False
|
||||
if node.annotation:
|
||||
if isinstance(node.annotation, ast.Attribute):
|
||||
if isinstance(node.annotation.value, ast.Name):
|
||||
if IsTModule(node.annotation.value.id, self.SymbolTable) and node.annotation.attr == 'CTypedef':
|
||||
is_ctypedef = True
|
||||
elif isinstance(node.annotation, ast.BinOp) and isinstance(node.annotation.op, ast.BitOr):
|
||||
def _has_ctype(ann: ast.AST) -> bool:
|
||||
if isinstance(ann, ast.Attribute):
|
||||
return hasattr(ann.value, 'id') and IsTModule(ann.value.id) and ann.attr == 'CTypedef'
|
||||
if isinstance(ann, ast.BinOp) and isinstance(ann.op, ast.BitOr):
|
||||
return _has_ctype(ann.left) or _has_ctype(ann.right)
|
||||
if isinstance(ann, ast.Subscript):
|
||||
return _has_ctype(ann.value)
|
||||
return False
|
||||
if _has_ctype(node.annotation):
|
||||
is_ctypedef = True
|
||||
if not is_ctypedef:
|
||||
# 如果符号已经存在且是 typedef，不要覆盖它
|
||||
existing = self.SymbolTable.lookup(VarName)
|
||||
if existing and existing.IsTypedef and existing.OriginalType:
|
||||
pass
|
||||
else:
|
||||
IsPtr: bool = False
|
||||
if node.annotation:
|
||||
try:
|
||||
TypeInfo: CTypeInfo | None = self.TypeMergeHandler.GetCTypeInfo(node.annotation)
|
||||
IsPtr = TypeInfo.IsPtr if TypeInfo else False
|
||||
except Exception as _e:
|
||||
if _ConfigMode == "strict":
|
||||
raise
|
||||
_vlog().warning(f"获取类型信息失败(ParsePythonFile): {_e}", "Exception")
|
||||
var_node: CTypeInfo = CTypeInfo()
|
||||
var_node.Name = VarName
|
||||
var_node.IsVariable = True
|
||||
var_node.set('IsPtr', IsPtr)
|
||||
var_node.set('file', '<stdin>')
|
||||
self.SymbolTable.insert(VarName, var_node)
|
||||
except Exception as e:
|
||||
_vlog().warning(f'解析 Python 文件失败 {FilePath}: {e}')
|
||||
|
||||
# ParseCFile 已移除 - .h/.c 文件的旧正则解析路径不再使用
|
||||
# 类型解析现在直接通过 CTypeInfo 对象进行，不经过中间字符串格式
|
||||
Reference in New Issue
Block a user