可用的回归测试通过的标准版本
This commit is contained in:
460
lib/core/Translator/PythonParser.py
Normal file
460
lib/core/Translator/PythonParser.py
Normal file
@@ -0,0 +1,460 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import struct
|
||||
|
||||
from lib.utils.helpers import DetectFileType
|
||||
from lib.core.SymbolNode import SymbolNode
|
||||
from lib.core.Handles.HandlesBase import CTypeInfo
|
||||
from lib.includes import t
|
||||
from lib.constants.config import mode as _ConfigMode
|
||||
from lib.core.Translator.BaseTranslator import _StrictLog
|
||||
|
||||
|
||||
class PythonParserMixin:
|
||||
"""Python 文件解析 Mixin
|
||||
|
||||
提供辅助文件解析、symbin 文件加载与 Python 源文件解析能力。
|
||||
"""
|
||||
|
||||
def ParseHelperFiles(self, HelperFiles, encoding='utf-8'):
|
||||
"""解析辅助文件，提取符号信息
|
||||
|
||||
支持 .py, .c, .h 和 .symbin 文件
|
||||
"""
|
||||
for FilePath in HelperFiles:
|
||||
ext = DetectFileType(FilePath)
|
||||
if ext == '.py':
|
||||
self.ParsePythonFile(FilePath, encoding, UpdateCurrentFile=False)
|
||||
elif ext == '.c':
|
||||
pass # .c 文件解析已移除，不再使用旧的正则解析路径
|
||||
elif FilePath.endswith('.symbin'):
|
||||
self.LoadSymbinFile(FilePath)
|
||||
|
||||
def LoadSymbinFile(self, FilePath):
|
||||
"""从.symbin文件加载符号表
|
||||
|
||||
Args:
|
||||
FilePath: .symbin文件路径
|
||||
"""
|
||||
try:
|
||||
with open(FilePath, 'rb') as f:
|
||||
data = f.read()
|
||||
|
||||
# 导入序列化函数（避免循环导入）
|
||||
import sys
|
||||
import os
|
||||
import json
|
||||
import struct
|
||||
|
||||
# 解析二进制格式
|
||||
if len(data) < 4:
|
||||
print(f"Warning: Invalid symbin file (too short): {FilePath}")
|
||||
return
|
||||
|
||||
# 解析4字节长度头（小端序）
|
||||
length = struct.unpack('<I', data[:4])[0]
|
||||
JsonData = data[4:4+length]
|
||||
symbols = json.loads(JsonData.decode('utf-8'))
|
||||
|
||||
# 合并到当前符号表
|
||||
self.SymbolTable.update(symbols)
|
||||
self.DebugPrint(f"[SYMBIN] Loaded {len(symbols)} symbols from {FilePath}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Warning: Failed to Load symbin file {FilePath}: {e}")
|
||||
|
||||
def ParsePythonFile(self, FilePath, encoding='utf-8', UpdateCurrentFile=True):
|
||||
"""解析Python文件，提取类、函数、变量信息
|
||||
|
||||
Args:
|
||||
FilePath: 要解析的文件路径
|
||||
encoding: 文件编码
|
||||
UpdateCurrentFile: 是否更新 CurrentFile（默认为 True，处理导入时设为 False）
|
||||
"""
|
||||
try:
|
||||
with open(FilePath, 'r', encoding=encoding) as f:
|
||||
content = f.read()
|
||||
|
||||
# 保存当前处理的文件路径（仅在主文件处理时更新）
|
||||
if UpdateCurrentFile:
|
||||
self.CurrentFile = FilePath
|
||||
|
||||
# 解析Python代码为AST
|
||||
tree = ast.parse(content)
|
||||
|
||||
# 保存 AST 树供后续使用
|
||||
self.tree = tree
|
||||
|
||||
# 第一遍：检测特殊赋值语句，收集跟在 class 后面的
|
||||
# 遍历 tree.body，找到所有 ClassDef，然后收集紧随其后的赋值语句
|
||||
i = 0
|
||||
while i < len(tree.body):
|
||||
node = tree.body[i]
|
||||
|
||||
# 检测 ClassDef
|
||||
if isinstance(node, ast.ClassDef):
|
||||
ClassName = node.name
|
||||
# 使用类名_行号作为唯一 key
|
||||
ClassKey = f"{ClassName}_{node.lineno}"
|
||||
|
||||
# 检查后面的语句，收集特殊赋值
|
||||
j = i + 1
|
||||
while j < len(tree.body):
|
||||
NextNode = tree.body[j]
|
||||
|
||||
if isinstance(NextNode, ast.Assign):
|
||||
# 检测 xxx = xxx 或 xxx = yyy
|
||||
if len(NextNode.targets) == 1 and isinstance(NextNode.targets[0], ast.Name):
|
||||
TargetName = NextNode.targets[0].id
|
||||
if isinstance(NextNode.value, ast.Name):
|
||||
ValueName = NextNode.value.id
|
||||
if ValueName == ClassName:
|
||||
# 这是 class 的实例化
|
||||
if ClassKey not in self.EmbeddedAssignments:
|
||||
self.EmbeddedAssignments[ClassKey] = []
|
||||
self.EmbeddedAssignments[ClassKey].append(TargetName)
|
||||
j += 1
|
||||
continue
|
||||
elif isinstance(NextNode, ast.AnnAssign):
|
||||
# 检测 xxx: t.Postdefinition(yyy) 或 xxx: t.CTypedef = xxx 或 xxx: t.CPtr = xxx
|
||||
if isinstance(NextNode.target, ast.Name):
|
||||
TargetName = NextNode.target.id
|
||||
# 先处理新格式：t.Postdefinition(...)
|
||||
annotation = NextNode.annotation
|
||||
original_AnnotationStr = ast.dump(NextNode.annotation)
|
||||
PostdefNode = None
|
||||
|
||||
# 递归搜索整个注解树，找到 t.Postdefinition(...) 调用
|
||||
def find_PostdefNode(node):
|
||||
if isinstance(node, ast.Call):
|
||||
if isinstance(node.func, ast.Attribute):
|
||||
if hasattr(node.func.value, 'id') and node.func.value.id == 't' and node.func.attr == 'Postdefinition':
|
||||
return node
|
||||
elif isinstance(node, ast.BinOp) and isinstance(node.op, ast.BitOr):
|
||||
LeftResult = find_PostdefNode(node.left)
|
||||
if LeftResult:
|
||||
return LeftResult
|
||||
RightResult = find_PostdefNode(node.right)
|
||||
if RightResult:
|
||||
return RightResult
|
||||
return None
|
||||
|
||||
PostdefNode = find_PostdefNode(annotation)
|
||||
# 检查是否找到 t.Postdefinition(...) 调用
|
||||
if PostdefNode:
|
||||
if (isinstance(PostdefNode.func, ast.Attribute) and
|
||||
hasattr(PostdefNode.func, 'value') and
|
||||
hasattr(PostdefNode.func.value, 'id') and
|
||||
PostdefNode.func.value.id == 't' and
|
||||
PostdefNode.func.attr == 'Postdefinition' and
|
||||
PostdefNode.args):
|
||||
# 检查第一个参数是否是当前类名
|
||||
arg = PostdefNode.args[0]
|
||||
if isinstance(arg, ast.Name) and arg.id == ClassName:
|
||||
# 检查是否包含 CTypedef 或 CPtr
|
||||
HasCtypedef = False
|
||||
IsPtr = False
|
||||
if 'CTypedef' in original_AnnotationStr:
|
||||
HasCtypedef = True
|
||||
if 'CPtr' in original_AnnotationStr:
|
||||
IsPtr = True
|
||||
|
||||
# 解析维度列表（第二个参数）
|
||||
dimensions = []
|
||||
if len(PostdefNode.args) > 1:
|
||||
# 第二个参数应该是一个列表
|
||||
dim_arg = PostdefNode.args[1]
|
||||
if isinstance(dim_arg, ast.List):
|
||||
for DimExpr in dim_arg.elts:
|
||||
dimensions.append(DimExpr)
|
||||
|
||||
# 如果没有从 Postdefinition 参数中获取维度，尝试从类型注解中的 Subscript 获取
|
||||
# 例如：memory_block_t[MAX_ORDER + 1] | t.CPtr
|
||||
if not dimensions:
|
||||
def extract_subscript_dims(node):
|
||||
"""递归提取 Subscript 节点的维度"""
|
||||
dims = []
|
||||
if isinstance(node, ast.Subscript):
|
||||
# 提取当前维度
|
||||
if isinstance(node.slice, ast.Constant):
|
||||
dims.append(ast.Constant(value=node.slice.value))
|
||||
elif isinstance(node.slice, ast.Name):
|
||||
dims.append(ast.Name(id=node.slice.id, ctx=ast.Load()))
|
||||
elif isinstance(node.slice, ast.BinOp):
|
||||
dims.append(node.slice)
|
||||
# 递归提取更外层的维度
|
||||
dims.extend(extract_subscript_dims(node.value))
|
||||
elif isinstance(node, ast.BinOp) and isinstance(node.op, ast.BitOr):
|
||||
# 处理 Type | t.CPtr 的情况，检查左侧
|
||||
dims.extend(extract_subscript_dims(node.left))
|
||||
return dims
|
||||
|
||||
# 从注解中提取维度
|
||||
subscript_dims = extract_subscript_dims(annotation)
|
||||
if subscript_dims:
|
||||
# 反转维度顺序（从内到外）
|
||||
dimensions = list(reversed(subscript_dims))
|
||||
|
||||
# 获取赋值值
|
||||
value = None
|
||||
if hasattr(NextNode, 'value') and NextNode.value is not None:
|
||||
value = NextNode.value
|
||||
|
||||
# 添加到对应的 assignments
|
||||
# 对于 t.Postdefinition(...)，只有包含 CTypedef 或 CPtr 时才视为 typedef 类型
|
||||
if HasCtypedef or IsPtr:
|
||||
if ClassKey not in self.TypedefAssignments:
|
||||
self.TypedefAssignments[ClassKey] = []
|
||||
# 如果有维度信息，使用字典存储
|
||||
if dimensions:
|
||||
self.TypedefAssignments[ClassKey].append({
|
||||
'name': TargetName,
|
||||
'IsPtr': IsPtr,
|
||||
'dimensions': dimensions,
|
||||
'value': value
|
||||
})
|
||||
else:
|
||||
self.TypedefAssignments[ClassKey].append((TargetName, IsPtr))
|
||||
else:
|
||||
if ClassKey not in self.EmbeddedAssignments:
|
||||
self.EmbeddedAssignments[ClassKey] = []
|
||||
# 如果有维度信息，使用字典存储
|
||||
if dimensions:
|
||||
self.EmbeddedAssignments[ClassKey].append({
|
||||
'name': TargetName,
|
||||
'dimensions': dimensions,
|
||||
'value': value
|
||||
})
|
||||
else:
|
||||
self.EmbeddedAssignments[ClassKey].append(TargetName)
|
||||
|
||||
# 检查是否有 __set_default__ 初始化
|
||||
if '__set_default__' in original_AnnotationStr:
|
||||
self._handle_set_default_in_parse(ClassKey, TargetName, ClassName, NextNode)
|
||||
j += 1
|
||||
continue
|
||||
# 再处理旧格式：xxx: t.CTypedef = xxx 或 xxx: t.CPtr = xxx
|
||||
# 注意：这种格式的 typedef 不应该被合并到结构体定义中，
|
||||
# 而应该生成独立的 typedef 语句，所以不添加到 TypedefAssignments
|
||||
elif NextNode.annotation and isinstance(NextNode.annotation, ast.Attribute):
|
||||
if isinstance(NextNode.annotation.value, ast.Name):
|
||||
annot_id = NextNode.annotation.value.id
|
||||
annot_attr = NextNode.annotation.attr
|
||||
if annot_id == 't' and NextNode.value:
|
||||
if isinstance(NextNode.value, ast.Name):
|
||||
ValueName = NextNode.value.id
|
||||
if ValueName == ClassName:
|
||||
# 检查是 CTypedef 还是 CPtr
|
||||
if annot_attr == t.CPtr.__name__:
|
||||
# t.CPtr 表示指针变量，加入 TypedefAssignments，设置 IsPtr=True
|
||||
if ClassKey not in self.TypedefAssignments:
|
||||
self.TypedefAssignments[ClassKey] = []
|
||||
self.TypedefAssignments[ClassKey].append((TargetName, True))
|
||||
j += 1
|
||||
continue
|
||||
# 注意：t.CTypedef 不在这里处理，由 HandlesAnnAssign 生成独立的 typedef 语句
|
||||
# 如果不是特殊赋值，停止收集
|
||||
break
|
||||
|
||||
i = j
|
||||
continue
|
||||
|
||||
i += 1
|
||||
|
||||
# 提取类定义（作为结构体）
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.ClassDef):
|
||||
ClassName = node.name
|
||||
# 先把类名加入 GeneratedTypes，这样 GetTypeName 可以正确识别
|
||||
self.GeneratedTypes.add(ClassName)
|
||||
|
||||
# 检测是否是 CEnum
|
||||
IsCenum = False
|
||||
for base in node.bases:
|
||||
if isinstance(base, ast.Attribute):
|
||||
BaseStr = ast.dump(base)
|
||||
if 'CEnum' in BaseStr or 'REnum' in BaseStr:
|
||||
IsCenum = True
|
||||
break
|
||||
elif isinstance(base, ast.Name):
|
||||
if 'CEnum' in base.id or 'REnum' in base.id:
|
||||
IsCenum = True
|
||||
break
|
||||
|
||||
# 检测是否有 @t.Object 装饰器或继承自 t.Object
|
||||
IsCpythonObject = False
|
||||
if hasattr(node, 'decorator_list') and node.decorator_list:
|
||||
for decorator in node.decorator_list:
|
||||
if isinstance(decorator, ast.Attribute):
|
||||
if (hasattr(decorator.value, 'id') and
|
||||
decorator.value.id == 't' and
|
||||
decorator.attr == 'Object'):
|
||||
IsCpythonObject = True
|
||||
break
|
||||
elif isinstance(decorator, ast.Call):
|
||||
if isinstance(decorator.func, ast.Attribute):
|
||||
if (hasattr(decorator.func.value, 'id') and
|
||||
decorator.func.value.id == 't' and
|
||||
decorator.func.attr == 'Object'):
|
||||
IsCpythonObject = True
|
||||
break
|
||||
elif isinstance(decorator.func, ast.Name):
|
||||
if decorator.func.id == 'Object':
|
||||
IsCpythonObject = True
|
||||
break
|
||||
elif isinstance(decorator, ast.Name):
|
||||
if decorator.id == 'Object':
|
||||
IsCpythonObject = True
|
||||
break
|
||||
if not IsCpythonObject and hasattr(node, 'bases') and node.bases:
|
||||
for base in node.bases:
|
||||
if isinstance(base, ast.Attribute):
|
||||
if (hasattr(base.value, 'id') and
|
||||
base.value.id == 't' and
|
||||
base.attr == 'Object'):
|
||||
IsCpythonObject = True
|
||||
break
|
||||
elif isinstance(base, ast.Name):
|
||||
if base.id == 'Object':
|
||||
IsCpythonObject = True
|
||||
break
|
||||
|
||||
if IsCenum:
|
||||
# 注册为 enum 类型
|
||||
EnumNode = SymbolNode.CreateClass(
|
||||
name=ClassName,
|
||||
TypeKind='enum',
|
||||
lineno=0
|
||||
)
|
||||
EnumNode.set('file', '<stdin>')
|
||||
self.SymbolTable[ClassName] = EnumNode.attributes
|
||||
|
||||
# 注册 enum 成员
|
||||
for item in node.body:
|
||||
if isinstance(item, ast.AnnAssign) and isinstance(item.target, ast.Name):
|
||||
MemberName = item.target.id
|
||||
MemberNode = CTypeInfo()
|
||||
MemberNode.Name = MemberName
|
||||
MemberNode.BaseType = t.CEnum(ClassName)
|
||||
MemberNode.value = None
|
||||
MemberNode.EnumName = ClassName
|
||||
MemberNode.library_name = None
|
||||
MemberNode.IsEnumMember = True
|
||||
self.SymbolTable[MemberName] = MemberNode
|
||||
self.SymbolTable[f"{ClassName}.{MemberName}"] = MemberNode
|
||||
self.SymbolTable[f"{ClassName}_{MemberName}"] = MemberNode
|
||||
|
||||
continue
|
||||
|
||||
# 检测是否是 Anonymous
|
||||
IsAnonymous = False
|
||||
|
||||
# 提取结构体成员信息
|
||||
members = {}
|
||||
for item in node.body:
|
||||
if isinstance(item, ast.AnnAssign) and isinstance(item.target, ast.Name):
|
||||
VarName = item.target.id
|
||||
try:
|
||||
TypeInfo = self.TypeMergeHandler.GetCTypeInfo(item.annotation)
|
||||
if TypeInfo is None:
|
||||
TypeInfo = CTypeInfo()
|
||||
TypeInfo.BaseType = t.CInt()
|
||||
IsPtr = TypeInfo.IsPtr
|
||||
if not IsPtr:
|
||||
if isinstance(item.annotation, ast.BinOp) and isinstance(item.annotation.op, ast.BitOr):
|
||||
LeftStr = ast.dump(item.annotation.left)
|
||||
RightStr = ast.dump(item.annotation.right)
|
||||
if 'CPtr' in LeftStr or 'CPtr' in RightStr:
|
||||
IsPtr = True
|
||||
dims = []
|
||||
if hasattr(item.annotation, 'slice') and isinstance(item.annotation.slice, ast.Constant):
|
||||
if isinstance(item.annotation.value, ast.Name):
|
||||
if item.annotation.value.id == 't':
|
||||
if hasattr(item.annotation, 'slice'):
|
||||
try:
|
||||
dims.append(int(item.annotation.slice.value))
|
||||
except Exception as _e:
|
||||
from lib.core.VLogger import get_logger as _vlog
|
||||
from lib.constants.config import mode as _ConfigMode
|
||||
if _ConfigMode == "strict":
|
||||
raise
|
||||
_vlog().warning(f"解析数组维度失败: {_e}", "Exception")
|
||||
MemberInfo = TypeInfo.Copy()
|
||||
if dims:
|
||||
MemberInfo.ArrayDims = [str(d) for d in dims]
|
||||
members[VarName] = MemberInfo
|
||||
except Exception as _e:
|
||||
from lib.core.VLogger import get_logger as _vlog
|
||||
from lib.constants.config import mode as _ConfigMode
|
||||
if _ConfigMode == "strict":
|
||||
raise
|
||||
_vlog().warning(f"解析结构体成员失败: {_e}", "Exception")
|
||||
StructNode = SymbolNode.CreateClass(
|
||||
name=ClassName,
|
||||
TypeKind='struct',
|
||||
members=members,
|
||||
lineno=0,
|
||||
IsCpythonObject=IsCpythonObject
|
||||
)
|
||||
StructNode.set('file', '<stdin>')
|
||||
StructNode.set('IsAnonymous', IsAnonymous)
|
||||
self.SymbolTable[ClassName] = StructNode.attributes
|
||||
elif isinstance(node, ast.FunctionDef):
|
||||
FuncName = node.name
|
||||
FuncNode = SymbolNode.CreateClass(
|
||||
name=FuncName,
|
||||
TypeKind='function',
|
||||
lineno=0
|
||||
)
|
||||
FuncNode.set('file', '<stdin>')
|
||||
self.SymbolTable[FuncName] = FuncNode.attributes
|
||||
elif isinstance(node, ast.AnnAssign):
|
||||
if isinstance(node.target, ast.Name):
|
||||
VarName = node.target.id
|
||||
# 检查是否是 t.CTypedef 格式，如果是则跳过（由 HandleClassDef 处理）
|
||||
is_ctypedef = False
|
||||
if node.annotation:
|
||||
if isinstance(node.annotation, ast.Attribute):
|
||||
if isinstance(node.annotation.value, ast.Name):
|
||||
if node.annotation.value.id == 't' and node.annotation.attr == 'CTypedef':
|
||||
is_ctypedef = True
|
||||
elif isinstance(node.annotation, ast.BinOp) and isinstance(node.annotation.op, ast.BitOr):
|
||||
def _has_ctype(ann):
|
||||
if isinstance(ann, ast.Attribute):
|
||||
return hasattr(ann.value, 'id') and ann.value.id == 't' and ann.attr == 'CTypedef'
|
||||
if isinstance(ann, ast.BinOp) and isinstance(ann.op, ast.BitOr):
|
||||
return _has_ctype(ann.left) or _has_ctype(ann.right)
|
||||
if isinstance(ann, ast.Subscript):
|
||||
return _has_ctype(ann.value)
|
||||
return False
|
||||
if _has_ctype(node.annotation):
|
||||
is_ctypedef = True
|
||||
if not is_ctypedef:
|
||||
IsPtr = False
|
||||
if node.annotation:
|
||||
try:
|
||||
TypeInfo = self.TypeMergeHandler.GetCTypeInfo(node.annotation)
|
||||
IsPtr = TypeInfo.IsPtr if TypeInfo else False
|
||||
except Exception as _e:
|
||||
from lib.core.VLogger import get_logger as _vlog
|
||||
from lib.constants.config import mode as _ConfigMode
|
||||
if _ConfigMode == "strict":
|
||||
raise
|
||||
_vlog().warning(f"获取类型信息失败(ParsePythonFile): {_e}", "Exception")
|
||||
var_node = SymbolNode.CreateClass(
|
||||
name=VarName,
|
||||
TypeKind='variable',
|
||||
lineno=0
|
||||
)
|
||||
var_node.set('IsPtr', IsPtr)
|
||||
var_node.set('file', '<stdin>')
|
||||
self.SymbolTable[VarName] = var_node.attributes
|
||||
except Exception as e:
|
||||
print(f'Warning: Failed to parse Python file {FilePath}: {e}')
|
||||
|
||||
# ParseCFile 已移除 - .h/.c 文件的旧正则解析路径不再使用
|
||||
# 类型解析现在直接通过 CTypeInfo 对象进行，不经过中间字符串格式
|
||||
Reference in New Issue
Block a user