可用的回归测试通过的标准版本

This commit is contained in:
2026-06-18 00:39:43 +08:00
parent bffb0cb6b7
commit e02c867edf
365 changed files with 22562 additions and 24532 deletions

42
lib/StubGen/Config.py Normal file
View File

@@ -0,0 +1,42 @@
"""StubGen 配置模块"""
import json
from dataclasses import dataclass, field
from typing import List, Dict, Optional
@dataclass
class StubGenConfig:
"""存根生成器配置"""
InputDir: Optional[str] = None
OutputDir: str = "./stubs"
InputFiles: List[str] = field(default_factory=list)
IncludePatterns: List[str] = field(default_factory=lambda: ["*.h", "*.c", "*.py"])
ExcludePatterns: List[str] = field(default_factory=list)
TypeMappings: Dict[str, str] = field(default_factory=dict)
PreserveStructure: bool = True # 保持目录结构
GenerateGuards: bool = True # 生成宏守卫
verbose: bool = False
DryRun: bool = False
@classmethod
def from_dict(cls, data: Dict) -> 'StubGenConfig':
"""从字典创建配置"""
return cls(
InputDir=data.get('InputDir'),
OutputDir=data.get('OutputDir', './stubs'),
InputFiles=data.get('InputFiles', []),
IncludePatterns=data.get('IncludePatterns', ['*.h', '*.c', '*.py']),
ExcludePatterns=data.get('ExcludePatterns', []),
TypeMappings=data.get('TypeMappings', {}),
PreserveStructure=data.get('PreserveStructure', True),
GenerateGuards=data.get('GenerateGuards', True),
verbose=data.get('verbose', False),
DryRun=data.get('DryRun', False),
)
@classmethod
def from_file(cls, FilePath: str) -> 'StubGenConfig':
"""从 JSON 文件加载配置"""
with open(FilePath, 'r', encoding='utf-8') as f:
data = json.load(f)
return cls.from_dict(data)

967
lib/StubGen/Converter.py Normal file
View File

@@ -0,0 +1,967 @@
"""StubGen 转换器模块 - 将 Python 文件转换为存根格式"""
import ast
import re
from typing import List, Dict, Optional, Tuple
class PythonToStubConverter:
"""将 Python 文件转换为存根格式"""
# __include 别名映射表 {alias: ModuleName}
IncludeAliasMap: Dict[str, str] = {}
# 变量排除列表 - 匹配这些模式的变量不会被添加到存根文件
VarExcludeList: List[str] = []
# 宏排除列表 - 匹配这些模式的宏不会被添加到存根文件
MacroExcludeList: List[str] = []
@classmethod
def SetIncludeAliasMap(cls, alias_map: Dict[str, str]):
"""设置 __include 别名映射表"""
cls.IncludeAliasMap = alias_map
@classmethod
def SetVarExcludeList(cls, exclude_list: List[str]):
"""设置变量排除列表"""
cls.VarExcludeList = exclude_list
@classmethod
def SetMacroExcludeList(cls, exclude_list: List[str]):
"""设置宏排除列表"""
cls.MacroExcludeList = exclude_list
@classmethod
def _ShouldExcludeVar(cls, VarName: str) -> bool:
"""检查变量是否应该被排除"""
import fnmatch
for pattern in cls.VarExcludeList:
if fnmatch.fnmatch(VarName, pattern):
return True
return False
@classmethod
def _ShouldExcludeMacro(cls, MacroName: str) -> bool:
"""检查宏是否应该被排除"""
import fnmatch
for pattern in cls.MacroExcludeList:
if fnmatch.fnmatch(MacroName, pattern):
return True
return False
@staticmethod
def convert(PyContent: str, ModuleName: str) -> str:
"""将 Python 代码转换为存根格式"""
lines = PyContent.split('\n')
StubLines = []
# 添加文件头
StubLines.append('"""')
StubLines.append(f'Auto-generated Python stub file from {ModuleName}.py')
StubLines.append(f'Module: {ModuleName}')
StubLines.append('"""')
StubLines.append('')
# 解析 Python 代码,检查是否已经导入了 t 和 c
import ast
HasImportT = False
HasImportC = False
try:
tree = ast.parse(PyContent)
for node in tree.body:
if isinstance(node, ast.Import):
for alias in node.names:
if alias.name == 't':
HasImportT = True
elif alias.name == 'c':
HasImportC = True
except:
pass
# 添加默认导入(如果源代码中没有)
if not HasImportT:
StubLines.append('import t')
if not HasImportC:
StubLines.append('import c')
if not HasImportT or not HasImportC:
StubLines.append('')
# 添加 c.CPragma("once")
# StubLines.append('c.CPragma("once")')
StubLines.append('')
# 生成文件级别宏守卫名称
#FileGuardName = f'__{ModuleName.upper()}_DEFINE__'
# 添加文件开头宏守卫
#StubLines.append(f'c.CIfndef({FileGuardName})')
#StubLines.append(f'{FileGuardName}: t.CDefine')
#StubLines.append('')
# 解析 Python 代码,提取类型定义
try:
tree = ast.parse(PyContent)
# 第一遍扫描:收集 Postdefinition 变量
PostdefVars = {} # {ClassName: [(VarName, TypeStr, node), ...]}
for node in tree.body:
if isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name):
TypeStr = PythonToStubConverter._GetTypeString(node.annotation)
# 检查是否是 Postdefinition 类型
if 't.Postdefinition' in TypeStr:
# 提取引用的类名
ClassName = PythonToStubConverter._ExtractPostdefClass(node.annotation)
if ClassName:
if ClassName not in PostdefVars:
PostdefVars[ClassName] = []
PostdefVars[ClassName].append((node.target.id, TypeStr, node))
# 按原始顺序处理节点,保持代码的原始顺序
PrevType = None # 记录前一个节点的类型
for node in tree.body:
CurrentType = None
if isinstance(node, ast.ClassDef):
CurrentType = 'class'
# 如果前一个不是类,添加空行分隔
if PrevType is not None and PrevType != 'class':
StubLines.append('')
# 检查是否有对应的 Postdefinition 变量
ClassPostdefs = PostdefVars.get(node.name, [])
ClassLines = PythonToStubConverter._ConvertClass(
node, SourceLines=lines, PostdefVars=ClassPostdefs
)
StubLines.extend(ClassLines)
elif isinstance(node, ast.FunctionDef):
CurrentType = 'function'
# 如果前一个是类或者是第一个函数,添加空行分隔
if PrevType == 'class' or (PrevType is not None and PrevType != 'function'):
StubLines.append('')
FuncLines = PythonToStubConverter._ConvertFunction(node, SourceLines=lines)
StubLines.extend(FuncLines)
StubLines.append('') # 函数后空行
elif isinstance(node, ast.AnnAssign):
CurrentType = 'variable'
# 跳过 Postdefinition 变量(已经在类中处理)
TypeStr = PythonToStubConverter._GetTypeString(node.annotation)
if 't.Postdefinition' in TypeStr:
continue
# 检查是否带有 t.CStatic 标志,如果有则不排除
HasStatic = 't.CStatic' in TypeStr
# 检查变量是否应该被排除
ShouldExclude = False
if isinstance(node.target, ast.Name) and not HasStatic:
VarName = node.target.id
if PythonToStubConverter._ShouldExcludeVar(VarName):
ShouldExclude = True
if ShouldExclude:
continue
# 如果前一个不是变量,添加空行分隔
if PrevType is not None and PrevType != 'variable':
StubLines.append('')
VarLines = PythonToStubConverter._ConvertVariable(node, SourceLines=lines)
# 移除多余的空行
VarLines = [line for line in VarLines if line.strip()]
StubLines.extend(VarLines)
elif isinstance(node, ast.Import):
# 处理导入语句
CurrentType = 'import'
ImportStr = PythonToStubConverter._GetImportString(node, SourceLines=lines)
if ImportStr:
if PrevType is not None and PrevType != 'import':
StubLines.append('')
StubLines.append(ImportStr)
elif isinstance(node, ast.Assign):
CurrentType = 'variable'
# 处理模块级别的全局变量(无类型标注)
if PrevType is not None and PrevType != 'variable':
StubLines.append('')
AssignLines = PythonToStubConverter._ConvertGlobalAssign(node)
StubLines.extend(AssignLines)
elif isinstance(node, ast.ImportFrom):
# 处理 from ... import ... 语句
CurrentType = 'import'
ImportStr = PythonToStubConverter._GetImportFromString(node, SourceLines=lines)
if ImportStr:
if PrevType is not None and PrevType != 'import':
StubLines.append('')
StubLines.append(ImportStr)
elif isinstance(node, ast.Expr):
# 处理模块级别的宏调用,如 c.CIf(...), c.CEndif()
ExprStr = PythonToStubConverter._GetExprString(node.value)
if ExprStr:
CurrentType = 'macro'
StubLines.append(ExprStr)
StubLines.append('') # 宏后空行
elif isinstance(node, ast.If):
# 处理模块级别的 if 宏条件,如 if c.CIf(FF_MULTI_PARTITION):
IfStr = PythonToStubConverter._GetModuleIfMacroString(node)
if IfStr:
CurrentType = 'macro'
StubLines.append(IfStr)
StubLines.append('') # 宏后空行
if CurrentType:
PrevType = CurrentType
# 添加文件结尾宏守卫
#StubLines.append('')
#StubLines.append('c.CEndif()')
except SyntaxError as e:
# 如果解析失败,添加注释说明
StubLines.append(f'# Warning: Failed to parse Python code: {e}')
StubLines.append('# Original content:')
StubLines.append('"""')
StubLines.append(PyContent[:1000]) # 只显示前1000字符
StubLines.append('"""')
return '\n'.join(StubLines)
@staticmethod
def _ConvertClass(node: ast.ClassDef, SourceLines: List[str] = None, PostdefVars: List[Tuple[str, str, ast.AnnAssign]] = None, indent: int = 0) -> List[str]:
"""转换类定义
Args:
node: 类定义节点
SourceLines: 源代码行列表
PostdefVars: Postdefinition 变量列表 [(VarName, TypeStr, node), ...]
indent: 缩进级别(用于嵌套类)
"""
lines = []
PostdefVars = PostdefVars or []
IndentStr = ' ' * indent
# 处理类装饰器
for decorator in node.decorator_list:
DecoratorStr = PythonToStubConverter._GetDecoratorString(decorator)
if DecoratorStr:
lines.append(f'{IndentStr}@{DecoratorStr}')
# 检查是否是结构体、联合体或枚举
BaseNames = [base.attr if isinstance(base, ast.Attribute) else base.id
for base in node.bases
if isinstance(base, (ast.Name, ast.Attribute))]
# 保留原始基类
if node.bases:
BaseStrs = []
for base in node.bases:
if isinstance(base, ast.Name):
BaseStrs.append(base.id)
elif isinstance(base, ast.Attribute):
BaseStrs.append(f'{base.value.id}.{base.attr}')
elif isinstance(base, ast.Subscript):
# 处理泛型类型,如 memory_block_t[MAX_ORDER + 1]
BaseStrs.append(PythonToStubConverter._GetTypeString(base))
ClassTypeParamStr = ''
if hasattr(node, 'type_params') and node.type_params:
param_names = [tp.name for tp in node.type_params]
ClassTypeParamStr = f'[{", ".join(param_names)}]'
if BaseStrs:
lines.append(f'{IndentStr}class {node.name}{ClassTypeParamStr}({", ".join(BaseStrs)}):')
else:
lines.append(f'{IndentStr}class {node.name}{ClassTypeParamStr}:')
else:
ClassTypeParamStr = ''
if hasattr(node, 'type_params') and node.type_params:
param_names = [tp.name for tp in node.type_params]
ClassTypeParamStr = f'[{", ".join(param_names)}]'
lines.append(f'{IndentStr}class {node.name}{ClassTypeParamStr}:')
# 处理类成员
HasMembers = False
seen_member_names = set()
for item in node.body:
if isinstance(item, ast.AnnAssign) and isinstance(item.target, ast.Name):
seen_member_names.add(item.target.id)
init_members = []
for item in node.body:
if isinstance(item, ast.FunctionDef) and item.name == '__init__':
for stmt in item.body:
if (isinstance(stmt, ast.AnnAssign)
and isinstance(stmt.target, ast.Attribute)
and isinstance(stmt.target.value, ast.Name)
and stmt.target.value.id == 'self'):
init_members.append(stmt)
untyped_self_members = []
for m_name in [m.target.attr for m in init_members if isinstance(m.target, ast.Attribute)]:
seen_member_names.add(m_name)
for item in node.body:
if isinstance(item, ast.FunctionDef):
for stmt in item.body:
if (isinstance(stmt, ast.Assign)
and len(stmt.targets) == 1
and isinstance(stmt.targets[0], ast.Attribute)
and isinstance(stmt.targets[0].value, ast.Name)
and stmt.targets[0].value.id == 'self'):
attr_name = stmt.targets[0].attr
if attr_name not in seen_member_names:
seen_member_names.add(attr_name)
untyped_self_members.append(stmt)
for item in list(node.body) + init_members + untyped_self_members:
if isinstance(item, ast.Assign):
if (len(item.targets) == 1
and isinstance(item.targets[0], ast.Attribute)
and isinstance(item.targets[0].value, ast.Name)
and item.targets[0].value.id == 'self'):
HasMembers = True
MemberIndent = IndentStr + ' '
attr_name = item.targets[0].attr
if attr_name in seen_member_names:
continue
InferredType = 'int'
if item.value and isinstance(item.value, ast.Constant):
if isinstance(item.value.value, float):
InferredType = 'float'
elif isinstance(item.value.value, bool):
InferredType = 'bool'
elif isinstance(item.value.value, str):
InferredType = 'str'
lines.append(f'{MemberIndent}{attr_name}: {InferredType}')
else:
HasMembers = True
MemberIndent = IndentStr + ' '
if SourceLines and hasattr(item, 'lineno'):
StartLine = item.lineno - 1
EndLine = item.EndLineno if hasattr(item, 'EndLineno') and item.EndLineno else StartLine + 1
for i in range(StartLine, min(EndLine, len(SourceLines))):
lines.append(MemberIndent + SourceLines[i].lstrip())
elif isinstance(item, ast.AnnAssign):
HasMembers = True
TypeStr = PythonToStubConverter._GetTypeString(item.annotation)
MemberIndent = IndentStr + ' '
# 检查是否是宏变量(类型包含 t.CDefine
if 't.CDefine' in TypeStr:
# 检查宏变量是否应该被排除
if isinstance(item.target, ast.Name):
MacroName = item.target.id
if PythonToStubConverter._ShouldExcludeMacro(MacroName):
continue # 跳过该宏
# 宏变量保持原样(包括赋值)
if SourceLines and hasattr(item, 'lineno'):
StartLine = item.lineno - 1
EndLine = item.EndLineno if hasattr(item, 'EndLineno') and item.EndLineno else StartLine + 1
for i in range(StartLine, min(EndLine, len(SourceLines))):
lines.append(MemberIndent + SourceLines[i].lstrip())
else:
lines.append(f'{MemberIndent}{item.target.id}: {TypeStr}')
else:
if isinstance(item.target, ast.Attribute):
MemberName = item.target.attr
else:
MemberName = item.target.id
FinalTypeStr = TypeStr
if (isinstance(item.annotation, ast.Subscript)
and isinstance(item.annotation.value, ast.Name)
and item.annotation.value.id == 'list'):
slice_node = item.annotation.slice
has_size = False
if isinstance(slice_node, ast.Tuple) and len(slice_node.elts) == 2:
count_node = slice_node.elts[1]
if isinstance(count_node, ast.Constant) and isinstance(count_node.value, int) and count_node.value > 0:
has_size = True
else:
try:
count_expr = ast.unparse(count_node)
except Exception:
count_expr = ''
if count_expr and count_expr != 'None':
has_size = True
elem_type_str = PythonToStubConverter._GetTypeString(slice_node.elts[0])
FinalTypeStr = f'list[{elem_type_str}, {count_expr}]'
if not has_size:
elem_type_str = PythonToStubConverter._GetTypeString(slice_node if not isinstance(slice_node, ast.Tuple) else slice_node.elts[0])
init_len = 0
if item.value is not None:
if isinstance(item.value, ast.List):
init_len = len(item.value.elts)
elif isinstance(item.value, ast.Constant) and isinstance(item.value.value, str):
init_len = len(item.value.value) + 1
if init_len > 0:
FinalTypeStr = f'list[{elem_type_str}, {init_len}]'
else:
FinalTypeStr = f'list[{elem_type_str}, None]'
lines.append(f'{MemberIndent}{MemberName}: {FinalTypeStr}')
elif isinstance(item, ast.FunctionDef):
HasMembers = True
FuncStub = PythonToStubConverter._ConvertFunction(item, indent=indent+1, SourceLines=SourceLines, class_name=node.name)
lines.extend(FuncStub)
elif isinstance(item, ast.Expr):
# 处理宏调用,如 c.CIf(...), c.CEndif(), c.CElif(), c.CElse()
ExprStr = PythonToStubConverter._GetExprString(item.value)
if ExprStr:
lines.append(f'{IndentStr} {ExprStr}')
elif isinstance(item, ast.If):
# 处理 if c.CIf(...): 形式的宏条件
IfStr = PythonToStubConverter._get_if_macro_string(item, indent=indent+1)
if IfStr:
lines.append(IfStr)
elif isinstance(item, ast.ClassDef):
# 处理嵌套类
HasMembers = True
NestedClassLines = PythonToStubConverter._ConvertClass(item, SourceLines=SourceLines, PostdefVars=[], indent=indent+1)
# 移除嵌套类的宏守卫(因为已经在父类的宏守卫内)
FilteredLines = []
for line in NestedClassLines:
# 跳过宏守卫相关行
if line.strip().startswith('c.CIfndef(') or line.strip().startswith('c.CEndif()'):
continue
if ': t.CDefine' in line and '__' in line:
continue
FilteredLines.append(line)
lines.extend(FilteredLines)
if not HasMembers:
lines.append(f'{IndentStr} pass')
# 添加 Postdefinition 变量
if PostdefVars:
lines.append('')
for VarName, TypeStr, _ in PostdefVars:
lines.append(f'{VarName}: {TypeStr}')
return lines
@staticmethod
def _ConvertFunction(node: ast.FunctionDef, indent: int = 0, SourceLines: List[str] = None, class_name: str = None) -> List[str]:
"""转换函数定义"""
lines = []
IndentStr = ' ' * indent
# 检查是否是宏函数(返回类型包含 t.CDefine
is_macro_func = False
if node.returns:
ReturnType = PythonToStubConverter._GetTypeString(node.returns)
if 't.CDefine' in ReturnType:
is_macro_func = True
# 如果是宏函数,检查是否应该被排除
if is_macro_func:
if PythonToStubConverter._ShouldExcludeMacro(node.name):
return lines # 返回空列表,排除该宏
# 保持原样(包括代码块)
if SourceLines:
StartLine = node.lineno - 1
EndLine = node.EndLineno if hasattr(node, 'EndLineno') and node.EndLineno else StartLine + 1
for i in range(StartLine, min(EndLine, len(SourceLines))):
lines.append(SourceLines[i])
if lines and lines[-1].strip() and not lines[-1].rstrip().endswith(':'):
pass
else:
lines.append(f'{IndentStr} pass')
return lines
# 处理函数装饰器
for decorator in node.decorator_list:
DecoratorStr = PythonToStubConverter._GetDecoratorString(decorator)
if DecoratorStr:
lines.append(f'{IndentStr}@{DecoratorStr}')
# 构建参数列表
params = []
for arg_idx, arg in enumerate(node.args.args):
ArgName = arg.arg
if arg.annotation:
TypeStr = PythonToStubConverter._GetTypeString(arg.annotation)
params.append(f'{ArgName}: {TypeStr}')
elif class_name and arg_idx == 0 and ArgName == 'self':
params.append(f'self: {class_name}')
else:
params.append(ArgName)
# 处理 *args
if node.args.vararg:
ArgName = node.args.vararg.arg
if node.args.vararg.annotation:
TypeStr = PythonToStubConverter._GetTypeString(node.args.vararg.annotation)
params.append(f'*{ArgName}: {TypeStr}')
else:
params.append(f'*{ArgName}')
# 处理 **kwargs
if node.args.kwarg:
ArgName = node.args.kwarg.arg
if node.args.kwarg.annotation:
TypeStr = PythonToStubConverter._GetTypeString(node.args.kwarg.annotation)
params.append(f'**{ArgName}: {TypeStr}')
else:
params.append(f'**{ArgName}')
ParamStr = ', '.join(params)
# 返回类型 - 保持原始类型不添加t.State
if node.returns:
ReturnType = PythonToStubConverter._GetTypeString(node.returns)
else:
ReturnType = 't.CInt'
TypeParamStr = ''
if hasattr(node, 'type_params') and node.type_params:
param_names = [tp.name for tp in node.type_params]
TypeParamStr = f'[{", ".join(param_names)}]'
lines.append(f'{IndentStr}def {node.name}{TypeParamStr}({ParamStr}) -> {ReturnType}: pass')
return lines
@staticmethod
def _ConvertVariable(node: ast.AnnAssign, SourceLines: List[str] = None) -> List[str]:
"""转换变量定义"""
lines = []
if isinstance(node.target, ast.Name):
VarName = node.target.id
TypeStr = PythonToStubConverter._GetTypeString(node.annotation)
# 检查是否是宏变量(类型包含 t.CDefine
if 't.CDefine' in TypeStr:
# 检查宏变量是否应该被排除
if PythonToStubConverter._ShouldExcludeMacro(VarName):
return lines # 返回空列表,排除该宏
# 宏变量保持原样(包括赋值)
if SourceLines and hasattr(node, 'lineno'):
StartLine = node.lineno - 1
EndLine = node.EndLineno if hasattr(node, 'EndLineno') and node.EndLineno else StartLine + 1
for i in range(StartLine, min(EndLine, len(SourceLines))):
lines.append(SourceLines[i])
else:
lines.append(f'{VarName}: {TypeStr}')
return lines
# 非宏变量:不加 t.State
# 含有 typedef 的变量不加 extern普通变量需要加 extern
has_typedef = 't.CTypedef' in TypeStr or 'CTypedef' in TypeStr
if has_typedef and node.value is not None:
ValueTypeStr = PythonToStubConverter._GetTypeString(node.value)
lines.append(f'{VarName}: {TypeStr} = {ValueTypeStr}')
else:
if not has_typedef and 't.CExtern' not in TypeStr and 't.CStatic' not in TypeStr:
TypeStr = f't.CExtern | {TypeStr}'
lines.append(f'{VarName}: {TypeStr}')
return lines
@staticmethod
def _ConvertGlobalAssign(node: ast.Assign) -> List[str]:
"""转换模块级别的全局变量(无类型标注)"""
lines = []
if not node.targets or not isinstance(node.targets[0], ast.Name):
return lines
VarName = node.targets[0].id
# 跳过私有变量(以下划线开头)
if VarName.startswith('_'):
return lines
# 从值推断类型
inferred_type = PythonToStubConverter._InferTypeFromValue(node.value)
if inferred_type:
lines.append(f'{VarName}: {inferred_type}')
else:
lines.append(f'{VarName}: t.CInt')
return lines
@staticmethod
def _InferTypeFromValue(value: ast.AST) -> str:
"""从值推断 C 类型"""
if isinstance(value, ast.Constant):
if isinstance(value.value, int):
return 't.CInt'
elif isinstance(value.value, float):
return 't.CDouble'
elif isinstance(value.value, str):
return 't.CCharPtr'
elif isinstance(value.value, bool):
return 't.CInt'
elif isinstance(value, ast.List):
return 't.CVoidPtr'
elif isinstance(value, ast.Dict):
return 't.CVoidPtr'
elif isinstance(value, ast.Name):
return 't.CInt'
elif isinstance(value, ast.BinOp):
left_type = PythonToStubConverter._InferTypeFromValue(value.left)
if left_type:
return left_type
return 't.CInt'
elif isinstance(value, ast.Call):
# 函数调用返回值,假设为指针类型
if isinstance(value.func, ast.Name):
func_name = value.func.id
if func_name == 'malloc':
return 't.CVoidPtr'
elif func_name in ('ctypes.cast', 'cast'):
return 't.CVoidPtr'
return 't.CVoidPtr'
return None
@staticmethod
def _GetTypeString(annotation: ast.AST) -> str:
"""获取类型字符串"""
if isinstance(annotation, ast.Name):
return annotation.id
elif isinstance(annotation, ast.Attribute):
if isinstance(annotation.value, ast.Name):
return f'{annotation.value.id}.{annotation.attr}'
else:
# 处理嵌套属性访问,如 t.attr.packed
ParentStr = PythonToStubConverter._GetTypeString(annotation.value)
return f'{ParentStr}.{annotation.attr}'
elif isinstance(annotation, ast.Subscript):
base = PythonToStubConverter._GetTypeString(annotation.value)
if isinstance(annotation.slice, ast.Tuple):
slice_strs = [PythonToStubConverter._GetTypeString(e) for e in annotation.slice.elts]
SliceStr = ', '.join(slice_strs)
else:
SliceStr = PythonToStubConverter._GetTypeString(annotation.slice)
return f'{base}[{SliceStr}]'
elif isinstance(annotation, ast.BinOp):
# 处理 BinOp如 t.CExtern | fd_table.__set_default__(...)
left = annotation.left
right = annotation.right
# 检查右侧是否是 __set_default__ 调用
if isinstance(right, ast.Call) and isinstance(right.func, ast.Attribute) and right.func.attr in ('__set_default__', '__set_default'):
# 去除 __set_default__保留左侧右侧替换为函数名部分
LeftStr = PythonToStubConverter._GetTypeString(left)
RightStr = PythonToStubConverter._GetTypeString(right.func.value)
if isinstance(annotation.op, ast.BitOr):
return f'{LeftStr} | {RightStr}'
LeftStr = PythonToStubConverter._GetTypeString(left)
RightStr = PythonToStubConverter._GetTypeString(right)
if isinstance(annotation.op, ast.BitOr):
return f'{LeftStr} | {RightStr}'
elif isinstance(annotation.op, ast.Add):
return f'{LeftStr} + {RightStr}'
elif isinstance(annotation.op, ast.Sub):
return f'{LeftStr} - {RightStr}'
elif isinstance(annotation.op, ast.Mult):
return f'{LeftStr} * {RightStr}'
elif isinstance(annotation.op, ast.Div):
return f'{LeftStr} / {RightStr}'
elif isinstance(annotation.op, ast.Mod):
return f'{LeftStr} % {RightStr}'
elif isinstance(annotation, ast.UnaryOp):
if isinstance(annotation.op, ast.Not):
operand = PythonToStubConverter._GetTypeString(annotation.operand)
return f'not {operand}'
elif isinstance(annotation.op, ast.Invert):
operand = PythonToStubConverter._GetTypeString(annotation.operand)
return f'~{operand}'
elif isinstance(annotation.op, ast.USub):
operand = PythonToStubConverter._GetTypeString(annotation.operand)
return f'-{operand}'
elif isinstance(annotation, ast.Compare):
# 处理比较操作,如 FF_MAX_SS != FF_MIN_SS
left = PythonToStubConverter._GetTypeString(annotation.left)
comparators = [PythonToStubConverter._GetTypeString(c) for c in annotation.comparators]
ops = []
for op in annotation.ops:
if isinstance(op, ast.Eq):
ops.append('==')
elif isinstance(op, ast.NotEq):
ops.append('!=')
elif isinstance(op, ast.Lt):
ops.append('<')
elif isinstance(op, ast.LtE):
ops.append('<=')
elif isinstance(op, ast.Gt):
ops.append('>')
elif isinstance(op, ast.GtE):
ops.append('>=')
else:
ops.append('?')
result = left
for i, op in enumerate(ops):
result += f' {op} {comparators[i]}'
return result
elif isinstance(annotation, ast.Constant):
return repr(annotation.value)
elif isinstance(annotation, ast.Index):
return PythonToStubConverter._GetTypeString(annotation.value)
elif isinstance(annotation, ast.List):
# 处理列表,如 [MAX_FILE_DESCRIPTORS]
elements = [PythonToStubConverter._GetTypeString(elem) for elem in annotation.elts]
return f'[{", ".join(elements)}]'
elif isinstance(annotation, ast.Call):
# 处理函数调用形式的类型注解,如 t.Postdefinition(xxa) 或 callable(t.CVoid, app_id = t.CInt)
# 如果是 __set_default__ 调用,只返回函数名部分
if isinstance(annotation.func, ast.Attribute) and annotation.func.attr in ('__set_default__', '__set_default'):
return PythonToStubConverter._GetTypeString(annotation.func.value)
FuncStr = PythonToStubConverter._GetTypeString(annotation.func)
ArgsStr = ', '.join([PythonToStubConverter._GetTypeString(arg) for arg in annotation.args])
# 处理关键字参数
keywords_str = ', '.join([f'{kw.arg} = {PythonToStubConverter._GetTypeString(kw.value)}' for kw in annotation.keywords])
# 合并位置参数和关键字参数
if ArgsStr and keywords_str:
return f'{FuncStr}({ArgsStr}, {keywords_str})'
elif keywords_str:
return f'{FuncStr}({keywords_str})'
else:
return f'{FuncStr}({ArgsStr})'
return 't.CVoid'
@staticmethod
def _ExtractPostdefClass(annotation: ast.AST) -> Optional[str]:
"""从 Postdefinition 类型注解中提取类名
例如t.Postdefinition(__buddy_system) -> '__buddy_system'
t.Postdefinition(xxx) | t.CTypedef -> 'xxx'
"""
# 处理 BinOp (如: t.Postdefinition(xxx) | t.CTypedef)
if isinstance(annotation, ast.BinOp):
# 递归检查左操作数
left_result = PythonToStubConverter._ExtractPostdefClass(annotation.left)
if left_result:
return left_result
# 递归检查右操作数
right_result = PythonToStubConverter._ExtractPostdefClass(annotation.right)
if right_result:
return right_result
return None
# 处理函数调用 (如: t.Postdefinition(__buddy_system))
if isinstance(annotation, ast.Call):
FuncStr = PythonToStubConverter._GetTypeString(annotation.func)
if FuncStr == 't.Postdefinition' and annotation.args:
# 提取第一个参数
first_arg = annotation.args[0]
if isinstance(first_arg, ast.Name):
return first_arg.id
elif isinstance(first_arg, ast.Attribute):
return f'{first_arg.value.id}.{first_arg.attr}'
return None
@staticmethod
def _GetImportString(node: ast.Import, SourceLines: List[str] = None) -> str:
"""获取导入语句字符串"""
parts = []
IncludeAliasMap = PythonToStubConverter.IncludeAliasMap
for alias in node.names:
ModuleName = alias.name
# 检查是否是 __include.xxx 格式的导入
if ModuleName.startswith('__include.'):
# 提取别名xxx 部分)
SubModule = ModuleName[len('__include.'):]
if SubModule in IncludeAliasMap:
# 替换为实际模块路径
RealModule = IncludeAliasMap[SubModule]
if alias.asname:
parts.append(f'{RealModule} as {alias.asname}')
else:
parts.append(f'{RealModule} as {SubModule}')
else:
# 不在映射表中,保持原样
if alias.asname:
parts.append(f'{ModuleName} as {alias.asname}')
else:
parts.append(ModuleName)
else:
if alias.asname:
parts.append(f'{alias.name} as {alias.asname}')
else:
parts.append(alias.name)
result = f'import {", ".join(parts)}'
# 提取并保留注释
if SourceLines and hasattr(node, 'lineno'):
LineIdx = node.lineno - 1
if 0 <= LineIdx < len(SourceLines):
line = SourceLines[LineIdx]
CommentMatch = re.search(r'#.*$', line)
if CommentMatch:
result += ' ' + CommentMatch.group(0)
return result
@staticmethod
def _GetImportFromString(node: ast.ImportFrom, SourceLines: List[str] = None) -> str:
"""获取 from ... import ... 语句字符串"""
module = node.module or ''
parts = []
IncludeAliasMap = PythonToStubConverter.IncludeAliasMap
# 处理相对导入 (from . import xxx 或 from ..package import xxx)
if node.level > 0:
if module:
if node.level == 1:
full_module = f'.{module}'
else:
full_module = '.' * node.level + module
else:
full_module = '.' * node.level
for alias in node.names:
if alias.asname:
parts.append(f'{alias.name} as {alias.asname}')
else:
parts.append(alias.name)
return f'from {full_module} import {", ".join(parts)}'
# 提取注释(在生成结果前获取)
comment = ''
if SourceLines and hasattr(node, 'lineno'):
LineIdx = node.lineno - 1
if 0 <= LineIdx < len(SourceLines):
line = SourceLines[LineIdx]
CommentMatch = re.search(r'#.*$', line)
if CommentMatch:
comment = ' ' + CommentMatch.group(0)
# 检查是否是 from __include import xxx 格式
if module == '__include':
for alias in node.names:
name = alias.name
if name in IncludeAliasMap:
# 替换为实际模块路径
RealModule = IncludeAliasMap[name]
if alias.asname:
parts.append(f'import {RealModule} as {alias.asname}{comment}')
else:
parts.append(f'import {RealModule} as {name}{comment}')
else:
# 不在映射表中,保持原样
if alias.asname:
parts.append(f'from {module} import {name} as {alias.asname}{comment}')
else:
parts.append(f'from {module} import {name}{comment}')
return '\n'.join(parts)
# 普通 from ... import ... 语句
for alias in node.names:
if alias.asname:
parts.append(f'{alias.name} as {alias.asname}')
else:
parts.append(alias.name)
return f'from {module} import {", ".join(parts)}{comment}'
@staticmethod
def _GetDecoratorString(decorator: ast.AST) -> str:
"""获取装饰器字符串"""
if isinstance(decorator, ast.Name):
return decorator.id
elif isinstance(decorator, ast.Attribute):
if isinstance(decorator.value, ast.Name):
return f'{decorator.value.id}.{decorator.attr}'
return decorator.attr
elif isinstance(decorator, ast.Call):
# 处理带参数的装饰器,如 @c.Attribute(t.attr.packed)
FuncStr = PythonToStubConverter._GetDecoratorString(decorator.func)
ArgsStr = ', '.join([PythonToStubConverter._GetTypeString(arg) for arg in decorator.args])
return f'{FuncStr}({ArgsStr})'
return ''
@staticmethod
def _GetExprString(expr: ast.AST) -> str:
"""获取表达式字符串(用于宏调用)"""
if isinstance(expr, ast.Call):
# 处理函数调用,如 c.CIf(...), c.CEndif(), c.CUndef(...), c.CPragma(...)
FuncStr = PythonToStubConverter._GetDecoratorString(expr.func)
if FuncStr and ('CIf' in FuncStr or 'CEndif' in FuncStr or 'CElif' in FuncStr or 'CElse' in FuncStr or 'CUndef' in FuncStr or 'CPragma' in FuncStr):
ArgsStr = ', '.join([PythonToStubConverter._GetTypeString(arg) for arg in expr.args])
return f'{FuncStr}({ArgsStr})' if ArgsStr else f'{FuncStr}()'
return ''
@staticmethod
def _get_if_macro_string(node: ast.If, indent: int = 1) -> str:
"""获取类内部的 if 宏字符串(如 if c.CIf(...):"""
IndentStr = ' ' * indent
InnerIndent = ' ' * (indent + 1)
# 检查条件是否是宏调用
if isinstance(node.test, ast.Call):
FuncStr = PythonToStubConverter._GetDecoratorString(node.test.func)
if FuncStr and 'CIf' in FuncStr:
ArgsStr = ', '.join([PythonToStubConverter._GetTypeString(arg) for arg in node.test.args])
result = [f'{IndentStr}if {FuncStr}({ArgsStr}):' if ArgsStr else f'{IndentStr}if {FuncStr}():']
# 处理 if 体内的语句
for item in node.body:
if isinstance(item, ast.AnnAssign):
TypeStr = PythonToStubConverter._GetTypeString(item.annotation)
result.append(f'{InnerIndent}{item.target.id}: {TypeStr}')
elif isinstance(item, ast.Expr):
ExprStr = PythonToStubConverter._GetExprString(item.value)
if ExprStr:
result.append(f'{InnerIndent}{ExprStr}')
return '\n'.join(result)
return ''
@staticmethod
def _GetModuleIfMacroString(node: ast.If) -> str:
"""获取模块级别的 if 宏字符串(如 if c.CIf(FF_MULTI_PARTITION):"""
return PythonToStubConverter._ProcessIfMacro(node, indent=0)
@staticmethod
def _ProcessIfMacro(node: ast.If, indent: int = 0) -> str:
"""处理 if 宏节点,支持 elif 和 else"""
IndentStr = ' ' * indent
# 检查条件是否是宏调用
if isinstance(node.test, ast.Call):
FuncStr = PythonToStubConverter._GetDecoratorString(node.test.func)
if FuncStr and 'CIf' in FuncStr:
ArgsStr = ', '.join([PythonToStubConverter._GetTypeString(arg) for arg in node.test.args])
result = [f'{IndentStr}if {FuncStr}({ArgsStr}):' if ArgsStr else f'{IndentStr}if {FuncStr}():']
# 处理 if 体内的语句
for item in node.body:
result.extend(PythonToStubConverter._ProcessMacroBodyItem(item, indent + 1))
# 处理 elif 和 else
current = node
while current.orelse:
if len(current.orelse) == 1 and isinstance(current.orelse[0], ast.If):
# elif 情况
elif_node = current.orelse[0]
if isinstance(elif_node.test, ast.Call):
ElifFuncStr = PythonToStubConverter._GetDecoratorString(elif_node.test.func)
if ElifFuncStr and 'CIf' in ElifFuncStr:
ElifArgsStr = ', '.join([PythonToStubConverter._GetTypeString(arg) for arg in elif_node.test.args])
result.append(f'{IndentStr}elif {ElifFuncStr}({ElifArgsStr}):' if ElifArgsStr else f'{IndentStr}elif {ElifFuncStr}():')
for item in elif_node.body:
result.extend(PythonToStubConverter._ProcessMacroBodyItem(item, indent + 1))
current = elif_node
continue
break
else:
# else 情况
result.append(f'{IndentStr}else:')
for item in current.orelse:
result.extend(PythonToStubConverter._ProcessMacroBodyItem(item, indent + 1))
break
return '\n'.join(result)
return ''
@staticmethod
def _ProcessMacroBodyItem(item: ast.AST, indent: int = 0) -> list:
"""处理宏体内的单个语句"""
IndentStr = ' ' * indent
result = []
if isinstance(item, ast.ClassDef):
class_stub = PythonToStubConverter._ConvertClass(item)
for line in class_stub:
if line.strip():
result.append(f'{IndentStr}{line}')
else:
result.append(line)
elif isinstance(item, ast.FunctionDef):
FuncStub = PythonToStubConverter._ConvertFunction(item, indent=indent)
result.extend(FuncStub)
elif isinstance(item, ast.AnnAssign):
TypeStr = PythonToStubConverter._GetTypeString(item.annotation)
result.append(f'{IndentStr}{item.target.id}: {TypeStr}')
elif isinstance(item, ast.Expr):
ExprStr = PythonToStubConverter._GetExprString(item.value)
if ExprStr:
result.append(f'{IndentStr}{ExprStr}')
elif isinstance(item, ast.If):
# 处理嵌套的 if 宏
NestedIf = PythonToStubConverter._ProcessIfMacro(item, indent=indent)
if NestedIf:
result.append(NestedIf)
return result

311
lib/StubGen/Generator.py Normal file
View File

@@ -0,0 +1,311 @@
"""StubGen 生成器模块 - 存根生成器主类和命令行入口"""
import sys
import os
import re
import argparse
import logging
from pathlib import Path
from typing import List, Tuple
from lib.core.stub_generator import CHeaderParser, PythonStubGenerator, CTypeMapper
from lib.StubGen.Config import StubGenConfig
from lib.StubGen.Converter import PythonToStubConverter
class StubGen:
"""存根生成器主类"""
def __init__(self, config: StubGenConfig):
self.config = config
self.logger = self._SetupLogger()
self.GeneratedFiles: List[str] = []
self.FailedFiles: List[str] = []
# 应用自定义类型映射
if config.TypeMappings:
CTypeMapper.BASIC_TYPE_MAP.update(config.TypeMappings)
def _SetupLogger(self) -> logging.Logger:
"""设置日志"""
logger = logging.getLogger('StubGen')
logger.setLevel(logging.DEBUG if self.config.verbose else logging.INFO)
if not logger.handlers:
handler = logging.StreamHandler(sys.stdout)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s',
datefmt='%H:%M:%S'
)
handler.setFormatter(formatter)
logger.addHandler(handler)
return logger
def FindInputFiles(self) -> List[Tuple[str, str]]:
"""查找输入文件,返回 (文件路径, 相对路径) 列表"""
files = []
# 添加显式指定的文件
for FilePath in self.config.InputFiles:
if os.path.exists(FilePath):
RelPath = os.path.relpath(FilePath, self.config.InputDir or '.')
files.append((FilePath, RelPath))
else:
self.logger.warning(f"Input file not found: {FilePath}")
# 从输入目录查找
if self.config.InputDir and os.path.exists(self.config.InputDir):
InputPath = Path(self.config.InputDir)
for pattern in self.config.IncludePatterns:
for FilePath in InputPath.rglob(pattern):
# 检查是否在排除列表中
excluded = False
RelPath = os.path.relpath(str(FilePath), self.config.InputDir)
for exclude_pattern in self.config.ExcludePatterns:
if FilePath.match(exclude_pattern) or RelPath == exclude_pattern:
excluded = True
break
if not excluded:
files.append((str(FilePath), RelPath))
# 去重并保持顺序
seen = set()
UniqueFiles = []
for FilePath, RelPath in files:
key = FilePath
if key not in seen:
seen.add(key)
UniqueFiles.append((FilePath, RelPath))
return UniqueFiles
def _GenerateGuardName(self, FilePath: str) -> str:
"""生成宏守卫名称"""
# 获取文件名(不含扩展名)
BaseName = os.path.splitext(os.path.basename(FilePath))[0]
# 转换为大写,替换特殊字符为下划线
guard = re.sub(r'[^a-zA-Z0-9_]', '_', BaseName).upper()
guard = re.sub(r'_+', '_', guard) # 合并多个下划线
guard = guard.strip('_')
return f'{guard}_DEFINE_H'
def _GetOutputPath(self, RelPath: str) -> str:
"""获取输出文件路径"""
# 更改扩展名为 .pyi (Python stub file)
BaseName = os.path.splitext(RelPath)[0]
OutputRelPath = BaseName + '.pyi'
# 构建完整输出路径
if self.config.OutputDir:
OutputPath = os.path.join(self.config.OutputDir, OutputRelPath)
else:
OutputPath = OutputRelPath
return OutputPath
def GenerateStub(self, InputFile: str, RelPath: str) -> bool:
"""生成单个存根文件"""
try:
self.logger.info(f"Processing: {InputFile}")
# 确定输出文件路径
OutputFile = self._GetOutputPath(RelPath)
# 确保输出目录存在
OutputDir = os.path.dirname(OutputFile)
if OutputDir and not os.path.exists(OutputDir):
if not self.config.DryRun:
os.makedirs(OutputDir, exist_ok=True)
self.logger.debug(f"Created directory: {OutputDir}")
if self.config.DryRun:
self.logger.info(f"[DRY RUN] Would generate: {OutputFile}")
return True
# 根据文件类型选择处理方式
ext = os.path.splitext(InputFile)[1].lower()
if ext in ['.h', '.c']:
# C 文件:使用 CHeaderParser
content = self._GenerateFromC(InputFile, RelPath)
elif ext == '.py':
# Python 文件:使用 PythonToStubConverter
content = self._GenerateFromPy(InputFile, RelPath)
else:
self.logger.warning(f"Unsupported file type: {ext}")
return False
# 写入文件
with open(OutputFile, 'w', encoding='utf-8') as f:
f.write(content)
self.GeneratedFiles.append(OutputFile)
self.logger.info(f"Generated: {OutputFile}")
return True
except Exception as e:
self.logger.error(f"Failed to process {InputFile}: {e}")
import traceback
traceback.print_exc()
self.FailedFiles.append(InputFile)
return False
def _GenerateFromC(self, InputFile: str, RelPath: str) -> str:
"""从 C 文件生成存根"""
parser = CHeaderParser()
parser.parse_file(InputFile)
generator = PythonStubGenerator(parser)
ModuleName = os.path.splitext(os.path.basename(InputFile))[0]
content = generator.generate(ModuleName)
# 添加宏守卫(如果需要)
if self.config.GenerateGuards:
guard_name = self._GenerateGuardName(InputFile)
guard_comment = f"\n# Guard: {guard_name}\n"
content = content + guard_comment
return content
def _GenerateFromPy(self, InputFile: str, RelPath: str) -> str:
"""从 Python 文件生成存根"""
with open(InputFile, 'r', encoding='utf-8') as f:
PyContent = f.read()
ModuleName = os.path.splitext(os.path.basename(InputFile))[0]
content = PythonToStubConverter.convert(PyContent, ModuleName)
# 添加宏守卫(如果需要)
if self.config.GenerateGuards:
guard_name = self._GenerateGuardName(InputFile)
guard_comment = f"\n# Guard: {guard_name}\n"
content = content + guard_comment
return content
def run(self) -> bool:
"""运行生成器"""
self.logger.info("=" * 60)
self.logger.info("StubGen - C/H/Py to Python Stub Generator")
self.logger.info("=" * 60)
# 查找输入文件
InputFiles = self.FindInputFiles()
if not InputFiles:
self.logger.warning("No input files found!")
return False
self.logger.info(f"Found {len(InputFiles)} input file(s)")
for FilePath, RelPath in InputFiles:
self.logger.debug(f" - {RelPath}")
# 处理每个文件
SuccessCount = 0
for FilePath, RelPath in InputFiles:
if self.GenerateStub(FilePath, RelPath):
SuccessCount += 1
# 输出统计信息
self.logger.info("=" * 60)
self.logger.info(f"Summary: {SuccessCount}/{len(InputFiles)} files generated successfully")
if self.FailedFiles:
self.logger.warning(f"Failed files ({len(self.FailedFiles)}):")
for f in self.FailedFiles:
self.logger.warning(f" - {f}")
return SuccessCount == len(InputFiles)
def main():
parser = argparse.ArgumentParser(
description='StubGen - Generate Python stub files from C/H/Py sources',
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog='''
Examples:
# 使用配置文件
%(prog)s -c config.json
# 处理单个文件
%(prog)s -i input.h -o output.py
# 批量处理目录(保持目录结构)
%(prog)s -d ./kernel -o ./kernel/include
# 干运行(不实际生成文件)
%(prog)s -d ./kernel --dry-run
# 不生成宏守卫
%(prog)s -d ./kernel --no-guards
'''
)
# 输入选项
InputGroup = parser.add_mutually_exclusive_group(required=True)
InputGroup.add_argument('-c', '--config', help='Configuration file (JSON)')
InputGroup.add_argument('-i', '--input', help='Input file')
InputGroup.add_argument('-d', '--directory', help='Input directory')
# 输出选项
parser.add_argument('-o', '--output', help='Output directory')
# 其他选项
parser.add_argument('--include', action='append',
help='Include patterns (default: *.h, *.c, *.py)')
parser.add_argument('--exclude', action='append',
help='Exclude patterns')
parser.add_argument('--no-guards', action='store_true',
help='Do not generate guard macros')
parser.add_argument('--no-structure', action='store_true',
help='Do not preserve directory structure')
parser.add_argument('-v', '--verbose', action='store_true',
help='Verbose output')
parser.add_argument('--dry-run', action='store_true',
help='Dry run (do not create files)')
args = parser.parse_args()
# 加载配置
if args.config:
if not os.path.exists(args.config):
print(f"Error: Config file not found: {args.config}")
sys.exit(1)
config = StubGenConfig.from_file(args.config)
else:
config = StubGenConfig()
config.IncludePatterns = args.include or ['*.h', '*.c', '*.py']
config.ExcludePatterns = args.exclude or []
config.GenerateGuards = not args.no_guards
config.PreserveStructure = not args.no_structure
config.verbose = args.verbose
config.DryRun = args.dry_run
if args.input:
config.InputFiles = [args.input]
if args.output:
config.OutputDir = os.path.dirname(args.output) or '.'
elif args.directory:
config.InputDir = args.directory
if args.output:
config.OutputDir = args.output
# 创建生成器并运行
generator = StubGen(config)
# 如果指定了单个输出文件,直接处理
if args.input and args.output and not os.path.isdir(args.output):
RelPath = os.path.relpath(args.input, config.InputDir or '.')
success = generator.GenerateStub(args.input, RelPath)
else:
success = generator.run()
sys.exit(0 if success else 1)
if __name__ == '__main__':
main()

6
lib/StubGen/__init__.py Normal file
View File

@@ -0,0 +1,6 @@
"""StubGen 包 - 存根文件生成器"""
from lib.StubGen.Config import StubGenConfig
from lib.StubGen.Converter import PythonToStubConverter
from lib.StubGen.Generator import StubGen, main
__all__ = ['StubGenConfig', 'PythonToStubConverter', 'StubGen', 'main']