实现了切片存储

This commit is contained in:
2026-07-22 21:56:33 +08:00
parent 751dc72d61
commit 909792bc8f
80 changed files with 584 additions and 26479 deletions

View File

@@ -1,43 +0,0 @@
"""StubGen 配置模块"""
from __future__ import annotations
import json
from dataclasses import dataclass, field
from typing import List, Dict, Optional
@dataclass
class StubGenConfig:
"""存根生成器配置"""
InputDir: Optional[str] = None
OutputDir: str = "./stubs"
InputFiles: List[str] = field(default_factory=list)
IncludePatterns: List[str] = field(default_factory=lambda: ["*.h", "*.c", "*.py"])
ExcludePatterns: List[str] = field(default_factory=list)
TypeMappings: Dict[str, str] = field(default_factory=dict)
PreserveStructure: bool = True # 保持目录结构
GenerateGuards: bool = True # 生成宏守卫
verbose: bool = False
DryRun: bool = False
@classmethod
def from_dict(cls, data: dict) -> StubGenConfig:
"""从字典创建配置"""
return cls(
InputDir=data.get('InputDir'),
OutputDir=data.get('OutputDir', './stubs'),
InputFiles=data.get('InputFiles', []),
IncludePatterns=data.get('IncludePatterns', ['*.h', '*.c', '*.py']),
ExcludePatterns=data.get('ExcludePatterns', []),
TypeMappings=data.get('TypeMappings', {}),
PreserveStructure=data.get('PreserveStructure', True),
GenerateGuards=data.get('GenerateGuards', True),
verbose=data.get('verbose', False),
DryRun=data.get('DryRun', False),
)
@classmethod
def from_file(cls, FilePath: str) -> StubGenConfig:
"""从 JSON 文件加载配置"""
with open(FilePath, 'r', encoding='utf-8') as f:
data: dict = json.load(f)
return cls.from_dict(data)

View File

@@ -127,6 +127,23 @@ def _PyiTypeStr(buf: bytes, buf_size: t.CSizeT, pos: t.CSizeT,
ti += 1
return pos
# Call: 函数调用表达式 (用于装饰器如 @t.NoVTable())
if k == ast.ASTKind.Call:
cl: ast.Call | t.CPtr = (ast.Call | t.CPtr)(annot)
pos = _PyiTypeStr(buf, buf_size, pos, cl.func)
pos = _PyiEmit(buf, buf_size, pos, "(")
if cl.args is not None:
can: t.CSizeT = cl.args.__len__()
cai: t.CSizeT = 0
while cai < can:
if cai > 0:
pos = _PyiEmit(buf, buf_size, pos, ", ")
ca: ast.AST | t.CPtr = cl.args.get(cai)
pos = _PyiTypeStr(buf, buf_size, pos, ca)
cai += 1
pos = _PyiEmit(buf, buf_size, pos, ")")
return pos
# 未知类型,回退到 t.CInt
return _PyiEmit(buf, buf_size, pos, "t.CInt")
@@ -140,6 +157,19 @@ def _PyiFunction(buf: bytes, buf_size: t.CSizeT, pos: t.CSizeT,
fn: ast.FunctionDef | t.CPtr,
indent_level: int, class_name: str) -> t.CSizeT:
"""生成函数签名到 buf返回新的 pos"""
# 装饰器
if fn.decorator_list is not None:
dn: t.CSizeT = fn.decorator_list.__len__()
di: t.CSizeT = 0
while di < dn:
dec: ast.AST | t.CPtr = fn.decorator_list.get(di)
if dec is not None:
pos = _PyiIndent(buf, buf_size, pos, indent_level)
pos = _PyiEmit(buf, buf_size, pos, "@")
pos = _PyiTypeStr(buf, buf_size, pos, dec)
pos = _PyiEmit(buf, buf_size, pos, "\n")
di += 1
pos = _PyiIndent(buf, buf_size, pos, indent_level)
pos = _PyiEmit(buf, buf_size, pos, "def ")
if fn.name is not None:
@@ -210,11 +240,39 @@ def _PyiFunction(buf: bytes, buf_size: t.CSizeT, pos: t.CSizeT,
def _PyiClass(buf: bytes, buf_size: t.CSizeT, pos: t.CSizeT,
cls: ast.ClassDef | t.CPtr, indent_level: int) -> t.CSizeT:
"""生成类定义到 buf返回新的 pos"""
# 装饰器
if cls.decorator_list is not None:
dcn: t.CSizeT = cls.decorator_list.__len__()
dci: t.CSizeT = 0
while dci < dcn:
ddec: ast.AST | t.CPtr = cls.decorator_list.get(dci)
if ddec is not None:
pos = _PyiIndent(buf, buf_size, pos, indent_level)
pos = _PyiEmit(buf, buf_size, pos, "@")
pos = _PyiTypeStr(buf, buf_size, pos, ddec)
pos = _PyiEmit(buf, buf_size, pos, "\n")
dci += 1
pos = _PyiIndent(buf, buf_size, pos, indent_level)
pos = _PyiEmit(buf, buf_size, pos, "class ")
if cls.name is not None:
pos = _PyiEmit(buf, buf_size, pos, cls.name)
# PEP 695 类型参数 (class Foo[T]:)
if cls.type_params is not None:
tpn: t.CSizeT = cls.type_params.__len__()
if tpn > 0:
pos = _PyiEmit(buf, buf_size, pos, "[")
tpi: t.CSizeT = 0
while tpi < tpn:
if tpi > 0:
pos = _PyiEmit(buf, buf_size, pos, ", ")
tpname: str = cls.type_params.get(tpi)
if tpname is not None:
pos = _PyiEmit(buf, buf_size, pos, tpname)
tpi += 1
pos = _PyiEmit(buf, buf_size, pos, "]")
# 基类
if cls.bases is not None:
bn: t.CSizeT = cls.bases.__len__()
@@ -245,6 +303,29 @@ def _PyiClass(buf: bytes, buf_size: t.CSizeT, pos: t.CSizeT,
# 方法
if sk == ast.ASTKind.FunctionDef:
mfn: ast.FunctionDef | t.CPtr = (ast.FunctionDef | t.CPtr)(stmt)
# __init__ 方法:先提取 self.x: Type 属性
if mfn.name is not None and string.strcmp(mfn.name, "__init__") == 0:
if mfn.children is not None:
icn: t.CSizeT = mfn.children.__len__()
ici: t.CSizeT = 0
while ici < icn:
ibody: ast.AST | t.CPtr = mfn.children.get(ici)
if ibody is not None and ibody.kind() == ast.ASTKind.AnnAssign:
iaa: ast.AnnAssign | t.CPtr = (ast.AnnAssign | t.CPtr)(ibody)
if iaa.target is not None and iaa.target.kind() == ast.ASTKind.Attribute:
iat: ast.Attribute | t.CPtr = (ast.Attribute | t.CPtr)(iaa.target)
# 检查 target.value 是否为 Name("self")
if iat.value is not None and iat.value.kind() == ast.ASTKind.Name:
iavn: ast.Name | t.CPtr = (ast.Name | t.CPtr)(iat.value)
if iavn.id is not None and string.strcmp(iavn.id, "self") == 0:
pos = _PyiIndent(buf, buf_size, pos, indent_level + 1)
if iat.attr is not None:
pos = _PyiEmit(buf, buf_size, pos, iat.attr)
pos = _PyiEmit(buf, buf_size, pos, ": ")
pos = _PyiTypeStr(buf, buf_size, pos, iaa.annotation)
pos = _PyiEmit(buf, buf_size, pos, "\n")
has_member = 1
ici += 1
pos = _PyiFunction(buf, buf_size, pos, mfn, indent_level + 1, cls.name)
has_member = 1
@@ -260,6 +341,12 @@ def _PyiClass(buf: bytes, buf_size: t.CSizeT, pos: t.CSizeT,
pos = _PyiTypeStr(buf, buf_size, pos, aa.annotation)
pos = _PyiEmit(buf, buf_size, pos, "\n")
has_member = 1
# 嵌套类
elif sk == ast.ASTKind.ClassDef:
ncls: ast.ClassDef | t.CPtr = (ast.ClassDef | t.CPtr)(stmt)
pos = _PyiClass(buf, buf_size, pos, ncls, indent_level + 1)
has_member = 1
ci += 1
# 空类补 pass
@@ -411,6 +498,45 @@ def _GeneratePyiFromAst(mb: memhub.MemBuddy | t.CPtr, tree: ast.AST | t.CPtr,
pos = _PyiTypeStr(buf, buf_size, pos, aa.annotation)
pos = _PyiEmit(buf, buf_size, pos, "\n")
# 全局赋值(无注解)— 从 Constant value 推断类型
elif k == ast.ASTKind.Assign:
asg: ast.Assign | t.CPtr = (ast.Assign | t.CPtr)(stmt)
if asg.targets is not None and asg.value is not None:
atn: t.CSizeT = asg.targets.__len__()
if atn == 1:
atg: ast.AST | t.CPtr = asg.targets.get(0)
if atg is not None and atg.kind() == ast.ASTKind.Name:
agn: ast.Name | t.CPtr = (ast.Name | t.CPtr)(atg)
aval: ast.AST | t.CPtr = asg.value
avk: int = aval.kind()
inferred: int = 0
if avk == ast.ASTKind.Constant:
acv: ast.Constant | t.CPtr = (ast.Constant | t.CPtr)(aval)
if acv.const_kind == ast.CONST_INT:
inferred = 1
elif acv.const_kind == ast.CONST_FLOAT:
inferred = 2
elif acv.const_kind == ast.CONST_STR:
inferred = 3
elif acv.const_kind == ast.CONST_BOOL:
inferred = 4
elif acv.const_kind == ast.CONST_NONE:
inferred = 5
if inferred > 0 and agn.id is not None:
pos = _PyiEmit(buf, buf_size, pos, agn.id)
pos = _PyiEmit(buf, buf_size, pos, ": ")
if inferred == 1:
pos = _PyiEmit(buf, buf_size, pos, "t.CInt")
elif inferred == 2:
pos = _PyiEmit(buf, buf_size, pos, "t.CDouble")
elif inferred == 3:
pos = _PyiEmit(buf, buf_size, pos, "str")
elif inferred == 4:
pos = _PyiEmit(buf, buf_size, pos, "bool")
elif inferred == 5:
pos = _PyiEmit(buf, buf_size, pos, "None")
pos = _PyiEmit(buf, buf_size, pos, "\n")
i += 1
# NUL 终止

View File

@@ -1,313 +0,0 @@
"""StubGen 生成器模块 - 存根生成器主类和命令行入口"""
from __future__ import annotations
import sys
import os
import re
import argparse
import logging
import traceback
from pathlib import Path
from typing import List
from lib.core.VLogger import get_logger as _vlog
from lib.core.stub_generator import CHeaderParser, PythonStubGenerator, CTypeMapper
from lib.StubGen.Config import StubGenConfig
from lib.StubGen.Converter import PythonToStubConverter
class StubGen:
"""存根生成器主类"""
def __init__(self, config: StubGenConfig) -> None:
self.config: StubGenConfig = config
self.logger: logging.Logger = self._SetupLogger()
self.GeneratedFiles: List[str] = []
self.FailedFiles: List[str] = []
# 应用自定义类型映射
if config.TypeMappings:
CTypeMapper.BASIC_TYPE_MAP.update(config.TypeMappings)
def _SetupLogger(self) -> logging.Logger:
"""设置日志"""
logger: logging.Logger = logging.getLogger('StubGen')
logger.setLevel(logging.DEBUG if self.config.verbose else logging.INFO)
if not logger.handlers:
handler: logging.StreamHandler = logging.StreamHandler(sys.stdout)
formatter: logging.Formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s',
datefmt='%H:%M:%S'
)
handler.setFormatter(formatter)
logger.addHandler(handler)
return logger
def FindInputFiles(self) -> list[tuple[str, str]]:
"""查找输入文件,返回 (文件路径, 相对路径) 列表"""
files: list[tuple[str, str]] = []
# 添加显式指定的文件
for FilePath in self.config.InputFiles:
if os.path.exists(FilePath):
RelPath: str = os.path.relpath(FilePath, self.config.InputDir or '.')
files.append((FilePath, RelPath))
else:
self.logger.warning(f"Input file not found: {FilePath}")
# 从输入目录查找
if self.config.InputDir and os.path.exists(self.config.InputDir):
InputPath: Path = Path(self.config.InputDir)
for pattern in self.config.IncludePatterns:
for FilePath in InputPath.rglob(pattern):
# 检查是否在排除列表中
excluded: bool = False
RelPath: str = os.path.relpath(str(FilePath), self.config.InputDir)
for exclude_pattern in self.config.ExcludePatterns:
if FilePath.match(exclude_pattern) or RelPath == exclude_pattern:
excluded = True
break
if not excluded:
files.append((str(FilePath), RelPath))
# 去重并保持顺序
seen: set[str] = set()
UniqueFiles: list[tuple[str, str]] = []
for FilePath, RelPath in files:
key: str = FilePath
if key not in seen:
seen.add(key)
UniqueFiles.append((FilePath, RelPath))
return UniqueFiles
def _GenerateGuardName(self, FilePath: str) -> str:
"""生成宏守卫名称"""
# 获取文件名(不含扩展名)
BaseName: str = os.path.splitext(os.path.basename(FilePath))[0]
# 转换为大写,替换特殊字符为下划线
guard: str = re.sub(r'[^a-zA-Z0-9_]', '_', BaseName).upper()
guard = re.sub(r'_+', '_', guard) # 合并多个下划线
guard = guard.strip('_')
return f'{guard}_DEFINE_H'
def _GetOutputPath(self, RelPath: str) -> str:
"""获取输出文件路径"""
# 更改扩展名为 .pyi (Python stub file)
BaseName: str = os.path.splitext(RelPath)[0]
OutputRelPath: str = BaseName + '.pyi'
# 构建完整输出路径
if self.config.OutputDir:
OutputPath: str = os.path.join(self.config.OutputDir, OutputRelPath)
else:
OutputPath = OutputRelPath
return OutputPath
def GenerateStub(self, InputFile: str, RelPath: str) -> bool:
"""生成单个存根文件"""
try:
self.logger.info(f"Processing: {InputFile}")
# 确定输出文件路径
OutputFile: str = self._GetOutputPath(RelPath)
# 确保输出目录存在
OutputDir: str = os.path.dirname(OutputFile)
if OutputDir and not os.path.exists(OutputDir):
if not self.config.DryRun:
os.makedirs(OutputDir, exist_ok=True)
self.logger.debug(f"Created directory: {OutputDir}")
if self.config.DryRun:
self.logger.info(f"[DRY RUN] Would generate: {OutputFile}")
return True
# 根据文件类型选择处理方式
ext: str = os.path.splitext(InputFile)[1].lower()
if ext in ['.h', '.c']:
# C 文件:使用 CHeaderParser
content: str = self._GenerateFromC(InputFile, RelPath)
elif ext == '.py':
# Python 文件:使用 PythonToStubConverter
content = self._GenerateFromPy(InputFile, RelPath)
else:
self.logger.warning(f"Unsupported file type: {ext}")
return False
# 写入文件
with open(OutputFile, 'w', encoding='utf-8') as f:
f.write(content)
self.GeneratedFiles.append(OutputFile)
self.logger.info(f"Generated: {OutputFile}")
return True
except Exception as e:
self.logger.error(f"Failed to process {InputFile}: {e}")
traceback.print_exc()
self.FailedFiles.append(InputFile)
return False
def _GenerateFromC(self, InputFile: str, RelPath: str) -> str:
"""从 C 文件生成存根"""
parser: CHeaderParser = CHeaderParser()
parser.parse_file(InputFile)
generator: PythonStubGenerator = PythonStubGenerator(parser)
ModuleName: str = os.path.splitext(os.path.basename(InputFile))[0]
content: str = generator.generate(ModuleName)
# 添加宏守卫(如果需要)
if self.config.GenerateGuards:
guard_name: str = self._GenerateGuardName(InputFile)
guard_comment: str = f"\n# Guard: {guard_name}\n"
content = content + guard_comment
return content
def _GenerateFromPy(self, InputFile: str, RelPath: str) -> str:
"""从 Python 文件生成存根"""
with open(InputFile, 'r', encoding='utf-8') as f:
PyContent: str = f.read()
ModuleName: str = os.path.splitext(os.path.basename(InputFile))[0]
content: str = PythonToStubConverter.convert(PyContent, ModuleName)
# 添加宏守卫(如果需要)
if self.config.GenerateGuards:
guard_name: str = self._GenerateGuardName(InputFile)
guard_comment: str = f"\n# Guard: {guard_name}\n"
content = content + guard_comment
return content
def run(self) -> bool:
"""运行生成器"""
self.logger.info("=" * 60)
self.logger.info("StubGen - C/H/Py to Python Stub Generator")
self.logger.info("=" * 60)
# 查找输入文件
InputFiles: list[tuple[str, str]] = self.FindInputFiles()
if not InputFiles:
self.logger.warning("No input files found!")
return False
self.logger.info(f"Found {len(InputFiles)} input file(s)")
for FilePath, RelPath in InputFiles:
self.logger.debug(f" - {RelPath}")
# 处理每个文件
SuccessCount: int = 0
for FilePath, RelPath in InputFiles:
if self.GenerateStub(FilePath, RelPath):
SuccessCount += 1
# 输出统计信息
self.logger.info("=" * 60)
self.logger.info(f"Summary: {SuccessCount}/{len(InputFiles)} files generated successfully")
if self.FailedFiles:
self.logger.warning(f"Failed files ({len(self.FailedFiles)}):")
for f in self.FailedFiles:
self.logger.warning(f" - {f}")
return SuccessCount == len(InputFiles)
def main() -> None:
parser: argparse.ArgumentParser = argparse.ArgumentParser(
description='StubGen - Generate Python stub files from C/H/Py sources',
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog='''
Examples:
# 使用配置文件
%(prog)s -c config.json
# 处理单个文件
%(prog)s -i input.h -o output.py
# 批量处理目录(保持目录结构)
%(prog)s -d ./kernel -o ./kernel/include
# 干运行(不实际生成文件)
%(prog)s -d ./kernel --dry-run
# 不生成宏守卫
%(prog)s -d ./kernel --no-guards
'''
)
# 输入选项
InputGroup: argparse._MutuallyExclusiveGroup = parser.add_mutually_exclusive_group(required=True)
InputGroup.add_argument('-c', '--config', help='Configuration file (JSON)')
InputGroup.add_argument('-i', '--input', help='Input file')
InputGroup.add_argument('-d', '--directory', help='Input directory')
# 输出选项
parser.add_argument('-o', '--output', help='Output directory')
# 其他选项
parser.add_argument('--include', action='append',
help='Include patterns (default: *.h, *.c, *.py)')
parser.add_argument('--exclude', action='append',
help='Exclude patterns')
parser.add_argument('--no-guards', action='store_true',
help='Do not generate guard macros')
parser.add_argument('--no-structure', action='store_true',
help='Do not preserve directory structure')
parser.add_argument('-v', '--verbose', action='store_true',
help='Verbose output')
parser.add_argument('--dry-run', action='store_true',
help='Dry run (do not create files)')
args: argparse.Namespace = parser.parse_args()
# 加载配置
if args.config:
if not os.path.exists(args.config):
_vlog().error(f"配置文件未找到: {args.config}")
sys.exit(1)
config: StubGenConfig = StubGenConfig.from_file(args.config)
else:
config = StubGenConfig()
config.IncludePatterns = args.include or ['*.h', '*.c', '*.py']
config.ExcludePatterns = args.exclude or []
config.GenerateGuards = not args.no_guards
config.PreserveStructure = not args.no_structure
config.verbose = args.verbose
config.DryRun = args.dry_run
if args.input:
config.InputFiles = [args.input]
if args.output:
config.OutputDir = os.path.dirname(args.output) or '.'
elif args.directory:
config.InputDir = args.directory
if args.output:
config.OutputDir = args.output
# 创建生成器并运行
generator: StubGen = StubGen(config)
# 如果指定了单个输出文件,直接处理
if args.input and args.output and not os.path.isdir(args.output):
RelPath: str = os.path.relpath(args.input, config.InputDir or '.')
success: bool = generator.GenerateStub(args.input, RelPath)
else:
success = generator.run()
sys.exit(0 if success else 1)
if __name__ == '__main__':
main()