394 lines
21 KiB
Python
394 lines
21 KiB
Python
from __future__ import annotations
|
|
|
|
import os
|
|
import ast
|
|
import traceback
|
|
from typing import Any
|
|
|
|
from lib.Projectrans.Utils import compute_sha1, get_file_dependencies, find_reachable_files_from_entries
|
|
from lib.Projectrans.DeclarationGenerator import DeclarationGenerator
|
|
from lib.core.VLogger import get_logger as _vlog
|
|
from lib.constants.config import mode as _ConfigMode
|
|
from StubGen import PythonToStubConverter
|
|
|
|
|
|
class Phase1Generator:
|
|
"""阶段一：从源文件生成声明接口"""
|
|
|
|
def __init__(self, src_root: str, temp_dir: str, include_dirs: list[str] | None = None, entry_files: list[str] | None = None, target_triple: str | None = None, target_datalayout: str | None = None) -> None:
|
|
self.src_root: str = os.path.abspath(src_root)
|
|
self.temp_dir: str = os.path.abspath(temp_dir)
|
|
self.include_dirs: list[str] = include_dirs or []
|
|
self.sha1_map: dict[str, str] = {}
|
|
self.include_py_map: dict[str, str] = {}
|
|
self.entry_files: list[str] | None = entry_files
|
|
self.target_triple: str | None = target_triple
|
|
self.target_datalayout: str | None = target_datalayout
|
|
os.makedirs(self.temp_dir, exist_ok=True)
|
|
|
|
def _get_needed_include_files(self, reachable_source_files: set[str]) -> list[tuple[str, str, str]]:
|
|
"""从可达源文件收集所有被引用（含传递依赖）的 include 文件"""
|
|
include_file_map: dict[str, tuple[str, str, str]] = {}
|
|
for includes_dir in self.include_dirs:
|
|
if not os.path.isdir(includes_dir):
|
|
continue
|
|
for root, dirs, files in os.walk(includes_dir):
|
|
dirs[:] = [d for d in dirs if not d.startswith('.') and d != '__pycache__']
|
|
for fname in files:
|
|
if fname.endswith('.py') or fname.endswith('.pyi'):
|
|
src_path: str = os.path.join(root, fname)
|
|
rel_from_inc: str = os.path.relpath(src_path, includes_dir)
|
|
ModulePath: str = rel_from_inc.replace(os.sep, '.').replace('/', '.')
|
|
module_name: str = os.path.splitext(ModulePath)[0]
|
|
info: tuple[str, str, str] = (src_path, rel_from_inc, includes_dir)
|
|
include_file_map[module_name] = info
|
|
top_pkg: str = module_name.split('.')[0]
|
|
if top_pkg not in include_file_map:
|
|
include_file_map[top_pkg] = info
|
|
|
|
imported_from_src: set[str] = set()
|
|
for src_path in reachable_source_files:
|
|
deps: set[str] = get_file_dependencies(src_path, self.src_root)
|
|
imported_from_src.update(deps)
|
|
|
|
# 检测 dict/list 容器使用，自动添加 _dict/_list/json 依赖
|
|
for src_path in reachable_source_files:
|
|
try:
|
|
with open(src_path, 'r', encoding='utf-8') as f:
|
|
content: str = f.read()
|
|
tree = ast.parse(content)
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.Subscript) and isinstance(node.value, ast.Name):
|
|
if node.value.id == 'dict':
|
|
imported_from_src.add('_dict')
|
|
imported_from_src.add('json')
|
|
break
|
|
elif node.value.id == 'list':
|
|
imported_from_src.add('_list')
|
|
break
|
|
if isinstance(node, ast.Call) and isinstance(node.func, ast.Name):
|
|
if node.func.id == 'dict':
|
|
imported_from_src.add('_dict')
|
|
imported_from_src.add('json')
|
|
break
|
|
elif node.func.id == 'list':
|
|
imported_from_src.add('_list')
|
|
break
|
|
# 检测类型注解 d: dict / l: list (非泛型形式)
|
|
if isinstance(node, ast.AnnAssign) and isinstance(node.annotation, ast.Name):
|
|
if node.annotation.id == 'dict':
|
|
imported_from_src.add('_dict')
|
|
imported_from_src.add('json')
|
|
break
|
|
elif node.annotation.id == 'list':
|
|
imported_from_src.add('_list')
|
|
break
|
|
except Exception:
|
|
pass
|
|
|
|
needed: set[str] = set()
|
|
queue: list[str] = list(imported_from_src)
|
|
while queue:
|
|
mod_name: str = queue.pop(0)
|
|
if mod_name in needed:
|
|
continue
|
|
if mod_name in include_file_map:
|
|
needed.add(mod_name)
|
|
src_path: str = include_file_map[mod_name][0]
|
|
includes_dir: str = include_file_map[mod_name][2]
|
|
deps: set[str] = get_file_dependencies(src_path, includes_dir)
|
|
for dep in deps:
|
|
if dep in include_file_map and dep not in needed:
|
|
queue.append(dep)
|
|
|
|
needed_infos: list[tuple[str, str, str]] = []
|
|
for mod_name in sorted(needed):
|
|
if mod_name in include_file_map:
|
|
info: tuple[str, str, str] = include_file_map[mod_name]
|
|
if info not in needed_infos:
|
|
needed_infos.append(info)
|
|
|
|
return needed_infos
|
|
|
|
def run(self) -> None:
|
|
"""扫描源目录，只处理入口文件可达的 .py 文件（导入遍历）"""
|
|
py_files: list[str]
|
|
if self.entry_files:
|
|
py_files = list(self.entry_files)
|
|
else:
|
|
main_py: str = os.path.join(self.src_root, 'main.py')
|
|
if os.path.exists(main_py):
|
|
py_files = [main_py]
|
|
else:
|
|
py_files = []
|
|
for root, dirs, files in os.walk(self.src_root):
|
|
dirs[:] = [d for d in dirs if d not in ('__pycache__',)]
|
|
for file in files:
|
|
if file.endswith('.py'):
|
|
py_files.append(os.path.join(root, file))
|
|
|
|
if not py_files:
|
|
_vlog().warning("未找到入口文件或源文件")
|
|
return
|
|
|
|
reachable: set[str]
|
|
if self.entry_files:
|
|
reachable = find_reachable_files_from_entries(self.src_root, py_files)
|
|
else:
|
|
reachable = find_reachable_files_from_entries(self.src_root, py_files)
|
|
|
|
_vlog().info(f"找到 {len(reachable)} 个可达源文件（从入口遍历）")
|
|
|
|
for i, src_path in enumerate(sorted(reachable), 1):
|
|
rel: str = os.path.relpath(src_path, self.src_root)
|
|
_vlog().info(f"[{i}/{len(reachable)}] 生成签名: {rel}")
|
|
try:
|
|
self._process_file_pyi(src_path, rel)
|
|
except Exception as e:
|
|
_vlog().error(f"生成签名失败: {e}")
|
|
traceback.print_exc()
|
|
|
|
needed_includes: list[tuple[str, str, str]] = self._get_needed_include_files(reachable)
|
|
self._process_include_py_files_pyi(needed_includes)
|
|
|
|
struct_names: set[str]
|
|
enum_names: set[str]
|
|
struct_sha1_map: dict[str, str]
|
|
exception_names: set[str]
|
|
struct_names, enum_names, struct_sha1_map, exception_names = self._build_struct_registry()
|
|
_vlog().info(f"结构体注册表: {len(struct_names)} 个类型, {len(enum_names)} 个枚举")
|
|
for name in sorted(struct_names):
|
|
_vlog().debug(f" {name}")
|
|
|
|
for i, src_path in enumerate(sorted(reachable), 1):
|
|
rel: str = os.path.relpath(src_path, self.src_root)
|
|
try:
|
|
self._process_file_stub(src_path, rel, struct_names, enum_names=enum_names, struct_sha1_map=struct_sha1_map, exception_names=exception_names)
|
|
except Exception as e:
|
|
_vlog().error(f"生成声明失败: {e}")
|
|
traceback.print_exc()
|
|
|
|
self._process_include_py_files_stub(struct_names, needed_includes, enum_names=enum_names, struct_sha1_map=struct_sha1_map, exception_names=exception_names)
|
|
|
|
_vlog().success(f"声明接口生成到: {self.temp_dir}")
|
|
_vlog().info(f"SHA1 映射表 ({len(self.sha1_map)} 个文件):")
|
|
for sha1, rel in sorted(self.sha1_map.items()):
|
|
_vlog().debug(f" {sha1} -> {rel}")
|
|
|
|
def _collect_include_py_files(self) -> list[tuple[str, str, str]]:
|
|
include_py_files: list[tuple[str, str, str]] = []
|
|
for includes_dir in self.include_dirs:
|
|
if not os.path.isdir(includes_dir):
|
|
continue
|
|
for root, dirs, files in os.walk(includes_dir):
|
|
dirs[:] = [d for d in dirs if not d.startswith('.') and d != '__pycache__']
|
|
for fname in files:
|
|
if fname.endswith('.py') or fname.endswith('.pyi'):
|
|
src_path: str = os.path.join(root, fname)
|
|
rel_from_inc: str = os.path.relpath(src_path, includes_dir)
|
|
include_py_files.append((src_path, rel_from_inc, includes_dir))
|
|
return include_py_files
|
|
|
|
def _process_include_py_files_pyi(self, needed_includes: list[tuple[str, str, str]] | None = None) -> None:
|
|
include_py_files: list[tuple[str, str, str]]
|
|
if needed_includes is not None:
|
|
include_py_files = needed_includes
|
|
else:
|
|
include_py_files = self._collect_include_py_files()
|
|
if not include_py_files:
|
|
return
|
|
_vlog().info(f"处理 {len(include_py_files)} 个被引用的 Python 库文件")
|
|
for src_path, rel_from_inc, includes_dir in include_py_files:
|
|
ModulePath: str = rel_from_inc.replace(os.sep, '.').replace('/', '.')
|
|
module_name: str = os.path.splitext(ModulePath)[0]
|
|
try:
|
|
with open(src_path, 'r', encoding='utf-8') as f:
|
|
content: str = f.read()
|
|
sha1: str = compute_sha1(content)
|
|
self.sha1_map[sha1] = f"includes/{rel_from_inc}"
|
|
top_module: str = rel_from_inc.split(os.sep)[0].split('/')[0]
|
|
if top_module not in self.include_py_map:
|
|
self.include_py_map[top_module] = sha1
|
|
self.include_py_map[module_name] = sha1
|
|
|
|
sig_path: str = os.path.join(self.temp_dir, f"{sha1}.pyi")
|
|
if os.path.isfile(sig_path):
|
|
_vlog().info(f" 缓存命中: {rel_from_inc} -> {sha1}.pyi")
|
|
continue
|
|
|
|
sig_content: str = PythonToStubConverter.convert(content, module_name)
|
|
with open(sig_path, 'w', encoding='utf-8', newline='\n') as f:
|
|
f.write(sig_content)
|
|
_vlog().info(f" 生成签名: {rel_from_inc} -> {sha1}.pyi")
|
|
except Exception as e:
|
|
_vlog().error(f"处理 include 文件失败 {rel_from_inc}: {e}")
|
|
|
|
def _collect_typedef_map(self) -> dict[str, ast.AST]:
|
|
typedef_map: dict[str, ast.AST] = {}
|
|
if not os.path.isdir(self.temp_dir):
|
|
return typedef_map
|
|
for fname in os.listdir(self.temp_dir):
|
|
if not fname.endswith('.pyi'):
|
|
continue
|
|
pyi_path: str = os.path.join(self.temp_dir, fname)
|
|
try:
|
|
with open(pyi_path, 'r', encoding='utf-8') as f:
|
|
content: str = f.read()
|
|
tree: ast.Module = ast.parse(content)
|
|
for node in ast.iter_child_nodes(tree):
|
|
if isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name):
|
|
var_name: str = node.target.id
|
|
is_typedef: bool = False
|
|
if isinstance(node.annotation, ast.Attribute) and hasattr(node.annotation, 'attr') and node.annotation.attr == 'CTypedef':
|
|
is_typedef = True
|
|
elif isinstance(node.annotation, ast.Name) and node.annotation.id == 'CTypedef':
|
|
is_typedef = True
|
|
elif isinstance(node.annotation, ast.BinOp) and isinstance(node.annotation.left, ast.Attribute) and hasattr(node.annotation.left, 'attr') and node.annotation.left.attr == 'CTypedef':
|
|
is_typedef = True
|
|
if is_typedef and node.value and var_name not in typedef_map:
|
|
typedef_map[var_name] = node.value
|
|
except Exception as _e:
|
|
if _ConfigMode == "strict":
|
|
raise
|
|
_vlog().warning(f"收集 typedef 映射失败: {_e}", "Exception")
|
|
return typedef_map
|
|
|
|
def _process_include_py_files_stub(self, struct_names: set[str], needed_includes: list[tuple[str, str, str]] | None = None, enum_names: set[str] | None = None, struct_sha1_map: dict[str, str] | None = None, exception_names: set[str] | None = None) -> None:
|
|
include_py_files: list[tuple[str, str, str]]
|
|
if needed_includes is not None:
|
|
include_py_files = needed_includes
|
|
else:
|
|
include_py_files = self._collect_include_py_files()
|
|
if not include_py_files:
|
|
return
|
|
typedef_map: dict[str, ast.AST] = self._collect_typedef_map()
|
|
for src_path, rel_from_inc, includes_dir in include_py_files:
|
|
ModulePath: str = rel_from_inc.replace(os.sep, '.').replace('/', '.')
|
|
module_name: str = os.path.splitext(ModulePath)[0]
|
|
try:
|
|
with open(src_path, 'r', encoding='utf-8') as f:
|
|
content: str = f.read()
|
|
sha1: str = compute_sha1(content)
|
|
|
|
stub_path: str = os.path.join(self.temp_dir, f"{sha1}.stub.ll")
|
|
if os.path.isfile(stub_path):
|
|
_vlog().info(f" 缓存命中: {rel_from_inc} -> {sha1}.stub.ll")
|
|
continue
|
|
|
|
sig_path: str = os.path.join(self.temp_dir, f"{sha1}.pyi")
|
|
with open(sig_path, 'r', encoding='utf-8') as f:
|
|
sig_content: str = f.read()
|
|
|
|
self._generate_decl_ll(sig_content, stub_path, src_path, struct_names, enum_names=enum_names, module_sha1=sha1, struct_sha1_map=struct_sha1_map, exception_names=exception_names, typedef_map=typedef_map)
|
|
_vlog().info(f" 生成声明: {rel_from_inc} -> {sha1}.stub.ll")
|
|
except Exception as e:
|
|
_vlog().error(f"处理 include 文件失败 {rel_from_inc}: {e}")
|
|
|
|
def _process_file_pyi(self, src_path: str, rel_path: str) -> None:
|
|
with open(src_path, 'r', encoding='utf-8') as f:
|
|
content: str = f.read()
|
|
|
|
sha1: str = compute_sha1(content)
|
|
self.sha1_map[sha1] = rel_path
|
|
|
|
sig_path: str = os.path.join(self.temp_dir, f"{sha1}.pyi")
|
|
if os.path.isfile(sig_path):
|
|
_vlog().info(f" -> {sha1}.pyi (缓存)")
|
|
return
|
|
|
|
module_name: str = os.path.splitext(rel_path)[0].replace(os.sep, '.').replace('/', '.')
|
|
sig_content: str = PythonToStubConverter.convert(content, module_name)
|
|
with open(sig_path, 'w', encoding='utf-8', newline='\n') as f:
|
|
f.write(sig_content)
|
|
_vlog().info(f" -> {sha1}.pyi (签名)")
|
|
|
|
def _process_file_stub(self, src_path: str, rel_path: str, struct_names: set[str], enum_names: set[str] | None = None, struct_sha1_map: dict[str, str] | None = None, exception_names: set[str] | None = None) -> None:
|
|
with open(src_path, 'r', encoding='utf-8') as f:
|
|
content: str = f.read()
|
|
|
|
sha1: str = compute_sha1(content)
|
|
stub_path: str = os.path.join(self.temp_dir, f"{sha1}.stub.ll")
|
|
if os.path.isfile(stub_path):
|
|
_vlog().info(f" -> {sha1}.stub.ll (缓存)")
|
|
return
|
|
|
|
sig_path: str = os.path.join(self.temp_dir, f"{sha1}.pyi")
|
|
with open(sig_path, 'r', encoding='utf-8') as f:
|
|
sig_content: str = f.read()
|
|
|
|
typedef_map: dict[str, ast.AST] = self._collect_typedef_map()
|
|
self._generate_decl_ll(sig_content, stub_path, src_path, struct_names, enum_names=enum_names, module_sha1=sha1, struct_sha1_map=struct_sha1_map, exception_names=exception_names, typedef_map=typedef_map)
|
|
_vlog().info(f" -> {sha1}.stub.ll (声明)")
|
|
|
|
def _build_struct_registry(self) -> tuple[set[str], set[str], dict[str, str], set[str]]:
|
|
struct_names: set[str] = set()
|
|
enum_names: set[str] = set()
|
|
exception_names: set[str] = set()
|
|
struct_sha1_map: dict[str, str] = {}
|
|
valid_sha1_keys: set[str] = set(self.sha1_map.keys())
|
|
for fname in os.listdir(self.temp_dir):
|
|
if not fname.endswith('.pyi'):
|
|
continue
|
|
sha1_key: str = fname.replace('.pyi', '')
|
|
if sha1_key not in valid_sha1_keys:
|
|
continue
|
|
pyi_path: str = os.path.join(self.temp_dir, fname)
|
|
try:
|
|
with open(pyi_path, 'r', encoding='utf-8') as f:
|
|
content: str = f.read()
|
|
tree: ast.Module = ast.parse(content)
|
|
for node in ast.iter_child_nodes(tree):
|
|
if isinstance(node, ast.ClassDef):
|
|
is_enum: bool = False
|
|
is_exception: bool = False
|
|
if node.bases:
|
|
for base in node.bases:
|
|
if isinstance(base, ast.Attribute) and hasattr(base, 'attr'):
|
|
if base.attr in ('CEnum', 'Enum', 'REnum'):
|
|
is_enum = True
|
|
break
|
|
elif base.attr == 'Exception' or base.attr in exception_names:
|
|
is_exception = True
|
|
break
|
|
elif isinstance(base, ast.Name) and hasattr(base, 'id'):
|
|
if base.id in ('CEnum', 'Enum', 'REnum'):
|
|
is_enum = True
|
|
break
|
|
elif base.id == 'Exception' or base.id in exception_names:
|
|
is_exception = True
|
|
break
|
|
# 也检查装饰器形式 @t.CEnum / @t.REnum
|
|
if not is_enum and not is_exception and hasattr(node, 'decorator_list') and node.decorator_list:
|
|
for deco in node.decorator_list:
|
|
deco_attr: str | None = None
|
|
if isinstance(deco, ast.Attribute) and hasattr(deco, 'attr'):
|
|
deco_attr = deco.attr
|
|
elif isinstance(deco, ast.Name) and hasattr(deco, 'id'):
|
|
deco_attr = deco.id
|
|
if deco_attr in ('CEnum', 'Enum', 'REnum'):
|
|
is_enum = True
|
|
break
|
|
if is_enum:
|
|
enum_names.add(node.name)
|
|
elif is_exception:
|
|
exception_names.add(node.name)
|
|
else:
|
|
struct_names.add(node.name)
|
|
struct_sha1_map[node.name] = sha1_key
|
|
except Exception as _e:
|
|
if _ConfigMode == "strict":
|
|
raise
|
|
_vlog().warning(f"构建结构体/枚举注册表失败: {_e}", "Exception")
|
|
return struct_names, enum_names, struct_sha1_map, exception_names
|
|
|
|
def _generate_decl_ll(self, pyi_content: str, ll_path: str, src_path: str, struct_names: set[str] | None = None, enum_names: set[str] | None = None, module_sha1: str | None = None, struct_sha1_map: dict[str, str] | None = None, exception_names: set[str] | None = None, typedef_map: dict[str, ast.AST] | None = None) -> None:
|
|
try:
|
|
decl_gen: DeclarationGenerator = DeclarationGenerator(struct_names=struct_names, enum_names=enum_names, module_sha1=module_sha1, target_triple=self.target_triple, target_datalayout=self.target_datalayout, struct_sha1_map=struct_sha1_map, exception_names=exception_names, typedef_map=typedef_map)
|
|
decl_ll: str = decl_gen.generate(pyi_content, src_path)
|
|
with open(ll_path, 'w', encoding='utf-8', newline='\n') as f:
|
|
f.write(decl_ll)
|
|
except Exception as e:
|
|
_vlog().warning(f".ll 声明生成失败: {e}")
|
|
with open(ll_path, 'w', encoding='utf-8') as f:
|
|
f.write(f"; declaration for {os.path.basename(src_path)}\n; error: {e}\n")
|