from __future__ import annotations import os import ast import traceback from typing import Any from lib.Projectrans.Utils import compute_sha1, get_file_dependencies, find_reachable_files_from_entries from lib.Projectrans.DeclarationGenerator import DeclarationGenerator from lib.core.VLogger import get_logger as _vlog from lib.constants.config import mode as _ConfigMode from StubGen import PythonToStubConverter class Phase1Generator: """阶段一:从源文件生成声明接口""" def __init__(self, src_root: str, temp_dir: str, include_dirs: list[str] | None = None, entry_files: list[str] | None = None, target_triple: str | None = None, target_datalayout: str | None = None) -> None: self.src_root: str = os.path.abspath(src_root) self.temp_dir: str = os.path.abspath(temp_dir) self.include_dirs: list[str] = include_dirs or [] self.sha1_map: dict[str, str] = {} self.include_py_map: dict[str, str] = {} self.entry_files: list[str] | None = entry_files self.target_triple: str | None = target_triple self.target_datalayout: str | None = target_datalayout os.makedirs(self.temp_dir, exist_ok=True) def _get_needed_include_files(self, reachable_source_files: set[str]) -> list[tuple[str, str, str]]: """从可达源文件收集所有被引用(含传递依赖)的 include 文件""" include_file_map: dict[str, tuple[str, str, str]] = {} for includes_dir in self.include_dirs: if not os.path.isdir(includes_dir): continue for root, dirs, files in os.walk(includes_dir): dirs[:] = [d for d in dirs if not d.startswith('.') and d != '__pycache__'] for fname in files: if fname.endswith('.py') or fname.endswith('.pyi'): src_path: str = os.path.join(root, fname) rel_from_inc: str = os.path.relpath(src_path, includes_dir) ModulePath: str = rel_from_inc.replace(os.sep, '.').replace('/', '.') module_name: str = os.path.splitext(ModulePath)[0] info: tuple[str, str, str] = (src_path, rel_from_inc, includes_dir) include_file_map[module_name] = info top_pkg: str = module_name.split('.')[0] if top_pkg not in include_file_map: include_file_map[top_pkg] = info imported_from_src: set[str] = set() for src_path in reachable_source_files: deps: set[str] = get_file_dependencies(src_path, self.src_root) imported_from_src.update(deps) # 检测 dict/list 容器使用,自动添加 _dict/_list/json 依赖 for src_path in reachable_source_files: try: with open(src_path, 'r', encoding='utf-8') as f: content: str = f.read() tree = ast.parse(content) for node in ast.walk(tree): if isinstance(node, ast.Subscript) and isinstance(node.value, ast.Name): if node.value.id == 'dict': imported_from_src.add('_dict') imported_from_src.add('json') break elif node.value.id == 'list': imported_from_src.add('_list') break if isinstance(node, ast.Call) and isinstance(node.func, ast.Name): if node.func.id == 'dict': imported_from_src.add('_dict') imported_from_src.add('json') break elif node.func.id == 'list': imported_from_src.add('_list') break # 检测类型注解 d: dict / l: list (非泛型形式) if isinstance(node, ast.AnnAssign) and isinstance(node.annotation, ast.Name): if node.annotation.id == 'dict': imported_from_src.add('_dict') imported_from_src.add('json') break elif node.annotation.id == 'list': imported_from_src.add('_list') break except Exception: pass needed: set[str] = set() queue: list[str] = list(imported_from_src) while queue: mod_name: str = queue.pop(0) if mod_name in needed: continue if mod_name in include_file_map: needed.add(mod_name) src_path: str = include_file_map[mod_name][0] includes_dir: str = include_file_map[mod_name][2] deps: set[str] = get_file_dependencies(src_path, includes_dir) for dep in deps: if dep in include_file_map and dep not in needed: queue.append(dep) needed_infos: list[tuple[str, str, str]] = [] for mod_name in sorted(needed): if mod_name in include_file_map: info: tuple[str, str, str] = include_file_map[mod_name] if info not in needed_infos: needed_infos.append(info) return needed_infos def run(self) -> None: """扫描源目录,只处理入口文件可达的 .py 文件(导入遍历)""" py_files: list[str] if self.entry_files: py_files = list(self.entry_files) else: main_py: str = os.path.join(self.src_root, 'main.py') if os.path.exists(main_py): py_files = [main_py] else: py_files = [] for root, dirs, files in os.walk(self.src_root): dirs[:] = [d for d in dirs if d not in ('__pycache__',)] for file in files: if file.endswith('.py'): py_files.append(os.path.join(root, file)) if not py_files: _vlog().warning("未找到入口文件或源文件") return reachable: set[str] if self.entry_files: reachable = find_reachable_files_from_entries(self.src_root, py_files) else: reachable = find_reachable_files_from_entries(self.src_root, py_files) _vlog().info(f"找到 {len(reachable)} 个可达源文件(从入口遍历)") for i, src_path in enumerate(sorted(reachable), 1): rel: str = os.path.relpath(src_path, self.src_root) _vlog().info(f"[{i}/{len(reachable)}] 生成签名: {rel}") try: self._process_file_pyi(src_path, rel) except Exception as e: _vlog().error(f"生成签名失败: {e}") traceback.print_exc() needed_includes: list[tuple[str, str, str]] = self._get_needed_include_files(reachable) self._process_include_py_files_pyi(needed_includes) struct_names: set[str] enum_names: set[str] struct_sha1_map: dict[str, str] exception_names: set[str] struct_names, enum_names, struct_sha1_map, exception_names = self._build_struct_registry() _vlog().info(f"结构体注册表: {len(struct_names)} 个类型, {len(enum_names)} 个枚举") for name in sorted(struct_names): _vlog().debug(f" {name}") for i, src_path in enumerate(sorted(reachable), 1): rel: str = os.path.relpath(src_path, self.src_root) try: self._process_file_stub(src_path, rel, struct_names, enum_names=enum_names, struct_sha1_map=struct_sha1_map, exception_names=exception_names) except Exception as e: _vlog().error(f"生成声明失败: {e}") traceback.print_exc() self._process_include_py_files_stub(struct_names, needed_includes, enum_names=enum_names, struct_sha1_map=struct_sha1_map, exception_names=exception_names) _vlog().success(f"声明接口生成到: {self.temp_dir}") _vlog().info(f"SHA1 映射表 ({len(self.sha1_map)} 个文件):") for sha1, rel in sorted(self.sha1_map.items()): _vlog().debug(f" {sha1} -> {rel}") def _collect_include_py_files(self) -> list[tuple[str, str, str]]: include_py_files: list[tuple[str, str, str]] = [] for includes_dir in self.include_dirs: if not os.path.isdir(includes_dir): continue for root, dirs, files in os.walk(includes_dir): dirs[:] = [d for d in dirs if not d.startswith('.') and d != '__pycache__'] for fname in files: if fname.endswith('.py') or fname.endswith('.pyi'): src_path: str = os.path.join(root, fname) rel_from_inc: str = os.path.relpath(src_path, includes_dir) include_py_files.append((src_path, rel_from_inc, includes_dir)) return include_py_files def _process_include_py_files_pyi(self, needed_includes: list[tuple[str, str, str]] | None = None) -> None: include_py_files: list[tuple[str, str, str]] if needed_includes is not None: include_py_files = needed_includes else: include_py_files = self._collect_include_py_files() if not include_py_files: return _vlog().info(f"处理 {len(include_py_files)} 个被引用的 Python 库文件") for src_path, rel_from_inc, includes_dir in include_py_files: ModulePath: str = rel_from_inc.replace(os.sep, '.').replace('/', '.') module_name: str = os.path.splitext(ModulePath)[0] try: with open(src_path, 'r', encoding='utf-8') as f: content: str = f.read() sha1: str = compute_sha1(content) self.sha1_map[sha1] = f"includes/{rel_from_inc}" top_module: str = rel_from_inc.split(os.sep)[0].split('/')[0] if top_module not in self.include_py_map: self.include_py_map[top_module] = sha1 self.include_py_map[module_name] = sha1 sig_path: str = os.path.join(self.temp_dir, f"{sha1}.pyi") if os.path.isfile(sig_path): _vlog().info(f" 缓存命中: {rel_from_inc} -> {sha1}.pyi") continue sig_content: str = PythonToStubConverter.convert(content, module_name) with open(sig_path, 'w', encoding='utf-8', newline='\n') as f: f.write(sig_content) _vlog().info(f" 生成签名: {rel_from_inc} -> {sha1}.pyi") except Exception as e: _vlog().error(f"处理 include 文件失败 {rel_from_inc}: {e}") def _collect_typedef_map(self) -> dict[str, ast.AST]: typedef_map: dict[str, ast.AST] = {} if not os.path.isdir(self.temp_dir): return typedef_map for fname in os.listdir(self.temp_dir): if not fname.endswith('.pyi'): continue pyi_path: str = os.path.join(self.temp_dir, fname) try: with open(pyi_path, 'r', encoding='utf-8') as f: content: str = f.read() tree: ast.Module = ast.parse(content) for node in ast.iter_child_nodes(tree): if isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name): var_name: str = node.target.id is_typedef: bool = False if isinstance(node.annotation, ast.Attribute) and hasattr(node.annotation, 'attr') and node.annotation.attr == 'CTypedef': is_typedef = True elif isinstance(node.annotation, ast.Name) and node.annotation.id == 'CTypedef': is_typedef = True elif isinstance(node.annotation, ast.BinOp) and isinstance(node.annotation.left, ast.Attribute) and hasattr(node.annotation.left, 'attr') and node.annotation.left.attr == 'CTypedef': is_typedef = True if is_typedef and node.value and var_name not in typedef_map: typedef_map[var_name] = node.value except Exception as _e: if _ConfigMode == "strict": raise _vlog().warning(f"收集 typedef 映射失败: {_e}", "Exception") return typedef_map def _process_include_py_files_stub(self, struct_names: set[str], needed_includes: list[tuple[str, str, str]] | None = None, enum_names: set[str] | None = None, struct_sha1_map: dict[str, str] | None = None, exception_names: set[str] | None = None) -> None: include_py_files: list[tuple[str, str, str]] if needed_includes is not None: include_py_files = needed_includes else: include_py_files = self._collect_include_py_files() if not include_py_files: return typedef_map: dict[str, ast.AST] = self._collect_typedef_map() for src_path, rel_from_inc, includes_dir in include_py_files: ModulePath: str = rel_from_inc.replace(os.sep, '.').replace('/', '.') module_name: str = os.path.splitext(ModulePath)[0] try: with open(src_path, 'r', encoding='utf-8') as f: content: str = f.read() sha1: str = compute_sha1(content) stub_path: str = os.path.join(self.temp_dir, f"{sha1}.stub.ll") if os.path.isfile(stub_path): _vlog().info(f" 缓存命中: {rel_from_inc} -> {sha1}.stub.ll") continue sig_path: str = os.path.join(self.temp_dir, f"{sha1}.pyi") with open(sig_path, 'r', encoding='utf-8') as f: sig_content: str = f.read() self._generate_decl_ll(sig_content, stub_path, src_path, struct_names, enum_names=enum_names, module_sha1=sha1, struct_sha1_map=struct_sha1_map, exception_names=exception_names, typedef_map=typedef_map) _vlog().info(f" 生成声明: {rel_from_inc} -> {sha1}.stub.ll") except Exception as e: _vlog().error(f"处理 include 文件失败 {rel_from_inc}: {e}") def _process_file_pyi(self, src_path: str, rel_path: str) -> None: with open(src_path, 'r', encoding='utf-8') as f: content: str = f.read() sha1: str = compute_sha1(content) self.sha1_map[sha1] = rel_path sig_path: str = os.path.join(self.temp_dir, f"{sha1}.pyi") if os.path.isfile(sig_path): _vlog().info(f" -> {sha1}.pyi (缓存)") return module_name: str = os.path.splitext(rel_path)[0].replace(os.sep, '.').replace('/', '.') sig_content: str = PythonToStubConverter.convert(content, module_name) with open(sig_path, 'w', encoding='utf-8', newline='\n') as f: f.write(sig_content) _vlog().info(f" -> {sha1}.pyi (签名)") def _process_file_stub(self, src_path: str, rel_path: str, struct_names: set[str], enum_names: set[str] | None = None, struct_sha1_map: dict[str, str] | None = None, exception_names: set[str] | None = None) -> None: with open(src_path, 'r', encoding='utf-8') as f: content: str = f.read() sha1: str = compute_sha1(content) stub_path: str = os.path.join(self.temp_dir, f"{sha1}.stub.ll") if os.path.isfile(stub_path): _vlog().info(f" -> {sha1}.stub.ll (缓存)") return sig_path: str = os.path.join(self.temp_dir, f"{sha1}.pyi") with open(sig_path, 'r', encoding='utf-8') as f: sig_content: str = f.read() typedef_map: dict[str, ast.AST] = self._collect_typedef_map() self._generate_decl_ll(sig_content, stub_path, src_path, struct_names, enum_names=enum_names, module_sha1=sha1, struct_sha1_map=struct_sha1_map, exception_names=exception_names, typedef_map=typedef_map) _vlog().info(f" -> {sha1}.stub.ll (声明)") def _build_struct_registry(self) -> tuple[set[str], set[str], dict[str, str], set[str]]: struct_names: set[str] = set() enum_names: set[str] = set() exception_names: set[str] = set() struct_sha1_map: dict[str, str] = {} valid_sha1_keys: set[str] = set(self.sha1_map.keys()) for fname in os.listdir(self.temp_dir): if not fname.endswith('.pyi'): continue sha1_key: str = fname.replace('.pyi', '') if sha1_key not in valid_sha1_keys: continue pyi_path: str = os.path.join(self.temp_dir, fname) try: with open(pyi_path, 'r', encoding='utf-8') as f: content: str = f.read() tree: ast.Module = ast.parse(content) for node in ast.iter_child_nodes(tree): if isinstance(node, ast.ClassDef): is_enum: bool = False is_exception: bool = False if node.bases: for base in node.bases: if isinstance(base, ast.Attribute) and hasattr(base, 'attr'): if base.attr in ('CEnum', 'Enum', 'REnum'): is_enum = True break elif base.attr == 'Exception' or base.attr in exception_names: is_exception = True break elif isinstance(base, ast.Name) and hasattr(base, 'id'): if base.id in ('CEnum', 'Enum', 'REnum'): is_enum = True break elif base.id == 'Exception' or base.id in exception_names: is_exception = True break # 也检查装饰器形式 @t.CEnum / @t.REnum if not is_enum and not is_exception and hasattr(node, 'decorator_list') and node.decorator_list: for deco in node.decorator_list: deco_attr: str | None = None if isinstance(deco, ast.Attribute) and hasattr(deco, 'attr'): deco_attr = deco.attr elif isinstance(deco, ast.Name) and hasattr(deco, 'id'): deco_attr = deco.id if deco_attr in ('CEnum', 'Enum', 'REnum'): is_enum = True break if is_enum: enum_names.add(node.name) elif is_exception: exception_names.add(node.name) else: struct_names.add(node.name) struct_sha1_map[node.name] = sha1_key except Exception as _e: if _ConfigMode == "strict": raise _vlog().warning(f"构建结构体/枚举注册表失败: {_e}", "Exception") return struct_names, enum_names, struct_sha1_map, exception_names def _generate_decl_ll(self, pyi_content: str, ll_path: str, src_path: str, struct_names: set[str] | None = None, enum_names: set[str] | None = None, module_sha1: str | None = None, struct_sha1_map: dict[str, str] | None = None, exception_names: set[str] | None = None, typedef_map: dict[str, ast.AST] | None = None) -> None: try: decl_gen: DeclarationGenerator = DeclarationGenerator(struct_names=struct_names, enum_names=enum_names, module_sha1=module_sha1, target_triple=self.target_triple, target_datalayout=self.target_datalayout, struct_sha1_map=struct_sha1_map, exception_names=exception_names, typedef_map=typedef_map) decl_ll: str = decl_gen.generate(pyi_content, src_path) with open(ll_path, 'w', encoding='utf-8', newline='\n') as f: f.write(decl_ll) except Exception as e: _vlog().warning(f".ll 声明生成失败: {e}") with open(ll_path, 'w', encoding='utf-8') as f: f.write(f"; declaration for {os.path.basename(src_path)}\n; error: {e}\n")