snapshot before regression test

This commit is contained in:
t
2026-07-18 19:25:40 +08:00
commit 796222a300
2295 changed files with 206453 additions and 0 deletions

280
lib/Projectrans/Config.py Normal file
View File

@@ -0,0 +1,280 @@
from __future__ import annotations
import os
import sys
import json
import glob as glob_module
import argparse
import shutil
import subprocess
from typing import Any
from lib.core.VLogger import get_logger as _vlog
from lib.Projectrans.Phase1Generator import Phase1Generator
from lib.Projectrans.Phase2Translator import Phase2Translator
def save_sha1_map(temp_dir: str, sha1_map: dict[str, str]) -> None:
"""将 SHA1 映射表保存到文件"""
map_path: str = os.path.join(temp_dir, '_sha1_map.txt')
with open(map_path, 'w', encoding='utf-8') as f:
for sha1, rel in sorted(sha1_map.items()):
f.write(f"{sha1}:{rel}\n")
def Load_sha1_map(temp_dir: str) -> dict[str, str]:
"""从文件加载 SHA1 映射表"""
map_path: str = os.path.join(temp_dir, '_sha1_map.txt')
result: dict[str, str] = {}
if os.path.exists(map_path):
with open(map_path, 'r', encoding='utf-8') as f:
for line in f:
line: str = line.strip()
if ':' in line:
sha1: str
rel: str
sha1, rel = line.split(':', 1)
result[sha1] = rel
return result
def Load_project_config(project_file: str) -> dict[str, Any]:
"""从 project.json 加载配置"""
with open(project_file, 'r', encoding='utf-8') as f:
return json.load(f)
def resolve_paths(config: dict[str, Any], project_file: str) -> dict[str, Any]:
"""将配置中的相对路径转换为绝对路径"""
project_dir: str = os.path.dirname(os.path.abspath(project_file))
result: dict[str, Any] = dict(config)
def resolve(p: str | None) -> str | None:
if p is None:
return None
if os.path.isabs(p):
return p
return os.path.normpath(os.path.join(project_dir, p))
if 'source_dir' in result:
result['source_dir'] = resolve(result.get('source_dir'))
elif 'sources' in result:
resolved_sources: list[str] = []
for src in result['sources']:
if '*' in src or '?' in src:
for matched in glob_module.glob(os.path.join(project_dir, src), recursive=True):
if os.path.isfile(matched):
resolved_sources.append(matched)
else:
resolved_sources.append(resolve(src))
result['sources'] = [s for s in resolved_sources if os.path.isfile(s)]
result['temp_dir'] = resolve(config.get('temp_dir'))
result['output_dir'] = resolve(config.get('output_dir'))
if 'includes' in result:
result['includes'] = [resolve(inc) for inc in result['includes']]
result['_project_dir'] = project_dir
return result
def main() -> None:
parser: argparse.ArgumentParser = argparse.ArgumentParser(
description='TransPyC 工程翻译器 - 基于 project.json 的两阶段翻译',
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog='''
示例:
# 从 project.json 读取配置并执行(默认查找当前目录)
python ProjectTrans.py
# 指定 project.json 文件
python ProjectTrans.py --project ./TestProject/project.json
# 使用命令行参数覆盖 project.json 配置
python ProjectTrans.py --src ./src --temp ./decl --output ./out --phase all
# 仅运行阶段一
python ProjectTrans.py --phase 1
# 仅运行阶段二
python ProjectTrans.py --phase 2
# 清理临时目录
python ProjectTrans.py --clean
# 编译并立即执行
python Projectrans.py --project ./TestProject/project.json --clean --run
# 删除 includes.binary 预编译缓存并重新编译所有 includes
python Projectrans.py --project ./TestProject/project.json --rebuild-includes
'''
)
parser.add_argument('--project', help='project.json 路径(默认查找当前目录)')
parser.add_argument('--src', help='源文件目录(覆盖 project.json')
parser.add_argument('--temp', help='声明接口临时目录(覆盖 project.json')
parser.add_argument('--output', help='输出目录(覆盖 project.json')
parser.add_argument('--phase', choices=['1', '2', 'all'], help='阶段: 1=生成声明, 2=翻译+编译, all=全部')
parser.add_argument('--cc', help='LLVM 编译器命令(覆盖 project.json')
parser.add_argument('--clean', action='store_true', help='清理 output 和 temp 目录')
parser.add_argument('--run', action='store_true', help='编译成功后立即执行生成的可执行文件')
parser.add_argument('--rebuild-includes', action='store_true', help='删除 includes.binary 预编译缓存并重新编译所有 includes')
parser.add_argument('--clear-cache', action='store_true', help='清除 .transpyc_cache 全局缓存(.pyi/.stub.ll/.doc.json')
args: argparse.Namespace = parser.parse_args()
project_file: str | None = args.project
if not project_file:
candidates: list[str] = ['project.json', './project.json']
for c in candidates:
if os.path.exists(c):
project_file = c
break
if project_file and os.path.exists(project_file):
_vlog().info(f"读取配置文件: {project_file}")
config: dict[str, Any] = Load_project_config(project_file)
project_dir: str = os.path.dirname(os.path.abspath(project_file))
_vlog().info(f"项目目录: {project_dir}")
else:
_vlog().error("未找到 project.json请使用 --project 指定")
sys.exit(1)
resolved: dict[str, Any] = resolve_paths(config, project_file)
src: str | list[str] | None = args.src or resolved.get('source_dir') or resolved.get('sources')
temp_dir: str | None = args.temp or resolved.get('temp_dir')
output_dir: str | None = args.output or resolved.get('output_dir')
phase: str = args.phase or 'all'
cc_cmd: str = args.cc or resolved.get('compiler', {}).get('cmd', 'llc')
cc_flags: list[str] = resolved.get('compiler', {}).get('flags', ['-filetype=obj'])
slice_level: int = resolved.get('options', {}).get('slice_level', 3)
project_include_dirs: list[str] = resolved.get('includes', [])
target_triple: str | None = resolved.get('target', {}).get('triple')
target_datalayout: str | None = resolved.get('target', {}).get('datalayout')
if isinstance(src, list):
if len(src) == 1 and os.path.isdir(src[0]):
src = src[0]
else:
src = src[0] if len(src) == 1 else os.path.dirname(os.path.commonpath(src)) if all(os.path.exists(p) for p in src) else src[0]
_vlog().info(f"源目录: {src}")
_vlog().info(f"临时目录: {temp_dir}")
_vlog().info(f"输出目录: {output_dir}")
_vlog().info(f"阶段: {phase}")
_vlog().info(f"编译器: {cc_cmd} {' '.join(cc_flags)}")
if target_triple:
_vlog().info(f"目标三元组: {target_triple}")
if target_datalayout:
_vlog().info(f"数据布局: {target_datalayout}")
if args.clean:
if output_dir and os.path.exists(output_dir):
for f in os.listdir(output_dir):
fpath: str = os.path.join(output_dir, f)
try:
if os.path.isfile(fpath) or os.path.islink(fpath):
os.remove(fpath)
elif os.path.isdir(fpath):
shutil.rmtree(fpath, ignore_errors=True)
except:
pass
_vlog().info(f"output: 已清空 {output_dir}")
if temp_dir and os.path.exists(temp_dir):
for f in os.listdir(temp_dir):
fpath: str = os.path.join(temp_dir, f)
try:
if os.path.isfile(fpath) or os.path.islink(fpath):
os.remove(fpath)
elif os.path.isdir(fpath):
shutil.rmtree(fpath, ignore_errors=True)
except:
pass
_vlog().info(f"temp: 已清空 {temp_dir}")
_vlog().info("清理完成")
if (not phase or phase == 'all') and not args.project:
return
if args.clear_cache:
# 清除 .transpyc_cache 全局缓存
cache_dir: str = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), '.transpyc_cache')
if os.path.isdir(cache_dir):
shutil.rmtree(cache_dir, ignore_errors=True)
os.makedirs(cache_dir, exist_ok=True)
_vlog().info(f"已清除全局缓存: {cache_dir}")
else:
_vlog().info(f"全局缓存目录不存在: {cache_dir}")
if phase in ('1', 'all'):
if isinstance(src, list):
_vlog().error("阶段一需要指定源目录,不支持文件列表")
sys.exit(1)
all_include_dirs: list[str] = list(project_include_dirs) if project_include_dirs else []
for d in Phase2Translator.INCLUDE_DIRS:
if os.path.isdir(d):
d = os.path.abspath(d)
if d not in [os.path.abspath(x) for x in all_include_dirs]:
all_include_dirs.append(d)
gen: Phase1Generator = Phase1Generator(src, temp_dir, include_dirs=all_include_dirs, target_triple=target_triple, target_datalayout=target_datalayout)
gen.run()
save_sha1_map(temp_dir, gen.sha1_map)
if phase in ('2', 'all'):
if not output_dir:
_vlog().error("阶段二需要指定 --output 目录")
sys.exit(1)
if not os.path.exists(temp_dir):
_vlog().error(f"声明目录不存在,请先运行阶段一: {temp_dir}")
sys.exit(1)
if isinstance(src, list):
_vlog().error("阶段二需要指定源目录,不支持文件列表")
sys.exit(1)
linker_cmd: str | None = resolved.get('linker', {}).get('cmd')
linker_flags: list[str] = resolved.get('linker', {}).get('flags', [])
linker_output: str | None = resolved.get('linker', {}).get('output')
target_triple = resolved.get('target', {}).get('triple')
target_datalayout = resolved.get('target', {}).get('datalayout')
force_recompile_includes: bool = bool(resolved.get('options', {}).get('force_recompile_includes', False))
if args.rebuild_includes:
force_recompile_includes = True
all_inc_dirs: list[str] = list(project_include_dirs) + list(Phase2Translator.INCLUDE_DIRS)
seen_dirs: set[str] = set()
for inc_dir in all_inc_dirs:
abs_inc: str = os.path.abspath(inc_dir)
if abs_inc in seen_dirs:
continue
seen_dirs.add(abs_inc)
if not os.path.isdir(abs_inc):
continue
inc_parent: str = os.path.dirname(abs_inc)
inc_base: str = os.path.basename(abs_inc)
precompiled_dir: str = os.path.join(inc_parent, inc_base + '.binary')
if os.path.isdir(precompiled_dir):
cleared: int = 0
for fname in os.listdir(precompiled_dir):
fpath: str = os.path.join(precompiled_dir, fname)
try:
if os.path.isfile(fpath) or os.path.islink(fpath):
os.remove(fpath)
cleared += 1
except:
pass
_vlog().info(f"includes.binary: 已清空 {precompiled_dir} ({cleared} 个文件)")
trans: Phase2Translator = Phase2Translator(src, temp_dir, output_dir, compile_cmd=cc_cmd, compile_flags=cc_flags, linker_cmd=linker_cmd, linker_flags=linker_flags, linker_output=linker_output, include_dirs=project_include_dirs, target_triple=target_triple, target_datalayout=target_datalayout, force_recompile_includes=force_recompile_includes)
trans.sha1_map = Load_sha1_map(temp_dir)
trans.slice_level = slice_level
trans.run()
if args.run and linker_output and not linker_output.endswith('.bin'):
exe_path: str = os.path.join(output_dir, linker_output)
if os.path.isfile(exe_path):
_vlog().info(f"\n[运行] 执行: {exe_path}")
try:
result: subprocess.CompletedProcess = subprocess.run([exe_path], cwd=output_dir)
_vlog().info(f"[运行] 退出码: {result.returncode}")
except Exception as e:
_vlog().error(f"[运行] 执行失败: {e}")
sys.exit(1)
else:
_vlog().error(f"[运行] 找不到可执行文件: {exe_path}")
sys.exit(1)

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,595 @@
from __future__ import annotations
import os
import ast
import json
import shutil
import traceback
from lib.Projectrans.Utils import compute_sha1, get_file_dependencies, find_reachable_files_from_entries, parse_python_file
from lib.Projectrans.DeclarationGenerator import DeclarationGenerator
from lib.core.VLogger import get_logger as _vlog
from lib.constants.config import mode as _ConfigMode
from StubGen import PythonToStubConverter
# 全局缓存目录(不被 --clean 清除),用于缓存库文件的 .pyi / .stub.ll / .doc.json
_COMPILER_ROOT: str = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
_GLOBAL_CACHE_DIR: str = os.path.join(_COMPILER_ROOT, '.transpyc_cache')
def _TryGlobalCache(Sha1: str, Ext: str, TempPath: str) -> bool:
"""检查全局缓存中是否存在指定文件,若存在则复制到 temp_dir 并返回 True。"""
GlobalPath: str = os.path.join(_GLOBAL_CACHE_DIR, f"{Sha1}.{Ext}")
if os.path.isfile(GlobalPath):
try:
os.makedirs(os.path.dirname(TempPath), exist_ok=True)
shutil.copy2(GlobalPath, TempPath)
return True
except Exception:
pass
return False
def _SaveToGlobalCache(Sha1: str, Ext: str, TempPath: str) -> None:
"""将文件保存到全局缓存供后续使用。"""
GlobalPath: str = os.path.join(_GLOBAL_CACHE_DIR, f"{Sha1}.{Ext}")
try:
os.makedirs(_GLOBAL_CACHE_DIR, exist_ok=True)
shutil.copy2(TempPath, GlobalPath)
except Exception:
pass
class Phase1Generator:
"""阶段一:从源文件生成声明接口"""
def __init__(self, src_root: str, temp_dir: str, include_dirs: list[str] | None = None, entry_files: list[str] | None = None, target_triple: str | None = None, target_datalayout: str | None = None) -> None:
self.src_root: str = os.path.abspath(src_root)
self.temp_dir: str = os.path.abspath(temp_dir)
self.include_dirs: list[str] = include_dirs or []
self.sha1_map: dict[str, str] = {}
self.include_py_map: dict[str, str] = {}
self.entry_files: list[str] | None = entry_files
self.target_triple: str | None = target_triple
self.target_datalayout: str | None = target_datalayout
os.makedirs(self.temp_dir, exist_ok=True)
# 增量编译 manifest{abs_path: {sha1, mtime, size}}
# 未变更文件mtime+size 匹配)跳过 open+read+sha1仅需 stat
self._manifest: dict[str, dict] = {}
self._typedef_map_cache: dict[str, ast.AST] | None = None
self._manifest_path: str = os.path.join(self.temp_dir, '_phase1_manifest.json')
self._load_manifest()
def _load_manifest(self) -> None:
"""加载上次运行的 manifestmtime+size → sha1 缓存)"""
try:
if os.path.isfile(self._manifest_path):
with open(self._manifest_path, 'r', encoding='utf-8') as f:
self._manifest = json.load(f)
except Exception:
self._manifest = {}
def _save_manifest(self) -> None:
"""保存 manifest 供下次增量编译使用"""
try:
with open(self._manifest_path, 'w', encoding='utf-8') as f:
json.dump(self._manifest, f)
except Exception:
pass
def _get_sha1(self, src_path: str) -> str:
"""用 mtime+size 缓存加速 sha1 计算。
未变更文件mtime+size 匹配 manifest直接返回缓存的 sha1
跳过 open+read+compute_sha1仅需一次 stat 调用。
"""
abs_path: str = os.path.abspath(src_path)
try:
st: os.stat_result = os.stat(abs_path)
except OSError:
return ''
mtime: float = st.st_mtime
size: int = st.st_size
entry: dict = self._manifest.get(abs_path, {})
if entry.get('mtime') == mtime and entry.get('size') == size:
sha1: str = entry.get('sha1', '')
if sha1:
return sha1
# mtime/size 不匹配,需读取内容重新计算
try:
with open(abs_path, 'r', encoding='utf-8') as f:
content: str = f.read()
except Exception:
return ''
sha1 = compute_sha1(content)
self._manifest[abs_path] = {'sha1': sha1, 'mtime': mtime, 'size': size}
return sha1
def _get_needed_include_files(self, reachable_source_files: set[str]) -> list[tuple[str, str, str]]:
"""从可达源文件收集所有被引用(含传递依赖)的 include 文件"""
include_file_map: dict[str, tuple[str, str, str]] = {}
for includes_dir in self.include_dirs:
if not os.path.isdir(includes_dir):
continue
for root, dirs, files in os.walk(includes_dir):
dirs[:] = [d for d in dirs if not d.startswith('.') and d != '__pycache__']
for fname in files:
if fname.endswith('.py') or fname.endswith('.pyi'):
src_path: str = os.path.join(root, fname)
rel_from_inc: str = os.path.relpath(src_path, includes_dir)
ModulePath: str = rel_from_inc.replace(os.sep, '.').replace('/', '.')
module_name: str = os.path.splitext(ModulePath)[0]
info: tuple[str, str, str] = (src_path, rel_from_inc, includes_dir)
include_file_map[module_name] = info
top_pkg: str = module_name.split('.')[0]
# __init__.py 是包入口,应优先作为 top_pkg 的代表
# (参考 find_reachable_files_from_entries 的优先级逻辑),
# 否则若 os/path.py 先被遍历,'os' 会错误指向 path.py
# 而非 os/__init__.py导致包入口 SHA1 未注册到 sha1_map
if module_name == f"{top_pkg}.__init__":
include_file_map[top_pkg] = info
elif top_pkg not in include_file_map:
include_file_map[top_pkg] = info
imported_from_src: set[str] = set()
for src_path in reachable_source_files:
deps: set[str] = get_file_dependencies(src_path, self.src_root)
imported_from_src.update(deps)
# 检测 dict/list 容器使用,自动添加 _dict/_list/json 依赖
for src_path in reachable_source_files:
try:
# 复用 parse_python_file 的 AST 缓存get_file_dependencies 已解析过同一文件)
content, tree = parse_python_file(src_path)
if tree is None:
continue
for node in ast.walk(tree):
if isinstance(node, ast.Subscript) and isinstance(node.value, ast.Name):
if node.value.id == 'dict':
imported_from_src.add('_dict')
imported_from_src.add('json')
break
elif node.value.id == 'list':
imported_from_src.add('_list')
break
if isinstance(node, ast.Call) and isinstance(node.func, ast.Name):
if node.func.id == 'dict':
imported_from_src.add('_dict')
imported_from_src.add('json')
break
elif node.func.id == 'list':
imported_from_src.add('_list')
break
# 检测类型注解 d: dict / l: list (非泛型形式)
if isinstance(node, ast.AnnAssign) and isinstance(node.annotation, ast.Name):
if node.annotation.id == 'dict':
imported_from_src.add('_dict')
imported_from_src.add('json')
break
elif node.annotation.id == 'list':
imported_from_src.add('_list')
break
except Exception:
pass
needed: set[str] = set()
queue: list[str] = list(imported_from_src)
while queue:
mod_name: str = queue.pop(0)
if mod_name in needed:
continue
if mod_name in include_file_map:
needed.add(mod_name)
src_path: str = include_file_map[mod_name][0]
includes_dir: str = include_file_map[mod_name][2]
deps: set[str] = get_file_dependencies(src_path, includes_dir)
for dep in deps:
if dep in include_file_map and dep not in needed:
queue.append(dep)
needed_infos: list[tuple[str, str, str]] = []
for mod_name in sorted(needed):
if mod_name in include_file_map:
info: tuple[str, str, str] = include_file_map[mod_name]
if info not in needed_infos:
needed_infos.append(info)
return needed_infos
def run(self) -> None:
"""扫描源目录,只处理入口文件可达的 .py 文件(导入遍历)"""
py_files: list[str]
if self.entry_files:
py_files = list(self.entry_files)
else:
main_py: str = os.path.join(self.src_root, 'main.py')
if os.path.exists(main_py):
py_files = [main_py]
else:
py_files = []
for root, dirs, files in os.walk(self.src_root):
dirs[:] = [d for d in dirs if d not in ('__pycache__',)]
for file in files:
if file.endswith('.py'):
py_files.append(os.path.join(root, file))
if not py_files:
_vlog().warning("未找到入口文件或源文件")
return
reachable: set[str]
if self.entry_files:
reachable = find_reachable_files_from_entries(self.src_root, py_files)
else:
reachable = find_reachable_files_from_entries(self.src_root, py_files)
_vlog().info(f"找到 {len(reachable)} 个可达源文件(从入口遍历)")
for i, src_path in enumerate(sorted(reachable), 1):
rel: str = os.path.relpath(src_path, self.src_root)
_vlog().info(f"[{i}/{len(reachable)}] 生成签名: {rel}")
try:
self._process_file_pyi(src_path, rel)
except Exception as e:
_vlog().error(f"生成签名失败: {e}")
traceback.print_exc()
needed_includes: list[tuple[str, str, str]] = self._get_needed_include_files(reachable)
self._process_include_py_files_pyi(needed_includes)
struct_names: set[str]
enum_names: set[str]
struct_sha1_map: dict[str, str]
exception_names: set[str]
struct_names, enum_names, struct_sha1_map, exception_names, class_def_map = self._build_struct_registry()
_vlog().info(f"结构体注册表: {len(struct_names)} 个类型, {len(enum_names)} 个枚举")
for name in sorted(struct_names):
_vlog().debug(f" {name}")
# 预先收集 typedef 映射,避免在 _process_file_stub 中对每个源文件重复扫描所有 .pyi
self._typedef_map_cache: dict[str, ast.AST] = self._collect_typedef_map()
for i, src_path in enumerate(sorted(reachable), 1):
rel: str = os.path.relpath(src_path, self.src_root)
try:
self._process_file_stub(src_path, rel, struct_names, enum_names=enum_names, struct_sha1_map=struct_sha1_map, exception_names=exception_names, class_def_map=class_def_map)
except Exception as e:
_vlog().error(f"生成声明失败: {e}")
traceback.print_exc()
self._process_include_py_files_stub(struct_names, needed_includes, enum_names=enum_names, struct_sha1_map=struct_sha1_map, exception_names=exception_names, class_def_map=class_def_map)
# 保存 manifest 供下次增量编译使用
self._save_manifest()
_vlog().success(f"声明接口生成到: {self.temp_dir}")
_vlog().info(f"SHA1 映射表 ({len(self.sha1_map)} 个文件):")
for sha1, rel in sorted(self.sha1_map.items()):
_vlog().debug(f" {sha1} -> {rel}")
def _collect_include_py_files(self) -> list[tuple[str, str, str]]:
include_py_files: list[tuple[str, str, str]] = []
for includes_dir in self.include_dirs:
if not os.path.isdir(includes_dir):
continue
for root, dirs, files in os.walk(includes_dir):
dirs[:] = [d for d in dirs if not d.startswith('.') and d != '__pycache__']
for fname in files:
if fname.endswith('.py') or fname.endswith('.pyi'):
src_path: str = os.path.join(root, fname)
rel_from_inc: str = os.path.relpath(src_path, includes_dir)
include_py_files.append((src_path, rel_from_inc, includes_dir))
return include_py_files
def _process_include_py_files_pyi(self, needed_includes: list[tuple[str, str, str]] | None = None) -> None:
include_py_files: list[tuple[str, str, str]]
if needed_includes is not None:
include_py_files = needed_includes
else:
include_py_files = self._collect_include_py_files()
if not include_py_files:
return
_vlog().info(f"处理 {len(include_py_files)} 个被引用的 Python 库文件")
for src_path, rel_from_inc, includes_dir in include_py_files:
ModulePath: str = rel_from_inc.replace(os.sep, '.').replace('/', '.')
module_name: str = os.path.splitext(ModulePath)[0]
try:
# 增量优化:用 mtime+size 获取 sha1未变更文件跳过 open+read
sha1: str = self._get_sha1(src_path)
if not sha1:
continue
self.sha1_map[sha1] = f"includes/{rel_from_inc}"
top_module: str = rel_from_inc.split(os.sep)[0].split('/')[0]
if top_module not in self.include_py_map:
self.include_py_map[top_module] = sha1
self.include_py_map[module_name] = sha1
sig_path: str = os.path.join(self.temp_dir, f"{sha1}.pyi")
if os.path.isfile(sig_path):
_vlog().info(f" 缓存命中: {rel_from_inc} -> {sha1}.pyi")
continue
# 全局缓存检查(不被 --clean 清除)
if _TryGlobalCache(sha1, 'pyi', sig_path):
_vlog().info(f" 缓存命中(全局): {rel_from_inc} -> {sha1}.pyi")
continue
# 缓存未命中,需读取 content 生成 .pyi
with open(src_path, 'r', encoding='utf-8') as f:
content: str = f.read()
sig_content: str = PythonToStubConverter.convert(content, module_name)
with open(sig_path, 'w', encoding='utf-8', newline='\n') as f:
f.write(sig_content)
_SaveToGlobalCache(sha1, 'pyi', sig_path)
_vlog().info(f" 生成签名: {rel_from_inc} -> {sha1}.pyi")
except Exception as e:
_vlog().error(f"处理 include 文件失败 {rel_from_inc}: {e}")
def _collect_typedef_map(self) -> dict[str, ast.AST]:
typedef_map: dict[str, ast.AST] = {}
if not os.path.isdir(self.temp_dir):
return typedef_map
for fname in os.listdir(self.temp_dir):
if not fname.endswith('.pyi'):
continue
pyi_path: str = os.path.join(self.temp_dir, fname)
try:
with open(pyi_path, 'r', encoding='utf-8') as f:
content: str = f.read()
tree: ast.Module = ast.parse(content)
for node in ast.iter_child_nodes(tree):
if isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name):
var_name: str = node.target.id
is_typedef: bool = False
if isinstance(node.annotation, ast.Attribute) and hasattr(node.annotation, 'attr') and node.annotation.attr == 'CTypedef':
is_typedef = True
elif isinstance(node.annotation, ast.Name) and node.annotation.id == 'CTypedef':
is_typedef = True
elif isinstance(node.annotation, ast.BinOp) and isinstance(node.annotation.left, ast.Attribute) and hasattr(node.annotation.left, 'attr') and node.annotation.left.attr == 'CTypedef':
is_typedef = True
if is_typedef and node.value and var_name not in typedef_map:
typedef_map[var_name] = node.value
except Exception as _e:
if _ConfigMode == "strict":
raise
_vlog().warning(f"收集 typedef 映射失败: {_e}", "Exception")
return typedef_map
def _process_include_py_files_stub(self, struct_names: set[str], needed_includes: list[tuple[str, str, str]] | None = None, enum_names: set[str] | None = None, struct_sha1_map: dict[str, str] | None = None, exception_names: set[str] | None = None, class_def_map: dict[str, ast.ClassDef] | None = None) -> None:
include_py_files: list[tuple[str, str, str]]
if needed_includes is not None:
include_py_files = needed_includes
else:
include_py_files = self._collect_include_py_files()
if not include_py_files:
return
typedef_map: dict[str, ast.AST] = self._collect_typedef_map()
for src_path, rel_from_inc, includes_dir in include_py_files:
ModulePath: str = rel_from_inc.replace(os.sep, '.').replace('/', '.')
module_name: str = os.path.splitext(ModulePath)[0]
try:
# 增量优化:用 mtime+size 获取 sha1未变更文件跳过 open+read
sha1: str = self._get_sha1(src_path)
if not sha1:
continue
stub_path: str = os.path.join(self.temp_dir, f"{sha1}.stub.ll")
if os.path.isfile(stub_path):
_vlog().info(f" 缓存命中: {rel_from_inc} -> {sha1}.stub.ll")
continue
# 全局缓存检查(不被 --clean 清除)
if _TryGlobalCache(sha1, 'stub.ll', stub_path):
_vlog().info(f" 缓存命中(全局): {rel_from_inc} -> {sha1}.stub.ll")
continue
sig_path: str = os.path.join(self.temp_dir, f"{sha1}.pyi")
if not os.path.isfile(sig_path):
_TryGlobalCache(sha1, 'pyi', sig_path)
with open(sig_path, 'r', encoding='utf-8') as f:
sig_content: str = f.read()
self._generate_decl_ll(sig_content, stub_path, src_path, struct_names, enum_names=enum_names, module_sha1=sha1, struct_sha1_map=struct_sha1_map, exception_names=exception_names, typedef_map=typedef_map, class_def_map=class_def_map)
_SaveToGlobalCache(sha1, 'stub.ll', stub_path)
_vlog().info(f" 生成声明: {rel_from_inc} -> {sha1}.stub.ll")
except Exception as e:
_vlog().error(f"处理 include 文件失败 {rel_from_inc}: {e}")
def _extract_docstrings(self, content: str) -> dict[str, str]:
"""从源码 AST 提取所有函数/结构体/方法的首条裸字符串字面量 docstring。
- 顶层 FunctionDef: key = 函数名
- 顶层 ClassDef: key = 类名同时遍历方法key = "ClassName.method"
- docstring 必须是 body[0] 为 ast.Expr + ast.Constant(str)
- 空字符串不计入
"""
result: dict[str, str] = {}
try:
tree = ast.parse(content)
except SyntaxError:
return result
for node in tree.body:
if isinstance(node, ast.FunctionDef):
doc = self._get_first_docstring(node.body)
if doc is not None and doc != '':
result[node.name] = doc
elif isinstance(node, ast.ClassDef):
doc = self._get_first_docstring(node.body)
if doc is not None and doc != '':
result[node.name] = doc
for sub in node.body:
if isinstance(sub, ast.FunctionDef):
m_doc = self._get_first_docstring(sub.body)
if m_doc is not None and m_doc != '':
result[f"{node.name}.{sub.name}"] = m_doc
return result
def _get_first_docstring(self, body: list[ast.stmt]) -> str | None:
"""返回 body 首条裸字符串字面量,否则 None"""
if body and isinstance(body[0], ast.Expr) and isinstance(body[0].value, ast.Constant) and isinstance(body[0].value.value, str):
return body[0].value.value
return None
def _process_file_pyi(self, src_path: str, rel_path: str) -> None:
# 增量优化:先用 mtime+size 获取 sha1未变更文件跳过 open+read
sha1: str = self._get_sha1(src_path)
if not sha1:
return
self.sha1_map[sha1] = rel_path
sig_path: str = os.path.join(self.temp_dir, f"{sha1}.pyi")
if os.path.isfile(sig_path):
_vlog().info(f" -> {sha1}.pyi (缓存)")
# 缓存命中时补全缺失的 .doc.json
doc_path: str = os.path.join(self.temp_dir, f"{sha1}.doc.json")
if not os.path.isfile(doc_path):
if not _TryGlobalCache(sha1, 'doc.json', doc_path):
# doc.json 缺失时需读 content 提取 docstring
with open(src_path, 'r', encoding='utf-8') as f:
content: str = f.read()
docs: dict[str, str] = self._extract_docstrings(content)
try:
with open(doc_path, 'w', encoding='utf-8', newline='\n') as f:
json.dump(docs, f, ensure_ascii=False)
_SaveToGlobalCache(sha1, 'doc.json', doc_path)
except Exception as e:
_vlog().warning(f"补写 {sha1}.doc.json 失败: {e}")
return
# 全局缓存检查
if _TryGlobalCache(sha1, 'pyi', sig_path):
_vlog().info(f" -> {sha1}.pyi (全局缓存)")
doc_path: str = os.path.join(self.temp_dir, f"{sha1}.doc.json")
if not os.path.isfile(doc_path):
_TryGlobalCache(sha1, 'doc.json', doc_path)
return
# 缓存未命中,需读取 content 生成 .pyi
with open(src_path, 'r', encoding='utf-8') as f:
content = f.read()
module_name: str = os.path.splitext(rel_path)[0].replace(os.sep, '.').replace('/', '.')
sig_content: str = PythonToStubConverter.convert(content, module_name)
with open(sig_path, 'w', encoding='utf-8', newline='\n') as f:
f.write(sig_content)
_SaveToGlobalCache(sha1, 'pyi', sig_path)
_vlog().info(f" -> {sha1}.pyi (签名)")
# 提取 docstring 写入 {sha1}.doc.json供 Phase2 跨模块 __doc__ 加载
docs: dict[str, str] = self._extract_docstrings(content)
doc_path: str = os.path.join(self.temp_dir, f"{sha1}.doc.json")
try:
with open(doc_path, 'w', encoding='utf-8', newline='\n') as f:
json.dump(docs, f, ensure_ascii=False)
_SaveToGlobalCache(sha1, 'doc.json', doc_path)
except Exception as e:
_vlog().warning(f"写入 {sha1}.doc.json 失败: {e}")
def _process_file_stub(self, src_path: str, rel_path: str, struct_names: set[str], enum_names: set[str] | None = None, struct_sha1_map: dict[str, str] | None = None, exception_names: set[str] | None = None, class_def_map: dict[str, ast.ClassDef] | None = None) -> None:
# 增量优化:用 mtime+size 获取 sha1未变更文件跳过 open+read
sha1: str = self._get_sha1(src_path)
if not sha1:
return
stub_path: str = os.path.join(self.temp_dir, f"{sha1}.stub.ll")
if os.path.isfile(stub_path):
_vlog().info(f" -> {sha1}.stub.ll (缓存)")
return
sig_path: str = os.path.join(self.temp_dir, f"{sha1}.pyi")
with open(sig_path, 'r', encoding='utf-8') as f:
sig_content: str = f.read()
typedef_map: dict[str, ast.AST] = self._typedef_map_cache or self._collect_typedef_map()
self._generate_decl_ll(sig_content, stub_path, src_path, struct_names, enum_names=enum_names, module_sha1=sha1, struct_sha1_map=struct_sha1_map, exception_names=exception_names, typedef_map=typedef_map, class_def_map=class_def_map)
_vlog().info(f" -> {sha1}.stub.ll (声明)")
def _build_struct_registry(self) -> tuple[set[str], set[str], dict[str, str], set[str], dict[str, ast.ClassDef]]:
struct_names: set[str] = set()
enum_names: set[str] = set()
exception_names: set[str] = set()
struct_sha1_map: dict[str, str] = {}
class_def_map: dict[str, ast.ClassDef] = {}
valid_sha1_keys: set[str] = set(self.sha1_map.keys())
for fname in os.listdir(self.temp_dir):
if not fname.endswith('.pyi'):
continue
sha1_key: str = fname.replace('.pyi', '')
if sha1_key not in valid_sha1_keys:
continue
pyi_path: str = os.path.join(self.temp_dir, fname)
try:
with open(pyi_path, 'r', encoding='utf-8') as f:
content: str = f.read()
tree: ast.Module = ast.parse(content)
for node in ast.iter_child_nodes(tree):
if isinstance(node, ast.ClassDef):
is_enum: bool = False
is_renum: bool = False
is_exception: bool = False
if node.bases:
for base in node.bases:
if isinstance(base, ast.Attribute) and hasattr(base, 'attr'):
if base.attr == 'REnum':
is_renum = True
break
elif base.attr in ('CEnum', 'Enum'):
is_enum = True
break
elif base.attr == 'Exception' or base.attr in exception_names:
is_exception = True
break
elif isinstance(base, ast.Name) and hasattr(base, 'id'):
if base.id == 'REnum':
is_renum = True
break
elif base.id in ('CEnum', 'Enum'):
is_enum = True
break
elif base.id == 'Exception' or base.id in exception_names:
is_exception = True
break
# 也检查装饰器形式 @t.CEnum / @t.REnum
if not is_enum and not is_renum and not is_exception and hasattr(node, 'decorator_list') and node.decorator_list:
for deco in node.decorator_list:
deco_attr: str | None = None
if isinstance(deco, ast.Attribute) and hasattr(deco, 'attr'):
deco_attr = deco.attr
elif isinstance(deco, ast.Name) and hasattr(deco, 'id'):
deco_attr = deco.id
if deco_attr == 'REnum':
is_renum = True
break
elif deco_attr in ('CEnum', 'Enum'):
is_enum = True
break
# REnum 是结构体类型(带 __tag + payload需加入 struct_names 以便
# DeclarationGenerator._get_type_str 返回 %Name* 而非 i32
if is_renum:
struct_names.add(node.name)
struct_sha1_map[node.name] = sha1_key
elif is_enum:
enum_names.add(node.name)
elif is_exception:
exception_names.add(node.name)
else:
struct_names.add(node.name)
struct_sha1_map[node.name] = sha1_key
# 收集所有类(含枚举/异常)的 ClassDef 节点,供 DeclarationGenerator
# 跨模块字段展平使用(如 exprs.py 的 Name(AST) 需查找 base.py 的 AST
class_def_map[node.name] = node
except Exception as _e:
if _ConfigMode == "strict":
raise
_vlog().warning(f"构建结构体/枚举注册表失败: {_e}", "Exception")
return struct_names, enum_names, struct_sha1_map, exception_names, class_def_map
def _generate_decl_ll(self, pyi_content: str, ll_path: str, src_path: str, struct_names: set[str] | None = None, enum_names: set[str] | None = None, module_sha1: str | None = None, struct_sha1_map: dict[str, str] | None = None, exception_names: set[str] | None = None, typedef_map: dict[str, ast.AST] | None = None, class_def_map: dict[str, ast.ClassDef] | None = None) -> None:
try:
decl_gen: DeclarationGenerator = DeclarationGenerator(struct_names=struct_names, enum_names=enum_names, module_sha1=module_sha1, target_triple=self.target_triple, target_datalayout=self.target_datalayout, struct_sha1_map=struct_sha1_map, exception_names=exception_names, typedef_map=typedef_map, class_def_map=class_def_map)
decl_ll: str = decl_gen.generate(pyi_content, src_path)
with open(ll_path, 'w', encoding='utf-8', newline='\n') as f:
f.write(decl_ll)
except Exception as e:
_vlog().warning(f".ll 声明生成失败: {e}")
with open(ll_path, 'w', encoding='utf-8') as f:
f.write(f"; declaration for {os.path.basename(src_path)}\n; error: {e}\n")

File diff suppressed because it is too large Load Diff

231
lib/Projectrans/Utils.py Normal file
View File

@@ -0,0 +1,231 @@
from __future__ import annotations
import hashlib
import os
import ast
from lib.core.VLogger import get_logger as _vlog
from lib.constants.config import mode as _ConfigMode
from lib.core.SymbolUtils import AnnotationContainsName
def compute_sha1(content: str) -> str:
return hashlib.sha1(content.encode('utf-8')).hexdigest()[:16]
# get_file_dependencies 缓存:键 (filepath, src_root),值 set[str]
# 单次编译中同一文件被 find_reachable_files 和 topological_sort 重复解析,
# 缓存避免重复 open+read+ast.parse+ast.walk~350 次调用→~175 次)
_FILE_DEPS_CACHE: dict[tuple[str, str], set[str]] = {}
# Python 源文件 AST 缓存:键 filepath值 (mtime, content, tree)
# 多处解析同一 .py 文件get_file_dependencies、used_includes 收集、
# _collect_inline_symbols 等)共享 parse 结果,避免重复 ast.parse~18s→~9s
_FILE_AST_CACHE: dict[str, tuple[float, str, ast.Module]] = {}
def _check_annotation_for_export(annotation: ast.AST | None) -> bool:
"""递归检查类型注解中是否包含 CExport/CExtern/Statet.State = t.CExtern | t.CExport"""
if not annotation:
return False
return (AnnotationContainsName(annotation, 'CExport') or
AnnotationContainsName(annotation, 'CExtern') or
AnnotationContainsName(annotation, 'State'))
def sha1_file(filepath: str) -> str:
with open(filepath, 'r', encoding='utf-8') as f:
content: str = f.read()
return compute_sha1(content)
def parse_python_file(filepath: str) -> tuple[str, ast.Module] | tuple[str, None]:
"""读取并解析 Python 文件,带 mtime 校验的 AST 缓存
多处调用共享同一文件的 parse 结果,避免重复 open+read+ast.parse。
返回 (content, tree);文件读取或解析失败返回 (content, None)。
"""
try:
st: os.stat_result = os.stat(filepath)
except OSError:
return ('', None)
mtime: float = st.st_mtime
cached: tuple[float, str, ast.Module] | None = _FILE_AST_CACHE.get(filepath)
if cached is not None and cached[0] == mtime:
return (cached[1], cached[2])
try:
with open(filepath, 'r', encoding='utf-8') as f:
content: str = f.read()
except Exception:
return ('', None)
try:
tree: ast.Module = ast.parse(content)
except SyntaxError:
return (content, None)
_FILE_AST_CACHE[filepath] = (mtime, content, tree)
return (content, tree)
def get_file_dependencies(filepath: str, src_root: str) -> set[str]:
"""解析 Python 文件的导入依赖(带缓存)
单次编译中 find_reachable_files_from_entries 和 topological_sort_files
会对同一批文件重复调用,缓存避免重复 ast.parse~8.6s→~4.3s)。
"""
cache_key: tuple[str, str] = (filepath, src_root)
cached: set[str] | None = _FILE_DEPS_CACHE.get(cache_key)
if cached is not None:
return cached
dependencies: set[str] = set()
content, tree = parse_python_file(filepath)
if tree is not None:
try:
for node in ast.walk(tree):
if isinstance(node, ast.Import):
for alias in node.names:
module: str = alias.name
dependencies.add(module)
elif isinstance(node, ast.ImportFrom):
if node.module:
module: str = node.module
if node.level > 0:
# from .module import name → 依赖 pkg.module模块本身
pkg_dir: str = os.path.dirname(filepath)
pkg_rel: str = os.path.relpath(pkg_dir, src_root).replace(os.sep, '.').replace('/', '.')
if pkg_rel == '.':
pkg_rel = ''
dep_module: str
if pkg_rel:
dep_module = f'{pkg_rel}.{module}'
else:
dep_module = module
dependencies.add(dep_module)
else:
if not module.startswith('_'):
dependencies.add(module)
elif node.level > 0 and node.names:
# from . import name → 依赖 pkg.name子模块
for alias in node.names:
pkg_dir: str = os.path.dirname(filepath)
pkg_rel: str = os.path.relpath(pkg_dir, src_root).replace(os.sep, '.').replace('/', '.')
if pkg_rel == '.':
pkg_rel = ''
dep_module: str = f'{pkg_rel}.{alias.name}' if pkg_rel else alias.name
dependencies.add(dep_module)
except Exception as _e:
if _ConfigMode == "strict":
raise
_vlog().warning(f"解析文件导入依赖失败: {_e}", "Exception")
_FILE_DEPS_CACHE[cache_key] = dependencies
return dependencies
def clear_file_deps_cache() -> None:
"""清除 get_file_dependencies 和 AST 缓存(跨编译或文件变更时调用)"""
_FILE_DEPS_CACHE.clear()
_FILE_AST_CACHE.clear()
def find_reachable_files_from_entries(src_root: str, entry_files: list[str]) -> set[str]:
"""从入口文件出发,找出所有可达的 .py 文件(通过导入关系)"""
all_files: dict[str, str] = {} # module_name -> filepath
for root, dirs, files in os.walk(src_root):
dirs[:] = [d for d in dirs if d not in ('__pycache__',)]
for file in files:
if file.endswith('.py'):
filepath: str = os.path.join(root, file)
rel: str = os.path.relpath(filepath, src_root)
module: str = os.path.splitext(rel)[0].replace(os.sep, '.').replace('/', '.')
if filepath.endswith('__init__.py'):
all_files[module] = filepath
parent_module: str = module.rsplit('.', 1)[0] if '.' in module else module
if parent_module not in all_files or not all_files[parent_module].endswith('__init__.py'):
all_files[parent_module] = filepath
elif module not in all_files:
all_files[module] = filepath
top_module: str = module.split('.')[0]
if top_module not in all_files:
all_files[top_module] = filepath
reachable: set[str] = set()
queue: list[str] = list(entry_files)
# BFS trace: record each file processed and unresolved deps
_bfs_trace: list[str] = []
_unresolved_deps: list[tuple[str, str]] = [] # (current_file, dep)
while queue:
current: str = queue.pop(0)
if current in reachable:
continue
reachable.add(current)
deps: set[str] = get_file_dependencies(current, src_root)
_bfs_trace.append(f" {os.path.basename(current)} deps={len(deps)}")
for dep in deps:
if dep in all_files and all_files[dep] not in reachable:
queue.append(all_files[dep])
else:
_matched = False
for key, val in all_files.items():
if key.endswith('.' + dep) or key == dep:
if val not in reachable:
queue.append(val)
_matched = True
break
if not _matched:
_unresolved_deps.append((os.path.basename(current), dep))
return reachable
def topological_sort_files(py_files: list[str], src_root: str) -> list[str]:
"""对 Python 文件进行拓扑排序,确保依赖模块先被处理"""
# 构建模块名到文件路径的映射
module_to_file: dict[str, str] = {}
for filepath in py_files:
rel: str = os.path.relpath(filepath, src_root)
module: str = os.path.splitext(rel)[0].replace(os.sep, '.').replace('/', '.')
module_to_file[module] = filepath
# 也添加顶层模块名
top_module: str = module.split('.')[0]
if top_module not in module_to_file:
module_to_file[top_module] = filepath
# 构建依赖图
graph: dict[str, set[str]] = {f: set() for f in py_files}
for filepath in py_files:
deps: set[str] = get_file_dependencies(filepath, src_root)
for dep in deps:
if dep in module_to_file:
dep_file: str = module_to_file[dep]
if dep_file != filepath: # 避免自依赖
graph[filepath].add(dep_file)
# 拓扑排序 (Kahn's algorithm)
in_degree: dict[str, int] = {f: 0 for f in py_files}
for f, deps in graph.items():
for dep in deps:
if dep in in_degree:
in_degree[f] += 1
queue: list[str] = [f for f, degree in in_degree.items() if degree == 0]
sorted_files: list[str] = []
while queue:
# 按字母顺序处理同级别的文件,确保确定性
queue.sort()
current: str = queue.pop(0)
sorted_files.append(current)
for f, deps in graph.items():
if current in deps:
in_degree[f] -= 1
if in_degree[f] == 0:
queue.append(f)
# 如果有环,添加剩余文件
if len(sorted_files) < len(py_files):
remaining: list[str] = [f for f in py_files if f not in sorted_files]
sorted_files.extend(remaining)
return sorted_files

View File

@@ -0,0 +1,31 @@
"""Projectrans 包 - 工程级 TransPyC 翻译器"""
import lib._bootstrap # noqa: F401 设置 sys.path项目根 + lib/includes
from lib.core.VLogger import Logger, LogLevel, set_logger
# 创建全局日志实例
logger = Logger(
name="TransPyC",
log_file="build.log",
level=LogLevel.INFO,
use_colors=True
)
set_logger(logger)
from lib.Projectrans.Utils import (
compute_sha1, _check_annotation_for_export, sha1_file,
get_file_dependencies, find_reachable_files_from_entries,
topological_sort_files
)
from lib.Projectrans.Phase1Generator import Phase1Generator
from lib.Projectrans.DeclarationGenerator import DeclarationGenerator
from lib.Projectrans.Phase2Translator import Phase2Translator, _parallel_translate_worker
from lib.Projectrans.Config import save_sha1_map, Load_sha1_map, Load_project_config, resolve_paths, main
__all__ = [
'compute_sha1', '_check_annotation_for_export', 'sha1_file',
'get_file_dependencies', 'find_reachable_files_from_entries',
'topological_sort_files', 'Phase1Generator', 'DeclarationGenerator',
'Phase2Translator', '_parallel_translate_worker',
'save_sha1_map', 'Load_sha1_map', 'Load_project_config', 'resolve_paths', 'main'
]