from __future__ import annotations import sys import os import re import shutil import subprocess import ast import traceback import json import pickle import time from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed, wait, FIRST_COMPLETED from lib.core.VLogger import get_logger as _vlog from lib.constants.config import mode as _config_mode from lib.core.Handles.HandlesBase import CTypeInfo from lib.includes import t from lib.core.SymbolUtils import AnnotationContainsName from lib.Projectrans.Utils import compute_sha1, _check_annotation_for_export, find_reachable_files_from_entries, topological_sort_files, parse_python_file from lib.Projectrans.Phase1Generator import _TryGlobalCache, _SaveToGlobalCache, _GLOBAL_CACHE_DIR import TransPyC from StubGen import PythonToStubConverter def _ExtractCDefineConstantsFromPyi(pyi_path: str, all_dc: dict[str, object]) -> None: """从 .pyi 文件中提取 CDefine 常量到 all_dc。文件不存在时静默跳过。""" if not os.path.exists(pyi_path): return try: # 复用 parse_python_file 的 AST 缓存（.pyi 文件被多处解析） _, tree = parse_python_file(pyi_path) if tree is None: return for node in ast.iter_child_nodes(tree): if isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name): if AnnotationContainsName(node.annotation, 'CDefine') and node.value: val = None if isinstance(node.value, ast.Constant): val = node.value.value elif isinstance(node.value, ast.Call) and isinstance(node.value.func, ast.Name) and node.value.func.id == 'float' and node.value.args and isinstance(node.value.args[0], ast.Constant): val = node.value.args[0].value if val is not None: all_dc[f"{node.target.id}"] = val except Exception as e: _vlog().warning(f"Phase2Translator: 忽略异常 {e}", exc_info=e) def _parallel_translate_worker(src_path: str, out_path: str, src_root: str, temp_dir: str, output_dir: str, include_dirs: list[str], triple: str | None, datalayout: str | None, sha1_map: dict[str, str], sig_files: dict[str, str], stub_files: dict[str, str], include_py_map: dict[str, str], slice_level: int, shared_sym_pickle_path: str, struct_sha1_map: dict[str, str], prebuilt_ModuleSha1Map: dict[str, str] | None = None) -> tuple[str, str, str]: try: trans = Phase2Translator(src_root, temp_dir, output_dir, compile_cmd='llc', include_dirs=include_dirs, target_triple=triple, target_datalayout=datalayout) trans.sha1_map = sha1_map trans.sig_files = sig_files trans.stub_files = stub_files trans.include_py_map = include_py_map trans.slice_level = slice_level trans.struct_sha1_map = struct_sha1_map if shared_sym_pickle_path and os.path.exists(shared_sym_pickle_path): # 使用 pickle.load(f) 流式加载，避免 pickle.loads(f.read()) 一次性 # 将整个 pickle 文件读入内存（磁盘副本 + 反序列化对象同时占用内存）导致 MemoryError。 # 同时提高递归限制，防止深层嵌套对象反序列化失败。 sys.setrecursionlimit(max(sys.getrecursionlimit(), 100000)) with open(shared_sym_pickle_path, 'rb') as f: shared_data = pickle.load(f) trans._shared_symbol_table = shared_data['symbol_table'] trans._shared_source_module_sig_files = shared_data['source_module_sig_files'] trans._shared_all_dc = shared_data['all_dc'] trans._shared_export_extern_funcs = shared_data['export_extern_funcs'] trans.inline_func_symbols = shared_data['inline_func_symbols'] trans.function_default_args = shared_data['function_default_args'] trans.function_param_names = shared_data.get('function_param_names', {}) trans._shared_generic_class_templates = shared_data.get('generic_class_templates', {}) trans._shared_doc_meta = shared_data.get('shared_doc_meta', {}) trans._shared_class_vtable = shared_data.get('class_vtable', set()) trans._shared_cross_module_vtable = shared_data.get('cross_module_vtable', set()) trans._shared_class_methods = shared_data.get('class_methods', {}) trans._shared_cross_module_novtable = shared_data.get('cross_module_novtable', set()) else: trans._precollect_inline_symbols() trans._build_shared_symbol_data() trans._translate_file(src_path, out_path, prebuilt_ModuleSha1Map=prebuilt_ModuleSha1Map) return ('ok', src_path, '') except Exception as e: return ('error', src_path, f'{e}\n{traceback.format_exc()}') def _count_braces_outside_strings(text: str) -> int: """统计花括号差值，跳过 c"..." 字符串字面量内部的括号""" count = 0 i = 0 while i < len(text): if text[i:i+2] == 'c"' and (i == 0 or text[i-1] != '\\'): # 跳过 c"..." 字面量 i += 2 while i < len(text): if text[i] == '"' and (i == 0 or text[i-1] != '\\'): i += 1 break i += 1 continue if text[i] == '{': count += 1 elif text[i] == '}': count -= 1 i += 1 return count class Phase2Translator: """阶段二：使用声明接口翻译源文件""" INCLUDE_LIB_EXTENSIONS: tuple[str, ...] = ('.dll', '.ll', '.so', '.o', '.a', '.lib') INCLUDE_SRC_EXTENSIONS: tuple[str, ...] = ('.c', '.cpp', '.cc', '.cxx', '.m', '.mm') INCLUDE_DIRS: list[str] = [ os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), 'includes'), ] def __init__(self, src_root: str, temp_dir: str, output_dir: str, compile_cmd: str = 'llc', compile_flags: list[str] | None = None, linker_cmd: str | None = None, linker_flags: list[str] | None = None, linker_output: str | None = None, include_dirs: list[str] | None = None, entry_files: list[str] | None = None, target_triple: str | None = None, target_datalayout: str | None = None, force_recompile_includes: bool = False) -> None: self.src_root = os.path.abspath(src_root) self.temp_dir = os.path.abspath(temp_dir) self.output_dir = os.path.abspath(output_dir) self.compile_cmd = compile_cmd self.compile_flags = compile_flags or ['-filetype=obj'] self.triple = target_triple self.datalayout = target_datalayout if not self.triple: for i, flag in enumerate(self.compile_flags): if flag.startswith('-mtriple='): self.triple = flag[len('-mtriple='):] elif flag == '-mtriple' and i + 1 < len(self.compile_flags): self.triple = self.compile_flags[i + 1] self.linker_cmd = linker_cmd self.linker_flags = linker_flags or [] self.linker_output = linker_output self.slice_level = 3 self.sha1_map: dict[str, str] = {} self.sig_files: dict[str, str] = {} self.stub_files: dict[str, str] = {} self.include_py_map: dict[str, str] = {} self.used_includes: set[str] = set() self._last_ErrorStack: list | None = None self.function_default_args: dict[str, list] = {} self.function_param_names: dict[str, list] = {} self.inline_func_symbols: dict[str, CTypeInfo] = {} self.extra_link_files: list[str] = [] self.extra_compile_files: list[str] = [] self.extra_py_files: list[str] = [] self._include_sha1s: set[str] = set() # 持有 ProcessPoolExecutor 引用，避免局部变量 GC 时 finalizer 阻塞 ~99s # （join_executor_internals 在 GC 时同步等待 worker 退出队列） self._executor_pool: ProcessPoolExecutor | None = None self.entry_files: list[str] | None = entry_files self.force_recompile_includes: bool = force_recompile_includes self._shared_symbol_table: object | None = None self._shared_source_module_sig_files: dict[str, str] = {} self._shared_all_dc: dict[str, dict] = {} self._shared_export_extern_funcs: dict[str, str] = {} self.include_dirs: list[str] = [] self.struct_sha1_map: dict[str, str] = {} self._shared_generic_class_templates: dict[str, dict] = {} self._shared_doc_meta: dict[str, dict[str, str]] = {} # 共享 vtable 状态跨模块（修复 pool.alloc(48) 虚表分派失败导致段错误） self._shared_class_vtable: set[str] = set() self._shared_cross_module_vtable: set[str] = set() self._shared_class_methods: dict[str, list[str]] = {} self._shared_cross_module_novtable: set[str] = set() default_dirs = [d for d in self.INCLUDE_DIRS if os.path.isdir(d)] if include_dirs: seen = set() self.include_dirs = [] for d in list(include_dirs) + default_dirs: d = os.path.abspath(d) if d not in seen and os.path.isdir(d): self.include_dirs.append(d) seen.add(d) else: self.include_dirs = default_dirs self._LoadSha1Map() def _LoadSha1Map(self) -> None: """从 temp_dir 加载 SHA1 映射、签名文件和 stub 声明文件列表。 temp_dir 中缺失的文件会从全局缓存补充。 """ if not os.path.exists(self.temp_dir): _vlog().error(f"声明目录不存在: {self.temp_dir}") return for fname in os.listdir(self.temp_dir): fpath = os.path.join(self.temp_dir, fname) if fname.startswith('_'): continue if fname.endswith('.pyi'): sha1 = fname[:-4] self.sig_files[sha1] = fpath elif fname.endswith('.stub.ll'): sha1 = fname[:-8] self.stub_files[sha1] = fpath # 从全局缓存补充 temp_dir 中缺失的 .pyi / .stub.ll if os.path.isdir(_GLOBAL_CACHE_DIR): for fname in os.listdir(_GLOBAL_CACHE_DIR): if fname.startswith('_'): continue if fname.endswith('.pyi'): sha1 = fname[:-4] if sha1 not in self.sig_files: self.sig_files[sha1] = os.path.join(_GLOBAL_CACHE_DIR, fname) elif fname.endswith('.stub.ll'): sha1 = fname[:-8] if sha1 not in self.stub_files: self.stub_files[sha1] = os.path.join(_GLOBAL_CACHE_DIR, fname) sha1_file_path = os.path.join(self.temp_dir, '_sha1_map.txt') if os.path.exists(sha1_file_path): with open(sha1_file_path, 'r', encoding='utf-8') as f: for line in f: line = line.strip() if ':' in line: sha1, rel = line.split(':', 1) self.sha1_map[sha1] = rel if rel.startswith('includes/'): module_name = os.path.splitext(os.path.basename(rel))[0] self.include_py_map[module_name] = sha1 else: for sha1, stub_path in self.stub_files.items(): self.sha1_map[sha1] = sha1 def _build_struct_sha1_map(self) -> dict[str, str]: struct_sha1_map = {} valid_sha1_keys = set(self.sha1_map.keys()) # 收集 temp_dir 和全局缓存中的 .pyi 文件 pyi_dirs: list[str] = [self.temp_dir] if os.path.isdir(_GLOBAL_CACHE_DIR): pyi_dirs.append(_GLOBAL_CACHE_DIR) for scan_dir in pyi_dirs: if not os.path.isdir(scan_dir): continue for fname in os.listdir(scan_dir): if not fname.endswith('.pyi'): continue sha1_key = fname.replace('.pyi', '') if sha1_key not in valid_sha1_keys: continue if sha1_key in struct_sha1_map: continue pyi_path = os.path.join(scan_dir, fname) # 复用 parse_python_file 的 AST 缓存（.pyi 文件可能被多处解析） _, tree = parse_python_file(pyi_path) if tree is None: continue for node in ast.walk(tree): if isinstance(node, ast.ClassDef): struct_sha1_map[node.name] = sha1_key return struct_sha1_map def run(self) -> None: """扫描源目录，翻译并生成 .ll 含代码文件（导入遍历）""" import sys as _sys if self.entry_files: py_files = list(self.entry_files) else: main_py = os.path.join(self.src_root, 'main.py') if os.path.exists(main_py): py_files = [main_py] else: py_files = [] for root, dirs, files in os.walk(self.src_root): dirs[:] = [d for d in dirs if d not in ('__pycache__',)] for file in files: if file.endswith('.py'): py_files.append(os.path.join(root, file)) if not py_files: _vlog().info("未找到入口文件或源文件") return reachable = find_reachable_files_from_entries(self.src_root, py_files) py_files = list(reachable) py_files = topological_sort_files(py_files, self.src_root) _vlog().info(f"找到 {len(py_files)} 个可达源文件（已按依赖排序）") _vlog().info("处理顺序:") for i, f in enumerate(py_files[:10], 1): rel = os.path.relpath(f, self.src_root) _vlog().info(f" {i}. {rel}") os.makedirs(self.output_dir, exist_ok=True) valid_sha1s = set(self.sha1_map.keys()) old_stub_count = len(self.stub_files) old_sig_count = len(self.sig_files) self.stub_files = {k: v for k, v in self.stub_files.items() if k in valid_sha1s} self.sig_files = {k: v for k, v in self.sig_files.items() if k in valid_sha1s} if old_stub_count != len(self.stub_files) or old_sig_count != len(self.sig_files): _vlog().info(f"过滤旧文件: stub {old_stub_count}->{len(self.stub_files)}, sig {old_sig_count}->{len(self.sig_files)}") active_sha1s = set() self._precollect_inline_symbols() self._build_shared_symbol_data() self._precollect_vtable_state() self.struct_sha1_map = self._build_struct_sha1_map() need_translate = [] for i, src_path in enumerate(py_files, 1): rel = os.path.relpath(src_path, self.src_root) # 复用 parse_python_file 的 AST 缓存（get_file_dependencies 已 parse 过） content, tree = parse_python_file(src_path) sha1 = compute_sha1(content) active_sha1s.add(sha1) out_path = os.path.join(self.output_dir, f"{sha1}.ll") current_ModuleSha1Map = self._build_current_ModuleSha1Map(sha1) if tree is not None: try: for node in ast.walk(tree): if isinstance(node, ast.Import): for alias in node.names: self.used_includes.add(alias.name.split('.')[0]) elif isinstance(node, ast.ImportFrom): if node.module: self.used_includes.add(node.module.split('.')[0]) # 检测 list/dict 容器使用，自动添加 _list/_dict/json 到 used_includes # 以便 _scan_include_libraries 能发现并编译这些库文件 if isinstance(node, ast.Subscript) and isinstance(node.value, ast.Name): if node.value.id == 'dict': self.used_includes.add('_dict') self.used_includes.add('json') elif node.value.id == 'list': self.used_includes.add('_list') if isinstance(node, ast.Call) and isinstance(node.func, ast.Name): if node.func.id == 'dict': self.used_includes.add('_dict') self.used_includes.add('json') elif node.func.id == 'list': self.used_includes.add('_list') # 检测类型注解 d: dict / l: list (非泛型形式) if isinstance(node, ast.AnnAssign) and isinstance(node.annotation, ast.Name): if node.annotation.id == 'dict': self.used_includes.add('_dict') self.used_includes.add('json') elif node.annotation.id == 'list': self.used_includes.add('_list') except Exception as _e: if _config_mode == "strict": raise _vlog().warning(f"收集源文件导入的 include 模块失败: {_e}", "Exception") if os.path.isfile(out_path): deps_changed = self._check_deps_changed(sha1, current_ModuleSha1Map) if not deps_changed: _vlog().info(f"[{i}/{len(py_files)}] 跳过(缓存): {rel} -> {sha1}.ll") continue else: if self._recombine_ll(sha1, current_ModuleSha1Map): _vlog().info(f"[{i}/{len(py_files)}] 重组(依赖变化): {rel} -> {sha1}.ll") continue _vlog().info(f"[{i}/{len(py_files)}] 重译(依赖变化): {rel} -> {sha1}.ll") else: stub_cache = os.path.join(self.output_dir, f"{sha1}.stub.ll") text_cache = os.path.join(self.output_dir, f"{sha1}.text.ll") if os.path.isfile(stub_cache) and os.path.isfile(text_cache): if self._recombine_ll(sha1, current_ModuleSha1Map): _vlog().info(f"[{i}/{len(py_files)}] 重组(缓存恢复): {rel} -> {sha1}.ll") continue need_translate.append((i, src_path, out_path, sha1, current_ModuleSha1Map)) if need_translate: # 提前从 include 文件源码中提取函数参数名，供主项目编译时 keyword 参数匹配使用 self._extract_include_param_names_from_dirs() # 提前从 App 源文件中提取函数默认参数，供跨模块调用参数检查使用 # （避免并行编译时，被调用函数的文件尚未编译完，导致默认参数信息缺失） self._extract_app_default_args(need_translate) _vlog().info(f"\n需要翻译 {len(need_translate)} 个文件") t_start = time.time() n_workers = min(8, len(need_translate)) if n_workers > 1 and len(need_translate) > 1: _vlog().info(f" 并行翻译 (workers={n_workers})") shared_pickle_path = os.path.join(self.temp_dir, '_shared_sym.json') try: shared_data = { 'symbol_table': self._shared_symbol_table, 'source_module_sig_files': self._shared_source_module_sig_files, 'all_dc': self._shared_all_dc, 'export_extern_funcs': self._shared_export_extern_funcs, 'inline_func_symbols': self.inline_func_symbols, 'function_default_args': self.function_default_args, 'function_param_names': self.function_param_names, 'generic_class_templates': self._shared_generic_class_templates, 'shared_doc_meta': self._shared_doc_meta, 'class_vtable': self._shared_class_vtable, 'cross_module_vtable': self._shared_cross_module_vtable, 'class_methods': self._shared_class_methods, 'cross_module_novtable': self._shared_cross_module_novtable, } with open(shared_pickle_path, 'wb') as f: f.write(pickle.dumps(shared_data)) _vlog().info(f" 共享符号表已序列化 ({os.path.getsize(shared_pickle_path)//1024}KB)") except Exception as e: _vlog().warning(f"符号表序列化失败({e})，worker将自行构建") shared_pickle_path = '' errors = [] # 复用 self._executor_pool：避免局部变量 GC 时 finalizer 阻塞 ~99s # （join_executor_internals 在 GC 时同步等待 worker 退出队列） if self._executor_pool is None: self._executor_pool = ProcessPoolExecutor(max_workers=n_workers) executor = self._executor_pool try: futures = {} for i, src_path, out_path, sha1, msm in need_translate: rel = os.path.relpath(src_path, self.src_root) future = executor.submit( _parallel_translate_worker, src_path, out_path, self.src_root, self.temp_dir, self.output_dir, self.include_dirs, self.triple, self.datalayout, self.sha1_map, self.sig_files, self.stub_files, self.include_py_map, self.slice_level, shared_pickle_path, self.struct_sha1_map, prebuilt_ModuleSha1Map=msm ) futures[future] = (i, rel) done_count = 0 for future in as_completed(futures): i, rel = futures[future] done_count += 1 try: status, _, err = future.result() if status == 'ok': _vlog().info(f" [{done_count}/{len(need_translate)}] 完成: {rel}") else: _vlog().error(f" {rel}: {err}") errors.append((rel, err)) except Exception as e: _vlog().error(f" {rel}: {e}") errors.append((rel, str(e))) except Exception: # 异常时立即清理，正常流程不 shutdown（留给 includes 阶段复用） if self._executor_pool is not None: self._executor_pool.shutdown(wait=False, cancel_futures=True) self._executor_pool = None raise if errors: _vlog().error(f"\n{len(errors)} 个文件翻译失败，继续编译 includes") else: for idx, (i, src_path, out_path, sha1, msm) in enumerate(need_translate): rel = os.path.relpath(src_path, self.src_root) t0 = time.time() try: self._translate_file(src_path, out_path, prebuilt_ModuleSha1Map=msm) t1 = time.time() _vlog().info(f" [{idx+1}/{len(need_translate)}] {rel} ({t1-t0:.1f}s)") except Exception as e: _vlog().error(f" {rel}: {e}") traceback.print_exc() t_end = time.time() _vlog().info(f" 翻译总耗时: {t_end-t_start:.1f}s") _vlog().info(f"\n[阶段二完成] .ll 文件生成到: {self.output_dir}") self._scan_include_libraries() self._compile_include_py_files(active_sha1s) self._compile_ll_files(active_sha1s) self._compile_include_sources() if self.linker_cmd: self._link_obj_files(active_sha1s) # 清理 ProcessPoolExecutor：所有翻译已完成，shutdown(wait=False) 让 worker # 后台退出。避免 GC finalizer 在程序末尾阻塞 ~99s（join_executor_internals） if self._executor_pool is not None: self._executor_pool.shutdown(wait=False, cancel_futures=True) self._executor_pool = None def _build_current_ModuleSha1Map(self, self_sha1: str | None) -> dict[str, str]: """构建当前模块的依赖 SHA1 映射（与 _translate_file 中逻辑一致）""" ModuleSha1Map = {} for sha1_key, sig_path in self.sig_files.items(): if sha1_key == self_sha1: continue rel_path = self.sha1_map.get(sha1_key, sha1_key) if rel_path.startswith('includes/'): inc_mod_name = os.path.splitext(rel_path)[0].replace(os.sep, '.').replace('/', '.') inc_mod_name = inc_mod_name[len('includes/'):] inc_short_name = inc_mod_name.split('.')[-1] if '.' in inc_mod_name else inc_mod_name actual_sha1 = sha1_key for inc_dir in self.include_dirs: if os.path.isdir(inc_dir): py_path = os.path.join(inc_dir, rel_path[len('includes/'):].replace(os.sep, '/')) py_path = os.path.splitext(py_path)[0] + '.py' if os.path.isfile(py_path): with open(py_path, 'r', encoding='utf-8') as f: py_sha1 = compute_sha1(f.read()) actual_sha1 = py_sha1 break ModuleSha1Map[inc_mod_name] = actual_sha1 ModuleSha1Map[inc_short_name] = actual_sha1 # Also add parent package name (e.g., 'hashlib' for 'hashlib.__init__') if inc_short_name == '__init__' and '.' in inc_mod_name: parent_pkg = inc_mod_name.rsplit('.', 1)[0] ModuleSha1Map[parent_pkg] = actual_sha1 continue module_name = os.path.splitext(rel_path)[0].replace(os.sep, '.').replace('/', '.') ModuleSha1Map[module_name] = sha1_key short_name = module_name.split('.')[-1] if '.' in module_name else module_name ModuleSha1Map[short_name] = sha1_key # 包入口文件：注册父包名（如 'os.__init__' -> 'os'），使 import os 能解析到包级符号 if short_name == '__init__' and '.' in module_name: parent_pkg = module_name.rsplit('.', 1)[0] ModuleSha1Map[parent_pkg] = sha1_key return ModuleSha1Map @staticmethod def _extract_llvm_type_name(line: str) -> str | None: """从 LLVM IR 类型定义行提取类型名（字符串操作，不使用正则） 支持 list[str]、dict[str, int] 等含特殊字符的类型名。 例如: %"18e6990aa81cc5b6.list[str]" = type { ... } → "18e6990aa81cc5b6.list[str]" %struct.foo = type { ... } → "struct.foo" """ stripped = line.strip() if not stripped.startswith('%'): return None # 处理带引号的情况: %"name" = type ... if stripped.startswith('%"'): end_quote = stripped.find('"', 2) if end_quote > 0: return stripped[2:end_quote] return None # 处理不带引号的情况: %name = type ... eq_pos = stripped.find('=') if eq_pos > 1: return stripped[1:eq_pos].strip() return None @staticmethod def _find_all_type_refs(text: str) -> set[str]: """查找文本中所有 %"..."/%name 类型引用（字符串操作，不使用正则） 支持 list[str]、dict[str, int] 等含特殊字符的类型名。 """ refs: set[str] = set() i: int = 0 n: int = len(text) while i < n: if text[i] == '%': if i + 1 < n and text[i + 1] == '"': # 带引号: %"name" end: int = text.find('"', i + 2) if end > 0: name: str = text[i + 2:end] if name: refs.add(name) i = end + 1 continue else: # 不带引号: %name (仅匹配字母、数字、下划线、点) j: int = i + 1 while j < n and (text[j].isalnum() or text[j] in '._'): j += 1 if j > i + 1: refs.add(text[i + 1:j]) i = j continue i += 1 return refs @staticmethod def _split_ll(ll_content: str) -> tuple[str, str]: """将 .ll 内容拆分为 stub（声明）和 text（代码）两部分 stub: target, 注释, %type = type, declare, @xxx = external global text: define ... { ... }, @xxx = global/constant (带初始化器) """ stub_lines = [] text_lines = [] in_define = False brace_depth = 0 module_sha1 = None defined_names = set() for line in ll_content.splitlines(True): stripped = line.strip() if stripped.startswith('define '): dm = re.match(r'define\s+[^@]*@"?([a-f0-9]+)\.', stripped) if dm and module_sha1 is None: module_sha1 = dm.group(1) dm2 = re.match(r'define\s+[^@]*@"?([^"\s(]+)"?', stripped) if dm2: defined_names.add(dm2.group(1)) in_global_def = False global_var_name = None global_type_parts = [] global_brace_depth = 0 for line in ll_content.splitlines(True): stripped = line.strip() if in_global_def: text_lines.append(line) global_type_parts.append(stripped) global_brace_depth += _count_braces_outside_strings(stripped) if global_brace_depth <= 0: full_type = ' '.join(global_type_parts).strip() depth = 0 type_end = len(full_type) for ci, cc in enumerate(full_type): if cc in ('{', '['): depth += 1 elif cc in ('}', ']'): depth -= 1 elif depth == 0 and cc == ' ': type_end = ci break var_type = full_type[:type_end].strip() if var_type: stub_lines.append(f'{global_var_name} = external global {var_type}\n') in_global_def = False global_var_name = None global_type_parts = [] global_brace_depth = 0 continue if in_define: text_lines.append(line) brace_depth += _count_braces_outside_strings(stripped) if brace_depth <= 0: in_define = False continue if stripped.startswith('define '): text_lines.append(line) in_define = True brace_depth = _count_braces_outside_strings(stripped) decl_line = re.sub(r'^define\s+', 'declare ', stripped) decl_line = re.sub(r'\b(linkonce_odr|weak_odr|linkonce|weak|common|appending|internal|private)\s+', '', decl_line) decl_line = re.sub(r'\balwaysinline\s+', '', decl_line) paren_pos = decl_line.rfind(')') if paren_pos >= 0: decl_line = decl_line[:paren_pos + 1] stub_lines.append(decl_line + '\n') continue if stripped == '{' and text_lines and not in_define: prev_stripped = text_lines[-1].strip() if text_lines else '' if prev_stripped.startswith('define ') or prev_stripped.endswith('noredzone') or prev_stripped.endswith(')'): text_lines.append(line) brace_depth = 1 in_define = True if prev_stripped.startswith('define '): decl_line = re.sub(r'^define\s+', 'declare ', prev_stripped) decl_line = re.sub(r'\b(linkonce_odr|weak_odr|linkonce|weak|common|appending|internal|private)\s+', '', decl_line) decl_line = re.sub(r'\balwaysinline\s+', '', decl_line) paren_pos = decl_line.rfind(')') if paren_pos >= 0: decl_line = decl_line[:paren_pos + 1] stub_lines.append(decl_line + '\n') continue if stripped.startswith('@') and ' global ' in stripped and 'external global' not in stripped: text_lines.append(line) if ' internal ' not in stripped and ' private ' not in stripped: name_match = re.match(r'(@\S+)', stripped) if name_match: var_name = name_match.group(1) global_pos = stripped.find(' global ') after_global = stripped[global_pos + 8:] depth = 0 type_end = len(after_global) for ci, cc in enumerate(after_global): if cc in ('{', '['): depth += 1 elif cc in ('}', ']'): depth -= 1 elif depth == 0 and cc == ' ': type_end = ci break var_type = after_global[:type_end].strip() brace_depth_gv = 0 for cc in var_type: if cc == '{': brace_depth_gv += 1 elif cc == '}': brace_depth_gv -= 1 if brace_depth_gv > 0: in_global_def = True global_var_name = var_name global_type_parts = [var_type] global_brace_depth = brace_depth_gv elif var_type: stub_lines.append(f'{var_name} = external global {var_type}\n') continue if stripped.startswith('@') and ' constant ' in stripped: text_lines.append(line) continue if stripped.startswith('declare'): dm = re.match(r'declare\s+[^@]*@"?([^"\s(]+)"?', stripped) if dm: fname = dm.group(1) if fname in defined_names: stub_lines.append(line) continue dot_pos = fname.find('.') if dot_pos > 0: fname_sha1 = fname[:dot_pos] if module_sha1 and fname_sha1 != module_sha1 and len(fname_sha1) >= 16: continue if re.match(r'^[a-f0-9]{16}\.', fname): continue stub_lines.append(line) continue stub_lines.append(line) return ''.join(stub_lines), ''.join(text_lines) def _save_deps(self, sha1: str, ModuleSha1Map: dict[str, str]) -> None: """保存依赖指纹到 .deps.json""" deps_path = os.path.join(self.output_dir, f"{sha1}.deps.json") deps = {} for mod_name, dep_sha1 in ModuleSha1Map.items(): deps[mod_name] = dep_sha1 with open(deps_path, 'w', encoding='utf-8') as f: json.dump(deps, f) def _LoadDeps(self, sha1: str) -> dict[str, str] | None: """加载依赖指纹""" deps_path = os.path.join(self.output_dir, f"{sha1}.deps.json") if os.path.isfile(deps_path): try: with open(deps_path, 'r', encoding='utf-8') as f: return json.load(f) except Exception as _e: if _config_mode == "strict": raise _vlog().warning(f"加载依赖指纹 JSON 失败: {_e}", "Exception") return None def _check_deps_changed(self, sha1: str, current_ModuleSha1Map: dict[str, str]) -> bool: """检查依赖指纹是否变化""" saved_deps = self._LoadDeps(sha1) if saved_deps is None: return True for mod_name, dep_sha1 in current_ModuleSha1Map.items(): if saved_deps.get(mod_name) != dep_sha1: return True for mod_name, dep_sha1 in saved_deps.items(): if current_ModuleSha1Map.get(mod_name) != dep_sha1: return True return False def _recombine_ll(self, sha1: str, current_ModuleSha1Map: dict[str, str]) -> bool: """依赖变化时重新组合 .ll：更新 .stub.ll 中的 SHA1 前缀，拼接 .stub.ll + .text.ll""" saved_deps = self._LoadDeps(sha1) if saved_deps is None: return False stub_path = os.path.join(self.output_dir, f"{sha1}.stub.ll") text_path = os.path.join(self.output_dir, f"{sha1}.text.ll") ll_path = os.path.join(self.output_dir, f"{sha1}.ll") if not os.path.isfile(stub_path) or not os.path.isfile(text_path): return False with open(stub_path, 'r', encoding='utf-8') as f: stub_content = f.read() with open(text_path, 'r', encoding='utf-8') as f: text_content = f.read() sha1_replacements = {} for mod_name, old_sha1 in saved_deps.items(): new_sha1 = current_ModuleSha1Map.get(mod_name) if new_sha1 and old_sha1 != new_sha1: sha1_replacements[old_sha1] = new_sha1 if sha1_replacements: for old_sha1, new_sha1 in sha1_replacements.items(): stub_content = stub_content.replace(old_sha1, new_sha1) text_content = text_content.replace(old_sha1, new_sha1) combined = stub_content if not combined.endswith('\n'): combined += '\n' combined += text_content with open(ll_path, 'w', encoding='utf-8', newline='\n') as f: f.write(combined) with open(stub_path, 'w', encoding='utf-8', newline='\n') as f: f.write(stub_content) with open(text_path, 'w', encoding='utf-8', newline='\n') as f: f.write(text_content) self._save_deps(sha1, current_ModuleSha1Map) obj_path = os.path.join(self.output_dir, f"{sha1}.obj") if os.path.isfile(obj_path): os.remove(obj_path) return True def _inject_auto_imports(self, code: str, src_path: str) -> str: """检测源码中对 list/dict 容器的使用，自动注入 import _list / import _dict。 当使用 dict 时，同时注入 import json (用于 dict 的 JSON 处理功能)。 检测模式: - list[T] / dict[V] 作为类型注解 (ast.Subscript, value=Name(id='list'/'dict')) - list[T](...) / dict[V](...) 作为构造函数调用 (ast.Call, func=Subscript) - list(...) / dict(...) 作为构造函数调用 (ast.Call, func=Name) """ try: tree = ast.parse(code) except SyntaxError: return code # 检查是否已经导入了 _list / _dict / json has_list_import: bool = False has_dict_import: bool = False has_json_import: bool = False for node in ast.iter_child_nodes(tree): if isinstance(node, ast.Import): for alias in node.names: if alias.name == '_list': has_list_import = True elif alias.name == '_dict': has_dict_import = True elif alias.name == 'json': has_json_import = True elif isinstance(node, ast.ImportFrom): if node.module == '_list': has_list_import = True elif node.module == '_dict': has_dict_import = True elif node.module == 'json': has_json_import = True if has_list_import and has_dict_import and has_json_import: break # 检查是否使用了 list / dict 容器 uses_list: bool = False uses_dict: bool = False for node in ast.walk(tree): if isinstance(node, ast.Subscript): if isinstance(node.value, ast.Name): if node.value.id == 'list': uses_list = True elif node.value.id == 'dict': uses_dict = True if isinstance(node, ast.Call) and isinstance(node.func, ast.Name): if node.func.id == 'list': uses_list = True elif node.func.id == 'dict': uses_dict = True # 检测类型注解 d: dict / l: list (非泛型形式) if isinstance(node, ast.AnnAssign) and isinstance(node.annotation, ast.Name): if node.annotation.id == 'dict': uses_dict = True elif node.annotation.id == 'list': uses_list = True # 注入 import 到源码开头 prefix: str = '' if uses_list and not has_list_import: prefix += 'import _list\n' if uses_dict and not has_dict_import: prefix += 'import _dict\n' if uses_dict and not has_json_import: prefix += 'import json\n' if prefix: return prefix + code return code def _translate_file(self, src_path: str, out_path: str, prebuilt_ModuleSha1Map: dict[str, str] | None = None) -> None: """翻译单个源文件，嵌入相关 .stub.ll 声明""" with open(src_path, 'r', encoding='utf-8') as f: code = f.read() # SHA1 必须在自动导入注入之前计算，以保持与 Phase1 生成的文件名/函数名前缀一致 sha1 = compute_sha1(code) # 自动导入: 检测 list 容器使用，自动注入 import _list code = self._inject_auto_imports(code, src_path) if prebuilt_ModuleSha1Map is not None: ModuleSha1Map = prebuilt_ModuleSha1Map else: ModuleSha1Map = self._build_current_ModuleSha1Map(sha1) trans = TransPyC.TransPyC(code=code, triple=self.triple, datalayout=self.datalayout) trans.translator.CurrentFile = src_path trans.SliceLevel = self.slice_level trans.translator.SliceLevel = self.slice_level trans.translator.SliceCount = 0 trans.translator.SliceInfos = [] trans.translator.LlvmGen = None trans.translator._module_sha1 = sha1 trans.translator._ModuleSha1Map = ModuleSha1Map trans.translator._struct_sha1_map = self.struct_sha1_map trans.translator._global_function_default_args = self.function_default_args trans.translator._global_function_param_names = self.function_param_names trans.translator._temp_dir = self.temp_dir trans.translator._shared_doc_meta = self._shared_doc_meta # 传递共享 vtable 状态 trans.translator._shared_class_vtable = self._shared_class_vtable trans.translator._shared_cross_module_vtable = self._shared_cross_module_vtable trans.translator._shared_class_methods = self._shared_class_methods trans.translator._shared_cross_module_novtable = self._shared_cross_module_novtable if self._shared_symbol_table is not None: trans.translator.SymbolTable = self._shared_symbol_table # 确保共享符号表的 translator 指向当前 translator，以便 import 别名解析正确 if hasattr(self._shared_symbol_table, 'translator'): self._shared_symbol_table.translator = trans.translator trans.translator._source_module_sig_files = self._shared_source_module_sig_files trans.translator._all_define_constants = self._shared_all_dc trans.translator._export_extern_funcs = self._shared_export_extern_funcs if self._shared_generic_class_templates: if not hasattr(trans.translator.ClassHandler, '_generic_class_templates'): trans.translator.ClassHandler._generic_class_templates = {} trans.translator.ClassHandler._generic_class_templates.update(self._shared_generic_class_templates) else: export_extern_funcs = {} for sha1_key, sig_path in self.sig_files.items(): if sha1_key == sha1: continue try: with open(sig_path, 'r', encoding='utf-8') as f: sig_content = f.read() sig_tree = ast.parse(sig_content) for node in ast.iter_child_nodes(sig_tree): if isinstance(node, ast.FunctionDef): if _check_annotation_for_export(node.returns): export_extern_funcs[node.name] = sha1_key except Exception as _e: if _config_mode == "strict": raise _vlog().warning(f"收集导出外部函数声明失败: {_e}", "Exception") if export_extern_funcs: trans.translator._export_extern_funcs = export_extern_funcs for includes_dir in self.include_dirs: if os.path.isdir(includes_dir): for pyi_file in os.listdir(includes_dir): if pyi_file.endswith('.pyi'): pyi_path = os.path.join(includes_dir, pyi_file) module_name = os.path.splitext(pyi_file)[0] trans.translator.SymbolTable.LoadModuleSymbols(pyi_path, module_name, lineno=0) for py_file in os.listdir(includes_dir): if py_file.endswith('.py') and (not py_file.startswith('_') or py_file == '_list.py' or py_file == '_dict.py'): module_name = os.path.splitext(py_file)[0] sha1_key = self.include_py_map.get(module_name) if sha1_key and sha1_key in self.sig_files: sig_path = self.sig_files[sha1_key] trans.translator.SymbolTable.LoadModuleSymbols(sig_path, module_name, lineno=0) # 收集需要重导出的包，等所有模块符号加载完后再处理 _pending_reexports = [] for sha1_key, sig_path in self.sig_files.items(): rel_path = self.sha1_map.get(sha1_key, sha1_key) if rel_path.startswith('includes/'): mod_name = os.path.splitext(rel_path)[0].replace(os.sep, '.').replace('/', '.') mod_name = mod_name[len('includes/'):] trans.translator.SymbolTable.LoadModuleSymbols(sig_path, mod_name, lineno=0) if mod_name.endswith('.__init__'): package_name = mod_name[:-len('.__init__')] _pending_reexports.append((sig_path, package_name)) # 所有 includes 模块符号已加载，现在处理包重导出 for sig_path, package_name in _pending_reexports: self._register_package_reexports(trans, sig_path, package_name) for sha1_key, sig_path in self.sig_files.items(): if sha1_key == sha1: continue rel_path = self.sha1_map.get(sha1_key, sha1_key) if rel_path.startswith('includes/'): continue module_name = os.path.splitext(rel_path)[0].replace(os.sep, '.').replace('/', '.') trans.translator.SymbolTable.LoadModuleSymbols(sig_path, module_name, lineno=0) trans.translator._source_module_sig_files[module_name] = sig_path short_name = module_name.split('.')[-1] if '.' in module_name else module_name if short_name != module_name: trans.translator._source_module_sig_files[short_name] = sig_path all_dc = {} for sha1_key, stub_path in self.stub_files.items(): if sha1_key == sha1: continue if os.path.exists(stub_path): try: with open(stub_path, 'r', encoding='utf-8') as f: stub_content = f.read() for line in stub_content.splitlines(): line = line.strip() if line.startswith('@') and '= global' in line and 'i32' in line and not line.startswith('@llvm'): var_name = line.split('=')[0].strip().lstrip('@') val_part = line.split('i32')[-1].strip().rstrip(']').lstrip('[').lstrip('i32') try: val = int(val_part) all_dc[var_name] = val except Exception as e: _vlog().warning(f"Phase2Translator: 忽略异常 {e}", exc_info=e) except Exception as e: _vlog().warning(f"Phase2Translator: 忽略异常 {e}", exc_info=e) for sha1_key, pyi_path in self.sig_files.items(): if sha1_key == sha1: continue _ExtractCDefineConstantsFromPyi(pyi_path, all_dc) if all_dc: trans.translator._all_define_constants = all_dc for sym_name, sym_info in self.inline_func_symbols.items(): if sym_name not in trans.translator.SymbolTable: trans.translator.SymbolTable[sym_name] = sym_info else: existing = trans.translator.SymbolTable[sym_name] if not getattr(existing, 'InlineBody', None) and getattr(sym_info, 'InlineBody', None): existing.IsInline = sym_info.IsInline existing.InlineBody = sym_info.InlineBody existing.InlineParams = sym_info.InlineParams try: result = trans.Convert( OutputFilename=out_path, SourceFilename=src_path, target='llvm' ) except Exception as e: error_stack = getattr(trans.translator, '_ErrorStack', []) if error_stack: self._last_ErrorStack = error_stack chain_lines = [] for entry in reversed(error_stack): if len(entry) >= 3: exc_msg, line_info = entry[1], entry[2] chain_lines.append(" %s\n %s" % (line_info, exc_msg)) if chain_lines: enriched = str(e) + "\n" + "\n".join(chain_lines) raise type(e)(enriched) from e node_info = "" if hasattr(trans.translator, 'LlvmGen') and trans.translator.LlvmGen: node_info = trans.translator.LlvmGen._get_node_info() if node_info: enriched = "%s\n %s" % (str(e), node_info.strip()) raise type(e)(enriched) from e raise if result and isinstance(result, str): for func_name, func_def in trans.translator.FunctionDefCache.items(): if hasattr(func_def, 'args'): mangled = trans.translator.LlvmGen._mangle_func_name(func_name) if trans.translator.LlvmGen else func_name # 提取参数名 if hasattr(func_def.args, 'args'): param_names = [arg.arg for arg in func_def.args.args] self.function_param_names[mangled] = param_names self.function_param_names[func_name] = param_names # 提取默认值 if hasattr(func_def.args, 'defaults') and func_def.args.defaults: defaults = [] for d in func_def.args.defaults: if isinstance(d, ast.Constant): defaults.append(d.value) else: defaults.append(None) self.function_default_args[mangled] = defaults self.function_default_args[func_name] = defaults for sym_name, sym_info in trans.translator.SymbolTable.items(): if isinstance(sym_info, CTypeInfo) and getattr(sym_info, 'IsInline', False) and getattr(sym_info, 'InlineBody', None): self.inline_func_symbols[sym_name] = sym_info out_dir = os.path.dirname(out_path) if out_dir: os.makedirs(out_dir, exist_ok=True) # 修复 LLVM 22+: external global 不能有初值，改为 linkonce result = re.sub(r'^(@[\w.]+)\s*=\s*external\s+global\s+(\w[\w*]*)\s+(-?\d+|null|zeroinitializer|true|false)$', r'\1 = linkonce global \2 \3', result, flags=re.MULTILINE) # 后处理：添加 llvmlite 不支持的 LLVM 属性（如 willreturn, mustprogress） pending_str_attrs = getattr(trans.translator.LlvmGen, '_pending_str_attrs', None) if pending_str_attrs: for func_name, attrs in pending_str_attrs.items(): attrs_str = ' '.join(attrs) # 匹配 define ... @func_name(...) ... { 并在 { 之前添加属性 pattern = rf'(define\s+[^@]*@{re.escape(func_name)}\s*\([^)]*\))([^{{]*)(\{{)' result, n = re.subn(pattern, rf'\1\2 {attrs_str}\3', result, count=1) if n == 0: _vlog().warning(f"后处理添加属性失败: 未找到函数 {func_name} 的定义") with open(out_path, 'w', encoding='utf-8', newline='\n') as f: f.write(result) stub_content, text_content = self._split_ll(result) base = os.path.splitext(out_path)[0] with open(base + '.stub.ll', 'w', encoding='utf-8', newline='\n') as f: f.write(stub_content) with open(base + '.text.ll', 'w', encoding='utf-8', newline='\n') as f: f.write(text_content) self._save_deps(sha1, ModuleSha1Map) _vlog().info(f" -> {os.path.basename(out_path)}") else: stub_path = self.stub_files.get(sha1) if stub_path and os.path.exists(stub_path): out_dir = os.path.dirname(out_path) if out_dir: os.makedirs(out_dir, exist_ok=True) with open(stub_path, 'r', encoding='utf-8') as f: stub_content = f.read() with open(out_path, 'w', encoding='utf-8', newline='\n') as f: f.write(stub_content) _vlog().info(f" -> {os.path.basename(out_path)} (仅声明)") def _register_package_reexports(self, trans, init_pyi_path: str, package_name: str) -> None: """解析 __init__.pyi 中的 from .xxx import yyy 或 from __xxx import yyy，在包级别注册导出符号""" try: # 复用 parse_python_file 的 AST 缓存 _, tree = parse_python_file(init_pyi_path) if tree is None: return for node in ast.walk(tree): if isinstance(node, ast.ImportFrom) and node.module: sub_module = node.module.lstrip('.') for alias in node.names: symbol_name = alias.name exported_name = alias.asname if alias.asname else symbol_name source_keys = [ f"{package_name}.{sub_module}.{symbol_name}", f"{package_name}.__{sub_module}.{symbol_name}" if not sub_module.startswith('__') else None, symbol_name, ] target_key = f"{package_name}.{exported_name}" if target_key not in trans.translator.SymbolTable: for source_key in source_keys: if source_key and source_key in trans.translator.SymbolTable: trans.translator.SymbolTable[target_key] = trans.translator.SymbolTable[source_key] break if exported_name not in trans.translator.SymbolTable: for source_key in source_keys: if source_key and source_key in trans.translator.SymbolTable: trans.translator.SymbolTable[exported_name] = trans.translator.SymbolTable[source_key] break except Exception as e: _vlog().warning(f" [警告] 解析包重导出失败 {init_pyi_path}: {e}") def _compile_ll_files(self, active_sha1s: set[str]) -> None: """将 output_dir 中属于 active_sha1s 的 .ll 文件编译为 .obj（跳过已编译过的 include 文件）""" # 第一次 walk：清理 stale 文件（.obj/.deps.json/纯 .ll，排除 .stub.ll/.text.ll） stale_files = [] for root, dirs, files in os.walk(self.output_dir): for file in files: if file.endswith('.obj'): sha1 = file[:-4] if sha1 not in active_sha1s: stale_files.append(os.path.join(root, file)) elif file.endswith('.deps.json'): sha1 = file[:-10] if sha1 not in active_sha1s: stale_files.append(os.path.join(root, file)) elif file.endswith('.ll') and not file.endswith('.stub.ll') and not file.endswith('.text.ll'): sha1 = file[:-3] if sha1 not in active_sha1s: stale_files.append(os.path.join(root, file)) if stale_files: for fpath in stale_files: try: os.remove(fpath) except Exception as e: _vlog().warning(f"Phase2Translator: 忽略异常 {e}", exc_info=e) _vlog().info(f"[清理] 删除 {len(stale_files)} 个过时文件") # 第二次 walk：收集需要编译的 .ll 文件（排除 .stub.ll/.text.ll） ll_files = [] for root, dirs, files in os.walk(self.output_dir): for file in files: if file.endswith('.ll') and not file.endswith('.stub.ll') and not file.endswith('.text.ll'): sha1 = file[:-3] if sha1 in active_sha1s: fpath = os.path.join(root, file) obj_path = os.path.join(root, file[:-3] + '.obj') if not os.path.isfile(obj_path): ll_files.append(fpath) if not ll_files: _vlog().info("[编译] 无 .ll 文件需要重新编译") else: _vlog().info(f"\n[编译] 找到 {len(ll_files)} 个 .ll 文件需要编译") # 第三次 walk：收集 all_ll_files（用于 alwaysinline 检测，排除 include sha1s） all_ll_files = [] for root, dirs, files in os.walk(self.output_dir): for file in files: if file.endswith('.ll') and not file.endswith('.stub.ll') and not file.endswith('.text.ll'): sha1 = file[:-3] if sha1 in active_sha1s and sha1 not in self._include_sha1s: all_ll_files.append(os.path.join(root, file)) has_inline = False for ll_path in all_ll_files: try: with open(ll_path, 'r', encoding='utf-8') as f: content = f.read() if 'alwaysinline' in content: has_inline = True break except Exception as e: _vlog().warning(f"Phase2Translator: 忽略异常 {e}", exc_info=e) if has_inline: llvm_link = shutil.which('llvm-link') or shutil.which('llvm-link.exe') opt_cmd = shutil.which('opt') or shutil.which('opt.exe') if llvm_link and opt_cmd: _vlog().info(" [内联] 检测到 alwaysinline 函数，执行跨模块内联优化...") for ll_path in all_ll_files: try: with open(ll_path, 'r', encoding='utf-8') as f: content = f.read() type_defs = [] type_def_indices = set() for i, line in enumerate(content.splitlines()): stripped = line.strip() if stripped.startswith('%') and '= type' in stripped: type_defs.append(line) type_def_indices.add(i) if type_def_indices: lines = content.splitlines(True) content_no_types = ''.join(line for i, line in enumerate(lines) if i not in type_def_indices) header_end = 0 for i, line in enumerate(content_no_types.splitlines()): stripped = line.strip() if stripped.startswith(';') or stripped.startswith('target ') or stripped.startswith('source_filename') or not stripped: header_end = i + 1 continue break lines2 = content_no_types.splitlines(True) fixed = ''.join(lines2[:header_end]) + '\n'.join(type_defs) + '\n' + ''.join(lines2[header_end:]) with open(ll_path, 'w', encoding='utf-8') as f: f.write(fixed) except Exception as e: _vlog().warning(f"Phase2Translator: 忽略异常 {e}", exc_info=e) merged_ll = os.path.join(self.output_dir, '_merged.ll') optimized_ll = os.path.join(self.output_dir, '_merged_opt.ll') try: link_cmd = [llvm_link] + all_ll_files + ['-o', merged_ll] result = subprocess.run(link_cmd, capture_output=True, text=True) if result.returncode != 0: _vlog().warning(f" llvm-link 失败: {result.stderr.strip()}，回退到单独编译") has_inline = False else: opt_result_cmd = [opt_cmd, '-passes=always-inline', merged_ll, '-S', '-o', optimized_ll] result = subprocess.run(opt_result_cmd, capture_output=True, text=True) if result.returncode != 0: _vlog().warning(f" opt 失败: {result.stderr.strip()}，回退到单独编译") has_inline = False else: _vlog().info(" [内联] 跨模块内联优化完成") with open(optimized_ll, 'r', encoding='utf-8') as f: opt_content = f.read() opt_content = re.sub(r'^(define|declare)\s+(private|internal|available_externally|linkonce|weak|common|appending|extern_weak|linkonce_odr|weak_odr|external)\s+', r'\1 \2 dso_local ', opt_content, flags=re.MULTILINE) opt_content = re.sub(r'^(define|declare)\s+(?!private|internal|available_externally|linkonce|weak|common|appending|extern_weak|linkonce_odr|weak_odr|external|dso_local|@)', r'\1 dso_local ', opt_content, flags=re.MULTILINE) # LLVM 22+: external 不允许在带初值的全局定义上显式写出，改为 linkonce 保留多模块合并语义 opt_content = re.sub(r'(@[\w.]+)\s*=\s*external\s+global\s+(\w[\w*]*)\s+(-?\d+|null|zeroinitializer|true|false)$', r'\1 = linkonce global \2 \3', opt_content, flags=re.MULTILINE) opt_content = re.sub(r'(@[\w.]+)\s*=\s*(private|internal|available_externally|linkonce|weak|common|appending|extern_weak|linkonce_odr|weak_odr)\s+(global|constant)', r'\1 = \2 dso_local \3', opt_content) opt_content = re.sub(r'(@[\w.]+)\s*=\s+(?!private|internal|available_externally|linkonce|weak|common|appending|extern_weak|linkonce_odr|weak_odr|dso_local)(global|constant)', r'\1 = dso_local \2', opt_content) with open(optimized_ll, 'w', encoding='utf-8') as f: f.write(opt_content) merged_obj = os.path.join(self.output_dir, '_merged.obj') try: cmd = [self.compile_cmd] + self.compile_flags + ['-o', merged_obj, optimized_ll] result = subprocess.run(cmd, capture_output=True, text=True, cwd=self.output_dir) if result.returncode == 0: _vlog().success(f" 合并模块编译完成") for ll_path in ll_files: obj_path = ll_path[:-3] + '.obj' sha1_obj = os.path.join(self.output_dir, os.path.basename(ll_path)[:-3] + '.obj') if os.path.isfile(sha1_obj): os.remove(sha1_obj) return else: _vlog().warning(f" 合并模块编译失败: {result.stderr.strip()}，回退到单独编译") has_inline = False except Exception as e: _vlog().warning(f" 合并模块编译异常: {e}，回退到单独编译") has_inline = False except FileNotFoundError: _vlog().warning(f" 找不到 llvm-link 或 opt，回退到单独编译") has_inline = False else: _vlog().warning(" 未找到 llvm-link/opt 工具，回退到单独编译（alwaysinline 可能不生效）") def _compile_one(ll_path: str) -> tuple[str, str, str]: rel = os.path.relpath(ll_path, self.output_dir) obj_path = ll_path[:-3] + '.obj' if os.path.isfile(obj_path): return (rel, 'cached', '') try: with open(ll_path, 'r', encoding='utf-8') as f: ll_content = f.read() sha1 = os.path.basename(ll_path)[:-3] deps = self._LoadDeps(sha1) stub_prefix = '' if deps: # 去重：同一个全局变量如果同时有 external 声明和实际定义， # 移除 external 声明，避免 "redefinition of global" 错误 # 注意：此去重必须在收集现有符号之前执行，否则行号偏移会导致 # lines_to_remove 误删其他行（如全局变量定义） ll_lines_for_dedup = ll_content.splitlines() external_globals: dict[str, int] = {} non_external_globals: dict[str, int] = {} for i, line in enumerate(ll_lines_for_dedup): stripped = line.strip() if stripped.startswith('@') and (' global ' in stripped or ' constant ' in stripped): gm = re.match(r'(@\S+)', stripped) if gm: gname = gm.group(1) is_external = re.match(r'@\S+\s*=\s*external\s+', stripped) is not None if is_external: external_globals[gname] = i else: non_external_globals[gname] = i lines_to_remove_global_dedup: set[int] = set() for gname in external_globals: if gname in non_external_globals: lines_to_remove_global_dedup.add(external_globals[gname]) if lines_to_remove_global_dedup: ll_content = '\n'.join( line for i, line in enumerate(ll_lines_for_dedup) if i not in lines_to_remove_global_dedup ) + '\n' existing_symbols: dict[str, str] = {} opaque_line_indices: dict[str, int] = {} existing_declares: set[str] = set() existing_declare_lines: dict[str, set[int]] = {} for i, line in enumerate(ll_content.splitlines()): stripped = line.strip() if stripped.startswith('%') and '= type' in stripped: name = Phase2Translator._extract_llvm_type_name(stripped) if name: if '= type opaque' in stripped: existing_symbols[name] = 'opaque' opaque_line_indices[name] = i else: existing_symbols[name] = 'full' elif stripped.startswith('declare'): dm = re.match(r'declare\s+[^@]*@"?([^"\s(]+)"?', stripped) if dm: fname = dm.group(1) existing_declares.add(fname) existing_declare_lines.setdefault(fname, set()).add(i) seen_stub_sha1: set[str] = set() replaced_opaque_names: set[str] = set() replaced_declare_names: set[str] = set() all_stub_lines: list[str | None] = [] stub_type_map: dict[str, tuple[bool, int]] = {} stub_declare_names: set[str] = set() stub_declare_lines: dict[str, str] = {} existing_defines: set[str] = set() existing_global_defs: set[str] = set() stub_global_defs: set[str] = set() for line in ll_content.splitlines(): stripped = line.strip() if stripped.startswith('define'): dm = re.match(r'define\s+[^@]*@"?([^"\s(]+)"?', stripped) if dm: existing_defines.add(dm.group(1)) elif stripped.startswith('@') and (' global ' in stripped or ' constant ' in stripped): gm = re.match(r'(@\S+)', stripped) if gm: existing_global_defs.add(gm.group(1)) for dep_name, dep_sha1 in deps.items(): if dep_sha1 in seen_stub_sha1: continue seen_stub_sha1.add(dep_sha1) stub_path = os.path.join(self.output_dir, f"{dep_sha1}.stub.ll") if not os.path.isfile(stub_path): stub_path = os.path.join(self.temp_dir, f"{dep_sha1}.stub.ll") if os.path.isfile(stub_path): with open(stub_path, 'r', encoding='utf-8') as sf: stub_content = sf.read() for line in stub_content.splitlines(): stripped = line.strip() if stripped.startswith(';') or stripped.startswith('target ') or stripped.startswith('source_filename') or not stripped: continue if stripped.startswith('declare'): dm = re.match(r'declare\s+[^@]*@"?([^"\s(]+)"?', stripped) if dm: fname = dm.group(1) if 'zdef_alloc' in fname: _vlog().debug(f"[DEBUG] stub declare for {fname}: existing_declares={fname in existing_declares}, stub_declare_names={fname in stub_declare_names}, has_struct={'%' in stripped}, existing={existing_declare_lines.get(fname)}") if fname in existing_defines: continue if fname in existing_declares or fname in stub_declare_names: existing_decl_line = stub_declare_lines.get(fname) if existing_decl_line is not None: has_struct = '%' in stripped existing_has_struct = '%' in existing_decl_line if existing_decl_line else False if has_struct and not existing_has_struct: for i, l in enumerate(all_stub_lines): if l is not None and l.strip().startswith('declare'): em = re.match(r'declare\s+[^@]*@"?([^"\s(]+)"?', l.strip()) if em and em.group(1) == fname: all_stub_lines[i] = line break continue elif fname in existing_declares: has_struct = '%' in stripped if has_struct: replaced_declare_names.add(fname) stub_declare_names.add(fname) stub_declare_lines[fname] = line all_stub_lines.append(line) continue continue stub_declare_names.add(fname) stub_declare_lines[fname] = line all_stub_lines.append(line) continue if stripped.startswith('%') and '= type' in stripped: name = Phase2Translator._extract_llvm_type_name(stripped) if name: is_opaque = '= type opaque' in stripped if name in existing_symbols: if existing_symbols[name] == 'opaque' and not is_opaque: replaced_opaque_names.add(name) existing_symbols[name] = 'full' idx = len(all_stub_lines) all_stub_lines.append(line) stub_type_map[name] = (is_opaque, idx) continue if name in stub_type_map: prev_opaque, prev_idx = stub_type_map[name] if prev_opaque and not is_opaque: all_stub_lines[prev_idx] = None idx = len(all_stub_lines) all_stub_lines.append(line) stub_type_map[name] = (is_opaque, idx) continue idx = len(all_stub_lines) all_stub_lines.append(line) stub_type_map[name] = (is_opaque, idx) else: all_stub_lines.append(line) elif stripped.startswith('@') and (' global ' in stripped or ' constant ' in stripped): gm = re.match(r'(@\S+)', stripped) if gm and (gm.group(1) in existing_global_defs or gm.group(1) in stub_global_defs): continue if gm: stub_global_defs.add(gm.group(1)) all_stub_lines.append(line) else: all_stub_lines.append(line) filtered = [l for l in all_stub_lines if l is not None] if filtered: stub_prefix = '\n'.join(filtered) + '\n' lines_to_remove = set() if replaced_opaque_names: for name in replaced_opaque_names: if name in opaque_line_indices: lines_to_remove.add(opaque_line_indices[name]) if replaced_declare_names: _vlog().debug(f"[DEBUG] replaced_declare_names: {replaced_declare_names}") for fname in replaced_declare_names: if fname in existing_declare_lines: lines_to_remove.update(existing_declare_lines[fname]) _vlog().debug(f"[DEBUG] removing lines {existing_declare_lines[fname]} for {fname}") else: _vlog().debug(f"[DEBUG] fname {fname} not in existing_declare_lines: {list(existing_declare_lines.keys())}") if lines_to_remove: ll_lines = ll_content.splitlines(True) ll_content = ''.join(line for i, line in enumerate(ll_lines) if i not in lines_to_remove) type_defs_in_ll = [] type_def_line_indices = set() for i, line in enumerate(ll_content.splitlines()): stripped = line.strip() if stripped.startswith('%') and '= type' in stripped: type_defs_in_ll.append(line) type_def_line_indices.add(i) if type_def_line_indices: ll_lines = ll_content.splitlines(True) ll_content = ''.join(line for i, line in enumerate(ll_lines) if i not in type_def_line_indices) if type_defs_in_ll: stub_prefix += '\n'.join(type_defs_in_ll) + '\n' if stub_prefix: header_end = 0 for i, line in enumerate(ll_content.splitlines()): stripped = line.strip() if stripped.startswith(';') or stripped.startswith('target ') or stripped.startswith('source_filename') or not stripped: header_end = i + 1 continue break ll_lines = ll_content.splitlines(True) ll_content = ''.join(ll_lines[:header_end]) + stub_prefix + ''.join(ll_lines[header_end:]) ll_content = re.sub(r'^(define|declare)\s+(private|internal|available_externally|linkonce|weak|common|appending|extern_weak|linkonce_odr|weak_odr|external)\s+', r'\1 \2 dso_local ', ll_content, flags=re.MULTILINE) ll_content = re.sub(r'^(define|declare)\s+(?!private|internal|available_externally|linkonce|weak|common|appending|extern_weak|linkonce_odr|weak_odr|external|dso_local|@)', r'\1 dso_local ', ll_content, flags=re.MULTILINE) # LLVM 22+: external 不允许在带初值的全局定义上显式写出，改为 linkonce 保留多模块合并语义 ll_content = re.sub(r'^(@[\w.]+)\s*=\s*external\s+global\s+(\w[\w*]*)\s+(-?\d+|null|zeroinitializer|true|false)$', r'\1 = linkonce global \2 \3', ll_content, flags=re.MULTILINE) ll_content = re.sub(r'^(@[\w.]+)\s*=\s*(private|internal|available_externally|linkonce|weak|common|appending|extern_weak|linkonce_odr|weak_odr)\s+(global|constant)', r'\1 = \2 dso_local \3', ll_content, flags=re.MULTILINE) ll_content = re.sub(r'^(@[\w.]+)\s*=\s+(?!private|internal|available_externally|linkonce|weak|common|appending|extern_weak|linkonce_odr|weak_odr|dso_local)(global|constant)', r'\1 = dso_local \2', ll_content, flags=re.MULTILINE) # LLVM 22+: 移除同一模块中已有 define 的 declare 语句（LLVM 22 将 declare+define 视为重复定义） defined_funcs: set[str] = set() for m in re.finditer(r'^define\s+(?:dso_local\s+)?[^@]*@\"?([^"\s\(]+)\"?\s*\(', ll_content, re.MULTILINE): defined_funcs.add(m.group(1)) if defined_funcs: lines: list[str] = ll_content.splitlines(True) kept_lines: list[str] = [] for line in lines: m = re.match(r'^declare\s+(?:dso_local\s+)?[^@]*@\"?([^"\s\(]+)\"?\s*\(', line) if m and m.group(1) in defined_funcs: continue kept_lines.append(line) ll_content = ''.join(kept_lines) # 移除重复的 declare 语句（同一函数多次 declare 但没有 define） seen_declares: set[str] = set() dedup_lines: list[str] = ll_content.splitlines(True) dedup_kept: list[str] = [] for line in dedup_lines: dm = re.match(r'^declare\s+(?:dso_local\s+)?[^@]*@\"?([^"\s\(]+)\"?\s*\(', line) if dm: fname = dm.group(1) if fname in seen_declares: continue seen_declares.add(fname) dedup_kept.append(line) ll_content = ''.join(dedup_kept) final_type_defs = [] final_type_def_indices = set() seen_type_names_main = set() for i, line in enumerate(ll_content.splitlines()): stripped = line.strip() if stripped.startswith('%') and '= type' in stripped: type_name_main = stripped.split('=')[0].strip() final_type_def_indices.add(i) if type_name_main in seen_type_names_main: continue seen_type_names_main.add(type_name_main) final_type_defs.append(line) if final_type_def_indices: final_lines = ll_content.splitlines(True) ll_content_no_types = ''.join(line for i, line in enumerate(final_lines) if i not in final_type_def_indices) header_end = 0 for i, line in enumerate(ll_content_no_types.splitlines()): stripped = line.strip() if stripped.startswith(';') or stripped.startswith('target ') or stripped.startswith('source_filename') or not stripped: header_end = i + 1 continue break final_lines2 = ll_content_no_types.splitlines(True) ll_content = ''.join(final_lines2[:header_end]) + '\n'.join(final_type_defs) + '\n' + ''.join(final_lines2[header_end:]) dso_ll_path = ll_path[:-3] + '_dso.ll' with open(dso_ll_path, 'w', encoding='utf-8') as f: f.write(ll_content) cmd = [self.compile_cmd] + self.compile_flags + ['-o', obj_path, dso_ll_path] result = subprocess.run(cmd, capture_output=True, text=True, cwd=os.path.dirname(ll_path) or '.') if result.returncode == 0: return (rel, 'ok', '') else: return (rel, 'error', result.stderr.strip()) except FileNotFoundError: return (rel, 'error', f'找不到编译器: {self.compile_cmd}') except Exception as e: return (rel, 'error', str(e)) need_compile = [] for ll_path in ll_files: obj_path = ll_path[:-3] + '.obj' if not os.path.isfile(obj_path): need_compile.append(ll_path) if need_compile: n_workers = min(os.cpu_count() or 4, len(need_compile), 8) _vlog().info(f" 并行编译 {len(need_compile)} 个文件 (workers={n_workers})") errors = [] with ThreadPoolExecutor(max_workers=n_workers) as executor: futures = {executor.submit(_compile_one, ll_path): ll_path for ll_path in need_compile} for future in as_completed(futures): rel, status, err = future.result() if status == 'ok': _vlog().success(f" {rel}") elif status == 'error': _vlog().error(f" {rel}: {err}") errors.append((rel, err)) elif status == 'cached': pass if errors: _vlog().error(f"\n{len(errors)} 个文件编译失败") sys.exit(1) def _precollect_vtable_state(self) -> None: """预扫描 includes .py 文件，收集 @t.CVTable 类的虚表信息。 并行编译（ProcessPoolExecutor）创建独立内存的 worker 进程， vtable 状态必须预收集并通过 pickle 共享。 复用 parse_python_file 的 AST 缓存。 """ for includes_dir in self.include_dirs: if not os.path.isdir(includes_dir): continue for root, dirs, files in os.walk(includes_dir): dirs[:] = [d for d in dirs if not d.startswith('.') and d != '__pycache__'] for fname in files: if not fname.endswith('.py'): continue py_path: str = os.path.join(root, fname) code, tree = parse_python_file(py_path) # 字符串预过滤：不含 CVTable 的文件跳过 AST 遍历 if tree is None or 'CVTable' not in code: continue try: for node in ast.iter_child_nodes(tree): if not isinstance(node, ast.ClassDef): continue has_cvtable: bool = False for dec in node.decorator_list: if isinstance(dec, ast.Attribute): if getattr(dec.value, 'id', None) == 't' and dec.attr == 'CVTable': has_cvtable = True elif isinstance(dec, ast.Name): if dec.id == 'CVTable': has_cvtable = True if not has_cvtable: continue class_name: str = node.name has_methods: bool = False if class_name not in self._shared_class_methods: self._shared_class_methods[class_name] = [] for item in node.body: if isinstance(item, ast.FunctionDef): has_methods = True method_name: str = item.name # 构造函数不放入 vtable（避免子类与基类 vtable 大小不一致破坏多态分派） if method_name in ('__new__', '__init__', '__before_init__'): continue if method_name == '__init__': func_full_name: str = f"{class_name}.__init__" elif method_name == '__call__': func_full_name = f"{class_name}.__call__" else: func_full_name = f"{class_name}.{method_name}" if func_full_name not in self._shared_class_methods[class_name]: self._shared_class_methods[class_name].append(func_full_name) if has_methods: self._shared_class_vtable.add(class_name) self._shared_cross_module_vtable.add(class_name) except Exception: pass def _build_include_py_index(self, includes_dir: str) -> set[str]: """预扫描 includes 目录，构建所有 .py 文件路径集合 用 set 查询替代 os.path.isfile，避免 3490 次磁盘 stat（83s→<0.1s）。 同时规范化路径为小写（Windows 不区分大小写）以加速查找。 """ py_index: set[str] = set() for root, dirs, files in os.walk(includes_dir): dirs[:] = [d for d in dirs if not d.startswith('.') and d != '__pycache__'] for fname in files: if fname.endswith('.py'): py_index.add(os.path.normcase(os.path.join(root, fname))) return py_index def _scan_include_libraries(self) -> None: """扫描 includes 目录，检测被使用的模块对应的库文件和源文件""" if not self.used_includes: return for includes_dir in self.include_dirs: if not os.path.isdir(includes_dir): continue # 预构建 .py 文件索引，避免递归发现时反复 os.path.isfile py_index = self._build_include_py_index(includes_dir) for module_name in self.used_includes: module_dir = os.path.join(includes_dir, module_name) if os.path.isdir(module_dir): self._scan_dir_for_libs(module_dir, module_name) for root, dirs, files in os.walk(module_dir): dirs[:] = [d for d in dirs if not d.startswith('.') and d != '__pycache__'] for fname in files: if fname.endswith('.py'): py_path = os.path.join(root, fname) if py_path not in self.extra_py_files: self.extra_py_files.append(py_path) _vlog().info(f" [includes] 发现 Python 包文件: {os.path.relpath(py_path, includes_dir)}") for ext in self.INCLUDE_LIB_EXTENSIONS: lib_path = os.path.join(includes_dir, module_name + ext) if os.path.isfile(lib_path): self.extra_link_files.append(lib_path) _vlog().info(f" [includes] 发现库文件: {os.path.relpath(lib_path, includes_dir)}") for ext in self.INCLUDE_SRC_EXTENSIONS: src_path = os.path.join(includes_dir, module_name + ext) if os.path.isfile(src_path): self.extra_compile_files.append(src_path) _vlog().info(f" [includes] 发现源文件: {os.path.relpath(src_path, includes_dir)}") py_path = os.path.join(includes_dir, module_name + '.py') if os.path.normcase(py_path) in py_index: self.extra_py_files.append(py_path) _vlog().info(f" [includes] 发现 Python 源文件: {os.path.relpath(py_path, includes_dir)}") discovered = set() for ef in list(self.extra_py_files): self._discover_transitive_deps(ef, includes_dir, discovered, py_index) if self.extra_link_files: _vlog().info(f"\n[includes] 需要链接的库文件: {len(self.extra_link_files)} 个") if self.extra_compile_files: _vlog().info(f"[includes] 需要编译的源文件: {len(self.extra_compile_files)} 个") def _scan_dir_for_libs(self, directory: str, module_name: str) -> None: """递归扫描目录中的库文件和源文件""" for root, dirs, files in os.walk(directory): for fname in files: fpath = os.path.join(root, fname) _, ext = os.path.splitext(fname) ext = ext.lower() if ext in self.INCLUDE_LIB_EXTENSIONS: self.extra_link_files.append(fpath) _vlog().info(f" [includes] 发现库文件: {os.path.relpath(fpath, directory)}") elif ext in self.INCLUDE_SRC_EXTENSIONS: self.extra_compile_files.append(fpath) _vlog().info(f" [includes] 发现源文件: {os.path.relpath(fpath, directory)}") def _discover_transitive_deps(self, py_path: str, includes_dir: str, discovered: set[str], py_index: set[str] | None = None) -> None: """递归发现 Python 文件的间接依赖 复用 parse_python_file 的 AST 缓存，避免重复 open+read+ast.parse。 py_index 为预构建的 .py 文件路径集合，避免 os.path.isfile 磁盘 stat。 """ if py_path in discovered: return discovered.add(py_path) _, tree = parse_python_file(py_path) if tree is None: return try: for node in ast.walk(tree): if isinstance(node, ast.Import): for alias in node.names: self._try_add_include_dep(alias.name, includes_dir, discovered, py_index) elif isinstance(node, ast.ImportFrom): if node.module and node.level == 0: self._try_add_include_dep(node.module, includes_dir, discovered, py_index) except Exception as _e: if _config_mode == "strict": raise _vlog().warning(f"发现 include 传递依赖失败: {_e}", "Exception") def _try_add_include_dep(self, module_name: str, includes_dir: str, discovered: set, py_index: set[str] | None = None): """尝试将模块添加为 include 依赖（支持子包，如 w32.win32base） py_index 非空时用 set 查询替代 os.path.isfile（O(1) vs 24ms/次）。 """ # 候选路径：子包路径（w32/win32base.py）、顶层模块（w32.py）、子包 __init__.py candidates: list[str] = [] # 子包路径：w32.win32base → w32/win32base.py candidates.append(os.path.join(includes_dir, module_name.replace('.', os.sep) + '.py')) # 子包 __init__.py：w32.win32base → w32/win32base/__init__.py candidates.append(os.path.join(includes_dir, module_name.replace('.', os.sep), '__init__.py')) # 顶层模块：取第一个组件 → w32.py top_module = module_name.split('.')[0] if top_module != module_name: candidates.append(os.path.join(includes_dir, top_module + '.py')) for py_path in candidates: exists: bool if py_index is not None: exists = os.path.normcase(py_path) in py_index else: exists = os.path.isfile(py_path) if exists and py_path not in self.extra_py_files: self.extra_py_files.append(py_path) _vlog().info(f" [includes] 发现间接依赖: {os.path.relpath(py_path, includes_dir)}") self._discover_transitive_deps(py_path, includes_dir, discovered, py_index) return # 找到第一个匹配即可 def _is_decl_only_file(self, src_path: str) -> bool: """复用 parse_python_file 的 AST 缓存""" _, tree = parse_python_file(src_path) if tree is None: return False try: for node in ast.walk(tree): if isinstance(node, ast.ClassDef): return False if isinstance(node, ast.FunctionDef): body = node.body if len(body) == 1: stmt = body[0] if isinstance(stmt, ast.Expr) and isinstance(stmt.value, ast.Constant) and stmt.value.value is ...: continue if isinstance(stmt, ast.Pass): continue return False return True except Exception as _e: if _config_mode == "strict": raise _vlog().warning(f"判断是否为声明文件失败: {_e}", "Exception") return False def _sort_include_files_by_deps(self, file_list: list[str]) -> tuple[list[str], dict[str, set[str]]]: """按依赖关系对 include 文件进行拓扑排序，确保被依赖的文件先编译。 返回 (排序后的文件列表, 依赖字典)。 依赖字典: {fpath: set(依赖的fpath)}，仅包含 file_list 中的依赖。 """ # 构建文件名到路径的映射 name_to_path = {} for fpath in file_list: basename = os.path.splitext(os.path.basename(fpath))[0] name_to_path[basename] = fpath # 也支持包内模块: vpsdk/window -> path for fpath in file_list: # 尝试从 includes 目录推断模块全名 for includes_dir in self.include_dirs: try: rel = os.path.relpath(fpath, includes_dir) mod_name = os.path.splitext(rel)[0].replace(os.sep, '.').replace('/', '.') name_to_path[mod_name] = fpath except ValueError: pass # 分析每个文件的 import 依赖（复用 parse_python_file 的 AST 缓存） deps = {} # path -> set of paths it depends on for fpath in file_list: deps[fpath] = set() try: _, tree = parse_python_file(fpath) if tree is None: continue for node in ast.iter_child_nodes(tree): if isinstance(node, ast.Import): for alias in node.names: # 尝试完整路径和各级前缀，确保同包内子模块依赖能被正确解析 # 例如 import zlib.zhuff -> 应先匹配 zlib.zhuff，再回退到 zlib parts = alias.name.split('.') for i in range(len(parts), 0, -1): mod = '.'.join(parts[:i]) if mod in name_to_path and name_to_path[mod] != fpath: deps[fpath].add(name_to_path[mod]) break elif isinstance(node, ast.ImportFrom): if node.module and node.level == 0: # 绝对导入：同样尝试完整路径和各级前缀 parts = node.module.split('.') for i in range(len(parts), 0, -1): mod = '.'.join(parts[:i]) if mod in name_to_path and name_to_path[mod] != fpath: deps[fpath].add(name_to_path[mod]) break elif node.level > 0: # 相对导入：from .xxx import yyy 或 from ..xxx import yyy # 根据当前文件位置推断目标模块的完整路径 current_dir = os.path.dirname(fpath) # node.level == 1 表示当前包，node.level == 2 表示上一级包 package_dir = current_dir for _ in range(node.level - 1): package_dir = os.path.dirname(package_dir) # 推断当前包名（从 includes 目录的相对路径） package_name: str | None = None for includes_dir in self.include_dirs: try: rel = os.path.relpath(package_dir, includes_dir) if rel and rel != '.': package_name = rel.replace(os.sep, '.').replace('/', '.') break except ValueError: pass # 构建目标模块名并查找 target_mod: str | None = None if node.module: target_mod = f"{package_name}.{node.module}" if package_name else node.module else: # from . import yyy：yyy 是子模块，在 names 中 target_mod = package_name if target_mod: parts = target_mod.split('.') for i in range(len(parts), 0, -1): mod = '.'.join(parts[:i]) if mod in name_to_path and name_to_path[mod] != fpath: deps[fpath].add(name_to_path[mod]) break # from . import yyy：yyy 可能是子模块 if node.level > 0 and not node.module: for alias in node.names: sub_mod = f"{package_name}.{alias.name}" if package_name else alias.name if sub_mod in name_to_path and name_to_path[sub_mod] != fpath: deps[fpath].add(name_to_path[sub_mod]) except Exception as _e: if _config_mode == "strict": raise _vlog().warning(f"分析 include 文件导入依赖失败: {_e}", "Exception") # 拓扑排序 (Kahn's algorithm) # in_degree 只计入 file_list 中的依赖——decl-only 文件（如 stdint.py） # 不在 file_list 中，不应计入入度，否则依赖它们的文件入度永远不为 0， # 被误判为循环依赖通过 fallback 添加，导致编译顺序错误。 file_set = set(file_list) in_degree = {fpath: sum(1 for dep in deps[fpath] if dep in file_set) for fpath in file_list} # 反向邻接表：如果 A 依赖 B，则 B -> A reverse_adj = {fpath: [] for fpath in file_list} for fpath in file_list: for dep in deps[fpath]: if dep in reverse_adj: reverse_adj[dep].append(fpath) result = [] queue = [fpath for fpath in file_list if in_degree[fpath] == 0] # 对队列排序以保持确定性 queue.sort(key=lambda x: x) while queue: current = queue.pop(0) result.append(current) for neighbor in reverse_adj[current]: in_degree[neighbor] -= 1 if in_degree[neighbor] == 0: queue.append(neighbor) queue.sort(key=lambda x: x) # 如果有循环依赖，把剩余文件也加入 for fpath in file_list: if fpath not in result: result.append(fpath) # 仅保留 file_list 内的依赖（用于波次并行翻译） file_set_result = set(file_list) deps_in_set = {fpath: (deps[fpath] & file_set_result) for fpath in file_list} return result, deps_in_set def _get_precompiled_obj_path(self, src_path: str, sha1: str) -> str | None: """计算 include 文件在 includes.binary/ 中的预编译 .obj 路径。 预编译目录位于 include 目录同级，命名为 {basename}.binary/。 文件命名为 {sha1}.{triple_safe}.{rel_model}.obj，区分目标平台和重定位模型。 """ AbsSrcPath: str = os.path.abspath(src_path) RelModel: str = 'static' for _flag in self.compile_flags: if _flag.startswith('-relocation-model='): RelModel = _flag[len('-relocation-model='):] break for inc_dir in self.include_dirs: abs_inc: str = os.path.abspath(inc_dir) try: rel: str = os.path.relpath(AbsSrcPath, abs_inc) if rel.startswith('..'): continue inc_parent: str = os.path.dirname(abs_inc) inc_base: str = os.path.basename(abs_inc) precompiled_dir: str = os.path.join(inc_parent, inc_base + '.binary') triple_safe: str = (self.triple or 'default').replace(os.sep, '_').replace('/', '_').replace('\\', '_').replace(':', '_') return os.path.join(precompiled_dir, f"{sha1}.{triple_safe}.{RelModel}.obj") except ValueError: continue return None def _compile_include_py_files(self, active_sha1s: set[str]) -> None: """通过 TransPyC 翻译器编译 includes 目录中发现的 .py 文件 优化流程：预注册符号 → 并行翻译 → 串行后处理 → 并行编译 → 注册+缓存 """ if not self.extra_py_files: return # 按依赖关系排序 include 文件，确保被依赖的文件先编译 sorted_files, file_deps = self._sort_include_files_by_deps(self.extra_py_files) _vlog().info(f"\n[includes 编译] 找到 {len(sorted_files)} 个 Python 源文件") # 构建 include 文件的模块 SHA1 映射（预计算所有文件，构建完整映射） include_ModuleSha1Map = self._build_current_ModuleSha1Map(None) need_translate = [] # [(src_path, sha1, module_name, ll_path, obj_path, precompiled_obj)] for src_path in sorted_files: module_name = os.path.splitext(os.path.basename(src_path))[0] # 复用 parse_python_file 缓存的 content，避免重复 open+read py_content, _ = parse_python_file(src_path) sha1 = compute_sha1(py_content) self.include_py_map[module_name] = sha1 active_sha1s.add(sha1) self._include_sha1s.add(sha1) for includes_dir in self.include_dirs: try: rel = os.path.relpath(src_path, includes_dir) mod_full = os.path.splitext(rel)[0].replace(os.sep, '.').replace('/', '.') include_ModuleSha1Map[mod_full] = sha1 include_ModuleSha1Map[module_name] = sha1 # 包入口文件：注册父包名（如 'os.__init__' -> 'os'） if module_name == '__init__' and '.' in mod_full: parent_pkg = mod_full.rsplit('.', 1)[0] include_ModuleSha1Map[parent_pkg] = sha1 except ValueError: pass include_ModuleSha1Map[module_name] = sha1 ll_path = os.path.join(self.output_dir, f"{sha1}.ll") obj_path = os.path.join(self.output_dir, f"{sha1}.obj") precompiled_obj = self._get_precompiled_obj_path(src_path, sha1) if self._is_decl_only_file(src_path): _vlog().info(f" 跳过(声明文件): {module_name}.py") # 声明文件（如 stdint.py）虽不编译为 .obj，但其 typedef/类型别名 # 必须注册到共享符号表，否则依赖它的 include 文件编译时 # 类型解析失败（如 UINT32PTR 被当作命名结构体而非 i32*） self._register_compiled_include(src_path, sha1, None, include_ModuleSha1Map) self._collect_inline_symbols(src_path) continue # 优先检查 includes.binary 预编译缓存（跨项目共享） if not self.force_recompile_includes and precompiled_obj and os.path.isfile(precompiled_obj): _vlog().info(f" 跳过(预编译): {module_name}.py -> {os.path.basename(precompiled_obj)}") # 预编译跳过的文件仍需注册 .pyi/.stub.ll 声明和符号， # 否则依赖它的 include 文件（如 vpui.py import vpsdk.window） # 在 _EmitModuleDeclarationsLlvm 中找不到声明，导致 Undefined symbol self._register_compiled_include(src_path, sha1, None, include_ModuleSha1Map) self._collect_inline_symbols(src_path) self.extra_link_files.append(precompiled_obj) continue if not self.force_recompile_includes and os.path.isfile(obj_path): _vlog().info(f" 跳过(缓存): {module_name}.py -> {sha1}.obj") self._register_compiled_include(src_path, sha1, None, include_ModuleSha1Map) self._collect_inline_symbols(src_path) self.extra_link_files.append(obj_path) continue need_translate.append((src_path, sha1, module_name, ll_path, obj_path, precompiled_obj)) if not need_translate: return # 预注册所有待翻译文件的符号（从 Phase1 缓存加载 .pyi / .stub.ll） # 这样并行翻译时所有跨模块符号都已可用 for src_path, sha1, module_name, ll_path, obj_path, precompiled_obj in need_translate: self._register_compiled_include(src_path, sha1, None, include_ModuleSha1Map) # 预收集 stub_decls（所有文件共用，避免重复扫描 .stub.ll） stub_decls = [] for other_sha1, other_stub in self.stub_files.items(): if os.path.exists(other_stub): try: with open(other_stub, 'r', encoding='utf-8') as sf: for sline in sf: sl = sline.strip() if sl.startswith('%') and '= type' in sl and 'opaque' not in sl: struct_name = Phase2Translator._extract_llvm_type_name(sl) if struct_name: stub_decls.append(sl) except Exception as e: _vlog().warning(f"Phase2Translator: 忽略异常 {e}", exc_info=e) all_type_defs = {} for sdecl in stub_decls: tname = Phase2Translator._extract_llvm_type_name(sdecl) if tname: all_type_defs[tname] = sdecl # ========== 阶段1：并行翻译 ========== _vlog().info(f" 翻译 {len(need_translate)} 个文件") if len(need_translate) > 1: # 序列化共享符号表供 worker 使用 shared_pickle_path = os.path.join(self.output_dir, '_shared_sym_includes.pickle') try: shared_data = { 'symbol_table': self._shared_symbol_table, 'source_module_sig_files': self._shared_source_module_sig_files, 'all_dc': self._shared_all_dc, 'export_extern_funcs': self._shared_export_extern_funcs, 'inline_func_symbols': self.inline_func_symbols, 'function_default_args': self.function_default_args, 'function_param_names': self.function_param_names, 'generic_class_templates': self._shared_generic_class_templates, 'shared_doc_meta': self._shared_doc_meta, 'class_vtable': self._shared_class_vtable, 'cross_module_vtable': self._shared_cross_module_vtable, 'class_methods': self._shared_class_methods, 'cross_module_novtable': self._shared_cross_module_novtable, } with open(shared_pickle_path, 'wb') as f: f.write(pickle.dumps(shared_data)) except Exception as e: _vlog().warning(f"符号表序列化失败({e})，include将串行翻译") shared_pickle_path = '' if shared_pickle_path and os.path.exists(shared_pickle_path): n_workers = min(8, len(need_translate)) errors = [] # 波次并行翻译：按依赖关系分批提交，确保被依赖的文件先翻译完成。 # 根因：_TryLoadStructFromStub 扫描 output 目录加载跨模块 struct 定义， # 若依赖文件尚未完成翻译（output stub 未写入），只能取到 Phase1 temp # 中的不完整 stub（缺少跨模块继承字段），导致运行时崩溃。 item_by_path = {item[0]: item for item in need_translate} translate_paths = set(item_by_path.keys()) # 每个文件的依赖（仅限本轮待翻译集合内） translate_deps = { p: (file_deps.get(p, set()) & translate_paths) for p in translate_paths } # 复用 self._executor_pool（与 App 源文件阶段共享），避免 GC 阻塞 if self._executor_pool is None: self._executor_pool = ProcessPoolExecutor(max_workers=n_workers) executor = self._executor_pool try: completed_paths: set[str] = set() remaining = list(need_translate) # 保持拓扑序 in_progress: dict = {} # future -> item done_count = 0 while remaining or in_progress: # 提交所有依赖已完成的文件 ready = [ item for item in remaining if translate_deps[item[0]] <= completed_paths ] for item in ready: remaining.remove(item) src_path, sha1, module_name, ll_path, obj_path, precompiled_obj = item future = executor.submit( _parallel_translate_worker, src_path, ll_path, self.src_root, self.temp_dir, self.output_dir, self.include_dirs, self.triple, self.datalayout, self.sha1_map, self.sig_files, self.stub_files, self.include_py_map, self.slice_level, shared_pickle_path, self.struct_sha1_map, include_ModuleSha1Map ) in_progress[future] = item if not in_progress: # 无可提交且无进行中：循环依赖 fallback for item in remaining: src_path, sha1, module_name, ll_path, obj_path, precompiled_obj = item future = executor.submit( _parallel_translate_worker, src_path, ll_path, self.src_root, self.temp_dir, self.output_dir, self.include_dirs, self.triple, self.datalayout, self.sha1_map, self.sig_files, self.stub_files, self.include_py_map, self.slice_level, shared_pickle_path, self.struct_sha1_map, include_ModuleSha1Map ) in_progress[future] = item remaining.clear() # 等待至少一个完成 done_set, _ = wait(in_progress, return_when=FIRST_COMPLETED) for future in done_set: item = in_progress.pop(future) src_path, sha1, module_name, ll_path, obj_path, precompiled_obj = item completed_paths.add(src_path) done_count += 1 try: status, _, err = future.result() if status == 'ok': _vlog().info(f" [{done_count}/{len(need_translate)}] 翻译完成: {module_name}.py") else: _vlog().error(f" {module_name}.py: {err}") errors.append((module_name, err)) except Exception as e: _vlog().error(f" {module_name}.py: {e}") errors.append((module_name, str(e))) except Exception: if self._executor_pool is not None: self._executor_pool.shutdown(wait=False, cancel_futures=True) self._executor_pool = None raise if errors: _vlog().error(f"\n{len(errors)} 个 include 文件翻译失败") sys.exit(1) else: # 串行回退 for item in need_translate: src_path, sha1, module_name, ll_path, obj_path, precompiled_obj = item _vlog().info(f" 翻译: {module_name}.py -> {sha1}.ll") self._translate_file(src_path, ll_path, prebuilt_ModuleSha1Map=include_ModuleSha1Map) else: item = need_translate[0] src_path, sha1, module_name, ll_path, obj_path, precompiled_obj = item _vlog().info(f" 翻译: {module_name}.py -> {sha1}.ll") self._translate_file(src_path, ll_path, prebuilt_ModuleSha1Map=include_ModuleSha1Map) # ========== 阶段2：串行后处理（dso_local + stub_decls 注入）========== compile_tasks = [] # [(dso_ll_path, obj_path, src_path, sha1, module_name, ll_path, precompiled_obj, ll_content)] for item in need_translate: src_path, sha1, module_name, ll_path, obj_path, precompiled_obj = item if not os.path.isfile(ll_path): _vlog().error(f" 翻译未生成 .ll 文件: {module_name}.py") continue with open(ll_path, 'r', encoding='utf-8') as f: ll_content = f.read() # dso_local 正则标注（使用 ^ + MULTILINE 避免匹配字符串常量内容中的 define/declare） ll_content = re.sub(r'^(define|declare)\s+(private|internal|available_externally|linkonce|weak|common|appending|extern_weak|linkonce_odr|weak_odr|external)\s+', r'\1 \2 dso_local ', ll_content, flags=re.MULTILINE) ll_content = re.sub(r'^(define|declare)\s+(?!private|internal|available_externally|linkonce|weak|common|appending|extern_weak|linkonce_odr|weak_odr|external|dso_local)', r'\1 dso_local ', ll_content, flags=re.MULTILINE) # LLVM 22+: external 不允许在带初值的全局定义上显式写出，改为 linkonce 保留多模块合并语义 ll_content = re.sub(r'^(@[\w.]+)\s*=\s*external\s+global\s+(\w[\w*]*)\s+(-?\d+|null|zeroinitializer|true|false)$', r'\1 = linkonce global \2 \3', ll_content, flags=re.MULTILINE) ll_content = re.sub(r'^(@[\w.]+)\s*=\s*(private|internal|available_externally|linkonce|weak|common|appending|extern_weak|linkonce_odr|weak_odr)\s+(global|constant)', r'\1 = \2 dso_local \3', ll_content, flags=re.MULTILINE) ll_content = re.sub(r'^(@[\w.]+)\s*=\s+(?!private|internal|available_externally|linkonce|weak|common|appending|extern_weak|linkonce_odr|weak_odr|dso_local)(global|constant)', r'\1 = dso_local \2', ll_content, flags=re.MULTILINE) # stub_decls 注入（复用预收集的 stub_decls） if stub_decls: for sdecl in stub_decls: sname = Phase2Translator._extract_llvm_type_name(sdecl) if sname: short = sname.split('.')[-1] if '.' in sname else sname ll_content = re.sub( r'%"?' + re.escape(sname) + r'"?\s*=\s*type\s*opaque', sdecl, ll_content) if short != sname: ll_content = re.sub( r'%"?' + re.escape(short) + r'"?\s*=\s*type\s*opaque', sdecl, ll_content) max_iterations = 10 for _ in range(max_iterations): defined_types = set() for line in ll_content.splitlines(): sl = line.strip() if sl.startswith('%') and '= type' in sl: tname = Phase2Translator._extract_llvm_type_name(sl) if tname: defined_types.add(tname) # 同时添加短名，防止 missing 循环因全名/短名不匹配而重复添加 if '.' in tname: parts = tname.split('.', 1) if len(parts[0]) == 16 and all(c in '0123456789abcdef' for c in parts[0]): defined_types.add(parts[1]) referenced_types = Phase2Translator._find_all_type_refs(ll_content) missing = referenced_types - defined_types new_defs = [] for tname in sorted(missing): if tname in all_type_defs: new_defs.append(all_type_defs[tname]) else: short_key = tname.split('.')[-1] if '.' in tname else tname for full_name, defn in all_type_defs.items(): if full_name.split('.')[-1] == short_key: if short_key not in defined_types: new_defs.append(defn) break else: # 未在 stub 中找到定义（如泛型类 list[T]）， # 声明为 opaque 以满足 llc 的类型前向声明要求 new_defs.append(f'%"{tname}" = type opaque') if not new_defs: break inject_block = '\n'.join(new_defs) + '\n' lines = ll_content.splitlines(True) insert_pos = 0 for i, line in enumerate(lines): stripped = line.strip() if stripped and not stripped.startswith(';') and not stripped.startswith('target ') and not stripped.startswith('source_filename') and not stripped.startswith('declare ') and not stripped.startswith('@') and not stripped.startswith('define ') and not stripped.startswith('attributes ') and not stripped.startswith('!') and not stripped.startswith('module '): insert_pos = i break if insert_pos == 0: insert_pos = len(lines) ll_content = ''.join(lines[:insert_pos]) + inject_block + ''.join(lines[insert_pos:]) # 将所有类型定义移到 declares 之前 # （llc 要求 by-value struct 参数的类型在 declare 前定义，否则报 invalid type for function argument） lines = ll_content.splitlines(True) type_def_lines = [] other_lines = [] seen_type_names_inc = set() for line in lines: stripped = line.strip() if stripped.startswith('%') and '= type' in stripped: type_name_inc = stripped.split('=')[0].strip() if type_name_inc in seen_type_names_inc: continue seen_type_names_inc.add(type_name_inc) type_def_lines.append(line) else: other_lines.append(line) if type_def_lines: insert_pos = 0 for i, line in enumerate(other_lines): stripped = line.strip() if stripped and not stripped.startswith(';') and not stripped.startswith('target ') and not stripped.startswith('source_filename') and not stripped.startswith('!'): insert_pos = i break if insert_pos == 0: insert_pos = len(other_lines) ll_content = ''.join(other_lines[:insert_pos]) + ''.join(type_def_lines) + ''.join(other_lines[insert_pos:]) dso_ll_path = ll_path[:-3] + '_dso.ll' with open(dso_ll_path, 'w', encoding='utf-8') as f: f.write(ll_content) compile_tasks.append((dso_ll_path, obj_path, src_path, sha1, module_name, ll_path, precompiled_obj, ll_content)) if not compile_tasks: return # ========== 阶段3：并行编译（llc）========== _vlog().info(f" 编译 {len(compile_tasks)} 个 .ll 文件") compile_results = {} # task -> result def _compile_one_include(task): dso_ll_path, obj_path = task[0], task[1] cmd = [self.compile_cmd] + self.compile_flags + ['-o', obj_path, dso_ll_path] result = subprocess.run(cmd, capture_output=True, text=True, cwd=self.output_dir) return (task, result) if len(compile_tasks) > 1: n_workers = min(8, len(compile_tasks)) with ThreadPoolExecutor(max_workers=n_workers) as executor: futures = {executor.submit(_compile_one_include, task): task for task in compile_tasks} done_count = 0 for future in as_completed(futures): done_count += 1 try: task, result = future.result() module_name = task[4] if result.returncode == 0: _vlog().success(f" [{done_count}/{len(compile_tasks)}] 编译成功: {module_name}.py") else: _vlog().warning(f" [{done_count}/{len(compile_tasks)}] 编译失败: {module_name}.py: {result.stderr.strip()}") compile_results[task] = result except Exception as e: task = futures[future] module_name = task[4] _vlog().error(f" [{done_count}/{len(compile_tasks)}] 编译异常: {module_name}.py: {e}") compile_results[task] = None else: task = compile_tasks[0] task_result, result = _compile_one_include(task) module_name = task[4] if result.returncode == 0: _vlog().success(f" 编译成功: {module_name}.py") else: _vlog().warning(f" 编译失败: {module_name}.py: {result.stderr.strip()}") compile_results[task] = result # ========== 阶段4：注册 + 缓存 ========== has_error = False for task in compile_tasks: dso_ll_path, obj_path, src_path, sha1, module_name, ll_path, precompiled_obj, ll_content = task result = compile_results.get(task) if result and result.returncode == 0: self.extra_link_files.append(obj_path) # 复制到 includes.binary 预编译目录 if precompiled_obj: precompiled_dir = os.path.dirname(precompiled_obj) try: if not os.path.exists(precompiled_dir): os.makedirs(precompiled_dir, exist_ok=True) shutil.copy2(obj_path, precompiled_obj) except Exception as _e: if _config_mode == "strict": _vlog().warning(f"预编译缓存写入失败: {_e}") # 注册符号 self._register_compiled_include(src_path, sha1, ll_content, include_ModuleSha1Map) # 清理临时文件（仅成功时清理，失败时保留以便调试） # DEBUG: 保留 .ll 文件以便静态分析 # for ext in ['.ll', '_dso.ll']: # p = ll_path[:-3] + ext # if os.path.isfile(p): # os.remove(p) else: has_error = True _vlog().warning(f" 保留失败文件以便调试: {ll_path}") if has_error: _vlog().error(f"\nincludes 文件编译失败") sys.exit(1) # 从 include 文件源码中提取函数参数名，供主项目编译时 keyword 参数匹配使用 self._extract_include_param_names(sorted_files) def _extract_include_param_names_from_dirs(self) -> None: """从 include 目录的 Python 源码中提取函数参数名，存入 function_param_names 优化：单次递归遍历 AST，维护当前类上下文，O(N) 复杂度。 复用 parse_python_file 的 AST 缓存。 """ for includes_dir in self.include_dirs: if not os.path.isdir(includes_dir): continue for filename in os.listdir(includes_dir): if not filename.endswith('.py'): continue src_path = os.path.join(includes_dir, filename) try: source, tree = parse_python_file(src_path) if tree is None: continue sha1 = compute_sha1(source) self._collect_function_params(tree, sha1) except Exception as e: _vlog().warning(f" 提取参数名失败 {src_path}: {e}") def _collect_function_params(self, tree: ast.AST, sha1: str | None) -> None: """单次递归遍历 AST 收集函数参数名（含类方法），O(N) 复杂度 遍历时维护 current_class 上下文：遇到 ClassDef 进入类作用域， 遇到 FunctionDef 记录参数名并标注所属类，函数内部的嵌套定义 不再属于外层类。 """ def visit(node: ast.AST, current_class: str | None) -> None: for child in ast.iter_child_nodes(node): if isinstance(child, ast.ClassDef): # 进入类作用域，递归处理类体 visit(child, child.name) elif isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)): func_name = child.name param_names = [arg.arg for arg in child.args.args] self.function_param_names[func_name] = param_names if sha1: self.function_param_names[f'{sha1}.{func_name}'] = param_names if current_class: self.function_param_names[f'{current_class}.{func_name}'] = param_names if sha1: self.function_param_names[f'{sha1}.{current_class}.{func_name}'] = param_names # 函数内部的嵌套定义不属于外层类 visit(child, None) else: visit(child, current_class) visit(tree, None) def _extract_app_default_args(self, need_translate: list) -> None: """从 App 源文件中提取函数默认参数，存入 function_default_args 在 Phase 2 并行编译开始前调用，确保跨模块调用参数检查能正确识别默认参数。 need_translate 中每个元素是 (i, src_path, out_path, sha1, current_ModuleSha1Map) 复用 parse_python_file 的 AST 缓存（run() 循环已 parse 过）。 """ for entry in need_translate: src_path = entry[1] sha1 = entry[3] _, tree = parse_python_file(src_path) if tree is None: continue try: for node in ast.walk(tree): if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): func_name = node.name defaults = getattr(node.args, 'defaults', []) if defaults: default_vals = [] for d in defaults: if isinstance(d, ast.Constant): default_vals.append(d.value) else: default_vals.append(None) # 存储多种键格式，确保跨模块调用检查能找到 self.function_default_args[func_name] = default_vals self.function_default_args[f'{sha1}.{func_name}'] = default_vals # 也更新参数名（预扫描时补充） param_names = [arg.arg for arg in node.args.args] if func_name not in self.function_param_names: self.function_param_names[func_name] = param_names if f'{sha1}.{func_name}' not in self.function_param_names: self.function_param_names[f'{sha1}.{func_name}'] = param_names except Exception as e: _vlog().warning(f" 提取默认参数失败 {src_path}: {e}") def _extract_include_param_names(self, sorted_files: list[str]) -> None: """从 include 文件的 Python 源码中提取函数参数名，存入 function_param_names 优化：复用 _collect_function_params 的 O(N) 单次遍历 + parse_python_file 缓存。 """ for src_path in sorted_files: if self._is_decl_only_file(src_path): continue try: _, tree = parse_python_file(src_path) if tree is None: continue module_name = os.path.splitext(os.path.basename(src_path))[0] sha1 = self.include_py_map.get(module_name) self._collect_function_params(tree, sha1) except Exception as e: _vlog().warning(f" 提取参数名失败 {src_path}: {e}") def _register_compiled_include(self, src_path: str, sha1: str, ll_content: str, include_ModuleSha1Map: dict[str, str]) -> None: """编译 include 文件成功后，生成签名和 stub 文件并注册到符号表中""" module_name = os.path.splitext(os.path.basename(src_path))[0] # 确定完整模块名（如 vpsdk.window） mod_full = module_name for includes_dir in self.include_dirs: try: rel = os.path.relpath(src_path, includes_dir) mod_full = os.path.splitext(rel)[0].replace(os.sep, '.').replace('/', '.') break except ValueError: pass # 1. 生成 .pyi 签名文件（如果还没有的话） sig_path = os.path.join(self.temp_dir, f"{sha1}.pyi") if not os.path.isfile(sig_path): # 全局缓存检查（不被 --clean 清除） _TryGlobalCache(sha1, 'pyi', sig_path) if not os.path.isfile(sig_path): try: with open(src_path, 'r', encoding='utf-8') as f: py_content = f.read() sig_content = PythonToStubConverter.convert(py_content, mod_full) os.makedirs(self.temp_dir, exist_ok=True) with open(sig_path, 'w', encoding='utf-8', newline='\n') as f: f.write(sig_content) _SaveToGlobalCache(sha1, 'pyi', sig_path) except Exception as e: if _config_mode == "strict": raise _vlog().warning(f"签名生成失败不阻塞编译: {e}", "Exception") # 注册到 sig_files 和 _source_module_sig_files if os.path.isfile(sig_path): self.sig_files[sha1] = sig_path self.sha1_map[sha1] = f"includes/{mod_full.replace('.', os.sep)}.py" if self._shared_source_module_sig_files is not None: self._shared_source_module_sig_files[mod_full] = sig_path self._shared_source_module_sig_files[module_name] = sig_path # 将签名信息加载到共享符号表中 if self._shared_symbol_table is not None: try: self._shared_symbol_table.LoadModuleSymbols(sig_path, mod_full, lineno=0) except Exception as _e: if _config_mode == "strict": raise _vlog().warning(f"加载模块签名到共享符号表失败: {_e}", "Exception") # 2. 生成 .stub.ll 文件 stub_path = os.path.join(self.output_dir, f"{sha1}.stub.ll") if not os.path.isfile(stub_path): # 全局缓存检查（不被 --clean 清除） _TryGlobalCache(sha1, 'stub.ll', stub_path) if not os.path.isfile(stub_path) and ll_content: try: stub_content, _ = self._split_ll(ll_content) with open(stub_path, 'w', encoding='utf-8', newline='\n') as f: f.write(stub_content) _SaveToGlobalCache(sha1, 'stub.ll', stub_path) except Exception as _e: if _config_mode == "strict": raise _vlog().warning(f"生成 .stub.ll 文件失败: {_e}", "Exception") # 注册到 stub_files if os.path.isfile(stub_path): self.stub_files[sha1] = stub_path elif os.path.isfile(sig_path): # 如果 stub 文件不存在，也尝试从 temp 目录查找 temp_stub = os.path.join(self.temp_dir, f"{sha1}.stub.ll") if os.path.isfile(temp_stub): self.stub_files[sha1] = temp_stub else: # 最后尝试从全局缓存直接读取 _TryGlobalCache(sha1, 'stub.ll', temp_stub) if os.path.isfile(temp_stub): self.stub_files[sha1] = temp_stub def _precollect_inline_symbols(self) -> None: """在主翻译循环之前，扫描includes目录中used_includes相关的模块收集内联函数AST""" if not self.used_includes: return for includes_dir in self.include_dirs: if not os.path.isdir(includes_dir): continue for module_name in self.used_includes: module_dir = os.path.join(includes_dir, module_name) if os.path.isdir(module_dir): for root, dirs, files in os.walk(module_dir): dirs[:] = [d for d in dirs if not d.startswith('.') and d != '__pycache__'] for fname in files: if fname.endswith('.py'): src_path = os.path.join(root, fname) self._collect_inline_symbols(src_path) py_path = os.path.join(includes_dir, module_name + '.py') if os.path.isfile(py_path): self._collect_inline_symbols(py_path) for ef in self.extra_py_files: self._collect_inline_symbols(ef) def _collect_inline_symbols(self, src_path: str): """解析源文件，收集内联函数的AST信息到inline_func_symbols 复用 parse_python_file 的 AST 缓存，避免重复 open+read+ast.parse。 """ try: _, tree = parse_python_file(src_path) if tree is None: return module_name = os.path.splitext(os.path.basename(src_path))[0] for node in ast.iter_child_nodes(tree): if isinstance(node, ast.FunctionDef) and node.returns: is_inline = False if isinstance(node.returns, ast.BinOp) and isinstance(node.returns.op, ast.BitOr): for part in [node.returns.left, node.returns.right]: if isinstance(part, ast.Attribute) and part.attr == 'CInline': is_inline = True break if isinstance(part, ast.Name) and part.id == 'CInline': is_inline = True break elif isinstance(node.returns, ast.Attribute) and node.returns.attr == 'CInline': is_inline = True elif isinstance(node.returns, ast.Name) and node.returns.id == 'CInline': is_inline = True if is_inline: func_name = node.name sym_key = f"{module_name}.{func_name}" info = CTypeInfo() info.Name = func_name info.IsFunction = True info.IsInline = True info.InlineBody = node.body info.InlineParams = [arg.arg for arg in node.args.args] info.Storage = t.CInline() self.inline_func_symbols[func_name] = info self.inline_func_symbols[sym_key] = info except Exception as _e: if _config_mode == "strict": raise _vlog().warning(f"预收集内联函数符号失败: {_e}", "Exception") def _build_shared_symbol_data(self) -> None: """一次性构建所有文件共享的符号表数据，避免每个文件重复加载""" t0 = time.time() _vlog().info("[缓存] 构建共享符号表数据...") base_trans = TransPyC.TransPyC(code="pass", triple=self.triple, datalayout=self.datalayout) base_trans.translator.LlvmGen = None base_trans.translator._ModuleSha1Map = {} base_trans.translator._global_function_default_args = self.function_default_args base_trans.translator._global_function_param_names = self.function_param_names for includes_dir in self.include_dirs: if os.path.isdir(includes_dir): for pyi_file in os.listdir(includes_dir): if pyi_file.endswith('.pyi'): pyi_path = os.path.join(includes_dir, pyi_file) module_name = os.path.splitext(pyi_file)[0] base_trans.translator.SymbolTable.LoadModuleSymbols(pyi_path, module_name, lineno=0) for py_file in os.listdir(includes_dir): if py_file.endswith('.py') and (not py_file.startswith('_') or py_file == '_list.py' or py_file == '_dict.py'): module_name = os.path.splitext(py_file)[0] sha1_key = self.include_py_map.get(module_name) if sha1_key and sha1_key in self.sig_files: sig_path = self.sig_files[sha1_key] base_trans.translator.SymbolTable.LoadModuleSymbols(sig_path, module_name, lineno=0) # 收集需要重导出的包，等所有模块符号加载完后再处理 _pending_reexports = [] for sha1_key, sig_path in self.sig_files.items(): rel_path = self.sha1_map.get(sha1_key, sha1_key) if rel_path.startswith('includes/'): mod_name = os.path.splitext(rel_path)[0].replace(os.sep, '.').replace('/', '.') mod_name = mod_name[len('includes/'):] base_trans.translator.SymbolTable.LoadModuleSymbols(sig_path, mod_name, lineno=0) if mod_name.endswith('.__init__'): package_name = mod_name[:-len('.__init__')] _pending_reexports.append((sig_path, package_name)) # 所有 includes 模块符号已加载，现在处理包重导出 for sig_path, package_name in _pending_reexports: self._register_package_reexports(base_trans, sig_path, package_name) for sha1_key, sig_path in self.sig_files.items(): rel_path = self.sha1_map.get(sha1_key, sha1_key) if rel_path.startswith('includes/'): continue module_name = os.path.splitext(rel_path)[0].replace(os.sep, '.').replace('/', '.') base_trans.translator.SymbolTable.LoadModuleSymbols(sig_path, module_name, lineno=0) base_trans.translator._source_module_sig_files[module_name] = sig_path short_name = module_name.split('.')[-1] if '.' in module_name else module_name if short_name != module_name: base_trans.translator._source_module_sig_files[short_name] = sig_path all_dc = {} for sha1_key, stub_path in self.stub_files.items(): if os.path.exists(stub_path): try: with open(stub_path, 'r', encoding='utf-8') as f: stub_content = f.read() for line in stub_content.splitlines(): line = line.strip() if line.startswith('@') and '= global' in line and 'i32' in line and not line.startswith('@llvm'): var_name = line.split('=')[0].strip().lstrip('@') val_part = line.split('i32')[-1].strip().rstrip(']').lstrip('[').lstrip('i32') try: val = int(val_part) all_dc[var_name] = val except Exception as e: _vlog().warning(f"Phase2Translator: 忽略异常 {e}", exc_info=e) except Exception as e: _vlog().warning(f"Phase2Translator: 忽略异常 {e}", exc_info=e) for sha1_key, pyi_path in self.sig_files.items(): _ExtractCDefineConstantsFromPyi(pyi_path, all_dc) export_extern_funcs = {} for sha1_key, sig_path in self.sig_files.items(): try: # 复用 parse_python_file 的 AST 缓存（sig 文件被多处解析） _, sig_tree = parse_python_file(sig_path) if sig_tree is None: continue for node in ast.iter_child_nodes(sig_tree): if isinstance(node, ast.FunctionDef): if _check_annotation_for_export(node.returns): export_extern_funcs[node.name] = sha1_key except Exception as _e: if _config_mode == "strict": raise _vlog().warning(f"收集共享导出外部函数失败: {_e}", "Exception") for sym_name, sym_info in self.inline_func_symbols.items(): if sym_name not in base_trans.translator.SymbolTable: base_trans.translator.SymbolTable[sym_name] = sym_info else: existing = base_trans.translator.SymbolTable[sym_name] if not getattr(existing, 'InlineBody', None) and getattr(sym_info, 'InlineBody', None): existing.IsInline = sym_info.IsInline existing.InlineBody = sym_info.InlineBody existing.InlineParams = sym_info.InlineParams self._shared_symbol_table = base_trans.translator.SymbolTable self._shared_source_module_sig_files = dict(base_trans.translator._source_module_sig_files) self._shared_all_dc = all_dc self._shared_export_extern_funcs = export_extern_funcs generic_class_templates = {} for includes_dir in self.include_dirs: if os.path.isdir(includes_dir): abs_includes = os.path.join(self.src_root, includes_dir) if not os.path.isabs(includes_dir) else includes_dir if not os.path.isdir(abs_includes): abs_includes = includes_dir for py_file in os.listdir(abs_includes): if py_file.endswith('.py') and (not py_file.startswith('_') or py_file == '_list.py' or py_file == '_dict.py'): py_path = os.path.join(abs_includes, py_file) try: # 复用 parse_python_file 的 AST 缓存 _, py_tree = parse_python_file(py_path) if py_tree is None: continue for node in py_tree.body: if isinstance(node, ast.ClassDef) and hasattr(node, 'type_params') and node.type_params: type_params = [tp.name for tp in node.type_params] generic_class_templates[node.name] = { 'node': node, 'type_params': type_params, } except Exception as _e: if _config_mode == "strict": raise _vlog().warning(f"收集泛型类模板失败: {_e}", "Exception") self._shared_generic_class_templates = generic_class_templates # 加载所有 {sha1}.doc.json 到 _shared_doc_meta，供跨模块 __doc__ 查询 shared_doc_meta: dict[str, dict[str, str]] = {} if os.path.isdir(self.temp_dir): for fname in os.listdir(self.temp_dir): if fname.endswith('.doc.json'): sha1_key = fname[:-len('.doc.json')] try: with open(os.path.join(self.temp_dir, fname), 'r', encoding='utf-8') as f: data = json.load(f) if isinstance(data, dict): shared_doc_meta[sha1_key] = {str(k): str(v) for k, v in data.items()} except Exception as _e: if _config_mode == "strict": raise _vlog().warning(f"加载 {fname} 失败: {_e}", "Exception") # 也从全局缓存加载 .doc.json（temp_dir 被 --clean 清空时补充） if os.path.isdir(_GLOBAL_CACHE_DIR): for fname in os.listdir(_GLOBAL_CACHE_DIR): if fname.endswith('.doc.json'): sha1_key = fname[:-len('.doc.json')] if sha1_key in shared_doc_meta: continue try: with open(os.path.join(_GLOBAL_CACHE_DIR, fname), 'r', encoding='utf-8') as f: data = json.load(f) if isinstance(data, dict): shared_doc_meta[sha1_key] = {str(k): str(v) for k, v in data.items()} except Exception as _e: if _config_mode == "strict": raise _vlog().warning(f"加载全局缓存 {fname} 失败: {_e}", "Exception") self._shared_doc_meta = shared_doc_meta t1 = time.time() _vlog().info(f"[缓存] 共享符号表构建完成 ({t1-t0:.2f}s, {len(self._shared_symbol_table)} 符号, {len(generic_class_templates)} 泛型模板, {len(shared_doc_meta)} doc 元数据)") def _compile_include_sources(self): """扫描项目目录中的汇编存根，编译 includes 目录中发现的 C/C++ 源文件""" extra_sources = list(self.extra_compile_files) scan_dirs = [] if self.src_root and os.path.isdir(self.src_root): scan_dirs.append(self.src_root) project_dir = os.path.dirname(self.output_dir) if project_dir and os.path.isdir(project_dir) and project_dir not in scan_dirs: scan_dirs.append(project_dir) for scan_dir in scan_dirs: for root, dirs, files in os.walk(scan_dir): dirs[:] = [d for d in dirs if not d.startswith('.') and d != '__pycache__'] for fname in files: if fname.endswith(('.S', '.s', '.c', '.cpp')): if fname.startswith('test_'): continue fpath = os.path.join(root, fname) if fpath not in extra_sources: extra_sources.append(fpath) if not extra_sources: return cc_map = { '.c': 'gcc', '.cpp': 'g++', '.cc': 'g++', '.cxx': 'g++', '.m': 'clang', '.mm': 'clang++', '.s': 'gcc', '.S': 'gcc', } _vlog().info(f"\n[includes 编译] 找到 {len(extra_sources)} 个源文件") for src_path in extra_sources: _, ext = os.path.splitext(src_path) ext = ext.lower() obj_path = os.path.join(self.output_dir, os.path.splitext(os.path.basename(src_path))[0] + '.obj') if ext in ('.s', '.S') and any('oformat' in f for f in self.linker_flags): cc = 'clang' compile_src = src_path extra_flags = ['-c', '-target', 'x86_64-none-elf', '-o', obj_path, compile_src] else: cc = cc_map.get(ext, 'gcc') extra_flags = ['-c', '-o', obj_path, src_path] _vlog().info(f" 编译: {os.path.basename(src_path)} -> {os.path.basename(obj_path)}") try: cmd = [cc] + extra_flags result = subprocess.run(cmd, capture_output=True, text=True) if result.returncode == 0: _vlog().success(" 编译成功") self.extra_link_files.append(obj_path) else: _vlog().error(f" 编译失败: {result.stderr.strip()}") _vlog().error(f"\nC/C++ 源文件编译失败，立即终止编译。") sys.exit(1) except FileNotFoundError: _vlog().error(f" 找不到编译器: {cc}") _vlog().error(f"\n找不到编译器，立即终止编译。") sys.exit(1) except Exception as e: _vlog().error(f" {e}") _vlog().error(f"\n编译异常，立即终止编译。") sys.exit(1) def _strip_debug_sections(self, exe_path: str) -> None: """剥离 .exe 中的调试段（跳过裸机二进制）""" if not os.path.exists(exe_path): return ext = os.path.splitext(exe_path)[1].lower() if ext not in ('.exe', '.dll', '.obj', '.o', '.elf'): _vlog().info(f" [剥离] 跳过（{ext} 格式无需剥离）") return for tool in ['llvm-strip', 'strip']: strip_cmd = shutil.which(tool) if strip_cmd: try: result = subprocess.run( [strip_cmd, '--strip-debug', exe_path], capture_output=True, text=True ) if result.returncode == 0: _vlog().info(f" [剥离] 调试段已移除 ({tool})") else: _vlog().warning(f" [剥离] 警告: {result.stderr.strip()}") except Exception as _e: if _config_mode == "strict": raise _vlog().warning(f"剥离可执行文件调试段失败: {_e}", "Exception") break def _link_obj_files(self, active_sha1s: set[str]) -> None: """将 active_sha1s 对应的 .obj 文件和 includes 库文件链接为可执行文件""" obj_files = [] merged_obj = os.path.join(self.output_dir, '_merged.obj') if os.path.isfile(merged_obj): obj_files.append(merged_obj) else: for root, dirs, files in os.walk(self.output_dir): for file in files: if file.endswith('.obj'): sha1 = file[:-4] if sha1 in active_sha1s and sha1 not in self._include_sha1s: obj_files.append(os.path.join(root, file)) if not obj_files and not self.extra_link_files: _vlog().info("[链接] 无文件可链接") return _vlog().info(f"\n[链接] 找到 {len(obj_files)} 个 .obj 文件") all_link_files = obj_files + self.extra_link_files seen = set() all_link_files = [f for f in all_link_files if not (f in seen or seen.add(f))] if self.extra_link_files: _vlog().info(f"[链接] 额外库文件: {len(self.extra_link_files)} 个") for lib in self.extra_link_files: _vlog().info(f" - {os.path.basename(lib)}") if not self.linker_cmd: _vlog().info("[链接] 未配置链接器") return exe_path = os.path.join(self.output_dir, self.linker_output) if self.linker_output else os.path.join(self.output_dir, 'a.exe') is_binary = self.linker_output and self.linker_output.endswith('.bin') # 检查是否直接输出二进制（使用 --oformat binary / -oformat binary 标志） is_direct_binary = is_binary and any('oformat' in f for f in self.linker_flags) if is_binary and not is_direct_binary: link_output = exe_path[:-4] + '_pe.exe' else: link_output = exe_path # 自动复制或生成 linker.ld 到输出目录 linker_ld_src = os.path.join(os.path.dirname(self.output_dir), 'linker.ld') linker_ld_dst = os.path.join(self.output_dir, 'linker.ld') if os.path.isfile(linker_ld_src) and not os.path.isfile(linker_ld_dst): shutil.copy2(linker_ld_src, linker_ld_dst) _vlog().info(f" [链接] 使用链接脚本: linker.ld") elif not os.path.isfile(linker_ld_dst) and any('-T' in f or '-T' in f.replace(',', ' ') for f in self.linker_flags): default_ld = """ENTRY(_start) SECTIONS { . = 0x100000; .text : { *(.text) *(.text.*) } .rodata : { *(.rodata) *(.rodata.*) } .data : { *(.data) *(.data.*) } .bss : { *(.bss) *(.bss.*) } /DISCARD/ : { *(.comment) *(.note) *(.eh_frame) *(.eh_frame_hdr) *(.reloc) *(.rela) *(.debug*) } } """ with open(linker_ld_dst, 'w', encoding='utf-8', newline='\n') as f: f.write(default_ld) _vlog().info(f" [链接] 自动创建默认链接脚本: linker.ld") try: # 将 linker_flags 拆分为非库标志和库标志（-l/-L），库标志放在目标文件之后 # 这样 Unix 风格链接器才能正确解析库中的符号 # -Wl, 选项是链接器全局选项，放在目标文件之前 non_lib_flags = [] lib_flags = [] i = 0 while i < len(self.linker_flags): flag = self.linker_flags[i] if flag in ('-l', '-L') and i + 1 < len(self.linker_flags): # -l xxx / -L xxx 分开的形式 lib_flags.extend([flag, self.linker_flags[i + 1]]) i += 2 elif flag.startswith('-l') or flag.startswith('-L'): # -lxxx / -Lxxx 合并的形式 lib_flags.append(flag) i += 1 else: # 其他所有标志（包括 -Wl, 选项）放在目标文件之前 non_lib_flags.append(flag) i += 1 cmd = [self.linker_cmd] + non_lib_flags + ['-o', link_output] + all_link_files + lib_flags _vlog().info(f" 执行: {' '.join(cmd)}") result = subprocess.run( cmd, capture_output=True, text=True, cwd=self.output_dir or '.' ) if result.returncode == 0: if is_binary and not is_direct_binary: objcopy = shutil.which('llvm-objcopy') or shutil.which('objcopy') if objcopy: conv = subprocess.run( [objcopy, '-O', 'binary', link_output, exe_path], capture_output=True, text=True ) if conv.returncode == 0: os.remove(link_output) _vlog().success(f" 生成裸机二进制: {exe_path}") file_size = os.path.getsize(exe_path) _vlog().info(f" [大小] {file_size} 字节") else: _vlog().error(f" 二进制转换失败: {conv.stderr.strip()}") sys.exit(1) else: _vlog().error(" 找不到 llvm-objcopy，无法生成裸机二进制") sys.exit(1) else: _vlog().success(f" 生成: {exe_path}") file_size = os.path.getsize(exe_path) _vlog().info(f" [大小] {file_size} 字节") if not is_binary: self._strip_debug_sections(exe_path) else: _vlog().error(f" 链接失败: {result.stderr.strip()}") _vlog().error(f"\n链接失败，立即终止编译。") sys.exit(1) except FileNotFoundError: _vlog().error(f" 找不到链接器: {self.linker_cmd}") _vlog().error(f"\n找不到链接器，立即终止编译。") sys.exit(1) except Exception as e: _vlog().error(f" {e}") _vlog().error(f"\n链接异常，立即终止编译。") sys.exit(1)