TermiNexus
d4fc66fe3c
_merged.obj 只合并非 include 模块；此前命中该分支后跳过 include 的 .obj，导致 argparse.print_help 等符号未链接进 exe。
5114 lines
280 KiB
Python
5114 lines
280 KiB
Python
from __future__ import annotations
|
||
|
||
import sys
|
||
import os
|
||
import re
|
||
import shutil
|
||
import subprocess
|
||
import vast as ast
|
||
import traceback
|
||
import json
|
||
import pickle
|
||
import time
|
||
import cProfile
|
||
from collections import OrderedDict
|
||
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed, wait, FIRST_COMPLETED
|
||
from lib.core.VLogger import get_logger as _vlog
|
||
from lib.constants.config import mode as _config_mode
|
||
from lib.core.Handles.HandlesBase import CTypeInfo
|
||
from lib.core.SymbolData import ModuleSymbols
|
||
from lib.core import TypeSystem
|
||
from lib.core.SymbolUtils import ExpandSoftKeywords, DefineDeclIntTypeFromAnnotation
|
||
from lib.core.SymbolUtils import ClassHasFuncLevelCVTable, IsVtableSlotMethod
|
||
from lib.Projectrans.Utils import compute_sha1, find_reachable_files_from_entries, topological_sort_files, parse_python_file
|
||
from lib.Projectrans.Phase1Generator import _TryGlobalCache, _SaveToGlobalCache, _GLOBAL_CACHE_DIR
|
||
import TransPyC
|
||
from StubGen import PythonToStubConverter
|
||
from lib.core.LLVMCG.IRMetadataFix import SanitizeIRMetadata
|
||
|
||
|
||
class ReportedCompileError(RuntimeError):
|
||
"""主进程已输出完整编译错误（含源码定位）的终止信号。
|
||
|
||
该异常只用于"错误已经打印过"的路径：main()/run() 抛出后，
|
||
顶层入口（Projectrans.py）不得再次打印错误内容，直接以失败退出码结束，
|
||
否则同一错误会在 worker/主进程/顶层各出现一遍。
|
||
"""
|
||
|
||
|
||
# PROF 热点：nt._path_isfile 2.6 万次 16.4s（Windows 磁盘 stat ~0.6ms/次）。
|
||
# 编译进程内文件状态稳定（obj/ll 一旦生成不删除、不被移动），用模块级缓存
|
||
# 避免重复 stat。仅用于"存在性检查"类调用（不用于内容变化的判断）。
|
||
_FILE_STAT_CACHE: dict[str, bool] = {}
|
||
|
||
|
||
# --traceback（Ctrl+C 调用栈）强制纳入编译+链接的 include 模块源文件名。
|
||
# 该模块对外只有 Install() 入口，且无任何用户代码 import 它 —— 若不强制纳入，
|
||
# 产物里既没有 Install 符号、也没有调用它的启动期构造器，功能静默失效。
|
||
# .vp 优先，.py 仅为向后兼容的回退名。
|
||
_TRACEBACK_MODULE_FILES: tuple[str, ...] = ('traceback.vp', 'traceback.py')
|
||
|
||
|
||
def _CachedIsFile(path: str) -> bool:
|
||
r: bool | None = _FILE_STAT_CACHE.get(path)
|
||
if r is None:
|
||
r = os.path.isfile(path)
|
||
_FILE_STAT_CACHE[path] = r
|
||
return r
|
||
|
||
|
||
# ---------------------------------------------------------------
|
||
# 文本读缓存（性能）：同一个文件在一次构建里被**多个阶段反复读**
|
||
# （prof 基线 PROF/20260930_224857：_io.open 12,740 次 / self 28.8s、
|
||
# read 11,902 次 / self 30.4s、TextIOWrapper.__exit__ 16.6s —— 128 个模块
|
||
# 却产生上万次读，主要是同一份 .stub.ll / .ll / deps.json 被
|
||
# _compile_one / _recombine_ll / 依赖解析等重复打开）。
|
||
# 键 = **路径**（不做 stat / 时间戳校验）。性能前提（用户口径）：TPC 单次运行内
|
||
# 非本进程写出的文件内容不变 —— 因此不需要"每次读都 stat 一次"来判新旧；
|
||
# 若再 stat，等于把省下的 1.2 万次 open 换成 1.2 万次 stat（仍是系统调用热点）。
|
||
# 正确性前提：**本进程写出/改写/删除过的文件，一律在该动作后调用
|
||
# _InvalidateTextCache**（写点见文件内全部 open(..., 'w') 与 os.remove 处）。
|
||
# 只缓存 newline=None 的默认换行翻译读，与直接 open(...).read() 语义逐字相同。
|
||
# 内存：按总字节预算 + LRU 淘汰；单文件超过 _TEXT_CACHE_FILE_MAX 不入缓存
|
||
# （合并后的 .ll 可达数十 MB，避免把内存吃光）。
|
||
# ---------------------------------------------------------------
|
||
_TEXT_CACHE_MAX_BYTES: int = 64 * 1024 * 1024
|
||
_TEXT_CACHE_FILE_MAX: int = 32 * 1024 * 1024
|
||
_TextReadCache: OrderedDict[str, str] = OrderedDict()
|
||
_TextReadCacheBytes: int = 0
|
||
|
||
|
||
def _ReadTextCached(path: str, newline: str | None = None) -> str:
|
||
"""读文本文件（按路径缓存，零 stat）；newline 非 None 时退化为直接读（不进缓存）。"""
|
||
global _TextReadCacheBytes
|
||
if newline is not None:
|
||
# 逐字节保真的读（newline='' 等）语义特殊：不进缓存，避免与默认换行翻译混用
|
||
with open(path, 'r', encoding='utf-8', newline=newline) as _f:
|
||
return _f.read()
|
||
_hit: str | None = _TextReadCache.get(path)
|
||
if _hit is not None:
|
||
_TextReadCache.move_to_end(path)
|
||
return _hit
|
||
with open(path, 'r', encoding='utf-8') as _f:
|
||
_text: str = _f.read()
|
||
if len(_text) <= _TEXT_CACHE_FILE_MAX:
|
||
_TextReadCache[path] = _text
|
||
_TextReadCacheBytes += len(_text)
|
||
while _TextReadCacheBytes > _TEXT_CACHE_MAX_BYTES and len(_TextReadCache) > 1:
|
||
_, _old = _TextReadCache.popitem(last=False)
|
||
_TextReadCacheBytes -= len(_old)
|
||
return _text
|
||
|
||
|
||
def _InvalidateTextCache(path: str) -> None:
|
||
"""丢弃 path 的文本读缓存条目：本进程**写出/改写/删除**该文件后必须调用。
|
||
|
||
这是本缓存唯一的正确性保证（读侧不再 stat 判新旧）：漏掉任一写点，后续读会
|
||
拿到该路径的旧文本。典型必失效场景：先读 → 同一 tick 内原地改写 → 再读，
|
||
且改写前后字节数相同（实例：_compile_one 前的 type 定义重排，行集合不变仅次序变；
|
||
_recombine_ll 的旧 sha1 前缀替换，16 位 hex 等长替换）。
|
||
"""
|
||
global _TextReadCacheBytes
|
||
_old = _TextReadCache.pop(path, None)
|
||
if _old is not None:
|
||
_TextReadCacheBytes -= len(_old)
|
||
|
||
|
||
_ASAN_DLL_A = 'libclang_rt.asan_dynamic-x86_64.dll.a'
|
||
|
||
|
||
def _FindAsanRuntime() -> tuple[str, str] | None:
|
||
"""通过 PATH 定位 clang64 的 ASan 运行库（按 LLVM 大版本取最新的那个）
|
||
|
||
mingw64 的 clang 只为 x86_64-w64-windows-gnu 目标提供 builtins / fuzzer /
|
||
profile / orc 运行库，**没有 ASan runtime**（<resource-dir>/lib/
|
||
x86_64-w64-windows-gnu/ 是空目录），所以不能靠 `-fsanitize=address` 让驱动
|
||
自动去找 —— 那只会得到
|
||
"could not open .../libclang_rt.asan_dynamic.dll.a: No such file or directory"。
|
||
clang64(MSVC ABI) 那边才有 libclang_rt.asan_dynamic-x86_64.dll.a 与
|
||
libclang_rt.asan_dynamic_runtime_thunk-x86_64.a，只能借用这两份。
|
||
|
||
路径**不再写死**：遍历 PATH 的每个目录 E，按 `E` 与 `dirname(E)` 两种可能取 base，
|
||
拼出 `<base>/lib/clang/<大版本>/lib/windows`，命中即为链接库目录；对应的运行期
|
||
DLL 目录取 `<base>/bin`。
|
||
|
||
Returns:
|
||
(链接库目录, 运行期 DLL 目录)；都找不到返回 None
|
||
"""
|
||
hits: list[tuple[str, str]] = []
|
||
seen: set[str] = set()
|
||
for entry in os.environ.get('PATH', '').split(os.pathsep):
|
||
if not entry:
|
||
continue
|
||
for base in (entry, os.path.dirname(entry)):
|
||
if not base:
|
||
continue
|
||
root = os.path.join(base, 'lib', 'clang')
|
||
if root in seen or not os.path.isdir(root):
|
||
continue
|
||
seen.add(root)
|
||
for name in os.listdir(root):
|
||
cand = os.path.join(root, name, 'lib', 'windows')
|
||
if os.path.isfile(os.path.join(cand, _ASAN_DLL_A)):
|
||
hits.append((cand, os.path.join(base, 'bin')))
|
||
if not hits:
|
||
return None
|
||
|
||
def _Ver(item: tuple[str, str]) -> int:
|
||
# 目录名是 LLVM 大版本号（如 "22"）——按数值比，否则 "9" 会排在 "22" 之后
|
||
v = item[0].split(os.sep)[-3]
|
||
return int(v) if v.isdigit() else 0
|
||
|
||
hits.sort(key=_Ver)
|
||
return hits[-1]
|
||
|
||
|
||
# ---------- --traceback：地址→源行 旁文件（<exe>.stackmap）----------
|
||
#
|
||
# 开启 --traceback 时，链接成功后用 llvm-dwarfdump 解析最终 PE 的
|
||
# .debug_line/.debug_info，把已有的"函数级/语句级 DWARF 行标记"导出成
|
||
# <exe>.stackmap，供 exe 内运行时处理器在 Ctrl+C/Ctrl+Break 时把 CRT 到当前
|
||
# 的返回地址逐帧翻译回 ".vp:行/函数"。以下函数只在开关开启时被调用；未开启时
|
||
# 不引入任何新依赖、任何新行为。
|
||
|
||
_PE_MAGIC: bytes = b'PE\0\0'
|
||
|
||
|
||
def _ReadPEImageLayout(exe_path: str) -> tuple[int, int, dict[str, tuple[int, int]]] | None:
|
||
"""读取 PE 的 (ImageBase, SizeOfImage, 节表) ；非 PE / 解析失败返回 None。
|
||
|
||
节表为 {节名: (节 RVA, 节 VirtualSize)}。写端需要这三样才能把 DWARF 地址严格
|
||
归一到"运行期 RVA"（运行期 traceback.vp 用 `帧地址 − GetModuleHandleA(NULL)`）：
|
||
* 合法行 ⇔ 地址落在映像 VA 区间 [ImageBase, ImageBase + SizeOfImage)；
|
||
* "运行期可能出现的 RVA 区间"的粗判据 = [.text 节 RVA, SizeOfImage)。
|
||
（原 _ReadPEImageBase 只返回 ImageBase，不足以做上述判定，故扩展为本函数。）
|
||
"""
|
||
import struct
|
||
try:
|
||
with open(exe_path, 'rb') as f:
|
||
head: bytes = f.read(0x1000)
|
||
except Exception:
|
||
return None
|
||
if len(head) < 0x40 or head[:2] != b'MZ':
|
||
return None
|
||
e_lfanew: int = struct.unpack_from('<I', head, 0x3C)[0]
|
||
if e_lfanew + 24 > len(head) or head[e_lfanew:e_lfanew + 4] != _PE_MAGIC:
|
||
return None
|
||
coff_off: int = e_lfanew + 4
|
||
n_sections: int = struct.unpack_from('<H', head, coff_off + 2)[0]
|
||
opt_size: int = struct.unpack_from('<H', head, coff_off + 16)[0]
|
||
opt_off: int = coff_off + 20 # COFF 文件头固定 20 字节
|
||
if opt_off + 32 > len(head):
|
||
return None
|
||
magic: int = struct.unpack_from('<H', head, opt_off)[0]
|
||
if magic == 0x20B: # PE32+
|
||
image_base: int = struct.unpack_from('<Q', head, opt_off + 24)[0]
|
||
elif magic == 0x10B: # PE32
|
||
image_base = struct.unpack_from('<I', head, opt_off + 28)[0]
|
||
else:
|
||
return None
|
||
# SizeOfImage 在两种可选头里都在 +56（PE32 多出的 BaseOfData 在 ImageBase 之前）
|
||
size_of_image: int = struct.unpack_from('<I', head, opt_off + 56)[0]
|
||
sections: dict[str, tuple[int, int]] = {}
|
||
sec_off: int = opt_off + opt_size
|
||
for i in range(n_sections):
|
||
base: int = sec_off + i * 40
|
||
if base + 40 > len(head):
|
||
break
|
||
name: str = head[base:base + 8].split(b'\x00', 1)[0].decode('ascii', 'replace')
|
||
sections[name] = (struct.unpack_from('<I', head, base + 12)[0], # VirtualAddress = 节 RVA
|
||
struct.unpack_from('<I', head, base + 8)[0]) # VirtualSize
|
||
return image_base, size_of_image, sections
|
||
|
||
|
||
# llvm-dwarfdump --debug-line 行表；行记录列形如
|
||
# Address Line Column File ISA Discriminator Flags
|
||
# 0x0000000140001000 12 0 1 0 0 is_stmt
|
||
_DWARF_LINE_HEADER_RE = re.compile(r'^Address\s+Line\s+Column\s+File\b')
|
||
_DWARF_LINE_ROW_RE = re.compile(r'^(0x[0-9A-Fa-f]+)\s+(\d+)\s+(\d+)\s+(\d+)\b')
|
||
_DWARF_FILENAMES_RE = re.compile(r'^\s*file_names\[\s*(\d+)\]\s*:')
|
||
_DWARF_DIRNAMES_RE = re.compile(r'^\s*(?:directory_names|include_directories)\[\s*(\d+)\]\s*[:=]\s*(.*)$')
|
||
_DWARF_NAME_RE = re.compile(r'^\s*name:\s*(.*)$')
|
||
_DWARF_DIRINDEX_RE = re.compile(r'^\s*dir_index:\s*(\d+)\s*$')
|
||
# llvm-dwarfdump --debug-info 里的 DISubprogram（DW_AT_low_pc/high_pc/name）
|
||
_DWARF_TAG_ANY_RE = re.compile(r'DW_TAG_\w+')
|
||
_DWARF_TAG_SUBPROGRAM_RE = re.compile(r'DW_TAG_subprogram\b')
|
||
_DWARF_LOWPC_RE = re.compile(r'DW_AT_low_pc\s*\(\s*(0x[0-9A-Fa-f]+)\s*\)')
|
||
_DWARF_HIGHPC_RE = re.compile(r'DW_AT_high_pc\s*\(\s*(0x[0-9A-Fa-f]+)\s*\)')
|
||
_DWARF_ATNAME_RE = re.compile(r'DW_AT_name\s*\("((?:[^"\\]|\\.)*)"\)')
|
||
|
||
# ---------------------------------------------------------------
|
||
# `_split_ll` 的逐行正则**预编译**（性能：该函数对合并后的 .ll 逐行跑两遍，
|
||
# prof 实测 43.9万行级输入下 re 模块级函数的缓存查找（re._compile）达 1970 万次
|
||
# ≈ 7s，另有 3100 万次 re.Pattern.match；每次 `re.match(字面量, ...)` 都要做
|
||
# "模式串哈希 → 缓存查找"。预编译成模块级常量后省掉这层查找，且匹配路径更短。
|
||
# 语义与逐字写法完全一致（模式串一字未改）。
|
||
# ---------------------------------------------------------------
|
||
_RE_DEFINE_SHA1: re.Pattern[str] = re.compile(r'define\s+[^@]*@"?([a-f0-9]+)\.')
|
||
_RE_DEFINE_NAME: re.Pattern[str] = re.compile(r'define\s+[^@]*@"?([^"\s(]+)"?')
|
||
_RE_DECLARE_NAME: re.Pattern[str] = re.compile(r'declare\s+[^@]*@"?([^"\s(]+)"?')
|
||
_RE_LLVM_INTRINSIC: re.Pattern[str] = re.compile(r'@("?)llvm\.')
|
||
_RE_AT_NAME: re.Pattern[str] = re.compile(r'(@\S+)')
|
||
_RE_SHA1_16_DOT: re.Pattern[str] = re.compile(r'^[a-f0-9]{16}\.')
|
||
_RE_DEFINE_PREFIX: re.Pattern[str] = re.compile(r'^define\s+')
|
||
_RE_DEFINE_LINKAGE: re.Pattern[str] = re.compile(
|
||
r'\b(linkonce_odr|weak_odr|linkonce|weak|common|appending|internal|private)\s+')
|
||
_RE_ALWAYSINLINE: re.Pattern[str] = re.compile(r'\balwaysinline\s+')
|
||
# `_compile_one` 的逐行符号扫描/全局去重里用到的两条（同一预编译理由）
|
||
_RE_AT_EXTERNAL: re.Pattern[str] = re.compile(r'@\S+\s*=\s*external\s+')
|
||
_RE_DECLARE_FULL: re.Pattern[str] = re.compile(
|
||
r'^declare\s+(?:dso_local\s+)?[^@]*@\"?([^"\s\(]+)\"?\s*\(')
|
||
|
||
# 其余字面量正则的模块级预编译（同一性能理由：省掉每次调用的"模式串哈希 →
|
||
# re 缓存查找"；模式串一字未改，flags 原样保留）。凡模式串不同或 flags 不同
|
||
# （有/无 re.MULTILINE）之处都各自独立成常量，保证匹配语义逐字不变。
|
||
# `_AddWeakComdat`：定位函数头第一个元数据（`!...`）附着点 / 全局定义尾部属性插入点
|
||
_RE_METADATA_ATTACH: re.Pattern[str] = re.compile(r'\s![\w.]')
|
||
_RE_COMDAT_TAIL_ATTRS: re.Pattern[str] = re.compile(r',\s*(?:align|section|partition|code_model)\b')
|
||
# LLVM 22+ external global 带初值 → linkonce（带行首锚 + MULTILINE）
|
||
_RE_EXTERNAL_GLOBAL_VALUE: re.Pattern[str] = re.compile(
|
||
r'^(@[\w.]+)\s*=\s*external\s+global\s+(\w[\w*]*)\s+(-?\d+|null|zeroinitializer|true|false)$',
|
||
re.MULTILINE)
|
||
# 同上但**不带行首锚**（opt 内联后整段内容替换处）
|
||
_RE_EXTERNAL_GLOBAL_VALUE_PLAIN: re.Pattern[str] = re.compile(
|
||
r'(@[\w.]+)\s*=\s*external\s+global\s+(\w[\w*]*)\s+(-?\d+|null|zeroinitializer|true|false)$',
|
||
re.MULTILINE)
|
||
# define/declare 后跟显式链接性关键字 → 其后补 dso_local
|
||
_RE_DEFINE_DECLARE_LINKAGE: re.Pattern[str] = re.compile(
|
||
r'^(define|declare)\s+(private|internal|available_externally|linkonce|weak|common|appending|extern_weak|linkonce_odr|weak_odr|external)\s+',
|
||
re.MULTILINE)
|
||
# define/declare 后无链接性/dso_local（含 `@` 分支）→ 直接补 dso_local
|
||
_RE_DEFINE_DECLARE_NO_LINKAGE_AT: re.Pattern[str] = re.compile(
|
||
r'^(define|declare)\s+(?!private|internal|available_externally|linkonce|weak|common|appending|extern_weak|linkonce_odr|weak_odr|external|dso_local|@)',
|
||
re.MULTILINE)
|
||
# 同上但**无 `@` 分支**（traceback include 模块专用）
|
||
_RE_DEFINE_DECLARE_NO_LINKAGE: re.Pattern[str] = re.compile(
|
||
r'^(define|declare)\s+(?!private|internal|available_externally|linkonce|weak|common|appending|extern_weak|linkonce_odr|weak_odr|external|dso_local)',
|
||
re.MULTILINE)
|
||
# 全局定义（非 llvm.*）带链接性关键字 → 补 dso_local（无 MULTILINE）
|
||
_RE_GLOBAL_LINKAGE: re.Pattern[str] = re.compile(
|
||
r'(@(?!llvm\.)[\w.]+)\s*=\s*(private|internal|available_externally|linkonce|weak|common|appending|extern_weak|linkonce_odr|weak_odr)\s+(global|constant)')
|
||
# 同上但带 MULTILINE
|
||
_RE_GLOBAL_LINKAGE_MULTILINE: re.Pattern[str] = re.compile(
|
||
r'(@(?!llvm\.)[\w.]+)\s*=\s*(private|internal|available_externally|linkonce|weak|common|appending|extern_weak|linkonce_odr|weak_odr)\s+(global|constant)',
|
||
re.MULTILINE)
|
||
# 全局定义无链接性 → 补 dso_local（无 MULTILINE）
|
||
_RE_GLOBAL_DSOLOCAL: re.Pattern[str] = re.compile(
|
||
r'(@[\w.]+)\s*=\s+(?!private|internal|available_externally|linkonce|weak|common|appending|extern_weak|linkonce_odr|weak_odr|dso_local)(global|constant)')
|
||
# 同上但带 MULTILINE（原文此处带行首锚 `^`：避免匹配到字符串常量内容里的 global/constant）
|
||
_RE_GLOBAL_DSOLOCAL_MULTILINE: re.Pattern[str] = re.compile(
|
||
r'^(@[\w.]+)\s*=\s+(?!private|internal|available_externally|linkonce|weak|common|appending|extern_weak|linkonce_odr|weak_odr|dso_local)(global|constant)',
|
||
re.MULTILINE)
|
||
# 收集已定义函数名（define 头）
|
||
_RE_DEFINE_HEAD: re.Pattern[str] = re.compile(
|
||
r'^define\s+(?:dso_local\s+)?[^@]*@\"?([^"\s\(]+)\"?\s*\(',
|
||
re.MULTILINE)
|
||
# 判定 IR 是否含 define 行（traceback 警告）
|
||
_RE_DEFINE_LINE: re.Pattern[str] = re.compile(r'^define ', re.MULTILINE)
|
||
|
||
|
||
def _SourceHasRealDef(src_path: str) -> bool:
|
||
"""模块是否含有真正的 `def`（而不是只有 `extern def`）。
|
||
|
||
extern-only 模块（如 includes/win32/win32console.vp）里的 define 是编译器合成的转发体，
|
||
没有源级函数体，本来就不会有 DWARF 行表；对它们发 "--traceback 未携带 DWARF" 属于误报。
|
||
读不到源码时返回 True，保持旧行为（宁可多提示，不漏提示）。
|
||
"""
|
||
try:
|
||
content, _ = parse_python_file(src_path)
|
||
except Exception:
|
||
return True
|
||
if not content:
|
||
return True
|
||
for raw in content.splitlines():
|
||
s: str = raw.lstrip()
|
||
if s.startswith('def ') or s.startswith('async def '):
|
||
return True
|
||
return False
|
||
|
||
|
||
def _ParseDwarfLineRows(dump_text: str) -> list[tuple[int, str, int]]:
|
||
"""从 llvm-dwarfdump 输出解析出 [(地址, 文件绝对路径, 行号)]。
|
||
|
||
逐个 CU（debug_line[0x..]）重建目录/文件表；行记录过滤掉 line==0 的
|
||
end_sequence 行。文件路径优先用文件表里的绝对 name；name 为相对时按 dir_index
|
||
指向的目录拼接（DWARF v5 取 directory_names，v4 取 include_directories）。
|
||
|
||
⚠ 地址口径：**原样返回 dwarfdump 打印的地址**（本例为含 ImageBase 的 VA
|
||
0x14000xxxx，但被链接器按 COMDAT 去重丢弃的泛型副本其地址恰为 0）。
|
||
本函数**不**做"整表 VA/RVA"的判定——那正是旧实现失真的根因（全局最小值一旦为
|
||
0，整表都不减基址）。归一化（逐条 `地址 − ImageBase`，并丢弃不能归一的条目）
|
||
由调用方 `_WriteStackMap` 依 PE.ImageBase/SizeOfImage 完成。
|
||
"""
|
||
rows: list[tuple[int, str, int]] = []
|
||
dirs: dict[int, str] = {}
|
||
files: dict[int, tuple[str, int]] = {}
|
||
pending_kind: str | None = None
|
||
pending_idx: int = -1
|
||
pending_name: str | None = None
|
||
pending_dir_index: int = 0
|
||
in_line_section: bool = False
|
||
|
||
def _Unquote(text: str) -> str:
|
||
text = text.strip()
|
||
if len(text) >= 2 and text[0] == '"' and text[-1] == '"':
|
||
text = text[1:-1]
|
||
return text
|
||
|
||
def _Commit() -> None:
|
||
nonlocal pending_kind, pending_name, pending_dir_index
|
||
if pending_kind == 'file' and pending_name is not None:
|
||
files[pending_idx] = (pending_name, pending_dir_index)
|
||
pending_kind = None
|
||
pending_name = None
|
||
pending_dir_index = 0
|
||
|
||
def _Resolve(idx: int) -> str:
|
||
entry = files.get(idx)
|
||
if entry is None:
|
||
return '??'
|
||
name, dir_index = entry
|
||
if os.path.isabs(name):
|
||
return name
|
||
base: str = dirs.get(dir_index, '')
|
||
if base and not os.path.isabs(base):
|
||
root: str = dirs.get(0, '')
|
||
if root and os.path.isabs(root):
|
||
base = os.path.join(root, base)
|
||
return os.path.normpath(os.path.join(base, name)) if base else name
|
||
|
||
for line in dump_text.splitlines():
|
||
stripped: str = line.strip()
|
||
if not stripped:
|
||
continue
|
||
if 'debug_line contents' in stripped:
|
||
in_line_section = True
|
||
continue
|
||
if 'contents:' in stripped and 'debug_' in stripped:
|
||
in_line_section = False
|
||
continue
|
||
if not in_line_section:
|
||
continue
|
||
if stripped.startswith('debug_line['):
|
||
_Commit()
|
||
dirs = {}
|
||
files = {}
|
||
continue
|
||
m = _DWARF_FILENAMES_RE.match(line)
|
||
if m:
|
||
_Commit()
|
||
pending_kind = 'file'
|
||
pending_idx = int(m.group(1))
|
||
continue
|
||
m = _DWARF_DIRNAMES_RE.match(line)
|
||
if m:
|
||
_Commit()
|
||
value: str = _Unquote(m.group(2))
|
||
if value:
|
||
dirs[int(m.group(1))] = value
|
||
else:
|
||
pending_kind = 'dir'
|
||
pending_idx = int(m.group(1))
|
||
continue
|
||
m = _DWARF_NAME_RE.match(line)
|
||
if m and pending_kind is not None:
|
||
name: str = _Unquote(m.group(1))
|
||
if pending_kind == 'file':
|
||
pending_name = name
|
||
else:
|
||
dirs[pending_idx] = name
|
||
pending_kind = None
|
||
continue
|
||
m = _DWARF_DIRINDEX_RE.match(line)
|
||
if m and pending_kind == 'file':
|
||
pending_dir_index = int(m.group(1))
|
||
continue
|
||
if _DWARF_LINE_HEADER_RE.match(line):
|
||
_Commit()
|
||
continue
|
||
m = _DWARF_LINE_ROW_RE.match(line)
|
||
if m:
|
||
_Commit()
|
||
row_line: int = int(m.group(2))
|
||
if row_line > 0:
|
||
rows.append((int(m.group(1), 16), _Resolve(int(m.group(4))), row_line))
|
||
return rows
|
||
|
||
|
||
def _ParseDwarfSubprograms(dump_text: str) -> list[tuple[int, int, str]]:
|
||
"""从 llvm-dwarfdump --debug-info 输出解析 [(low_pc, high_pc, 函数名)]。
|
||
|
||
DW_AT_high_pc 有两种形态且 dwarfdump 都打成 `(0x…)`：`DW_FORM_addr`（绝对
|
||
结束地址，值 > low_pc）与常量形式（相对 low_pc 的**长度**，值通常 < low_pc）。
|
||
本函数以 `high_pc <= low_pc` 判定后者并按 `low_pc + 长度` 还原；缺 high_pc 时
|
||
退化为 [low_pc, low_pc + 1)。
|
||
|
||
⚠ 地址口径同 `_ParseDwarfLineRows`：**原样返回**，不做整表 VA/RVA 判定；被
|
||
链接器 COMDAT 去重丢弃的泛型副本其 low_pc 恰为 0（实测 viperc.exe 有 756 个），
|
||
由调用方 `_WriteStackMap` 逐条归一化并丢弃。
|
||
"""
|
||
result: list[tuple[int, int, str]] = []
|
||
cur_low: int | None = None
|
||
cur_high: int | None = None
|
||
cur_name: str | None = None
|
||
in_prog: bool = False
|
||
|
||
def _Flush() -> None:
|
||
nonlocal cur_low, cur_high, cur_name, in_prog
|
||
if in_prog and cur_low is not None and cur_name:
|
||
hi: int = cur_high if cur_high is not None else cur_low + 1
|
||
if hi <= cur_low:
|
||
hi = cur_low + cur_high # high_pc 为 offset 形式
|
||
if hi > cur_low:
|
||
result.append((cur_low, hi, cur_name))
|
||
cur_low = None
|
||
cur_high = None
|
||
cur_name = None
|
||
in_prog = False
|
||
|
||
for line in dump_text.splitlines():
|
||
if _DWARF_TAG_ANY_RE.search(line):
|
||
if _DWARF_TAG_SUBPROGRAM_RE.search(line):
|
||
_Flush()
|
||
in_prog = True
|
||
else:
|
||
_Flush()
|
||
continue
|
||
if not in_prog:
|
||
continue
|
||
m = _DWARF_LOWPC_RE.search(line)
|
||
if m:
|
||
cur_low = int(m.group(1), 16)
|
||
continue
|
||
m = _DWARF_HIGHPC_RE.search(line)
|
||
if m:
|
||
cur_high = int(m.group(1), 16)
|
||
continue
|
||
m = _DWARF_ATNAME_RE.search(line)
|
||
if m and cur_name is None:
|
||
cur_name = m.group(1)
|
||
_Flush()
|
||
return result
|
||
|
||
|
||
# 显式并行度覆盖：TPC_JOBS=N（N>=1）——使用者明确指定 worker 数并**自行承担**
|
||
# 内存风险（含 N=1 即显式接受串行）。设了它就不再走内存校验/预分配。
|
||
_JOBS_ENV: str = 'TPC_JOBS'
|
||
# 已通过"预分配校验"的预算（MB）：同一预算不重复校验（翻译/编译两阶段各查一次）
|
||
_MemProbeDone: set[int] = set()
|
||
# 预分配校验的分块大小（MB）：分块提交并**全部持有**到校验结束（峰值 = 预算），
|
||
# 结束后立即整体释放 —— 紧接其后就创建 worker 进程，OS 直接复用这批刚释放的页。
|
||
_MEM_PROBE_CHUNK_MB: int = 256
|
||
# 自动并行度上限（用户 2026-10-01 定：16 worker 太多）。只约束**自动**取值，
|
||
# 显式 TPC_JOBS=N 仍按硬件/文件数上限生效。
|
||
_MAX_WORKERS: int = 8
|
||
|
||
|
||
def _MemStatus() -> tuple[int, int]:
|
||
"""(可用物理内存 MB, 物理内存总量 MB)；读取失败返回 (-1, -1)，调用方按"未知"处理"""
|
||
try:
|
||
import ctypes
|
||
|
||
class _MemStat(ctypes.Structure):
|
||
_fields_ = [
|
||
("dwLength", ctypes.c_ulong),
|
||
("dwMemoryLoad", ctypes.c_ulong),
|
||
("ullTotalPhys", ctypes.c_ulonglong),
|
||
("ullAvailPhys", ctypes.c_ulonglong),
|
||
("ullTotalPageFile", ctypes.c_ulonglong),
|
||
("ullAvailPageFile", ctypes.c_ulonglong),
|
||
("ullTotalVirtual", ctypes.c_ulonglong),
|
||
("ullAvailVirtual", ctypes.c_ulonglong),
|
||
("ullAvailExtendedVirtual", ctypes.c_ulonglong),
|
||
]
|
||
|
||
stat = _MemStat()
|
||
stat.dwLength = ctypes.sizeof(_MemStat)
|
||
if ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(stat)):
|
||
return (stat.ullAvailPhys // (1024 * 1024),
|
||
stat.ullTotalPhys // (1024 * 1024))
|
||
except Exception:
|
||
pass
|
||
return (-1, -1)
|
||
|
||
|
||
def _ProbeReserveMb(mb: int) -> bool:
|
||
"""预分配校验：一次性提交 mb MB（分块持有后整体释放），成功返回 True。
|
||
|
||
为什么要"预分配"而不是只读 ullAvailPhys：可用物理内存含大量**可回收**的
|
||
缓存页/其他进程随时会吃掉的余量，只按它决策会出现"这一秒够、真跑起来不够"
|
||
的波动（编译时间在 400s / 1180s 两档之间随机跳的直接来源）。先真正把预算
|
||
拿到手（页级提交）再放掉，等于向 OS 确认过这笔预算确实可分配。
|
||
"""
|
||
if mb <= 0:
|
||
return True
|
||
Bufs = []
|
||
Left = mb
|
||
try:
|
||
while Left > 0:
|
||
Cur = min(_MEM_PROBE_CHUNK_MB, Left)
|
||
Bufs.append(bytearray(Cur * 1024 * 1024))
|
||
Left -= Cur
|
||
return True
|
||
except MemoryError:
|
||
return False
|
||
finally:
|
||
del Bufs
|
||
|
||
|
||
def _PickWorkerCount(n_files: int, per_worker_mb: int = 350) -> int:
|
||
"""并行 worker 数的**唯一**决策入口（CPU 上限 + 内存预分配双约束）。
|
||
|
||
口径（用户 2026-10-01 定，取代旧"内存紧就降级串行"）：
|
||
· **自动上限 `_MAX_WORKERS`=8**：16 worker 收益不足、内存压力翻倍（用户 2026-10-01 定）；
|
||
显式 TPC_JOBS 不受此限（只受硬件/文件数约束）。
|
||
1. **不降级**：内存不足**绝不**静默退回单 worker 串行 —— 旧实现 `by_mem =
|
||
max(1, avail_mb // per_worker_mb)` 在可用内存稍紧时就整体降为串行，
|
||
同一条命令 76 文件逐个翻译 1180s、并行 ~400s，时间随机跳 2-3 倍
|
||
（profile 实证：224857 那次主进程 0 次 _translate_file＝并行，438s；
|
||
231916/235354 两次主进程 75 次 _translate_file＝串行，1048s/1168s）。
|
||
2. **预分配 + 强制预留**：先按 base×per_worker_mb 做一次性提交校验
|
||
（_ProbeReserveMb），拿到就并行、拿不到就 fail-fast 报出"可用/需要"
|
||
数字，由使用者释放内存或**显式** TPC_JOBS=N 接受其它并行度。
|
||
3. 显式 TPC_JOBS=N 优先且跳过校验（使用者自行承担内存风险，N=1 即显式串行）。
|
||
|
||
单 worker 预估 per_worker_mb（Python 解释器 + llvmlite module + 共享符号表
|
||
快照）；实测翻译阶段 worker 稳态 ~110MB、峰值 ~300MB，故取 350（编译阶段
|
||
调用方传 400）。Windows 用 GlobalMemoryStatusEx 读物理内存。
|
||
"""
|
||
hw_max = min(os.cpu_count() or 4, n_files)
|
||
base = min(hw_max, _MAX_WORKERS)
|
||
if base <= 1:
|
||
return base
|
||
env_jobs: str | None = os.environ.get(_JOBS_ENV)
|
||
if env_jobs:
|
||
try:
|
||
want = int(env_jobs)
|
||
except ValueError:
|
||
want = base
|
||
# 显式指定只受硬件/文件数约束（不受 _MAX_WORKERS 限制），并跳过内存校验
|
||
want = max(1, min(want, hw_max))
|
||
_vlog().info(f" [并行] {_JOBS_ENV}={env_jobs} → workers={want}（显式指定，跳过校验）")
|
||
return want
|
||
avail_mb, total_mb = _MemStatus()
|
||
need_mb = base * per_worker_mb
|
||
if avail_mb < 0:
|
||
# 读不到内存信息：不阻塞编译（按上限并行），但必须显式告知
|
||
_vlog().warning(f" [并行] 读不到物理内存，取 workers={base}（预算 {need_mb}MB）")
|
||
return base
|
||
if avail_mb < need_mb:
|
||
raise RuntimeError(
|
||
f"[并行翻译] 内存不足：需 {need_mb}MB（{base}×{per_worker_mb}MB），"
|
||
f"可用 {avail_mb}MB/{total_mb}MB；释放内存或设 {_JOBS_ENV}=N 重跑")
|
||
if need_mb not in _MemProbeDone:
|
||
_t0: float = _time.time()
|
||
if not _ProbeReserveMb(need_mb):
|
||
raise RuntimeError(
|
||
f"[并行翻译] 预分配 {need_mb}MB 失败（{base}×{per_worker_mb}MB，"
|
||
f"可用 {avail_mb}MB/{total_mb}MB）；释放内存或设 {_JOBS_ENV}=N 重跑")
|
||
_MemProbeDone.add(need_mb)
|
||
_vlog().info(f" [并行] 预分配 {need_mb}MB 通过（{_time.time() - _t0:.2f}s）")
|
||
_vlog().info(f" [并行] workers={base}（预算 {need_mb}MB / 可用 {avail_mb}MB）")
|
||
return base
|
||
|
||
|
||
def _ExtractCDefineConstantsFromPyi(pyi_path: str, all_dc: dict[str, object],
|
||
symtab: object | None = None,
|
||
all_dt: dict[str, tuple[int, bool]] | None = None) -> None:
|
||
"""从 .pyi 文件中提取 define 常量到 all_dc。文件不存在时静默跳过。
|
||
|
||
治本修复：原实现只支持 ast.Constant 和 float() 调用，
|
||
无法处理 CUnsignedLong(-11)（ast.Call）和
|
||
FOREGROUND_GREEN | FOREGROUND_INTENSITY（ast.BinOp）等复杂表达式。
|
||
现使用 ConstEvaluator.eval_full 统一求值，支持：
|
||
- ast.Constant: 简单常量 (0x0002)
|
||
- ast.Call: 类型构造 (CUnsignedLong(-11))
|
||
- ast.BinOp: 位运算 (FOREGROUND_RED | FOREGROUND_GREEN)
|
||
- ast.UnaryOp: 一元运算 (-11)
|
||
- ast.Name: 引用其他 define 常量
|
||
"""
|
||
if not os.path.exists(pyi_path):
|
||
return
|
||
try:
|
||
# 复用 parse_python_file 的 AST 缓存（.pyi 文件被多处解析）
|
||
_, tree = parse_python_file(pyi_path)
|
||
if tree is None:
|
||
return
|
||
# 治本修复：使用 ConstEvaluator 处理复杂表达式
|
||
# 创建 mock Gen 对象，其 _define_constants 指向 all_dc，
|
||
# 使 ConstEvaluator 能查找同文件中已提取的常量（如 FOREGROUND_RED 引用 FOREGROUND_BLUE）
|
||
from lib.core.ConstEvaluator import ConstEvaluator, EvalContext
|
||
|
||
class _MockGen:
|
||
"""轻量 mock，仅提供 _define_constants 供 ConstEvaluator 查找"""
|
||
def __init__(self) -> None:
|
||
self._define_constants: dict[str, object] = all_dc
|
||
|
||
mock_gen: _MockGen = _MockGen()
|
||
ctx: EvalContext = EvalContext(Gen=mock_gen)
|
||
|
||
for node in ast.iter_child_nodes(tree):
|
||
if isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name):
|
||
Markers: tuple[str, ...] = (tuple(getattr(node, 'Markers', None) or ())
|
||
+ tuple(getattr(node.annotation, 'Markers', None) or ()))
|
||
# 新语法 `define NAME = v` 由解析器写入 Markers。
|
||
if 'define' in Markers and node.value:
|
||
val: object = ConstEvaluator.eval_full(node.value, ctx)
|
||
if val is None and isinstance(node.value, ast.Call) and len(node.value.args) == 1:
|
||
# 类型构造的**裸别名**形态（如 `DWORD(-11)`，非 `CUnsignedLong(-11)`）：
|
||
# eval_full 只认 `X(...)` / `from t import X`（见 ConstEvaluator
|
||
# _eval_compile_time_call），裸 typedef 别名既不在 _t_module 也不在
|
||
# mock ctx 的 _builtin_type_symbols 里 → 返回 None → 该常量被整条丢弃。
|
||
# 现场：win32file.vp 的 STD_INPUT/OUTPUT/ERROR_HANDLE 是全仓**唯一**用
|
||
# 别名构造写的 define → 唯独这三个进不了 _all_define_constants，导致
|
||
# 模块属性访问 `wf.STD_ERROR_HANDLE` 解析落空、退化成未定义的全局符号
|
||
# （链接失败）。define 关心的是常量值，故回退求值其唯一实参（位型一致）。
|
||
val = ConstEvaluator.eval_full(node.value.args[0], ctx)
|
||
if val is not None and isinstance(val, (int, float, str)):
|
||
all_dc[node.target.id] = val
|
||
# 带类型注册：记录该 define 的**声明整数类型**（如 `DWORD` → (32, False)），
|
||
# 供使用点按声明类型物化（u32 用 i32 + 位型，而不是按值大小猜 i32/i64）。
|
||
if all_dt is not None and isinstance(val, int) and not isinstance(val, bool):
|
||
decl_type: tuple[int, bool] | None = DefineDeclIntTypeFromAnnotation(symtab, node.annotation)
|
||
if decl_type is not None:
|
||
all_dt[node.target.id] = decl_type
|
||
except Exception as e:
|
||
_vlog().warning(f"Phase2Translator: 忽略异常 {e}", exc_info=e)
|
||
|
||
|
||
def _IsExportExternFuncNode(node: ast.FunctionDef) -> bool:
|
||
"""函数是否导出/外部可见：读 `export def` / `extern def` 的 AST 标记与 Markers。"""
|
||
if bool(getattr(node, 'Export', False)) or bool(getattr(node, 'Extern', False)):
|
||
return True
|
||
Markers: tuple[str, ...] = getattr(node, 'Markers', None) or ()
|
||
return 'export' in Markers or 'extern' in Markers or 'state' in Markers
|
||
|
||
|
||
def _CountUnresolvedTypedefs(module_symbols: object) -> int:
|
||
"""该模块"有值注解却解析不出类型"的 typedef 个数（供声明阶段判断是否要下一轮）。
|
||
|
||
typedef 右侧可以引用**其它模块**的 typedef：例 win32base 的
|
||
`typedef HANDLE = VOIDPTR`（VOIDPTR 在 stdint）——本模块先 stage 时右侧
|
||
解析为空（original_type_kind/original_class 均为 None），要等依赖模块入表后
|
||
重 stage 才能补全。返回 0 表示本模块声明已完整（不再需要重跑）。
|
||
"""
|
||
miss: int = 0
|
||
for td in getattr(module_symbols, 'typedefs', []):
|
||
if getattr(td, 'value', None) is None and getattr(td, 'annotation', None) is None:
|
||
continue
|
||
if getattr(td, 'original_type_kind', None) is None and getattr(td, 'original_class', None) is None:
|
||
miss += 1
|
||
return miss
|
||
|
||
|
||
def _parallel_translate_worker(src_path: str, out_path: str, src_root: str, temp_dir: str, output_dir: str,
|
||
include_dirs: list[str], triple: str | None, datalayout: str | None,
|
||
sha1_map: dict[str, str], sig_files: dict[str, str],
|
||
stub_files: dict[str, str], include_py_map: dict[str, str], slice_level: int,
|
||
shared_sym_pickle_path: str, struct_sha1_map: dict[str, str],
|
||
prebuilt_ModuleSha1Map: dict[str, str] | None = None,
|
||
hotspot_sites_file: str | None = None,
|
||
hotspot_sites_all: bool = False,
|
||
hotspot_callees: list[str] | None = None,
|
||
traceback_enabled: bool = False,
|
||
dwarf_enabled: bool = False) -> tuple[str, str, str]:
|
||
# 注意：不可跨任务复用 Phase2Translator/共享符号表！_translate_file 编译过程中
|
||
# 会向共享 SymbolTable 写入符号（import insert/类注册等），复用导致前文件的
|
||
# 符号污染后文件 → 跨模块 struct 类型前缀错位（如 7cadfe07e983fd90.CompilerFactory
|
||
# 混入 _list 前缀，Type of #1 arg mismatch）。每任务必须从 pickle 反序列化出
|
||
# 独立副本（原实现；worker 复用省掉的 pickle.load 时间仅 ~0.5s/次，远小于
|
||
# worker 内翻译计算，不值得以正确性为代价）。
|
||
try:
|
||
trans = Phase2Translator(src_root, temp_dir, output_dir,
|
||
compile_cmd='llc', include_dirs=include_dirs,
|
||
target_triple=triple, target_datalayout=datalayout,
|
||
build_pyi_cache=False)
|
||
trans.sha1_map = sha1_map
|
||
trans.sig_files = sig_files
|
||
trans.stub_files = stub_files
|
||
trans.include_py_map = include_py_map
|
||
trans.slice_level = slice_level
|
||
trans.struct_sha1_map = struct_sha1_map
|
||
# 调用点热点统计与每函数计时（--hotspot-sites）：worker 子进程无父进程属性，
|
||
# 须显式透传（None=未启用，零影响）；生成 IR 时各模块处理器统一追加写最终 <path>
|
||
trans.hotspot_sites_file = hotspot_sites_file
|
||
trans.hotspot_sites_all = hotspot_sites_all
|
||
trans.hotspot_callees = hotspot_callees
|
||
# Ctrl+C 调用栈（--traceback）：worker 子进程无父进程属性，须显式透传
|
||
# （False=未启用，零影响）；traceback 模块据此注入 Install() 启动期构造器
|
||
trans.traceback_enabled = traceback_enabled
|
||
# 函数级 DWARF（--traceback 时 include 模块也必须带行表，否则栈里落到
|
||
# includes/* 的帧取不到源行）：worker 子进程无父进程属性，须显式透传
|
||
# （False=未启用，零影响），仅用于 _translate_file 的 dwarf_enabled 形参
|
||
trans.dwarf_enabled = dwarf_enabled
|
||
|
||
if shared_sym_pickle_path and os.path.exists(shared_sym_pickle_path):
|
||
# 使用 pickle.load(f) 流式加载，避免 pickle.loads(f.read()) 一次性
|
||
# 将整个 pickle 文件读入内存（磁盘副本 + 反序列化对象同时占用内存）导致 MemoryError。
|
||
# 同时提高递归限制，防止深层嵌套对象反序列化失败。
|
||
sys.setrecursionlimit(max(sys.getrecursionlimit(), 100000))
|
||
with open(shared_sym_pickle_path, 'rb') as f:
|
||
shared_data = pickle.load(f)
|
||
_sym_bytes: bytes | None = shared_data.get('symbol_table_bytes')
|
||
if _sym_bytes is not None:
|
||
trans._shared_symbol_table = pickle.loads(_sym_bytes)
|
||
trans._shared_source_module_sig_files = shared_data['source_module_sig_files']
|
||
trans._shared_all_dc = shared_data['all_dc']
|
||
trans._shared_all_decl_types = shared_data.get('all_decl_types', {})
|
||
trans._shared_export_extern_funcs = shared_data['export_extern_funcs']
|
||
trans.inline_func_symbols = shared_data['inline_func_symbols']
|
||
trans.function_default_args = shared_data['function_default_args']
|
||
trans.function_param_names = shared_data.get('function_param_names', {})
|
||
trans._shared_generic_class_templates = shared_data.get('generic_class_templates', {})
|
||
trans._shared_doc_meta = shared_data.get('shared_doc_meta', {})
|
||
trans._shared_class_vtable = shared_data.get('class_vtable', set())
|
||
trans._shared_cross_module_vtable = shared_data.get('cross_module_vtable', set())
|
||
trans._shared_class_methods = shared_data.get('class_methods', {})
|
||
trans._shared_cross_module_novtable = shared_data.get('cross_module_novtable', set())
|
||
else:
|
||
trans._precollect_inline_symbols()
|
||
trans._build_shared_symbol_data()
|
||
|
||
trans._translate_file(src_path, out_path, prebuilt_ModuleSha1Map=prebuilt_ModuleSha1Map, dwarf_enabled=dwarf_enabled)
|
||
return ('ok', src_path, '')
|
||
except Exception as e:
|
||
# fail-fast：worker 不写 stderr，只把错误（含源码定位）返回主进程；
|
||
# 由主进程统一输出一次，避免同一错误打印两遍。
|
||
return ('error', src_path, f'{type(e).__name__}: {e}')
|
||
|
||
|
||
def _parallel_translate_batch(batch: list, src_root: str, temp_dir: str, output_dir: str,
|
||
include_dirs: list[str], triple: str | None, datalayout: str | None,
|
||
sha1_map: dict[str, str], sig_files: dict[str, str],
|
||
stub_files: dict[str, str], include_py_map: dict[str, str], slice_level: int,
|
||
shared_sym_pickle_path: str, struct_sha1_map: dict[str, str],
|
||
hotspot_sites_file: str | None = None,
|
||
hotspot_sites_all: bool = False,
|
||
hotspot_callees: list[str] | None = None,
|
||
dwarf_enabled: bool = False) -> tuple[list, list]:
|
||
"""批次子任务：worker 内创建一个 Phase2Translator + pickle.load 一次，顺序翻译批内全部文件。
|
||
|
||
batch 每项: (i, src_path, out_path, sha1, prebuilt_ModuleSha1Map)
|
||
|
||
防污染：SymbolTable 是编译期唯一被写的共享对象（import insert/类注册），
|
||
批内每个文件翻译前从 symbol_table_bytes 反序列化出干净副本，前文件的
|
||
符号写入不会残留到后文件（否则跨模块 struct 类型前缀错位）。
|
||
返回 (错误列表[(src_path, err)], batch, per_file_times[(src_path, elapsed, rel)]）。
|
||
错误只回传、不写 stderr：完整错误由主进程统一输出一次（含源码定位）。
|
||
"""
|
||
_current_src: str = batch[0][1] if batch else ''
|
||
per_file_times: list[tuple[str, float, str]] = []
|
||
try:
|
||
trans = Phase2Translator(src_root, temp_dir, output_dir,
|
||
compile_cmd='llc', include_dirs=include_dirs,
|
||
target_triple=triple, target_datalayout=datalayout,
|
||
build_pyi_cache=False)
|
||
trans.sha1_map = sha1_map
|
||
trans.sig_files = sig_files
|
||
trans.stub_files = stub_files
|
||
trans.include_py_map = include_py_map
|
||
trans.slice_level = slice_level
|
||
trans.struct_sha1_map = struct_sha1_map
|
||
# 调用点热点统计与每函数计时（--hotspot-sites）：批内每文件同样透传
|
||
trans.hotspot_sites_file = hotspot_sites_file
|
||
trans.hotspot_sites_all = hotspot_sites_all
|
||
trans.hotspot_callees = hotspot_callees
|
||
|
||
symbol_table_bytes: bytes | None = None
|
||
if shared_sym_pickle_path and os.path.exists(shared_sym_pickle_path):
|
||
sys.setrecursionlimit(max(sys.getrecursionlimit(), 100000))
|
||
with open(shared_sym_pickle_path, 'rb') as f:
|
||
shared_data = pickle.load(f)
|
||
symbol_table_bytes = shared_data.get('symbol_table_bytes')
|
||
if symbol_table_bytes is not None:
|
||
trans._shared_symbol_table = pickle.loads(symbol_table_bytes)
|
||
trans._shared_source_module_sig_files = shared_data['source_module_sig_files']
|
||
trans._shared_all_dc = shared_data['all_dc']
|
||
trans._shared_all_decl_types = shared_data.get('all_decl_types', {})
|
||
trans._shared_export_extern_funcs = shared_data['export_extern_funcs']
|
||
trans.inline_func_symbols = shared_data['inline_func_symbols']
|
||
trans.function_default_args = shared_data['function_default_args']
|
||
trans.function_param_names = shared_data.get('function_param_names', {})
|
||
trans._shared_generic_class_templates = shared_data.get('generic_class_templates', {})
|
||
trans._shared_doc_meta = shared_data.get('shared_doc_meta', {})
|
||
trans._shared_class_vtable = shared_data.get('class_vtable', set())
|
||
trans._shared_cross_module_vtable = shared_data.get('cross_module_vtable', set())
|
||
trans._shared_class_methods = shared_data.get('class_methods', {})
|
||
trans._shared_cross_module_novtable = shared_data.get('cross_module_novtable', set())
|
||
else:
|
||
trans._precollect_inline_symbols()
|
||
trans._build_shared_symbol_data()
|
||
|
||
batch_errors: list = []
|
||
for _i, src_path, out_path, _sha1, msm in batch:
|
||
_f0: float = time.time()
|
||
_current_src = src_path
|
||
# 每文件前重置 SymbolTable 为干净副本（防跨文件污染）
|
||
if symbol_table_bytes is not None:
|
||
trans._shared_symbol_table = pickle.loads(symbol_table_bytes)
|
||
trans._shared_symbol_table.translator = None
|
||
trans._translate_file(src_path, out_path, prebuilt_ModuleSha1Map=msm, dwarf_enabled=dwarf_enabled)
|
||
per_file_times.append((src_path, time.time() - _f0, os.path.relpath(src_path, src_root)))
|
||
return (batch_errors, batch, per_file_times)
|
||
except Exception as e:
|
||
# fail-fast：任意文件编译出错立即终止本批（不再翻译批内剩余文件）。
|
||
# worker 不写 stderr —— 错误（含源码定位）随返回值交给主进程统一输出一次。
|
||
_src: str = _current_src or (batch[0][1] if batch else '')
|
||
return ([(_src, f'{type(e).__name__}: {e}')], batch, per_file_times)
|
||
|
||
|
||
# ==================================================================
|
||
# 子进程性能剖析：主进程 cProfile 只能看到进程池等待（WaitForMultipleObjects/
|
||
# as_completed），看不到 worker 进程内部 CPU 分布（vast parse 等热点）。
|
||
# 固定启用（不设环境变量开关），worker 每个任务导出 cProfile dump
|
||
# （worker_prof_{pid}_{seq}.prof），主进程导出 main_prof_{pid}.prof + main_time.txt。
|
||
# 输出落在 PROF_DIR/<启动时间>/ 下的**本次编译专属目录**（2026-09-29 起）。
|
||
# ==================================================================
|
||
import functools as _functools
|
||
import time as _time
|
||
|
||
# 硬编码 PROF 根目录（2026-08-18 起不再用 TPC_WORKER_PROF 环境变量控制）
|
||
PROF_DIR: str = r'D:\Users\TermiNexus\Desktop\TransPyC\PROF'
|
||
|
||
# 本次编译的 run 目录：按启动时间在 PROF_DIR 下新建子目录，**不清理旧目录**
|
||
# （2026-09-29 用户要求保留历次 profile 以便跨次对比，旧行为是每次编译删掉
|
||
# PROF 根目录下全部 *.prof ⇒ 只能看最近一次）。
|
||
# 子目录名经环境变量传给 worker：ProcessPoolExecutor 用 spawn 起新解释器，
|
||
# 进程内模块态不共享；环境变量随子进程继承 ⇒ 同一次编译的所有 worker 落在
|
||
# 同一个 run 目录，不会各建各的目录。
|
||
_PROF_RUN_ENV: str = 'TPC_PROF_RUN_DIR'
|
||
_prof_run_dir: str | None = None
|
||
|
||
|
||
def StartProfRun() -> str:
|
||
"""每次编译开始时调用一次：创建本次编译专属的 PROF 子目录，返回其路径。
|
||
|
||
目录名 = PROF 根目录下 `YYYYmmdd_HHMMSS`（编译启动时刻）；同名（同一秒内
|
||
二次启动）补 `_N` 序号。创建失败时回落到 PROF 根目录，保证不因 profiling
|
||
影响编译本身。成功后把目录写进环境变量，供 worker 子进程复用。
|
||
"""
|
||
global _prof_run_dir
|
||
if _prof_run_dir:
|
||
return _prof_run_dir
|
||
_name: str = _time.strftime('%Y%m%d_%H%M%S')
|
||
_dir: str = os.path.join(PROF_DIR, _name)
|
||
_seq: int = 1
|
||
while os.path.isdir(_dir):
|
||
_seq += 1
|
||
_dir = os.path.join(PROF_DIR, f'{_name}_{_seq}')
|
||
try:
|
||
os.makedirs(_dir, exist_ok=True)
|
||
except OSError:
|
||
return PROF_DIR
|
||
_prof_run_dir = _dir
|
||
os.environ[_PROF_RUN_ENV] = _dir
|
||
return _dir
|
||
|
||
|
||
def GetProfRunDir() -> str:
|
||
"""当前进程写 profile 应落的目录。
|
||
|
||
主进程 = StartProfRun 选定的 run 目录；worker = 继承来的环境变量指向的目录。
|
||
两者都取不到（未先调 StartProfRun，如 worker 被单独拉起）时回落到 PROF 根目录，
|
||
宁可文件落在根目录也不丢 dump。
|
||
"""
|
||
global _prof_run_dir
|
||
if _prof_run_dir:
|
||
return _prof_run_dir
|
||
_env_dir: str = os.environ.get(_PROF_RUN_ENV, '')
|
||
if _env_dir:
|
||
_prof_run_dir = _env_dir
|
||
return _env_dir
|
||
return PROF_DIR
|
||
|
||
|
||
def _ProfileWorker(_fn):
|
||
@_functools.wraps(_fn)
|
||
def _wrapper(*args, **kwargs):
|
||
_prof = None
|
||
try:
|
||
# cProfile 默认不启用（环境变量 TPC_WORKER_PROF=1 开启）：
|
||
# Python 只允许一个 profiling tool 同时活跃，TPC 的 cProfile 与
|
||
# viztracer --log_multiprocess 冲突（cProfile.enable() 检测到已激活
|
||
# profiler 抛 "Another profiling tool is already active" → worker 崩溃
|
||
# → batch 异常 → 编译终止）。外部 profiler 活跃（sys.getprofile()
|
||
# 非 None）时即使开启也跳过让位；enable() 异常兜底吞掉保编译。
|
||
if os.environ.get('TPC_WORKER_PROF', '0') == '1' and sys.getprofile() is None:
|
||
_prof = cProfile.Profile()
|
||
try:
|
||
_prof.enable()
|
||
except RuntimeError:
|
||
_prof = None
|
||
return _fn(*args, **kwargs)
|
||
finally:
|
||
if _prof is not None:
|
||
_prof.disable()
|
||
_seq: int = getattr(_wrapper, '_seq', 0) + 1
|
||
_wrapper._seq = _seq
|
||
try:
|
||
_out_dir: str = GetProfRunDir()
|
||
if not os.path.isdir(_out_dir):
|
||
os.makedirs(_out_dir, exist_ok=True)
|
||
_out: str = os.path.join(_out_dir, f'worker_prof_{os.getpid()}_{_seq}.prof')
|
||
_prof.dump_stats(_out)
|
||
except Exception:
|
||
pass
|
||
|
||
return _wrapper
|
||
|
||
|
||
_parallel_translate_worker = _ProfileWorker(_parallel_translate_worker)
|
||
_parallel_translate_batch = _ProfileWorker(_parallel_translate_batch)
|
||
|
||
|
||
# ==================================================================
|
||
# worker pyi 缓存预热：ProcessPoolExecutor(spawn) 下每个 worker 重新
|
||
# import 模块后，HandlesImports.ImportHandle 的类级 _pyi_cache 全空，
|
||
# 每进程首次任务重新 vast.parse 全部依赖 .vpi/.pyi
|
||
# （cProfile: 4479 次 parse / 245s）。
|
||
# 方案：主进程把 temp 下全部 .vpi 的 vast AST 树 pickle 落盘
|
||
# （temp/_pyi_cache_v1.pkl，任务 B），worker initializer 启动时
|
||
# pickle.load 一次填充 _pyi_cache（任务 A）；load 失败回退逐文件
|
||
# vast.parse。文件系统共享即为"共用缓存"，不做跨进程共享内存（任务 C）。
|
||
# 环境变量 TPC_WORKER_WARMUP=0 可关闭（回归对比基线）。
|
||
# ==================================================================
|
||
_PYI_CACHE_PKL_NAME: str = '_pyi_cache_v1.pkl'
|
||
_PYI_CACHE_SIG_NAME: str = '_pyi_cache_v1.sig.json'
|
||
_PYI_CACHE_VERSION: int = 1
|
||
|
||
|
||
def _ScanPyiFiles(temp_dir: str) -> list[str]:
|
||
"""扫描 temp 目录下全部 .vpi/.pyi 文件，返回绝对路径列表（与
|
||
_TryLoadClassMembersFromPyi 的 _pyi_cache 键格式一致）。"""
|
||
try:
|
||
names: list[str] = os.listdir(temp_dir)
|
||
except OSError:
|
||
return []
|
||
paths: list[str] = []
|
||
for fname in sorted(names):
|
||
if fname.startswith('_'):
|
||
continue
|
||
if fname.endswith('.vpi') or fname.endswith('.pyi'):
|
||
paths.append(os.path.join(temp_dir, fname))
|
||
return paths
|
||
|
||
|
||
def _PyiCacheSignature(paths: list[str]) -> list[tuple[str, int, float]]:
|
||
"""vpi 清单签名：(basename, size, mtime) 元组列表，用于缓存失效判断。"""
|
||
sig: list[tuple[str, int, float]] = []
|
||
for p in paths:
|
||
try:
|
||
st = os.stat(p)
|
||
sig.append((os.path.basename(p), st.st_size, st.st_mtime))
|
||
except OSError:
|
||
sig.append((os.path.basename(p), -1, -1.0))
|
||
return sig
|
||
|
||
|
||
def _LoadPyiCachePkl(temp_dir: str) -> dict | None:
|
||
"""pickle.load temp/_pyi_cache_v1.pkl，返回 {abs_path: vast树|None}。
|
||
文件不存在/损坏/版本不符时返回 None（调用方回退逐文件解析）。"""
|
||
pkl_path: str = os.path.join(temp_dir, _PYI_CACHE_PKL_NAME)
|
||
try:
|
||
with open(pkl_path, 'rb') as f:
|
||
data: dict = pickle.load(f)
|
||
if not isinstance(data, dict) or data.get('version') != _PYI_CACHE_VERSION:
|
||
return None
|
||
cache = data.get('cache')
|
||
if not isinstance(cache, dict):
|
||
return None
|
||
return cache
|
||
except Exception:
|
||
return None
|
||
|
||
|
||
def _BuildPyiCacheFile(temp_dir: str) -> str | None:
|
||
"""构建/刷新 temp/_pyi_cache_v1.pkl：遍历全部 .vpi 做 vast.parse 后
|
||
pickle.dump（含版本号与签名）。签名（basename,size,mtime）未变时跳过。
|
||
返回 pkl 路径；失败（无 vpi / pickle 不兼容等）返回 None。
|
||
注意：vast AST 可 pickle 性由 Nodes.py 侧保证；若实际 pickle.dump
|
||
失败，此处静默回退，worker 的 initializer 将改为逐文件 vast.parse。"""
|
||
paths: list[str] = _ScanPyiFiles(temp_dir)
|
||
if not paths:
|
||
return None
|
||
sig: list[tuple[str, int, float]] = _PyiCacheSignature(paths)
|
||
pkl_path: str = os.path.join(temp_dir, _PYI_CACHE_PKL_NAME)
|
||
sig_path: str = os.path.join(temp_dir, _PYI_CACHE_SIG_NAME)
|
||
try:
|
||
if _CachedIsFile(pkl_path) and _CachedIsFile(sig_path):
|
||
stored = json.loads(_ReadTextCached(sig_path))
|
||
# json 往返后 tuple→list，需规范化再比较
|
||
if stored == [[n, s, m] for n, s, m in sig]:
|
||
return pkl_path
|
||
except Exception:
|
||
pass
|
||
cache: dict[str, object] = {}
|
||
# 增量复用：Phase2 过程会新增 include .vpi（签名每次变 → 全量重建 12.2s）。
|
||
# 加载旧 pkl，按 (basename, size, mtime) 匹配未变化的文件复用 vast 树，
|
||
# 只对新增/变化的文件重新 parse。
|
||
old_by_name: dict[str, tuple[int, float, object | None]] = {}
|
||
try:
|
||
old = _LoadPyiCachePkl(temp_dir)
|
||
if isinstance(old, dict):
|
||
old_sig = old.get('signature') or []
|
||
old_trees = list((old.get('cache') or {}).values())
|
||
for _i, (_name, _size, _mtime) in enumerate(old_sig):
|
||
_tree = old_trees[_i] if _i < len(old_trees) else None
|
||
old_by_name[_name] = (_size, _mtime, _tree)
|
||
except Exception:
|
||
pass
|
||
for p, (name, size, mtime) in zip(paths, sig):
|
||
old_entry = old_by_name.get(name)
|
||
if (old_entry is not None and old_entry[0] == size
|
||
and old_entry[1] == mtime and old_entry[2] is not None):
|
||
cache[p] = old_entry[2]
|
||
continue
|
||
try:
|
||
# 走 LRU 文本缓存：同一 .vpi 在增量复用/多 worker 预热路径会被反复读
|
||
cache[p] = ast.parse(_ReadTextCached(p), p)
|
||
except Exception:
|
||
cache[p] = None
|
||
try:
|
||
with open(pkl_path, 'wb') as f:
|
||
pickle.dump({'version': _PYI_CACHE_VERSION, 'signature': sig, 'cache': cache}, f)
|
||
with open(sig_path, 'w', encoding='utf-8') as f:
|
||
json.dump(sig, f)
|
||
return pkl_path
|
||
except Exception as e:
|
||
_vlog().warning(f"Phase2Translator: pyi 缓存落盘失败({e})，worker 将回退逐文件解析")
|
||
return None
|
||
|
||
|
||
def _PyiCacheWorkerInitializer(temp_dir: str) -> None:
|
||
"""worker 进程启动时预热 ImportHandle._pyi_cache（类级缓存）。
|
||
|
||
spawn 下每 worker 重新 import 模块后 _pyi_cache 为空，首次任务会
|
||
重新 vast.parse 全部依赖 .vpi（4479 次 parse / 245s）。此处启动时
|
||
一次性填充：优先 pickle.load 主进程落盘的 _pyi_cache_v1.pkl；
|
||
失败（文件不存在/损坏/版本不符/pickle 不兼容）回退逐文件 vast.parse。
|
||
预热只解析 .vpi（pyi 树小）；stub 索引保留 _build_stub_index 懒加载。
|
||
TPC_WORKER_WARMUP=0 时关闭（便于回归对比）。
|
||
"""
|
||
if os.environ.get('TPC_WORKER_WARMUP', '1') != '1':
|
||
return
|
||
if not os.path.isdir(temp_dir):
|
||
return
|
||
from lib.core.Handles.HandlesImports import ImportHandle
|
||
cache: dict | None = _LoadPyiCachePkl(temp_dir)
|
||
if cache is not None:
|
||
ImportHandle._pyi_cache = cache
|
||
ImportHandle._pyi_cache_dir = os.path.abspath(temp_dir)
|
||
return
|
||
# 回退：逐文件 vast.parse 预热（worker 单线程，无竞争）
|
||
try:
|
||
new_cache: dict[str, object] = {}
|
||
for p in _ScanPyiFiles(temp_dir):
|
||
try:
|
||
new_cache[p] = ast.parse(_ReadTextCached(p), p)
|
||
except Exception:
|
||
new_cache[p] = None
|
||
if new_cache:
|
||
ImportHandle._pyi_cache = new_cache
|
||
ImportHandle._pyi_cache_dir = os.path.abspath(temp_dir)
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
def _count_braces_outside_strings(text: str) -> int:
|
||
"""统计花括号差值，跳过 c"..." 字符串字面量内部的括号
|
||
|
||
快路径（prof 基线：本函数 102,397 次调用 / self 2.0s，占 _split_ll cumtime
|
||
3.35s 的 83%，是全流程 str.strip 之后第二大的"纯 Python 逐字符"热点）：
|
||
行内不含 `c"` 时，"跳过字符串内部括号"这一语义退化为"整行括号计数"，
|
||
直接用 C 级 str.count 一次算完。含 `c"` 的行（主要见于 c"..." 全局串常量）
|
||
仍走下面的逐字符扫描，语义逐字不变。
|
||
"""
|
||
if 'c"' not in text:
|
||
return text.count('{') - text.count('}')
|
||
count = 0
|
||
i = 0
|
||
while i < len(text):
|
||
if text[i:i+2] == 'c"' and (i == 0 or text[i-1] != '\\'):
|
||
# 跳过 c"..." 字面量
|
||
i += 2
|
||
while i < len(text):
|
||
if text[i] == '"' and (i == 0 or text[i-1] != '\\'):
|
||
i += 1
|
||
break
|
||
i += 1
|
||
continue
|
||
if text[i] == '{':
|
||
count += 1
|
||
elif text[i] == '}':
|
||
count -= 1
|
||
i += 1
|
||
return count
|
||
|
||
|
||
# LLVM IR "无引号标识符"的精确规则（LangRef "Identifiers"；实现即
|
||
# LLLexer::ReadVarName）：首字符 ∈ [-a-zA-Z$._]，后续字符 ∈ [-a-zA-Z$._0-9]。
|
||
# 首字符**不允许是数字**：`$884f3d94ea4d0123.md5_F` 会让 LLLexer::LexDollar
|
||
# 走到 ReadVarName() 返回 false ⇒ 产出 error token ⇒ 顶层解析报
|
||
# "expected top-level entity"。故 comdat 名必须以同一规则判定是否加引号
|
||
# （与 LLVM 自身 AsmWriter 的输出形态一致：`$"884f3d94ea4d0123.md5_F"`）。
|
||
# 另：引号名 @"..." 不支持转义，名字含 `"`/`\` 时两种形态都非法 ⇒ 见
|
||
# _AddWeakComdat 内的 fail-fast（不静默替换、不跳过）。
|
||
_LLVM_UNQUOTED_NAME_RE: re.Pattern[str] = re.compile(
|
||
r'[-a-zA-Z$._][-a-zA-Z$._0-9]*\Z')
|
||
|
||
# comdat 声明行：`$name = comdat any` / `$"name" = comdat any`（允许行首缩进）
|
||
_RE_COMDAT_DECL: re.Pattern[str] = re.compile(r'^\s*\$("?)([^"=\s]+)\1\s*=\s*comdat\b')
|
||
|
||
|
||
def _DedupComdatDecls(ll_content: str) -> str:
|
||
"""删除重复的 `$name = comdat any` 声明（保留首次出现）。
|
||
|
||
落盘的 `_dso.ll` 是「本模块 + 各依赖 .stub.ll/.text.ll 片段」拼接而成，
|
||
每个片段各自带一份同名 comdat 声明（片段生成时也会各自补 comdat）。同名
|
||
声明出现两次即 llc 报 `redefinition of comdat '$…'`（现场：泛型特化
|
||
`list[<sha1>.AST*].__before_init__` 在同一文件里出现 4 次）。comdat 声明
|
||
在顶层可位于任意位置，保留首次出现即可，语义不变。
|
||
"""
|
||
if '= comdat' not in ll_content:
|
||
return ll_content
|
||
seen: set[str] = set()
|
||
kept: list[str] = []
|
||
for line in ll_content.splitlines(True):
|
||
m = _RE_COMDAT_DECL.match(line)
|
||
if m is not None:
|
||
name: str = m.group(2)
|
||
if name in seen:
|
||
continue
|
||
seen.add(name)
|
||
kept.append(line)
|
||
return ''.join(kept)
|
||
|
||
|
||
def _AddWeakComdat(ll_content: str) -> str:
|
||
"""为 weak/linkonce 系列 函数/全局 定义补 COMDAT（COFF/PE 必需）。
|
||
|
||
COFF 下无 COMDAT 的 weak/linkonce 定义会被 llc 展开成强符号
|
||
`.weak.<name>.default`（weak external 的实现副本）。同一弱定义出现在
|
||
多个模块时（泛型特化方法如 `e94ef9e139461915.list[rawptr[AST]].
|
||
__before_init__`、`linkonce`/`weak` 标注的跨模块共享常量），副本同名 →
|
||
ld.lld 报 duplicate symbol。补 COMDAT(any) 后 llc 输出单一 COMDAT
|
||
成员，链接器自动去重（llc/ld.lld 端到端已实验验证）。
|
||
"""
|
||
if (' linkonce ' not in ll_content and ' linkonce_odr ' not in ll_content
|
||
and ' weak_odr ' not in ll_content and ' weak ' not in ll_content):
|
||
return _DedupComdatDecls(ll_content)
|
||
# 拼接而来的片段可能各自已带同名 comdat 声明 ⇒ 先去重，再做本次补全
|
||
ll_content = _DedupComdatDecls(ll_content)
|
||
weak_links = 'linkonce|linkonce_odr|weak|weak_odr|extern_weak'
|
||
comdat_names: list[str] = []
|
||
|
||
# 1) 函数定义：define <linkage> [dso_local] ... @name(...) { → @name(...) comdat {
|
||
def _fn_repl(m: re.Match) -> str:
|
||
comdat_names.append(m.group(3))
|
||
head: str = m.group(1)
|
||
# comdat 是函数头关键字，必须写在 !dbg 等元数据附着**之前**：LLVM 解析器
|
||
# 读完 `!...` 附着后就只接受 `{`，附着之后写 comdat 会报
|
||
# "expected '{' in function body"（`[^{]*` 会跨行吞掉换行，--dwarf 给
|
||
# 每个 define 行尾加了 !dbg 后插入点就落到了附着之后）。故插入点前移到
|
||
# 第一个元数据附着之前。
|
||
attach = _RE_METADATA_ATTACH.search(head)
|
||
if attach is None:
|
||
return head + ' comdat ' + m.group(4)
|
||
return head[:attach.start()] + ' comdat' + head[attach.start():] + m.group(4)
|
||
ll_content = re.sub(
|
||
r'^(define\s+(?:dso_local\s+)?(?:%s)\s+[^@]*@("?)([^"\s\(]+)\2[^{]*)(\{)' % weak_links,
|
||
_fn_repl, ll_content, flags=re.MULTILINE)
|
||
|
||
# 2) 全局定义：@name = <linkage> [dso_local] (global|constant) <init> → 值后补 ', comdat'
|
||
# （comdat 需位于 align/section 等尾部属性之前，符合 LLVM IR 文法）
|
||
def _g_repl(m: re.Match) -> str:
|
||
line = m.group(0)
|
||
if ' comdat' in line:
|
||
return line
|
||
comdat_names.append(m.group(3))
|
||
ins = _RE_COMDAT_TAIL_ATTRS.search(line)
|
||
if ins:
|
||
return line[:ins.start()] + ', comdat' + line[ins.start():]
|
||
return line + ', comdat'
|
||
ll_content = re.sub(
|
||
r'^(@("?)([^"=\s]+)\2\s*=\s*(?:%s)\s+(?:dso_local\s+)?(?:global|constant)[^\n]*)' % weak_links,
|
||
_g_repl, ll_content, flags=re.MULTILINE)
|
||
|
||
if not comdat_names:
|
||
return ll_content
|
||
decls: list[str] = []
|
||
for name in dict.fromkeys(comdat_names):
|
||
# fail-fast：LLVM 的引号标识符 @"..." **不支持转义**（LLLexer 读到 `"` 即认为名字
|
||
# 结束），故名字里含 `"` / `\` / 换行时，加不加引号都产不出合法 IR。此处不静默
|
||
# 跳过、也不做字符替换（那会把符号名改得与源码/其它模块不一致，链接期更难查），
|
||
# 直接指名报错。名字来自弱符号命名 `<sha1>.<标识符>`，出现这些字符说明上游符号名
|
||
# 构造有误，应在上游修正。
|
||
if '"' in name or '\\' in name or '\n' in name or '\r' in name:
|
||
raise RuntimeError(
|
||
f'comdat 名无法表达为 LLVM 标识符（含双引号/反斜杠/换行）: {name!r}；'
|
||
'LLVM 引号名 @"..." 不支持转义，产出即非法 IR（opt 解析报 '
|
||
'"expected top-level entity"）。请在上游符号名构造处修正。')
|
||
if _LLVM_UNQUOTED_NAME_RE.match(name):
|
||
decls.append(f'${name} = comdat any')
|
||
else:
|
||
decls.append(f'$"{name}" = comdat any')
|
||
comdat_block = '\n'.join(decls) + '\n'
|
||
lines = ll_content.splitlines(True)
|
||
header_end = 0
|
||
for i, line in enumerate(lines):
|
||
stripped = line.strip()
|
||
if stripped.startswith(';') or stripped.startswith('target ') or stripped.startswith('source_filename') or not stripped:
|
||
header_end = i + 1
|
||
continue
|
||
break
|
||
return ''.join(lines[:header_end]) + comdat_block + ''.join(lines[header_end:])
|
||
|
||
|
||
class Phase2Translator:
|
||
"""阶段二：使用声明接口翻译源文件"""
|
||
|
||
INCLUDE_LIB_EXTENSIONS: tuple[str, ...] = ('.dll', '.ll', '.so', '.o', '.a', '.lib')
|
||
INCLUDE_SRC_EXTENSIONS: tuple[str, ...] = ('.c', '.cpp', '.cc', '.cxx', '.m', '.mm')
|
||
# includes 只认项目配置（project.vpj includes），禁止仓库硬编码回退。
|
||
# INCLUDE_DIRS 保留为空以兼容旧引用；实际目录由 __init__ 写入 PROJECT_INCLUDE_DIRS。
|
||
INCLUDE_DIRS: list[str] = []
|
||
PROJECT_INCLUDE_DIRS: list[str] = []
|
||
# include 源文件 sha1/存在性缓存（_build_current_ModuleSha1Map）：
|
||
# 编译期间 include .vp/.py 源文件不变，避免每文件重复 isfile + open +
|
||
# compute_sha1（PROF 显示 9306 isfile + 4653 open + TextIOWrapper.read
|
||
# 35.9s 的大头来源，99 次调用 × 全量 include 重算）。
|
||
_include_sha1_cache: dict[str, str] = {}
|
||
_include_exist_cache: dict[str, bool] = {}
|
||
|
||
def __init__(self, src_root: str, temp_dir: str, output_dir: str, compile_cmd: str = 'llc', compile_flags: list[str] | None = None, linker_cmd: str | None = None, linker_flags: list[str] | None = None, linker_output: str | None = None, static_libs: list[str] | None = None, dynamic_libs: list[str] | None = None, include_dirs: list[str] | None = None, entry_files: list[str] | None = None, target_triple: str | None = None, target_datalayout: str | None = None, force_recompile_includes: bool = False, build_pyi_cache: bool = True, asan_mode: bool = False, dwarf_enabled: bool = False, traceback_enabled: bool = False) -> None:
|
||
self.src_root = os.path.abspath(src_root)
|
||
self.temp_dir = os.path.abspath(temp_dir)
|
||
self.output_dir = os.path.abspath(output_dir)
|
||
self.compile_cmd = compile_cmd
|
||
self.compile_flags = compile_flags or ['-filetype=obj']
|
||
self.triple = target_triple
|
||
self.datalayout = target_datalayout
|
||
if not self.triple:
|
||
for i, flag in enumerate(self.compile_flags):
|
||
if flag.startswith('-mtriple='):
|
||
self.triple = flag[len('-mtriple='):]
|
||
elif flag == '-mtriple' and i + 1 < len(self.compile_flags):
|
||
self.triple = self.compile_flags[i + 1]
|
||
self.linker_cmd = linker_cmd
|
||
self.linker_flags = linker_flags or []
|
||
self.linker_output = linker_output
|
||
# ASan 模式：由 Config 的 --asan 参数直传（不再用 TPC_ASAN 环境变量，
|
||
# 环境变量在 worker 子进程/外部调用链下不可靠）。
|
||
# 检测 stdlib 堆/栈/全局越界与垃圾指针（MemBuddy pool 内部由 canary 覆盖）。
|
||
# 注意：mingw64 的 clang 不提供 x86_64-w64-windows-gnu 目标的 ASan runtime，
|
||
# 只能借用 clang64(MSVC ABI) 的那两份：PATH 末尾追加 clang64/bin（运行期 DLL），
|
||
# 链接时显式 -l: 那两份 .a/.dll.a（见 _FindAsanRuntime 的说明）。
|
||
self.asan_mode = asan_mode
|
||
self.asan_rt_dir: str | None = None
|
||
if self.asan_mode:
|
||
_rt = _FindAsanRuntime()
|
||
if _rt is not None:
|
||
self.asan_rt_dir, _asan_bin = _rt
|
||
# clang64/bin 必须**追加**到 PATH 末尾：既要让运行期能找到
|
||
# libclang_rt.asan_dynamic-x86_64.dll，又不能前置 —— 前置会让裸名
|
||
# clang++.exe / llc / opt 解析到 clang64 的 libc++ 版本，与 includes/
|
||
# 里 mingw64(libstdc++) 编的静态库链接不上。
|
||
if _asan_bin and os.path.isdir(_asan_bin):
|
||
os.environ['PATH'] = os.environ.get('PATH', '') + os.pathsep + _asan_bin
|
||
self.slice_level = 3
|
||
# 函数级 DWARF 调试元数据（--dwarf；--asan 时自动开启）：False=未启用
|
||
# （零影响，不构造任何 DI 元数据）；True=App 源模块的 .ll 携带
|
||
# !llvm.dbg.cu + 每函数 DISubprogram + 入口 DILocation，供 ASan /
|
||
# llvm-symbolizer 反查 ".vp:函数起始行"。
|
||
self.dwarf_enabled: bool = dwarf_enabled
|
||
# Ctrl+C 调用栈（--traceback）：False=未启用（零影响，不调用任何外部
|
||
# 工具、不写任何旁文件）；True=链接成功后额外用 llvm-dwarfdump 把最终
|
||
# exe 的 DWARF 行表导出为 "<exe>.stackmap"（地址→源行旁文件，供 exe 内
|
||
# 运行时在 Ctrl+C/Ctrl+Break 时逐帧翻译返回地址）。要求 --dwarf 已开启
|
||
# （Config 侧联动，否则行表为空），且目标为 Windows PE。
|
||
self.traceback_enabled: bool = traceback_enabled
|
||
# 调用点热点统计（--hotspot-sites）：None=未启用（零影响）；str=调用点计数
|
||
# 落盘文件路径（每函数计时落同目录 hotspot_time.txt）。各模块运行时自带
|
||
# Ctrl+C 处理器、统一以追加方式写该路径；含 main 的模块在 main 入口清空一次
|
||
# → 每次运行一份结果。
|
||
self.hotspot_sites_file: str | None = None
|
||
# 是否对全部直接 call 点插桩（忽略白名单）；白名单被调函数基名列表（None=默认）
|
||
self.hotspot_sites_all: bool = False
|
||
self.hotspot_callees: list[str] | None = None
|
||
self.sha1_map: dict[str, str] = {}
|
||
self.sig_files: dict[str, str] = {}
|
||
self.stub_files: dict[str, str] = {}
|
||
self.include_py_map: dict[str, str] = {}
|
||
self.used_includes: set[str] = set()
|
||
self._last_ErrorStack: list | None = None
|
||
self.function_default_args: dict[str, list] = {}
|
||
self.function_param_names: dict[str, list] = {}
|
||
self.inline_func_symbols: dict[str, CTypeInfo] = {}
|
||
self.extra_link_files: list[str] = []
|
||
self.extra_compile_files: list[str] = []
|
||
self.extra_py_files: list[str] = []
|
||
# project.vpj linker.static_libs：显式静态链接文件（.a/.obj），
|
||
# 加入 extra_link_files 参与最终链接（位于 -o 之后、-l 库标志之前）。
|
||
if static_libs:
|
||
for lib in static_libs:
|
||
if lib not in self.extra_link_files:
|
||
self.extra_link_files.append(lib)
|
||
# project.vpj linker.dynamic_libs：显式动态链接文件（.dll/.so），
|
||
# 同样作为链接输入（GNU ld 从 DLL/SO 提取导入符号）；运行时需
|
||
# DLL/SO 位于 exe 同目录 / PATH / LD_LIBRARY_PATH。
|
||
if dynamic_libs:
|
||
for lib in dynamic_libs:
|
||
if lib not in self.extra_link_files:
|
||
self.extra_link_files.append(lib)
|
||
self._include_sha1s: set[str] = set()
|
||
# 持有 ProcessPoolExecutor 引用，避免局部变量 GC 时 finalizer 阻塞 ~99s
|
||
# （join_executor_internals 在 GC 时同步等待 worker 退出队列）
|
||
self._executor_pool: ProcessPoolExecutor | None = None
|
||
self.entry_files: list[str] | None = entry_files
|
||
self.force_recompile_includes: bool = force_recompile_includes
|
||
# pyi cache 构建开关：仅主进程编译启动时构建一次（_LoadSha1Map 内），
|
||
# worker 子进程只读主进程落盘的 temp/_pyi_cache_v1.pkl（_PyiCacheWorkerInitializer），
|
||
# 不重复构建——worker 每任务重建 pkl 的 pickle.load/dump（43 次 × ~0.5s ≈ 23s
|
||
# + pickle.loads 10.4s）纯属浪费；worker 缺 includes vpi 时运行期按需 parse 回退。
|
||
self._build_pyi_cache: bool = build_pyi_cache
|
||
self._shared_symbol_table: object | None = None
|
||
self._shared_source_module_sig_files: dict[str, str] = {}
|
||
self._shared_all_dc: dict[str, dict] = {}
|
||
self._shared_all_decl_types: dict[str, tuple[int, bool]] = {}
|
||
self._shared_export_extern_funcs: dict[str, str] = {}
|
||
self.include_dirs: list[str] = []
|
||
self.struct_sha1_map: dict[str, str] = {}
|
||
self._shared_generic_class_templates: dict[str, dict] = {}
|
||
self._shared_doc_meta: dict[str, dict[str, str]] = {}
|
||
# 共享 vtable 状态跨模块（修复 pool.alloc(48) 虚表分派失败导致段错误）
|
||
self._shared_class_vtable: set[str] = set()
|
||
self._shared_cross_module_vtable: set[str] = set()
|
||
self._shared_class_methods: dict[str, list[str]] = {}
|
||
self._shared_cross_module_novtable: set[str] = set()
|
||
default_dirs: list[str] = []
|
||
if include_dirs:
|
||
seen = set()
|
||
self.include_dirs = []
|
||
for d in list(include_dirs) + default_dirs:
|
||
d = os.path.abspath(d)
|
||
if d not in seen and os.path.isdir(d):
|
||
self.include_dirs.append(d)
|
||
seen.add(d)
|
||
else:
|
||
self.include_dirs = default_dirs
|
||
# 严格：只认项目 includes；供 builtins.vp 扫描等静态路径使用
|
||
Phase2Translator.PROJECT_INCLUDE_DIRS = list(self.include_dirs)
|
||
self._LoadSha1Map()
|
||
|
||
def _LoadSha1Map(self) -> None:
|
||
"""从 temp_dir 加载 SHA1 映射、签名文件和 stub 声明文件列表。
|
||
temp_dir 中缺失的文件会从全局缓存补充。
|
||
"""
|
||
if not os.path.exists(self.temp_dir):
|
||
_vlog().error(f"声明目录不存在: {self.temp_dir}")
|
||
return
|
||
|
||
for fname in os.listdir(self.temp_dir):
|
||
fpath = os.path.join(self.temp_dir, fname)
|
||
if fname.startswith('_'):
|
||
continue
|
||
if fname.endswith('.vpi') or fname.endswith('.pyi'):
|
||
sha1 = fname[:-4]
|
||
self.sig_files[sha1] = fpath
|
||
elif fname.endswith('.stub.ll'):
|
||
sha1 = fname[:-8]
|
||
self.stub_files[sha1] = fpath
|
||
|
||
# 从内存存储器读取 SHA1 映射（_sha1_map.txt 仅人类可读，不再作为机器数据源）
|
||
# 延迟导入避免循环依赖（Config 导入 Phase2Translator）
|
||
from lib.Projectrans.Config import get_sha1_map_store
|
||
mem_store: dict[str, str] = get_sha1_map_store()
|
||
if mem_store:
|
||
for sha1, rel in mem_store.items():
|
||
self.sha1_map[sha1] = rel
|
||
if rel.startswith('includes/'):
|
||
module_name = os.path.splitext(os.path.basename(rel))[0]
|
||
self.include_py_map[module_name] = sha1
|
||
else:
|
||
# 内存存储器为空（独立运行 Phase2 的回退）：从 stub 文件名推断
|
||
for sha1, stub_path in self.stub_files.items():
|
||
self.sha1_map[sha1] = sha1
|
||
|
||
# 从全局缓存补充 temp_dir 中缺失的 .vpi / .stub.ll。
|
||
# 只补充"当前编译有效"的 sha1（在 sha1_map 中）：.transpyc_cache 会残留
|
||
# 旧版源码的缓存（如旧 __builder.vp 的 9026a429f5e2d0e0.pyi/.stub.ll 与新
|
||
# d9b3d06934cd1898 共存），不加过滤会把旧版 .vpi/.stub.ll 也加载进
|
||
# sig_files/stub_files → 同名类（IRBuilder）按旧 sha1 前缀注册 →
|
||
# ld.lld undefined symbol: 9026a429f5e2d0e0.new_builder。
|
||
if os.path.isdir(_GLOBAL_CACHE_DIR):
|
||
for fname in os.listdir(_GLOBAL_CACHE_DIR):
|
||
if fname.startswith('_'):
|
||
continue
|
||
if fname.endswith('.vpi') or fname.endswith('.pyi'):
|
||
sha1 = fname[:-4]
|
||
if sha1 in self.sha1_map and sha1 not in self.sig_files:
|
||
self.sig_files[sha1] = os.path.join(_GLOBAL_CACHE_DIR, fname)
|
||
elif fname.endswith('.stub.ll'):
|
||
sha1 = fname[:-8]
|
||
if sha1 in self.sha1_map and sha1 not in self.stub_files:
|
||
self.stub_files[sha1] = os.path.join(_GLOBAL_CACHE_DIR, fname)
|
||
|
||
# 构建 pyi AST pickle 缓存（temp/_pyi_cache_v1.pkl）供 worker 预热：
|
||
# Phase1 已生成全部 .vpi，此处把 vast 树落盘，避免每个 worker 首次任务
|
||
# 重新 vast.parse（4479 次 parse / 245s）。签名未变时零开销跳过。
|
||
# 仅主进程构建（build_pyi_cache=True）；worker 只读 pkl，不重复构建。
|
||
if self._build_pyi_cache and os.environ.get('TPC_WORKER_WARMUP', '1') == '1':
|
||
_BuildPyiCacheFile(self.temp_dir)
|
||
|
||
def _build_struct_sha1_map(self) -> dict[str, str]:
|
||
struct_sha1_map = {}
|
||
valid_sha1_keys = set(self.sha1_map.keys())
|
||
# 收集 temp_dir 和全局缓存中的 .vpi/.pyi 文件
|
||
pyi_dirs: list[str] = [self.temp_dir]
|
||
if os.path.isdir(_GLOBAL_CACHE_DIR):
|
||
pyi_dirs.append(_GLOBAL_CACHE_DIR)
|
||
_scanned_sha1: set[str] = set()
|
||
for scan_dir in pyi_dirs:
|
||
if not os.path.isdir(scan_dir):
|
||
continue
|
||
for fname in os.listdir(scan_dir):
|
||
if not (fname.endswith('.vpi') or fname.endswith('.pyi')):
|
||
continue
|
||
sha1_key = fname.replace('.vpi', '').replace('.pyi', '')
|
||
if sha1_key not in valid_sha1_keys:
|
||
continue
|
||
# 同一模块在两处目录都有文件时只解析一次（与下面的类名归属判据无关）
|
||
if sha1_key in _scanned_sha1:
|
||
continue
|
||
_scanned_sha1.add(sha1_key)
|
||
pyi_path = os.path.join(scan_dir, fname)
|
||
# 复用 parse_python_file 的 AST 缓存（.pyi 文件可能被多处解析）
|
||
_, tree = parse_python_file(pyi_path)
|
||
if tree is None:
|
||
continue
|
||
for node in ast.iter_child_nodes(tree):
|
||
# 每类只登记一次，**先扫到的为准**：判据必须是**类名**。
|
||
# 原实现写成 `if sha1_key in struct_sha1_map` —— 用模块 SHA1 去查一张
|
||
# 按**类名**建的表，判据恒假 ⇒ 实际是"后扫到的覆盖先扫到的"，
|
||
# 同名类的归属随 os.listdir 顺序漂移（现场：includes/ast/astaux.vp 与
|
||
# includes/ast/tokens.vp 各有一个 `class Keyword`，定义端拿到的前缀
|
||
# 可能是**另一个模块**的 sha1 ⇒ 同一类出现两个 struct 实例、
|
||
# `__before_init__` 声明与调用点分叉）。
|
||
if isinstance(node, ast.ClassDef):
|
||
# 嵌套类按扁平名 `{外类}_{内类}` 登记（与定义端
|
||
# HandlesClassDef._EmitNestedClassDefs、Phase1Generator.
|
||
# _RegisterNestedStructNames 同源）。此前用 ast.walk 会把嵌套类按
|
||
# **短名** `Inner` 登记 —— 既占住平坦命名空间，又与定义端的
|
||
# `外类_内类` 不同源（跨模块前缀分裂）。
|
||
self._RegisterStructSha1Rec(node, node.name, sha1_key, struct_sha1_map)
|
||
return struct_sha1_map
|
||
|
||
def _RegisterStructSha1Rec(self, Node, FlatName: str, Sha1Key: str,
|
||
StructSha1Map: dict) -> None:
|
||
"""递归登记"类名 → 定义模块 SHA1"（嵌套类用扁平名 `{外类}_{内类}`）"""
|
||
if not FlatName:
|
||
return
|
||
if FlatName not in StructSha1Map:
|
||
StructSha1Map[FlatName] = Sha1Key
|
||
Body = getattr(Node, 'body', None)
|
||
if not Body:
|
||
return
|
||
for Item in Body:
|
||
if not isinstance(Item, ast.ClassDef):
|
||
continue
|
||
InnerName: str | None = getattr(Item, 'name', None)
|
||
if not InnerName:
|
||
continue
|
||
self._RegisterStructSha1Rec(Item, f'{FlatName}_{InnerName}', Sha1Key, StructSha1Map)
|
||
|
||
def run(self) -> None:
|
||
"""扫描源目录，翻译并生成 .ll 含代码文件（导入遍历）"""
|
||
import sys as _sys
|
||
if self.entry_files:
|
||
py_files = list(self.entry_files)
|
||
else:
|
||
main_py = os.path.join(self.src_root, 'main.vp')
|
||
if not os.path.exists(main_py):
|
||
# 回退到 main.py（向后兼容）
|
||
main_py = os.path.join(self.src_root, 'main.py')
|
||
if os.path.exists(main_py):
|
||
py_files = [main_py]
|
||
else:
|
||
py_files = []
|
||
for root, dirs, files in os.walk(self.src_root):
|
||
dirs[:] = [d for d in dirs if d not in ('__pycache__',)]
|
||
for file in files:
|
||
if file.endswith('.vp') or file.endswith('.py'):
|
||
py_files.append(os.path.join(root, file))
|
||
|
||
if not py_files:
|
||
_vlog().info("未找到入口文件或源文件")
|
||
return
|
||
|
||
reachable = find_reachable_files_from_entries(self.src_root, py_files)
|
||
py_files = list(reachable)
|
||
|
||
py_files = topological_sort_files(py_files, self.src_root)
|
||
|
||
_vlog().info(f"找到 {len(py_files)} 个可达源文件（已按依赖排序）")
|
||
_vlog().info("处理顺序:")
|
||
for i, f in enumerate(py_files[:10], 1):
|
||
rel = os.path.relpath(f, self.src_root)
|
||
_vlog().info(f" {i}. {rel}")
|
||
os.makedirs(self.output_dir, exist_ok=True)
|
||
|
||
valid_sha1s = set(self.sha1_map.keys())
|
||
old_stub_count = len(self.stub_files)
|
||
old_sig_count = len(self.sig_files)
|
||
self.stub_files = {k: v for k, v in self.stub_files.items() if k in valid_sha1s}
|
||
self.sig_files = {k: v for k, v in self.sig_files.items() if k in valid_sha1s}
|
||
if old_stub_count != len(self.stub_files) or old_sig_count != len(self.sig_files):
|
||
_vlog().info(f"过滤旧文件: stub {old_stub_count}->{len(self.stub_files)}, sig {old_sig_count}->{len(self.sig_files)}")
|
||
|
||
active_sha1s = set()
|
||
self._precollect_inline_symbols()
|
||
self._build_shared_symbol_data()
|
||
self._precollect_vtable_state()
|
||
self.struct_sha1_map = self._build_struct_sha1_map()
|
||
|
||
need_translate = []
|
||
# --hotspot-sites：缓存 .ll / .stub.ll+.text.ll 可能不含热点计数/计时 IR，
|
||
# 启用时一律重译，保证每个模块都嵌入 internal 计数器、计时器与
|
||
# @llvm.global_ctors 自注册。未启用时 hotspot_on=False，缓存与重组分支
|
||
# 行为与改动前完全一致（零影响）。
|
||
# --dwarf（函数级 DWARF）同理：缓存 .ll/.obj 不含 !llvm.dbg.cu，必须一并
|
||
# 强制重译，否则开启开关后仍复用旧产物、看不到任何 .vp 行号。
|
||
# --traceback 依赖同一份 .debug_line 行表（Config 已联动开启 --dwarf），
|
||
# 这里显式纳入，避免将来联动改动后静默复用无 DI 的旧产物。
|
||
hotspot_on: bool = bool(self.hotspot_sites_file) or bool(self.dwarf_enabled) or bool(self.traceback_enabled)
|
||
for i, src_path in enumerate(py_files, 1):
|
||
rel = os.path.relpath(src_path, self.src_root)
|
||
# 复用 parse_python_file 的 AST 缓存（get_file_dependencies 已 parse 过）
|
||
content, tree = parse_python_file(src_path)
|
||
sha1 = compute_sha1(content)
|
||
active_sha1s.add(sha1)
|
||
out_path = os.path.join(self.output_dir, f"{sha1}.ll")
|
||
|
||
current_ModuleSha1Map = self._build_current_ModuleSha1Map(sha1)
|
||
|
||
if tree is not None:
|
||
try:
|
||
for node in ast.walk(tree):
|
||
if isinstance(node, ast.Import):
|
||
for alias in node.names:
|
||
self.used_includes.add(alias.name.split('.')[0])
|
||
elif isinstance(node, ast.ImportFrom):
|
||
if node.module:
|
||
self.used_includes.add(node.module.split('.')[0])
|
||
# 检测 assert 使用，自动添加 testcheck 到 used_includes：
|
||
# assert 译为 testcheck.check(...)，该库须被发现、编译并链接。
|
||
if isinstance(node, ast.Assert):
|
||
self.used_includes.add('testcheck')
|
||
# 检测 list/dict 容器使用，自动添加 _list/_dict/json 到 used_includes
|
||
# 以便 _scan_include_libraries 能发现并编译这些库文件
|
||
if isinstance(node, ast.Subscript) and isinstance(node.value, ast.Name):
|
||
if node.value.id == 'dict':
|
||
self.used_includes.add('_dict')
|
||
self.used_includes.add('json')
|
||
elif node.value.id == 'list':
|
||
self.used_includes.add('_list')
|
||
if isinstance(node, ast.Call) and isinstance(node.func, ast.Name):
|
||
if node.func.id == 'dict':
|
||
self.used_includes.add('_dict')
|
||
self.used_includes.add('json')
|
||
elif node.func.id == 'list':
|
||
self.used_includes.add('_list')
|
||
# 检测类型注解 d: dict / l: list (非泛型形式)
|
||
if isinstance(node, ast.AnnAssign) and isinstance(node.annotation, ast.Name):
|
||
if node.annotation.id == 'dict':
|
||
self.used_includes.add('_dict')
|
||
self.used_includes.add('json')
|
||
elif node.annotation.id == 'list':
|
||
self.used_includes.add('_list')
|
||
except Exception as _e:
|
||
if _config_mode == "strict":
|
||
raise
|
||
_vlog().warning(f"收集源文件导入的 include 模块失败: {_e}", "Exception")
|
||
|
||
# --hotspot-sites：上面两个缓存/重组分支在 hotspot_on 时被跳过，
|
||
# 本轮每个模块都会强制重译 .ll；此时必须让旧的 {sha1}.obj 一并失效，
|
||
# 否则 _compile_ll_files 会因 .obj 已存在而跳过重编，复用上一次未插桩
|
||
# 的目标文件，导致部分模块热点计时/计数静默缺失、耗时被计入调用者 self。
|
||
# 这里用 os.path.isfile 而非 _CachedIsFile：避免把"已存在"写进存在性
|
||
# 缓存后又被本函数删除，使后续 _compile_ll_files 读到过期缓存而漏编。
|
||
if hotspot_on:
|
||
obj_path = os.path.join(self.output_dir, f"{sha1}.obj")
|
||
if os.path.isfile(obj_path):
|
||
try:
|
||
os.remove(obj_path)
|
||
except OSError as e:
|
||
# 不中断构建，但真实错误（如权限不足）必须打印告警，不静默吞掉
|
||
_vlog().warning(
|
||
f"Phase2Translator: 删除热点插桩模块的旧目标文件失败: {obj_path}: {e}")
|
||
|
||
if not hotspot_on and _CachedIsFile(out_path):
|
||
deps_changed = self._check_deps_changed(sha1, current_ModuleSha1Map)
|
||
if not deps_changed:
|
||
_vlog().info(f"[{i}/{len(py_files)}] 跳过(缓存): {rel} -> {sha1}.ll")
|
||
continue
|
||
else:
|
||
if self._recombine_ll(sha1, current_ModuleSha1Map):
|
||
_vlog().info(f"[{i}/{len(py_files)}] 重组(依赖变化): {rel} -> {sha1}.ll")
|
||
continue
|
||
_vlog().info(f"[{i}/{len(py_files)}] 重译(依赖变化): {rel} -> {sha1}.ll")
|
||
|
||
elif not hotspot_on:
|
||
stub_cache = os.path.join(self.output_dir, f"{sha1}.stub.ll")
|
||
text_cache = os.path.join(self.output_dir, f"{sha1}.text.ll")
|
||
if _CachedIsFile(stub_cache) and _CachedIsFile(text_cache):
|
||
if self._recombine_ll(sha1, current_ModuleSha1Map):
|
||
_vlog().info(f"[{i}/{len(py_files)}] 重组(缓存恢复): {rel} -> {sha1}.ll")
|
||
continue
|
||
|
||
need_translate.append((i, src_path, out_path, sha1, current_ModuleSha1Map))
|
||
|
||
if need_translate:
|
||
# 提前从 include 文件源码中提取函数参数名，供主项目编译时 keyword 参数匹配使用
|
||
self._extract_include_param_names_from_dirs()
|
||
# 提前从 App 源文件中提取函数默认参数，供跨模块调用参数检查使用
|
||
# （避免并行编译时，被调用函数的文件尚未编译完，导致默认参数信息缺失）
|
||
self._extract_app_default_args(need_translate)
|
||
|
||
_vlog().info(f"\n需要翻译 {len(need_translate)} 个文件")
|
||
t_start = time.time()
|
||
|
||
n_workers = _PickWorkerCount(len(need_translate))
|
||
|
||
if n_workers > 1 and len(need_translate) > 1:
|
||
_vlog().info(f" 并行翻译 (workers={n_workers})")
|
||
|
||
shared_pickle_path = os.path.join(self.temp_dir, '_shared_sym.json')
|
||
try:
|
||
# symbol_table_bytes：SymbolTable 独立 pickle 字节，供批次 worker
|
||
# 每文件前反序列化出干净副本（SymbolTable 是编译期唯一被写的共享
|
||
# 对象，批次内复用 trans 时必须在每个文件前重置，防止前文件 import
|
||
# 写入的符号污染后文件 → 跨模块 struct 类型前缀错位）。
|
||
symbol_table_bytes: bytes = pickle.dumps(self._shared_symbol_table)
|
||
shared_data = {
|
||
'symbol_table_bytes': symbol_table_bytes,
|
||
'source_module_sig_files': self._shared_source_module_sig_files,
|
||
'all_dc': self._shared_all_dc,
|
||
'all_decl_types': self._shared_all_decl_types,
|
||
'export_extern_funcs': self._shared_export_extern_funcs,
|
||
'inline_func_symbols': self.inline_func_symbols,
|
||
'function_default_args': self.function_default_args,
|
||
'function_param_names': self.function_param_names,
|
||
'generic_class_templates': self._shared_generic_class_templates,
|
||
'shared_doc_meta': self._shared_doc_meta,
|
||
'class_vtable': self._shared_class_vtable,
|
||
'cross_module_vtable': self._shared_cross_module_vtable,
|
||
'class_methods': self._shared_class_methods,
|
||
'cross_module_novtable': self._shared_cross_module_novtable,
|
||
}
|
||
with open(shared_pickle_path, 'wb') as f:
|
||
f.write(pickle.dumps(shared_data))
|
||
_vlog().info(f" 共享符号表已序列化 ({os.path.getsize(shared_pickle_path)//1024}KB)")
|
||
except Exception as e:
|
||
_vlog().warning(f"符号表序列化失败({e})，worker将自行构建")
|
||
shared_pickle_path = ''
|
||
|
||
# 复用 self._executor_pool：避免局部变量 GC 时 finalizer 阻塞 ~99s
|
||
# （join_executor_internals 在 GC 时同步等待 worker 退出队列）
|
||
if self._executor_pool is None:
|
||
# initializer 预热 ImportHandle._pyi_cache（pickle.load 落盘缓存），
|
||
# 避免每个 worker 首次任务重新 vast.parse 全部依赖 .vpi
|
||
self._executor_pool = ProcessPoolExecutor(max_workers=n_workers,
|
||
initializer=_PyiCacheWorkerInitializer,
|
||
initargs=(self.temp_dir,))
|
||
executor = self._executor_pool
|
||
try:
|
||
# 批次子任务模型：多个文件包装为一个子任务（batch），
|
||
# 每 worker 处理一批——Phase2Translator 创建 + pickle.load 只做
|
||
# 一次，批内文件顺序翻译（每文件前重置 SymbolTable 防污染）。
|
||
# 相比"一文件一任务"：减少进程池调度/通信/构造开销
|
||
# （200+ 文件 → n_workers 个批次任务）。
|
||
batches: list[list] = []
|
||
# 按源文件大小加权分桶（LPT 贪心：最大文件优先放入当前总成本最小的桶），
|
||
# 使各批次总成本均匀。原按文件数均分（batch_size=ceil(N/n)）导致大文件
|
||
# 集中的批次拖长尾，其他 worker 提前完成空等（任务颗粒度不均匀）。
|
||
if len(need_translate) <= n_workers:
|
||
batches = [[item] for item in need_translate]
|
||
else:
|
||
sized_items: list = []
|
||
for item in need_translate:
|
||
try:
|
||
_size: int = os.path.getsize(item[1])
|
||
except OSError:
|
||
_size = 0
|
||
sized_items.append((item, _size))
|
||
buckets: list[list] = [[] for _ in range(n_workers)]
|
||
bucket_cost: list[int] = [0] * n_workers
|
||
for item, _size in sorted(sized_items, key=lambda t: t[1], reverse=True):
|
||
target: int = min(range(n_workers), key=lambda w: bucket_cost[w])
|
||
buckets[target].append(item)
|
||
bucket_cost[target] += _size
|
||
batches = [b for b in buckets if b]
|
||
futures = {}
|
||
for bidx, batch in enumerate(batches):
|
||
rel_first = os.path.relpath(batch[0][1], self.src_root)
|
||
future = executor.submit(
|
||
_parallel_translate_batch,
|
||
batch,
|
||
self.src_root, self.temp_dir, self.output_dir,
|
||
self.include_dirs, self.triple, self.datalayout,
|
||
self.sha1_map, self.sig_files, self.stub_files,
|
||
self.include_py_map, self.slice_level,
|
||
shared_pickle_path, self.struct_sha1_map,
|
||
hotspot_sites_file=self.hotspot_sites_file,
|
||
hotspot_sites_all=self.hotspot_sites_all,
|
||
hotspot_callees=self.hotspot_callees,
|
||
dwarf_enabled=self.dwarf_enabled,
|
||
)
|
||
futures[future] = (bidx, rel_first, len(batch))
|
||
|
||
done_files = 0
|
||
total_files = len(need_translate)
|
||
for future in as_completed(futures):
|
||
bidx, rel_first, bcount = futures[future]
|
||
try:
|
||
batch_errors, batch, per_file_times = future.result()
|
||
except ReportedCompileError:
|
||
raise
|
||
except Exception as e:
|
||
# fail-fast：进程池已 broken（worker 异常退出）→ 取消
|
||
# 全部剩余批次并终止编译，不空等。
|
||
_vlog().error(f" 批次 {bidx} 异常（worker 已终止）: {e}")
|
||
for fut in futures:
|
||
if not fut.done():
|
||
fut.cancel()
|
||
if self._executor_pool is not None:
|
||
self._executor_pool.shutdown(wait=False, cancel_futures=True)
|
||
self._executor_pool = None
|
||
raise ReportedCompileError(
|
||
f"批次 {bidx} worker 异常终止，编译立即停止") from None
|
||
if batch_errors:
|
||
# worker 只回传错误、不写 stderr：完整错误（含源码定位）
|
||
# 只在这里输出一次，随后立即终止（fail-fast）。
|
||
src, err = batch_errors[0]
|
||
rel = os.path.relpath(src, self.src_root) if src else rel_first
|
||
_vlog().error(f"\n文件翻译失败: {rel}\n{err}")
|
||
for fut in futures:
|
||
if not fut.done():
|
||
fut.cancel()
|
||
if self._executor_pool is not None:
|
||
self._executor_pool.shutdown(wait=False, cancel_futures=True)
|
||
self._executor_pool = None
|
||
raise ReportedCompileError(
|
||
f"{rel} 翻译失败，编译立即停止") from None
|
||
for src, elapsed, rel in per_file_times:
|
||
done_files += 1
|
||
_vlog().info(f" [{done_files}/{total_files}] {rel} ({elapsed:.1f}s)")
|
||
_vlog().info(f" [批次完成] {bcount} 文件, 首个 {rel_first}")
|
||
except Exception:
|
||
# 异常时立即清理，正常流程不 shutdown（留给 includes 阶段复用）
|
||
if self._executor_pool is not None:
|
||
self._executor_pool.shutdown(wait=False, cancel_futures=True)
|
||
self._executor_pool = None
|
||
raise
|
||
|
||
else:
|
||
for idx, (i, src_path, out_path, sha1, msm) in enumerate(need_translate):
|
||
rel = os.path.relpath(src_path, self.src_root)
|
||
t0 = time.time()
|
||
try:
|
||
self._translate_file(src_path, out_path, prebuilt_ModuleSha1Map=msm, dwarf_enabled=self.dwarf_enabled)
|
||
t1 = time.time()
|
||
_vlog().info(f" [{idx+1}/{len(need_translate)}] {rel} ({t1-t0:.1f}s)")
|
||
except Exception as e:
|
||
# 单点输出：完整错误（含源码定位）只打印这一次
|
||
_vlog().error(f"\n文件翻译失败: {rel}\n{type(e).__name__}: {e}")
|
||
raise ReportedCompileError(
|
||
f"翻译 {rel} 失败，已终止编译") from None
|
||
t_end = time.time()
|
||
_vlog().info(f" 翻译总耗时: {t_end-t_start:.1f}s")
|
||
|
||
_vlog().info(f"\n[阶段二完成] .ll 文件生成到: {self.output_dir}")
|
||
self._scan_include_libraries()
|
||
self._compile_include_py_files(active_sha1s)
|
||
self._compile_ll_files(active_sha1s)
|
||
self._compile_include_sources()
|
||
if self.linker_cmd:
|
||
self._link_obj_files(active_sha1s)
|
||
# 清理 ProcessPoolExecutor：所有翻译已完成，shutdown(wait=False) 让 worker
|
||
# 后台退出。避免 GC finalizer 在程序末尾阻塞 ~99s（join_executor_internals）
|
||
if self._executor_pool is not None:
|
||
self._executor_pool.shutdown(wait=False, cancel_futures=True)
|
||
self._executor_pool = None
|
||
|
||
def _build_current_ModuleSha1Map(self, self_sha1: str | None) -> dict[str, str]:
|
||
"""构建当前模块的依赖 SHA1 映射（与 _translate_file 中逻辑一致）"""
|
||
ModuleSha1Map = {}
|
||
for sha1_key, sig_path in self.sig_files.items():
|
||
if sha1_key == self_sha1:
|
||
continue
|
||
rel_path = self.sha1_map.get(sha1_key, sha1_key)
|
||
if rel_path.startswith('includes/'):
|
||
inc_mod_name = os.path.splitext(rel_path)[0].replace(os.sep, '.').replace('/', '.')
|
||
inc_mod_name = inc_mod_name[len('includes/'):]
|
||
inc_short_name = inc_mod_name.split('.')[-1] if '.' in inc_mod_name else inc_mod_name
|
||
actual_sha1 = sha1_key
|
||
for inc_dir in self.include_dirs:
|
||
if os.path.isdir(inc_dir):
|
||
py_path = os.path.join(inc_dir, rel_path[len('includes/'):].replace(os.sep, '/'))
|
||
py_path = os.path.splitext(py_path)[0] + '.vp'
|
||
# 存在性/内容缓存：编译期 include 源文件不变，命中跳过磁盘 I/O
|
||
exists = Phase2Translator._include_exist_cache.get(py_path)
|
||
if exists is None:
|
||
exists = _CachedIsFile(py_path)
|
||
Phase2Translator._include_exist_cache[py_path] = exists
|
||
if not exists:
|
||
# 回退到 .py（向后兼容）
|
||
py_path = os.path.splitext(py_path)[0] + '.py'
|
||
exists = Phase2Translator._include_exist_cache.get(py_path)
|
||
if exists is None:
|
||
exists = _CachedIsFile(py_path)
|
||
Phase2Translator._include_exist_cache[py_path] = exists
|
||
if exists:
|
||
py_sha1 = Phase2Translator._include_sha1_cache.get(py_path)
|
||
if py_sha1 is None:
|
||
py_sha1 = compute_sha1(_ReadTextCached(py_path))
|
||
Phase2Translator._include_sha1_cache[py_path] = py_sha1
|
||
actual_sha1 = py_sha1
|
||
break
|
||
ModuleSha1Map[inc_mod_name] = actual_sha1
|
||
ModuleSha1Map[inc_short_name] = actual_sha1
|
||
# Also add parent package name (e.g., 'hashlib' for 'hashlib.__init__')
|
||
if inc_short_name == '__init__' and '.' in inc_mod_name:
|
||
parent_pkg = inc_mod_name.rsplit('.', 1)[0]
|
||
ModuleSha1Map[parent_pkg] = actual_sha1
|
||
continue
|
||
module_name = os.path.splitext(rel_path)[0].replace(os.sep, '.').replace('/', '.')
|
||
ModuleSha1Map[module_name] = sha1_key
|
||
short_name = module_name.split('.')[-1] if '.' in module_name else module_name
|
||
ModuleSha1Map[short_name] = sha1_key
|
||
# 包入口文件：注册父包名（如 'os.__init__' -> 'os'），使 import os 能解析到包级符号
|
||
if short_name == '__init__' and '.' in module_name:
|
||
parent_pkg = module_name.rsplit('.', 1)[0]
|
||
ModuleSha1Map[parent_pkg] = sha1_key
|
||
return ModuleSha1Map
|
||
|
||
@staticmethod
|
||
def _extract_llvm_type_name(line: str) -> str | None:
|
||
"""从 LLVM IR 类型定义行提取类型名（字符串操作，不使用正则）
|
||
|
||
支持 list[str]、dict[str, int] 等含特殊字符的类型名。
|
||
例如:
|
||
%"18e6990aa81cc5b6.list[str]" = type { ... } → "18e6990aa81cc5b6.list[str]"
|
||
%struct.foo = type { ... } → "struct.foo"
|
||
"""
|
||
stripped = line.strip()
|
||
if not stripped.startswith('%'):
|
||
return None
|
||
# 处理带引号的情况: %"name" = type ...
|
||
if stripped.startswith('%"'):
|
||
end_quote = stripped.find('"', 2)
|
||
if end_quote > 0:
|
||
return stripped[2:end_quote]
|
||
return None
|
||
# 处理不带引号的情况: %name = type ...
|
||
eq_pos = stripped.find('=')
|
||
if eq_pos > 1:
|
||
return stripped[1:eq_pos].strip()
|
||
return None
|
||
|
||
@staticmethod
|
||
def _find_all_type_refs(text: str) -> set[str]:
|
||
"""查找文本中所有 %"..."/%name 类型引用（字符串操作，不使用正则）
|
||
|
||
支持 list[str]、dict[str, int] 等含特殊字符的类型名。
|
||
"""
|
||
refs: set[str] = set()
|
||
i: int = 0
|
||
n: int = len(text)
|
||
while i < n:
|
||
if text[i] == '%':
|
||
if i + 1 < n and text[i + 1] == '"':
|
||
# 带引号: %"name"
|
||
end: int = text.find('"', i + 2)
|
||
if end > 0:
|
||
name: str = text[i + 2:end]
|
||
if name:
|
||
refs.add(name)
|
||
i = end + 1
|
||
continue
|
||
else:
|
||
# 不带引号: %name (仅匹配字母、数字、下划线、点)
|
||
j: int = i + 1
|
||
while j < n and (text[j].isalnum() or text[j] in '._'):
|
||
j += 1
|
||
if j > i + 1:
|
||
refs.add(text[i + 1:j])
|
||
i = j
|
||
continue
|
||
i += 1
|
||
return refs
|
||
|
||
@staticmethod
|
||
def _split_ll(ll_content: str) -> tuple[str, str]:
|
||
"""将 .ll 内容拆分为 stub（声明）和 text（代码）两部分
|
||
|
||
stub: target, 注释, %type = type, declare, @xxx = external global
|
||
text: define ... { ... }, @xxx = global/constant (带初始化器)
|
||
"""
|
||
stub_lines = []
|
||
text_lines = []
|
||
in_define = False
|
||
brace_depth = 0
|
||
module_sha1 = None
|
||
defined_names = set()
|
||
|
||
for line in ll_content.splitlines(True):
|
||
# 第一遍只关心 `define ...` 行，而合并后的 .ll 里这类行占比极低。
|
||
# `'define' not in line` 是 C 级子串扫描 ⇒ 绝大多数行连 strip 都不做
|
||
# （prof 实测 str.strip 1.79 亿次 / str.startswith 2.37 亿次是全流程
|
||
# 最大的 Python 侧热点，本函数两遍逐行扫描是其主要来源）。
|
||
if 'define' not in line:
|
||
continue
|
||
stripped = line.strip()
|
||
|
||
if stripped.startswith('define '):
|
||
dm = _RE_DEFINE_SHA1.match(stripped)
|
||
if dm and module_sha1 is None:
|
||
module_sha1 = dm.group(1)
|
||
dm2 = _RE_DEFINE_NAME.match(stripped)
|
||
if dm2:
|
||
defined_names.add(dm2.group(1))
|
||
|
||
in_global_def = False
|
||
global_var_name = None
|
||
global_type_parts = []
|
||
global_brace_depth = 0
|
||
|
||
for line in ll_content.splitlines(True):
|
||
# 续行（函数体指令 / 全局初始化器）在合并 .ll 里占绝大多数，而这两支
|
||
# 只需要"行内容 + 花括号计数" ⇒ 不进 strip（前导/尾随空白不含花括号，
|
||
# _count_braces_outside_strings 的结果逐字不变）。prof 实测 str.strip
|
||
# 1.79 亿次为最大 Python 侧热点，逐行 strip 是其主要来源。
|
||
if in_global_def:
|
||
text_lines.append(line)
|
||
global_type_parts.append(line.strip())
|
||
# 预筛：无花括号的行必然贡献 0（与计数器返回值逐字等价，见其快路径）
|
||
if '{' in line or '}' in line:
|
||
global_brace_depth += _count_braces_outside_strings(line)
|
||
if global_brace_depth <= 0:
|
||
full_type = ' '.join(global_type_parts).strip()
|
||
depth = 0
|
||
type_end = len(full_type)
|
||
for ci, cc in enumerate(full_type):
|
||
if cc in ('{', '['):
|
||
depth += 1
|
||
elif cc in ('}', ']'):
|
||
depth -= 1
|
||
elif depth == 0 and cc == ' ':
|
||
type_end = ci
|
||
break
|
||
var_type = full_type[:type_end].strip()
|
||
if var_type:
|
||
stub_lines.append(f'{global_var_name} = external global {var_type}\n')
|
||
in_global_def = False
|
||
global_var_name = None
|
||
global_type_parts = []
|
||
global_brace_depth = 0
|
||
continue
|
||
|
||
if in_define:
|
||
text_lines.append(line)
|
||
# 预筛：函数体指令行绝大多数不含花括号，跳过即省掉一次函数调用
|
||
# （prof：本函数 102k 次调用；有花括号的行仍按原语义计数）
|
||
if '{' in line or '}' in line:
|
||
brace_depth += _count_braces_outside_strings(line)
|
||
if brace_depth <= 0:
|
||
in_define = False
|
||
continue
|
||
|
||
stripped = line.strip()
|
||
if stripped.startswith('define '):
|
||
text_lines.append(line)
|
||
in_define = True
|
||
brace_depth = _count_braces_outside_strings(stripped)
|
||
decl_line = _RE_DEFINE_PREFIX.sub('declare ', stripped)
|
||
decl_line = _RE_DEFINE_LINKAGE.sub('', decl_line)
|
||
decl_line = _RE_ALWAYSINLINE.sub('', decl_line)
|
||
paren_pos = decl_line.rfind(')')
|
||
if paren_pos >= 0:
|
||
decl_line = decl_line[:paren_pos + 1]
|
||
stub_lines.append(decl_line + '\n')
|
||
continue
|
||
|
||
if stripped == '{' and text_lines and not in_define:
|
||
prev_stripped = text_lines[-1].strip() if text_lines else ''
|
||
if prev_stripped.startswith('define ') or prev_stripped.endswith('noredzone') or prev_stripped.endswith(')'):
|
||
text_lines.append(line)
|
||
brace_depth = 1
|
||
in_define = True
|
||
if prev_stripped.startswith('define '):
|
||
# 与上面 define 分支同一组预编译正则（模式串一字未改），
|
||
# 避免 re 模块级的"模式串哈希→缓存查找"逐行开销
|
||
decl_line = _RE_DEFINE_PREFIX.sub('declare ', prev_stripped)
|
||
decl_line = _RE_DEFINE_LINKAGE.sub('', decl_line)
|
||
decl_line = _RE_ALWAYSINLINE.sub('', decl_line)
|
||
paren_pos = decl_line.rfind(')')
|
||
if paren_pos >= 0:
|
||
decl_line = decl_line[:paren_pos + 1]
|
||
stub_lines.append(decl_line + '\n')
|
||
continue
|
||
|
||
if stripped.startswith('@') and ' global ' in stripped and 'external global' not in stripped:
|
||
text_lines.append(line)
|
||
# LLVM 保留全局（如 --hotspot-sites 的 @llvm.global_ctors；名字含点故
|
||
# 最终 IR 保留引号）是 appending 定义，禁止改写为 "external global"
|
||
# 存根（否则 .ll 重组后 llc 报错）
|
||
if _RE_LLVM_INTRINSIC.match(stripped):
|
||
continue
|
||
if ' internal ' not in stripped and ' private ' not in stripped:
|
||
name_match = _RE_AT_NAME.match(stripped)
|
||
if name_match:
|
||
var_name = name_match.group(1)
|
||
global_pos = stripped.find(' global ')
|
||
after_global = stripped[global_pos + 8:]
|
||
depth = 0
|
||
type_end = len(after_global)
|
||
for ci, cc in enumerate(after_global):
|
||
if cc in ('{', '['):
|
||
depth += 1
|
||
elif cc in ('}', ']'):
|
||
depth -= 1
|
||
elif depth == 0 and cc == ' ':
|
||
type_end = ci
|
||
break
|
||
var_type = after_global[:type_end].strip()
|
||
brace_depth_gv = 0
|
||
for cc in var_type:
|
||
if cc == '{':
|
||
brace_depth_gv += 1
|
||
elif cc == '}':
|
||
brace_depth_gv -= 1
|
||
if brace_depth_gv > 0:
|
||
in_global_def = True
|
||
global_var_name = var_name
|
||
global_type_parts = [var_type]
|
||
global_brace_depth = brace_depth_gv
|
||
elif var_type:
|
||
stub_lines.append(f'{var_name} = external global {var_type}\n')
|
||
continue
|
||
if stripped.startswith('@') and ' constant ' in stripped:
|
||
text_lines.append(line)
|
||
continue
|
||
|
||
if stripped.startswith('declare'):
|
||
dm = _RE_DECLARE_NAME.match(stripped)
|
||
if dm:
|
||
fname = dm.group(1)
|
||
if fname in defined_names:
|
||
stub_lines.append(line)
|
||
continue
|
||
dot_pos = fname.find('.')
|
||
if dot_pos > 0:
|
||
fname_sha1 = fname[:dot_pos]
|
||
if module_sha1 and fname_sha1 != module_sha1 and len(fname_sha1) >= 16:
|
||
continue
|
||
if _RE_SHA1_16_DOT.match(fname):
|
||
continue
|
||
stub_lines.append(line)
|
||
continue
|
||
|
||
stub_lines.append(line)
|
||
|
||
return ''.join(stub_lines), ''.join(text_lines)
|
||
|
||
def _save_deps(self, sha1: str, ModuleSha1Map: dict[str, str]) -> None:
|
||
"""保存依赖指纹到 .deps.json"""
|
||
deps_path = os.path.join(self.output_dir, f"{sha1}.deps.json")
|
||
deps = {}
|
||
for mod_name, dep_sha1 in ModuleSha1Map.items():
|
||
deps[mod_name] = dep_sha1
|
||
with open(deps_path, 'w', encoding='utf-8') as f:
|
||
json.dump(deps, f)
|
||
_InvalidateTextCache(deps_path)
|
||
|
||
def _LoadDeps(self, sha1: str) -> dict[str, str] | None:
|
||
"""加载依赖指纹"""
|
||
deps_path = os.path.join(self.output_dir, f"{sha1}.deps.json")
|
||
if _CachedIsFile(deps_path):
|
||
try:
|
||
return json.loads(_ReadTextCached(deps_path))
|
||
except Exception as _e:
|
||
if _config_mode == "strict":
|
||
raise
|
||
_vlog().warning(f"加载依赖指纹 JSON 失败: {_e}", "Exception")
|
||
return None
|
||
|
||
def _check_deps_changed(self, sha1: str, current_ModuleSha1Map: dict[str, str]) -> bool:
|
||
"""检查依赖指纹是否变化"""
|
||
saved_deps = self._LoadDeps(sha1)
|
||
if saved_deps is None:
|
||
return True
|
||
for mod_name, dep_sha1 in current_ModuleSha1Map.items():
|
||
if saved_deps.get(mod_name) != dep_sha1:
|
||
return True
|
||
for mod_name, dep_sha1 in saved_deps.items():
|
||
if current_ModuleSha1Map.get(mod_name) != dep_sha1:
|
||
return True
|
||
return False
|
||
|
||
def _recombine_ll(self, sha1: str, current_ModuleSha1Map: dict[str, str]) -> bool:
|
||
"""依赖变化时重新组合 .ll：更新 .stub.ll 中的 SHA1 前缀，拼接 .stub.ll + .text.ll"""
|
||
saved_deps = self._LoadDeps(sha1)
|
||
if saved_deps is None:
|
||
return False
|
||
|
||
stub_path = os.path.join(self.output_dir, f"{sha1}.stub.ll")
|
||
text_path = os.path.join(self.output_dir, f"{sha1}.text.ll")
|
||
ll_path = os.path.join(self.output_dir, f"{sha1}.ll")
|
||
|
||
if not _CachedIsFile(stub_path) or not _CachedIsFile(text_path):
|
||
return False
|
||
|
||
stub_content = _ReadTextCached(stub_path)
|
||
text_content = _ReadTextCached(text_path)
|
||
|
||
sha1_replacements = {}
|
||
for mod_name, old_sha1 in saved_deps.items():
|
||
new_sha1 = current_ModuleSha1Map.get(mod_name)
|
||
if new_sha1 and old_sha1 != new_sha1:
|
||
sha1_replacements[old_sha1] = new_sha1
|
||
|
||
if sha1_replacements:
|
||
for old_sha1, new_sha1 in sha1_replacements.items():
|
||
stub_content = stub_content.replace(old_sha1, new_sha1)
|
||
text_content = text_content.replace(old_sha1, new_sha1)
|
||
|
||
combined = stub_content
|
||
if not combined.endswith('\n'):
|
||
combined += '\n'
|
||
combined += text_content
|
||
|
||
with open(ll_path, 'w', encoding='utf-8', newline='\n') as f:
|
||
f.write(combined)
|
||
with open(stub_path, 'w', encoding='utf-8', newline='\n') as f:
|
||
f.write(stub_content)
|
||
with open(text_path, 'w', encoding='utf-8', newline='\n') as f:
|
||
f.write(text_content)
|
||
_InvalidateTextCache(ll_path)
|
||
_InvalidateTextCache(stub_path)
|
||
_InvalidateTextCache(text_path)
|
||
|
||
self._save_deps(sha1, current_ModuleSha1Map)
|
||
|
||
obj_path = os.path.join(self.output_dir, f"{sha1}.obj")
|
||
if _CachedIsFile(obj_path):
|
||
os.remove(obj_path)
|
||
|
||
return True
|
||
|
||
def _inject_auto_imports(self, code: str, src_path: str) -> str:
|
||
"""检测源码中对 list/dict 容器的使用，自动注入 import _list / import _dict。
|
||
当使用 dict 时，同时注入 import json (用于 dict 的 JSON 处理功能)。
|
||
检测到 assert 语句时自动注入 import testcheck（assert 译为
|
||
testcheck.check(...)，需该模块的跨模块符号声明）。
|
||
|
||
检测模式:
|
||
- list[T] / dict[V] 作为类型注解 (ast.Subscript, value=Name(id='list'/'dict'))
|
||
- list[T](...) / dict[V](...) 作为构造函数调用 (ast.Call, func=Subscript)
|
||
- list(...) / dict(...) 作为构造函数调用 (ast.Call, func=Name)
|
||
- assert 语句 (ast.Assert)
|
||
"""
|
||
try:
|
||
# `.vp` 源码须先过 Viper 软关键字层（`enum X:` / `fn` 等）再解析：
|
||
# 否则此处 SyntaxError 会静默 return 原 code（自动注入 import _list/_dict
|
||
# 失效），与主翻译路径（Shell/TransPyC 已展开）口径不一致。
|
||
tree = ast.parse(ExpandSoftKeywords(code), src_path)
|
||
except SyntaxError:
|
||
return code
|
||
|
||
# 检查是否已经导入了 _list / _dict / json / testcheck
|
||
has_list_import: bool = False
|
||
has_dict_import: bool = False
|
||
has_json_import: bool = False
|
||
has_testcheck_import: bool = False
|
||
for node in ast.iter_child_nodes(tree):
|
||
if isinstance(node, ast.Import):
|
||
for alias in node.names:
|
||
if alias.name == '_list':
|
||
has_list_import = True
|
||
elif alias.name == '_dict':
|
||
has_dict_import = True
|
||
elif alias.name == 'json':
|
||
has_json_import = True
|
||
elif alias.name == 'testcheck':
|
||
has_testcheck_import = True
|
||
elif isinstance(node, ast.ImportFrom):
|
||
if node.module == '_list':
|
||
has_list_import = True
|
||
elif node.module == '_dict':
|
||
has_dict_import = True
|
||
elif node.module == 'json':
|
||
has_json_import = True
|
||
elif node.module == 'testcheck':
|
||
has_testcheck_import = True
|
||
if has_list_import and has_dict_import and has_json_import and has_testcheck_import:
|
||
break
|
||
|
||
# 检查是否使用了 list / dict 容器，以及是否含 assert 语句
|
||
uses_list: bool = False
|
||
uses_dict: bool = False
|
||
uses_assert: bool = False
|
||
for node in ast.walk(tree):
|
||
if isinstance(node, ast.Assert):
|
||
uses_assert = True
|
||
if isinstance(node, ast.Subscript):
|
||
if isinstance(node.value, ast.Name):
|
||
if node.value.id == 'list':
|
||
uses_list = True
|
||
elif node.value.id == 'dict':
|
||
uses_dict = True
|
||
if isinstance(node, ast.Call) and isinstance(node.func, ast.Name):
|
||
if node.func.id == 'list':
|
||
uses_list = True
|
||
elif node.func.id == 'dict':
|
||
uses_dict = True
|
||
# 检测类型注解 d: dict / l: list (非泛型形式)
|
||
if isinstance(node, ast.AnnAssign) and isinstance(node.annotation, ast.Name):
|
||
if node.annotation.id == 'dict':
|
||
uses_dict = True
|
||
elif node.annotation.id == 'list':
|
||
uses_list = True
|
||
|
||
# 注入 import 到源码开头
|
||
prefix: str = ''
|
||
if uses_list and not has_list_import:
|
||
prefix += 'import _list\n'
|
||
if uses_dict and not has_dict_import:
|
||
prefix += 'import _dict\n'
|
||
if uses_dict and not has_json_import:
|
||
prefix += 'import json\n'
|
||
# 含 assert 的模块等价于在文件顶部写下 import testcheck（已导入则不重复）
|
||
if uses_assert and not has_testcheck_import:
|
||
prefix += 'import testcheck\n'
|
||
if prefix:
|
||
return prefix + code
|
||
return code
|
||
|
||
def _translate_file(self, src_path: str, out_path: str, prebuilt_ModuleSha1Map: dict[str, str] | None = None, dwarf_enabled: bool = False) -> None:
|
||
"""翻译单个源文件，嵌入相关 .stub.ll 声明
|
||
|
||
dwarf_enabled: 函数级 DWARF 调试元数据（--dwarf），默认 False（零影响）。
|
||
App 源模块在 --dwarf/--asan/--traceback 时传 True。include 文件默认
|
||
传 False（其 .obj 会落进跨项目共享的 includes.binary 预编译缓存，注入 DI
|
||
会污染该缓存）；仅 --traceback 时传 True——此时 include 的 .obj 改落
|
||
隔离目录（includes.binary_traceback，见 _get_precompiled_obj_path），
|
||
不会污染共享缓存。
|
||
"""
|
||
_t0: float = time.time()
|
||
# 每个文件翻译入口置脏：本文件/其它文件在此之间新写的 *.stub.ll 全部可见
|
||
# （ImportHandle._build_stub_index 的零 syscall 快路径据此判断是否重扫目录；
|
||
# 顺序模式与 worker 模式的每文件粒度皆覆盖，见 MarkStubSetDirty）。
|
||
from lib.core.Handles.HandlesImports import ImportHandle as _IH
|
||
_IH.MarkStubSetDirty()
|
||
code = _ReadTextCached(src_path)
|
||
|
||
# SHA1 以主流程 out_path 文件名为准（= 主流程 compute_sha1(parse_python_file 内容)），
|
||
# 必须在自动导入注入之前确定。不可在此重读文件重算：主流程收集（need_translate）
|
||
# 与本次实际翻译之间若文件内容变化（如编辑器保存），重算会得到不同 sha1，
|
||
# 导致函数定义前缀 ≠ 文件名/调用方引用前缀 → 链接 undefined symbol
|
||
# （实例：696d41ca86e0a29a.BuildTopoSortedIndices 定义被生成成 5536… 前缀）。
|
||
sha1 = os.path.splitext(os.path.basename(out_path))[0]
|
||
cur_sha1 = compute_sha1(code)
|
||
if cur_sha1 != sha1:
|
||
_vlog().warning(
|
||
f"模块内容在收集后已变化(文件名sha1={sha1}, 当前sha1={cur_sha1}): {src_path}，"
|
||
f"仍按主流程 sha1 生成前缀以保证链接一致；下次构建将采用新 sha1", "SHA1")
|
||
|
||
# 自动导入: 检测 list 容器使用，自动注入 import _list
|
||
# 注入行会让 AST 节点 lineno 整体下移 N（磁盘真实行号 = AST 行号 - N）；
|
||
# 记下 N 供报错渲染还原磁盘行号（如 _RaiseUnresolvedExpr）。
|
||
_raw_newlines: int = code.count('\n')
|
||
code = self._inject_auto_imports(code, src_path)
|
||
_line_offset: int = code.count('\n') - _raw_newlines
|
||
|
||
if prebuilt_ModuleSha1Map is not None:
|
||
ModuleSha1Map = prebuilt_ModuleSha1Map
|
||
else:
|
||
ModuleSha1Map = self._build_current_ModuleSha1Map(sha1)
|
||
|
||
trans = TransPyC.TransPyC(code=code, triple=self.triple, datalayout=self.datalayout)
|
||
trans.translator.CurrentFile = src_path
|
||
# 函数级 DWARF（--dwarf）：透传给 Translator，由其传给每个新 LlvmCodeGenerator；
|
||
# 未启用时不设置 → 与改动前逐字节一致（零影响）
|
||
if dwarf_enabled:
|
||
trans.translator.DwarfEnabled = True
|
||
# Ctrl+C 调用栈（--traceback）：仅当本文件就是 traceback 模块时置位——
|
||
# 只有该模块需要注入 @llvm.global_ctors 构造器调用自己的 Install()
|
||
# （见 BaseGen._RegisterTracebackInstallViaCtor）。其它模块不设置该属性
|
||
# → Gen.traceback_ctor 保持 False，IR 与改动前逐字节一致（零影响）。
|
||
_traceback_ctor: bool = bool(
|
||
self.traceback_enabled
|
||
and os.path.basename(src_path) in _TRACEBACK_MODULE_FILES)
|
||
if _traceback_ctor:
|
||
trans.translator.TracebackCtor = True
|
||
trans.SliceLevel = self.slice_level
|
||
trans.translator.SliceLevel = self.slice_level
|
||
trans.translator.SliceCount = 0
|
||
trans.translator.SliceInfos = []
|
||
trans.translator.LlvmGen = None
|
||
trans.translator._module_sha1 = sha1
|
||
trans.translator._line_offset = _line_offset
|
||
trans.translator._ModuleSha1Map = ModuleSha1Map
|
||
trans.translator._struct_sha1_map = self.struct_sha1_map
|
||
trans.translator._global_function_default_args = self.function_default_args
|
||
trans.translator._global_function_param_names = self.function_param_names
|
||
trans.translator._temp_dir = self.temp_dir
|
||
# includes 只认 project.vpj 配置的 include 目录（不再回退仓库 TransPyCRoot/includes）：
|
||
# 目录重组/缺失时避免声明解析从仓库或 sig 回退路径产生错误时序（memhub 未注册→i32）
|
||
_proj_inc = [os.path.abspath(d) for d in self.include_dirs if os.path.isdir(d)]
|
||
trans.translator.LibraryPaths = _proj_inc if _proj_inc else ['.']
|
||
trans.translator._shared_doc_meta = self._shared_doc_meta
|
||
# 传递共享 vtable 状态
|
||
trans.translator._shared_class_vtable = self._shared_class_vtable
|
||
trans.translator._shared_cross_module_vtable = self._shared_cross_module_vtable
|
||
trans.translator._shared_class_methods = self._shared_class_methods
|
||
trans.translator._shared_cross_module_novtable = self._shared_cross_module_novtable
|
||
|
||
# 调用点热点统计与每函数计时（--hotspot-sites，Projectrans 多模块）：
|
||
# 各模块处理器统一以追加方式写最终 <path>，注册策略为本模块
|
||
# @llvm.global_ctors 自注册（见 BaseGen._InstallHotspotSitesModuleLocal）；
|
||
# self.hotspot_sites_file 为 None 时不设置任何属性 → 与未启用时逐字节一致。
|
||
if self.hotspot_sites_file:
|
||
trans.translator.HotspotSitesFile = self.hotspot_sites_file
|
||
trans.translator.HotspotSitesAll = self.hotspot_sites_all
|
||
trans.translator.HotspotCallees = self.hotspot_callees
|
||
trans.translator.HotspotSitesModuleLocal = True
|
||
|
||
if self._shared_symbol_table is not None:
|
||
trans.translator.SymbolTable = self._shared_symbol_table
|
||
# 确保共享符号表的 translator 指向当前 translator，以便 import 别名解析正确
|
||
if hasattr(self._shared_symbol_table, 'translator'):
|
||
self._shared_symbol_table.translator = trans.translator
|
||
trans.translator._source_module_sig_files = self._shared_source_module_sig_files
|
||
trans.translator._all_define_constants = self._shared_all_dc
|
||
trans.translator._define_decl_types = self._shared_all_decl_types
|
||
trans.translator._export_extern_funcs = self._shared_export_extern_funcs
|
||
if self._shared_generic_class_templates:
|
||
if not hasattr(trans.translator.ClassHandler, '_generic_class_templates'):
|
||
trans.translator.ClassHandler._generic_class_templates = {}
|
||
trans.translator.ClassHandler._generic_class_templates.update(self._shared_generic_class_templates)
|
||
else:
|
||
export_extern_funcs = {}
|
||
for sha1_key, sig_path in self.sig_files.items():
|
||
if sha1_key == sha1:
|
||
continue
|
||
try:
|
||
sig_content = _ReadTextCached(sig_path)
|
||
sig_tree = ast.parse(sig_content, sig_path)
|
||
for node in ast.iter_child_nodes(sig_tree):
|
||
if isinstance(node, ast.FunctionDef):
|
||
if _IsExportExternFuncNode(node):
|
||
export_extern_funcs[node.name] = sha1_key
|
||
except Exception as _e:
|
||
if _config_mode == "strict":
|
||
raise
|
||
_vlog().warning(f"收集导出外部函数声明失败: {_e}", "Exception")
|
||
if export_extern_funcs:
|
||
trans.translator._export_extern_funcs = export_extern_funcs
|
||
|
||
for includes_dir in self.include_dirs:
|
||
if os.path.isdir(includes_dir):
|
||
for pyi_file in os.listdir(includes_dir):
|
||
if pyi_file.endswith('.vpi') or pyi_file.endswith('.pyi'):
|
||
pyi_path = os.path.join(includes_dir, pyi_file)
|
||
module_name = os.path.splitext(pyi_file)[0]
|
||
trans.translator.SymbolTable.LoadModuleSymbols(pyi_path, module_name, lineno=0)
|
||
for py_file in os.listdir(includes_dir):
|
||
if (py_file.endswith('.vp') or py_file.endswith('.py')) and (not py_file.startswith('_') or py_file == '_list.vp' or py_file == '_list.py' or py_file == '_dict.vp' or py_file == '_dict.py'):
|
||
module_name = os.path.splitext(py_file)[0]
|
||
sha1_key = self.include_py_map.get(module_name)
|
||
if sha1_key and sha1_key in self.sig_files:
|
||
sig_path = self.sig_files[sha1_key]
|
||
trans.translator.SymbolTable.LoadModuleSymbols(sig_path, module_name, lineno=0)
|
||
# 收集需要重导出的包，等所有模块符号加载完后再处理
|
||
_pending_reexports = []
|
||
for sha1_key, sig_path in self.sig_files.items():
|
||
rel_path = self.sha1_map.get(sha1_key, sha1_key)
|
||
if rel_path.startswith('includes/'):
|
||
mod_name = os.path.splitext(rel_path)[0].replace(os.sep, '.').replace('/', '.')
|
||
mod_name = mod_name[len('includes/'):]
|
||
trans.translator.SymbolTable.LoadModuleSymbols(sig_path, mod_name, lineno=0)
|
||
if mod_name.endswith('.__init__'):
|
||
package_name = mod_name[:-len('.__init__')]
|
||
_pending_reexports.append((sig_path, package_name))
|
||
|
||
# 所有 includes 模块符号已加载，现在处理包重导出
|
||
for sig_path, package_name in _pending_reexports:
|
||
self._register_package_reexports(trans, sig_path, package_name)
|
||
|
||
for sha1_key, sig_path in self.sig_files.items():
|
||
if sha1_key == sha1:
|
||
continue
|
||
rel_path = self.sha1_map.get(sha1_key, sha1_key)
|
||
if rel_path.startswith('includes/'):
|
||
continue
|
||
module_name = os.path.splitext(rel_path)[0].replace(os.sep, '.').replace('/', '.')
|
||
trans.translator.SymbolTable.LoadModuleSymbols(sig_path, module_name, lineno=0)
|
||
trans.translator._source_module_sig_files[module_name] = sig_path
|
||
short_name = module_name.split('.')[-1] if '.' in module_name else module_name
|
||
if short_name != module_name:
|
||
trans.translator._source_module_sig_files[short_name] = sig_path
|
||
|
||
all_dc = {}
|
||
all_dt: dict[str, tuple[int, bool]] = {}
|
||
for sha1_key, stub_path in self.stub_files.items():
|
||
if sha1_key == sha1:
|
||
continue
|
||
if os.path.exists(stub_path):
|
||
try:
|
||
stub_content = _ReadTextCached(stub_path)
|
||
for line in stub_content.splitlines():
|
||
line = line.strip()
|
||
if line.startswith('@') and '= global' in line and 'i32' in line and not line.startswith('@llvm'):
|
||
var_name = line.split('=')[0].strip().lstrip('@')
|
||
val_part = line.split('i32')[-1].strip().rstrip(']').lstrip('[').lstrip('i32')
|
||
try:
|
||
val = int(val_part)
|
||
all_dc[var_name] = val
|
||
except Exception as e:
|
||
_vlog().warning(f"Phase2Translator: 忽略异常 {e}", exc_info=e)
|
||
except Exception as e:
|
||
_vlog().warning(f"Phase2Translator: 忽略异常 {e}", exc_info=e)
|
||
for sha1_key, pyi_path in self.sig_files.items():
|
||
if sha1_key == sha1:
|
||
continue
|
||
_ExtractCDefineConstantsFromPyi(pyi_path, all_dc, trans.translator.SymbolTable, all_dt)
|
||
if all_dc:
|
||
trans.translator._all_define_constants = all_dc
|
||
if all_dt:
|
||
trans.translator._define_decl_types = all_dt
|
||
|
||
for sym_name, sym_info in self.inline_func_symbols.items():
|
||
if sym_name not in trans.translator.SymbolTable:
|
||
trans.translator.SymbolTable[sym_name] = sym_info
|
||
else:
|
||
existing = trans.translator.SymbolTable[sym_name]
|
||
if not getattr(existing, 'InlineBody', None) and getattr(sym_info, 'InlineBody', None):
|
||
existing.IsInline = sym_info.IsInline
|
||
existing.InlineBody = sym_info.InlineBody
|
||
existing.InlineParams = sym_info.InlineParams
|
||
|
||
try:
|
||
result = trans.Convert(
|
||
OutputFilename=out_path,
|
||
SourceFilename=src_path,
|
||
target='llvm'
|
||
)
|
||
except Exception as e:
|
||
error_stack = getattr(trans.translator, '_ErrorStack', [])
|
||
if error_stack:
|
||
self._last_ErrorStack = error_stack
|
||
# 语句栈渲染：error_stack 按"由内向外"入栈（内层语句先 append），
|
||
# 这里反转为"外层 → 内层"的调用路径；**异常消息只在最内层印一次**，
|
||
# 其余帧只印位置——否则同一段长 fail-fast 报文会被每个帧重复一遍
|
||
# （旧格式把消息贴在每一帧后面，输出里同一句话出现 N 次，不可读）。
|
||
entries: list = [entry for entry in error_stack if len(entry) >= 3]
|
||
chain_lines: list[str] = []
|
||
for chain_idx, entry in enumerate(reversed(entries)):
|
||
exc_msg, line_info = entry[1], entry[2]
|
||
if chain_idx == len(entries) - 1:
|
||
chain_lines.append(" %s\n %s" % (line_info, exc_msg))
|
||
else:
|
||
chain_lines.append(" %s" % (line_info,))
|
||
if chain_lines:
|
||
enriched = str(e) + "\n" + "\n".join(chain_lines)
|
||
raise type(e)(enriched) from e
|
||
node_info = ""
|
||
if hasattr(trans.translator, 'LlvmGen') and trans.translator.LlvmGen:
|
||
node_info = trans.translator.LlvmGen._get_node_info()
|
||
if node_info:
|
||
enriched = "%s\n %s" % (str(e), node_info.strip())
|
||
raise type(e)(enriched) from e
|
||
raise
|
||
|
||
if result and isinstance(result, str):
|
||
for func_name, func_def in trans.translator.FunctionDefCache.items():
|
||
if hasattr(func_def, 'args'):
|
||
mangled = trans.translator.LlvmGen._mangle_func_name(func_name) if trans.translator.LlvmGen else func_name
|
||
# 提取参数名
|
||
if hasattr(func_def.args, 'args'):
|
||
param_names = [arg.arg for arg in func_def.args.args]
|
||
self.function_param_names[mangled] = param_names
|
||
self.function_param_names[func_name] = param_names
|
||
# 提取默认值
|
||
if hasattr(func_def.args, 'defaults') and func_def.args.defaults:
|
||
defaults = []
|
||
for d in func_def.args.defaults:
|
||
if isinstance(d, ast.Constant):
|
||
defaults.append(d.value)
|
||
else:
|
||
defaults.append(None)
|
||
self.function_default_args[mangled] = defaults
|
||
self.function_default_args[func_name] = defaults
|
||
for sym_name, sym_info in trans.translator.SymbolTable.items():
|
||
if isinstance(sym_info, CTypeInfo) and getattr(sym_info, 'IsInline', False) and getattr(sym_info, 'InlineBody', None):
|
||
self.inline_func_symbols[sym_name] = sym_info
|
||
|
||
out_dir = os.path.dirname(out_path)
|
||
if out_dir:
|
||
os.makedirs(out_dir, exist_ok=True)
|
||
# --traceback：核对注入钩子确实生效——本模块要求注入 @llvm.global_ctors
|
||
# 构造器（StructGen.finalize 里落成 .ll 文本之前），若产物中没有该全局
|
||
# 说明钩子不可用（如序列化路径被改动绕过），显式报错终止，不静默产出
|
||
# 缺注册的产物（那会让整个功能静默失效）。
|
||
if _traceback_ctor and 'llvm.global_ctors' not in result:
|
||
raise RuntimeError(
|
||
'--traceback: traceback 模块未注入 @llvm.global_ctors '
|
||
'（模块级收尾钩子未生效，不静默降级）')
|
||
# 修复 LLVM 22+: external global 不能有初值，改为 linkonce
|
||
result = _RE_EXTERNAL_GLOBAL_VALUE.sub(r'\1 = linkonce global \2 \3', result)
|
||
# 后处理：添加 llvmlite 不支持的 LLVM 属性（如 willreturn, mustprogress）
|
||
pending_str_attrs = getattr(trans.translator.LlvmGen, '_pending_str_attrs', None)
|
||
if pending_str_attrs:
|
||
for func_name, attrs in pending_str_attrs.items():
|
||
attrs_str = ' '.join(attrs)
|
||
# 匹配 define ... @func_name(...) ... { 并在 { 之前添加属性
|
||
pattern = rf'(define\s+[^@]*@{re.escape(func_name)}\s*\([^)]*\))([^{{]*)(\{{)'
|
||
result, n = re.subn(pattern, rf'\1\2 {attrs_str}\3', result, count=1)
|
||
if n == 0:
|
||
_vlog().warning(f"后处理添加属性失败: 未找到函数 {func_name} 的定义")
|
||
with open(out_path, 'w', encoding='utf-8', newline='\n') as f:
|
||
f.write(result)
|
||
_InvalidateTextCache(out_path)
|
||
|
||
stub_content, text_content = self._split_ll(result)
|
||
base = os.path.splitext(out_path)[0]
|
||
with open(base + '.stub.ll', 'w', encoding='utf-8', newline='\n') as f:
|
||
f.write(stub_content)
|
||
_InvalidateTextCache(base + '.stub.ll')
|
||
# 新 stub 已落盘 → 置脏，使本文件后续（及下个文件）的索引 miss 能重扫到它
|
||
from lib.core.Handles.HandlesImports import ImportHandle as _IH2
|
||
_IH2.MarkStubSetDirty()
|
||
with open(base + '.text.ll', 'w', encoding='utf-8', newline='\n') as f:
|
||
f.write(text_content)
|
||
_InvalidateTextCache(base + '.text.ll')
|
||
self._save_deps(sha1, ModuleSha1Map)
|
||
|
||
# 进度日志由调用方统一打印（主进程按文件粒度 [x/total]），此处不再重复输出。
|
||
else:
|
||
stub_path = self.stub_files.get(sha1)
|
||
if stub_path and os.path.exists(stub_path):
|
||
out_dir = os.path.dirname(out_path)
|
||
if out_dir:
|
||
os.makedirs(out_dir, exist_ok=True)
|
||
stub_content = _ReadTextCached(stub_path)
|
||
with open(out_path, 'w', encoding='utf-8', newline='\n') as f:
|
||
f.write(stub_content)
|
||
# 进度日志由调用方统一打印。
|
||
|
||
def _register_package_reexports(self, trans, init_pyi_path: str, package_name: str) -> None:
|
||
"""解析 __init__.pyi 中的 from .xxx import yyy 或 from __xxx import yyy，在包级别注册导出符号"""
|
||
try:
|
||
# 复用 parse_python_file 的 AST 缓存
|
||
_, tree = parse_python_file(init_pyi_path)
|
||
if tree is None:
|
||
return
|
||
for node in ast.walk(tree):
|
||
if isinstance(node, ast.ImportFrom) and node.module:
|
||
sub_module = node.module.lstrip('.')
|
||
for alias in node.names:
|
||
symbol_name = alias.name
|
||
exported_name = alias.asname if alias.asname else symbol_name
|
||
source_keys = [
|
||
f"{package_name}.{sub_module}.{symbol_name}",
|
||
f"{package_name}.__{sub_module}.{symbol_name}" if not sub_module.startswith('__') else None,
|
||
symbol_name,
|
||
]
|
||
target_key = f"{package_name}.{exported_name}"
|
||
if target_key not in trans.translator.SymbolTable:
|
||
for source_key in source_keys:
|
||
if source_key and source_key in trans.translator.SymbolTable:
|
||
trans.translator.SymbolTable[target_key] = trans.translator.SymbolTable[source_key]
|
||
break
|
||
if exported_name not in trans.translator.SymbolTable:
|
||
for source_key in source_keys:
|
||
if source_key and source_key in trans.translator.SymbolTable:
|
||
trans.translator.SymbolTable[exported_name] = trans.translator.SymbolTable[source_key]
|
||
break
|
||
except Exception as e:
|
||
_vlog().warning(f" [警告] 解析包重导出失败 {init_pyi_path}: {e}")
|
||
|
||
def _SanitizeIRFile(self, ll_path: str) -> str:
|
||
"""送入 opt/llc 之前，对该 .ll 做一次元数据规整（见 IRMetadataFix）。
|
||
|
||
App 模块的 IR 是「本模块 IR + 各依赖 .stub.ll 片段」直接拼接而成，而
|
||
--dwarf（--asan 自动开启）给每份片段都加了从 !0 开始编号的 DI 元数据，
|
||
合并文件里同一个 !N 被定义多次 → opt/llc 报 "Metadata id is already
|
||
used"、命名元数据 "defined more than once"。规整后编号全局唯一、
|
||
命名元数据只留一份。无元数据 / 已规整的文件逐字节不变（no-op），
|
||
原地覆盖没有额外产物。
|
||
"""
|
||
with open(ll_path, 'r', encoding='utf-8', newline='') as f:
|
||
text = f.read()
|
||
fixed = SanitizeIRMetadata(text)
|
||
if fixed != text:
|
||
with open(ll_path, 'w', encoding='utf-8', newline='') as f:
|
||
f.write(fixed)
|
||
_InvalidateTextCache(ll_path)
|
||
return ll_path
|
||
|
||
def _llc_compile(self, ll_path: str, obj_path: str,
|
||
cwd: str | None = None) -> subprocess.CompletedProcess:
|
||
"""编译单个 .ll 为 obj。
|
||
|
||
ASan 模式（--asan）下先对 IR 执行 opt 插桩：
|
||
检测 stdlib 堆/栈/全局越界写与垃圾指针解引用（MemBuddy pool 内部
|
||
由块尾 canary 覆盖，两者互补）。opt 失败返回其结果（returncode!=0）。
|
||
|
||
LLVM >= 20 起 opt -passes=asan 只对带 sanitize_address 函数属性的
|
||
函数插桩（clang -fsanitize=address 自动加；手写 IR 没有）——
|
||
先 forceattrs 批量加属性再 asan，否则只链接 runtime 不插桩。
|
||
"""
|
||
ll_path = self._SanitizeIRFile(ll_path)
|
||
compile_input = ll_path
|
||
if self.asan_mode:
|
||
asan_ll = ll_path[:-3] + '_asan.ll'
|
||
# -asan-globals=false：TPC 对全局变量（如各模块 _g_gen）用直接地址寻址，
|
||
# ASan 全局 redzone 重定位后引用错位会导致读到垃圾指针（如
|
||
# register_struct 的 self=0xbebebebe...）。先关闭全局插桩排除该因素。
|
||
opt_cmd = ['opt', '-passes=forceattrs,asan', '-force-attribute=sanitize_address',
|
||
'-asan-globals=false', ll_path, '-o', asan_ll]
|
||
opt_r = subprocess.run(opt_cmd, capture_output=True, text=True, cwd=cwd)
|
||
if opt_r.returncode != 0:
|
||
_vlog().error(f" opt ASan 插桩失败: {opt_r.stderr.strip()}")
|
||
return opt_r
|
||
compile_input = asan_ll
|
||
cmd = [self.compile_cmd] + self.compile_flags + ['-o', obj_path, compile_input]
|
||
if self.traceback_enabled:
|
||
# --traceback：让 llc 保留帧指针（-frame-pointer=all）。dbghelp 主路径
|
||
# 不依赖帧指针，但符号表不可用时要靠 RBP 链兜底——llc 默认
|
||
# omit-frame-pointer 会让 RBP 链只走出 1~2 帧（实测"共 2 帧"）。
|
||
# 只在开启该开关时加，关闭时命令与改前逐字一致。
|
||
cmd = [self.compile_cmd] + self.compile_flags \
|
||
+ ['-frame-pointer=all', '-o', obj_path, compile_input]
|
||
return subprocess.run(cmd, capture_output=True, text=True, cwd=cwd)
|
||
|
||
def _compile_ll_files(self, active_sha1s: set[str]) -> None:
|
||
"""将 output_dir 中属于 active_sha1s 的 .ll 文件编译为 .obj（跳过已编译过的 include 文件）"""
|
||
# 第一次 walk：清理 stale 文件（.obj/.deps.json/纯 .ll，排除 .stub.ll/.text.ll）
|
||
stale_files = []
|
||
for root, dirs, files in os.walk(self.output_dir):
|
||
for file in files:
|
||
if file.endswith('.obj'):
|
||
sha1 = file[:-4]
|
||
if sha1 not in active_sha1s:
|
||
stale_files.append(os.path.join(root, file))
|
||
elif file.endswith('.deps.json'):
|
||
sha1 = file[:-10]
|
||
if sha1 not in active_sha1s:
|
||
stale_files.append(os.path.join(root, file))
|
||
elif file.endswith('.ll') and not file.endswith('.stub.ll') and not file.endswith('.text.ll'):
|
||
sha1 = file[:-3]
|
||
if sha1 not in active_sha1s:
|
||
stale_files.append(os.path.join(root, file))
|
||
if stale_files:
|
||
for fpath in stale_files:
|
||
try:
|
||
os.remove(fpath)
|
||
# 删除也是"本进程改变了该路径"：不失效会让后续读拿到已删文件的旧文本
|
||
_InvalidateTextCache(fpath)
|
||
except Exception as e:
|
||
_vlog().warning(f"Phase2Translator: 忽略异常 {e}", exc_info=e)
|
||
_vlog().info(f"[清理] 删除 {len(stale_files)} 个过时文件")
|
||
|
||
# 第二次 walk：收集需要编译的 .ll 文件（排除 .stub.ll/.text.ll）
|
||
ll_files = []
|
||
for root, dirs, files in os.walk(self.output_dir):
|
||
for file in files:
|
||
if file.endswith('.ll') and not file.endswith('.stub.ll') and not file.endswith('.text.ll'):
|
||
sha1 = file[:-3]
|
||
if sha1 in active_sha1s:
|
||
fpath = os.path.join(root, file)
|
||
obj_path = os.path.join(root, file[:-3] + '.obj')
|
||
if not _CachedIsFile(obj_path):
|
||
ll_files.append(fpath)
|
||
if not ll_files:
|
||
_vlog().info("[编译] 无 .ll 文件需要重新编译")
|
||
else:
|
||
_vlog().info(f"\n[编译] 找到 {len(ll_files)} 个 .ll 文件需要编译")
|
||
|
||
# 第三次 walk：收集 all_ll_files（用于 alwaysinline 检测，排除 include sha1s）
|
||
all_ll_files = []
|
||
for root, dirs, files in os.walk(self.output_dir):
|
||
for file in files:
|
||
if file.endswith('.ll') and not file.endswith('.stub.ll') and not file.endswith('.text.ll'):
|
||
sha1 = file[:-3]
|
||
if sha1 in active_sha1s and sha1 not in self._include_sha1s:
|
||
all_ll_files.append(os.path.join(root, file))
|
||
|
||
has_inline = False
|
||
for ll_path in all_ll_files:
|
||
try:
|
||
content = _ReadTextCached(ll_path)
|
||
if 'alwaysinline' in content:
|
||
has_inline = True
|
||
break
|
||
except Exception as e:
|
||
_vlog().warning(f"Phase2Translator: 忽略异常 {e}", exc_info=e)
|
||
|
||
if has_inline:
|
||
llvm_link = shutil.which('llvm-link') or shutil.which('llvm-link.exe')
|
||
opt_cmd = shutil.which('opt') or shutil.which('opt.exe')
|
||
if llvm_link and opt_cmd:
|
||
_vlog().info(" [内联] 检测到 alwaysinline 函数，执行跨模块内联优化...")
|
||
for ll_path in all_ll_files:
|
||
try:
|
||
content = _ReadTextCached(ll_path)
|
||
type_defs = []
|
||
type_def_indices = set()
|
||
for i, line in enumerate(content.splitlines()):
|
||
stripped = line.strip()
|
||
if stripped.startswith('%') and '= type' in stripped:
|
||
type_defs.append(line)
|
||
type_def_indices.add(i)
|
||
if type_def_indices:
|
||
lines = content.splitlines(True)
|
||
content_no_types = ''.join(line for i, line in enumerate(lines) if i not in type_def_indices)
|
||
header_end = 0
|
||
for i, line in enumerate(content_no_types.splitlines()):
|
||
stripped = line.strip()
|
||
if stripped.startswith(';') or stripped.startswith('target ') or stripped.startswith('source_filename') or not stripped:
|
||
header_end = i + 1
|
||
continue
|
||
break
|
||
lines2 = content_no_types.splitlines(True)
|
||
fixed = ''.join(lines2[:header_end]) + '\n'.join(type_defs) + '\n' + ''.join(lines2[header_end:])
|
||
with open(ll_path, 'w', encoding='utf-8') as f:
|
||
f.write(fixed)
|
||
_InvalidateTextCache(ll_path)
|
||
except Exception as e:
|
||
_vlog().warning(f"Phase2Translator: 忽略异常 {e}", exc_info=e)
|
||
merged_ll = os.path.join(self.output_dir, '_merged.ll')
|
||
optimized_ll = os.path.join(self.output_dir, '_merged_opt.ll')
|
||
try:
|
||
link_cmd = [llvm_link] + all_ll_files + ['-o', merged_ll]
|
||
result = subprocess.run(link_cmd, capture_output=True, text=True)
|
||
if result.returncode != 0:
|
||
_vlog().warning(f" llvm-link 失败: {result.stderr.strip()}，回退到单独编译")
|
||
has_inline = False
|
||
else:
|
||
opt_result_cmd = [opt_cmd, '-passes=always-inline', merged_ll, '-S', '-o', optimized_ll]
|
||
result = subprocess.run(opt_result_cmd, capture_output=True, text=True)
|
||
if result.returncode != 0:
|
||
_vlog().warning(f" opt 失败: {result.stderr.strip()}，回退到单独编译")
|
||
has_inline = False
|
||
else:
|
||
_vlog().info(" [内联] 跨模块内联优化完成")
|
||
opt_content = _ReadTextCached(optimized_ll)
|
||
opt_content = _RE_DEFINE_DECLARE_LINKAGE.sub(r'\1 \2 dso_local ', opt_content)
|
||
opt_content = _RE_DEFINE_DECLARE_NO_LINKAGE_AT.sub(r'\1 dso_local ', opt_content)
|
||
# LLVM 22+: external 不允许在带初值的全局定义上显式写出，改为 linkonce 保留多模块合并语义
|
||
opt_content = _RE_EXTERNAL_GLOBAL_VALUE_PLAIN.sub(r'\1 = linkonce global \2 \3', opt_content)
|
||
opt_content = _RE_GLOBAL_LINKAGE.sub(r'\1 = \2 dso_local \3', opt_content)
|
||
opt_content = _RE_GLOBAL_DSOLOCAL.sub(r'\1 = dso_local \2', opt_content)
|
||
opt_content = _AddWeakComdat(opt_content)
|
||
with open(optimized_ll, 'w', encoding='utf-8') as f:
|
||
f.write(opt_content)
|
||
merged_obj = os.path.join(self.output_dir, '_merged.obj')
|
||
try:
|
||
result = self._llc_compile(optimized_ll, merged_obj, self.output_dir)
|
||
if result.returncode == 0:
|
||
_vlog().success(f" 合并模块编译完成")
|
||
for ll_path in ll_files:
|
||
obj_path = ll_path[:-3] + '.obj'
|
||
sha1_obj = os.path.join(self.output_dir, os.path.basename(ll_path)[:-3] + '.obj')
|
||
if _CachedIsFile(sha1_obj):
|
||
os.remove(sha1_obj)
|
||
return
|
||
else:
|
||
_vlog().warning(f" 合并模块编译失败: {result.stderr.strip()}，回退到单独编译")
|
||
has_inline = False
|
||
except Exception as e:
|
||
_vlog().warning(f" 合并模块编译异常: {e}，回退到单独编译")
|
||
has_inline = False
|
||
except FileNotFoundError:
|
||
_vlog().warning(f" 找不到 llvm-link 或 opt，回退到单独编译")
|
||
has_inline = False
|
||
else:
|
||
_vlog().warning(" 未找到 llvm-link/opt 工具，回退到单独编译（alwaysinline 可能不生效）")
|
||
|
||
def _compile_one(ll_path: str) -> tuple[str, str, str]:
|
||
rel = os.path.relpath(ll_path, self.output_dir)
|
||
obj_path = ll_path[:-3] + '.obj'
|
||
if _CachedIsFile(obj_path):
|
||
return (rel, 'cached', '')
|
||
try:
|
||
ll_content = _ReadTextCached(ll_path)
|
||
sha1 = os.path.basename(ll_path)[:-3]
|
||
deps = self._LoadDeps(sha1)
|
||
stub_prefix = ''
|
||
if deps:
|
||
# 去重：同一个全局变量如果同时有 external 声明和实际定义，
|
||
# 移除 external 声明，避免 "redefinition of global" 错误
|
||
# 注意：此去重必须在收集现有符号之前执行，否则行号偏移会导致
|
||
# lines_to_remove 误删其他行（如全局变量定义）
|
||
ll_lines_for_dedup = ll_content.splitlines()
|
||
external_globals: dict[str, int] = {}
|
||
non_external_globals: dict[str, int] = {}
|
||
for i, line in enumerate(ll_lines_for_dedup):
|
||
stripped = line.strip()
|
||
if stripped.startswith('@') and (' global ' in stripped or ' constant ' in stripped):
|
||
gm = _RE_AT_NAME.match(stripped)
|
||
if gm:
|
||
gname = gm.group(1)
|
||
is_external = _RE_AT_EXTERNAL.match(stripped) is not None
|
||
if is_external:
|
||
external_globals[gname] = i
|
||
else:
|
||
non_external_globals[gname] = i
|
||
lines_to_remove_global_dedup: set[int] = set()
|
||
for gname in external_globals:
|
||
if gname in non_external_globals:
|
||
lines_to_remove_global_dedup.add(external_globals[gname])
|
||
if lines_to_remove_global_dedup:
|
||
ll_content = '\n'.join(
|
||
line for i, line in enumerate(ll_lines_for_dedup)
|
||
if i not in lines_to_remove_global_dedup
|
||
) + '\n'
|
||
existing_symbols: dict[str, str] = {}
|
||
opaque_line_indices: dict[str, int] = {}
|
||
existing_declares: set[str] = set()
|
||
existing_declare_lines: dict[str, set[int]] = {}
|
||
for i, line in enumerate(ll_content.splitlines()):
|
||
stripped = line.strip()
|
||
if stripped.startswith('%') and '= type' in stripped:
|
||
name = Phase2Translator._extract_llvm_type_name(stripped)
|
||
if name:
|
||
if '= type opaque' in stripped:
|
||
existing_symbols[name] = 'opaque'
|
||
opaque_line_indices[name] = i
|
||
else:
|
||
existing_symbols[name] = 'full'
|
||
elif stripped.startswith('declare'):
|
||
dm = _RE_DECLARE_NAME.match(stripped)
|
||
if dm:
|
||
fname = dm.group(1)
|
||
existing_declares.add(fname)
|
||
existing_declare_lines.setdefault(fname, set()).add(i)
|
||
seen_stub_sha1: set[str] = set()
|
||
replaced_opaque_names: set[str] = set()
|
||
replaced_declare_names: set[str] = set()
|
||
all_stub_lines: list[str | None] = []
|
||
stub_type_map: dict[str, tuple[bool, int]] = {}
|
||
stub_declare_names: set[str] = set()
|
||
stub_declare_lines: dict[str, str] = {}
|
||
existing_defines: set[str] = set()
|
||
existing_global_defs: set[str] = set()
|
||
stub_global_defs: set[str] = set()
|
||
for line in ll_content.splitlines():
|
||
stripped = line.strip()
|
||
if stripped.startswith('define'):
|
||
dm = _RE_DEFINE_NAME.match(stripped)
|
||
if dm:
|
||
existing_defines.add(dm.group(1))
|
||
elif stripped.startswith('@') and (' global ' in stripped or ' constant ' in stripped):
|
||
gm = _RE_AT_NAME.match(stripped)
|
||
if gm:
|
||
existing_global_defs.add(gm.group(1))
|
||
for dep_name, dep_sha1 in deps.items():
|
||
if dep_sha1 in seen_stub_sha1:
|
||
continue
|
||
seen_stub_sha1.add(dep_sha1)
|
||
stub_path = os.path.join(self.output_dir, f"{dep_sha1}.stub.ll")
|
||
if not _CachedIsFile(stub_path):
|
||
stub_path = os.path.join(self.temp_dir, f"{dep_sha1}.stub.ll")
|
||
if _CachedIsFile(stub_path):
|
||
stub_content = _ReadTextCached(stub_path)
|
||
for line in stub_content.splitlines():
|
||
stripped = line.strip()
|
||
if stripped.startswith(';') or stripped.startswith('target ') or stripped.startswith('source_filename') or not stripped:
|
||
continue
|
||
if stripped.startswith('declare'):
|
||
dm = _RE_DECLARE_NAME.match(stripped)
|
||
if dm:
|
||
fname = dm.group(1)
|
||
if 'zdef_alloc' in fname:
|
||
_vlog().debug(f"[DEBUG] stub declare for {fname}: existing_declares={fname in existing_declares}, stub_declare_names={fname in stub_declare_names}, has_struct={'%' in stripped}, existing={existing_declare_lines.get(fname)}")
|
||
if fname in existing_defines:
|
||
continue
|
||
if fname in existing_declares or fname in stub_declare_names:
|
||
existing_decl_line = stub_declare_lines.get(fname)
|
||
if existing_decl_line is not None:
|
||
has_struct = '%' in stripped
|
||
existing_has_struct = '%' in existing_decl_line if existing_decl_line else False
|
||
if has_struct and not existing_has_struct:
|
||
for i, l in enumerate(all_stub_lines):
|
||
if l is not None and l.strip().startswith('declare'):
|
||
em = _RE_DECLARE_NAME.match(l.strip())
|
||
if em and em.group(1) == fname:
|
||
all_stub_lines[i] = line
|
||
break
|
||
continue
|
||
elif fname in existing_declares:
|
||
has_struct = '%' in stripped
|
||
if has_struct:
|
||
replaced_declare_names.add(fname)
|
||
stub_declare_names.add(fname)
|
||
stub_declare_lines[fname] = line
|
||
all_stub_lines.append(line)
|
||
continue
|
||
continue
|
||
stub_declare_names.add(fname)
|
||
stub_declare_lines[fname] = line
|
||
all_stub_lines.append(line)
|
||
continue
|
||
if stripped.startswith('%') and '= type' in stripped:
|
||
name = Phase2Translator._extract_llvm_type_name(stripped)
|
||
if name:
|
||
is_opaque = '= type opaque' in stripped
|
||
if name in existing_symbols:
|
||
if existing_symbols[name] == 'opaque' and not is_opaque:
|
||
replaced_opaque_names.add(name)
|
||
existing_symbols[name] = 'full'
|
||
idx = len(all_stub_lines)
|
||
all_stub_lines.append(line)
|
||
stub_type_map[name] = (is_opaque, idx)
|
||
continue
|
||
if name in stub_type_map:
|
||
prev_opaque, prev_idx = stub_type_map[name]
|
||
if prev_opaque and not is_opaque:
|
||
all_stub_lines[prev_idx] = None
|
||
idx = len(all_stub_lines)
|
||
all_stub_lines.append(line)
|
||
stub_type_map[name] = (is_opaque, idx)
|
||
continue
|
||
idx = len(all_stub_lines)
|
||
all_stub_lines.append(line)
|
||
stub_type_map[name] = (is_opaque, idx)
|
||
else:
|
||
all_stub_lines.append(line)
|
||
elif stripped.startswith('@') and (' global ' in stripped or ' constant ' in stripped):
|
||
gm = _RE_AT_NAME.match(stripped)
|
||
if gm and (gm.group(1) in existing_global_defs or gm.group(1) in stub_global_defs):
|
||
continue
|
||
if gm:
|
||
stub_global_defs.add(gm.group(1))
|
||
all_stub_lines.append(line)
|
||
else:
|
||
all_stub_lines.append(line)
|
||
filtered = [l for l in all_stub_lines if l is not None]
|
||
if filtered:
|
||
stub_prefix = '\n'.join(filtered) + '\n'
|
||
lines_to_remove = set()
|
||
if replaced_opaque_names:
|
||
for name in replaced_opaque_names:
|
||
if name in opaque_line_indices:
|
||
lines_to_remove.add(opaque_line_indices[name])
|
||
if replaced_declare_names:
|
||
_vlog().debug(f"[DEBUG] replaced_declare_names: {replaced_declare_names}")
|
||
for fname in replaced_declare_names:
|
||
if fname in existing_declare_lines:
|
||
lines_to_remove.update(existing_declare_lines[fname])
|
||
_vlog().debug(f"[DEBUG] removing lines {existing_declare_lines[fname]} for {fname}")
|
||
else:
|
||
_vlog().debug(f"[DEBUG] fname {fname} not in existing_declare_lines: {list(existing_declare_lines.keys())}")
|
||
if lines_to_remove:
|
||
ll_lines = ll_content.splitlines(True)
|
||
ll_content = ''.join(line for i, line in enumerate(ll_lines) if i not in lines_to_remove)
|
||
type_defs_in_ll = []
|
||
type_def_line_indices = set()
|
||
for i, line in enumerate(ll_content.splitlines()):
|
||
stripped = line.strip()
|
||
if stripped.startswith('%') and '= type' in stripped:
|
||
type_defs_in_ll.append(line)
|
||
type_def_line_indices.add(i)
|
||
if type_def_line_indices:
|
||
ll_lines = ll_content.splitlines(True)
|
||
ll_content = ''.join(line for i, line in enumerate(ll_lines) if i not in type_def_line_indices)
|
||
if type_defs_in_ll:
|
||
stub_prefix += '\n'.join(type_defs_in_ll) + '\n'
|
||
if stub_prefix:
|
||
header_end = 0
|
||
for i, line in enumerate(ll_content.splitlines()):
|
||
stripped = line.strip()
|
||
if stripped.startswith(';') or stripped.startswith('target ') or stripped.startswith('source_filename') or not stripped:
|
||
header_end = i + 1
|
||
continue
|
||
break
|
||
ll_lines = ll_content.splitlines(True)
|
||
ll_content = ''.join(ll_lines[:header_end]) + stub_prefix + ''.join(ll_lines[header_end:])
|
||
ll_content = _RE_DEFINE_DECLARE_LINKAGE.sub(r'\1 \2 dso_local ', ll_content)
|
||
ll_content = _RE_DEFINE_DECLARE_NO_LINKAGE_AT.sub(r'\1 dso_local ', ll_content)
|
||
# LLVM 22+: external 不允许在带初值的全局定义上显式写出，改为 linkonce 保留多模块合并语义
|
||
ll_content = _RE_EXTERNAL_GLOBAL_VALUE.sub(r'\1 = linkonce global \2 \3', ll_content)
|
||
ll_content = _RE_GLOBAL_LINKAGE_MULTILINE.sub(r'\1 = \2 dso_local \3', ll_content)
|
||
ll_content = _RE_GLOBAL_DSOLOCAL_MULTILINE.sub(r'\1 = dso_local \2', ll_content)
|
||
ll_content = _AddWeakComdat(ll_content)
|
||
# LLVM 22+: 移除同一模块中已有 define 的 declare 语句（LLVM 22 将 declare+define 视为重复定义）
|
||
defined_funcs: set[str] = set()
|
||
for m in _RE_DEFINE_HEAD.finditer(ll_content):
|
||
defined_funcs.add(m.group(1))
|
||
if defined_funcs:
|
||
lines: list[str] = ll_content.splitlines(True)
|
||
kept_lines: list[str] = []
|
||
for line in lines:
|
||
m = _RE_DECLARE_FULL.match(line)
|
||
if m and m.group(1) in defined_funcs:
|
||
continue
|
||
kept_lines.append(line)
|
||
ll_content = ''.join(kept_lines)
|
||
# 移除重复的 declare 语句（同一函数多次 declare 但没有 define）
|
||
seen_declares: set[str] = set()
|
||
dedup_lines: list[str] = ll_content.splitlines(True)
|
||
dedup_kept: list[str] = []
|
||
for line in dedup_lines:
|
||
dm = _RE_DECLARE_FULL.match(line)
|
||
if dm:
|
||
fname = dm.group(1)
|
||
if fname in seen_declares:
|
||
continue
|
||
seen_declares.add(fname)
|
||
dedup_kept.append(line)
|
||
ll_content = ''.join(dedup_kept)
|
||
final_type_defs = []
|
||
final_type_def_indices = set()
|
||
seen_type_names_main = set()
|
||
for i, line in enumerate(ll_content.splitlines()):
|
||
stripped = line.strip()
|
||
if stripped.startswith('%') and '= type' in stripped:
|
||
type_name_main = stripped.split('=')[0].strip()
|
||
final_type_def_indices.add(i)
|
||
if type_name_main in seen_type_names_main:
|
||
continue
|
||
seen_type_names_main.add(type_name_main)
|
||
final_type_defs.append(line)
|
||
if final_type_def_indices:
|
||
final_lines = ll_content.splitlines(True)
|
||
ll_content_no_types = ''.join(line for i, line in enumerate(final_lines) if i not in final_type_def_indices)
|
||
header_end = 0
|
||
for i, line in enumerate(ll_content_no_types.splitlines()):
|
||
stripped = line.strip()
|
||
if stripped.startswith(';') or stripped.startswith('target ') or stripped.startswith('source_filename') or not stripped:
|
||
header_end = i + 1
|
||
continue
|
||
break
|
||
final_lines2 = ll_content_no_types.splitlines(True)
|
||
ll_content = ''.join(final_lines2[:header_end]) + '\n'.join(final_type_defs) + '\n' + ''.join(final_lines2[header_end:])
|
||
# 写盘前先规整元数据：本文件的 IR 是「本模块 + 各依赖 .stub.ll/
|
||
# .text.ll 片段」拼接而成，每份片段都自带从 !0 起编号的 DI 元数据，
|
||
# 未规整直接落盘的话，任何读该文件的 opt/llc（含另一并发构建进程）
|
||
# 都会报 "Metadata id is already used"。规整后盘上产物本身即合法 IR，
|
||
# _llc_compile 内的规整随之成为幂等 no-op。
|
||
ll_content = SanitizeIRMetadata(ll_content)
|
||
# 拼接片段各自带一份同名 comdat 声明 ⇒ 落盘前统一去重：
|
||
# llc 对同一 comdat 声明出现两次即报 redefinition of comdat
|
||
ll_content = _DedupComdatDecls(ll_content)
|
||
dso_ll_path = ll_path[:-3] + '_dso.ll'
|
||
with open(dso_ll_path, 'w', encoding='utf-8') as f:
|
||
f.write(ll_content)
|
||
result = self._llc_compile(dso_ll_path, obj_path, os.path.dirname(ll_path) or '.')
|
||
if result.returncode == 0:
|
||
return (rel, 'ok', '')
|
||
else:
|
||
return (rel, 'error', result.stderr.strip())
|
||
except FileNotFoundError:
|
||
return (rel, 'error', f'找不到编译器: {self.compile_cmd}')
|
||
except Exception as e:
|
||
return (rel, 'error', str(e))
|
||
|
||
need_compile = []
|
||
for ll_path in ll_files:
|
||
obj_path = ll_path[:-3] + '.obj'
|
||
if not _CachedIsFile(obj_path):
|
||
need_compile.append(ll_path)
|
||
|
||
if need_compile:
|
||
n_workers = _PickWorkerCount(len(need_compile), 400)
|
||
_vlog().info(f" 并行编译 {len(need_compile)} 个文件 (workers={n_workers})")
|
||
errors = []
|
||
_compile_total = len(need_compile)
|
||
_compile_done = 0
|
||
with ThreadPoolExecutor(max_workers=n_workers) as executor:
|
||
futures = {}
|
||
for ll_path in need_compile:
|
||
futures[executor.submit(_compile_one, ll_path)] = (ll_path, time.time())
|
||
for future in as_completed(futures):
|
||
ll_path, _t_sub = futures[future]
|
||
rel, status, err = future.result()
|
||
_compile_done += 1
|
||
_elapsed_c = time.time() - _t_sub
|
||
_sha1_c = os.path.basename(ll_path)[:-3]
|
||
_src_c = self.sha1_map.get(_sha1_c, '?')
|
||
if status == 'ok':
|
||
_vlog().success(f" [{_compile_done}/{_compile_total}] {os.path.basename(ll_path)} {_src_c} ({_elapsed_c:.1f}s)")
|
||
elif status == 'error':
|
||
_vlog().error(f" [{_compile_done}/{_compile_total}] {rel}: {err}")
|
||
errors.append((rel, err))
|
||
elif status == 'cached':
|
||
pass
|
||
if errors:
|
||
_vlog().error(f"\n{len(errors)} 个文件编译失败")
|
||
sys.exit(1)
|
||
|
||
def _precollect_vtable_state(self) -> None:
|
||
"""预扫描 includes .py 文件，收集 @CVTable 类的虚表信息。
|
||
|
||
并行编译（ProcessPoolExecutor）创建独立内存的 worker 进程，
|
||
vtable 状态必须预收集并通过 pickle 共享。
|
||
复用 parse_python_file 的 AST 缓存。
|
||
"""
|
||
for includes_dir in self.include_dirs:
|
||
if not os.path.isdir(includes_dir):
|
||
continue
|
||
for root, dirs, files in os.walk(includes_dir):
|
||
dirs[:] = [d for d in dirs if not d.startswith('.') and d != '__pycache__']
|
||
for fname in files:
|
||
if not (fname.endswith('.vp') or fname.endswith('.py')):
|
||
continue
|
||
py_path: str = os.path.join(root, fname)
|
||
code, tree = parse_python_file(py_path)
|
||
# 字符串预过滤：不含 CVTable 的文件跳过 AST 遍历。
|
||
# `virtual class` / `virtual def` 修饰符在源码里写的是 `virtual`
|
||
# （AST 上才变成 @CVTable 装饰器），故预过滤必须同时放行
|
||
# `virtual`/`novirtual`，否则含修饰符的 includes 类会被整段跳过——
|
||
# 静默不预收集虚表状态（与本题"不得静默丢弃修饰符"同族问题）。
|
||
if tree is None or ('CVTable' not in code and 'virtual' not in code):
|
||
continue
|
||
try:
|
||
for node in ast.iter_child_nodes(tree):
|
||
if not isinstance(node, ast.ClassDef):
|
||
continue
|
||
has_cvtable: bool = False
|
||
for dec in node.decorator_list:
|
||
if isinstance(dec, ast.Name):
|
||
if dec.id == 'CVTable':
|
||
has_cvtable = True
|
||
# 方法级 virtual 触发类级 vtable（与
|
||
# HandlesClassDef._resolve_class_flags 同口径：类级
|
||
# NoVTable / 无真实父类两种可判定形态）。
|
||
# 记录"类级 CVTable"单独一份：它是 vtable 槽筛选的
|
||
# VtableByClassLevel（忽略方法级标记），两者不可混用。
|
||
class_level_cvtable: bool = has_cvtable
|
||
if not has_cvtable:
|
||
_has_real_parent: bool = False
|
||
for _b in node.bases:
|
||
_bn: str = None
|
||
if hasattr(_b, 'id'):
|
||
_bn = _b.id
|
||
elif hasattr(_b, 'attr'):
|
||
_bn = _b.attr
|
||
if _bn and _bn not in ('Object', 'CVTable', 'NoVTable', 'CStruct', 'CUnion', 'CEnum', 'REnum', 'Exception', 'Enum'):
|
||
_has_real_parent = True
|
||
break
|
||
if ClassHasFuncLevelCVTable(node) and not _has_real_parent:
|
||
has_cvtable = True
|
||
if not has_cvtable:
|
||
continue
|
||
class_name: str = node.name
|
||
has_methods: bool = False
|
||
if class_name not in self._shared_class_methods:
|
||
self._shared_class_methods[class_name] = []
|
||
for item in node.body:
|
||
if isinstance(item, ast.FunctionDef):
|
||
has_methods = True
|
||
method_name: str = item.name
|
||
# 构造函数不放入 vtable（避免子类与基类 vtable 大小不一致破坏多态分派）
|
||
if method_name in ('__new__', '__init__', '__before_init__'):
|
||
continue
|
||
# 方法级修饰符：`novirtual def`（⇒ @NoVTable）不进槽；
|
||
# 判据唯一源 = SymbolUtils.IsVtableSlotMethod，与
|
||
# HandlesClassDef / VPC _should_method_be_virtual 同口径。
|
||
if not IsVtableSlotMethod(item, class_level_cvtable):
|
||
continue
|
||
if method_name == '__init__':
|
||
func_full_name: str = f"{class_name}.__init__"
|
||
elif method_name == '__call__':
|
||
func_full_name = f"{class_name}.__call__"
|
||
else:
|
||
func_full_name = f"{class_name}.{method_name}"
|
||
if func_full_name not in self._shared_class_methods[class_name]:
|
||
self._shared_class_methods[class_name].append(func_full_name)
|
||
if has_methods:
|
||
self._shared_class_vtable.add(class_name)
|
||
self._shared_cross_module_vtable.add(class_name)
|
||
except Exception:
|
||
pass
|
||
|
||
def _build_include_py_index(self, includes_dir: str) -> set[str]:
|
||
"""预扫描 includes 目录，构建所有 .py 文件路径集合
|
||
|
||
用 set 查询替代 os.path.isfile，避免 3490 次磁盘 stat（83s→<0.1s）。
|
||
同时规范化路径为小写（Windows 不区分大小写）以加速查找。
|
||
"""
|
||
py_index: set[str] = set()
|
||
for root, dirs, files in os.walk(includes_dir):
|
||
dirs[:] = [d for d in dirs if not d.startswith('.') and d != '__pycache__']
|
||
for fname in files:
|
||
if fname.endswith('.vp') or fname.endswith('.py'):
|
||
py_index.add(os.path.normcase(os.path.join(root, fname)))
|
||
return py_index
|
||
|
||
def _scan_include_libraries(self) -> None:
|
||
"""扫描 includes 目录，检测被使用的模块对应的库文件和源文件"""
|
||
# --traceback：强制纳入 traceback 模块 —— 无任何用户代码 import 它，
|
||
# 不纳入就不会被翻译 ⇒ 产物没有 Install 符号可注入启动期构造器。
|
||
# 直接借用既有的 used_includes 收集路径（下面按同一套 py_index /
|
||
# 传递依赖逻辑把它加入 extra_py_files，再交给 _compile_include_py_files
|
||
# 编译并把 .obj 追加进 extra_link_files）。开关关闭时不触碰 used_includes
|
||
# → 本函数行为与改动前逐字一致（零影响）。
|
||
if self.traceback_enabled:
|
||
self.used_includes.add('traceback')
|
||
if not self.used_includes:
|
||
return
|
||
|
||
for includes_dir in self.include_dirs:
|
||
if not os.path.isdir(includes_dir):
|
||
continue
|
||
|
||
# 预构建 .py 文件索引，避免递归发现时反复 os.path.isfile
|
||
py_index = self._build_include_py_index(includes_dir)
|
||
|
||
for module_name in self.used_includes:
|
||
module_dir = os.path.join(includes_dir, module_name)
|
||
if os.path.isdir(module_dir):
|
||
self._scan_dir_for_libs(module_dir, module_name)
|
||
for root, dirs, files in os.walk(module_dir):
|
||
dirs[:] = [d for d in dirs if not d.startswith('.') and d != '__pycache__']
|
||
for fname in files:
|
||
if fname.endswith('.vp') or fname.endswith('.py'):
|
||
py_path = os.path.join(root, fname)
|
||
if py_path not in self.extra_py_files:
|
||
self.extra_py_files.append(py_path)
|
||
_vlog().info(f" [includes] 发现 Python 包文件: {os.path.relpath(py_path, includes_dir)}")
|
||
|
||
for ext in self.INCLUDE_LIB_EXTENSIONS:
|
||
lib_path = os.path.join(includes_dir, module_name + ext)
|
||
if _CachedIsFile(lib_path):
|
||
self.extra_link_files.append(lib_path)
|
||
_vlog().info(f" [includes] 发现库文件: {os.path.relpath(lib_path, includes_dir)}")
|
||
|
||
for ext in self.INCLUDE_SRC_EXTENSIONS:
|
||
src_path = os.path.join(includes_dir, module_name + ext)
|
||
if _CachedIsFile(src_path):
|
||
self.extra_compile_files.append(src_path)
|
||
_vlog().info(f" [includes] 发现源文件: {os.path.relpath(src_path, includes_dir)}")
|
||
|
||
py_path = os.path.join(includes_dir, module_name + '.vp')
|
||
if os.path.normcase(py_path) not in py_index:
|
||
# 回退到 .py（向后兼容）
|
||
py_path = os.path.join(includes_dir, module_name + '.py')
|
||
if os.path.normcase(py_path) in py_index:
|
||
self.extra_py_files.append(py_path)
|
||
_vlog().info(f" [includes] 发现 Python 源文件: {os.path.relpath(py_path, includes_dir)}")
|
||
|
||
discovered = set()
|
||
for ef in list(self.extra_py_files):
|
||
self._discover_transitive_deps(ef, includes_dir, discovered, py_index)
|
||
|
||
if self.extra_link_files:
|
||
_vlog().info(f"\n[includes] 需要链接的库文件: {len(self.extra_link_files)} 个")
|
||
if self.extra_compile_files:
|
||
_vlog().info(f"[includes] 需要编译的源文件: {len(self.extra_compile_files)} 个")
|
||
# --traceback：断言 traceback 源文件确实被纳入待编译集合。找不到就必然
|
||
# 得不到 Install 符号、构造器无处注入（功能静默失效）——显式报错终止，
|
||
# 不静默跳过（多半是 project.json 的 includes 未包含仓库的 includes 目录）。
|
||
if self.traceback_enabled:
|
||
_traceback_found: bool = any(
|
||
os.path.basename(_p) in _TRACEBACK_MODULE_FILES
|
||
for _p in self.extra_py_files)
|
||
if not _traceback_found:
|
||
raise RuntimeError(
|
||
'--traceback: 在 includes 目录中找不到 traceback.vp/.py，'
|
||
'无法编译并链接 Install()（请确认 project.json 的 includes '
|
||
f'包含仓库 includes 目录；已扫描: {self.include_dirs}）')
|
||
|
||
def _scan_dir_for_libs(self, directory: str, module_name: str) -> None:
|
||
"""递归扫描目录中的库文件和源文件
|
||
|
||
模块目录含 `.no_autobuild` 标记时直接跳过：该模块的 C/C++ 源码
|
||
与库文件由 project.vpj linker.static_libs 显式链接（如 fltk 内嵌
|
||
完整 FLTK/C 源码树，避免误编译整个第三方库）。递归时跳过常见
|
||
构建/缓存目录（build、cmake-build*、.git、__pycache__、. 开头）。
|
||
"""
|
||
if _CachedIsFile(os.path.join(directory, '.no_autobuild')):
|
||
return
|
||
for root, dirs, files in os.walk(directory):
|
||
dirs[:] = [
|
||
d for d in dirs
|
||
if d not in ('.git', '__pycache__', 'build', 'cmake-build')
|
||
and not d.startswith('cmake-build-')
|
||
and not d.startswith('.')
|
||
]
|
||
for fname in files:
|
||
fpath = os.path.join(root, fname)
|
||
_, ext = os.path.splitext(fname)
|
||
ext = ext.lower()
|
||
if ext in self.INCLUDE_LIB_EXTENSIONS:
|
||
self.extra_link_files.append(fpath)
|
||
_vlog().info(f" [includes] 发现库文件: {os.path.relpath(fpath, directory)}")
|
||
elif ext in self.INCLUDE_SRC_EXTENSIONS:
|
||
self.extra_compile_files.append(fpath)
|
||
_vlog().info(f" [includes] 发现源文件: {os.path.relpath(fpath, directory)}")
|
||
|
||
def _discover_transitive_deps(self, py_path: str, includes_dir: str, discovered: set[str], py_index: set[str] | None = None) -> None:
|
||
"""递归发现 Python 文件的间接依赖
|
||
|
||
复用 parse_python_file 的 AST 缓存，避免重复 open+read+ast.parse。
|
||
py_index 为预构建的 .py 文件路径集合，避免 os.path.isfile 磁盘 stat。
|
||
"""
|
||
if py_path in discovered:
|
||
return
|
||
discovered.add(py_path)
|
||
_, tree = parse_python_file(py_path)
|
||
if tree is None:
|
||
return
|
||
try:
|
||
for node in ast.walk(tree):
|
||
if isinstance(node, ast.Import):
|
||
for alias in node.names:
|
||
self._try_add_include_dep(alias.name, includes_dir, discovered, py_index)
|
||
elif isinstance(node, ast.ImportFrom):
|
||
if node.module and node.level == 0:
|
||
self._try_add_include_dep(node.module, includes_dir, discovered, py_index)
|
||
except Exception as _e:
|
||
if _config_mode == "strict":
|
||
raise
|
||
_vlog().warning(f"发现 include 传递依赖失败: {_e}", "Exception")
|
||
|
||
def _try_add_include_dep(self, module_name: str, includes_dir: str, discovered: set, py_index: set[str] | None = None):
|
||
"""尝试将模块添加为 include 依赖（支持子包，如 win32.win32base）
|
||
|
||
py_index 非空时用 set 查询替代 os.path.isfile（O(1) vs 24ms/次）。
|
||
"""
|
||
# 候选路径：子包路径（win32/win32base.vp/.py）、顶层模块（win32.vp/.py）、子包 __init__.vp/__init__.py
|
||
candidates: list[str] = []
|
||
# 子包路径：win32.win32base → win32/win32base.vp / .py
|
||
candidates.append(os.path.join(includes_dir, module_name.replace('.', os.sep) + '.vp'))
|
||
candidates.append(os.path.join(includes_dir, module_name.replace('.', os.sep) + '.py'))
|
||
# 子包 __init__.vp/__init__.py：win32.win32base → win32/win32base/__init__.vp / .py
|
||
candidates.append(os.path.join(includes_dir, module_name.replace('.', os.sep), '__init__.vp'))
|
||
candidates.append(os.path.join(includes_dir, module_name.replace('.', os.sep), '__init__.py'))
|
||
# 顶层模块：取第一个组件 → win32.vp / .py
|
||
top_module = module_name.split('.')[0]
|
||
if top_module != module_name:
|
||
candidates.append(os.path.join(includes_dir, top_module + '.vp'))
|
||
candidates.append(os.path.join(includes_dir, top_module + '.py'))
|
||
|
||
for py_path in candidates:
|
||
exists: bool
|
||
if py_index is not None:
|
||
exists = os.path.normcase(py_path) in py_index
|
||
else:
|
||
exists = _CachedIsFile(py_path)
|
||
if exists and py_path not in self.extra_py_files:
|
||
self.extra_py_files.append(py_path)
|
||
_vlog().info(f" [includes] 发现间接依赖: {os.path.relpath(py_path, includes_dir)}")
|
||
self._discover_transitive_deps(py_path, includes_dir, discovered, py_index)
|
||
return # 找到第一个匹配即可
|
||
|
||
def _is_decl_only_file(self, src_path: str) -> bool:
|
||
"""复用 parse_python_file 的 AST 缓存"""
|
||
_, tree = parse_python_file(src_path)
|
||
if tree is None:
|
||
return False
|
||
try:
|
||
for node in ast.walk(tree):
|
||
if isinstance(node, ast.ClassDef):
|
||
return False
|
||
if isinstance(node, ast.FunctionDef):
|
||
body = node.body
|
||
if len(body) == 1:
|
||
stmt = body[0]
|
||
if isinstance(stmt, ast.Expr) and isinstance(stmt.value, ast.Constant) and stmt.value.value is ...:
|
||
continue
|
||
if isinstance(stmt, ast.Pass):
|
||
continue
|
||
return False
|
||
return True
|
||
except Exception as _e:
|
||
if _config_mode == "strict":
|
||
raise
|
||
_vlog().warning(f"判断是否为声明文件失败: {_e}", "Exception")
|
||
return False
|
||
|
||
def _sort_include_files_by_deps(self, file_list: list[str]) -> tuple[list[str], dict[str, set[str]]]:
|
||
"""按依赖关系对 include 文件进行拓扑排序，确保被依赖的文件先编译。
|
||
|
||
返回 (排序后的文件列表, 依赖字典)。
|
||
依赖字典: {fpath: set(依赖的fpath)}，仅包含 file_list 中的依赖。
|
||
"""
|
||
# 构建文件名到路径的映射
|
||
name_to_path = {}
|
||
for fpath in file_list:
|
||
basename = os.path.splitext(os.path.basename(fpath))[0]
|
||
name_to_path[basename] = fpath
|
||
|
||
# 也支持包内模块: vpsdk/window -> path
|
||
for fpath in file_list:
|
||
# 尝试从 includes 目录推断模块全名
|
||
for includes_dir in self.include_dirs:
|
||
try:
|
||
rel = os.path.relpath(fpath, includes_dir)
|
||
mod_name = os.path.splitext(rel)[0].replace(os.sep, '.').replace('/', '.')
|
||
name_to_path[mod_name] = fpath
|
||
except ValueError:
|
||
pass
|
||
|
||
# 分析每个文件的 import 依赖（复用 parse_python_file 的 AST 缓存）
|
||
deps = {} # path -> set of paths it depends on
|
||
for fpath in file_list:
|
||
deps[fpath] = set()
|
||
try:
|
||
_, tree = parse_python_file(fpath)
|
||
if tree is None:
|
||
continue
|
||
for node in ast.iter_child_nodes(tree):
|
||
if isinstance(node, ast.Import):
|
||
for alias in node.names:
|
||
# 尝试完整路径和各级前缀，确保同包内子模块依赖能被正确解析
|
||
# 例如 import zlib.zhuff -> 应先匹配 zlib.zhuff，再回退到 zlib
|
||
parts = alias.name.split('.')
|
||
for i in range(len(parts), 0, -1):
|
||
mod = '.'.join(parts[:i])
|
||
if mod in name_to_path and name_to_path[mod] != fpath:
|
||
deps[fpath].add(name_to_path[mod])
|
||
break
|
||
elif isinstance(node, ast.ImportFrom):
|
||
if node.module and node.level == 0:
|
||
# 绝对导入：同样尝试完整路径和各级前缀
|
||
parts = node.module.split('.')
|
||
for i in range(len(parts), 0, -1):
|
||
mod = '.'.join(parts[:i])
|
||
if mod in name_to_path and name_to_path[mod] != fpath:
|
||
deps[fpath].add(name_to_path[mod])
|
||
break
|
||
elif node.level > 0:
|
||
# 相对导入：from .xxx import yyy 或 from ..xxx import yyy
|
||
# 根据当前文件位置推断目标模块的完整路径
|
||
current_dir = os.path.dirname(fpath)
|
||
# node.level == 1 表示当前包，node.level == 2 表示上一级包
|
||
package_dir = current_dir
|
||
for _ in range(node.level - 1):
|
||
package_dir = os.path.dirname(package_dir)
|
||
# 推断当前包名（从 includes 目录的相对路径）
|
||
package_name: str | None = None
|
||
for includes_dir in self.include_dirs:
|
||
try:
|
||
rel = os.path.relpath(package_dir, includes_dir)
|
||
if rel and rel != '.':
|
||
package_name = rel.replace(os.sep, '.').replace('/', '.')
|
||
break
|
||
except ValueError:
|
||
pass
|
||
# 构建目标模块名并查找
|
||
target_mod: str | None = None
|
||
if node.module:
|
||
target_mod = f"{package_name}.{node.module}" if package_name else node.module
|
||
else:
|
||
# from . import yyy：yyy 是子模块，在 names 中
|
||
target_mod = package_name
|
||
if target_mod:
|
||
parts = target_mod.split('.')
|
||
for i in range(len(parts), 0, -1):
|
||
mod = '.'.join(parts[:i])
|
||
if mod in name_to_path and name_to_path[mod] != fpath:
|
||
deps[fpath].add(name_to_path[mod])
|
||
break
|
||
# from . import yyy：yyy 可能是子模块
|
||
if node.level > 0 and not node.module:
|
||
for alias in node.names:
|
||
sub_mod = f"{package_name}.{alias.name}" if package_name else alias.name
|
||
if sub_mod in name_to_path and name_to_path[sub_mod] != fpath:
|
||
deps[fpath].add(name_to_path[sub_mod])
|
||
except Exception as _e:
|
||
if _config_mode == "strict":
|
||
raise
|
||
_vlog().warning(f"分析 include 文件导入依赖失败: {_e}", "Exception")
|
||
|
||
# 拓扑排序 (Kahn's algorithm)
|
||
# in_degree 只计入 file_list 中的依赖——decl-only 文件（如 stdint.py）
|
||
# 不在 file_list 中，不应计入入度，否则依赖它们的文件入度永远不为 0，
|
||
# 被误判为循环依赖通过 fallback 添加，导致编译顺序错误。
|
||
file_set = set(file_list)
|
||
in_degree = {fpath: sum(1 for dep in deps[fpath] if dep in file_set) for fpath in file_list}
|
||
# 反向邻接表：如果 A 依赖 B，则 B -> A
|
||
reverse_adj = {fpath: [] for fpath in file_list}
|
||
for fpath in file_list:
|
||
for dep in deps[fpath]:
|
||
if dep in reverse_adj:
|
||
reverse_adj[dep].append(fpath)
|
||
|
||
result = []
|
||
queue = [fpath for fpath in file_list if in_degree[fpath] == 0]
|
||
# 对队列排序以保持确定性
|
||
queue.sort(key=lambda x: x)
|
||
|
||
while queue:
|
||
current = queue.pop(0)
|
||
result.append(current)
|
||
for neighbor in reverse_adj[current]:
|
||
in_degree[neighbor] -= 1
|
||
if in_degree[neighbor] == 0:
|
||
queue.append(neighbor)
|
||
queue.sort(key=lambda x: x)
|
||
|
||
# 如果有循环依赖，把剩余文件也加入
|
||
for fpath in file_list:
|
||
if fpath not in result:
|
||
result.append(fpath)
|
||
|
||
# 仅保留 file_list 内的依赖（用于波次并行翻译）
|
||
file_set_result = set(file_list)
|
||
deps_in_set = {fpath: (deps[fpath] & file_set_result) for fpath in file_list}
|
||
return result, deps_in_set
|
||
|
||
def _get_precompiled_obj_path(self, src_path: str, sha1: str) -> str | None:
|
||
"""计算 include 文件在 includes.binary/ 中的预编译 .obj 路径。
|
||
|
||
预编译目录位于 include 目录同级，命名为 {basename}.binary/。
|
||
文件命名为 {sha1}.{triple_safe}.{rel_model}.obj，区分目标平台和重定位模型。
|
||
|
||
--traceback：include 模块也带 DWARF 行表（栈帧要能落到 includes/* 的源行），
|
||
此类 .obj 不得回写进跨项目共享的 includes.binary（会污染其它项目的预编译
|
||
缓存），改落**兄弟目录** {basename}.binary + 后缀（不新增层级，形态与
|
||
ASan 变体目录 includes.binary_asan 一致）。
|
||
关闭时后缀为空 → 路径与改动前逐字一致（含 --dwarf/--asan 单独开启的情形）。
|
||
--asan 与 --traceback 同开：ASan 插桩也会改变 include obj 内容（见
|
||
_llc_compile），两者产物不可互相复用，故再叠加 _asan 后缀区分
|
||
（优先级：是否隔离由 --traceback 决定，--asan 只决定后缀再叠加 _asan）。
|
||
"""
|
||
AbsSrcPath: str = os.path.abspath(src_path)
|
||
RelModel: str = 'static'
|
||
for _flag in self.compile_flags:
|
||
if _flag.startswith('-relocation-model='):
|
||
RelModel = _flag[len('-relocation-model='):]
|
||
break
|
||
for inc_dir in self.include_dirs:
|
||
abs_inc: str = os.path.abspath(inc_dir)
|
||
try:
|
||
rel: str = os.path.relpath(AbsSrcPath, abs_inc)
|
||
if rel.startswith('..'):
|
||
continue
|
||
inc_parent: str = os.path.dirname(abs_inc)
|
||
inc_base: str = os.path.basename(abs_inc)
|
||
_iso_suffix: str = ''
|
||
if self.traceback_enabled:
|
||
_iso_suffix = '_traceback_asan' if self.asan_mode else '_traceback'
|
||
precompiled_dir: str = os.path.join(inc_parent, inc_base + '.binary' + _iso_suffix)
|
||
triple_safe: str = (self.triple or 'default').replace(os.sep, '_').replace('/', '_').replace('\\', '_').replace(':', '_')
|
||
return os.path.join(precompiled_dir, f"{sha1}.{triple_safe}.{RelModel}.obj")
|
||
except ValueError:
|
||
continue
|
||
return None
|
||
|
||
def _compile_include_py_files(self, active_sha1s: set[str]) -> None:
|
||
"""通过 TransPyC 翻译器编译 includes 目录中发现的 .py 文件
|
||
|
||
优化流程：预注册符号 → 并行翻译 → 串行后处理 → 并行编译 → 注册+缓存
|
||
"""
|
||
if not self.extra_py_files:
|
||
return
|
||
|
||
# 按依赖关系排序 include 文件，确保被依赖的文件先编译
|
||
sorted_files, file_deps = self._sort_include_files_by_deps(self.extra_py_files)
|
||
_vlog().info(f"\n[includes 编译] 找到 {len(sorted_files)} 个 Python 源文件")
|
||
|
||
# --hotspot-sites：include 模块同样被插桩（hotspot_sites_file 经 worker 透传），
|
||
# 故启用时必须让旧 .obj 失效并强制重译，否则会复用上一次未插桩的目标文件，
|
||
# 使 include 模块的热点计数/计时 IR 静默缺失。未启用时 hotspot_on=False，
|
||
# 本函数行为与改动前逐字节一致（零影响）。
|
||
hotspot_on: bool = bool(self.hotspot_sites_file)
|
||
|
||
# --traceback：include 模块也必须带 DWARF 行表（否则运行时 Ctrl+C 的栈里
|
||
# 落在 includes/*（memhub/string/viperlib/win32…）的帧取不到源行，只能落到
|
||
# 邻行或 "+0x…(unknown)"）。为此翻译时给 include 透传 dwarf_enabled=True
|
||
# （见下方 worker/串行四处的 dwarf_enabled=self.traceback_enabled），并把
|
||
# 旧目标文件（output 目录 {sha1}.obj 及隔离缓存 includes.binary_traceback
|
||
# 内的旧 .obj）一并失效重译——复用上一次无 DI 的产物会让行表静默缺失。
|
||
# 与 hotspot_on 处理方式一致；同时覆盖 traceback.vp（其 .obj 还须含
|
||
# @llvm.global_ctors 启动期构造器）。取舍：开启本开关时 include 目标文件每
|
||
# 次构建都重译（放弃隔离缓存的复用，换取"必然带 DWARF"的正确性保证），
|
||
# 仅调试模式可接受。开关关闭时 include_dwarf=False，本函数行为与改动前
|
||
# 逐字节一致（零影响，含 --dwarf/--asan 单独开启的情形）。
|
||
include_dwarf: bool = bool(self.traceback_enabled)
|
||
|
||
# 构建 include 文件的模块 SHA1 映射（预计算所有文件，构建完整映射）
|
||
include_ModuleSha1Map = self._build_current_ModuleSha1Map(None)
|
||
need_translate = [] # [(src_path, sha1, module_name, ll_path, obj_path, precompiled_obj)]
|
||
for src_path in sorted_files:
|
||
module_name = os.path.splitext(os.path.basename(src_path))[0]
|
||
# 复用 parse_python_file 缓存的 content，避免重复 open+read
|
||
py_content, _ = parse_python_file(src_path)
|
||
sha1 = compute_sha1(py_content)
|
||
self.include_py_map[module_name] = sha1
|
||
active_sha1s.add(sha1)
|
||
self._include_sha1s.add(sha1)
|
||
|
||
for includes_dir in self.include_dirs:
|
||
try:
|
||
rel = os.path.relpath(src_path, includes_dir)
|
||
mod_full = os.path.splitext(rel)[0].replace(os.sep, '.').replace('/', '.')
|
||
include_ModuleSha1Map[mod_full] = sha1
|
||
include_ModuleSha1Map[module_name] = sha1
|
||
# 包入口文件：注册父包名（如 'os.__init__' -> 'os'）
|
||
if module_name == '__init__' and '.' in mod_full:
|
||
parent_pkg = mod_full.rsplit('.', 1)[0]
|
||
include_ModuleSha1Map[parent_pkg] = sha1
|
||
except ValueError:
|
||
pass
|
||
include_ModuleSha1Map[module_name] = sha1
|
||
ll_path = os.path.join(self.output_dir, f"{sha1}.ll")
|
||
obj_path = os.path.join(self.output_dir, f"{sha1}.obj")
|
||
precompiled_obj = self._get_precompiled_obj_path(src_path, sha1)
|
||
|
||
if self._is_decl_only_file(src_path):
|
||
_vlog().info(f" 跳过(声明文件): {module_name}.py")
|
||
# 声明文件（如 stdint.py）虽不编译为 .obj，但其 typedef/类型别名
|
||
# 必须注册到共享符号表，否则依赖它的 include 文件编译时
|
||
# 类型解析失败（如 UINT32PTR 被当作命名结构体而非 i32*）
|
||
self._register_compiled_include(src_path, sha1, None, include_ModuleSha1Map)
|
||
self._collect_inline_symbols(src_path)
|
||
continue
|
||
|
||
# --hotspot-sites / --traceback：include 的 .py 未变时下面两处会因旧
|
||
# .obj 存在而直接跳过，复用上一次（未插桩 / 不含 DI）的目标文件。
|
||
# 启用时先让这两类旧 .obj 一并失效，使其必然走入 need_translate 重新
|
||
# 翻译并嵌入热点计数/计时 IR 或 DWARF 行表。
|
||
# 用 os.path.isfile 而非 _CachedIsFile：避免把"已存在"写入存在性缓存后又被
|
||
# 本处删除，使下面两处判定读到过期缓存而漏编（与 App 主循环同一写法）。
|
||
if hotspot_on or include_dwarf:
|
||
for _stale_obj in (precompiled_obj, obj_path):
|
||
if _stale_obj and os.path.isfile(_stale_obj):
|
||
try:
|
||
os.remove(_stale_obj)
|
||
except OSError as e:
|
||
# 不中断构建，但真实错误（如权限不足）必须打印告警，不静默吞掉
|
||
# （报文保持原样：关闭开关时与改动前逐字一致）
|
||
_vlog().warning(
|
||
f"Phase2Translator: 删除热点插桩 include 模块的旧目标文件失败: {_stale_obj}: {e}")
|
||
|
||
# 优先检查 includes.binary 预编译缓存（跨项目共享）
|
||
if not self.force_recompile_includes and precompiled_obj and _CachedIsFile(precompiled_obj):
|
||
_vlog().info(f" 跳过(预编译): {module_name}.py -> {os.path.basename(precompiled_obj)}")
|
||
# 预编译跳过的文件仍需注册 .pyi/.stub.ll 声明和符号，
|
||
# 否则依赖它的 include 文件（如 vpui.py import vpsdk.window）
|
||
# 在 _EmitModuleDeclarationsLlvm 中找不到声明，导致 Undefined symbol
|
||
self._register_compiled_include(src_path, sha1, None, include_ModuleSha1Map)
|
||
self._collect_inline_symbols(src_path)
|
||
self.extra_link_files.append(precompiled_obj)
|
||
continue
|
||
|
||
if not self.force_recompile_includes and _CachedIsFile(obj_path):
|
||
_vlog().info(f" 跳过(缓存): {module_name}.py -> {sha1}.obj")
|
||
self._register_compiled_include(src_path, sha1, None, include_ModuleSha1Map)
|
||
self._collect_inline_symbols(src_path)
|
||
self.extra_link_files.append(obj_path)
|
||
continue
|
||
|
||
need_translate.append((src_path, sha1, module_name, ll_path, obj_path, precompiled_obj))
|
||
|
||
if not need_translate:
|
||
return
|
||
|
||
# 预注册所有待翻译文件的符号（从 Phase1 缓存加载 .pyi / .stub.ll）
|
||
# 这样并行翻译时所有跨模块符号都已可用
|
||
for src_path, sha1, module_name, ll_path, obj_path, precompiled_obj in need_translate:
|
||
self._register_compiled_include(src_path, sha1, None, include_ModuleSha1Map)
|
||
|
||
# 预收集 stub_decls（所有文件共用，避免重复扫描 .stub.ll）
|
||
stub_decls = []
|
||
for other_sha1, other_stub in self.stub_files.items():
|
||
if os.path.exists(other_stub):
|
||
try:
|
||
for sline in _ReadTextCached(other_stub).splitlines(True):
|
||
sl = sline.strip()
|
||
if sl.startswith('%') and '= type' in sl and 'opaque' not in sl:
|
||
struct_name = Phase2Translator._extract_llvm_type_name(sl)
|
||
if struct_name:
|
||
stub_decls.append(sl)
|
||
except Exception as e:
|
||
_vlog().warning(f"Phase2Translator: 忽略异常 {e}", exc_info=e)
|
||
all_type_defs = {}
|
||
for sdecl in stub_decls:
|
||
tname = Phase2Translator._extract_llvm_type_name(sdecl)
|
||
if tname:
|
||
all_type_defs[tname] = sdecl
|
||
|
||
# ========== 阶段1：并行翻译 ==========
|
||
_vlog().info(f" 翻译 {len(need_translate)} 个文件")
|
||
if len(need_translate) > 1:
|
||
# 序列化共享符号表供 worker 使用
|
||
shared_pickle_path = os.path.join(self.output_dir, '_shared_sym_includes.pickle')
|
||
try:
|
||
# symbol_table 只以 pickle 字节传递（symbol_table_bytes），不重复序列化
|
||
# 对象本身（原双重序列化使文件体积/时间翻倍，4.65MB 中近半冗余）。
|
||
shared_data = {
|
||
'symbol_table_bytes': pickle.dumps(self._shared_symbol_table),
|
||
'source_module_sig_files': self._shared_source_module_sig_files,
|
||
'all_dc': self._shared_all_dc,
|
||
'all_decl_types': self._shared_all_decl_types,
|
||
'export_extern_funcs': self._shared_export_extern_funcs,
|
||
'inline_func_symbols': self.inline_func_symbols,
|
||
'function_default_args': self.function_default_args,
|
||
'function_param_names': self.function_param_names,
|
||
'generic_class_templates': self._shared_generic_class_templates,
|
||
'shared_doc_meta': self._shared_doc_meta,
|
||
'class_vtable': self._shared_class_vtable,
|
||
'cross_module_vtable': self._shared_cross_module_vtable,
|
||
'class_methods': self._shared_class_methods,
|
||
'cross_module_novtable': self._shared_cross_module_novtable,
|
||
}
|
||
with open(shared_pickle_path, 'wb') as f:
|
||
f.write(pickle.dumps(shared_data))
|
||
except Exception as e:
|
||
_vlog().warning(f"符号表序列化失败({e})，include将串行翻译")
|
||
shared_pickle_path = ''
|
||
|
||
if shared_pickle_path and os.path.exists(shared_pickle_path):
|
||
n_workers = _PickWorkerCount(len(need_translate))
|
||
errors = []
|
||
# 波次并行翻译：按依赖关系分批提交，确保被依赖的文件先翻译完成。
|
||
# 根因：_TryLoadStructFromStub 扫描 output 目录加载跨模块 struct 定义，
|
||
# 若依赖文件尚未完成翻译（output stub 未写入），只能取到 Phase1 temp
|
||
# 中的不完整 stub（缺少跨模块继承字段），导致运行时崩溃。
|
||
item_by_path = {item[0]: item for item in need_translate}
|
||
translate_paths = set(item_by_path.keys())
|
||
# 每个文件的依赖（仅限本轮待翻译集合内）
|
||
translate_deps = {
|
||
p: (file_deps.get(p, set()) & translate_paths)
|
||
for p in translate_paths
|
||
}
|
||
# 复用 self._executor_pool（与 App 源文件阶段共享），避免 GC 阻塞
|
||
if self._executor_pool is None:
|
||
# 新池：刷新 pyi pickle 缓存，把 _register_compiled_include 新增的
|
||
# include .vpi 一并落盘，使 worker 预热包含它们（签名未变时零开销）
|
||
if os.environ.get('TPC_WORKER_WARMUP', '1') == '1':
|
||
_BuildPyiCacheFile(self.temp_dir)
|
||
self._executor_pool = ProcessPoolExecutor(max_workers=n_workers,
|
||
initializer=_PyiCacheWorkerInitializer,
|
||
initargs=(self.temp_dir,))
|
||
executor = self._executor_pool
|
||
try:
|
||
completed_paths: set[str] = set()
|
||
remaining = list(need_translate) # 保持拓扑序
|
||
in_progress: dict = {} # future -> item
|
||
done_count = 0
|
||
|
||
while remaining or in_progress:
|
||
# 提交所有依赖已完成的文件
|
||
ready = [
|
||
item for item in remaining
|
||
if translate_deps[item[0]] <= completed_paths
|
||
]
|
||
for item in ready:
|
||
remaining.remove(item)
|
||
src_path, sha1, module_name, ll_path, obj_path, precompiled_obj = item
|
||
future = executor.submit(
|
||
_parallel_translate_worker,
|
||
src_path, ll_path,
|
||
self.src_root, self.temp_dir, self.output_dir,
|
||
self.include_dirs, self.triple, self.datalayout,
|
||
self.sha1_map, self.sig_files, self.stub_files,
|
||
self.include_py_map, self.slice_level,
|
||
shared_pickle_path, self.struct_sha1_map,
|
||
include_ModuleSha1Map,
|
||
hotspot_sites_file=self.hotspot_sites_file,
|
||
hotspot_sites_all=self.hotspot_sites_all,
|
||
hotspot_callees=self.hotspot_callees,
|
||
traceback_enabled=self.traceback_enabled,
|
||
dwarf_enabled=self.traceback_enabled
|
||
)
|
||
in_progress[future] = item
|
||
|
||
if not in_progress:
|
||
# 无可提交且无进行中：循环依赖 fallback
|
||
for item in remaining:
|
||
src_path, sha1, module_name, ll_path, obj_path, precompiled_obj = item
|
||
future = executor.submit(
|
||
_parallel_translate_worker,
|
||
src_path, ll_path,
|
||
self.src_root, self.temp_dir, self.output_dir,
|
||
self.include_dirs, self.triple, self.datalayout,
|
||
self.sha1_map, self.sig_files, self.stub_files,
|
||
self.include_py_map, self.slice_level,
|
||
shared_pickle_path, self.struct_sha1_map,
|
||
include_ModuleSha1Map,
|
||
hotspot_sites_file=self.hotspot_sites_file,
|
||
hotspot_sites_all=self.hotspot_sites_all,
|
||
hotspot_callees=self.hotspot_callees,
|
||
traceback_enabled=self.traceback_enabled,
|
||
dwarf_enabled=self.traceback_enabled
|
||
)
|
||
in_progress[future] = item
|
||
remaining.clear()
|
||
|
||
# 等待至少一个完成
|
||
done_set, _ = wait(in_progress, return_when=FIRST_COMPLETED)
|
||
for future in done_set:
|
||
item = in_progress.pop(future)
|
||
src_path, sha1, module_name, ll_path, obj_path, precompiled_obj = item
|
||
completed_paths.add(src_path)
|
||
done_count += 1
|
||
try:
|
||
status, _, err = future.result()
|
||
except ReportedCompileError:
|
||
raise
|
||
except Exception as e:
|
||
_vlog().error(f" {module_name}.py: {e}")
|
||
for f2 in in_progress:
|
||
f2.cancel()
|
||
if self._executor_pool is not None:
|
||
self._executor_pool.shutdown(wait=False, cancel_futures=True)
|
||
self._executor_pool = None
|
||
raise ReportedCompileError(
|
||
"include worker 异常终止，编译立即停止") from None
|
||
if status == 'ok':
|
||
_vlog().info(f" [{done_count}/{len(need_translate)}] 翻译完成: {module_name}.py")
|
||
else:
|
||
# worker 只回传错误、不写 stderr：完整错误（含源码定位）
|
||
# 只在这里输出一次，随后立即终止（fail-fast）。
|
||
_vlog().error(f"\n文件翻译失败: {module_name}.py\n{err}")
|
||
for f2 in in_progress:
|
||
f2.cancel()
|
||
if self._executor_pool is not None:
|
||
self._executor_pool.shutdown(wait=False, cancel_futures=True)
|
||
self._executor_pool = None
|
||
raise ReportedCompileError(
|
||
"include 翻译失败，编译立即停止") from None
|
||
except Exception:
|
||
if self._executor_pool is not None:
|
||
self._executor_pool.shutdown(wait=False, cancel_futures=True)
|
||
self._executor_pool = None
|
||
raise
|
||
|
||
if errors:
|
||
_vlog().error(f"\n{len(errors)} 个 include 文件翻译失败")
|
||
sys.exit(1)
|
||
else:
|
||
# 串行回退
|
||
for item in need_translate:
|
||
src_path, sha1, module_name, ll_path, obj_path, precompiled_obj = item
|
||
_vlog().info(f" 翻译: {module_name}.py -> {sha1}.ll")
|
||
try:
|
||
self._translate_file(src_path, ll_path, prebuilt_ModuleSha1Map=include_ModuleSha1Map, dwarf_enabled=self.traceback_enabled)
|
||
except Exception as e:
|
||
# 单点输出：完整错误（含源码定位）只打印这一次
|
||
_vlog().error(f"\n文件翻译失败: {module_name}.py\n{type(e).__name__}: {e}")
|
||
raise ReportedCompileError(
|
||
"include 翻译失败，编译立即停止") from None
|
||
else:
|
||
item = need_translate[0]
|
||
src_path, sha1, module_name, ll_path, obj_path, precompiled_obj = item
|
||
_vlog().info(f" 翻译: {module_name}.py -> {sha1}.ll")
|
||
try:
|
||
self._translate_file(src_path, ll_path, prebuilt_ModuleSha1Map=include_ModuleSha1Map, dwarf_enabled=self.traceback_enabled)
|
||
except Exception as e:
|
||
# 单点输出：完整错误（含源码定位）只打印这一次
|
||
_vlog().error(f"\n文件翻译失败: {module_name}.py\n{type(e).__name__}: {e}")
|
||
raise ReportedCompileError(
|
||
"include 翻译失败，编译立即停止") from None
|
||
|
||
# ========== 阶段2：串行后处理（dso_local + stub_decls 注入）==========
|
||
compile_tasks = [] # [(dso_ll_path, obj_path, src_path, sha1, module_name, ll_path, precompiled_obj, ll_content)]
|
||
for item in need_translate:
|
||
src_path, sha1, module_name, ll_path, obj_path, precompiled_obj = item
|
||
if not _CachedIsFile(ll_path):
|
||
# 翻译成功（worker 返回 ok）却没有产物 = 翻译失败：立即终止，
|
||
# 不允许跳过该模块继续后处理/编译/链接（否则只会得到链接期
|
||
# undefined symbol，真因与现场脱节）。
|
||
_vlog().error(f" 翻译未生成 .ll 文件: {module_name}.py")
|
||
raise ReportedCompileError(
|
||
f"include 翻译未生成 .ll 文件: {module_name}.py，编译立即停止")
|
||
|
||
ll_content = _ReadTextCached(ll_path)
|
||
|
||
# --traceback 提醒：include 模块若发射了 define 却不带 DI（!llvm.dbg.cu），
|
||
# 其 .debug_line 行为空 ⇒ 运行时 Ctrl+C 栈里该 include 帧取不到源行
|
||
# （会退化为 +0x… (unknown)）。
|
||
# **只警告、不报错**：像 includes/stdarg.vp 这类模块的 define 是"合成"的
|
||
# （由 extern/sugar 派生、无 .vp 函数体），本就不会有源级 DI —— 按 fail-fast
|
||
# 报错会把这类模块全部误杀；而"行号缺失"只降低可读性，不是编译正确性问题。
|
||
if self.traceback_enabled and _RE_DEFINE_LINE.search(ll_content) \
|
||
and '!llvm.dbg.cu' not in ll_content \
|
||
and _SourceHasRealDef(src_path):
|
||
_vlog().warning(
|
||
f"--traceback: include 模块 {module_name}.py 发射了 define 但 IR 未携带 "
|
||
f"DWARF (!llvm.dbg.cu)——该模块内联帧在 Ctrl+C 栈里将只显示地址: {ll_path}")
|
||
|
||
# dso_local 正则标注（使用 ^ + MULTILINE 避免匹配字符串常量内容中的 define/declare）
|
||
ll_content = _RE_DEFINE_DECLARE_LINKAGE.sub(r'\1 \2 dso_local ', ll_content)
|
||
ll_content = _RE_DEFINE_DECLARE_NO_LINKAGE.sub(r'\1 dso_local ', ll_content)
|
||
# LLVM 22+: external 不允许在带初值的全局定义上显式写出，改为 linkonce 保留多模块合并语义
|
||
ll_content = _RE_EXTERNAL_GLOBAL_VALUE.sub(r'\1 = linkonce global \2 \3', ll_content)
|
||
ll_content = _RE_GLOBAL_LINKAGE_MULTILINE.sub(r'\1 = \2 dso_local \3', ll_content)
|
||
ll_content = _RE_GLOBAL_DSOLOCAL_MULTILINE.sub(r'\1 = dso_local \2', ll_content)
|
||
ll_content = _AddWeakComdat(ll_content)
|
||
|
||
# stub_decls 注入（复用预收集的 stub_decls）
|
||
if stub_decls:
|
||
for sdecl in stub_decls:
|
||
sname = Phase2Translator._extract_llvm_type_name(sdecl)
|
||
if sname:
|
||
short = sname.split('.')[-1] if '.' in sname else sname
|
||
ll_content = re.sub(
|
||
r'%"?' + re.escape(sname) + r'"?\s*=\s*type\s*opaque',
|
||
sdecl, ll_content)
|
||
if short != sname:
|
||
ll_content = re.sub(
|
||
r'%"?' + re.escape(short) + r'"?\s*=\s*type\s*opaque',
|
||
sdecl, ll_content)
|
||
|
||
max_iterations = 10
|
||
for _ in range(max_iterations):
|
||
defined_types = set()
|
||
for line in ll_content.splitlines():
|
||
sl = line.strip()
|
||
if sl.startswith('%') and '= type' in sl:
|
||
tname = Phase2Translator._extract_llvm_type_name(sl)
|
||
if tname:
|
||
defined_types.add(tname)
|
||
# 同时添加短名，防止 missing 循环因全名/短名不匹配而重复添加
|
||
if '.' in tname:
|
||
parts = tname.split('.', 1)
|
||
if len(parts[0]) == 16 and all(c in '0123456789abcdef' for c in parts[0]):
|
||
defined_types.add(parts[1])
|
||
|
||
referenced_types = Phase2Translator._find_all_type_refs(ll_content)
|
||
missing = referenced_types - defined_types
|
||
new_defs = []
|
||
for tname in sorted(missing):
|
||
if tname in all_type_defs:
|
||
new_defs.append(all_type_defs[tname])
|
||
else:
|
||
short_key = tname.split('.')[-1] if '.' in tname else tname
|
||
for full_name, defn in all_type_defs.items():
|
||
if full_name.split('.')[-1] == short_key:
|
||
if short_key not in defined_types:
|
||
new_defs.append(defn)
|
||
break
|
||
else:
|
||
# 未在 stub 中找到定义（如泛型类 list[T]），
|
||
# 声明为 opaque 以满足 llc 的类型前向声明要求
|
||
new_defs.append(f'%"{tname}" = type opaque')
|
||
|
||
if not new_defs:
|
||
break
|
||
|
||
inject_block = '\n'.join(new_defs) + '\n'
|
||
lines = ll_content.splitlines(True)
|
||
insert_pos = 0
|
||
for i, line in enumerate(lines):
|
||
stripped = line.strip()
|
||
if stripped and not stripped.startswith(';') and not stripped.startswith('target ') and not stripped.startswith('source_filename') and not stripped.startswith('declare ') and not stripped.startswith('@') and not stripped.startswith('define ') and not stripped.startswith('attributes ') and not stripped.startswith('!') and not stripped.startswith('module '):
|
||
insert_pos = i
|
||
break
|
||
if insert_pos == 0:
|
||
insert_pos = len(lines)
|
||
ll_content = ''.join(lines[:insert_pos]) + inject_block + ''.join(lines[insert_pos:])
|
||
|
||
# 将所有类型定义移到 declares 之前
|
||
# （llc 要求 by-value struct 参数的类型在 declare 前定义，否则报 invalid type for function argument）
|
||
lines = ll_content.splitlines(True)
|
||
type_def_lines = []
|
||
other_lines = []
|
||
seen_type_names_inc = set()
|
||
for line in lines:
|
||
stripped = line.strip()
|
||
if stripped.startswith('%') and '= type' in stripped:
|
||
type_name_inc = stripped.split('=')[0].strip()
|
||
if type_name_inc in seen_type_names_inc:
|
||
continue
|
||
seen_type_names_inc.add(type_name_inc)
|
||
type_def_lines.append(line)
|
||
else:
|
||
other_lines.append(line)
|
||
if type_def_lines:
|
||
insert_pos = 0
|
||
for i, line in enumerate(other_lines):
|
||
stripped = line.strip()
|
||
if stripped and not stripped.startswith(';') and not stripped.startswith('target ') and not stripped.startswith('source_filename') and not stripped.startswith('!'):
|
||
insert_pos = i
|
||
break
|
||
if insert_pos == 0:
|
||
insert_pos = len(other_lines)
|
||
ll_content = ''.join(other_lines[:insert_pos]) + ''.join(type_def_lines) + ''.join(other_lines[insert_pos:])
|
||
|
||
# 同 App 路径：写盘前规整元数据（拼接片段各自带 !0..!N 的 DI 元数据），
|
||
# 让盘上产物本身即合法 IR，不依赖 _llc_compile 里那次事后规整。
|
||
ll_content = SanitizeIRMetadata(ll_content)
|
||
# 同 App 路径：拼接片段各自带一份同名 comdat 声明 ⇒ 落盘前统一去重
|
||
ll_content = _DedupComdatDecls(ll_content)
|
||
dso_ll_path = ll_path[:-3] + '_dso.ll'
|
||
with open(dso_ll_path, 'w', encoding='utf-8') as f:
|
||
f.write(ll_content)
|
||
_InvalidateTextCache(dso_ll_path)
|
||
compile_tasks.append((dso_ll_path, obj_path, src_path, sha1, module_name, ll_path, precompiled_obj, ll_content))
|
||
|
||
if not compile_tasks:
|
||
return
|
||
|
||
# ========== 阶段3：并行编译（llc）==========
|
||
_vlog().info(f" 编译 {len(compile_tasks)} 个 .ll 文件")
|
||
compile_results = {} # task -> result
|
||
|
||
def _compile_one_include(task):
|
||
dso_ll_path, obj_path = task[0], task[1]
|
||
result = self._llc_compile(dso_ll_path, obj_path, self.output_dir)
|
||
return (task, result)
|
||
|
||
if len(compile_tasks) > 1:
|
||
n_workers = _PickWorkerCount(len(compile_tasks), 400)
|
||
with ThreadPoolExecutor(max_workers=n_workers) as executor:
|
||
futures = {executor.submit(_compile_one_include, task): task for task in compile_tasks}
|
||
done_count = 0
|
||
for future in as_completed(futures):
|
||
done_count += 1
|
||
try:
|
||
task, result = future.result()
|
||
module_name = task[4]
|
||
if result.returncode == 0:
|
||
_vlog().success(f" [{done_count}/{len(compile_tasks)}] 编译成功: {module_name}.py")
|
||
else:
|
||
_vlog().warning(f" [{done_count}/{len(compile_tasks)}] 编译失败: {module_name}.py: {result.stderr.strip()}")
|
||
compile_results[task] = result
|
||
except Exception as e:
|
||
task = futures[future]
|
||
module_name = task[4]
|
||
_vlog().error(f" [{done_count}/{len(compile_tasks)}] 编译异常: {module_name}.py: {e}")
|
||
compile_results[task] = None
|
||
else:
|
||
task = compile_tasks[0]
|
||
task_result, result = _compile_one_include(task)
|
||
module_name = task[4]
|
||
if result.returncode == 0:
|
||
_vlog().success(f" 编译成功: {module_name}.py")
|
||
else:
|
||
_vlog().warning(f" 编译失败: {module_name}.py: {result.stderr.strip()}")
|
||
compile_results[task] = result
|
||
|
||
# ========== 阶段4：注册 + 缓存 ==========
|
||
has_error = False
|
||
for task in compile_tasks:
|
||
dso_ll_path, obj_path, src_path, sha1, module_name, ll_path, precompiled_obj, ll_content = task
|
||
result = compile_results.get(task)
|
||
if result and result.returncode == 0:
|
||
self.extra_link_files.append(obj_path)
|
||
# 复制到 includes.binary 预编译目录（--traceback 时 precompiled_obj
|
||
# 指向隔离目录 includes.binary_traceback，不会污染共享缓存）
|
||
if precompiled_obj:
|
||
precompiled_dir = os.path.dirname(precompiled_obj)
|
||
try:
|
||
if not os.path.exists(precompiled_dir):
|
||
os.makedirs(precompiled_dir, exist_ok=True)
|
||
shutil.copy2(obj_path, precompiled_obj)
|
||
except Exception as _e:
|
||
# --traceback fail-fast：隔离目录创建 / 写回失败必须显式终止，
|
||
# 不静默降级（否则本次构建的 include obj 拿不到、或落到共享缓存
|
||
# 之外的位置，DWARF 行表与后续复用的前提都被破坏）。
|
||
if self.traceback_enabled:
|
||
raise RuntimeError(
|
||
f"--traceback: include 预编译缓存写入隔离目录失败: "
|
||
f"{precompiled_obj}: {_e}") from _e
|
||
if _config_mode == "strict":
|
||
_vlog().warning(f"预编译缓存写入失败: {_e}")
|
||
# 注册符号
|
||
self._register_compiled_include(src_path, sha1, ll_content, include_ModuleSha1Map)
|
||
# 清理临时文件（仅成功时清理，失败时保留以便调试）
|
||
# DEBUG: 保留 .ll 文件以便静态分析
|
||
# for ext in ['.ll', '_dso.ll']:
|
||
# p = ll_path[:-3] + ext
|
||
# if _CachedIsFile(p):
|
||
# os.remove(p)
|
||
else:
|
||
has_error = True
|
||
_vlog().warning(f" 保留失败文件以便调试: {ll_path}")
|
||
|
||
if has_error:
|
||
_vlog().error(f"\nincludes 文件编译失败")
|
||
sys.exit(1)
|
||
|
||
# 从 include 文件源码中提取函数参数名，供主项目编译时 keyword 参数匹配使用
|
||
self._extract_include_param_names(sorted_files)
|
||
|
||
def _extract_include_param_names_from_dirs(self) -> None:
|
||
"""从 include 目录的 Python 源码中提取函数参数名与默认值。
|
||
|
||
顶层 .vp/.py 直接解析；包子目录（如 includes/json）只在该包被
|
||
used_includes 引用时递归解析——避免为每个工程全量递归 includes
|
||
（fltk/vqt6/openssl 等上百个文件）带来的解析开销。
|
||
默认值一并注册（跨模块缺参调用点据此填充，见 HandlesExprCall
|
||
._HandleClosureCallLlvm），此前只注册参数名，导致 include 模块的
|
||
尾随默认参数在调用点被零值填充。
|
||
"""
|
||
for includes_dir in self.include_dirs:
|
||
if not os.path.isdir(includes_dir):
|
||
continue
|
||
for entry in os.listdir(includes_dir):
|
||
entry_path = os.path.join(includes_dir, entry)
|
||
targets: list[str] = []
|
||
if os.path.isdir(entry_path):
|
||
if not self.used_includes or entry not in self.used_includes:
|
||
continue
|
||
for root, dirs, files in os.walk(entry_path):
|
||
dirs[:] = [d for d in dirs if not d.startswith('.') and d != '__pycache__']
|
||
targets.extend(os.path.join(root, f) for f in files
|
||
if f.endswith('.vp') or f.endswith('.py'))
|
||
elif entry.endswith('.vp') or entry.endswith('.py'):
|
||
targets.append(entry_path)
|
||
for src_path in targets:
|
||
try:
|
||
source, tree = parse_python_file(src_path)
|
||
if tree is None:
|
||
continue
|
||
sha1 = compute_sha1(source)
|
||
self._collect_function_params(tree, sha1)
|
||
except Exception as e:
|
||
_vlog().warning(f" 提取参数名失败 {src_path}: {e}")
|
||
|
||
def _collect_function_params(self, tree: ast.AST, sha1: str | None) -> None:
|
||
"""单次递归遍历 AST 收集函数参数名（含类方法）与默认值，O(N) 复杂度
|
||
|
||
遍历时维护 current_class 上下文：遇到 ClassDef 进入类作用域，
|
||
遇到 FunctionDef 记录参数名并标注所属类，函数内部的嵌套定义
|
||
不再属于外层类。
|
||
|
||
默认值只登记非方法（模块级函数）的裸名键，减少与其它模块同名函数的
|
||
误配；带 sha1 / 类名限定的键始终登记，查找时精确键优先。
|
||
"""
|
||
def visit(node: ast.AST, current_class: str | None) -> None:
|
||
for child in ast.iter_child_nodes(node):
|
||
if isinstance(child, ast.ClassDef):
|
||
# 进入类作用域，递归处理类体
|
||
visit(child, child.name)
|
||
elif isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
||
func_name = child.name
|
||
param_names = [arg.arg for arg in child.args.args]
|
||
self.function_param_names[func_name] = param_names
|
||
if sha1:
|
||
self.function_param_names[f'{sha1}.{func_name}'] = param_names
|
||
if current_class:
|
||
self.function_param_names[f'{current_class}.{func_name}'] = param_names
|
||
if sha1:
|
||
self.function_param_names[f'{sha1}.{current_class}.{func_name}'] = param_names
|
||
defaults = getattr(child.args, 'defaults', []) or []
|
||
if defaults:
|
||
default_vals = []
|
||
for d in defaults:
|
||
if isinstance(d, ast.Constant):
|
||
default_vals.append(d.value)
|
||
else:
|
||
default_vals.append(None)
|
||
if sha1:
|
||
self.function_default_args[f'{sha1}.{func_name}'] = default_vals
|
||
if current_class:
|
||
self.function_default_args[f'{current_class}.{func_name}'] = default_vals
|
||
if sha1:
|
||
self.function_default_args[f'{sha1}.{current_class}.{func_name}'] = default_vals
|
||
else:
|
||
self.function_default_args[func_name] = default_vals
|
||
# 函数内部的嵌套定义不属于外层类
|
||
visit(child, None)
|
||
else:
|
||
visit(child, current_class)
|
||
visit(tree, None)
|
||
|
||
def _extract_app_default_args(self, need_translate: list) -> None:
|
||
"""从 App 源文件中提取函数默认参数，存入 function_default_args
|
||
|
||
在 Phase 2 并行编译开始前调用，确保跨模块调用参数检查能正确识别默认参数。
|
||
need_translate 中每个元素是 (i, src_path, out_path, sha1, current_ModuleSha1Map)
|
||
复用 parse_python_file 的 AST 缓存（run() 循环已 parse 过）。
|
||
"""
|
||
for entry in need_translate:
|
||
src_path = entry[1]
|
||
sha1 = entry[3]
|
||
_, tree = parse_python_file(src_path)
|
||
if tree is None:
|
||
continue
|
||
try:
|
||
for node in ast.walk(tree):
|
||
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
||
func_name = node.name
|
||
defaults = getattr(node.args, 'defaults', [])
|
||
if defaults:
|
||
default_vals = []
|
||
for d in defaults:
|
||
if isinstance(d, ast.Constant):
|
||
default_vals.append(d.value)
|
||
else:
|
||
default_vals.append(None)
|
||
# 存储多种键格式，确保跨模块调用检查能找到
|
||
self.function_default_args[func_name] = default_vals
|
||
self.function_default_args[f'{sha1}.{func_name}'] = default_vals
|
||
# 也更新参数名（预扫描时补充）
|
||
param_names = [arg.arg for arg in node.args.args]
|
||
if func_name not in self.function_param_names:
|
||
self.function_param_names[func_name] = param_names
|
||
if f'{sha1}.{func_name}' not in self.function_param_names:
|
||
self.function_param_names[f'{sha1}.{func_name}'] = param_names
|
||
except Exception as e:
|
||
_vlog().warning(f" 提取默认参数失败 {src_path}: {e}")
|
||
|
||
def _extract_include_param_names(self, sorted_files: list[str]) -> None:
|
||
"""从 include 文件的 Python 源码中提取函数参数名，存入 function_param_names
|
||
|
||
优化：复用 _collect_function_params 的 O(N) 单次遍历 + parse_python_file 缓存。
|
||
"""
|
||
for src_path in sorted_files:
|
||
if self._is_decl_only_file(src_path):
|
||
continue
|
||
try:
|
||
_, tree = parse_python_file(src_path)
|
||
if tree is None:
|
||
continue
|
||
module_name = os.path.splitext(os.path.basename(src_path))[0]
|
||
sha1 = self.include_py_map.get(module_name)
|
||
self._collect_function_params(tree, sha1)
|
||
except Exception as e:
|
||
_vlog().warning(f" 提取参数名失败 {src_path}: {e}")
|
||
|
||
def _register_compiled_include(self, src_path: str, sha1: str, ll_content: str, include_ModuleSha1Map: dict[str, str]) -> None:
|
||
"""编译 include 文件成功后，生成签名和 stub 文件并注册到符号表中"""
|
||
module_name = os.path.splitext(os.path.basename(src_path))[0]
|
||
|
||
# 确定完整模块名（如 vpsdk.window）
|
||
mod_full = module_name
|
||
for includes_dir in self.include_dirs:
|
||
try:
|
||
rel = os.path.relpath(src_path, includes_dir)
|
||
mod_full = os.path.splitext(rel)[0].replace(os.sep, '.').replace('/', '.')
|
||
break
|
||
except ValueError:
|
||
pass
|
||
|
||
# 1. 生成 .pyi 签名文件（如果还没有的话）
|
||
sig_path = os.path.join(self.temp_dir, f"{sha1}.vpi")
|
||
if not _CachedIsFile(sig_path):
|
||
# 全局缓存检查（不被 --clean 清除）
|
||
_TryGlobalCache(sha1, 'pyi', sig_path)
|
||
if not _CachedIsFile(sig_path):
|
||
try:
|
||
py_content = _ReadTextCached(src_path)
|
||
sig_content = PythonToStubConverter.convert(py_content, mod_full)
|
||
os.makedirs(self.temp_dir, exist_ok=True)
|
||
with open(sig_path, 'w', encoding='utf-8', newline='\n') as f:
|
||
f.write(sig_content)
|
||
_InvalidateTextCache(sig_path)
|
||
_SaveToGlobalCache(sha1, 'pyi', sig_path)
|
||
except Exception as e:
|
||
if _config_mode == "strict":
|
||
raise
|
||
_vlog().warning(f"签名生成失败不阻塞编译: {e}", "Exception")
|
||
|
||
# 注册到 sig_files 和 _source_module_sig_files
|
||
if _CachedIsFile(sig_path):
|
||
self.sig_files[sha1] = sig_path
|
||
self.sha1_map[sha1] = f"includes/{mod_full.replace('.', '/')}.vp"
|
||
if self._shared_source_module_sig_files is not None:
|
||
self._shared_source_module_sig_files[mod_full] = sig_path
|
||
self._shared_source_module_sig_files[module_name] = sig_path
|
||
|
||
# 将签名信息加载到共享符号表中
|
||
if self._shared_symbol_table is not None:
|
||
try:
|
||
self._shared_symbol_table.LoadModuleSymbols(sig_path, mod_full, lineno=0)
|
||
except Exception as _e:
|
||
if _config_mode == "strict":
|
||
raise
|
||
_vlog().warning(f"加载模块签名到共享符号表失败: {_e}", "Exception")
|
||
|
||
# 2. 生成 .stub.ll 文件
|
||
stub_path = os.path.join(self.output_dir, f"{sha1}.stub.ll")
|
||
if not _CachedIsFile(stub_path):
|
||
# 全局缓存检查（不被 --clean 清除）
|
||
_TryGlobalCache(sha1, 'stub.ll', stub_path)
|
||
if not _CachedIsFile(stub_path) and ll_content:
|
||
try:
|
||
stub_content, _ = self._split_ll(ll_content)
|
||
with open(stub_path, 'w', encoding='utf-8', newline='\n') as f:
|
||
f.write(stub_content)
|
||
_SaveToGlobalCache(sha1, 'stub.ll', stub_path)
|
||
except Exception as _e:
|
||
if _config_mode == "strict":
|
||
raise
|
||
_vlog().warning(f"生成 .stub.ll 文件失败: {_e}", "Exception")
|
||
|
||
# 注册到 stub_files
|
||
if _CachedIsFile(stub_path):
|
||
self.stub_files[sha1] = stub_path
|
||
# output/<sha1>.stub.ll 存在（本步刚落盘或从全局缓存拷入）→ 置脏，
|
||
# 让后续索引 miss 重扫到新增的 *.stub.ll（见 _build_stub_index 快路径）
|
||
from lib.core.Handles.HandlesImports import ImportHandle as _IH3
|
||
_IH3.MarkStubSetDirty()
|
||
elif _CachedIsFile(sig_path):
|
||
# 如果 stub 文件不存在，也尝试从 temp 目录查找
|
||
temp_stub = os.path.join(self.temp_dir, f"{sha1}.stub.ll")
|
||
if _CachedIsFile(temp_stub):
|
||
self.stub_files[sha1] = temp_stub
|
||
else:
|
||
# 最后尝试从全局缓存直接读取
|
||
_TryGlobalCache(sha1, 'stub.ll', temp_stub)
|
||
if _CachedIsFile(temp_stub):
|
||
self.stub_files[sha1] = temp_stub
|
||
|
||
def _precollect_inline_symbols(self) -> None:
|
||
"""在主翻译循环之前，扫描includes目录中used_includes相关的模块收集内联函数AST"""
|
||
if not self.used_includes:
|
||
return
|
||
for includes_dir in self.include_dirs:
|
||
if not os.path.isdir(includes_dir):
|
||
continue
|
||
for module_name in self.used_includes:
|
||
module_dir = os.path.join(includes_dir, module_name)
|
||
if os.path.isdir(module_dir):
|
||
for root, dirs, files in os.walk(module_dir):
|
||
dirs[:] = [d for d in dirs if not d.startswith('.') and d != '__pycache__']
|
||
for fname in files:
|
||
if fname.endswith('.vp') or fname.endswith('.py'):
|
||
src_path = os.path.join(root, fname)
|
||
self._collect_inline_symbols(src_path)
|
||
py_path = os.path.join(includes_dir, module_name + '.vp')
|
||
if not _CachedIsFile(py_path):
|
||
# 回退到 .py（向后兼容）
|
||
py_path = os.path.join(includes_dir, module_name + '.py')
|
||
if _CachedIsFile(py_path):
|
||
self._collect_inline_symbols(py_path)
|
||
for ef in self.extra_py_files:
|
||
self._collect_inline_symbols(ef)
|
||
|
||
def _collect_inline_symbols(self, src_path: str):
|
||
"""解析源文件，收集内联函数的AST信息到inline_func_symbols
|
||
|
||
复用 parse_python_file 的 AST 缓存，避免重复 open+read+ast.parse。
|
||
"""
|
||
try:
|
||
_, tree = parse_python_file(src_path)
|
||
if tree is None:
|
||
return
|
||
module_name = os.path.splitext(os.path.basename(src_path))[0]
|
||
for node in ast.iter_child_nodes(tree):
|
||
if isinstance(node, ast.FunctionDef) and node.returns:
|
||
# 新语法 `inline` 由解析器写入返回注解的 Markers。
|
||
ReturnMarkers: tuple[str, ...] = getattr(node.returns, 'Markers', None) or ()
|
||
is_inline: bool = 'inline' in ReturnMarkers
|
||
if isinstance(node.returns, ast.BinOp) and isinstance(node.returns.op, ast.BitOr):
|
||
_Line: int = getattr(node.returns, 'lineno', 0) or 0
|
||
raise TypeError(
|
||
f"[TPC] 不支持类型联合 'T1 | T2'（第 {_Line} 行）；"
|
||
f"内联标记请写 `-> inline T`，类型注解只能是单一类型")
|
||
if is_inline:
|
||
func_name = node.name
|
||
sym_key = f"{module_name}.{func_name}"
|
||
info = CTypeInfo()
|
||
info.Name = func_name
|
||
info.IsFunction = True
|
||
info.IsInline = True
|
||
info.InlineBody = node.body
|
||
info.InlineParams = [arg.arg for arg in node.args.args]
|
||
info.Storage = TypeSystem.CInline()
|
||
self.inline_func_symbols[func_name] = info
|
||
self.inline_func_symbols[sym_key] = info
|
||
except TypeError:
|
||
# 类型注解 fail-fast（BinOp(BitOr) 联合）不得被 strict 开关吞掉
|
||
raise
|
||
except Exception as _e:
|
||
if _config_mode == "strict":
|
||
raise
|
||
_vlog().warning(f"预收集内联函数符号失败: {_e}", "Exception")
|
||
|
||
def _build_shared_symbol_data(self) -> None:
|
||
"""一次性构建所有文件共享的符号表数据，避免每个文件重复加载"""
|
||
t0 = time.time()
|
||
|
||
_vlog().info("[缓存] 构建共享符号表数据...")
|
||
|
||
base_trans = TransPyC.TransPyC(code="pass", triple=self.triple, datalayout=self.datalayout)
|
||
base_trans.translator.LlvmGen = None
|
||
base_trans.translator._ModuleSha1Map = {}
|
||
base_trans.translator._global_function_default_args = self.function_default_args
|
||
base_trans.translator._global_function_param_names = self.function_param_names
|
||
|
||
# ---- 两阶段加载：先收集全部模块，再「①声明」全跑完 / 「②签名」全跑完 ----
|
||
# 声明阶段把类/枚举/typedef 灌满符号表，签名阶段才解析函数签名 ⇒ 模块间
|
||
# 类型引用（`from X import *` 的裸名如 posix 的 `INT`、跨模块类如
|
||
# `rawptr[Constant]`）不再依赖 os.listdir 的字典序（此前 posix 排在 stdint
|
||
# 之前 ⇒ `INT` 解析不到，旧口径静默降级成 i32，fail-fast 后必须消除）。
|
||
staged: list[tuple[str, str]] = [] # (声明文件路径, 模块名)
|
||
# 需要重导出的包（在全部模块符号加载完后统一处理；提到循环外避免多个
|
||
# include_dir 时被反复覆盖/重复收集）
|
||
_pending_reexports: list[tuple[str, str]] = []
|
||
for includes_dir in self.include_dirs:
|
||
if os.path.isdir(includes_dir):
|
||
for pyi_file in os.listdir(includes_dir):
|
||
if pyi_file.endswith('.vpi') or pyi_file.endswith('.pyi'):
|
||
pyi_path = os.path.join(includes_dir, pyi_file)
|
||
module_name = os.path.splitext(pyi_file)[0]
|
||
staged.append((pyi_path, module_name))
|
||
for py_file in os.listdir(includes_dir):
|
||
if (py_file.endswith('.vp') or py_file.endswith('.py')) and (not py_file.startswith('_') or py_file == '_list.vp' or py_file == '_list.py' or py_file == '_dict.vp' or py_file == '_dict.py'):
|
||
module_name = os.path.splitext(py_file)[0]
|
||
sha1_key = self.include_py_map.get(module_name)
|
||
if sha1_key and sha1_key in self.sig_files:
|
||
staged.append((self.sig_files[sha1_key], module_name))
|
||
# 收集需要重导出的包，等所有模块符号加载完后再处理
|
||
for sha1_key, sig_path in self.sig_files.items():
|
||
rel_path = self.sha1_map.get(sha1_key, sha1_key)
|
||
if rel_path.startswith('includes/'):
|
||
mod_name = os.path.splitext(rel_path)[0].replace(os.sep, '.').replace('/', '.')
|
||
mod_name = mod_name[len('includes/'):]
|
||
_entry_mod: tuple[str, str] = (sig_path, mod_name)
|
||
if _entry_mod not in staged:
|
||
staged.append(_entry_mod)
|
||
if mod_name.endswith('.__init__'):
|
||
package_name = mod_name[:-len('.__init__')]
|
||
_entry_pkg: tuple[str, str] = (sig_path, package_name)
|
||
if _entry_pkg not in _pending_reexports:
|
||
_pending_reexports.append(_entry_pkg)
|
||
|
||
for sha1_key, sig_path in self.sig_files.items():
|
||
rel_path = self.sha1_map.get(sha1_key, sha1_key)
|
||
if rel_path.startswith('includes/'):
|
||
continue
|
||
module_name = os.path.splitext(rel_path)[0].replace(os.sep, '.').replace('/', '.')
|
||
staged.append((sig_path, module_name))
|
||
base_trans.translator._source_module_sig_files[module_name] = sig_path
|
||
short_name = module_name.split('.')[-1] if '.' in module_name else module_name
|
||
if short_name != module_name:
|
||
base_trans.translator._source_module_sig_files[short_name] = sig_path
|
||
|
||
# ① 声明阶段（全部模块；多轮直到"本轮没有新增可解析的声明"）
|
||
# 单轮仍会与模块顺序有关：typedef 右侧可以引用**其它模块**的 typedef，
|
||
# 例 win32base 的 `typedef HANDLE = VOIDPTR`，VOIDPTR 在 stdint
|
||
# ⇒ 本模块先 stage 时右侧解析为空；下一轮把它重 stage 一遍（重 stage 覆盖
|
||
# 登记，条目随依赖入表而变完整）。无进展即停，最多再跑一轮。
|
||
_stage_map: dict[tuple[str, str], ModuleSymbols] = {}
|
||
_round_pending = list(staged)
|
||
_prev_miss: int | None = None
|
||
while _round_pending:
|
||
_miss_total: int = 0
|
||
_next_round: list[tuple[str, str]] = []
|
||
for _stage_path, _stage_mod in _round_pending:
|
||
_stage_ms = base_trans.translator.SymbolTable.StageModuleSymbols(_stage_path, _stage_mod)
|
||
if _stage_ms is None:
|
||
continue
|
||
_stage_map[(_stage_path, _stage_mod)] = _stage_ms
|
||
_miss: int = _CountUnresolvedTypedefs(_stage_ms)
|
||
_miss_total += _miss
|
||
if _miss:
|
||
_next_round.append((_stage_path, _stage_mod))
|
||
if not _next_round or (_prev_miss is not None and _miss_total >= _prev_miss):
|
||
break
|
||
_prev_miss = _miss_total
|
||
_round_pending = _next_round
|
||
|
||
# ② 签名阶段（全部模块；此时任一模块的签名都能看到全部声明）
|
||
for (_commit_path, _commit_mod), _commit_ms in _stage_map.items():
|
||
base_trans.translator.SymbolTable.CommitModuleSymbols(_commit_ms, _commit_mod, _commit_path)
|
||
|
||
# 所有 includes 模块符号已加载，现在处理包重导出
|
||
for sig_path, package_name in _pending_reexports:
|
||
self._register_package_reexports(base_trans, sig_path, package_name)
|
||
|
||
all_dc = {}
|
||
all_dt: dict[str, tuple[int, bool]] = {}
|
||
for sha1_key, stub_path in self.stub_files.items():
|
||
if os.path.exists(stub_path):
|
||
try:
|
||
stub_content = _ReadTextCached(stub_path)
|
||
for line in stub_content.splitlines():
|
||
line = line.strip()
|
||
if line.startswith('@') and '= global' in line and 'i32' in line and not line.startswith('@llvm'):
|
||
var_name = line.split('=')[0].strip().lstrip('@')
|
||
val_part = line.split('i32')[-1].strip().rstrip(']').lstrip('[').lstrip('i32')
|
||
try:
|
||
val = int(val_part)
|
||
all_dc[var_name] = val
|
||
except Exception as e:
|
||
_vlog().warning(f"Phase2Translator: 忽略异常 {e}", exc_info=e)
|
||
except Exception as e:
|
||
_vlog().warning(f"Phase2Translator: 忽略异常 {e}", exc_info=e)
|
||
|
||
for sha1_key, pyi_path in self.sig_files.items():
|
||
_ExtractCDefineConstantsFromPyi(pyi_path, all_dc, base_trans.translator.SymbolTable, all_dt)
|
||
|
||
export_extern_funcs = {}
|
||
for sha1_key, sig_path in self.sig_files.items():
|
||
try:
|
||
# 复用 parse_python_file 的 AST 缓存（sig 文件被多处解析）
|
||
_, sig_tree = parse_python_file(sig_path)
|
||
if sig_tree is None:
|
||
continue
|
||
for node in ast.iter_child_nodes(sig_tree):
|
||
if isinstance(node, ast.FunctionDef):
|
||
if _IsExportExternFuncNode(node):
|
||
export_extern_funcs[node.name] = sha1_key
|
||
except Exception as _e:
|
||
if _config_mode == "strict":
|
||
raise
|
||
_vlog().warning(f"收集共享导出外部函数失败: {_e}", "Exception")
|
||
|
||
for sym_name, sym_info in self.inline_func_symbols.items():
|
||
if sym_name not in base_trans.translator.SymbolTable:
|
||
base_trans.translator.SymbolTable[sym_name] = sym_info
|
||
else:
|
||
existing = base_trans.translator.SymbolTable[sym_name]
|
||
if not getattr(existing, 'InlineBody', None) and getattr(sym_info, 'InlineBody', None):
|
||
existing.IsInline = sym_info.IsInline
|
||
existing.InlineBody = sym_info.InlineBody
|
||
existing.InlineParams = sym_info.InlineParams
|
||
|
||
self._shared_symbol_table = base_trans.translator.SymbolTable
|
||
self._shared_source_module_sig_files = dict(base_trans.translator._source_module_sig_files)
|
||
self._shared_all_dc = all_dc
|
||
self._shared_all_decl_types = all_dt
|
||
self._shared_export_extern_funcs = export_extern_funcs
|
||
|
||
generic_class_templates = {}
|
||
for includes_dir in self.include_dirs:
|
||
if os.path.isdir(includes_dir):
|
||
abs_includes = os.path.join(self.src_root, includes_dir) if not os.path.isabs(includes_dir) else includes_dir
|
||
if not os.path.isdir(abs_includes):
|
||
abs_includes = includes_dir
|
||
for py_file in os.listdir(abs_includes):
|
||
if (py_file.endswith('.vp') or py_file.endswith('.py')) and (not py_file.startswith('_') or py_file == '_list.vp' or py_file == '_list.py' or py_file == '_dict.vp' or py_file == '_dict.py'):
|
||
py_path = os.path.join(abs_includes, py_file)
|
||
try:
|
||
# 复用 parse_python_file 的 AST 缓存
|
||
py_content, py_tree = parse_python_file(py_path)
|
||
if py_tree is None:
|
||
continue
|
||
# 计算 include 文件的 SHA1，供特化时统一命名空间
|
||
py_sha1: str = compute_sha1(py_content)
|
||
for node in py_tree.body:
|
||
if isinstance(node, ast.ClassDef) and hasattr(node, 'type_params') and node.type_params:
|
||
type_params = [tp.name for tp in node.type_params]
|
||
generic_class_templates[node.name] = {
|
||
'node': node,
|
||
'type_params': type_params,
|
||
'sha1': py_sha1,
|
||
}
|
||
except Exception as _e:
|
||
if _config_mode == "strict":
|
||
raise
|
||
_vlog().warning(f"收集泛型类模板失败: {_e}", "Exception")
|
||
self._shared_generic_class_templates = generic_class_templates
|
||
|
||
# 加载所有 {sha1}.doc.json 到 _shared_doc_meta，供跨模块 __doc__ 查询
|
||
shared_doc_meta: dict[str, dict[str, str]] = {}
|
||
if os.path.isdir(self.temp_dir):
|
||
for fname in os.listdir(self.temp_dir):
|
||
if fname.endswith('.doc.json'):
|
||
sha1_key = fname[:-len('.doc.json')]
|
||
try:
|
||
with open(os.path.join(self.temp_dir, fname), 'r', encoding='utf-8') as f:
|
||
data = json.load(f)
|
||
if isinstance(data, dict):
|
||
shared_doc_meta[sha1_key] = {str(k): str(v) for k, v in data.items()}
|
||
except Exception as _e:
|
||
if _config_mode == "strict":
|
||
raise
|
||
_vlog().warning(f"加载 {fname} 失败: {_e}", "Exception")
|
||
# 也从全局缓存加载 .doc.json（temp_dir 被 --clean 清空时补充）
|
||
if os.path.isdir(_GLOBAL_CACHE_DIR):
|
||
for fname in os.listdir(_GLOBAL_CACHE_DIR):
|
||
if fname.endswith('.doc.json'):
|
||
sha1_key = fname[:-len('.doc.json')]
|
||
if sha1_key in shared_doc_meta:
|
||
continue
|
||
try:
|
||
data = json.loads(_ReadTextCached(os.path.join(_GLOBAL_CACHE_DIR, fname)))
|
||
if isinstance(data, dict):
|
||
shared_doc_meta[sha1_key] = {str(k): str(v) for k, v in data.items()}
|
||
except Exception as _e:
|
||
if _config_mode == "strict":
|
||
raise
|
||
_vlog().warning(f"加载全局缓存 {fname} 失败: {_e}", "Exception")
|
||
self._shared_doc_meta = shared_doc_meta
|
||
|
||
t1 = time.time()
|
||
_vlog().info(f"[缓存] 共享符号表构建完成 ({t1-t0:.2f}s, {len(self._shared_symbol_table)} 符号, {len(generic_class_templates)} 泛型模板, {len(shared_doc_meta)} doc 元数据)")
|
||
|
||
def _compile_include_sources(self):
|
||
"""扫描项目目录中的汇编存根，编译 includes 目录中发现的 C/C++ 源文件"""
|
||
extra_sources = list(self.extra_compile_files)
|
||
scan_dirs = []
|
||
if self.src_root and os.path.isdir(self.src_root):
|
||
scan_dirs.append(self.src_root)
|
||
# 不再扫描 output_dir 的父目录：当 output_dir 位于顶层时，
|
||
# 父目录是整个仓库根，会误扫到 ViperOS/includes/vqt6 等无关 C/C++ 源文件。
|
||
# 项目特定的 C/汇编文件应放在 src_root 下，或通过 extra_compile_files 指定。
|
||
for scan_dir in scan_dirs:
|
||
for root, dirs, files in os.walk(scan_dir):
|
||
dirs[:] = [d for d in dirs if not d.startswith('.') and d != '__pycache__']
|
||
for fname in files:
|
||
if fname.endswith(('.S', '.s', '.c', '.cpp')):
|
||
if fname.startswith('test_'):
|
||
continue
|
||
fpath = os.path.join(root, fname)
|
||
if fpath not in extra_sources:
|
||
extra_sources.append(fpath)
|
||
if not extra_sources:
|
||
return
|
||
|
||
cc_map = {
|
||
'.c': 'gcc',
|
||
'.cpp': 'g++', '.cc': 'g++', '.cxx': 'g++',
|
||
'.m': 'clang', '.mm': 'clang++',
|
||
'.s': 'gcc', '.S': 'gcc',
|
||
}
|
||
|
||
_vlog().info(f"\n[includes 编译] 找到 {len(extra_sources)} 个源文件")
|
||
|
||
for src_path in extra_sources:
|
||
_, ext = os.path.splitext(src_path)
|
||
ext = ext.lower()
|
||
obj_path = os.path.join(self.output_dir, os.path.splitext(os.path.basename(src_path))[0] + '.obj')
|
||
if ext in ('.s', '.S') and any('oformat' in f for f in self.linker_flags):
|
||
cc = 'clang'
|
||
compile_src = src_path
|
||
extra_flags = ['-c', '-target', 'x86_64-none-elf', '-o', obj_path, compile_src]
|
||
else:
|
||
cc = cc_map.get(ext, 'gcc')
|
||
extra_flags = ['-c', '-o', obj_path, src_path]
|
||
|
||
_vlog().info(f" 编译: {os.path.basename(src_path)} -> {os.path.basename(obj_path)}")
|
||
|
||
try:
|
||
cmd = [cc] + extra_flags
|
||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||
if result.returncode == 0:
|
||
_vlog().success(" 编译成功")
|
||
self.extra_link_files.append(obj_path)
|
||
else:
|
||
_vlog().error(f" 编译失败: {result.stderr.strip()}")
|
||
_vlog().error(f"\nC/C++ 源文件编译失败，立即终止编译。")
|
||
sys.exit(1)
|
||
except FileNotFoundError:
|
||
_vlog().error(f" 找不到编译器: {cc}")
|
||
_vlog().error(f"\n找不到编译器，立即终止编译。")
|
||
sys.exit(1)
|
||
except Exception as e:
|
||
_vlog().error(f" {e}")
|
||
_vlog().error(f"\n编译异常，立即终止编译。")
|
||
sys.exit(1)
|
||
|
||
def _strip_debug_sections(self, exe_path: str) -> None:
|
||
"""剥离 .exe 中的调试段（跳过裸机二进制）"""
|
||
if not os.path.exists(exe_path):
|
||
return
|
||
ext = os.path.splitext(exe_path)[1].lower()
|
||
if ext not in ('.exe', '.dll', '.obj', '.o', '.elf'):
|
||
_vlog().info(f" [剥离] 跳过（{ext} 格式无需剥离）")
|
||
return
|
||
for tool in ['llvm-strip', 'strip']:
|
||
strip_cmd = shutil.which(tool)
|
||
if strip_cmd:
|
||
try:
|
||
result = subprocess.run(
|
||
[strip_cmd, '--strip-debug', exe_path],
|
||
capture_output=True, text=True
|
||
)
|
||
if result.returncode == 0:
|
||
_vlog().info(f" [剥离] 调试段已移除 ({tool})")
|
||
else:
|
||
_vlog().warning(f" [剥离] 警告: {result.stderr.strip()}")
|
||
except Exception as _e:
|
||
if _config_mode == "strict":
|
||
raise
|
||
_vlog().warning(f"剥离可执行文件调试段失败: {_e}", "Exception")
|
||
break
|
||
|
||
def _WriteStackMap(self, exe_path: str) -> None:
|
||
"""--traceback：链接成功后导出"地址→源行"旁文件 <exe>.stackmap。
|
||
|
||
用 llvm-dwarfdump 解析最终 PE 的 .debug_line（地址→文件/行）与
|
||
.debug_info（DISubprogram 的 low/high_pc + DW_AT_name → 函数名），
|
||
换算为 RVA 后落盘。旁文件规格（全部小端 little-endian）：
|
||
|
||
头部 20 字节 = magic(4)="TPCS" + version(u32=1) + entry_count(u32)
|
||
+ str_count(u32) + str_bytes(u32)
|
||
记录 entry_count × 20 字节（按 rva 升序）:
|
||
rva(u64) + file_off(u32) + line(u32) + func_off(u32)
|
||
字符串池 str_bytes 字节（紧接记录区，UTF-8，全部以 '\\0' 结尾）
|
||
|
||
口径（必须与运行期严格一致）：运行期 traceback.vp 取
|
||
`rva = 帧地址 − GetModuleHandleA(NULL)`，即"链接期 ImageBase 之上的偏移"，
|
||
故写端唯一正确口径是 **`RVA = dwarfdump 打印的地址 − PE.ImageBase`**。
|
||
|
||
⚠ 实测 viperc.exe 的 DWARF 里同时存在两个地址域：
|
||
* 绝大多数行/子程序地址是**已重定位的 VA**（0x14000xxxx，含 ImageBase）；
|
||
* 另有 22638 条行记录 / 756 个 subprogram 的地址**恰为 0**——它们是**被
|
||
链接器按 COMDAT 去重丢弃的泛型实例副本**（实测 36 个 `list[T].*` 名字，
|
||
每个都有 1 份 VA 副本 + 15~21 份 0 地址副本），并非解析错误。
|
||
旧实现按"整段最小地址是否 >= ImageBase"**整表**判定：只要有一行 0，就整表都不
|
||
减基址 ⇒ 全部 VA 行留在 0x140000000+ 域、与运行期 RVA 域相差一个 ImageBase，
|
||
运行期下界查找对任意帧都落到同一条记录上（用户实测所有帧都显示同一行）。
|
||
现改为**逐条**归一化：地址落在映像 VA 区间 [ImageBase, ImageBase+SizeOfImage)
|
||
的记录保留（RVA = 地址 − ImageBase）；其余（未重定位的 0、越界值）一律**丢弃并
|
||
统计**（不静默留垃圾条目）。同一 RVA 只保留行号最大的一条。
|
||
|
||
写出前自检（任一不满足即 fail-fast 抛异常，遵循 --traceback 既有风格）：
|
||
记录数 / min~max RVA 与 SizeOfImage 的关系 / 所有 RVA ∈ [0, SizeOfImage) /
|
||
RVA 严格升序 / 至少若干条落在运行期可能出现的 RVA 区间
|
||
（粗判据 [.text 节 RVA, SizeOfImage)）。
|
||
|
||
任何一步失败都直接抛异常终止编译（--traceback 的硬要求，无兜底）。
|
||
"""
|
||
import struct
|
||
import bisect
|
||
|
||
def _Fail(msg: str) -> None:
|
||
full: str = msg + '（这是 --traceback 的要求）'
|
||
_vlog().error(full)
|
||
raise RuntimeError(full)
|
||
|
||
layout = _ReadPEImageLayout(exe_path)
|
||
if layout is None:
|
||
_Fail(f"--traceback 只支持 Windows PE 产物，但 {exe_path} 不是有效 PE")
|
||
image_base, size_of_image, sections = layout
|
||
if size_of_image <= 0:
|
||
_Fail(f"--traceback 读到的 PE.SizeOfImage 非正（{size_of_image}），无法界定 RVA 域")
|
||
img_lo: int = image_base
|
||
img_hi: int = image_base + size_of_image
|
||
# 运行期可能出现的 RVA 区间的**粗判据**下界：.text 节 RVA（缺失时退化为最小节 RVA）
|
||
text_rva: int = min((v[0] for v in sections.values()), default=0x1000)
|
||
if '.text' in sections:
|
||
text_rva = sections['.text'][0]
|
||
max_sec_end: int = max((v[0] + v[1] for v in sections.values()), default=size_of_image)
|
||
|
||
dwarfdump: str | None = shutil.which('llvm-dwarfdump') or shutil.which('llvm-dwarfdump.exe')
|
||
if not dwarfdump:
|
||
_Fail("--traceback 找不到 llvm-dwarfdump（请确认 PATH 内含 LLVM 工具）")
|
||
|
||
try:
|
||
proc = subprocess.run(
|
||
[dwarfdump, '--debug-line', '--debug-info', exe_path],
|
||
capture_output=True)
|
||
except Exception as e:
|
||
_Fail(f"--traceback 调用 llvm-dwarfdump 失败: {e}")
|
||
if proc.returncode != 0:
|
||
_Fail(f"--traceback 的 llvm-dwarfdump 返回 {proc.returncode}: "
|
||
+ proc.stderr.decode('utf-8', errors='replace').strip())
|
||
dump_text: str = proc.stdout.decode('utf-8', errors='replace')
|
||
|
||
rows: list[tuple[int, str, int]] = _ParseDwarfLineRows(dump_text)
|
||
if not rows:
|
||
_Fail(f"--traceback 未能从 {exe_path} 解析出任何 DWARF 行记录（.debug_line 为空？）")
|
||
|
||
# ---- 行记录：逐条归一化为"运行期 RVA"；不能归一的一律丢弃并统计 ----
|
||
dropped_rows: int = 0
|
||
dropped_row_samples: list[int] = []
|
||
best: dict[int, tuple[str, int]] = {}
|
||
for addr, path, line in rows:
|
||
if not (img_lo <= addr < img_hi):
|
||
dropped_rows += 1
|
||
if len(dropped_row_samples) < 5:
|
||
dropped_row_samples.append(addr)
|
||
continue
|
||
rva: int = addr - image_base
|
||
cur = best.get(rva)
|
||
if cur is None or line > cur[1]:
|
||
best[rva] = (path, line)
|
||
if dropped_rows:
|
||
_vlog().warning(
|
||
f" [--traceback] 丢弃 {dropped_rows} 条无法归一化的 DWARF 行记录"
|
||
f"（地址不在映像 VA 区间 [0x{img_lo:x}, 0x{img_hi:x}) 内；判定原因：多为被连接器"
|
||
f" COMDAT 去重丢弃的泛型副本，其地址恰为 0）示例地址: "
|
||
+ ", ".join(f"0x{a:x}" for a in dropped_row_samples))
|
||
if not best:
|
||
_Fail(f"--traceback 从 {exe_path} 得到的行记录全部无法归一到运行期 RVA 域")
|
||
|
||
# ---- 子程序（函数名）：同样逐条归一化；不能归一的丢弃 ----
|
||
funcs: list[tuple[int, int, str]] = _ParseDwarfSubprograms(dump_text)
|
||
func_ranges: list[tuple[int, int, str]] = []
|
||
dropped_funcs: int = 0
|
||
for low, high, name in funcs:
|
||
if not (img_lo <= low < img_hi):
|
||
dropped_funcs += 1
|
||
continue
|
||
lo_rva: int = low - image_base
|
||
hi_rva: int = high - image_base
|
||
if hi_rva <= lo_rva:
|
||
hi_rva = lo_rva + 1
|
||
if hi_rva > size_of_image:
|
||
hi_rva = size_of_image
|
||
func_ranges.append((lo_rva, hi_rva, name))
|
||
if dropped_funcs:
|
||
_vlog().warning(
|
||
f" [--traceback] 丢弃 {dropped_funcs} 个无法归一化的 DWARF 子程序"
|
||
f"（low_pc 不在映像 VA 区间 [0x{img_lo:x}, 0x{img_hi:x}) 内；同上为去重副本）")
|
||
func_ranges.sort(key=lambda f: f[0])
|
||
func_lows: list[int] = [f[0] for f in func_ranges]
|
||
|
||
def _FuncNameFor(rva: int) -> str | None:
|
||
i: int = bisect.bisect_right(func_lows, rva) - 1
|
||
if i >= 0:
|
||
low, high, name = func_ranges[i]
|
||
if low <= rva < high:
|
||
return name
|
||
return None
|
||
|
||
# 字符串池：偏移 0 固定为一个空串（'\0'）哨兵，使 func_off=0 可无歧义表示
|
||
# "无函数名"；真实串偏移恒 >= 1。
|
||
pool: bytearray = bytearray(b'\x00')
|
||
str_offsets: dict[str, int] = {'': 0}
|
||
|
||
def _Intern(s: str) -> int:
|
||
off: int | None = str_offsets.get(s)
|
||
if off is None:
|
||
off = len(pool)
|
||
pool.extend(s.encode('utf-8'))
|
||
pool.append(0)
|
||
str_offsets[s] = off
|
||
return off
|
||
|
||
records: list[tuple[int, int, int, int]] = []
|
||
for rva in sorted(best.keys()):
|
||
path, line = best[rva]
|
||
name = _FuncNameFor(rva)
|
||
records.append((rva, _Intern(path), line, _Intern(name) if name else 0))
|
||
|
||
# ---- 写端自检（fail-fast；与运行期 LoadMapFile 的校验口径一一对应）----
|
||
rvas: list[int] = [r[0] for r in records]
|
||
min_rva: int = rvas[0]
|
||
max_rva: int = rvas[-1]
|
||
for i in range(1, len(rvas)):
|
||
if rvas[i] <= rvas[i - 1]:
|
||
_Fail(f"--traceback 自检失败：记录 RVA 非严格升序"
|
||
f"（第 {i} 条 0x{rvas[i]:x} <= 前一条 0x{rvas[i - 1]:x}，会破坏运行期二分查找）")
|
||
outside: int = sum(1 for r in rvas if not (0 <= r < size_of_image))
|
||
if outside:
|
||
_Fail(f"--traceback 自检失败：有 {outside} 条记录的 RVA 落在 [0, SizeOfImage=0x{size_of_image:x}) 之外")
|
||
# 运行期可能出现的 RVA 区间（粗判据）：[.text 节 RVA, SizeOfImage)。一条都没有 ⇒ 口径又错了。
|
||
plausible: int = sum(1 for r in rvas if text_rva <= r < size_of_image)
|
||
if plausible <= 0:
|
||
_Fail(f"--traceback 自检失败：没有任何记录的 RVA 落在运行期可能出现的区间"
|
||
f" [0x{text_rva:x}(.text RVA), 0x{size_of_image:x}(SizeOfImage)) 内"
|
||
f"（RVA 口径疑似又错了）")
|
||
_vlog().info(
|
||
f" [--traceback] 自检: 记录={len(records)} RVA范围=[0x{min_rva:x}, 0x{max_rva:x}]"
|
||
f" PE.SizeOfImage=0x{size_of_image:x}({size_of_image})"
|
||
f" 各节最高结束 RVA=0x{max_sec_end:x}"
|
||
f" | 全部∈[0,SizeOfImage)=是 严格升序=是"
|
||
f" | 落在[.text 0x{text_rva:x},SizeOfImage)内={plausible} 条")
|
||
|
||
out: bytearray = bytearray()
|
||
out.extend(b'TPCS')
|
||
out.extend(struct.pack('<IIII', 1, len(records), len(str_offsets), len(pool)))
|
||
for rva, file_off, line, func_off in records:
|
||
out.extend(struct.pack('<QIII', rva, file_off, line, func_off))
|
||
out.extend(pool)
|
||
|
||
map_path: str = exe_path + '.stackmap'
|
||
try:
|
||
with open(map_path, 'wb') as f:
|
||
f.write(out)
|
||
except Exception as e:
|
||
_Fail(f"--traceback 写出旁文件失败 {map_path}: {e}")
|
||
_vlog().success(
|
||
f" [--traceback] 已写出地址→源行旁文件: {map_path}"
|
||
f"（{len(records)} 条记录，{len(out)} 字节）")
|
||
|
||
def _link_obj_files(self, active_sha1s: set[str]) -> None:
|
||
"""将 active_sha1s 对应的 .obj 文件和 includes 库文件链接为可执行文件"""
|
||
obj_files = []
|
||
merged_obj = os.path.join(self.output_dir, '_merged.obj')
|
||
has_merged: bool = _CachedIsFile(merged_obj)
|
||
if has_merged:
|
||
# _merged.obj 只合并了非 include 模块，include 的 .obj 仍需单独链接
|
||
obj_files.append(merged_obj)
|
||
for root, dirs, files in os.walk(self.output_dir):
|
||
for file in files:
|
||
if file.endswith('.obj'):
|
||
sha1 = file[:-4]
|
||
if sha1 not in active_sha1s:
|
||
continue
|
||
if has_merged and sha1 not in self._include_sha1s:
|
||
continue
|
||
if not has_merged and sha1 in self._include_sha1s:
|
||
continue
|
||
obj_files.append(os.path.join(root, file))
|
||
|
||
if not obj_files and not self.extra_link_files:
|
||
_vlog().info("[链接] 无文件可链接")
|
||
return
|
||
|
||
_vlog().info(f"\n[链接] 找到 {len(obj_files)} 个 .obj 文件")
|
||
|
||
all_link_files = obj_files + self.extra_link_files
|
||
seen = set()
|
||
all_link_files = [f for f in all_link_files if not (f in seen or seen.add(f))]
|
||
if self.extra_link_files:
|
||
_vlog().info(f"[链接] 额外库文件: {len(self.extra_link_files)} 个")
|
||
for lib in self.extra_link_files:
|
||
_vlog().info(f" - {os.path.basename(lib)}")
|
||
|
||
if not self.linker_cmd:
|
||
_vlog().info("[链接] 未配置链接器")
|
||
return
|
||
|
||
exe_path = os.path.join(self.output_dir, self.linker_output) if self.linker_output else os.path.join(self.output_dir, 'a.exe')
|
||
is_binary = self.linker_output and self.linker_output.endswith('.bin')
|
||
|
||
# 检查是否直接输出二进制（使用 --oformat binary / -oformat binary 标志）
|
||
is_direct_binary = is_binary and any('oformat' in f for f in self.linker_flags)
|
||
|
||
if is_binary and not is_direct_binary:
|
||
link_output = exe_path[:-4] + '_pe.exe'
|
||
else:
|
||
link_output = exe_path
|
||
|
||
# 自动复制或生成 linker.ld 到输出目录
|
||
linker_ld_src = os.path.join(os.path.dirname(self.output_dir), 'linker.ld')
|
||
linker_ld_dst = os.path.join(self.output_dir, 'linker.ld')
|
||
if _CachedIsFile(linker_ld_src) and not _CachedIsFile(linker_ld_dst):
|
||
shutil.copy2(linker_ld_src, linker_ld_dst)
|
||
_vlog().info(f" [链接] 使用链接脚本: linker.ld")
|
||
elif not _CachedIsFile(linker_ld_dst) and any('-T' in f or '-T' in f.replace(',', ' ') for f in self.linker_flags):
|
||
default_ld = """ENTRY(_start)
|
||
|
||
SECTIONS {
|
||
. = 0x100000;
|
||
|
||
.text : {
|
||
*(.text)
|
||
*(.text.*)
|
||
}
|
||
|
||
.rodata : {
|
||
*(.rodata)
|
||
*(.rodata.*)
|
||
}
|
||
|
||
.data : {
|
||
*(.data)
|
||
*(.data.*)
|
||
}
|
||
|
||
.bss : {
|
||
*(.bss)
|
||
*(.bss.*)
|
||
}
|
||
|
||
/DISCARD/ : {
|
||
*(.comment)
|
||
*(.note)
|
||
*(.eh_frame)
|
||
*(.eh_frame_hdr)
|
||
*(.reloc)
|
||
*(.rela)
|
||
*(.debug*)
|
||
}
|
||
}
|
||
"""
|
||
with open(linker_ld_dst, 'w', encoding='utf-8', newline='\n') as f:
|
||
f.write(default_ld)
|
||
_InvalidateTextCache(linker_ld_dst)
|
||
_vlog().info(f" [链接] 自动创建默认链接脚本: linker.ld")
|
||
|
||
try:
|
||
# 将 linker_flags 拆分为非库标志和库标志（-l/-L），库标志放在目标文件之后
|
||
# 这样 Unix 风格链接器才能正确解析库中的符号
|
||
# -Wl, 选项是链接器全局选项，放在目标文件之前
|
||
non_lib_flags = []
|
||
lib_flags = []
|
||
i = 0
|
||
while i < len(self.linker_flags):
|
||
flag = self.linker_flags[i]
|
||
if flag in ('-l', '-L') and i + 1 < len(self.linker_flags):
|
||
# -l xxx / -L xxx 分开的形式
|
||
lib_flags.extend([flag, self.linker_flags[i + 1]])
|
||
i += 2
|
||
elif flag.startswith('-l') or flag.startswith('-L'):
|
||
# -lxxx / -Lxxx 合并的形式
|
||
lib_flags.append(flag)
|
||
i += 1
|
||
else:
|
||
# 其他所有标志（包括 -Wl, 选项）放在目标文件之前
|
||
non_lib_flags.append(flag)
|
||
i += 1
|
||
# ASan 模式：链接 ASan 运行库（配合各 .ll 的 opt -passes=asan 插桩）
|
||
# 不能传 -fsanitize=address：那会让 clang++ 驱动去找
|
||
# <resource-dir>/lib/x86_64-w64-windows-gnu/libclang_rt.asan_dynamic.dll.a，
|
||
# 而该目录是空的（上游 LLVM 不为这个 triple 构建 ASan runtime），
|
||
# 直接报 "could not open ... No such file or directory"。
|
||
# 改为显式链接 clang64 的那两份（与 VPC / README §8.1 同口径）。
|
||
# 它们是库，必须排在目标文件之后，所以追加到 lib_flags 而不是 non_lib_flags。
|
||
if self.asan_mode:
|
||
if self.asan_rt_dir is None:
|
||
_vlog().error(
|
||
"ASan 模式在 PATH 里找不到 clang64 的 ASan 运行库目录 "
|
||
"(<MSYS2>\\clang64\\lib\\clang\\<ver>\\lib\\windows\\"
|
||
"libclang_rt.asan_dynamic-x86_64.dll.a)\n"
|
||
" 请确认 msys2 的 mingw-w64-clang-x86_64-compiler-rt 已安装，"
|
||
"且 <MSYS2>\\clang64（或其 bin）已在 PATH 中。")
|
||
sys.exit(1)
|
||
lib_flags.append('-L' + self.asan_rt_dir)
|
||
lib_flags.append('-l:libclang_rt.asan_dynamic-x86_64.dll.a')
|
||
lib_flags.append('-l:libclang_rt.asan_dynamic_runtime_thunk-x86_64.a')
|
||
# Ctrl+C 栈转储（--traceback）：includes/traceback.vp 的主回溯路径用
|
||
# StackWalk64 / SymInitialize / SymFunctionTableAccess64 / SymGetModuleBase64
|
||
# （均在 dbghelp），故开启该开关时必须显式链接 -ldbghelp，否则链接期报未定义符号。
|
||
# 与 ASan 同理：库要排在目标文件之后 ⇒ 追加到 lib_flags。
|
||
# （若确实无法链 dbghelp：把 includes/traceback.vp 的 TRACEBACK_USE_DBGHELP 置 0，
|
||
# 退化为纯 RBP 链回溯，此时不该再带本 flag。）
|
||
if self.traceback_enabled:
|
||
lib_flags.append('-ldbghelp')
|
||
cmd = [self.linker_cmd] + non_lib_flags + ['-o', link_output] + all_link_files + lib_flags
|
||
_vlog().info(f" 执行: {' '.join(cmd)}")
|
||
result = subprocess.run(
|
||
cmd,
|
||
capture_output=True,
|
||
text=True,
|
||
cwd=self.output_dir or '.'
|
||
)
|
||
if result.returncode == 0:
|
||
if is_binary and not is_direct_binary:
|
||
objcopy = shutil.which('llvm-objcopy') or shutil.which('objcopy')
|
||
if objcopy:
|
||
conv = subprocess.run(
|
||
[objcopy, '-O', 'binary', link_output, exe_path],
|
||
capture_output=True, text=True
|
||
)
|
||
if conv.returncode == 0:
|
||
os.remove(link_output)
|
||
_vlog().success(f" 生成裸机二进制: {exe_path}")
|
||
file_size = os.path.getsize(exe_path)
|
||
_vlog().info(f" [大小] {file_size} 字节")
|
||
else:
|
||
_vlog().error(f" 二进制转换失败: {conv.stderr.strip()}")
|
||
sys.exit(1)
|
||
else:
|
||
_vlog().error(" 找不到 llvm-objcopy，无法生成裸机二进制")
|
||
sys.exit(1)
|
||
else:
|
||
_vlog().success(f" 生成: {exe_path}")
|
||
file_size = os.path.getsize(exe_path)
|
||
_vlog().info(f" [大小] {file_size} 字节")
|
||
# --dwarf 构建（含 --asan 自动开启）必须保留 DWARF：
|
||
# 该模式存在的唯一意义是崩溃时反查 ".vp:行"，
|
||
# 而 _strip_debug_sections 会 llvm-strip --strip-debug 掉全部
|
||
# .debug_* 段（实测 exe 体积 13378048 → 11895296，
|
||
# llvm-symbolizer 对含 main 在内的所有地址返回 ??:0:0）。
|
||
# 非 dwarf 构建保持原有剥离行为（产物零变化）。
|
||
if not is_binary and not self.dwarf_enabled:
|
||
self._strip_debug_sections(exe_path)
|
||
else:
|
||
_vlog().error(f" 链接失败: {result.stderr.strip()}")
|
||
_vlog().error(f"\n链接失败，立即终止编译。")
|
||
sys.exit(1)
|
||
except FileNotFoundError:
|
||
_vlog().error(f" 找不到链接器: {self.linker_cmd}")
|
||
_vlog().error(f"\n找不到链接器，立即终止编译。")
|
||
sys.exit(1)
|
||
except Exception as e:
|
||
_vlog().error(f" {e}")
|
||
_vlog().error(f"\n链接异常，立即终止编译。")
|
||
sys.exit(1)
|
||
|
||
# --traceback：链接成功后导出"地址→源行"旁文件（<exe>.stackmap）。
|
||
# 刻意放在上面的链接 try/except 之外，使导出阶段的 fail-fast RuntimeError
|
||
# 直接向上抛出终止编译，而不被"链接异常兜底"吞掉。
|
||
if self.traceback_enabled:
|
||
self._WriteStackMap(exe_path)
|