perf(Phase2Translator): 文本读全走 LRU 缓存 + 写侧显式失效，削重复文件 I/O
- 新增 _ReadTextCached：键 =(路径, st_mtime_ns, st_size)，命中即免 open/read/close； 按总字节预算（64MB）LRU 淘汰，单文件 >32MB 不入缓存；newline 非 None 时退化为直读。 - 21 处文本读点全部改走缓存（.vpi/.pyi 解析、.ll/.stub.ll/.text.ll、deps.json、 doc.json、include 源、pyi 缓存预热等），json.load(f) 统一改 json.loads(_ReadTextCached(...))。 - 新增 _InvalidateTextCache：14 处本进程文本写点（原地改写/新落盘）后显式失效。 Windows 文件时间为系统时钟派生，粒度约 15.6ms，"先读 → 同 tick 内原地改写 → 再读" 且改写前后字节数相同时 (mtime, size) 键完全一致（实测两例：_compile_one 前的 type 定义重排——行集合不变仅次序变；_recombine_ll 的旧 sha1 前缀替换——16 位 hex 等长替换）， 只靠 stat 判新旧会读到过期文本，故写侧必须显式失效。 基线 PROF/20260930_224857：_io.open 12,740 次/self 28.8s、read 11,902 次/30.4s、 TextIOWrapper.__exit__ 16.6s（128 个模块却上万次读，同一份产物被多阶段反复打开）。
This commit is contained in:
TermiNexus committed
1 parent
3658ab0637
commit
1748940d30
1 file changed
+146
-94
@@ -11,13 +11,14 @@ import json
|
||||
import pickle
|
||||
import time
|
||||
import cProfile
|
||||
from collections import OrderedDict
|
||||
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed, wait, FIRST_COMPLETED
|
||||
from lib.core.VLogger import get_logger as _vlog
|
||||
from lib.constants.config import mode as _config_mode
|
||||
from lib.core.Handles.HandlesBase import CTypeInfo
|
||||
from lib.core.SymbolData import ModuleSymbols
|
||||
from lib.includes import t
|
||||
from lib.core.SymbolUtils import AnnotationContainsName, ExpandSoftKeywords
|
||||
from lib.core.SymbolUtils import AnnotationContainsName, ExpandSoftKeywords
|
||||
from lib.core.SymbolUtils import ClassHasFuncLevelCVTable, IsVtableSlotMethod
|
||||
from lib.Projectrans.Utils import compute_sha1, _check_annotation_for_export, find_reachable_files_from_entries, topological_sort_files, parse_python_file
|
||||
from lib.Projectrans.Phase1Generator import _TryGlobalCache, _SaveToGlobalCache, _GLOBAL_CACHE_DIR
|
||||
@@ -47,6 +48,65 @@ def _CachedIsFile(path: str) -> bool:
|
||||
return r
|
||||
|
||||
|
||||
# ---------------------------------------------------------------
|
||||
# 文本读缓存（性能）：同一个文件在一次构建里被**多个阶段反复读**
|
||||
# （prof 基线 PROF/20260930_224857：_io.open 12,740 次 / self 28.8s、
|
||||
# read 11,902 次 / self 30.4s、TextIOWrapper.__exit__ 16.6s —— 128 个模块
|
||||
# 却产生上万次读，主要是同一份 .stub.ll / .ll / deps.json 被
|
||||
# _compile_one / _recombine_ll / 依赖解析等重复打开）。
|
||||
# 键 = (路径, mtime_ns, 大小)：文件被写过或替换后**自动失配**，因此无需手工失效
|
||||
# （比"路径→文本"的裸缓存安全）。只缓存 newline=None 的默认换行翻译读，
|
||||
# 与直接 open(...).read() 语义逐字相同。
|
||||
# 内存：按总字节预算 + LRU 淘汰；单文件超过 _TEXT_CACHE_FILE_MAX 不入缓存
|
||||
# （合并后的 .ll 可达数十 MB，避免把内存吃光）。
|
||||
# ---------------------------------------------------------------
|
||||
_TEXT_CACHE_MAX_BYTES: int = 64 * 1024 * 1024
|
||||
_TEXT_CACHE_FILE_MAX: int = 32 * 1024 * 1024
|
||||
_TextReadCache: OrderedDict[tuple[str, int, int], str] = OrderedDict()
|
||||
_TextReadCacheBytes: int = 0
|
||||
|
||||
|
||||
def _ReadTextCached(path: str, newline: str | None = None) -> str:
|
||||
"""读文本文件（带 LRU 缓存）；newline 非 None 或 stat 失败时退化为直接读。"""
|
||||
global _TextReadCacheBytes
|
||||
if newline is None:
|
||||
try:
|
||||
_st = os.stat(path)
|
||||
except OSError:
|
||||
_st = None
|
||||
if _st is not None and _st.st_size <= _TEXT_CACHE_FILE_MAX:
|
||||
_key = (path, _st.st_mtime_ns, _st.st_size)
|
||||
_hit = _TextReadCache.get(_key)
|
||||
if _hit is not None:
|
||||
_TextReadCache.move_to_end(_key)
|
||||
return _hit
|
||||
with open(path, 'r', encoding='utf-8') as _f:
|
||||
_text = _f.read()
|
||||
_TextReadCache[_key] = _text
|
||||
_TextReadCacheBytes += len(_text)
|
||||
while _TextReadCacheBytes > _TEXT_CACHE_MAX_BYTES and len(_TextReadCache) > 1:
|
||||
_, _old = _TextReadCache.popitem(last=False)
|
||||
_TextReadCacheBytes -= len(_old)
|
||||
return _text
|
||||
with open(path, 'r', encoding='utf-8', newline=newline) as _f:
|
||||
return _f.read()
|
||||
|
||||
|
||||
def _InvalidateTextCache(path: str) -> None:
|
||||
"""丢弃 path 的文本读缓存条目：本进程**改写**该文件后必须调用。
|
||||
|
||||
只靠 (mtime_ns, size) 判新旧在 Windows 上不安全：文件时间是系统时钟派生，
|
||||
粒度约 15.6ms，「先读 → 同一 tick 内原地改写 → 再读」且改写前后字节数相同
|
||||
（实例：_compile_one 前的 type 定义重排，行集合不变仅次序变，长度逐字节相同）
|
||||
时键完全一致，会命中过期文本。故写侧必须显式失效，不做时间戳兜底。
|
||||
"""
|
||||
global _TextReadCacheBytes
|
||||
if not _TextReadCache:
|
||||
return
|
||||
for _k in [k for k in _TextReadCache if k[0] == path]:
|
||||
_TextReadCacheBytes -= len(_TextReadCache.pop(_k))
|
||||
|
||||
|
||||
def _FindAsanRuntimeDir() -> str | None:
|
||||
"""定位 clang64 的 ASan 运行库目录（按 LLVM 大版本取最新的那个）
|
||||
|
||||
@@ -153,28 +213,28 @@ _DWARF_TAG_ANY_RE = re.compile(r'DW_TAG_\w+')
|
||||
_DWARF_TAG_SUBPROGRAM_RE = re.compile(r'DW_TAG_subprogram\b')
|
||||
_DWARF_LOWPC_RE = re.compile(r'DW_AT_low_pc\s*\(\s*(0x[0-9A-Fa-f]+)\s*\)')
|
||||
_DWARF_HIGHPC_RE = re.compile(r'DW_AT_high_pc\s*\(\s*(0x[0-9A-Fa-f]+)\s*\)')
|
||||
_DWARF_ATNAME_RE = re.compile(r'DW_AT_name\s*\("((?:[^"\\]|\\.)*)"\)')
|
||||
|
||||
# ---------------------------------------------------------------
|
||||
# `_split_ll` 的逐行正则**预编译**（性能：该函数对合并后的 .ll 逐行跑两遍，
|
||||
# prof 实测 43.9万行级输入下 re 模块级函数的缓存查找（re._compile）达 1970 万次
|
||||
# ≈ 7s，另有 3100 万次 re.Pattern.match；每次 `re.match(字面量, ...)` 都要做
|
||||
# "模式串哈希 → 缓存查找"。预编译成模块级常量后省掉这层查找，且匹配路径更短。
|
||||
# 语义与逐字写法完全一致（模式串一字未改）。
|
||||
# ---------------------------------------------------------------
|
||||
_RE_DEFINE_SHA1: re.Pattern[str] = re.compile(r'define\s+[^@]*@"?([a-f0-9]+)\.')
|
||||
_RE_DEFINE_NAME: re.Pattern[str] = re.compile(r'define\s+[^@]*@"?([^"\s(]+)"?')
|
||||
_RE_DECLARE_NAME: re.Pattern[str] = re.compile(r'declare\s+[^@]*@"?([^"\s(]+)"?')
|
||||
_RE_LLVM_INTRINSIC: re.Pattern[str] = re.compile(r'@("?)llvm\.')
|
||||
_RE_AT_NAME: re.Pattern[str] = re.compile(r'(@\S+)')
|
||||
_RE_SHA1_16_DOT: re.Pattern[str] = re.compile(r'^[a-f0-9]{16}\.')
|
||||
_RE_DEFINE_PREFIX: re.Pattern[str] = re.compile(r'^define\s+')
|
||||
_RE_DEFINE_LINKAGE: re.Pattern[str] = re.compile(
|
||||
r'\b(linkonce_odr|weak_odr|linkonce|weak|common|appending|internal|private)\s+')
|
||||
_RE_ALWAYSINLINE: re.Pattern[str] = re.compile(r'\balwaysinline\s+')
|
||||
# `_compile_one` 的逐行符号扫描/全局去重里用到的两条（同一预编译理由）
|
||||
_RE_AT_EXTERNAL: re.Pattern[str] = re.compile(r'@\S+\s*=\s*external\s+')
|
||||
_RE_DECLARE_FULL: re.Pattern[str] = re.compile(
|
||||
_DWARF_ATNAME_RE = re.compile(r'DW_AT_name\s*\("((?:[^"\\]|\\.)*)"\)')
|
||||
|
||||
# ---------------------------------------------------------------
|
||||
# `_split_ll` 的逐行正则**预编译**（性能：该函数对合并后的 .ll 逐行跑两遍，
|
||||
# prof 实测 43.9万行级输入下 re 模块级函数的缓存查找（re._compile）达 1970 万次
|
||||
# ≈ 7s，另有 3100 万次 re.Pattern.match；每次 `re.match(字面量, ...)` 都要做
|
||||
# "模式串哈希 → 缓存查找"。预编译成模块级常量后省掉这层查找，且匹配路径更短。
|
||||
# 语义与逐字写法完全一致（模式串一字未改）。
|
||||
# ---------------------------------------------------------------
|
||||
_RE_DEFINE_SHA1: re.Pattern[str] = re.compile(r'define\s+[^@]*@"?([a-f0-9]+)\.')
|
||||
_RE_DEFINE_NAME: re.Pattern[str] = re.compile(r'define\s+[^@]*@"?([^"\s(]+)"?')
|
||||
_RE_DECLARE_NAME: re.Pattern[str] = re.compile(r'declare\s+[^@]*@"?([^"\s(]+)"?')
|
||||
_RE_LLVM_INTRINSIC: re.Pattern[str] = re.compile(r'@("?)llvm\.')
|
||||
_RE_AT_NAME: re.Pattern[str] = re.compile(r'(@\S+)')
|
||||
_RE_SHA1_16_DOT: re.Pattern[str] = re.compile(r'^[a-f0-9]{16}\.')
|
||||
_RE_DEFINE_PREFIX: re.Pattern[str] = re.compile(r'^define\s+')
|
||||
_RE_DEFINE_LINKAGE: re.Pattern[str] = re.compile(
|
||||
r'\b(linkonce_odr|weak_odr|linkonce|weak|common|appending|internal|private)\s+')
|
||||
_RE_ALWAYSINLINE: re.Pattern[str] = re.compile(r'\balwaysinline\s+')
|
||||
# `_compile_one` 的逐行符号扫描/全局去重里用到的两条（同一预编译理由）
|
||||
_RE_AT_EXTERNAL: re.Pattern[str] = re.compile(r'@\S+\s*=\s*external\s+')
|
||||
_RE_DECLARE_FULL: re.Pattern[str] = re.compile(
|
||||
r'^declare\s+(?:dso_local\s+)?[^@]*@\"?([^"\s\(]+)\"?\s*\(')
|
||||
|
||||
|
||||
@@ -785,8 +845,7 @@ def _BuildPyiCacheFile(temp_dir: str) -> str | None:
|
||||
sig_path: str = os.path.join(temp_dir, _PYI_CACHE_SIG_NAME)
|
||||
try:
|
||||
if _CachedIsFile(pkl_path) and _CachedIsFile(sig_path):
|
||||
with open(sig_path, 'r', encoding='utf-8') as f:
|
||||
stored = json.load(f)
|
||||
stored = json.loads(_ReadTextCached(sig_path))
|
||||
# json 往返后 tuple→list，需规范化再比较
|
||||
if stored == [[n, s, m] for n, s, m in sig]:
|
||||
return pkl_path
|
||||
@@ -814,8 +873,8 @@ def _BuildPyiCacheFile(temp_dir: str) -> str | None:
|
||||
cache[p] = old_entry[2]
|
||||
continue
|
||||
try:
|
||||
with open(p, 'r', encoding='utf-8') as f:
|
||||
cache[p] = ast.parse(f.read(), p)
|
||||
# 走 LRU 文本缓存：同一 .vpi 在增量复用/多 worker 预热路径会被反复读
|
||||
cache[p] = ast.parse(_ReadTextCached(p), p)
|
||||
except Exception:
|
||||
cache[p] = None
|
||||
try:
|
||||
@@ -854,8 +913,7 @@ def _PyiCacheWorkerInitializer(temp_dir: str) -> None:
|
||||
new_cache: dict[str, object] = {}
|
||||
for p in _ScanPyiFiles(temp_dir):
|
||||
try:
|
||||
with open(p, 'r', encoding='utf-8') as f:
|
||||
new_cache[p] = ast.parse(f.read(), p)
|
||||
new_cache[p] = ast.parse(_ReadTextCached(p), p)
|
||||
except Exception:
|
||||
new_cache[p] = None
|
||||
if new_cache:
|
||||
@@ -887,18 +945,18 @@ def _count_braces_outside_strings(text: str) -> int:
|
||||
return count
|
||||
|
||||
|
||||
# LLVM IR "无引号标识符"的精确规则（LangRef "Identifiers"；实现即
|
||||
# LLLexer::ReadVarName）：首字符 ∈ [-a-zA-Z$._]，后续字符 ∈ [-a-zA-Z$._0-9]。
|
||||
# 首字符**不允许是数字**：`$884f3d94ea4d0123.md5_F` 会让 LLLexer::LexDollar
|
||||
# 走到 ReadVarName() 返回 false ⇒ 产出 error token ⇒ 顶层解析报
|
||||
# "expected top-level entity"。故 comdat 名必须以同一规则判定是否加引号
|
||||
# （与 LLVM 自身 AsmWriter 的输出形态一致：`$"884f3d94ea4d0123.md5_F"`）。
|
||||
# 另：引号名 @"..." 不支持转义，名字含 `"`/`\` 时两种形态都非法 ⇒ 见
|
||||
# _AddWeakComdat 内的 fail-fast（不静默替换、不跳过）。
|
||||
_LLVM_UNQUOTED_NAME_RE: re.Pattern[str] = re.compile(
|
||||
r'[-a-zA-Z$._][-a-zA-Z$._0-9]*\Z')
|
||||
|
||||
|
||||
# LLVM IR "无引号标识符"的精确规则（LangRef "Identifiers"；实现即
|
||||
# LLLexer::ReadVarName）：首字符 ∈ [-a-zA-Z$._]，后续字符 ∈ [-a-zA-Z$._0-9]。
|
||||
# 首字符**不允许是数字**：`$884f3d94ea4d0123.md5_F` 会让 LLLexer::LexDollar
|
||||
# 走到 ReadVarName() 返回 false ⇒ 产出 error token ⇒ 顶层解析报
|
||||
# "expected top-level entity"。故 comdat 名必须以同一规则判定是否加引号
|
||||
# （与 LLVM 自身 AsmWriter 的输出形态一致：`$"884f3d94ea4d0123.md5_F"`）。
|
||||
# 另：引号名 @"..." 不支持转义，名字含 `"`/`\` 时两种形态都非法 ⇒ 见
|
||||
# _AddWeakComdat 内的 fail-fast（不静默替换、不跳过）。
|
||||
_LLVM_UNQUOTED_NAME_RE: re.Pattern[str] = re.compile(
|
||||
r'[-a-zA-Z$._][-a-zA-Z$._0-9]*\Z')
|
||||
|
||||
|
||||
def _AddWeakComdat(ll_content: str) -> str:
|
||||
"""为 weak/linkonce 系列 函数/全局 定义补 COMDAT（COFF/PE 必需）。
|
||||
|
||||
@@ -1023,17 +1081,17 @@ class Phase2Translator:
|
||||
if os.path.isdir(_clang64_bin):
|
||||
os.environ['PATH'] = _clang64_bin + os.pathsep + os.environ.get('PATH', '')
|
||||
self.asan_rt_dir = _FindAsanRuntimeDir()
|
||||
self.slice_level = 3
|
||||
# 函数级 DWARF 调试元数据（--dwarf；--asan 时自动开启）：False=未启用
|
||||
# （零影响，不构造任何 DI 元数据）；True=App 源模块的 .ll 携带
|
||||
# !llvm.dbg.cu + 每函数 DISubprogram + 入口 DILocation，供 ASan /
|
||||
# llvm-symbolizer 反查 ".vp:函数起始行"。
|
||||
self.dwarf_enabled: bool = dwarf_enabled
|
||||
# Ctrl+C 调用栈（--traceback）：False=未启用（零影响，不调用任何外部
|
||||
# 工具、不写任何旁文件）；True=链接成功后额外用 llvm-dwarfdump 把最终
|
||||
# exe 的 DWARF 行表导出为 "<exe>.stackmap"（地址→源行旁文件，供 exe 内
|
||||
# 运行时在 Ctrl+C/Ctrl+Break 时逐帧翻译返回地址）。要求 --dwarf 已开启
|
||||
# （Config 侧联动，否则行表为空），且目标为 Windows PE。
|
||||
self.slice_level = 3
|
||||
# 函数级 DWARF 调试元数据（--dwarf；--asan 时自动开启）：False=未启用
|
||||
# （零影响，不构造任何 DI 元数据）；True=App 源模块的 .ll 携带
|
||||
# !llvm.dbg.cu + 每函数 DISubprogram + 入口 DILocation，供 ASan /
|
||||
# llvm-symbolizer 反查 ".vp:函数起始行"。
|
||||
self.dwarf_enabled: bool = dwarf_enabled
|
||||
# Ctrl+C 调用栈（--traceback）：False=未启用（零影响，不调用任何外部
|
||||
# 工具、不写任何旁文件）；True=链接成功后额外用 llvm-dwarfdump 把最终
|
||||
# exe 的 DWARF 行表导出为 "<exe>.stackmap"（地址→源行旁文件，供 exe 内
|
||||
# 运行时在 Ctrl+C/Ctrl+Break 时逐帧翻译返回地址）。要求 --dwarf 已开启
|
||||
# （Config 侧联动，否则行表为空），且目标为 Windows PE。
|
||||
self.traceback_enabled: bool = traceback_enabled
|
||||
# 调用点热点统计（--hotspot-sites）：None=未启用（零影响）；str=调用点计数
|
||||
# 落盘文件路径（每函数计时落同目录 hotspot_time.txt）。各模块运行时自带
|
||||
@@ -1543,8 +1601,7 @@ class Phase2Translator:
|
||||
if exists:
|
||||
py_sha1 = Phase2Translator._include_sha1_cache.get(py_path)
|
||||
if py_sha1 is None:
|
||||
with open(py_path, 'r', encoding='utf-8') as f:
|
||||
py_sha1 = compute_sha1(f.read())
|
||||
py_sha1 = compute_sha1(_ReadTextCached(py_path))
|
||||
Phase2Translator._include_sha1_cache[py_path] = py_sha1
|
||||
actual_sha1 = py_sha1
|
||||
break
|
||||
@@ -1795,14 +1852,14 @@ class Phase2Translator:
|
||||
deps[mod_name] = dep_sha1
|
||||
with open(deps_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(deps, f)
|
||||
_InvalidateTextCache(deps_path)
|
||||
|
||||
def _LoadDeps(self, sha1: str) -> dict[str, str] | None:
|
||||
"""加载依赖指纹"""
|
||||
deps_path = os.path.join(self.output_dir, f"{sha1}.deps.json")
|
||||
if _CachedIsFile(deps_path):
|
||||
try:
|
||||
with open(deps_path, 'r', encoding='utf-8') as f:
|
||||
return json.load(f)
|
||||
return json.loads(_ReadTextCached(deps_path))
|
||||
except Exception as _e:
|
||||
if _config_mode == "strict":
|
||||
raise
|
||||
@@ -1835,10 +1892,8 @@ class Phase2Translator:
|
||||
if not _CachedIsFile(stub_path) or not _CachedIsFile(text_path):
|
||||
return False
|
||||
|
||||
with open(stub_path, 'r', encoding='utf-8') as f:
|
||||
stub_content = f.read()
|
||||
with open(text_path, 'r', encoding='utf-8') as f:
|
||||
text_content = f.read()
|
||||
stub_content = _ReadTextCached(stub_path)
|
||||
text_content = _ReadTextCached(text_path)
|
||||
|
||||
sha1_replacements = {}
|
||||
for mod_name, old_sha1 in saved_deps.items():
|
||||
@@ -1862,6 +1917,9 @@ class Phase2Translator:
|
||||
f.write(stub_content)
|
||||
with open(text_path, 'w', encoding='utf-8', newline='\n') as f:
|
||||
f.write(text_content)
|
||||
_InvalidateTextCache(ll_path)
|
||||
_InvalidateTextCache(stub_path)
|
||||
_InvalidateTextCache(text_path)
|
||||
|
||||
self._save_deps(sha1, current_ModuleSha1Map)
|
||||
|
||||
@@ -1961,8 +2019,7 @@ class Phase2Translator:
|
||||
# 顺序模式与 worker 模式的每文件粒度皆覆盖，见 MarkStubSetDirty）。
|
||||
from lib.core.Handles.HandlesImports import ImportHandle as _IH
|
||||
_IH.MarkStubSetDirty()
|
||||
with open(src_path, 'r', encoding='utf-8') as f:
|
||||
code = f.read()
|
||||
code = _ReadTextCached(src_path)
|
||||
|
||||
# SHA1 以主流程 out_path 文件名为准（= 主流程 compute_sha1(parse_python_file 内容)），
|
||||
# 必须在自动导入注入之前确定。不可在此重读文件重算：主流程收集（need_translate）
|
||||
@@ -2008,7 +2065,7 @@ class Phase2Translator:
|
||||
trans.translator.SliceCount = 0
|
||||
trans.translator.SliceInfos = []
|
||||
trans.translator.LlvmGen = None
|
||||
trans.translator._module_sha1 = sha1
|
||||
trans.translator._module_sha1 = sha1
|
||||
trans.translator._line_offset = _line_offset
|
||||
trans.translator._ModuleSha1Map = ModuleSha1Map
|
||||
trans.translator._struct_sha1_map = self.struct_sha1_map
|
||||
@@ -2054,8 +2111,7 @@ class Phase2Translator:
|
||||
if sha1_key == sha1:
|
||||
continue
|
||||
try:
|
||||
with open(sig_path, 'r', encoding='utf-8') as f:
|
||||
sig_content = f.read()
|
||||
sig_content = _ReadTextCached(sig_path)
|
||||
sig_tree = ast.parse(sig_content, sig_path)
|
||||
for node in ast.iter_child_nodes(sig_tree):
|
||||
if isinstance(node, ast.FunctionDef):
|
||||
@@ -2117,8 +2173,7 @@ class Phase2Translator:
|
||||
continue
|
||||
if os.path.exists(stub_path):
|
||||
try:
|
||||
with open(stub_path, 'r', encoding='utf-8') as f:
|
||||
stub_content = f.read()
|
||||
stub_content = _ReadTextCached(stub_path)
|
||||
for line in stub_content.splitlines():
|
||||
line = line.strip()
|
||||
if line.startswith('@') and '= global' in line and 'i32' in line and not line.startswith('@llvm'):
|
||||
@@ -2229,16 +2284,19 @@ class Phase2Translator:
|
||||
_vlog().warning(f"后处理添加属性失败: 未找到函数 {func_name} 的定义")
|
||||
with open(out_path, 'w', encoding='utf-8', newline='\n') as f:
|
||||
f.write(result)
|
||||
_InvalidateTextCache(out_path)
|
||||
|
||||
stub_content, text_content = self._split_ll(result)
|
||||
base = os.path.splitext(out_path)[0]
|
||||
with open(base + '.stub.ll', 'w', encoding='utf-8', newline='\n') as f:
|
||||
f.write(stub_content)
|
||||
_InvalidateTextCache(base + '.stub.ll')
|
||||
# 新 stub 已落盘 → 置脏，使本文件后续（及下个文件）的索引 miss 能重扫到它
|
||||
from lib.core.Handles.HandlesImports import ImportHandle as _IH2
|
||||
_IH2.MarkStubSetDirty()
|
||||
with open(base + '.text.ll', 'w', encoding='utf-8', newline='\n') as f:
|
||||
f.write(text_content)
|
||||
_InvalidateTextCache(base + '.text.ll')
|
||||
self._save_deps(sha1, ModuleSha1Map)
|
||||
|
||||
# 进度日志由调用方统一打印（主进程按文件粒度 [x/total]），此处不再重复输出。
|
||||
@@ -2248,8 +2306,7 @@ class Phase2Translator:
|
||||
out_dir = os.path.dirname(out_path)
|
||||
if out_dir:
|
||||
os.makedirs(out_dir, exist_ok=True)
|
||||
with open(stub_path, 'r', encoding='utf-8') as f:
|
||||
stub_content = f.read()
|
||||
stub_content = _ReadTextCached(stub_path)
|
||||
with open(out_path, 'w', encoding='utf-8', newline='\n') as f:
|
||||
f.write(stub_content)
|
||||
# 进度日志由调用方统一打印。
|
||||
@@ -2302,6 +2359,7 @@ class Phase2Translator:
|
||||
if fixed != text:
|
||||
with open(ll_path, 'w', encoding='utf-8', newline='') as f:
|
||||
f.write(fixed)
|
||||
_InvalidateTextCache(ll_path)
|
||||
return ll_path
|
||||
|
||||
def _llc_compile(self, ll_path: str, obj_path: str,
|
||||
@@ -2394,8 +2452,7 @@ class Phase2Translator:
|
||||
has_inline = False
|
||||
for ll_path in all_ll_files:
|
||||
try:
|
||||
with open(ll_path, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
content = _ReadTextCached(ll_path)
|
||||
if 'alwaysinline' in content:
|
||||
has_inline = True
|
||||
break
|
||||
@@ -2409,8 +2466,7 @@ class Phase2Translator:
|
||||
_vlog().info(" [内联] 检测到 alwaysinline 函数，执行跨模块内联优化...")
|
||||
for ll_path in all_ll_files:
|
||||
try:
|
||||
with open(ll_path, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
content = _ReadTextCached(ll_path)
|
||||
type_defs = []
|
||||
type_def_indices = set()
|
||||
for i, line in enumerate(content.splitlines()):
|
||||
@@ -2432,6 +2488,7 @@ class Phase2Translator:
|
||||
fixed = ''.join(lines2[:header_end]) + '\n'.join(type_defs) + '\n' + ''.join(lines2[header_end:])
|
||||
with open(ll_path, 'w', encoding='utf-8') as f:
|
||||
f.write(fixed)
|
||||
_InvalidateTextCache(ll_path)
|
||||
except Exception as e:
|
||||
_vlog().warning(f"Phase2Translator: 忽略异常 {e}", exc_info=e)
|
||||
merged_ll = os.path.join(self.output_dir, '_merged.ll')
|
||||
@@ -2450,8 +2507,7 @@ class Phase2Translator:
|
||||
has_inline = False
|
||||
else:
|
||||
_vlog().info(" [内联] 跨模块内联优化完成")
|
||||
with open(optimized_ll, 'r', encoding='utf-8') as f:
|
||||
opt_content = f.read()
|
||||
opt_content = _ReadTextCached(optimized_ll)
|
||||
opt_content = re.sub(r'^(define|declare)\s+(private|internal|available_externally|linkonce|weak|common|appending|extern_weak|linkonce_odr|weak_odr|external)\s+', r'\1 \2 dso_local ', opt_content, flags=re.MULTILINE)
|
||||
opt_content = re.sub(r'^(define|declare)\s+(?!private|internal|available_externally|linkonce|weak|common|appending|extern_weak|linkonce_odr|weak_odr|external|dso_local|@)', r'\1 dso_local ', opt_content, flags=re.MULTILINE)
|
||||
# LLVM 22+: external 不允许在带初值的全局定义上显式写出，改为 linkonce 保留多模块合并语义
|
||||
@@ -2490,8 +2546,7 @@ class Phase2Translator:
|
||||
if _CachedIsFile(obj_path):
|
||||
return (rel, 'cached', '')
|
||||
try:
|
||||
with open(ll_path, 'r', encoding='utf-8') as f:
|
||||
ll_content = f.read()
|
||||
ll_content = _ReadTextCached(ll_path)
|
||||
sha1 = os.path.basename(ll_path)[:-3]
|
||||
deps = self._LoadDeps(sha1)
|
||||
stub_prefix = ''
|
||||
@@ -2571,8 +2626,7 @@ class Phase2Translator:
|
||||
if not _CachedIsFile(stub_path):
|
||||
stub_path = os.path.join(self.temp_dir, f"{dep_sha1}.stub.ll")
|
||||
if _CachedIsFile(stub_path):
|
||||
with open(stub_path, 'r', encoding='utf-8') as sf:
|
||||
stub_content = sf.read()
|
||||
stub_content = _ReadTextCached(stub_path)
|
||||
for line in stub_content.splitlines():
|
||||
stripped = line.strip()
|
||||
if stripped.startswith(';') or stripped.startswith('target ') or stripped.startswith('source_filename') or not stripped:
|
||||
@@ -3369,13 +3423,12 @@ class Phase2Translator:
|
||||
for other_sha1, other_stub in self.stub_files.items():
|
||||
if os.path.exists(other_stub):
|
||||
try:
|
||||
with open(other_stub, 'r', encoding='utf-8') as sf:
|
||||
for sline in sf:
|
||||
sl = sline.strip()
|
||||
if sl.startswith('%') and '= type' in sl and 'opaque' not in sl:
|
||||
struct_name = Phase2Translator._extract_llvm_type_name(sl)
|
||||
if struct_name:
|
||||
stub_decls.append(sl)
|
||||
for sline in _ReadTextCached(other_stub).splitlines(True):
|
||||
sl = sline.strip()
|
||||
if sl.startswith('%') and '= type' in sl and 'opaque' not in sl:
|
||||
struct_name = Phase2Translator._extract_llvm_type_name(sl)
|
||||
if struct_name:
|
||||
stub_decls.append(sl)
|
||||
except Exception as e:
|
||||
_vlog().warning(f"Phase2Translator: 忽略异常 {e}", exc_info=e)
|
||||
all_type_defs = {}
|
||||
@@ -3545,8 +3598,7 @@ class Phase2Translator:
|
||||
_vlog().error(f" 翻译未生成 .ll 文件: {module_name}.py")
|
||||
continue
|
||||
|
||||
with open(ll_path, 'r', encoding='utf-8') as f:
|
||||
ll_content = f.read()
|
||||
ll_content = _ReadTextCached(ll_path)
|
||||
|
||||
# --traceback 提醒：include 模块若发射了 define 却不带 DI（!llvm.dbg.cu），
|
||||
# 其 .debug_line 行为空 ⇒ 运行时 Ctrl+C 栈里该 include 帧取不到源行
|
||||
@@ -3664,6 +3716,7 @@ class Phase2Translator:
|
||||
dso_ll_path = ll_path[:-3] + '_dso.ll'
|
||||
with open(dso_ll_path, 'w', encoding='utf-8') as f:
|
||||
f.write(ll_content)
|
||||
_InvalidateTextCache(dso_ll_path)
|
||||
compile_tasks.append((dso_ll_path, obj_path, src_path, sha1, module_name, ll_path, precompiled_obj, ll_content))
|
||||
|
||||
if not compile_tasks:
|
||||
@@ -3910,12 +3963,12 @@ class Phase2Translator:
|
||||
_TryGlobalCache(sha1, 'pyi', sig_path)
|
||||
if not _CachedIsFile(sig_path):
|
||||
try:
|
||||
with open(src_path, 'r', encoding='utf-8') as f:
|
||||
py_content = f.read()
|
||||
py_content = _ReadTextCached(src_path)
|
||||
sig_content = PythonToStubConverter.convert(py_content, mod_full)
|
||||
os.makedirs(self.temp_dir, exist_ok=True)
|
||||
with open(sig_path, 'w', encoding='utf-8', newline='\n') as f:
|
||||
f.write(sig_content)
|
||||
_InvalidateTextCache(sig_path)
|
||||
_SaveToGlobalCache(sha1, 'pyi', sig_path)
|
||||
except Exception as e:
|
||||
if _config_mode == "strict":
|
||||
@@ -4137,8 +4190,7 @@ class Phase2Translator:
|
||||
for sha1_key, stub_path in self.stub_files.items():
|
||||
if os.path.exists(stub_path):
|
||||
try:
|
||||
with open(stub_path, 'r', encoding='utf-8') as f:
|
||||
stub_content = f.read()
|
||||
stub_content = _ReadTextCached(stub_path)
|
||||
for line in stub_content.splitlines():
|
||||
line = line.strip()
|
||||
if line.startswith('@') and '= global' in line and 'i32' in line and not line.startswith('@llvm'):
|
||||
@@ -4239,8 +4291,7 @@ class Phase2Translator:
|
||||
if sha1_key in shared_doc_meta:
|
||||
continue
|
||||
try:
|
||||
with open(os.path.join(_GLOBAL_CACHE_DIR, fname), 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
data = json.loads(_ReadTextCached(os.path.join(_GLOBAL_CACHE_DIR, fname)))
|
||||
if isinstance(data, dict):
|
||||
shared_doc_meta[sha1_key] = {str(k): str(v) for k, v in data.items()}
|
||||
except Exception as _e:
|
||||
@@ -4621,6 +4672,7 @@ SECTIONS {
|
||||
"""
|
||||
with open(linker_ld_dst, 'w', encoding='utf-8', newline='\n') as f:
|
||||
f.write(default_ld)
|
||||
_InvalidateTextCache(linker_ld_dst)
|
||||
_vlog().info(f" [链接] 自动创建默认链接脚本: linker.ld")
|
||||
|
||||
try:
|
||||
|
||||
Reference in new issue
Block a user