fix(tpc): 符号表加载改两阶段，消除模块加载顺序依赖
a 类（编译器缺陷/机制漏豁免）：签名解析要通过符号表查其它模块的类型名 （from X import * 的裸名如 posix 的 INT；跨模块类如 rawptr[Constant]），而加载器 按 os.listdir 字典序逐个「提取→解析→插入」⇒ posix 排在 stdint 之前时 INT 查不到。 旧口径静默降级 i32 掩盖了它，fail-fast 后暴露。 修法：加载拆两趟 —— ① StageModuleSymbols：提取 + 只解析/插入声明 （类/枚举/typedef/匿名类型）② CommitModuleSymbols：解析签名 + 插入函数/define + 命名空间重导出。批量加载器（Phase2Translator._build_shared_symbol_data）先收集全部 模块、跑完所有①再统一跑② ⇒ 任一模块的签名都能看到全部声明，与顺序无关。 单模块 LoadModuleSymbols 语义不变（①紧接②）。 验证（探针，posix 先 stage、stdint 后 stage 的“坏顺序”）： posix.open ret=i32 params=[i8*,i32]（INT=stdint 的 typedef）；read ret=i64 params=[i32,i8*,i64]； 同样顺序走旧的单模块路径则 RAISE（对照组）。
This commit is contained in:
TermiNexus committed
1 parent
8b657b9ac2
commit
05f4adb233
4 files changed
+168
-53
No files matched your search
@@ -3942,47 +3942,69 @@ class Phase2Translator:
|
||||
base_trans.translator._global_function_default_args = self.function_default_args
|
||||
base_trans.translator._global_function_param_names = self.function_param_names
|
||||
|
||||
# ---- 两阶段加载：先收集全部模块，再「①声明」全跑完 / 「②签名」全跑完 ----
|
||||
# 声明阶段把类/枚举/typedef 灌满符号表，签名阶段才解析函数签名 ⇒ 模块间
|
||||
# 类型引用（`from X import *` 的裸名如 posix 的 `INT`、跨模块类如
|
||||
# `rawptr[Constant]`）不再依赖 os.listdir 的字典序（此前 posix 排在 stdint
|
||||
# 之前 ⇒ `INT` 解析不到，旧口径静默降级成 i32，fail-fast 后必须消除）。
|
||||
staged: list[tuple[str, str]] = [] # (声明文件路径, 模块名)
|
||||
# 需要重导出的包（在全部模块符号加载完后统一处理；提到循环外避免多个
|
||||
# include_dir 时被反复覆盖/重复收集）
|
||||
_pending_reexports: list[tuple[str, str]] = []
|
||||
for includes_dir in self.include_dirs:
|
||||
if os.path.isdir(includes_dir):
|
||||
for pyi_file in os.listdir(includes_dir):
|
||||
if pyi_file.endswith('.vpi') or pyi_file.endswith('.pyi'):
|
||||
pyi_path = os.path.join(includes_dir, pyi_file)
|
||||
module_name = os.path.splitext(pyi_file)[0]
|
||||
base_trans.translator.SymbolTable.LoadModuleSymbols(pyi_path, module_name, lineno=0)
|
||||
staged.append((pyi_path, module_name))
|
||||
for py_file in os.listdir(includes_dir):
|
||||
if (py_file.endswith('.vp') or py_file.endswith('.py')) and (not py_file.startswith('_') or py_file == '_list.vp' or py_file == '_list.py' or py_file == '_dict.vp' or py_file == '_dict.py'):
|
||||
module_name = os.path.splitext(py_file)[0]
|
||||
sha1_key = self.include_py_map.get(module_name)
|
||||
if sha1_key and sha1_key in self.sig_files:
|
||||
sig_path = self.sig_files[sha1_key]
|
||||
base_trans.translator.SymbolTable.LoadModuleSymbols(sig_path, module_name, lineno=0)
|
||||
staged.append((self.sig_files[sha1_key], module_name))
|
||||
# 收集需要重导出的包，等所有模块符号加载完后再处理
|
||||
_pending_reexports = []
|
||||
for sha1_key, sig_path in self.sig_files.items():
|
||||
rel_path = self.sha1_map.get(sha1_key, sha1_key)
|
||||
if rel_path.startswith('includes/'):
|
||||
mod_name = os.path.splitext(rel_path)[0].replace(os.sep, '.').replace('/', '.')
|
||||
mod_name = mod_name[len('includes/'):]
|
||||
base_trans.translator.SymbolTable.LoadModuleSymbols(sig_path, mod_name, lineno=0)
|
||||
_entry_mod: tuple[str, str] = (sig_path, mod_name)
|
||||
if _entry_mod not in staged:
|
||||
staged.append(_entry_mod)
|
||||
if mod_name.endswith('.__init__'):
|
||||
package_name = mod_name[:-len('.__init__')]
|
||||
_pending_reexports.append((sig_path, package_name))
|
||||
|
||||
# 所有 includes 模块符号已加载，现在处理包重导出
|
||||
for sig_path, package_name in _pending_reexports:
|
||||
self._register_package_reexports(base_trans, sig_path, package_name)
|
||||
_entry_pkg: tuple[str, str] = (sig_path, package_name)
|
||||
if _entry_pkg not in _pending_reexports:
|
||||
_pending_reexports.append(_entry_pkg)
|
||||
|
||||
for sha1_key, sig_path in self.sig_files.items():
|
||||
rel_path = self.sha1_map.get(sha1_key, sha1_key)
|
||||
if rel_path.startswith('includes/'):
|
||||
continue
|
||||
module_name = os.path.splitext(rel_path)[0].replace(os.sep, '.').replace('/', '.')
|
||||
base_trans.translator.SymbolTable.LoadModuleSymbols(sig_path, module_name, lineno=0)
|
||||
staged.append((sig_path, module_name))
|
||||
base_trans.translator._source_module_sig_files[module_name] = sig_path
|
||||
short_name = module_name.split('.')[-1] if '.' in module_name else module_name
|
||||
if short_name != module_name:
|
||||
base_trans.translator._source_module_sig_files[short_name] = sig_path
|
||||
|
||||
# ① 声明阶段（全部模块）
|
||||
_staged_modules = []
|
||||
for _stage_path, _stage_mod in staged:
|
||||
_stage_ms = base_trans.translator.SymbolTable.StageModuleSymbols(_stage_path, _stage_mod)
|
||||
if _stage_ms is not None:
|
||||
_staged_modules.append((_stage_ms, _stage_mod, _stage_path))
|
||||
|
||||
# ② 签名阶段（全部模块；此时任一模块的签名都能看到全部声明）
|
||||
for _commit_ms, _commit_mod, _commit_path in _staged_modules:
|
||||
base_trans.translator.SymbolTable.CommitModuleSymbols(_commit_ms, _commit_mod, _commit_path)
|
||||
|
||||
# 所有 includes 模块符号已加载，现在处理包重导出
|
||||
for sig_path, package_name in _pending_reexports:
|
||||
self._register_package_reexports(base_trans, sig_path, package_name)
|
||||
|
||||
all_dc = {}
|
||||
for sha1_key, stub_path in self.stub_files.items():
|
||||
if os.path.exists(stub_path):
|
||||
|
||||
@@ -20,7 +20,17 @@ class SymbolInserter:
|
||||
self._symtab.register_namespace(namespace)
|
||||
|
||||
def insert(self, module_symbols: ModuleSymbols) -> list[str]:
|
||||
"""插入所有符号到主命名空间，返回已加载的符号名列表"""
|
||||
"""插入所有符号到主命名空间，返回已加载的符号名列表（= 声明 + 函数两阶段）"""
|
||||
loaded: list[str] = self.insert_declarations(module_symbols)
|
||||
loaded.extend(self.insert_functions(module_symbols))
|
||||
return loaded
|
||||
|
||||
def insert_declarations(self, module_symbols: ModuleSymbols) -> list[str]:
|
||||
"""阶段 1：插入**声明**符号（类 / 枚举成员 / typedef / 匿名类型）。
|
||||
|
||||
与 insert_functions 分开：批量加载器先把所有模块的声明灌进符号表，再统一
|
||||
解析+插入函数 ⇒ 模块间类型引用与加载顺序无关。
|
||||
"""
|
||||
loaded: list[str] = []
|
||||
file_path: str = module_symbols.file_path
|
||||
ns: dict[str, object] = self._symtab._namespaces.get(self._namespace, {}) if self._namespace else {}
|
||||
@@ -62,6 +72,21 @@ class SymbolInserter:
|
||||
if self._namespace:
|
||||
ns[td.name] = self._symtab._symbols[td.name]
|
||||
|
||||
# 插入匿名类型符号（短名）
|
||||
for anon_name, anon_data in module_symbols.anonymous_types.items():
|
||||
self._symtab._InsertAnonymousSymbol(
|
||||
anon_name, anon_data.is_union, anon_data.members,
|
||||
anon_data.lineno, file_path
|
||||
)
|
||||
|
||||
return loaded
|
||||
|
||||
def insert_functions(self, module_symbols: ModuleSymbols) -> list[str]:
|
||||
"""阶段 2：插入**函数 / define** 符号（须已插入本模块声明）"""
|
||||
loaded: list[str] = []
|
||||
file_path: str = module_symbols.file_path
|
||||
ns: dict[str, object] = self._symtab._namespaces.get(self._namespace, {}) if self._namespace else {}
|
||||
|
||||
# 插入函数符号
|
||||
for func in module_symbols.functions:
|
||||
# 优先使用 CTypeInfo（保留符号性），回退到 LLVM 类型字符串
|
||||
@@ -83,11 +108,4 @@ class SymbolInserter:
|
||||
if self._namespace:
|
||||
ns[define.name] = self._symtab._symbols[define.name]
|
||||
|
||||
# 插入匿名类型符号（短名）
|
||||
for anon_name, anon_data in module_symbols.anonymous_types.items():
|
||||
self._symtab._InsertAnonymousSymbol(
|
||||
anon_name, anon_data.is_union, anon_data.members,
|
||||
anon_data.lineno, file_path
|
||||
)
|
||||
|
||||
return loaded
|
||||
+93
-33
@@ -373,46 +373,106 @@ class SymbolTable:
|
||||
|
||||
def LoadModuleSymbols(self, FullModulePath: str, asname: str, lineno: int = 0) -> list[str]:
|
||||
return self._LoadSymbolsFromFile(FullModulePath, asname, lineno)
|
||||
|
||||
def _LoadSymbolsFromFile(self, FilePath: str, namespace_prefix: str | list[str], lineno: int = 0) -> list[str]:
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# 两阶段加载（顺序无关）：声明阶段 / 签名阶段
|
||||
#
|
||||
# 为什么要拆：签名解析（Pass 2 的函数部分）会通过符号表查找**其它模块**的
|
||||
# 类型名（裸名 typedef，如 posix 的 `INT | t.State`；跨模块类，如
|
||||
# `rawptr[Constant]`）。单模块逐个「提取→解析→插入」时，这些名字要求依赖模块
|
||||
# **恰好先被加载**（Phase2Translator 按 os.listdir 字典序加载 ⇒ posix 排在
|
||||
# stdint 之前，`INT` 找不到）。旧口径把它静默降级成 i32，所以顺序依赖不可见；
|
||||
# 改成 fail-fast 后必须消除顺序依赖 —— 做法是把加载拆成两趟：
|
||||
# ① StageModuleSymbols：提取 + 只解析/插入**声明**（类/枚举/typedef/匿名类型）
|
||||
# ② CommitModuleSymbols：解析签名 + 插入函数/define + 命名空间重导出
|
||||
# 批量加载器先把**所有**模块跑完①再统一跑②，任一模块的签名都能看到全部声明。
|
||||
# 单模块加载（_LoadSymbolsFromFile）语义不变：①紧接②。
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def StageModuleSymbols(self, FilePath: str, asname: str) -> ModuleSymbols | None:
|
||||
"""两阶段加载·①：提取 + 声明解析/插入（类 / 枚举成员 / typedef / 匿名类型）。
|
||||
|
||||
失败返回 None（非类型错误的既有语义）或上抛 TypeNameUnresolvedError。
|
||||
"""
|
||||
try:
|
||||
# 提取模块名作为命名空间
|
||||
module_name: str = namespace_prefix[0] if isinstance(namespace_prefix, list) else namespace_prefix
|
||||
|
||||
# Pass 1: 从 AST 提取原始符号信息
|
||||
extractor: ASTSymbolExtractor = ASTSymbolExtractor(self)
|
||||
module_symbols: ModuleSymbols = extractor.extract(FilePath)
|
||||
|
||||
# Pass 2: 解析类型信息（typedef、函数签名、define 值）
|
||||
resolver: TypeResolver = TypeResolver(self)
|
||||
resolver.resolve(module_symbols)
|
||||
|
||||
# Pass 3: 插入符号到主命名空间（无前缀）
|
||||
inserter: SymbolInserter = SymbolInserter(self)
|
||||
inserter.set_namespace(module_name)
|
||||
loaded: list[str] = inserter.insert(module_symbols)
|
||||
|
||||
# Pass 4: 重新导出到命名空间前缀下
|
||||
prefixes: list[str] = [namespace_prefix] if isinstance(namespace_prefix, str) else namespace_prefix
|
||||
reexporter: PackageReexporter = PackageReexporter(self)
|
||||
reexporter.set_namespace(module_name)
|
||||
loaded.extend(reexporter.reexport(module_symbols, prefixes, lineno))
|
||||
|
||||
return loaded
|
||||
|
||||
return self._StageSymbolsFromFile(FilePath, asname)
|
||||
except TypeNameUnresolvedError as e:
|
||||
# 类型解析 fail-fast：**不再**吞成"加载模块失败"后返回空符号表
|
||||
# （那会让该模块的全部符号缺失，错误在下游以"找不到类/函数"的面目炸开，
|
||||
# 根因被掩盖）。补上文件/模块定位后上抛 → 上层终止编译。
|
||||
raise TypeNameUnresolvedError(
|
||||
f"[类型解析失败] 声明文件 {FilePath}\n"
|
||||
f" 模块命名空间：{namespace_prefix}\n"
|
||||
f" 第 {lineno} 行发起加载（lineno=0 表示解析声明文件本身）\n"
|
||||
f" {e}") from None
|
||||
self._TypeResolveFailText(FilePath, asname, 0, e)) from None
|
||||
except Exception as e:
|
||||
_vlog().error(traceback.format_exc())
|
||||
self.diagnostics.error(FilePath, 0, f"加载模块失败: {e}")
|
||||
return None
|
||||
|
||||
def CommitModuleSymbols(self, module_symbols: ModuleSymbols, asname: str,
|
||||
FilePath: str, lineno: int = 0) -> list[str]:
|
||||
"""两阶段加载·②：签名解析 + 函数/define 插入 + 命名空间重导出。"""
|
||||
try:
|
||||
return self._CommitSymbolsFromFile(module_symbols, FilePath, asname, lineno)
|
||||
except TypeNameUnresolvedError as e:
|
||||
raise TypeNameUnresolvedError(
|
||||
self._TypeResolveFailText(FilePath, asname, lineno, e)) from None
|
||||
except Exception as e:
|
||||
_vlog().error(traceback.format_exc())
|
||||
self.diagnostics.error(FilePath, lineno, f"加载模块失败: {e}")
|
||||
return []
|
||||
|
||||
def _StageSymbolsFromFile(self, FilePath: str,
|
||||
namespace_prefix: str | list[str]) -> ModuleSymbols:
|
||||
"""① 的裸实现（异常交调用方处理）"""
|
||||
# 提取模块名作为命名空间
|
||||
module_name: str = namespace_prefix[0] if isinstance(namespace_prefix, list) else namespace_prefix
|
||||
|
||||
# Pass 1: 从 AST 提取原始符号信息
|
||||
extractor: ASTSymbolExtractor = ASTSymbolExtractor(self)
|
||||
module_symbols: ModuleSymbols = extractor.extract(FilePath)
|
||||
|
||||
# Pass 2a: 解析声明（typedef；同时注入本模块类名/枚举宽度供签名解析用）
|
||||
resolver: TypeResolver = TypeResolver(self)
|
||||
resolver.resolve_declarations(module_symbols)
|
||||
|
||||
# Pass 3a: 插入声明符号到主命名空间（无前缀）
|
||||
inserter: SymbolInserter = SymbolInserter(self)
|
||||
inserter.set_namespace(module_name)
|
||||
inserter.insert_declarations(module_symbols)
|
||||
return module_symbols
|
||||
|
||||
def _CommitSymbolsFromFile(self, module_symbols: ModuleSymbols, FilePath: str,
|
||||
namespace_prefix: str | list[str], lineno: int = 0) -> list[str]:
|
||||
"""② 的裸实现（异常交调用方处理）"""
|
||||
module_name: str = namespace_prefix[0] if isinstance(namespace_prefix, list) else namespace_prefix
|
||||
|
||||
# Pass 2b: 解析函数签名与 define 值（此时全部声明已在符号表中）
|
||||
resolver: TypeResolver = TypeResolver(self)
|
||||
resolver.resolve_signatures(module_symbols)
|
||||
|
||||
# Pass 3b: 插入函数/define 符号到主命名空间（无前缀）
|
||||
inserter: SymbolInserter = SymbolInserter(self)
|
||||
inserter.set_namespace(module_name)
|
||||
loaded: list[str] = inserter.insert_functions(module_symbols)
|
||||
|
||||
# Pass 4: 重新导出到命名空间前缀下
|
||||
prefixes: list[str] = [namespace_prefix] if isinstance(namespace_prefix, str) else namespace_prefix
|
||||
reexporter: PackageReexporter = PackageReexporter(self)
|
||||
reexporter.set_namespace(module_name)
|
||||
loaded.extend(reexporter.reexport(module_symbols, prefixes, lineno))
|
||||
return loaded
|
||||
|
||||
@staticmethod
|
||||
def _TypeResolveFailText(FilePath: str, namespace_prefix: str | list[str],
|
||||
lineno: int, e: Exception) -> str:
|
||||
"""类型解析 fail-fast 的报头（文件 / 模块 / 发起行）+ 内层定位"""
|
||||
return (f"[类型解析失败] 声明文件 {FilePath}\n"
|
||||
f" 模块命名空间：{namespace_prefix}\n"
|
||||
f" 第 {lineno} 行发起加载（lineno=0 表示解析声明文件本身）\n"
|
||||
f" {e}")
|
||||
|
||||
def _LoadSymbolsFromFile(self, FilePath: str, namespace_prefix: str | list[str], lineno: int = 0) -> list[str]:
|
||||
"""单模块加载 = ① 紧随 ②（语义与拆分前一致）"""
|
||||
module_symbols: ModuleSymbols | None = self.StageModuleSymbols(FilePath, namespace_prefix)
|
||||
if module_symbols is None:
|
||||
return []
|
||||
return self.CommitModuleSymbols(module_symbols, namespace_prefix, FilePath, lineno)
|
||||
|
||||
# ==================================================================
|
||||
# 类型解析辅助方法（委托到 LLVMTypeMapper）
|
||||
|
||||
@@ -24,7 +24,19 @@ class TypeResolver:
|
||||
self._resolved_typedefs: dict[str, str | None] = {}
|
||||
|
||||
def resolve(self, module_symbols: ModuleSymbols) -> ModuleSymbols:
|
||||
"""解析所有类型信息，就地更新 ModuleSymbols"""
|
||||
"""解析所有类型信息（= 声明 + 签名两阶段，供单模块加载使用）"""
|
||||
self.resolve_declarations(module_symbols)
|
||||
self.resolve_signatures(module_symbols)
|
||||
return module_symbols
|
||||
|
||||
def resolve_declarations(self, module_symbols: ModuleSymbols) -> ModuleSymbols:
|
||||
"""阶段 1：只解析**声明**（typedef）。
|
||||
|
||||
与 resolve_signatures 分开，是为了让批量加载器先把所有模块的声明灌进
|
||||
符号表、再统一解析签名 ⇒ 模块间类型引用（`from X import *` 的裸名、
|
||||
`rawptr[跨模块类]`）与加载顺序无关（此前 posix 因字典序排在 stdint 之前，
|
||||
注解 `INT` 解析不到，被静默降级成 i32）。
|
||||
"""
|
||||
# 构建 class_members 查找表（typedef Postdefinition 需要）
|
||||
class_members: dict[str, dict[str, CTypeInfo]] = {cls.name: cls.members for cls in module_symbols.classes}
|
||||
|
||||
@@ -45,7 +57,10 @@ class TypeResolver:
|
||||
for td in module_symbols.typedefs:
|
||||
self._resolve_typedef(td, class_members)
|
||||
self._resolved_typedefs[td.name] = td.original_class
|
||||
return module_symbols
|
||||
|
||||
def resolve_signatures(self, module_symbols: ModuleSymbols) -> ModuleSymbols:
|
||||
"""阶段 2：解析函数签名与 define 值（依赖其它模块的声明已插入符号表）"""
|
||||
# 解析函数签名
|
||||
func: FuncSymbolData
|
||||
for func in module_symbols.functions:
|
||||
|
||||
Reference in new issue
Block a user