feat(backend): 新增 MLIR 后端开关/产物隔离/降级管线/模块别名(默认 llvm 逐字不变)

- Config: BackendTarget/BackendIsMlir/set_backend/is_mlir_backend/backend_dir_suffix/alias_module_name;读取 options.backend(回退 options.target)

- main.vp: 新增 --backend llvm|mlir(CLI 优先级最高,默认 llvm),加载工程后、resolve_paths 前生效

- 产物隔离: temp/output 追加 _mlir 后缀;includes 缓存追加后端后缀(.binary[_asan][_mlir])

- BuildPipeline: MLIR 降级 mlir-opt(校验) -> mlir-translate --mlir-to-llvmir,覆盖串行/并行/--phase2command,且置于 ASan opt 之前;LlcTask 增 MlirOptCmd/MlirTranslateCmd 字段

- 别名: ResolveModuleToPath 与 _FindSha1ByModName 经 Config.alias_module_name 把 llvmlite 重定向到 mlirlite(调用点零改动)
This commit is contained in:
TermiNexus committed 2026-10-03 07:53:04 +08:00
1 parent 4eec3f0908
commit 0ebe2e0752
5 files changed
+219 -14

No files matched your search

+112 -7
View File
@@ -53,6 +53,34 @@ _phase2_cmd_path: str = None
# bat 文件是否已写入 header(@echo off),首次写入时置 1
_phase2_bat_init: int = 0
# MLIR 后端(Config.is_mlir_backend())降级步骤:mlirlite 发射的是 MLIR 文本,
# 而 opt/llc 只接受 LLVM IR,故在两者之前插入:
# mlir-opt <in> -o <in>.mlir.opt (语法/语义校验)
# mlir-translate --mlir-to-llvmir <in>.mlir.opt -o <in>.ll.local (降级回 LLVM IR)
# 降级产物 <...>.ll.local 与输入 .ll 同目录,不占用 <sha1>.ll 缓存键。
MLIR_OPT_CMD: str = "mlir-opt"
MLIR_TRANSLATE_CMD: str = "mlir-translate"
def _ir_sibling_path(ir_path: str, suffix: str) -> str:
"""由 <...>.ll 派生同目录邻居路径:去掉末尾 ".ll" 后拼 suffix(如 ".mlir.opt")。
从 _mbuddy 分配(与 build_sliced_path 同池,随 pool reset 回收,无需手动 free)。
"""
if ir_path is None or suffix is None:
return None
n: sizet = string.strlen(ir_path)
if n < 3:
return None
suf_len: sizet = string.strlen(suffix)
buf: str = _mbuddy.alloc(n + suf_len + 1)
if buf is None:
return None
string.strncpy(buf, ir_path, n - 3)
buf[n - 3] = '\0'
string.strcat(buf, suffix)
return buf
# 源代码缓冲区大小(1MB)
SRC_BUF_SIZE: t.CDefine = 1048576 # 2^20 = 1MB
@@ -398,18 +426,50 @@ def compile_ll_to_obj(ir_path: str, output_dir: str, module_name: str, cc_cmd: s
# 用直接地址寻址,ASan 全局 redzone 重定位后引用错位会读到垃圾指针。
compile_input: str = ir_path
# ---- MLIR 后端降级:mlir-opt(校验)→ mlir-translate(降级回 LLVM IR)----
# 必须在 ASan opt 与 llc 之前,并把 compile_input 替换为 <...>.ll.local。
if Config.is_mlir_backend() != 0:
ll_local_path: str = _ir_sibling_path(ir_path, ".ll.local")
mlir_opt_out: str = _ir_sibling_path(ir_path, ".mlir.opt")
if ll_local_path is None or mlir_opt_out is None:
return 1
mo_len: sizet = len(MLIR_OPT_CMD) + len(ir_path) + len(mlir_opt_out) + 16
cmd_mo: str = _mbuddy.alloc(mo_len)
if cmd_mo is None:
return 1
viperlib.snprintf(cmd_mo, mo_len, "%s %s -o %s", MLIR_OPT_CMD, ir_path, mlir_opt_out)
mt_len: sizet = len(MLIR_TRANSLATE_CMD) + len(mlir_opt_out) + len(ll_local_path) + 40
cmd_mt: str = _mbuddy.alloc(mt_len)
if cmd_mt is None:
return 1
viperlib.snprintf(cmd_mt, mt_len, "%s --mlir-to-llvmir %s -o %s", MLIR_TRANSLATE_CMD, mlir_opt_out, ll_local_path)
if _phase2_cmd_path is not None:
# 只写命令(须排在下面的 ASan/llc 命令之前)
_phase2_write_cmd(cmd_mo)
_phase2_write_cmd(cmd_mt)
else:
mo_rc: int = _run_cmd_wait(cmd_mo)
if mo_rc != 0:
VLogger.error(f"mlir-opt 失败 (module={module_name}, exit={mo_rc}, cmd={cmd_mo})", "MLIR")
return 1
mt_rc: int = _run_cmd_wait(cmd_mt)
if mt_rc != 0:
VLogger.error(f"mlir-translate 失败 (module={module_name}, exit={mt_rc}, cmd={cmd_mt})", "MLIR")
return 1
compile_input = ll_local_path
# --phase2command 模式:不执行 opt/llc,把 opt(ASan)+ llc 命令写入 bat
if _phase2_cmd_path is not None:
p2_opt_cmd: str = None
if _asan_mode != 0:
p2_asan: str = build_sliced_path(output_dir, module_name, "asan.ll")
if p2_asan is not None:
p2_olen: sizet = len(p2_asan) + len(ir_path) + 192
p2_olen: sizet = len(p2_asan) + len(compile_input) + 192
p2_opt_cmd = _mbuddy.alloc(p2_olen)
if p2_opt_cmd is not None:
viperlib.snprintf(p2_opt_cmd, p2_olen,
"opt -passes=forceattrs,asan -force-attribute=sanitize_address -asan-globals=false -o %s %s",
p2_asan, ir_path)
p2_asan, compile_input)
compile_input = p2_asan
llc_fp: str = _llc_frame_ptr_flag()
p2_clen: sizet = len(cc_cmd) + len(cc_flags) + len(llc_fp) + len(obj_path) + len(compile_input) + 64
@@ -427,13 +487,13 @@ def compile_ll_to_obj(ir_path: str, output_dir: str, module_name: str, cc_cmd: s
asan_path: str = build_sliced_path(output_dir, module_name, "asan.ll")
if asan_path is None:
return 1
opt_cmd_len: sizet = len(asan_path) + len(ir_path) + 192
opt_cmd_len: sizet = len(asan_path) + len(compile_input) + 192
opt_cmd: bytes = _mbuddy.alloc(opt_cmd_len)
if opt_cmd is None:
return 1
viperlib.snprintf(opt_cmd, opt_cmd_len,
"opt -passes=forceattrs,asan -force-attribute=sanitize_address -asan-globals=false -o %s %s",
asan_path, ir_path)
asan_path, compile_input)
opt_ret: int = stdlib.system(opt_cmd)
if opt_ret != 0:
VLogger.error(f"opt ASan 插桩失败 (module={module_name})", "ASAN")
@@ -479,6 +539,9 @@ class LlcTask:
# ExitCode 追加在结尾:既有字段偏移不变;LlcTask.__sizeof__() 随之增大,
# 所有构造/初始化点均按 __sizeof__() 算首址并对整条 memset,无需改偏移。
ExitCode: int # worker 写:失败命令退出码(成功为 0)
# MLIR 后端降级命令(非 MLIR 为 None);同样追加在结尾,不影响既有偏移。
MlirOptCmd: bytes # mlir-opt 校验命令
MlirTranslateCmd: bytes # mlir-translate 降级命令
def init_llc_task(task: LlcTask | t.CPtr, ir_path: str, output_dir: str,
@@ -501,20 +564,39 @@ def init_llc_task(task: LlcTask | t.CPtr, ir_path: str, output_dir: str,
# compile_ll_to_obj 的 compile_input 同口径(否则 opt 生成了插桩产物
# 却被 llc 忽略,App 侧 obj 里 0 处 __asan,内存排查半盲)
compile_input: str = ir_path
# ASan 模式:opt 插桩命令(worker 先跑 opt 再跑 llc,故 LlcCmd 的输入
# MLIR 后端降级:worker 先跑 mlir-opt(校验)→ mlir-translate(降级回 LLVM IR),
# 之后的 ASan OptCmd / LlcCmd 输入必须是 <...>.ll.local。
if Config.is_mlir_backend() != 0:
ll_local_path: str = _ir_sibling_path(ir_path, ".ll.local")
mlir_opt_out: str = _ir_sibling_path(ir_path, ".mlir.opt")
if ll_local_path is None or mlir_opt_out is None:
return 1
mo_len: sizet = len(MLIR_OPT_CMD) + len(ir_path) + len(mlir_opt_out) + 16
task.MlirOptCmd = stdlib.malloc(mo_len)
if task.MlirOptCmd is None:
return 1
viperlib.snprintf(task.MlirOptCmd, mo_len, "%s %s -o %s", MLIR_OPT_CMD, ir_path, mlir_opt_out)
mt_len: sizet = len(MLIR_TRANSLATE_CMD) + len(mlir_opt_out) + len(ll_local_path) + 40
task.MlirTranslateCmd = stdlib.malloc(mt_len)
if task.MlirTranslateCmd is None:
return 1
viperlib.snprintf(task.MlirTranslateCmd, mt_len, "%s --mlir-to-llvmir %s -o %s",
MLIR_TRANSLATE_CMD, mlir_opt_out, ll_local_path)
compile_input = ll_local_path
# ASan 模式:opt 插桩命令(worker 先跑 MLIR 降级/opt 再跑 llc,故 LlcCmd 的输入
# 必须指向 asan_path;失败即置 Result)
if _asan_mode != 0:
asan_path: str = build_sliced_path(output_dir, module_name, "asan.ll")
if asan_path is None:
return 1
opt_len: sizet = len(asan_path) + len(ir_path) + 192
opt_len: sizet = len(asan_path) + len(compile_input) + 192
task.OptCmd = stdlib.malloc(opt_len)
if task.OptCmd is None:
return 1
viperlib.snprintf(task.OptCmd, opt_len,
"opt -passes=forceattrs,asan -force-attribute=sanitize_address"
" -asan-globals=false -o %s %s",
asan_path, ir_path)
asan_path, compile_input)
compile_input = asan_path
# llc 命令(输入 = compile_input:非 ASan 为 .ll,ASan 为 .asan.ll)
llc_fp3: str = _llc_frame_ptr_flag()
@@ -538,6 +620,12 @@ def free_llc_task(task: LlcTask | t.CPtr):
if task.LlcCmd is not None:
stdlib.free(task.LlcCmd)
task.LlcCmd = None
if task.MlirOptCmd is not None:
stdlib.free(task.MlirOptCmd)
task.MlirOptCmd = None
if task.MlirTranslateCmd is not None:
stdlib.free(task.MlirTranslateCmd)
task.MlirTranslateCmd = None
if task.ModuleSha1 is not None:
stdlib.free(task.ModuleSha1)
task.ModuleSha1 = None
@@ -628,6 +716,19 @@ def _llc_task_worker(arg: t.CPtr) -> t.CPtr:
task: LlcTask | t.CPtr = (LlcTask | t.CPtr)(arg)
if task is None:
return t.CPtr(0)
# MLIR 后端:先校验(mlir-opt)→ 降级(mlir-translate),再 ASan opt → llc
if task.MlirOptCmd is not None:
mo_rc: int = _run_cmd_wait(task.MlirOptCmd)
if mo_rc != 0:
task.ExitCode = mo_rc
task.Result = 1
return t.CPtr(0)
if task.MlirTranslateCmd is not None:
mt_rc: int = _run_cmd_wait(task.MlirTranslateCmd)
if mt_rc != 0:
task.ExitCode = mt_rc
task.Result = 1
return t.CPtr(0)
if task.OptCmd is not None:
opt_rc: int = _run_cmd_wait(task.OptCmd)
if opt_rc != 0:
@@ -657,6 +758,10 @@ def run_llc_parallel(tasks: bytes, task_count: int, max_threads: int) -> int:
tsk_p: LlcTask | t.CPtr = (LlcTask | t.CPtr)(uint64(tasks) + sizet(pi2) * LlcTask.__sizeof__())
if tsk_p is None:
continue
if tsk_p.MlirOptCmd is not None:
_phase2_write_cmd(tsk_p.MlirOptCmd)
if tsk_p.MlirTranslateCmd is not None:
_phase2_write_cmd(tsk_p.MlirTranslateCmd)
if tsk_p.OptCmd is not None:
_phase2_write_cmd(tsk_p.OptCmd)
if tsk_p.LlcCmd is not None:
+83 -7
View File
@@ -43,6 +43,17 @@ IncludesDirs: str = None # 全部 includes 目录(空格分隔,多
# includes 预编译缓存目录后缀(asan 模式用 .binary_asan 隔离 asan 插桩 obj,避免与非 asan obj 混链)
_inc_bin_suffix: str = ".binary"
# ============================================================
# 后端(Emitter Backend)
#
# "llvm"(默认):IR 发射走 includes/llvmlite(LLVM IR 文本),opt/llc 直接消费。
# "mlir" :模块名 llvmlite 在解析层重定向到 includes/mlirlite(MLIR 文本),
# 并在 opt/llc 之前插入 mlir-opt(校验)→ mlir-translate(降级回 LLVM IR)。
# 只认字符串 "mlir":缺省 / 未知值一律按 LLVM(BackendIsMlir=0),保证默认路径逐字不变。
# ============================================================
BackendTarget: str = "llvm"
BackendIsMlir: int = 0
def Load_project_config(path: str) -> int:
"""加载工程配置文件(project.vpj)
@@ -145,6 +156,16 @@ def Load_project_config(path: str) -> int:
sm: JsonValue | t.CPtr = options["strict_mode"]
if sm is not None and sm.is_bool():
StrictMode = 1 if sm.as_bool() else 0
# 后端:options.backend(新键)优先,回退 options.target(兼容旧键)。
# 只认 "mlir";其余(含缺省/未知)按 LLVM。CLI --backend 由 main.vp
# 在本函数返回后覆盖(CLI 优先级最高)。
be_val: JsonValue | t.CPtr = options["backend"]
if be_val is not None and be_val.is_string():
set_backend(be_val.as_string())
else:
tgt_be: JsonValue | t.CPtr = options["target"]
if tgt_be is not None and tgt_be.is_string():
set_backend(tgt_be.as_string())
# 读取 includes 数组(支持多目录:首元素存 IncludesDir,全部拼接存 IncludesDirs)
includes_val: JsonValue | t.CPtr = root["includes"]
@@ -424,17 +445,19 @@ def resolve_paths(project_dir: str) -> int:
SourceDir = _join_path(SourceDir, project_dir)
BuildDir = _join_path(BuildDir, project_dir)
# TempDir = {BuildDir}/temp
# TempDir = {BuildDir}/temp(非 LLVM 后端追加隔离后缀,如 _mlir;LLVM 时后缀为空串,路径逐字不变)
if BuildDir is not None:
bd_len: sizet = string.strlen(BuildDir)
temp_buf: str = _mbuddy.alloc(bd_len + 8)
suf_dir: str = backend_dir_suffix()
suf_dir_len: sizet = string.strlen(suf_dir)
temp_buf: str = _mbuddy.alloc(bd_len + suf_dir_len + 8)
if temp_buf is not None:
viperlib.snprintf(temp_buf, bd_len + 8, "%s/temp", BuildDir)
viperlib.snprintf(temp_buf, bd_len + suf_dir_len + 8, "%s/temp%s", BuildDir, suf_dir)
TempDir = temp_buf
# OutputDir = {BuildDir}/output
output_buf: str = _mbuddy.alloc(bd_len + 8)
output_buf: str = _mbuddy.alloc(bd_len + suf_dir_len + 8)
if output_buf is not None:
viperlib.snprintf(output_buf, bd_len + 8, "%s/output", BuildDir)
viperlib.snprintf(output_buf, bd_len + suf_dir_len + 8, "%s/output%s", BuildDir, suf_dir)
OutputDir = output_buf
# includes 路径可能是 ../includes 形式,需要基于 project_dir 解析
@@ -638,18 +661,71 @@ def get_includes_binary_dir() -> str:
includes 目录如 ../includes,默认返回 ../includes.binary;
asan 模式(set_inc_bin_suffix(".binary_asan"))返回 ../includes.binary_asan,
隔离 asan 插桩 obj,避免与非 asan obj 混链。
非 LLVM 后端再追加后端后缀(如 .binary_asan_mlir),隔离不同后端的 obj。
"""
if IncludesDir is None:
return None
inc_len: sizet = string.strlen(IncludesDir)
suf_len: sizet = string.strlen(_inc_bin_suffix)
buf: str = _mbuddy.alloc(inc_len + suf_len + 1)
be_suf: str = backend_dir_suffix()
be_suf_len: sizet = string.strlen(be_suf)
buf: str = _mbuddy.alloc(inc_len + suf_len + be_suf_len + 1)
if buf is None:
return None
viperlib.snprintf(buf, inc_len + suf_len + 1, "%s%s", IncludesDir, _inc_bin_suffix)
viperlib.snprintf(buf, inc_len + suf_len + be_suf_len + 1, "%s%s%s", IncludesDir, _inc_bin_suffix, be_suf)
return buf
# ============================================================
# 后端开关(Emitter Backend)
#
# set_backend: 只接受 "llvm" / "mlir"(其余返回 1 且不改动),大小写敏感。
# backend_dir_suffix: 非默认后端的目录隔离后缀;LLVM 后端返回空串(调用方拼出的
# 路径与改前逐字相同)。用于 TempDir/OutputDir/includes.binary。
# is_mlir_backend: 0/1 便捷视图(BuildPipeline 降级管线的门)。
# ============================================================
def set_backend(name: str) -> int:
"""设置后端;0=成功,1=非法(name 为 None 或非 llvm/mlir)"""
global BackendTarget, BackendIsMlir
if name is None:
return 1
if string.strcmp(name, "llvm") == 0:
BackendTarget = name
BackendIsMlir = 0
return 0
if string.strcmp(name, "mlir") == 0:
BackendTarget = name
BackendIsMlir = 1
return 0
return 1
def is_mlir_backend() -> int:
"""是否 MLIR 后端(1=mlir / 0=llvm)"""
return BackendIsMlir
def backend_dir_suffix() -> str:
"""非默认后端的目录隔离后缀(LLVM 后端返回空串 ⇒ 路径逐字不变)"""
if BackendIsMlir != 0:
return "_mlir"
return ""
def alias_module_name(name: str) -> str:
"""MLIR 后端下把模块名 "llvmlite" 重定向到 "mlirlite"(其余原样返回)。
这是"TransPyV 调用点零改动"的唯一开关:所有 **名称→模块/文件/SHA1** 的解析
都应经此函数(模块文件解析 IncludesScanner.ResolveModuleToPath、依赖名→SHA1
StubMergerIR._FindSha1ByModName 等)。LLVM 后端恒等返回,默认路径逐字不变。
"""
if name is None:
return None
if BackendIsMlir != 0 and string.strcmp(name, "llvmlite") == 0:
return "mlirlite"
return name
def asan_runtime_declared() -> int:
"""project.vpj 是否已声明 ASan 运行库(1 = 声明过)
+5
View File
@@ -335,6 +335,11 @@ def _FindSha1ByModName(sha1_arr: str, mod_arr: str,
return None
if count <= 0:
return None
# MLIR 后端:依赖名 "llvmlite" 重定向到 "mlirlite"
#(与 IncludesScanner.ResolveModuleToPath 同口径;LLVM 后端恒等)
mod_name = Config.alias_module_name(mod_name)
if mod_name is None:
return None
# 第一遍:精确匹配
for i in range(count):
idx: sizet = sizet(i) * 64
+7
View File
@@ -10,6 +10,7 @@ import hashlib
import viperlib
import lib.Projectrans.Utils as Utils
import lib.core.VLogger as VLogger
import lib.Projectrans.Config as Config
# ============================================================
@@ -452,6 +453,12 @@ def ResolveModuleToPath(includes_dirs: str, mod_name: str,
out_inc_dir: t.CPtr) -> int:
if includes_dirs is None or mod_name is None:
return 1
# MLIR 后端:模块名 "llvmlite" 重定向到 "mlirlite"(TransPyV 调用点零改动)。
# 重定向后的名字用于本次解析的路径段/相对路径,调用方拿到的 rel_path
# 即 "mlirlite/__init__.vp" ⇒ 下游模块身份/SHA1 全部随之一致。
mod_name = Config.alias_module_name(mod_name)
if mod_name is None:
return 1
mn_len: sizet = string.strlen(mod_name)
if mn_len == 0:
return 1
+12
View File
@@ -162,6 +162,7 @@ def main() -> int:
parser.add_argument("--repl", None, argparse.BOOL, 0, None, False, argparse.STORE_TRUE, "Start the interactive REPL (M2 session variables + M3 cross-fragment functions: fragment -> DLL -> call)")
parser.add_argument("--build-dir", None, argparse.STRING, 0, None, False, argparse.STORE, "Override project.vpj build_dir (default ./.tpv_build; relative paths resolve against the project dir)")
parser.add_argument("--no-llvmpeephole", None, argparse.BOOL, 0, None, False, argparse.STORE_TRUE, "Disable all local llvmlite peephole passes (T2 store/load forward & ret-zero, T3 no-op bitcast) in FunctionPrint serialization")
parser.add_argument("--backend", None, argparse.STRING, 0, None, False, argparse.STORE, "Emitter backend: llvm (default) | mlir. mlir redirects module 'llvmlite' to includes/mlirlite and inserts mlir-opt + mlir-translate before opt/llc")
parser.add_argument("--version", "-v", argparse.BOOL, 0, None, False, argparse.STORE_TRUE, "Show version and exit")
parser.add_argument("--help", "-h", argparse.BOOL, 0, None, False, argparse.STORE_TRUE, "Show help and exit")
@@ -369,6 +370,17 @@ Target: %s
proj_path = "../project.vpj"
if Config.Load_project_config(proj_path) == 0:
proj_loaded = 1
# === --backend: CLI 覆盖 project.vpj 的 options.backend(CLI 优先级最高)===
# 位置:工程配置加载之后(options.backend 已读入)、resolve_paths 之前
#(TempDir/OutputDir 的后端隔离后缀由此定型);非法值 fail-fast。
gb_backend: str = args.get_str("backend")
if gb_backend is not None:
if Config.set_backend(gb_backend) != 0:
log.error(f"非法的 --backend 值: '{gb_backend}'(只支持 llvm / mlir)", "backend")
argparse.release(args)
return 1
log.info(f"后端: {Config.BackendTarget}", "backend")
if proj_loaded == 1:
# 提取 project.vpj 所在目录,将相对路径解析为基于该目录的路径
proj_len: sizet = string.strlen(proj_path)