feat(backend): 新增 MLIR 后端开关/产物隔离/降级管线/模块别名（默认 llvm 逐字不变）
- Config: BackendTarget/BackendIsMlir/set_backend/is_mlir_backend/backend_dir_suffix/alias_module_name；读取 options.backend（回退 options.target） - main.vp: 新增 --backend llvm|mlir（CLI 优先级最高，默认 llvm），加载工程后、resolve_paths 前生效 - 产物隔离: temp/output 追加 _mlir 后缀；includes 缓存追加后端后缀（.binary[_asan][_mlir]） - BuildPipeline: MLIR 降级 mlir-opt(校验) -> mlir-translate --mlir-to-llvmir，覆盖串行/并行/--phase2command，且置于 ASan opt 之前；LlcTask 增 MlirOptCmd/MlirTranslateCmd 字段 - 别名: ResolveModuleToPath 与 _FindSha1ByModName 经 Config.alias_module_name 把 llvmlite 重定向到 mlirlite（调用点零改动）
This commit is contained in:
TermiNexus committed
1 parent
4eec3f0908
commit
0ebe2e0752
5 files changed
+219
-14
No files matched your search
@@ -53,6 +53,34 @@ _phase2_cmd_path: str = None
|
||||
# bat 文件是否已写入 header（@echo off），首次写入时置 1
|
||||
_phase2_bat_init: int = 0
|
||||
|
||||
# MLIR 后端（Config.is_mlir_backend()）降级步骤：mlirlite 发射的是 MLIR 文本，
|
||||
# 而 opt/llc 只接受 LLVM IR，故在两者之前插入：
|
||||
# mlir-opt <in> -o <in>.mlir.opt （语法/语义校验）
|
||||
# mlir-translate --mlir-to-llvmir <in>.mlir.opt -o <in>.ll.local （降级回 LLVM IR）
|
||||
# 降级产物 <...>.ll.local 与输入 .ll 同目录，不占用 <sha1>.ll 缓存键。
|
||||
MLIR_OPT_CMD: str = "mlir-opt"
|
||||
MLIR_TRANSLATE_CMD: str = "mlir-translate"
|
||||
|
||||
|
||||
def _ir_sibling_path(ir_path: str, suffix: str) -> str:
|
||||
"""由 <...>.ll 派生同目录邻居路径：去掉末尾 ".ll" 后拼 suffix（如 ".mlir.opt"）。
|
||||
|
||||
从 _mbuddy 分配（与 build_sliced_path 同池，随 pool reset 回收，无需手动 free）。
|
||||
"""
|
||||
if ir_path is None or suffix is None:
|
||||
return None
|
||||
n: sizet = string.strlen(ir_path)
|
||||
if n < 3:
|
||||
return None
|
||||
suf_len: sizet = string.strlen(suffix)
|
||||
buf: str = _mbuddy.alloc(n + suf_len + 1)
|
||||
if buf is None:
|
||||
return None
|
||||
string.strncpy(buf, ir_path, n - 3)
|
||||
buf[n - 3] = '\0'
|
||||
string.strcat(buf, suffix)
|
||||
return buf
|
||||
|
||||
# 源代码缓冲区大小（1MB）
|
||||
SRC_BUF_SIZE: t.CDefine = 1048576 # 2^20 = 1MB
|
||||
|
||||
@@ -398,18 +426,50 @@ def compile_ll_to_obj(ir_path: str, output_dir: str, module_name: str, cc_cmd: s
|
||||
# 用直接地址寻址，ASan 全局 redzone 重定位后引用错位会读到垃圾指针。
|
||||
compile_input: str = ir_path
|
||||
|
||||
# ---- MLIR 后端降级：mlir-opt（校验）→ mlir-translate（降级回 LLVM IR）----
|
||||
# 必须在 ASan opt 与 llc 之前，并把 compile_input 替换为 <...>.ll.local。
|
||||
if Config.is_mlir_backend() != 0:
|
||||
ll_local_path: str = _ir_sibling_path(ir_path, ".ll.local")
|
||||
mlir_opt_out: str = _ir_sibling_path(ir_path, ".mlir.opt")
|
||||
if ll_local_path is None or mlir_opt_out is None:
|
||||
return 1
|
||||
mo_len: sizet = len(MLIR_OPT_CMD) + len(ir_path) + len(mlir_opt_out) + 16
|
||||
cmd_mo: str = _mbuddy.alloc(mo_len)
|
||||
if cmd_mo is None:
|
||||
return 1
|
||||
viperlib.snprintf(cmd_mo, mo_len, "%s %s -o %s", MLIR_OPT_CMD, ir_path, mlir_opt_out)
|
||||
mt_len: sizet = len(MLIR_TRANSLATE_CMD) + len(mlir_opt_out) + len(ll_local_path) + 40
|
||||
cmd_mt: str = _mbuddy.alloc(mt_len)
|
||||
if cmd_mt is None:
|
||||
return 1
|
||||
viperlib.snprintf(cmd_mt, mt_len, "%s --mlir-to-llvmir %s -o %s", MLIR_TRANSLATE_CMD, mlir_opt_out, ll_local_path)
|
||||
if _phase2_cmd_path is not None:
|
||||
# 只写命令（须排在下面的 ASan/llc 命令之前）
|
||||
_phase2_write_cmd(cmd_mo)
|
||||
_phase2_write_cmd(cmd_mt)
|
||||
else:
|
||||
mo_rc: int = _run_cmd_wait(cmd_mo)
|
||||
if mo_rc != 0:
|
||||
VLogger.error(f"mlir-opt 失败 (module={module_name}, exit={mo_rc}, cmd={cmd_mo})", "MLIR")
|
||||
return 1
|
||||
mt_rc: int = _run_cmd_wait(cmd_mt)
|
||||
if mt_rc != 0:
|
||||
VLogger.error(f"mlir-translate 失败 (module={module_name}, exit={mt_rc}, cmd={cmd_mt})", "MLIR")
|
||||
return 1
|
||||
compile_input = ll_local_path
|
||||
|
||||
# --phase2command 模式：不执行 opt/llc，把 opt（ASan）+ llc 命令写入 bat
|
||||
if _phase2_cmd_path is not None:
|
||||
p2_opt_cmd: str = None
|
||||
if _asan_mode != 0:
|
||||
p2_asan: str = build_sliced_path(output_dir, module_name, "asan.ll")
|
||||
if p2_asan is not None:
|
||||
p2_olen: sizet = len(p2_asan) + len(ir_path) + 192
|
||||
p2_olen: sizet = len(p2_asan) + len(compile_input) + 192
|
||||
p2_opt_cmd = _mbuddy.alloc(p2_olen)
|
||||
if p2_opt_cmd is not None:
|
||||
viperlib.snprintf(p2_opt_cmd, p2_olen,
|
||||
"opt -passes=forceattrs,asan -force-attribute=sanitize_address -asan-globals=false -o %s %s",
|
||||
p2_asan, ir_path)
|
||||
p2_asan, compile_input)
|
||||
compile_input = p2_asan
|
||||
llc_fp: str = _llc_frame_ptr_flag()
|
||||
p2_clen: sizet = len(cc_cmd) + len(cc_flags) + len(llc_fp) + len(obj_path) + len(compile_input) + 64
|
||||
@@ -427,13 +487,13 @@ def compile_ll_to_obj(ir_path: str, output_dir: str, module_name: str, cc_cmd: s
|
||||
asan_path: str = build_sliced_path(output_dir, module_name, "asan.ll")
|
||||
if asan_path is None:
|
||||
return 1
|
||||
opt_cmd_len: sizet = len(asan_path) + len(ir_path) + 192
|
||||
opt_cmd_len: sizet = len(asan_path) + len(compile_input) + 192
|
||||
opt_cmd: bytes = _mbuddy.alloc(opt_cmd_len)
|
||||
if opt_cmd is None:
|
||||
return 1
|
||||
viperlib.snprintf(opt_cmd, opt_cmd_len,
|
||||
"opt -passes=forceattrs,asan -force-attribute=sanitize_address -asan-globals=false -o %s %s",
|
||||
asan_path, ir_path)
|
||||
asan_path, compile_input)
|
||||
opt_ret: int = stdlib.system(opt_cmd)
|
||||
if opt_ret != 0:
|
||||
VLogger.error(f"opt ASan 插桩失败 (module={module_name})", "ASAN")
|
||||
@@ -479,6 +539,9 @@ class LlcTask:
|
||||
# ExitCode 追加在结尾：既有字段偏移不变；LlcTask.__sizeof__() 随之增大，
|
||||
# 所有构造/初始化点均按 __sizeof__() 算首址并对整条 memset，无需改偏移。
|
||||
ExitCode: int # worker 写：失败命令退出码（成功为 0）
|
||||
# MLIR 后端降级命令（非 MLIR 为 None）；同样追加在结尾，不影响既有偏移。
|
||||
MlirOptCmd: bytes # mlir-opt 校验命令
|
||||
MlirTranslateCmd: bytes # mlir-translate 降级命令
|
||||
|
||||
|
||||
def init_llc_task(task: LlcTask | t.CPtr, ir_path: str, output_dir: str,
|
||||
@@ -501,20 +564,39 @@ def init_llc_task(task: LlcTask | t.CPtr, ir_path: str, output_dir: str,
|
||||
# compile_ll_to_obj 的 compile_input 同口径（否则 opt 生成了插桩产物
|
||||
# 却被 llc 忽略，App 侧 obj 里 0 处 __asan，内存排查半盲）
|
||||
compile_input: str = ir_path
|
||||
# ASan 模式：opt 插桩命令（worker 先跑 opt 再跑 llc，故 LlcCmd 的输入
|
||||
# MLIR 后端降级：worker 先跑 mlir-opt（校验）→ mlir-translate（降级回 LLVM IR），
|
||||
# 之后的 ASan OptCmd / LlcCmd 输入必须是 <...>.ll.local。
|
||||
if Config.is_mlir_backend() != 0:
|
||||
ll_local_path: str = _ir_sibling_path(ir_path, ".ll.local")
|
||||
mlir_opt_out: str = _ir_sibling_path(ir_path, ".mlir.opt")
|
||||
if ll_local_path is None or mlir_opt_out is None:
|
||||
return 1
|
||||
mo_len: sizet = len(MLIR_OPT_CMD) + len(ir_path) + len(mlir_opt_out) + 16
|
||||
task.MlirOptCmd = stdlib.malloc(mo_len)
|
||||
if task.MlirOptCmd is None:
|
||||
return 1
|
||||
viperlib.snprintf(task.MlirOptCmd, mo_len, "%s %s -o %s", MLIR_OPT_CMD, ir_path, mlir_opt_out)
|
||||
mt_len: sizet = len(MLIR_TRANSLATE_CMD) + len(mlir_opt_out) + len(ll_local_path) + 40
|
||||
task.MlirTranslateCmd = stdlib.malloc(mt_len)
|
||||
if task.MlirTranslateCmd is None:
|
||||
return 1
|
||||
viperlib.snprintf(task.MlirTranslateCmd, mt_len, "%s --mlir-to-llvmir %s -o %s",
|
||||
MLIR_TRANSLATE_CMD, mlir_opt_out, ll_local_path)
|
||||
compile_input = ll_local_path
|
||||
# ASan 模式：opt 插桩命令（worker 先跑 MLIR 降级/opt 再跑 llc，故 LlcCmd 的输入
|
||||
# 必须指向 asan_path；失败即置 Result）
|
||||
if _asan_mode != 0:
|
||||
asan_path: str = build_sliced_path(output_dir, module_name, "asan.ll")
|
||||
if asan_path is None:
|
||||
return 1
|
||||
opt_len: sizet = len(asan_path) + len(ir_path) + 192
|
||||
opt_len: sizet = len(asan_path) + len(compile_input) + 192
|
||||
task.OptCmd = stdlib.malloc(opt_len)
|
||||
if task.OptCmd is None:
|
||||
return 1
|
||||
viperlib.snprintf(task.OptCmd, opt_len,
|
||||
"opt -passes=forceattrs,asan -force-attribute=sanitize_address"
|
||||
" -asan-globals=false -o %s %s",
|
||||
asan_path, ir_path)
|
||||
asan_path, compile_input)
|
||||
compile_input = asan_path
|
||||
# llc 命令（输入 = compile_input：非 ASan 为 .ll，ASan 为 .asan.ll）
|
||||
llc_fp3: str = _llc_frame_ptr_flag()
|
||||
@@ -538,6 +620,12 @@ def free_llc_task(task: LlcTask | t.CPtr):
|
||||
if task.LlcCmd is not None:
|
||||
stdlib.free(task.LlcCmd)
|
||||
task.LlcCmd = None
|
||||
if task.MlirOptCmd is not None:
|
||||
stdlib.free(task.MlirOptCmd)
|
||||
task.MlirOptCmd = None
|
||||
if task.MlirTranslateCmd is not None:
|
||||
stdlib.free(task.MlirTranslateCmd)
|
||||
task.MlirTranslateCmd = None
|
||||
if task.ModuleSha1 is not None:
|
||||
stdlib.free(task.ModuleSha1)
|
||||
task.ModuleSha1 = None
|
||||
@@ -628,6 +716,19 @@ def _llc_task_worker(arg: t.CPtr) -> t.CPtr:
|
||||
task: LlcTask | t.CPtr = (LlcTask | t.CPtr)(arg)
|
||||
if task is None:
|
||||
return t.CPtr(0)
|
||||
# MLIR 后端：先校验（mlir-opt）→ 降级（mlir-translate），再 ASan opt → llc
|
||||
if task.MlirOptCmd is not None:
|
||||
mo_rc: int = _run_cmd_wait(task.MlirOptCmd)
|
||||
if mo_rc != 0:
|
||||
task.ExitCode = mo_rc
|
||||
task.Result = 1
|
||||
return t.CPtr(0)
|
||||
if task.MlirTranslateCmd is not None:
|
||||
mt_rc: int = _run_cmd_wait(task.MlirTranslateCmd)
|
||||
if mt_rc != 0:
|
||||
task.ExitCode = mt_rc
|
||||
task.Result = 1
|
||||
return t.CPtr(0)
|
||||
if task.OptCmd is not None:
|
||||
opt_rc: int = _run_cmd_wait(task.OptCmd)
|
||||
if opt_rc != 0:
|
||||
@@ -657,6 +758,10 @@ def run_llc_parallel(tasks: bytes, task_count: int, max_threads: int) -> int:
|
||||
tsk_p: LlcTask | t.CPtr = (LlcTask | t.CPtr)(uint64(tasks) + sizet(pi2) * LlcTask.__sizeof__())
|
||||
if tsk_p is None:
|
||||
continue
|
||||
if tsk_p.MlirOptCmd is not None:
|
||||
_phase2_write_cmd(tsk_p.MlirOptCmd)
|
||||
if tsk_p.MlirTranslateCmd is not None:
|
||||
_phase2_write_cmd(tsk_p.MlirTranslateCmd)
|
||||
if tsk_p.OptCmd is not None:
|
||||
_phase2_write_cmd(tsk_p.OptCmd)
|
||||
if tsk_p.LlcCmd is not None:
|
||||
|
||||
Reference in new issue
Block a user