diff --git a/App/lib/Phase/BuildPipeline.vp b/App/lib/Phase/BuildPipeline.vp index 9757226..7ab4804 100644 --- a/App/lib/Phase/BuildPipeline.vp +++ b/App/lib/Phase/BuildPipeline.vp @@ -53,6 +53,34 @@ _phase2_cmd_path: str = None # bat 文件是否已写入 header(@echo off),首次写入时置 1 _phase2_bat_init: int = 0 +# MLIR 后端(Config.is_mlir_backend())降级步骤:mlirlite 发射的是 MLIR 文本, +# 而 opt/llc 只接受 LLVM IR,故在两者之前插入: +# mlir-opt -o .mlir.opt (语法/语义校验) +# mlir-translate --mlir-to-llvmir .mlir.opt -o .ll.local (降级回 LLVM IR) +# 降级产物 <...>.ll.local 与输入 .ll 同目录,不占用 .ll 缓存键。 +MLIR_OPT_CMD: str = "mlir-opt" +MLIR_TRANSLATE_CMD: str = "mlir-translate" + + +def _ir_sibling_path(ir_path: str, suffix: str) -> str: + """由 <...>.ll 派生同目录邻居路径:去掉末尾 ".ll" 后拼 suffix(如 ".mlir.opt")。 + + 从 _mbuddy 分配(与 build_sliced_path 同池,随 pool reset 回收,无需手动 free)。 + """ + if ir_path is None or suffix is None: + return None + n: sizet = string.strlen(ir_path) + if n < 3: + return None + suf_len: sizet = string.strlen(suffix) + buf: str = _mbuddy.alloc(n + suf_len + 1) + if buf is None: + return None + string.strncpy(buf, ir_path, n - 3) + buf[n - 3] = '\0' + string.strcat(buf, suffix) + return buf + # 源代码缓冲区大小(1MB) SRC_BUF_SIZE: t.CDefine = 1048576 # 2^20 = 1MB @@ -398,18 +426,50 @@ def compile_ll_to_obj(ir_path: str, output_dir: str, module_name: str, cc_cmd: s # 用直接地址寻址,ASan 全局 redzone 重定位后引用错位会读到垃圾指针。 compile_input: str = ir_path + # ---- MLIR 后端降级:mlir-opt(校验)→ mlir-translate(降级回 LLVM IR)---- + # 必须在 ASan opt 与 llc 之前,并把 compile_input 替换为 <...>.ll.local。 + if Config.is_mlir_backend() != 0: + ll_local_path: str = _ir_sibling_path(ir_path, ".ll.local") + mlir_opt_out: str = _ir_sibling_path(ir_path, ".mlir.opt") + if ll_local_path is None or mlir_opt_out is None: + return 1 + mo_len: sizet = len(MLIR_OPT_CMD) + len(ir_path) + len(mlir_opt_out) + 16 + cmd_mo: str = _mbuddy.alloc(mo_len) + if cmd_mo is None: + return 1 + viperlib.snprintf(cmd_mo, mo_len, "%s %s -o %s", MLIR_OPT_CMD, ir_path, mlir_opt_out) + mt_len: sizet = len(MLIR_TRANSLATE_CMD) + len(mlir_opt_out) + len(ll_local_path) + 40 + cmd_mt: str = _mbuddy.alloc(mt_len) + if cmd_mt is None: + return 1 + viperlib.snprintf(cmd_mt, mt_len, "%s --mlir-to-llvmir %s -o %s", MLIR_TRANSLATE_CMD, mlir_opt_out, ll_local_path) + if _phase2_cmd_path is not None: + # 只写命令(须排在下面的 ASan/llc 命令之前) + _phase2_write_cmd(cmd_mo) + _phase2_write_cmd(cmd_mt) + else: + mo_rc: int = _run_cmd_wait(cmd_mo) + if mo_rc != 0: + VLogger.error(f"mlir-opt 失败 (module={module_name}, exit={mo_rc}, cmd={cmd_mo})", "MLIR") + return 1 + mt_rc: int = _run_cmd_wait(cmd_mt) + if mt_rc != 0: + VLogger.error(f"mlir-translate 失败 (module={module_name}, exit={mt_rc}, cmd={cmd_mt})", "MLIR") + return 1 + compile_input = ll_local_path + # --phase2command 模式:不执行 opt/llc,把 opt(ASan)+ llc 命令写入 bat if _phase2_cmd_path is not None: p2_opt_cmd: str = None if _asan_mode != 0: p2_asan: str = build_sliced_path(output_dir, module_name, "asan.ll") if p2_asan is not None: - p2_olen: sizet = len(p2_asan) + len(ir_path) + 192 + p2_olen: sizet = len(p2_asan) + len(compile_input) + 192 p2_opt_cmd = _mbuddy.alloc(p2_olen) if p2_opt_cmd is not None: viperlib.snprintf(p2_opt_cmd, p2_olen, "opt -passes=forceattrs,asan -force-attribute=sanitize_address -asan-globals=false -o %s %s", - p2_asan, ir_path) + p2_asan, compile_input) compile_input = p2_asan llc_fp: str = _llc_frame_ptr_flag() p2_clen: sizet = len(cc_cmd) + len(cc_flags) + len(llc_fp) + len(obj_path) + len(compile_input) + 64 @@ -427,13 +487,13 @@ def compile_ll_to_obj(ir_path: str, output_dir: str, module_name: str, cc_cmd: s asan_path: str = build_sliced_path(output_dir, module_name, "asan.ll") if asan_path is None: return 1 - opt_cmd_len: sizet = len(asan_path) + len(ir_path) + 192 + opt_cmd_len: sizet = len(asan_path) + len(compile_input) + 192 opt_cmd: bytes = _mbuddy.alloc(opt_cmd_len) if opt_cmd is None: return 1 viperlib.snprintf(opt_cmd, opt_cmd_len, "opt -passes=forceattrs,asan -force-attribute=sanitize_address -asan-globals=false -o %s %s", - asan_path, ir_path) + asan_path, compile_input) opt_ret: int = stdlib.system(opt_cmd) if opt_ret != 0: VLogger.error(f"opt ASan 插桩失败 (module={module_name})", "ASAN") @@ -479,6 +539,9 @@ class LlcTask: # ExitCode 追加在结尾:既有字段偏移不变;LlcTask.__sizeof__() 随之增大, # 所有构造/初始化点均按 __sizeof__() 算首址并对整条 memset,无需改偏移。 ExitCode: int # worker 写:失败命令退出码(成功为 0) + # MLIR 后端降级命令(非 MLIR 为 None);同样追加在结尾,不影响既有偏移。 + MlirOptCmd: bytes # mlir-opt 校验命令 + MlirTranslateCmd: bytes # mlir-translate 降级命令 def init_llc_task(task: LlcTask | t.CPtr, ir_path: str, output_dir: str, @@ -501,20 +564,39 @@ def init_llc_task(task: LlcTask | t.CPtr, ir_path: str, output_dir: str, # compile_ll_to_obj 的 compile_input 同口径(否则 opt 生成了插桩产物 # 却被 llc 忽略,App 侧 obj 里 0 处 __asan,内存排查半盲) compile_input: str = ir_path - # ASan 模式:opt 插桩命令(worker 先跑 opt 再跑 llc,故 LlcCmd 的输入 + # MLIR 后端降级:worker 先跑 mlir-opt(校验)→ mlir-translate(降级回 LLVM IR), + # 之后的 ASan OptCmd / LlcCmd 输入必须是 <...>.ll.local。 + if Config.is_mlir_backend() != 0: + ll_local_path: str = _ir_sibling_path(ir_path, ".ll.local") + mlir_opt_out: str = _ir_sibling_path(ir_path, ".mlir.opt") + if ll_local_path is None or mlir_opt_out is None: + return 1 + mo_len: sizet = len(MLIR_OPT_CMD) + len(ir_path) + len(mlir_opt_out) + 16 + task.MlirOptCmd = stdlib.malloc(mo_len) + if task.MlirOptCmd is None: + return 1 + viperlib.snprintf(task.MlirOptCmd, mo_len, "%s %s -o %s", MLIR_OPT_CMD, ir_path, mlir_opt_out) + mt_len: sizet = len(MLIR_TRANSLATE_CMD) + len(mlir_opt_out) + len(ll_local_path) + 40 + task.MlirTranslateCmd = stdlib.malloc(mt_len) + if task.MlirTranslateCmd is None: + return 1 + viperlib.snprintf(task.MlirTranslateCmd, mt_len, "%s --mlir-to-llvmir %s -o %s", + MLIR_TRANSLATE_CMD, mlir_opt_out, ll_local_path) + compile_input = ll_local_path + # ASan 模式:opt 插桩命令(worker 先跑 MLIR 降级/opt 再跑 llc,故 LlcCmd 的输入 # 必须指向 asan_path;失败即置 Result) if _asan_mode != 0: asan_path: str = build_sliced_path(output_dir, module_name, "asan.ll") if asan_path is None: return 1 - opt_len: sizet = len(asan_path) + len(ir_path) + 192 + opt_len: sizet = len(asan_path) + len(compile_input) + 192 task.OptCmd = stdlib.malloc(opt_len) if task.OptCmd is None: return 1 viperlib.snprintf(task.OptCmd, opt_len, "opt -passes=forceattrs,asan -force-attribute=sanitize_address" " -asan-globals=false -o %s %s", - asan_path, ir_path) + asan_path, compile_input) compile_input = asan_path # llc 命令(输入 = compile_input:非 ASan 为 .ll,ASan 为 .asan.ll) llc_fp3: str = _llc_frame_ptr_flag() @@ -538,6 +620,12 @@ def free_llc_task(task: LlcTask | t.CPtr): if task.LlcCmd is not None: stdlib.free(task.LlcCmd) task.LlcCmd = None + if task.MlirOptCmd is not None: + stdlib.free(task.MlirOptCmd) + task.MlirOptCmd = None + if task.MlirTranslateCmd is not None: + stdlib.free(task.MlirTranslateCmd) + task.MlirTranslateCmd = None if task.ModuleSha1 is not None: stdlib.free(task.ModuleSha1) task.ModuleSha1 = None @@ -628,6 +716,19 @@ def _llc_task_worker(arg: t.CPtr) -> t.CPtr: task: LlcTask | t.CPtr = (LlcTask | t.CPtr)(arg) if task is None: return t.CPtr(0) + # MLIR 后端:先校验(mlir-opt)→ 降级(mlir-translate),再 ASan opt → llc + if task.MlirOptCmd is not None: + mo_rc: int = _run_cmd_wait(task.MlirOptCmd) + if mo_rc != 0: + task.ExitCode = mo_rc + task.Result = 1 + return t.CPtr(0) + if task.MlirTranslateCmd is not None: + mt_rc: int = _run_cmd_wait(task.MlirTranslateCmd) + if mt_rc != 0: + task.ExitCode = mt_rc + task.Result = 1 + return t.CPtr(0) if task.OptCmd is not None: opt_rc: int = _run_cmd_wait(task.OptCmd) if opt_rc != 0: @@ -657,6 +758,10 @@ def run_llc_parallel(tasks: bytes, task_count: int, max_threads: int) -> int: tsk_p: LlcTask | t.CPtr = (LlcTask | t.CPtr)(uint64(tasks) + sizet(pi2) * LlcTask.__sizeof__()) if tsk_p is None: continue + if tsk_p.MlirOptCmd is not None: + _phase2_write_cmd(tsk_p.MlirOptCmd) + if tsk_p.MlirTranslateCmd is not None: + _phase2_write_cmd(tsk_p.MlirTranslateCmd) if tsk_p.OptCmd is not None: _phase2_write_cmd(tsk_p.OptCmd) if tsk_p.LlcCmd is not None: diff --git a/App/lib/Projectrans/Config.vp b/App/lib/Projectrans/Config.vp index 436b7fa..086509c 100644 --- a/App/lib/Projectrans/Config.vp +++ b/App/lib/Projectrans/Config.vp @@ -43,6 +43,17 @@ IncludesDirs: str = None # 全部 includes 目录(空格分隔,多 # includes 预编译缓存目录后缀(asan 模式用 .binary_asan 隔离 asan 插桩 obj,避免与非 asan obj 混链) _inc_bin_suffix: str = ".binary" +# ============================================================ +# 后端(Emitter Backend) +# +# "llvm"(默认):IR 发射走 includes/llvmlite(LLVM IR 文本),opt/llc 直接消费。 +# "mlir" :模块名 llvmlite 在解析层重定向到 includes/mlirlite(MLIR 文本), +# 并在 opt/llc 之前插入 mlir-opt(校验)→ mlir-translate(降级回 LLVM IR)。 +# 只认字符串 "mlir":缺省 / 未知值一律按 LLVM(BackendIsMlir=0),保证默认路径逐字不变。 +# ============================================================ +BackendTarget: str = "llvm" +BackendIsMlir: int = 0 + def Load_project_config(path: str) -> int: """加载工程配置文件(project.vpj) @@ -145,6 +156,16 @@ def Load_project_config(path: str) -> int: sm: JsonValue | t.CPtr = options["strict_mode"] if sm is not None and sm.is_bool(): StrictMode = 1 if sm.as_bool() else 0 + # 后端:options.backend(新键)优先,回退 options.target(兼容旧键)。 + # 只认 "mlir";其余(含缺省/未知)按 LLVM。CLI --backend 由 main.vp + # 在本函数返回后覆盖(CLI 优先级最高)。 + be_val: JsonValue | t.CPtr = options["backend"] + if be_val is not None and be_val.is_string(): + set_backend(be_val.as_string()) + else: + tgt_be: JsonValue | t.CPtr = options["target"] + if tgt_be is not None and tgt_be.is_string(): + set_backend(tgt_be.as_string()) # 读取 includes 数组(支持多目录:首元素存 IncludesDir,全部拼接存 IncludesDirs) includes_val: JsonValue | t.CPtr = root["includes"] @@ -424,17 +445,19 @@ def resolve_paths(project_dir: str) -> int: SourceDir = _join_path(SourceDir, project_dir) BuildDir = _join_path(BuildDir, project_dir) - # TempDir = {BuildDir}/temp + # TempDir = {BuildDir}/temp(非 LLVM 后端追加隔离后缀,如 _mlir;LLVM 时后缀为空串,路径逐字不变) if BuildDir is not None: bd_len: sizet = string.strlen(BuildDir) - temp_buf: str = _mbuddy.alloc(bd_len + 8) + suf_dir: str = backend_dir_suffix() + suf_dir_len: sizet = string.strlen(suf_dir) + temp_buf: str = _mbuddy.alloc(bd_len + suf_dir_len + 8) if temp_buf is not None: - viperlib.snprintf(temp_buf, bd_len + 8, "%s/temp", BuildDir) + viperlib.snprintf(temp_buf, bd_len + suf_dir_len + 8, "%s/temp%s", BuildDir, suf_dir) TempDir = temp_buf # OutputDir = {BuildDir}/output - output_buf: str = _mbuddy.alloc(bd_len + 8) + output_buf: str = _mbuddy.alloc(bd_len + suf_dir_len + 8) if output_buf is not None: - viperlib.snprintf(output_buf, bd_len + 8, "%s/output", BuildDir) + viperlib.snprintf(output_buf, bd_len + suf_dir_len + 8, "%s/output%s", BuildDir, suf_dir) OutputDir = output_buf # includes 路径可能是 ../includes 形式,需要基于 project_dir 解析 @@ -638,18 +661,71 @@ def get_includes_binary_dir() -> str: includes 目录如 ../includes,默认返回 ../includes.binary; asan 模式(set_inc_bin_suffix(".binary_asan"))返回 ../includes.binary_asan, 隔离 asan 插桩 obj,避免与非 asan obj 混链。 + 非 LLVM 后端再追加后端后缀(如 .binary_asan_mlir),隔离不同后端的 obj。 """ if IncludesDir is None: return None inc_len: sizet = string.strlen(IncludesDir) suf_len: sizet = string.strlen(_inc_bin_suffix) - buf: str = _mbuddy.alloc(inc_len + suf_len + 1) + be_suf: str = backend_dir_suffix() + be_suf_len: sizet = string.strlen(be_suf) + buf: str = _mbuddy.alloc(inc_len + suf_len + be_suf_len + 1) if buf is None: return None - viperlib.snprintf(buf, inc_len + suf_len + 1, "%s%s", IncludesDir, _inc_bin_suffix) + viperlib.snprintf(buf, inc_len + suf_len + be_suf_len + 1, "%s%s%s", IncludesDir, _inc_bin_suffix, be_suf) return buf +# ============================================================ +# 后端开关(Emitter Backend) +# +# set_backend: 只接受 "llvm" / "mlir"(其余返回 1 且不改动),大小写敏感。 +# backend_dir_suffix: 非默认后端的目录隔离后缀;LLVM 后端返回空串(调用方拼出的 +# 路径与改前逐字相同)。用于 TempDir/OutputDir/includes.binary。 +# is_mlir_backend: 0/1 便捷视图(BuildPipeline 降级管线的门)。 +# ============================================================ +def set_backend(name: str) -> int: + """设置后端;0=成功,1=非法(name 为 None 或非 llvm/mlir)""" + global BackendTarget, BackendIsMlir + if name is None: + return 1 + if string.strcmp(name, "llvm") == 0: + BackendTarget = name + BackendIsMlir = 0 + return 0 + if string.strcmp(name, "mlir") == 0: + BackendTarget = name + BackendIsMlir = 1 + return 0 + return 1 + + +def is_mlir_backend() -> int: + """是否 MLIR 后端(1=mlir / 0=llvm)""" + return BackendIsMlir + + +def backend_dir_suffix() -> str: + """非默认后端的目录隔离后缀(LLVM 后端返回空串 ⇒ 路径逐字不变)""" + if BackendIsMlir != 0: + return "_mlir" + return "" + + +def alias_module_name(name: str) -> str: + """MLIR 后端下把模块名 "llvmlite" 重定向到 "mlirlite"(其余原样返回)。 + + 这是"TransPyV 调用点零改动"的唯一开关:所有 **名称→模块/文件/SHA1** 的解析 + 都应经此函数(模块文件解析 IncludesScanner.ResolveModuleToPath、依赖名→SHA1 + StubMergerIR._FindSha1ByModName 等)。LLVM 后端恒等返回,默认路径逐字不变。 + """ + if name is None: + return None + if BackendIsMlir != 0 and string.strcmp(name, "llvmlite") == 0: + return "mlirlite" + return name + + def asan_runtime_declared() -> int: """project.vpj 是否已声明 ASan 运行库(1 = 声明过) diff --git a/App/lib/Stub/StubMerger/StubMergerIR.vp b/App/lib/Stub/StubMerger/StubMergerIR.vp index 8c8ead0..68df257 100644 --- a/App/lib/Stub/StubMerger/StubMergerIR.vp +++ b/App/lib/Stub/StubMerger/StubMergerIR.vp @@ -335,6 +335,11 @@ def _FindSha1ByModName(sha1_arr: str, mod_arr: str, return None if count <= 0: return None + # MLIR 后端:依赖名 "llvmlite" 重定向到 "mlirlite" + #(与 IncludesScanner.ResolveModuleToPath 同口径;LLVM 后端恒等) + mod_name = Config.alias_module_name(mod_name) + if mod_name is None: + return None # 第一遍:精确匹配 for i in range(count): idx: sizet = sizet(i) * 64 diff --git a/App/lib/core/IncludesScanner.vp b/App/lib/core/IncludesScanner.vp index f73102b..9774a28 100644 --- a/App/lib/core/IncludesScanner.vp +++ b/App/lib/core/IncludesScanner.vp @@ -10,6 +10,7 @@ import hashlib import viperlib import lib.Projectrans.Utils as Utils import lib.core.VLogger as VLogger +import lib.Projectrans.Config as Config # ============================================================ @@ -452,6 +453,12 @@ def ResolveModuleToPath(includes_dirs: str, mod_name: str, out_inc_dir: t.CPtr) -> int: if includes_dirs is None or mod_name is None: return 1 + # MLIR 后端:模块名 "llvmlite" 重定向到 "mlirlite"(TransPyV 调用点零改动)。 + # 重定向后的名字用于本次解析的路径段/相对路径,调用方拿到的 rel_path + # 即 "mlirlite/__init__.vp" ⇒ 下游模块身份/SHA1 全部随之一致。 + mod_name = Config.alias_module_name(mod_name) + if mod_name is None: + return 1 mn_len: sizet = string.strlen(mod_name) if mn_len == 0: return 1 diff --git a/App/main.vp b/App/main.vp index 8c7725c..19bae7a 100644 --- a/App/main.vp +++ b/App/main.vp @@ -162,6 +162,7 @@ def main() -> int: parser.add_argument("--repl", None, argparse.BOOL, 0, None, False, argparse.STORE_TRUE, "Start the interactive REPL (M2 session variables + M3 cross-fragment functions: fragment -> DLL -> call)") parser.add_argument("--build-dir", None, argparse.STRING, 0, None, False, argparse.STORE, "Override project.vpj build_dir (default ./.tpv_build; relative paths resolve against the project dir)") parser.add_argument("--no-llvmpeephole", None, argparse.BOOL, 0, None, False, argparse.STORE_TRUE, "Disable all local llvmlite peephole passes (T2 store/load forward & ret-zero, T3 no-op bitcast) in FunctionPrint serialization") + parser.add_argument("--backend", None, argparse.STRING, 0, None, False, argparse.STORE, "Emitter backend: llvm (default) | mlir. mlir redirects module 'llvmlite' to includes/mlirlite and inserts mlir-opt + mlir-translate before opt/llc") parser.add_argument("--version", "-v", argparse.BOOL, 0, None, False, argparse.STORE_TRUE, "Show version and exit") parser.add_argument("--help", "-h", argparse.BOOL, 0, None, False, argparse.STORE_TRUE, "Show help and exit") @@ -369,6 +370,17 @@ Target: %s proj_path = "../project.vpj" if Config.Load_project_config(proj_path) == 0: proj_loaded = 1 + # === --backend: CLI 覆盖 project.vpj 的 options.backend(CLI 优先级最高)=== + # 位置:工程配置加载之后(options.backend 已读入)、resolve_paths 之前 + #(TempDir/OutputDir 的后端隔离后缀由此定型);非法值 fail-fast。 + gb_backend: str = args.get_str("backend") + if gb_backend is not None: + if Config.set_backend(gb_backend) != 0: + log.error(f"非法的 --backend 值: '{gb_backend}'(只支持 llvm / mlir)", "backend") + argparse.release(args) + return 1 + log.info(f"后端: {Config.BackendTarget}", "backend") + if proj_loaded == 1: # 提取 project.vpj 所在目录,将相对路径解析为基于该目录的路径 proj_len: sizet = string.strlen(proj_path)