From ef4582dcfc62502f24013ebd30de415d4f318c11 Mon Sep 17 00:00:00 2001 From: TermiNexus Date: Fri, 30 Oct 2026 18:42:55 +0800 Subject: [PATCH] build(phaseb): compile modules in-process through ircore.dll PhaseB spawned three child processes per module (mlir-opt, mlir-translate, llc), and the process startup dominated the cost: a 0.3 KB module took ~170 ms, of which ~150 ms was just starting the tools; the 59-module VKernel set cost 18.4 s CPU (3.3 s wall on 16 workers). ircore.dll (IRCore) does MLIR parse + verify + LLVM IR translation + codegen in a single call. PhaseB now uses it from the worker threads: - loaded at runtime with LoadLibraryA/GetProcAddress, not through the import table, so the compiler still starts when the DLL is missing; one retry points the loader at clang64's bin directory because ircore.dll needs that exact LLVM build - falls back to the legacy flow automatically (with a warning) when the DLL cannot load - `--phaseb-legacy` forces the legacy backend; the old command path is kept intact (compile_ll_to_obj / init_llc_task / _llc_task_worker / run_llc_parallel) - ASan instrumentation, raw-IR merging (llvm-link/opt), --traceback frame pointers and --phase2command keep using the external tools - new failure stage LLC_STAGE_IRCORE so a failure is attributed to the FFI call Measured: 59 VKernel modules 18.4 s -> 4.8 s CPU, 3.3 s -> 1.6 s wall; AstTest --rebuild-includes 13.6 s -> 11.3 s; BuddyTest 4.8 s -> 2.9 s; ViperC self-host 125 s -> 119 s (PhaseA dominates it); 41 Test projects pass, plus the 5 pre-existing failures. --- App/lib/Phase/BuildPipeline.vp | 189 ++++++++++++++++++++++++++++++++- App/main.vp | 8 ++ 2 files changed, 196 insertions(+), 1 deletion(-) diff --git a/App/lib/Phase/BuildPipeline.vp b/App/lib/Phase/BuildPipeline.vp index ffeb2c6..c275722 100644 --- a/App/lib/Phase/BuildPipeline.vp +++ b/App/lib/Phase/BuildPipeline.vp @@ -51,6 +51,78 @@ _phase2_bat_init: int = 0 MLIR_OPT_CMD: str = "mlir-opt" MLIR_TRANSLATE_CMD: str = "mlir-translate" +# --------------------------------------------------------------------------- +# PhaseB backend +# +# ircore.dll (IRCore) compiles one MLIR module to an object file in-process: parse + +# verify + LLVM IR translation + codegen in a single FFI call. That replaces the three +# child processes (mlir-opt / mlir-translate / llc). +# +# The legacy command flow is kept intact for rollback (see compile_ll_to_obj, +# init_llc_task, _llc_task_worker and run_llc_parallel): set _phaseb_backend to +# PHASEB_BACKEND_LEGACY and PhaseB runs exactly the old commands again. Modules that +# need the external tools (ASan instrumentation, raw-IR merging via llvm-link/opt, +# --phase2command, --traceback frame pointers) automatically take the legacy path. +# --------------------------------------------------------------------------- +define PHASEB_BACKEND_IRCORE = 1 +define PHASEB_BACKEND_LEGACY = 0 + +_phaseb_backend: int = PHASEB_BACKEND_IRCORE + +# IRCore FFI (C ABI, ircore.dll). Loaded at runtime with LoadLibraryA instead of the import +# table: the compiler must still start (and PhaseB must still work) when the DLL or the LLVM +# build it needs is absent -- in that case PhaseB falls back to the legacy command flow. +# +# The signature of ircore_mlir_to_obj: +# int ircore_mlir_to_obj(mlir, mlir_len, out_obj, out_obj_len, out_err, out_err_len) +# Returned out_obj/out_err buffers are owned by the DLL; release with ircore_free. +_ircore_lib: win32.win32base.HANDLE = None +_ircore_ready: int = 0 +_ircore_obj_fn: callable[[char*, sizet, char**, sizet*, char**, sizet*], int] = None +_ircore_free_fn: callable[[char*], int] = None + +# ircore.dll imports the exact LLVM build it was compiled with (MSYS2 clang64 22.1.x). +# When another LLVM build (e.g. MSYS2 mingw64's) comes first on PATH the load fails with +# 0xC0000139; point the loader at this directory and retry once. +define IRCORE_LLVM_DLL_DIR = "D:\\msys64\\clang64\\bin" + +extern def SetDllDirectoryA(lpPathName: LPCSTR) -> BOOL + + +def _ircore_load() -> int: + """Load ircore.dll and resolve its two entry points; 1 = available, 0 = not available.""" + global _ircore_lib, _ircore_obj_fn, _ircore_free_fn + if _ircore_lib is not None: return 1 + h: win32.win32base.HANDLE = win32.win32base.LoadLibraryA("ircore.dll") + if h is None: + # Retry with clang64's bin directory ahead of PATH (the LLVM build ircore.dll needs). + SetDllDirectoryA(IRCORE_LLVM_DLL_DIR) + h = win32.win32base.LoadLibraryA("ircore.dll") + if h is None: + return 0 + p_obj: win32.win32base.VOIDPTR = win32.win32base.GetProcAddress(h, "ircore_mlir_to_obj") + p_free: win32.win32base.VOIDPTR = win32.win32base.GetProcAddress(h, "ircore_free") + if p_obj is None or p_free is None: + win32.win32base.FreeLibrary(h) + return 0 + _ircore_obj_fn = p_obj + _ircore_free_fn = p_free + _ircore_lib = h + return 1 + + +def _phaseb_ircore_active() -> int: + """1 when PhaseB should use the in-process ircore backend; switches to legacy on failure.""" + global _phaseb_backend, _ircore_ready + if _phaseb_backend != PHASEB_BACKEND_IRCORE: return 0 + if _ircore_ready != 0: return 1 + if _ircore_load() == 0: + _phaseb_backend = PHASEB_BACKEND_LEGACY + VLogger.warning("ircore.dll is unavailable (or its LLVM runtime could not be loaded); PhaseB falls back to the legacy mlir-opt/mlir-translate/llc flow", "PhaseB") + return 0 + _ircore_ready = 1 + return 1 + # Raw-IR support (__llvmir__/__mlir__): raw LLVM function bodies are split into .raw.ll # and merged with the lowered module; always-inline then removes the call boundary so the raw # body behaves like inline IR again. @@ -436,6 +508,70 @@ def write_ir_to_file(ir_buf: bytes, ir_len: sizet, output_dir: str, module_name: return 0 +# _ircore_compile_module - PhaseB through ircore.dll: one in-process MLIR -> object call +# replaces mlir-opt + mlir-translate + llc. Returns 0 on success, non-zero on failure. +# The worker context: only stdlib.malloc buffers, no MemBuddy allocation. +def _ircore_compile_module(ir_path: str, obj_path: str, module_name: str) -> int: + if ir_path is None or obj_path is None: return -1 + + rf: io.File* = io.File(ir_path, io.MODE.R) + if rf.closed: + VLogger.warning(f"ircore: cannot open module {ir_path}", "PhaseB") + return -1 + fsize: LONGLONG = rf.size() + rf.close() + if fsize <= 0: + VLogger.warning(f"ircore: empty module {ir_path}", "PhaseB") + return -1 + + # read_all() keeps the last byte for the NUL, so ask for fsize+1 and expect fsize bytes. + src: bytes = stdlib.malloc(sizet(fsize) + 1) + if src is None: + VLogger.warning(f"ircore: module buffer allocation failed: {ir_path}", "PhaseB") + return -1 + rf = io.File(ir_path, io.MODE.R) + if rf.closed: + stdlib.free(src) + VLogger.warning(f"ircore: cannot reopen module {ir_path}", "PhaseB") + return -1 + got: LONG = rf.read_all(src, sizet(fsize) + 1) + rf.close() + if got != LONG(fsize): + stdlib.free(src) + VLogger.warning(f"ircore: short read on {ir_path} ({got}/{fsize})", "PhaseB") + return -1 + + out_obj: char* = None + out_len: sizet = 0 + err: char* = None + err_len: sizet = 0 + rc: int = _ircore_obj_fn(src, sizet(fsize), &out_obj, &out_len, &err, &err_len) + stdlib.free(src) + + if rc != 0: + # The DLL's diagnostic is the authoritative message (MLIR verifier or LLVM codegen). + if err is not None: + VLogger.warning(f"ircore MLIR->obj failed (module={module_name}): {err}", "PhaseB") + _ircore_free_fn(err) + else: + VLogger.warning(f"ircore MLIR->obj failed (module={module_name}): no diagnostic", "PhaseB") + if out_obj is not None: _ircore_free_fn(out_obj) + return 1 + + wf: io.File* = io.File(obj_path, io.MODE.W) + if wf.closed: + VLogger.warning(f"ircore: cannot create object {obj_path}", "PhaseB") + _ircore_free_fn(out_obj) + return -1 + wr: LONG = wf.write(out_obj, out_len) + wf.close() + _ircore_free_fn(out_obj) + if wr != LONG(out_len): + VLogger.warning(f"ircore: short write on {obj_path} ({wr}/{out_len})", "PhaseB") + return -1 + return 0 + + def compile_ll_to_obj(ir_path: str, output_dir: str, module_name: str, cc_cmd: str, cc_flags: str) -> int: """Compile a .ll file to .obj via llc; 0 on success, non-zero on failure.""" if ir_path is None or cc_cmd is None or output_dir is None or module_name is None: return 1 @@ -444,6 +580,16 @@ def compile_ll_to_obj(ir_path: str, output_dir: str, module_name: str, cc_cmd: s obj_path: str = build_sliced_path(output_dir, module_name, "obj") if obj_path is None: return 1 + # IRCore backend: one in-process call for plain modules. ASan instrumentation, raw-IR + # merging (llvm-link/opt), --traceback frame pointers and --phase2command still need the + # external tools, so they fall through to the legacy command flow below (kept intact). + if _phaseb_ircore_active() != 0 and _asan_mode == 0 and _traceback_mode == 0 \ + and _phase2_cmd_path is None: + raw_sidecar: str = _ir_sibling_path(ir_path, ".raw.ll") + inline_marker: str = _ir_sibling_path(ir_path, ".inline") + if _file_exists(raw_sidecar) == 0 and _file_exists(inline_marker) == 0: + return _ircore_compile_module(ir_path, obj_path, module_name) + # ASan mode: run opt -passes=asan before llc. -asan-globals=false disables global # instrumentation: VPC addresses globals directly and ASan redzones would shift them. compile_input: str = ir_path @@ -583,6 +729,7 @@ define LLC_STAGE_ASAN_OPT = 3 define LLC_STAGE_LLC = 4 define LLC_STAGE_RAW_LINK = 5 define LLC_STAGE_RAW_INLINE = 6 +define LLC_STAGE_IRCORE = 7 class LlcTask: @@ -602,6 +749,12 @@ class LlcTask: # Raw-IR support (__llvmir__/__mlir__); appended at the end, None when the module has none. RawLinkCmd: bytes # llvm-link merge of .raw.ll into the lowered module RawInlineCmd: bytes # opt -passes=always-inline,globaldce (raw body -> inline) + # IRCore backend (PHASEB_BACKEND_IRCORE): when UseIrcore=1 the worker ignores every + # command field above and compiles IrPath -> ObjPath in-process. The commands stay + # allocated (error reporting / dead legacy path) and are never executed. + UseIrcore: int + IrPath: bytes + ObjPath: bytes def init_llc_task(task: LlcTask*, ir_path: str, output_dir: str, module_name: str, cc_cmd: str, cc_flags: str) -> int: @@ -675,6 +828,19 @@ def init_llc_task(task: LlcTask*, ir_path: str, output_dir: str, module_name: st task.LlcCmd = stdlib.malloc(cmd_len) if task.LlcCmd is None: return 1 viperlib.snprintf(task.LlcCmd, cmd_len, "%s %s%s -o %s %s", cc_cmd, cc_flags, llc_fp3, obj_path, compile_input) + + # IRCore backend: plain modules compile in-process (the commands above stay allocated for + # the legacy path / reporting but are not run). ASan instrumentation, raw-IR merging and + # --phase2command keep the external tools. + if _phaseb_ircore_active() != 0 and _asan_mode == 0 and _traceback_mode == 0 \ + and _phase2_cmd_path is None and has_raw == 0 and need_inline == 0: + task.UseIrcore = 1 + task.IrPath = stdlib.malloc(len(ir_path) + 1) + if task.IrPath is None: return 1 + string.strcpy(task.IrPath, ir_path) + task.ObjPath = stdlib.malloc(len(obj_path) + 1) + if task.ObjPath is None: return 1 + string.strcpy(task.ObjPath, obj_path) return 0 @@ -702,6 +868,12 @@ def free_llc_task(task: LlcTask*): if task.ModuleSha1 is not None: stdlib.free(task.ModuleSha1) task.ModuleSha1 = None + if task.IrPath is not None: + stdlib.free(task.IrPath) + task.IrPath = None + if task.ObjPath is not None: + stdlib.free(task.ObjPath) + task.ObjPath = None def _phase2_write_cmd(cmd: str) -> None: @@ -764,7 +936,19 @@ def _llc_task_worker(arg: rawptr) -> rawptr: """llc thread entry: only calls _run_cmd_wait, no pool allocation (thread-safe).""" task: LlcTask* = (LlcTask*)(arg) if task is None: return None - # MLIR backend: validate (mlir-opt) -> lower (mlir-translate), then ASan opt -> llc + # IRCore backend: one in-process call replaces the whole command sequence below. + if task.UseIrcore != 0: + ir_rc: int = _ircore_compile_module(task.IrPath, task.ObjPath, task.ModuleSha1) + task.ExitCode = ir_rc + if ir_rc != 0: + task.FailStage = LLC_STAGE_IRCORE + task.Result = 1 + else: + task.FailStage = LLC_STAGE_NONE + task.Result = 0 + return None + # Legacy backend (rollback path): validate (mlir-opt) -> lower (mlir-translate), then + # ASan opt -> llc if task.MlirOptCmd is not None: mo_rc: int = _run_cmd_wait(task.MlirOptCmd) if mo_rc != 0: @@ -880,6 +1064,9 @@ def run_llc_parallel(tasks: bytes, task_count: int, max_threads: int) -> int: elif st == LLC_STAGE_LLC: stage_name = "llc compile" stage_cmd = tsk2.LlcCmd + elif st == LLC_STAGE_IRCORE: + stage_name = "ircore MLIR->obj (in-process)" + stage_cmd = tsk2.IrPath VLogger.error(f"{stage_name} failed: {(tsk2.ModuleSha1 if tsk2.ModuleSha1 is not None else "?")} (exit={tsk2.ExitCode} cmd={(stage_cmd if stage_cmd is not None else "?")})", "LLC") ret = 1 return ret diff --git a/App/main.vp b/App/main.vp index 98fbf4b..8a4b865 100644 --- a/App/main.vp +++ b/App/main.vp @@ -177,6 +177,7 @@ def main(argc: int, argv: []str) -> int: parser.add_argument("--asan", None, argparse.BOOL, 0, None, False, argparse.STORE_TRUE, "ASan mode: .ll first opt -passes=asan; ASan runtime libs must be declared in project.vpj (linker.static_libs)") parser.add_argument("--traceback", None, argparse.BOOL, 0, None, False, argparse.STORE_TRUE, "Traceback mode: force-compile includes/traceback.vp + Install() ctor, llc -frame-pointer=all, link -ldbghelp, export .stackmap") parser.add_argument("--phase2command", None, argparse.STRING, 0, None, False, argparse.STORE, "Do not run llc/link: write Phase2 llc(opt+llc) and clang link commands to the given .bat path") + parser.add_argument("--phaseb-legacy", None, argparse.BOOL, 0, None, False, argparse.STORE_TRUE, "PhaseB: use the legacy mlir-opt/mlir-translate/llc child-process flow instead of the in-process ircore.dll backend") parser.add_argument("--nowarn-register", None, argparse.BOOL, 0, None, False, argparse.STORE_TRUE, "Suppress Phase A dependency WARN") parser.add_argument("--detection", None, argparse.BOOL, 0, None, False, argparse.STORE_TRUE, "Enable MemBuddy allocation detection (0xCDCD markers + record-table probe for out-of-bounds writes; applies to both mb and perm_pool)") # Registered only so argparse accepts the flag: the value is read via raw @@ -329,6 +330,13 @@ Target: %s else: BuildPipeline._asan_mode = False if gb_traceback: BuildPipeline._traceback_mode = True else: BuildPipeline._traceback_mode = False + # PhaseB backend: ircore.dll (single in-process MLIR -> obj call) or the legacy + # mlir-opt/mlir-translate/llc child-process flow (rollback switch). + if args.get_bool("phaseb-legacy"): + BuildPipeline._phaseb_backend = BuildPipeline.PHASEB_BACKEND_LEGACY + log.info("PhaseB: legacy backend (mlir-opt + mlir-translate + llc child processes)", "phaseb") + else: + BuildPipeline._phaseb_backend = BuildPipeline.PHASEB_BACKEND_IRCORE if gb_asan and gb_traceback: Config._inc_bin_suffix = ".binary_traceback_asan" elif gb_traceback: Config._inc_bin_suffix = ".binary_traceback" elif gb_asan: Config._inc_bin_suffix = ".binary_asan"