From d206abf3e1fee540e1e37c230451b4643b7260b8 Mon Sep 17 00:00:00 2001 From: TermiNexus Date: Mon, 12 Oct 2026 20:59:11 +0800 Subject: [PATCH] =?UTF-8?q?chore:=20=E5=85=A8=E4=BB=93=E5=90=8C=E6=AD=A5?= =?UTF-8?q?=20w32=E2=86=92win32=20=E6=A8=A1=E5=9D=97=E6=94=B9=E5=90=8D?= =?UTF-8?q?=E4=B8=8E=20range(int(...))=20=E6=98=BE=E5=BC=8F=E6=94=B6?= =?UTF-8?q?=E7=AA=84=E9=80=82=E9=85=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- App/lib/Phase/Phase1/Phase1Main.vp | 68 +- App/lib/Phase/Phase1/Phase1Scan.vp | 38 +- App/lib/Phase/Phase1/Phase1Topo.vp | 80 +- App/lib/Phase/Phase2/__init__.vp | 3074 +++++----- App/lib/Projectrans/Config.vp | 1200 ++-- App/lib/Projectrans/Utils.vp | 34 +- App/lib/Stub/StubGen/Converter.vp | 2008 +++---- App/lib/Stub/StubMerger/StubMergerBuild.vp | 10 +- App/lib/Stub/StubMerger/StubMergerCollect.vp | 250 +- App/lib/Stub/StubMerger/StubMergerIR.vp | 14 +- App/lib/Stub/StubMerger/StubMergerStore.vp | 30 +- App/lib/Stub/StubMerger/StubMergerTopo.vp | 34 +- App/lib/Stub/StubMerger/__init__.vp | 6 +- App/lib/core/Diag.vp | 10 +- App/lib/core/Gen/Call/CallGenBase.vp | 238 +- App/lib/core/Gen/Call/CallGenCBuiltin.vp | 42 +- App/lib/core/Gen/Call/CallGenCall.vp | 32 +- App/lib/core/Gen/Call/CallGenCtor.vp | 1930 +++---- App/lib/core/Gen/Call/CallGenDispatch.vp | 3054 +++++----- App/lib/core/Gen/Call/CallGenFd.vp | 8 +- App/lib/core/Gen/Call/CallGenLeafKit.vp | 40 +- App/lib/core/Gen/Call/CallGenLib.vp | 4 +- App/lib/core/Gen/Call/CallGenMixin.vp | 2836 +++++----- App/lib/core/Gen/Call/CallGenProp.vp | 4 +- App/lib/core/Gen/Call/CallGenVt.vp | 2 +- App/lib/core/Gen/Class/ClassGenBase.vp | 64 +- App/lib/core/Gen/Class/ClassGenDef.vp | 12 +- App/lib/core/Gen/Class/ClassGenEnum.vp | 79 +- App/lib/core/Gen/Class/ClassGenGeneric.vp | 64 +- App/lib/core/Gen/DebugInfoGen.vp | 8 +- App/lib/core/Gen/Expr/ExprGenCls.vp | 18 +- App/lib/core/Gen/Expr/ExprGenFstr.vp | 4 +- App/lib/core/Gen/Func/FuncGenBase.vp | 582 +- App/lib/core/Gen/ImportsGenMixin.vp | 26 +- App/lib/core/Gen/MatchGenMixin.vp | 2994 +++++----- App/lib/core/Gen/Stmt/StmtGenCifMixin.vp | 65 +- App/lib/core/Gen/Stmt/StmtGenDeclMixin.vp | 40 +- App/lib/core/Gen/Stmt/StmtGenMixin.vp | 26 +- App/lib/core/Gen/StructGenMixin.vp | 4210 +++++++------- App/lib/core/Gen/Translator.vp | 2016 +++---- App/lib/core/Gen/Type/TypeGenCType.vp | 1093 ++-- App/lib/core/Gen/Type/TypeGenErr.vp | 12 +- App/lib/core/Gen/Type/TypeGenMixin.vp | 2 +- App/lib/core/Gen/Type/TypeGenResolve.vp | 5296 +++++++++--------- App/lib/core/Gen/Type/TypeSpec.vp | 14 +- App/lib/core/IncludesScanner.vp | 786 +-- App/lib/core/VLogger.vp | 492 +- App/lib/core/builtin/ViperBuild.vp | 53 +- App/main.vp | 38 +- 49 files changed, 16545 insertions(+), 16495 deletions(-) diff --git a/App/lib/Phase/Phase1/Phase1Main.vp b/App/lib/Phase/Phase1/Phase1Main.vp index 01b01a9..e297ba0 100644 --- a/App/lib/Phase/Phase1/Phase1Main.vp +++ b/App/lib/Phase/Phase1/Phase1Main.vp @@ -4,8 +4,8 @@ import string import stdlib import memhub import viperlib -import w32.fileio as fileio -import w32.win32base +import win32.fileio as fileio +import win32.win32base import ast import mlirlite import lib.core.VLogger as VLogger @@ -133,7 +133,7 @@ def _BreakContainerCyclesInDeps(temp_dir: str) -> int: if StubMerger._includes_deps_sha1 is None or StubMerger._includes_deps_str is None: return 0 cnt: sizet = StubMerger._includes_deps_count ci: sizet - for ci in range(cnt): + for ci in range(int(cnt)): sha1_m: str = StubMerger._includes_deps_sha1 + sizet(ci) * 17 deps_m: str = StubMerger._includes_deps_str + sizet(ci) * StubMerger.MAX_DEPS_STR_LEN if sha1_m is None or deps_m is None: continue @@ -195,7 +195,7 @@ def _BuildReachableResult(mb: memhub.MemBuddy*, includes_dir: str) -> IncludesSc if result is None: return None entry_size: sizet = IncludesScanner.FileEntry.__sizeof__() i: sizet - for i in range(store_count): + for i in range(int(store_count)): sidx: sizet = sizet(i) * 17 if StubMerger._sha1_store_arr[sidx] == '\0': continue # Only includes entries (non-empty incdir); App entries do not enter result @@ -274,7 +274,7 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: VLogg # On-demand dependency scan (App-first): seed the queue from App imports, then BFS-resolve # reachable includes only (read + SHA1 + import parse); unreferenced ones are never read. - t_scan0: uint64 = w32.win32base.GetTickCount64() + t_scan0: uint64 = win32.win32base.GetTickCount64() reachable_set: str = None reachable_count: int = 0 use_reachable: int = 0 @@ -298,7 +298,7 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: VLogg else: VLogger.warning("SourceDir unavailable; cannot build the dependency graph; skipping Phase1", "Phase1") return 0 - t_scan1: uint64 = w32.win32base.GetTickCount64() + t_scan1: uint64 = win32.win32base.GetTickCount64() VLogger.info(f"dependency-graph scan total time: {t_scan1 - t_scan0} ms (reachable {reachable_count} includes)", "Phase1") # Build the reachable FileEntry list from the Sha1Store; 1a/1b-pre/1b iterate result @@ -354,10 +354,10 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: VLogg p1_sz_ds: sizet = sizet(Phase1.MAX_CLASSES * Phase1.SHA1_LEN) p1_sz_tp: sizet = sizet(4 * Phase1.MAX_FILE_DEPS) p1_sz_bx: sizet = sizet(4) - p1_o_cn: int = mb._order_for_size(p1_sz_cn + 16) - p1_o_ds: int = mb._order_for_size(p1_sz_ds + 16) - p1_o_tp: int = mb._order_for_size(p1_sz_tp + 16) - p1_o_bx: int = mb._order_for_size(p1_sz_bx + 16) + p1_o_cn: int = int(mb._order_for_size(p1_sz_cn + 16)) + p1_o_ds: int = int(mb._order_for_size(p1_sz_ds + 16)) + p1_o_tp: int = int(mb._order_for_size(p1_sz_tp + 16)) + p1_o_bx: int = int(mb._order_for_size(p1_sz_bx + 16)) p1_fl_cn: uint64 = uint64(mb.free_lists[p1_o_cn]) p1_fl_ds: uint64 = uint64(mb.free_lists[p1_o_ds]) VLogger.info(f"[EVIDENCE] order(class_names {p1_sz_cn}B+16)={p1_o_cn} fl={p1_fl_cn} " f"order(def_sha1s {p1_sz_ds}B+16)={p1_o_ds} fl={p1_fl_ds} " f"order(topo {p1_sz_tp}B+16)={p1_o_tp} order(box {p1_sz_bx}B+16)={p1_o_bx} " f"(fl==0 only means that order is empty and can still be split by a higher order; if alloc fails while dump_stats shows free>0 that is the definitive proof of metadata corruption; " f"any order>max_order={mb.max_order} means an abnormal size)", "Phase1") @@ -394,7 +394,7 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: VLogg td_len_ts_call: sizet = string.strlen(temp_dir) ret_topo: int = Phase1Topo._TopoSortFiles(mb, result, reachable_set, reachable_count, use_reachable, sha1_set, set_count, class_names_buf, parent_names_buf, def_sha1s_buf, class_count_scanned, topo_order, topo_count_box, temp_dir, td_len_ts_call) if ret_topo == 0: - tc_ptr: t.CPtr = (int*)(topo_count_box) + tc_ptr: int* = (int*)(topo_count_box) if tc_ptr is not None: topo_count = tc_ptr[0] VLogger.info(f"topological sort: {class_count_scanned} classes, {topo_count} files processed in dependency order", "Phase1") else: @@ -415,17 +415,17 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: VLogg GenApi.set_stdlib_alloc(1) # Iterate files: topo order preferred, falling back to alphabetical - iter_count: int = topo_count if topo_count > 0 else result.Count + iter_count: int = int(topo_count if topo_count > 0 else result.Count) # 1a total-time start (struct/enum/union registration) - t_a0: uint64 = w32.win32base.GetTickCount64() - for i in range(iter_count): + t_a0: uint64 = win32.win32base.GetTickCount64() + for i in range(int(iter_count)): # Per-file mark: free this file's AST after translating (registered data is in perm_pool) p1a_mark: sizet = mb.mark() # Get the file index: topo order or alphabetical - file_idx: int = i + file_idx: int = int(i) if topo_count > 0: od_addr: uint64 = uint64(topo_order) + i * 4 - od_ptr: t.CPtr = (int*)(t.CPtr(od_addr)) + od_ptr: int* = (int*)(t.CPtr(od_addr)) if od_ptr is not None: file_idx = od_ptr[0] entry_addr_r: uint64 = uint64(result.Entries) + file_idx * entry_size @@ -451,7 +451,7 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: VLogg continue # Per-file timer start (read + parse, to locate 1a stalls) - t_fa0: uint64 = w32.win32base.GetTickCount64() + t_fa0: uint64 = win32.win32base.GetTickCount64() # Read file contents file_path_r: str = entry_r.Path @@ -496,7 +496,7 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: VLogg continue # Timer 1: read + parse done - t_fa1: uint64 = w32.win32base.GetTickCount64() + t_fa1: uint64 = win32.win32base.GetTickCount64() # Create Translator with _declare_only=1 (register struct/enum/union) tr_r: HandlesTranslator.Translator* = HandlesTranslator.Translator() @@ -515,7 +515,7 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: VLogg GenApi.reset_visible_structs(mb, 0) ret_r: int = tr_r.translate(tree_r) # Timer 2: translate(declare=1) returned - t_fa2: uint64 = w32.win32base.GetTickCount64() + t_fa2: uint64 = win32.win32base.GetTickCount64() if ret_r != 0: p1a_fl += 1 else: p1a_reg += 1 @@ -540,7 +540,7 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: VLogg mb.reset_to(p1a_mark) # 1a total time (struct/enum/union registration, incl. per-file read+parse+declaration) - t_a1: uint64 = w32.win32base.GetTickCount64() + t_a1: uint64 = win32.win32base.GetTickCount64() VLogger.info(f"1a total time: {t_a1-t_a0} ms (registering struct/enum, reg={p1a_reg} skp={p1a_skp})", "Phase1") if p1a_fl > 0: @@ -567,8 +567,8 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: VLogg p1bpre_failed: int = 0 dir_len_pv: sizet = string.strlen(temp_dir) # 1b-pre total-time start (vpi pre-generation) - t_pv0: uint64 = w32.win32base.GetTickCount64() - for i_pv in range(result.Count): + t_pv0: uint64 = win32.win32base.GetTickCount64() + for i_pv in range(int(result.Count)): entry_addr_pv: uint64 = uint64(result.Entries) + i_pv * entry_size entry_pv: IncludesScanner.FileEntry* = IncludesScanner.FileEntry*(entry_addr_pv) if entry_pv is None or entry_pv.Sha1 is None: @@ -663,7 +663,7 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: VLogg mb.reset_to(mark1) # 1b-pre total time (vpi pre-generation) - t_pv1: uint64 = w32.win32base.GetTickCount64() + t_pv1: uint64 = win32.win32base.GetTickCount64() VLogger.info(f"1b-pre total time: {t_pv1-t_pv0} ms (vpi pre-generation)", "Phase1") if p1bpre_failed > 0: @@ -676,16 +676,16 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: VLogg skipped: int = 0 failed: int = 0 # 1b total-time start (full translation) - t_b0: uint64 = w32.win32base.GetTickCount64() + t_b0: uint64 = win32.win32base.GetTickCount64() - loop_n_b: int = result.Count + loop_n_b: int = int(result.Count) if topo_count > 0: loop_n_b = topo_count - for i in range(loop_n_b): + for i in range(int(loop_n_b)): # Get the entry (topo: file_idx from topo_order, else the scan index) - file_idx_b: int = i + file_idx_b: int = int(i) if topo_count > 0: od_addr_b: uint64 = uint64(topo_order) + sizet(i) * 4 - od_ptr_b: t.CPtr = (int*)(od_addr_b) + od_ptr_b: int* = (int*)(od_addr_b) if od_ptr_b is not None: file_idx_b = od_ptr_b[0] entry_addr: uint64 = uint64(result.Entries) + sizet(file_idx_b) * entry_size entry: IncludesScanner.FileEntry* = IncludesScanner.FileEntry*(entry_addr) @@ -793,7 +793,7 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: VLogg stdlib.free(stub_path) # Per-file timer start: read (parse/translate/dump segments, matching Phase2 instrumentation) - t_f0: uint64 = w32.win32base.GetTickCount64() + t_f0: uint64 = win32.win32base.GetTickCount64() # Read file contents file_path: str = entry.Path @@ -847,7 +847,7 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: VLogg continue # Timer 1: read + lexer/tokenize/parse done - t_f1: uint64 = w32.win32base.GetTickCount64() + t_f1: uint64 = win32.win32base.GetTickCount64() # Register CTypedef by scanning source text: AST string fields (attr/id) point into src_buf # and dangle across phases (reading attr faults), so match "NAME: t.CTypedef = TYPE" lines. @@ -901,7 +901,7 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: VLogg ret: int = tr.translate(tree) # Timer 2: translate returned - t_f2: uint64 = w32.win32base.GetTickCount64() + t_f2: uint64 = win32.win32base.GetTickCount64() # Diagnostic: translate returned normally (if it hangs inside, this line is not printed, # distinguishing an internal hang from a later dump hang). if ret == 0: VLogger.info(f"translation complete (translate returned): {file_path}", "Phase1") @@ -996,7 +996,7 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: VLogg stdlib.free(vpcall_text_p1) # Timer 3: dump (stub+text) done; print the per-file three-segment timing - t_f3: uint64 = w32.win32base.GetTickCount64() + t_f3: uint64 = win32.win32base.GetTickCount64() VLogger.info(f"[{i+1}/{loop_n_b}] translating: {file_path} (parse={t_f1-t_f0} ms, translate={t_f2-t_f1} ms, dump={t_f3-t_f2} ms, total={t_f3-t_f0} ms)", "Phase1") # save dependencies(_imported_modules) for Phase B @@ -1023,14 +1023,14 @@ def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: VLogg # tables moved to stdlib). Memory sentinels locate overruns deterministically. chk_rc: int = mb.check_alive_blocks(mark1) if chk_rc != 0: VLogger.error(f"memory corruption (live block header, rc={chk_rc}): {file_path}", "Phase1") - # Out-of-bounds writes are the crash root cause (unrelated to reclamation/UAF, confirmed + # Out-of-bounds writes are the crash root cause (unrelated to reclamation/UAF, confirmed # by A/B tests); reset_to plus the tail canary pins the offending file. mb.reset_to(mark1) sc_rc: int = mb.self_check() if sc_rc != 0: VLogger.error(f"memory corruption (free list, rc={sc_rc}): {file_path}", "Phase1") # 1b total time (full translation + dump) - t_b1: uint64 = w32.win32base.GetTickCount64() + t_b1: uint64 = win32.win32base.GetTickCount64() VLogger.info(f"1b total time: {t_b1-t_b0} ms (full translation, trans={translated} skip={skipped} fail={failed})", "Phase1") # Release the topo-sort resources used by Phase 1b diff --git a/App/lib/Phase/Phase1/Phase1Scan.vp b/App/lib/Phase/Phase1/Phase1Scan.vp index 9e90042..3c4985a 100644 --- a/App/lib/Phase/Phase1/Phase1Scan.vp +++ b/App/lib/Phase/Phase1/Phase1Scan.vp @@ -5,9 +5,9 @@ import stdlib import memhub import viperlib import sys -import w32.fileio as fileio -import w32.win32file -import w32.win32base +import win32.fileio as fileio +import win32.win32file +import win32.win32base import ast import mlirlite import lib.core.VLogger as VLogger @@ -50,7 +50,7 @@ def _ScanClassInheritance(mb: memhub.MemBuddy*, result: IncludesScanner.ScanResu entry_size: sizet = IncludesScanner.FileEntry.__sizeof__() class_count: int = 0 - for i in range(result.Count): + for i in range(int(result.Count)): # Per-file mark: reclaim this file's AST right after collecting inheritance # info (keeping all ASTs alive until mark1 would OOM the 1GB main pool). scan_mark: sizet = mb.mark() @@ -106,7 +106,7 @@ def _ScanClassInheritance(mb: memhub.MemBuddy*, result: IncludesScanner.ScanResu ch: list[ast.AST*]* = tree.children if ch is not None: cn: sizet = len(ch) - for ci in range(cn): + for ci in range(int(cn)): child: ast.AST* = ch.get(ci) if child is None: continue kd: int = child.kind() @@ -121,12 +121,12 @@ def _ScanClassInheritance(mb: memhub.MemBuddy*, result: IncludesScanner.ScanResu if tp_scan is not None: file_has_generic = 1 if generic_file_count_box is not None: - gfc_ptr_scan: t.CPtr = (int*)(generic_file_count_box) + gfc_ptr_scan: int* = (int*)(generic_file_count_box) if gfc_ptr_scan is not None: gfc_scan: int = gfc_ptr_scan[0] if gfc_scan < Phase1.MAX_FILE_DEPS and generic_file_indices is not None: gi_addr_scan: uint64 = uint64(generic_file_indices) + gfc_scan * 4 - gi_ptr_scan: t.CPtr = (int*)(gi_addr_scan) + gi_ptr_scan: int* = (int*)(gi_addr_scan) if gi_ptr_scan is not None: gi_ptr_scan[0] = i gfc_ptr_scan[0] = gfc_scan + 1 @@ -226,7 +226,7 @@ def _PreRegisterAllGenericTemplates(mb: memhub.MemBuddy*, result: IncludesScanne if perm is None: return 0 p1e_size: sizet = IncludesScanner.FileEntry.__sizeof__() handled_count: int = 0 - for i in range(result.Count): + for i in range(int(result.Count)): entry_addr_pg: uint64 = uint64(result.Entries) + i * p1e_size entry_pg: IncludesScanner.FileEntry* = (IncludesScanner.FileEntry*)(entry_addr_pg) if entry_pg is None or entry_pg.Sha1 is None: continue @@ -269,7 +269,7 @@ def _PreRegisterAllGenericTemplates(mb: memhub.MemBuddy*, result: IncludesScanne ch_pg: list[ast.AST*]* = tree_pg.children if ch_pg is not None: cn_pg: sizet = len(ch_pg) - for ci_pg in range(cn_pg): + for ci_pg in range(int(cn_pg)): child_pg: ast.AST* = ch_pg.get(ci_pg) if child_pg is None: continue ck_pg: int = child_pg.kind() @@ -298,7 +298,7 @@ def _PreRegisterAllGenericTemplates(mb: memhub.MemBuddy*, result: IncludesScanne def _FindClassDefSha1(class_names_buf: bytes, def_sha1s_buf: bytes, class_count: int, class_name: str) -> str: """Find a class name's defining file SHA1 in the flat arrays.""" if class_names_buf is None or def_sha1s_buf is None or class_name is None: return None - for i in range(class_count): + for i in range(int(class_count)): name_ptr: str = class_names_buf + i * Phase1.CLASS_NAME_LEN if string.strcmp(name_ptr, class_name) == 0: return def_sha1s_buf + i * Phase1.SHA1_LEN return None @@ -349,12 +349,12 @@ def _ScanTreeForContainers(node: ast.AST*, need_list_box: t.CPtr, need_dict_box: if cl_args_s is not None: na_s: sizet = len(cl_args_s) i_a2: sizet - for i_a2 in range(na_s): _ScanTreeForContainers(cl_args_s.get(i_a2), need_list_box, need_dict_box, depth + 1) + for i_a2 in range(int(na_s)): _ScanTreeForContainers(cl_args_s.get(i_a2), need_list_box, need_dict_box, depth + 1) cl_kws_s: list[ast.AST*]* = cl_s2.keywords if cl_kws_s is not None: nk_s: sizet = len(cl_kws_s) i_k2: sizet - for i_k2 in range(nk_s): + for i_k2 in range(int(nk_s)): kw_s: ast.Keyword* = ast.Keyword*(cl_kws_s.get(i_k2)) if kw_s is not None: _ScanTreeForContainers(kw_s.value, need_list_box, need_dict_box, depth + 1) elif k_s == ast.ASTKind.Assign: @@ -364,7 +364,7 @@ def _ScanTreeForContainers(node: ast.AST*, need_list_box: t.CPtr, need_dict_box: if tgts_s is not None: nt_s: sizet = len(tgts_s) i_t2: sizet - for i_t2 in range(nt_s): _ScanTreeForContainers(tgts_s.get(i_t2), need_list_box, need_dict_box, depth + 1) + for i_t2 in range(int(nt_s)): _ScanTreeForContainers(tgts_s.get(i_t2), need_list_box, need_dict_box, depth + 1) elif k_s == ast.ASTKind.AnnAssign: aa_s2: ast.AnnAssign* = ast.AnnAssign*(node) _ScanTreeForContainers(aa_s2.annotation, need_list_box, need_dict_box, depth + 1) @@ -378,7 +378,7 @@ def _ScanTreeForContainers(node: ast.AST*, need_list_box: t.CPtr, need_dict_box: if fa_s is not None: nf_s: sizet = len(fa_s) i_f2: sizet - for i_f2 in range(nf_s): + for i_f2 in range(int(nf_s)): arg_s: ast.Arg* = ast.Arg*(fa_s.get(i_f2)) if arg_s is not None: _ScanTreeForContainers(arg_s.annotation, need_list_box, need_dict_box, depth + 1) elif k_s == ast.ASTKind.ClassDef: @@ -387,7 +387,7 @@ def _ScanTreeForContainers(node: ast.AST*, need_list_box: t.CPtr, need_dict_box: if bases_s is not None: nb_s: sizet = len(bases_s) i_b2: sizet - for i_b2 in range(nb_s): _ScanTreeForContainers(bases_s.get(i_b2), need_list_box, need_dict_box, depth + 1) + for i_b2 in range(int(nb_s)): _ScanTreeForContainers(bases_s.get(i_b2), need_list_box, need_dict_box, depth + 1) elif k_s == ast.ASTKind.Attribute: at_s: ast.Attribute* = ast.Attribute*(node) _ScanTreeForContainers(at_s.value, need_list_box, need_dict_box, depth + 1) @@ -397,7 +397,7 @@ def _ScanTreeForContainers(node: ast.AST*, need_list_box: t.CPtr, need_dict_box: if elts_s is not None: ne_s: sizet = len(elts_s) i_e2: sizet - for i_e2 in range(ne_s): _ScanTreeForContainers(elts_s.get(i_e2), need_list_box, need_dict_box, depth + 1) + for i_e2 in range(int(ne_s)): _ScanTreeForContainers(elts_s.get(i_e2), need_list_box, need_dict_box, depth + 1) elif k_s == ast.ASTKind.UnaryOp: uop_s: ast.UnaryOp* = ast.UnaryOp*(node) _ScanTreeForContainers(uop_s.operand, need_list_box, need_dict_box, depth + 1) @@ -414,18 +414,18 @@ def _ScanTreeForContainers(node: ast.AST*, need_list_box: t.CPtr, need_dict_box: if cps_s is not None: nc_s: sizet = len(cps_s) i_c2: sizet - for i_c2 in range(nc_s): _ScanTreeForContainers(cps_s.get(i_c2), need_list_box, need_dict_box, depth + 1) + for i_c2 in range(int(nc_s)): _ScanTreeForContainers(cps_s.get(i_c2), need_list_box, need_dict_box, depth + 1) elif k_s == ast.ASTKind.BoolOp: bo_s: ast.BoolOp* = ast.BoolOp*(node) bvs_s: list[ast.AST*]* = bo_s.values if bvs_s is not None: nv_s: sizet = len(bvs_s) i_v2: sizet - for i_v2 in range(nv_s): _ScanTreeForContainers(bvs_s.get(i_v2), need_list_box, need_dict_box, depth + 1) + for i_v2 in range(int(nv_s)): _ScanTreeForContainers(bvs_s.get(i_v2), need_list_box, need_dict_box, depth + 1) # `[...]`/`{}` literals do not trigger _list/_dict imports (compile-time constant lists, array init); # recurse into statement-block children. if node.children is not None: ch_s: list[ast.AST*]* = node.children n_s: sizet = len(ch_s) ci_s: sizet - for ci_s in range(n_s): _ScanTreeForContainers(ch_s.get(ci_s), need_list_box, need_dict_box, depth + 1) + for ci_s in range(int(n_s)): _ScanTreeForContainers(ch_s.get(ci_s), need_list_box, need_dict_box, depth + 1) diff --git a/App/lib/Phase/Phase1/Phase1Topo.vp b/App/lib/Phase/Phase1/Phase1Topo.vp index 6f5acf6..2a12ce2 100644 --- a/App/lib/Phase/Phase1/Phase1Topo.vp +++ b/App/lib/Phase/Phase1/Phase1Topo.vp @@ -4,9 +4,9 @@ import string import stdlib import memhub import viperlib -import w32.fileio as fileio -import w32.win32file -import w32.win32base +import win32.fileio as fileio +import win32.win32file +import win32.win32base import ast import mlirlite import lib.core.VLogger as VLogger @@ -62,7 +62,7 @@ def _TopoSortFiles(mb: memhub.MemBuddy*, result: IncludesScanner.ScanResult*, re file_count: int = 0 # sha1 -> file_index map (simple linear search; file count is small) - for i in range(result.Count): + for i in range(int(result.Count)): entry_addr: uint64 = uint64(result.Entries) + i * entry_size entry: IncludesScanner.FileEntry* = IncludesScanner.FileEntry*(entry_addr) if entry is None or entry.Sha1 is None: continue @@ -83,7 +83,7 @@ def _TopoSortFiles(mb: memhub.MemBuddy*, result: IncludesScanner.ScanResult*, re file_count += 1 # 2. Build dependency edges: for each class, find the parent's defining file and add a child->parent edge - for ci in range(class_count): + for ci in range(int(class_count)): # Read parent class name (offset = ci * Phase1.CLASS_NAME_LEN) parent_ptr: str = parent_names_buf + ci * Phase1.CLASS_NAME_LEN if parent_ptr[0] == '\0': continue @@ -100,19 +100,19 @@ def _TopoSortFiles(mb: memhub.MemBuddy*, result: IncludesScanner.ScanResult*, re if string.strcmp(parent_sha1, child_def_sha1) == 0: continue # Find the child file's index and add the parent file SHA1 to its dependency list - for fi in range(file_count): + for fi in range(int(file_count)): fsha1: str = file_sha1s_buf + fi * Phase1.SHA1_LEN if string.strcmp(fsha1, child_def_sha1) != 0: continue # Read the current dependency count cnt_addr: uint64 = uint64(file_dep_counts) + fi * 4 - cnt_ptr: t.CPtr = (int*)(cnt_addr) + cnt_ptr: int* = (int*)(cnt_addr) if cnt_ptr is None: break cur_cnt: int = cnt_ptr[0] # Check whether the dependency is already present (dedupe) already: int = 0 - for di in range(cur_cnt): + for di in range(int(cur_cnt)): dep_slot: str = file_deps_buf + fi * Phase1.MAX_DEP_EDGES * Phase1.SHA1_LEN + di * Phase1.SHA1_LEN if string.strcmp(dep_slot, parent_sha1) == 0: already = 1 @@ -132,7 +132,7 @@ def _TopoSortFiles(mb: memhub.MemBuddy*, result: IncludesScanner.ScanResult*, re map_count_ts: int = 0 if sha1_arr_ts is not None and mod_arr_ts is not None: map_count_ts = StubMergerIR._BuildIncludesSha1Map(temp_dir, td_len_ts, sha1_arr_ts, mod_arr_ts) - for fi_imp in range(file_count): + for fi_imp in range(int(file_count)): if map_count_ts <= 0: break fsha1_imp: str = file_sha1s_buf + fi_imp * Phase1.SHA1_LEN deps_imp: str = StubMergerStore.LookupIncludesDeps(fsha1_imp) @@ -153,16 +153,16 @@ def _TopoSortFiles(mb: memhub.MemBuddy*, result: IncludesScanner.ScanResult*, re stdlib.free(dname_imp) if dsha1_imp is None: continue # Find the dependency file's index in the list (skip self) - for fj_imp in range(file_count): + for fj_imp in range(int(file_count)): if string.strcmp(file_sha1s_buf + fj_imp * Phase1.SHA1_LEN, dsha1_imp) != 0: continue if fj_imp == fi_imp: break # Dedupe and add the edge: fi_imp depends on fj_imp (InDegree[fi_imp]++) cnt_addr_imp: uint64 = uint64(file_dep_counts) + fi_imp * 4 - cnt_ptr_imp: t.CPtr = (int*)(cnt_addr_imp) + cnt_ptr_imp: int* = (int*)(cnt_addr_imp) if cnt_ptr_imp is None: break cur_cnt_imp: int = cnt_ptr_imp[0] already_imp: int = 0 - for di_imp in range(cur_cnt_imp): + for di_imp in range(int(cur_cnt_imp)): dep_slot_imp: str = file_deps_buf + fi_imp * Phase1.MAX_DEP_EDGES * Phase1.SHA1_LEN + di_imp * Phase1.SHA1_LEN if string.strcmp(dep_slot_imp, dsha1_imp) == 0: already_imp = 1 @@ -179,11 +179,11 @@ def _TopoSortFiles(mb: memhub.MemBuddy*, result: IncludesScanner.ScanResult*, re # 3. Kahn: InDegree = DepCount (number of parent files depended on). # Process zero-dependency files first, then decrement the in-degree of dependents. - for fi in range(file_count): + for fi in range(int(file_count)): cnt_addr3: uint64 = uint64(file_dep_counts) + fi * 4 - cnt_ptr3: t.CPtr = (int*)(cnt_addr3) + cnt_ptr3: int* = (int*)(cnt_addr3) ind_addr3: uint64 = uint64(file_indegrees) + fi * 4 - ind_ptr3: t.CPtr = (int*)(ind_addr3) + ind_ptr3: int* = (int*)(ind_addr3) if cnt_ptr3 is not None and ind_ptr3 is not None: ind_ptr3[0] = cnt_ptr3[0] # 4. Topological sort output @@ -191,27 +191,27 @@ def _TopoSortFiles(mb: memhub.MemBuddy*, result: IncludesScanner.ScanResult*, re changed: int = 1 while changed != 0: changed = 0 - for fi in range(file_count): + for fi in range(int(file_count)): # Check processed flag proc_addr4: uint64 = uint64(file_processed) + fi * 4 - proc_ptr4: t.CPtr = (int*)(proc_addr4) + proc_ptr4: int* = (int*)(proc_addr4) if proc_ptr4 is None or proc_ptr4[0] != 0: continue # Check in-degree ind_addr4: uint64 = uint64(file_indegrees) + fi * 4 - ind_ptr4: t.CPtr = (int*)(ind_addr4) + ind_ptr4: int* = (int*)(ind_addr4) if ind_ptr4 is None or ind_ptr4[0] > 0: continue # In-degree 0: can be processed # Find the matching result.Entries index target_sha1: str = file_sha1s_buf + fi * Phase1.SHA1_LEN - for ri in range(result.Count): + for ri in range(int(result.Count)): ea_r: uint64 = uint64(result.Entries) + ri * entry_size ent_r: IncludesScanner.FileEntry* = IncludesScanner.FileEntry*(ea_r) if ent_r is not None and ent_r.Sha1 is not None: if string.strcmp(ent_r.Sha1, target_sha1) == 0: # Write into the output index array out_idx_addr: uint64 = uint64(out_order) + out_idx * 4 - out_int_ptr: t.CPtr = (int*)(out_idx_addr) + out_int_ptr: int* = (int*)(out_idx_addr) if out_int_ptr is not None: out_int_ptr[0] = ri out_idx += 1 break @@ -221,56 +221,56 @@ def _TopoSortFiles(mb: memhub.MemBuddy*, result: IncludesScanner.ScanResult*, re changed = 1 # Decrement the in-degree of files that depend on this one - for fj in range(file_count): + for fj in range(int(file_count)): proc_addr5: uint64 = uint64(file_processed) + fj * 4 - proc_ptr5: t.CPtr = (int*)(proc_addr5) + proc_ptr5: int* = (int*)(proc_addr5) if proc_ptr5 is None or proc_ptr5[0] != 0: continue # Read fj's dependency count cnt_addr5: uint64 = uint64(file_dep_counts) + fj * 4 - cnt_ptr5: t.CPtr = (int*)(cnt_addr5) + cnt_ptr5: int* = (int*)(cnt_addr5) if cnt_ptr5 is None: continue fj_dep_cnt: int = cnt_ptr5[0] - for di in range(fj_dep_cnt): + for di in range(int(fj_dep_cnt)): dep_slot5: str = file_deps_buf + fj * Phase1.MAX_DEP_EDGES * Phase1.SHA1_LEN + di * Phase1.SHA1_LEN if string.strcmp(dep_slot5, target_sha1) == 0: ind_addr5: uint64 = uint64(file_indegrees) + fj * 4 - ind_ptr5: t.CPtr = (int*)(ind_addr5) + ind_ptr5: int* = (int*)(ind_addr5) if ind_ptr5 is not None: ind_ptr5[0] = ind_ptr5[0] - 1 break # Handle cycles: within a cycle emit the more-depended-upon files first. # On-demand container injection (e.g. string -> _list) plus real imports form pseudo-cycles Kahn cannot fully drain; scan order would translate dependents first -> unregistered cross-module vtable TypeStr. if out_idx < file_count: - rem_idx: t.CPtr = stdlib.malloc(4 * file_count) - rem_depcnt: t.CPtr = stdlib.malloc(4 * file_count) + rem_idx: int* = (int*)(stdlib.malloc(4 * file_count)) + rem_depcnt: int* = (int*)(stdlib.malloc(4 * file_count)) rem_count: int = 0 if rem_idx is not None and rem_depcnt is not None: - for fi in range(file_count): + for fi in range(int(file_count)): proc_addr7: uint64 = uint64(file_processed) + fi * 4 - proc_ptr7: t.CPtr = (int*)(proc_addr7) + proc_ptr7: int* = (int*)(proc_addr7) if proc_ptr7 is not None and proc_ptr7[0] == 0: rem_idx[rem_count] = fi rem_depcnt[rem_count] = 0 rem_count += 1 # Compute each remaining file's dependents count (among the remaining files) - for i_a in range(rem_count): + for i_a in range(int(rem_count)): fi_a: int = rem_idx[i_a] target_sha1_a: str = file_sha1s_buf + fi_a * Phase1.SHA1_LEN - for i_b in range(rem_count): + for i_b in range(int(rem_count)): fi_b: int = rem_idx[i_b] if fi_b == fi_a: continue cnt_addr_b: uint64 = uint64(file_dep_counts) + fi_b * 4 - cnt_ptr_b: t.CPtr = (int*)(cnt_addr_b) + cnt_ptr_b: int* = (int*)(cnt_addr_b) if cnt_ptr_b is None: continue cnt_b: int = cnt_ptr_b[0] - for di_b in range(cnt_b): + for di_b in range(int(cnt_b)): dep_slot_b: str = file_deps_buf + fi_b * Phase1.MAX_DEP_EDGES * Phase1.SHA1_LEN + di_b * Phase1.SHA1_LEN if string.strcmp(dep_slot_b, target_sha1_a) == 0: rem_depcnt[i_a] = rem_depcnt[i_a] + 1 break # Bubble sort descending (more dependents first) - for i_a in range(rem_count): - for j_a in range(rem_count - 1): + for i_a in range(int(rem_count)): + for j_a in range(int(rem_count - 1)): if rem_depcnt[j_a] < rem_depcnt[j_a + 1]: tmp_i: int = rem_idx[j_a] rem_idx[j_a] = rem_idx[j_a + 1] @@ -279,19 +279,19 @@ def _TopoSortFiles(mb: memhub.MemBuddy*, result: IncludesScanner.ScanResult*, re rem_depcnt[j_a] = rem_depcnt[j_a + 1] rem_depcnt[j_a + 1] = tmp_d # Emit the remaining files in order - for i_a in range(rem_count): + for i_a in range(int(rem_count)): fi_a2: int = rem_idx[i_a] proc_addr8: uint64 = uint64(file_processed) + fi_a2 * 4 - proc_ptr8: t.CPtr = (int*)(proc_addr8) + proc_ptr8: int* = (int*)(proc_addr8) if proc_ptr8 is None or proc_ptr8[0] != 0: continue target_sha1_2: str = file_sha1s_buf + fi_a2 * Phase1.SHA1_LEN - for ri in range(result.Count): + for ri in range(int(result.Count)): ea_r2: uint64 = uint64(result.Entries) + ri * entry_size ent_r2: IncludesScanner.FileEntry* = IncludesScanner.FileEntry*(ea_r2) if ent_r2 is not None and ent_r2.Sha1 is not None: if string.strcmp(ent_r2.Sha1, target_sha1_2) == 0: out_idx_addr2: uint64 = uint64(out_order) + out_idx * 4 - out_int_ptr2: t.CPtr = (int*)(out_idx_addr2) + out_int_ptr2: int* = (int*)(out_idx_addr2) if out_int_ptr2 is not None: out_int_ptr2[0] = ri out_idx += 1 break @@ -300,7 +300,7 @@ def _TopoSortFiles(mb: memhub.MemBuddy*, result: IncludesScanner.ScanResult*, re stdlib.free(rem_depcnt) # Write the output count - out_count_ptr: t.CPtr = (int*)(out_count) + out_count_ptr: int* = (int*)(out_count) if out_count_ptr is not None: out_count_ptr[0] = out_idx # Free the flat arrays diff --git a/App/lib/Phase/Phase2/__init__.vp b/App/lib/Phase/Phase2/__init__.vp index b80f416..a9054d6 100644 --- a/App/lib/Phase/Phase2/__init__.vp +++ b/App/lib/Phase/Phase2/__init__.vp @@ -1,1537 +1,1537 @@ -import lib.core.Gen.GenApi as GenApi -import t, c -from stdint import * -import stdio -import string -import stdlib -import memhub -import viperlib -import w32.fileio as fileio -import w32.win32file -import w32.win32base -import sys -import ast -import mlirlite -import subprocess -import argparse -import lib.core.VLogger as VLogger -import lib.core.Gen.Translator as HandlesTranslator -import lib.Phase.BuildPipeline as BuildPipeline -import lib.core.IncludesScanner as IncludesScanner -import lib.core.CompilerFactory as CompilerFactory -import lib.Stub.StubMerger as StubMerger -import lib.Projectrans.Utils as Utils -import lib.Projectrans.Config as Config -import lib.Stub.StubMerger.StubMergerBuild as StubMergerBuild -import lib.Stub.StubMerger.StubMergerCollect as StubMergerCollect -import lib.Stub.StubMerger.StubMergerIR as StubMergerIR -import lib.Stub.StubMerger.StubMergerStore as StubMergerStore -import lib.Stub.StubMerger.StubMergerTopo as StubMergerTopo -import lib.Stub.StubGen.Converter as StubConverter - -# Global mbuddy pointer -_mbuddy: memhub.MemBuddy* - -# Source buffer size (1MB) -define SRC_BUF_SIZE = 1048576 - -# Max source files (recursive scanning grows the count; too small truncates the list -> undefined symbol) -define MAX_SRC_FILES = 128 - -@t.NoVTable -class SrcFileEntry: - """Source file entry""" - Path: str # full path - Sha1: str # first 16 chars of SHA1 - -# _ScanDirForPyFiles - recursively scan a dir and collect .vp/.py files into entries, returning the new count. -# Args: mb pool; dir_path; entries SrcFileEntry array; entry_size; file_count; max_files. -def _ScanDirForPyFiles(mb: memhub.MemBuddy*, dir_path: str, entries: SrcFileEntry*, entry_size: sizet, file_count: int, max_files: int) -> int: - """Recursively scan a directory, collect .vp/.py files into entries, return the new file_count""" - if dir_path is None or entries is None: return file_count - if file_count >= max_files: return file_count - - dir_len: sizet = string.strlen(dir_path) - pattern: bytes = stdlib.malloc(dir_len + 16) - if pattern is None: return file_count - viperlib.snprintf(pattern, dir_len + 16, "%s/*", dir_path) - - find_data_size: sizet = w32.win32file.WIN32_FIND_DATAA.__sizeof__() - find_data: w32.win32file.WIN32_FIND_DATAA* = stdlib.malloc(find_data_size + 16) - if find_data is None: - stdlib.free(pattern) - return file_count - string.memset(find_data, 0, find_data_size + 16) - - handle: w32.win32base.HANDLE = w32.win32file.FindFirstFileA(pattern, find_data) - if handle == w32.win32base.INVALID_HANDLE_VALUE: - stdlib.free(pattern) - stdlib.free(find_data) - return file_count - - while True: - fname: str = find_data.cFileName - if fname is None: - if w32.win32file.FindNextFileA(handle, find_data) == 0: break - continue - fname_len: sizet = string.strlen(fname) - if fname_len <= 0: - if w32.win32file.FindNextFileA(handle, find_data) == 0: break - continue - # Skip . and .. - is_dot: int = 0 - if fname_len == 1 and fname[0] == '.': is_dot = 1 - elif fname_len == 2 and fname[0] == '.' and fname[1] == '.': is_dot = 1 - if is_dot != 0: - if w32.win32file.FindNextFileA(handle, find_data) == 0: break - continue - # Check whether it is a directory - attrs: ULONG = find_data.dwFileAttributes - is_dir: int = 0 - if (attrs & w32.win32file.FILE_ATTRIBUTE_DIRECTORY) != 0: is_dir = 1 - if is_dir != 0: - # Recurse into the subdirectory - sub_dir: bytes = Utils.JoinPath(dir_path, fname) - if sub_dir is not None: - file_count = _ScanDirForPyFiles(mb, sub_dir, entries, entry_size, file_count, max_files) - stdlib.free(sub_dir) - if w32.win32file.FindNextFileA(handle, find_data) == 0: break - continue - # Check for a .vp source file (Viper sources are only .vp now; .py is no longer recognized) - if fname_len > 3 and Utils.is_vp_at(fname, 0, fname_len) != 0 and file_count < max_files: - full_path: bytes = Utils.JoinPath(dir_path, fname) - if full_path is not None: - sbuf: bytes = Utils.read_file_to_buf(full_path, SRC_BUF_SIZE) - if sbuf is None: - if w32.win32file.FindNextFileA(handle, find_data) == 0: break - continue - # Use IncludesScanner.compute_file_sha1 (CRLF->LF normalized) as StubMerger does for the same - # App sources; raw-byte hashing makes stub/define symbol prefixes differ -> undefined reference. - sha1_val: str = IncludesScanner.compute_file_sha1(mb, full_path) - if sha1_val is not None: - ea: uint64 = uint64(entries) + file_count * entry_size - ent: SrcFileEntry* = (SrcFileEntry*)(ea) - if ent is not None: - ent.Path = full_path - ent.Sha1 = sha1_val - file_count += 1 - - VLogger.info(f"{fname} (sha1={sha1_val})", "project") - stdlib.free(sbuf) - # full_path is not freed: ent.Path references it - - if w32.win32file.FindNextFileA(handle, find_data) == 0: break - - w32.win32file.FindClose(handle) - stdlib.free(pattern) - stdlib.free(find_data) - return file_count - - -# _ScanDirForAsmFiles - recursively scan a dir and collect .s assembly files. -# Like _ScanDirForPyFiles; TPV does not translate .s but assembles it to .obj via llvm-mc for linking. -def _ScanDirForAsmFiles(mb: memhub.MemBuddy*, dir_path: str, entries: SrcFileEntry*, entry_size: sizet, file_count: int, max_files: int) -> int: - """Recursively scan a directory, collect .s assembly files into entries, return the new file_count""" - if dir_path is None or entries is None: return file_count - if file_count >= max_files: return file_count - - dir_len: sizet = string.strlen(dir_path) - pattern: bytes = stdlib.malloc(dir_len + 16) - if pattern is None: return file_count - viperlib.snprintf(pattern, dir_len + 16, "%s/*", dir_path) - - find_data_size: sizet = w32.win32file.WIN32_FIND_DATAA.__sizeof__() - find_data: w32.win32file.WIN32_FIND_DATAA* = stdlib.malloc(find_data_size + 16) - if find_data is None: - stdlib.free(pattern) - return file_count - string.memset(find_data, 0, find_data_size + 16) - - handle: w32.win32base.HANDLE = w32.win32file.FindFirstFileA(pattern, find_data) - if handle == w32.win32base.INVALID_HANDLE_VALUE: - stdlib.free(pattern) - stdlib.free(find_data) - return file_count - - while True: - fname: str = find_data.cFileName - if fname is None: - if w32.win32file.FindNextFileA(handle, find_data) == 0: break - continue - fname_len: sizet = string.strlen(fname) - if fname_len <= 0: - if w32.win32file.FindNextFileA(handle, find_data) == 0: break - continue - # Skip . and .. - is_dot: int = 0 - if fname_len == 1 and fname[0] == '.': is_dot = 1 - elif fname_len == 2 and fname[0] == '.' and fname[1] == '.': is_dot = 1 - if is_dot != 0: - if w32.win32file.FindNextFileA(handle, find_data) == 0: break - continue - # Check whether it is a directory - attrs: ULONG = find_data.dwFileAttributes - is_dir: int = 0 - if (attrs & w32.win32file.FILE_ATTRIBUTE_DIRECTORY) != 0: is_dir = 1 - if is_dir != 0: - sub_dir: bytes = Utils.JoinPath(dir_path, fname) - if sub_dir is not None: - file_count = _ScanDirForAsmFiles(mb, sub_dir, entries, entry_size, file_count, max_files) - stdlib.free(sub_dir) - if w32.win32file.FindNextFileA(handle, find_data) == 0: break - continue - # Check for the .s suffix (assembly source) - is_asm: int = 0 - if fname_len > 2 and fname[fname_len - 2] == '.' and fname[fname_len - 1] == 's': is_asm = 1 - if is_asm != 0 and file_count < max_files: - full_path: bytes = Utils.JoinPath(dir_path, fname) - if full_path is not None: - # Same SHA1 convention as .vp (CRLF->LF normalized), used for the sliced path - sha1_val: str = IncludesScanner.compute_file_sha1(mb, full_path) - if sha1_val is not None: - ea: uint64 = uint64(entries) + file_count * entry_size - ent: SrcFileEntry* = (SrcFileEntry*)(ea) - if ent is not None: - ent.Path = full_path - ent.Sha1 = sha1_val - file_count += 1 - - VLogger.info(f"{fname} (asm sha1={sha1_val})", "project") - # full_path is not freed: ent.Path references it - - if w32.win32file.FindNextFileA(handle, find_data) == 0: break - - w32.win32file.FindClose(handle) - stdlib.free(pattern) - stdlib.free(find_data) - return file_count - - -# _PreRegisterAppGenericTemplates - pre-register App generic class templates before mark2. -# The generic tables live in the pool; registering after mark2 would be reclaimed by reset_to, corrupting later passes. -def _PreRegisterAppGenericTemplates(mb: memhub.MemBuddy*, entries: SrcFileEntry*, file_count: int, entry_size: sizet) -> int: - """Pre-register App source generic class templates before mark2, return the processed count""" - if mb is None or entries is None or file_count <= 0: return 0 - handled: int = 0 - for i in range(file_count): - ea_pa: uint64 = uint64(entries) + i * entry_size - ent_pa: SrcFileEntry* = (SrcFileEntry*)(ea_pa) - if ent_pa is None or ent_pa.Path is None or ent_pa.Sha1 is None: continue - # Read the file - f_pa: fileio.File* = fileio.File(ent_pa.Path, fileio.MODE.R) - if f_pa.closed: continue - # read_all's capacity includes the terminator (reads at most cap-1 bytes); longer files are truncated - if f_pa.size() >= SRC_BUF_SIZE: - VLogger.error(f"source file exceeds {SRC_BUF_SIZE} byte limit, refusing to read: {ent_pa.Path}", "PhaseA") - f_pa.close() - continue - src_buf_pa: bytes = stdlib.malloc(SRC_BUF_SIZE) - if src_buf_pa is None: - f_pa.close() - continue - bytes_read_pa: LONG = f_pa.read_all(src_buf_pa, SRC_BUF_SIZE) - f_pa.close() - if bytes_read_pa <= 0: - stdlib.free(src_buf_pa) - continue - if bytes_read_pa < SRC_BUF_SIZE: src_buf_pa[bytes_read_pa] = 0 - # Parse the AST only (perm pool keeps template nodes alive across the mark2 reset) - perm_pa: memhub.MemBuddy* = CompilerFactory.factory.PermPool - if perm_pa is None: perm_pa = mb - lx_pa: ast.Lexer* = ast.new_lexer(perm_pa) - if lx_pa is None: - stdlib.free(src_buf_pa) - continue - ast._lexer_init(lx_pa, src_buf_pa, perm_pa) - tokens_pa: ast.Token* = ast.tokenize(lx_pa) - tree_pa: ast.AST* = ast.parse_tokens(perm_pa, tokens_pa) - if tree_pa is None: - stdlib.free(src_buf_pa) - continue - # Walk top-level ClassDef/FunctionDef and register generic templates (type_params != None) - ch_pa: list[ast.AST*]* = tree_pa.children - if ch_pa is not None: - cn_pa: sizet = len(ch_pa) - for ci_pa in range(cn_pa): - child_pa: ast.AST* = ch_pa.get(ci_pa) - if child_pa is None: continue - ck_pa: int = child_pa.kind() - if ck_pa == ast.ASTKind.ClassDef: - cd_pa: ast.ClassDef* = ast.ClassDef*(child_pa) - if cd_pa is None or cd_pa.name is None: continue - tp_pa: list[str]* = cd_pa.type_params - if tp_pa is None: continue - GenApi.set_current_file_type(ent_pa.Path) - GenApi._register_generic_template(perm_pa, cd_pa, ent_pa.Sha1) - handled += 1 - elif ck_pa == ast.ASTKind.FunctionDef: - fd_pa: ast.FunctionDef* = ast.FunctionDef*(child_pa) - if fd_pa is None or fd_pa.name is None: continue - tf_pa: list[str]* = fd_pa.type_params - if tf_pa is None: continue - GenApi.set_current_file_type(ent_pa.Path) - GenApi._register_generic_func_template(perm_pa, fd_pa, ent_pa.Sha1) - handled += 1 - stdlib.free(src_buf_pa) - if handled > 0: VLogger.info(f"pre-registered {handled} App generic templates before mark2 (classes + functions)", "PhaseA") - return handled - - -# Phase A convergence-failure dependency-graph diagnostics. -# Each file's deps and missing details are persisted to a stdlib table (survives reset) and dumped on failure. -PA_DEPS_MAX_STR: sizet = 1024 -PA_MISS_MAX_STR: sizet = 768 -PA_GRAPH_DEPTH_MAX: int = 3 # missing-chain drill-down depth (counts from layer 1) -PA_GRAPH_OK_PROBE: int = 2 # ready nodes are probed 1 more level (to find hidden misses) -_pa_diag_max: sizet = 0 -_pa_diag_count: int = 0 -_pa_diag_sha1: bytes = None -_pa_diag_deps: bytes = None -_pa_diag_miss: bytes = None - - -def _PhaseADiagInit(max_count: sizet) -> None: - """Allocate the persistent Phase A convergence-failure diagnostic table""" - global _pa_diag_max, _pa_diag_count, _pa_diag_sha1, _pa_diag_deps, _pa_diag_miss - _pa_diag_max = max_count - _pa_diag_count = 0 - _pa_diag_sha1 = stdlib.malloc(max_count * 17) - _pa_diag_deps = stdlib.malloc(max_count * PA_DEPS_MAX_STR) - _pa_diag_miss = stdlib.malloc(max_count * PA_MISS_MAX_STR) - if _pa_diag_sha1 is not None: string.memset(_pa_diag_sha1, 0, max_count * 17) - if _pa_diag_deps is not None: string.memset(_pa_diag_deps, 0, max_count * PA_DEPS_MAX_STR) - if _pa_diag_miss is not None: string.memset(_pa_diag_miss, 0, max_count * PA_MISS_MAX_STR) - - -def _PhaseADiagFree() -> None: - """Free the persistent diagnostic table""" - global _pa_diag_max, _pa_diag_count, _pa_diag_sha1, _pa_diag_deps, _pa_diag_miss - if _pa_diag_sha1 is not None: - stdlib.free(_pa_diag_sha1) - _pa_diag_sha1 = None - if _pa_diag_deps is not None: - stdlib.free(_pa_diag_deps) - _pa_diag_deps = None - if _pa_diag_miss is not None: - stdlib.free(_pa_diag_miss) - _pa_diag_miss = None - _pa_diag_count = 0 - _pa_diag_max = 0 - - -def _PhaseADiagRecord(sha1: str, deps: str, miss: str) -> None: - """Record or update one file's diagnostics: sha1 to deps plus missing detail""" - global _pa_diag_count - if _pa_diag_sha1 is None or _pa_diag_deps is None or sha1 is None: return - if string.strlen(sha1) < 16: return - for i in range(_pa_diag_count): - es: bytes = _pa_diag_sha1 + sizet(i) * 17 - if string.strcmp(es, sha1) == 0: - if miss is not None and _pa_diag_miss is not None: - maddr: bytes = _pa_diag_miss + sizet(i) * PA_MISS_MAX_STR - string.strncpy(maddr, miss, PA_MISS_MAX_STR - 1) - maddr[PA_MISS_MAX_STR - 1] = 0 - return - if _pa_diag_count >= _pa_diag_max: return - saddr: bytes = _pa_diag_sha1 + sizet(_pa_diag_count) * 17 - string.strncpy(saddr, sha1, 16) - saddr[16] = 0 - daddr: bytes = _pa_diag_deps + sizet(_pa_diag_count) * PA_DEPS_MAX_STR - if deps is not None: string.strncpy(daddr, deps, PA_DEPS_MAX_STR - 1) - daddr[PA_DEPS_MAX_STR - 1] = 0 - if miss is not None and _pa_diag_miss is not None: - maddr2: bytes = _pa_diag_miss + sizet(_pa_diag_count) * PA_MISS_MAX_STR - string.strncpy(maddr2, miss, PA_MISS_MAX_STR - 1) - maddr2[PA_MISS_MAX_STR - 1] = 0 - _pa_diag_count += 1 - - -def _PhaseADiagDepsLookup(sha1: str) -> str: - """Look up the persisted deps string by sha1; caller frees the copy, None if absent""" - if _pa_diag_sha1 is None or _pa_diag_deps is None or sha1 is None: return None - for i in range(_pa_diag_count): - es: bytes = _pa_diag_sha1 + sizet(i) * 17 - if string.strcmp(es, sha1) == 0: - src: bytes = _pa_diag_deps + sizet(i) * PA_DEPS_MAX_STR - ln: sizet = string.strlen(src) - out: bytes = stdlib.malloc(ln + 1) - if out is None: return None - string.strcpy(out, src) - return out - return None - - -def _PhaseADiagMissLookup(sha1: str) -> str: - """Look up the persisted missing detail by sha1; caller frees the copy, None if absent""" - if _pa_diag_sha1 is None or _pa_diag_miss is None or sha1 is None: return None - for i in range(_pa_diag_count): - es: bytes = _pa_diag_sha1 + sizet(i) * 17 - if string.strcmp(es, sha1) == 0: - src: bytes = _pa_diag_miss + sizet(i) * PA_MISS_MAX_STR - ln: sizet = string.strlen(src) - if ln == 0: return None - out: bytes = stdlib.malloc(ln + 1) - if out is None: return None - string.strcpy(out, src) - return out - return None - - -def _PhaseATextLLExists(temp_dir: str, td_len: sizet, sha1: str) -> int: - """Probe whether {temp_dir}/{sha1 first 2 chars}/{sha1}.text.ll exists, 1=yes""" - if temp_dir is None or td_len == 0 or sha1 is None or string.strlen(sha1) < 2: return 0 - tpath: str = StubMergerCollect._sliced_path(temp_dir, td_len, sha1, "text.ll") - if tpath is None: return 0 - tf: fileio.File* = fileio.File(tpath, fileio.MODE.R) - exists: int = 0 - if not tf.closed: - tf.close() - exists = 1 - stdlib.free(tpath) - return exists - - -def _PhaseAResolveDepSha1(mod_name: str) -> str: - """Resolve a dependency module name to SHA1 (exact then suffix), None if absent""" - if mod_name is None: return None - s1: str = StubMergerStore.LookupSha1ByModName(mod_name) - if s1 is not None: return s1 - return StubMergerStore.LookupSha1ByModNameSuffix(mod_name) - - -def _PhaseADumpNode(temp_dir: str, td_len: sizet, sha1: str, depth: int, prefix: bytes) -> None: - """Recursively print the dependency subgraph; [MISS]=missing text.ll, [OK]=ready""" - deps: bytes = None - if sha1 is not None: deps = _PhaseADiagDepsLookup(sha1) - if deps is None and sha1 is not None: - # Fallback: read {temp_dir}/{prefix}/{sha1}.deps.txt (Phase1 already wrote it for includes) - dpath: str = StubMergerCollect._sliced_path(temp_dir, td_len, sha1, "deps.txt") - if dpath is not None: - df: fileio.File* = fileio.File(dpath, fileio.MODE.R) - if not df.closed: - deps = stdlib.malloc(PA_DEPS_MAX_STR) - if deps is not None: - rl: int64 = df.read_all(deps, PA_DEPS_MAX_STR) - if rl < 0: rl = 0 - # read_all's size arg is the buffer capacity (incl. terminator): it reads at most MAX-1 - deps[rl] = 0 - df.close() - stdlib.free(dpath) - if deps is None or deps[0] == 0: - if deps is not None: stdlib.free(deps) - return - dl: sizet = string.strlen(deps) - token_count: int = 0 - pos: sizet = 0 - while pos < dl: - while pos < dl and deps[pos] == ' ': pos += 1 - if pos >= dl: break - token_count += 1 - while pos < dl and deps[pos] != ' ': pos += 1 - pos = 0 - idx: int = 0 - while pos < dl: - while pos < dl and deps[pos] == ' ': pos += 1 - if pos >= dl: break - ts: sizet = pos - while pos < dl and deps[pos] != ' ': pos += 1 - tl: sizet = pos - ts - nm: bytes = stdlib.malloc(tl + 1) - if nm is not None: - string.strncpy(nm, deps + ts, tl) - nm[tl] = 0 - is_last: int = 0 - if idx + 1 >= token_count: is_last = 1 - builtin: int = 0 - if string.strcmp(nm, "t") == 0 or string.strcmp(nm, "c") == 0: builtin = 1 - dep_sha1: str = _PhaseAResolveDepSha1(nm) - txt_ok: int = 0 - if dep_sha1 is not None: txt_ok = _PhaseATextLLExists(temp_dir, td_len, dep_sha1) - # Branch characters are inlined (don't return string literals: TPV would return a dangling - # stack pointer for a literal -> printf %s crashes and the graph output is swallowed) - br_1: str = "|-- " - br_2: str = "`-- " - if builtin != 0: - if is_last != 0: stdio.printf("%s%s%s (builtin)\n", prefix, br_2, nm) - else: stdio.printf("%s%s%s (builtin)\n", prefix, br_1, nm) - elif dep_sha1 is None: - if is_last != 0: stdio.printf("%s%s%s (unresolved)\n", prefix, br_2, nm) - else: stdio.printf("%s%s%s (unresolved)\n", prefix, br_1, nm) - elif txt_ok != 0: - if is_last != 0: stdio.printf("%s%s%s [OK]\n", prefix, br_2, nm) - else: stdio.printf("%s%s%s [OK]\n", prefix, br_1, nm) - else: - if is_last != 0: stdio.printf("%s%s%s [MISS] text.ll=%s\n", prefix, br_2, nm, dep_sha1) - else: stdio.printf("%s%s%s [MISS] text.ll=%s\n", prefix, br_1, nm, dep_sha1) - # Drill down: MISS follows the whole chain (to PA_GRAPH_DEPTH_MAX); OK probes 1 level only - can_expand: int = 0 - if dep_sha1 is not None and builtin == 0: - if txt_ok == 0: - if depth + 1 < PA_GRAPH_DEPTH_MAX: can_expand = 1 - else: - if depth + 1 < PA_GRAPH_OK_PROBE: can_expand = 1 - if can_expand != 0: - # +5: strcat appends "| "/" " (4 chars) + NUL; strlen+4 would overflow 1 byte and - # corrupt heap metadata, crashing later malloc/free and aborting the graph output - npl: sizet = string.strlen(prefix) + 5 - new_prefix: bytes = stdlib.malloc(npl) - if new_prefix is not None: - string.strcpy(new_prefix, prefix) - if is_last != 0: string.strcat(new_prefix, " ") - else: string.strcat(new_prefix, "| ") - _PhaseADumpNode(temp_dir, td_len, dep_sha1, depth + 1, new_prefix) - stdlib.free(new_prefix) - stdlib.free(nm) - idx += 1 - stdlib.free(deps) - - -def _PhaseADumpFailGraph(temp_dir: str, td_len: sizet, path: str, sha1: str) -> None: - """Convergence-failure diagnostics: pending detail plus dependency tree. - Flush after every printf so the trace survives a later crash.""" - stdio.printf("[PhaseA-Fail] ==============================================================\n") - stdio.fflush(None) - stdio.printf("[PhaseA-Fail] FAIL TO TRANSLATE: %s [sha1=%s]\n", path, sha1) - stdio.fflush(None) - miss: bytes = _PhaseADiagMissLookup(sha1) - if miss is not None: - if miss[0] != 0: stdio.printf("[PhaseA-Fail] pending reason: %s\n", miss) - stdlib.free(miss) - stdio.fflush(None) - stdio.printf("[PhaseA-Fail] dep graph ([MISS]=missing text.ll, [OK]=ready, depth<=%d):\n", PA_GRAPH_DEPTH_MAX) - stdio.fflush(None) - root_prefix: bytes = stdlib.malloc(1) - if root_prefix is not None: - root_prefix[0] = 0 - _PhaseADumpNode(temp_dir, td_len, sha1, 0, root_prefix) - stdlib.free(root_prefix) - stdio.fflush(None) - - -def RunMultiFileProject(mb: memhub.MemBuddy*, source_dir: str, temp_dir: str, output_dir: str, cc_cmd: str, cc_flags: str, linker_cmd: str, linker_flags: str, linker_output: str, includes_binary_dir: str, includes_dir: str, do_phase1: int, do_phase2: int, log: VLogger.Logger*, args: argparse.ParsedArgs*, linker_static_libs: str = None, linker_dynamic_libs: str = None) -> int: - """Compile a multi-file project; return 0 on success, non-zero on failure""" - if source_dir is None: - VLogger.error("source_dir is empty", "project") - return 1 - - VLogger.info(f"multi-file project compilation: {source_dir}", "project") - - # === 1. Recursively scan .vp/.py files under source_dir (including subdirectories) === - entry_size: sizet = SrcFileEntry.__sizeof__() - entries: SrcFileEntry* = stdlib.malloc(MAX_SRC_FILES * entry_size) - if entries is None: return 1 - string.memset(entries, 0, MAX_SRC_FILES * entry_size) - - file_count: int = _ScanDirForPyFiles(mb, source_dir, entries, entry_size, 0, MAX_SRC_FILES) - VLogger.info(f"{file_count} source files in total", "project") - - # Scan .s assembly files (separate array; not translated in Phase A, assembled by llvm-mc after Phase B) - asm_entries: SrcFileEntry* = stdlib.malloc(MAX_SRC_FILES * entry_size) - if asm_entries is None: - stdlib.free(entries) - return 1 - string.memset(asm_entries, 0, MAX_SRC_FILES * entry_size) - asm_file_count: int = _ScanDirForAsmFiles(mb, source_dir, asm_entries, entry_size, 0, MAX_SRC_FILES) - if asm_file_count > 0: VLogger.info(f"{asm_file_count} assembly files (assembled by llvm-mc and linked)", "project") - - if file_count == 0: - stdlib.free(entries) - stdlib.free(asm_entries) - return 1 - - # Initialize the AST tables - ast._init_tables(mb) - - # Pre-register App generic class templates before mark2: the tables/nodes are pool-allocated, so they - # must be registered before mark2 to survive reset_to(mark2); otherwise later passes hit dangling memory. - _PreRegisterAppGenericTemplates(mb, entries, file_count, entry_size) - - # OOM cleanup (mirrors Phase1's mark/reset pattern): without reclamation the AST/IR of all includes - # pile up in the pool. mark2 lets each file's transient data be freed by reset_to after translation. - mark2: sizet = mb.mark() - # struct/enum entries registered after mark2 use stdlib allocation (survive reset, as in Phase1) - GenApi.set_stdlib_alloc(1) - - # Ensure the temp/output directories exist - BuildPipeline.ensure_dir(temp_dir) - BuildPipeline.ensure_dir(output_dir) - - # === 0.5 Fill Sha1Store on demand from the dependency graph (removes the Phase1 dependency) === - # If Sha1Store is empty, scan the dependency graph (App-first BFS) so later "compile missing includes" works. - if includes_dir is not None: - if StubMergerStore.GetSha1StoreCount() == 0: - rs_05: bytes = stdlib.malloc(StubMerger.MAX_INCLUDES_SHA1 * 17) - if rs_05 is None: VLogger.error("reachable_set allocation failed, cannot fill Sha1Store", "project") - else: - string.memset(rs_05, 0, StubMerger.MAX_INCLUDES_SHA1 * 17) - rc_05: int = StubMergerBuild._BuildReachableSha1Set(mb, source_dir, temp_dir, rs_05, None, includes_dir) - if rc_05 < 0: VLogger.error("dependency graph scan failed, cannot fill Sha1Store", "project") - elif rc_05 > 0: VLogger.info(f"Phase2 dependency graph filled Sha1Store on demand: {rc_05} entries", "project") - stdlib.free(rs_05) - - # Append App source SHA1s to _sha1_map.txt (human-readable only; format {sha1}:{rel_path}) and to the - # in-memory store (AppendToSha1MapStore) for cross-module CDefine lookups. - td_len_am: sizet = string.strlen(temp_dir) - src_dir_len_am: sizet = string.strlen(source_dir) - map_path_am: bytes = stdlib.malloc(td_len_am + 32) - if map_path_am is not None: - viperlib.snprintf(map_path_am, td_len_am + 32, "%s/_sha1_map.txt", temp_dir) - mf: fileio.File* = fileio.File(map_path_am, fileio.MODE.A) - line_am: bytes = stdlib.malloc(512) - # First fill the in-memory store (no file I/O), then write _sha1_map.txt - app_filled: int = 0 - for i in range(file_count): - ea_am: uint64 = uint64(entries) + i * entry_size - ent_am: SrcFileEntry* = (SrcFileEntry*)(ea_am) - if ent_am is None or ent_am.Path is None or ent_am.Sha1 is None: continue - # Compute the relative path (strip the source_dir prefix, keep the directory structure) - p_str: str = ent_am.Path - p_len: sizet = string.strlen(p_str) - rel_path_am: str = p_str - if p_len > src_dir_len_am + 1: - if string.strncmp(p_str, source_dir, src_dir_len_am) == 0: rel_path_am = p_str + src_dir_len_am + 1 - # Also append to the in-memory store (for cross-module CDefine lookups) - # rel_path_am looks like "lib/core/StubMerger.vp", matching _sha1_map.txt - if StubMergerStore.AppendToSha1MapStore(ent_am.Sha1, rel_path_am) != 0: VLogger.error(f"AppendToSha1MapStore failed: {rel_path_am}", "project") - app_filled += 1 - # Write _sha1_map.txt (human-readable output) - if not mf.closed and line_am is not None: - viperlib.snprintf(line_am, 512, "%s:%s\n", ent_am.Sha1, rel_path_am) - ll_am: sizet = string.strlen(line_am) - mf.write(line_am, ll_am) - if line_am is not None: stdlib.free(line_am) - if not mf.closed: - mf.close() - VLogger.info(f"appended {app_filled} App files to _sha1_map.txt and the in-memory store", "project") - stdlib.free(map_path_am) - - # The module SHA1 map is filled into the StubMerger global store by PopulateSha1MapStore + - # AppendToSha1MapStore; HandlesExprCall._lookup_module_sha1 queries it directly, no local copy needed. - - - # === Phase A-pre-vpi: pre-generate .vpi stubs for all App sources === - # Deps' .vpi must exist before their dependents translate; pre-generating avoids missing dep declarations. - td_len_vpa: sizet = string.strlen(temp_dir) - PYI_BUF_SIZE_P2: sizet = 262144 - vpi_failed_p2: int = 0 - for i_vp2 in range(file_count): - ea_vp2: uint64 = uint64(entries) + i_vp2 * entry_size - ent_vp2: SrcFileEntry* = (SrcFileEntry*)(ea_vp2) - if ent_vp2 is None or ent_vp2.Path is None or ent_vp2.Sha1 is None: - vpi_failed_p2 += 1 - VLogger.error("[Phase A-pre-vpi] invalid source entry (Path/Sha1 missing), skipping", "PhaseA") - continue - # Skip if the vpi already exists (sha1 is content-based, so a same-named vpi is valid; cross-run cache) - vpi_path_vp2: str = StubMergerCollect._sliced_path(temp_dir, td_len_vpa, ent_vp2.Sha1, "vpi") - if vpi_path_vp2 is None: - vpi_failed_p2 += 1 - VLogger.error(f"[Phase A-pre-vpi] {ent_vp2.Path} | failed to build vpi path", "PhaseA") - continue - vf_vp2: fileio.File* = fileio.File(vpi_path_vp2, fileio.MODE.R) - if not vf_vp2.closed: - vf_vp2.close() - stdlib.free(vpi_path_vp2) - continue - stdlib.free(vpi_path_vp2) - # Read the source -> parse the AST -> generate the vpi (rel_path = ent_vp2.Path: vpi content does - # not depend on it; it is only used for fatal_error attribution, pointing at this source file) - f_vp2: fileio.File* = fileio.File(ent_vp2.Path, fileio.MODE.R) - if f_vp2.closed: - vpi_failed_p2 += 1 - VLogger.error(f"[Phase A-pre-vpi] {ent_vp2.Path} | cannot open source file", "PhaseA") - continue - # read_all's capacity includes the terminator (reads at most cap-1 bytes); longer files are truncated - if f_vp2.size() >= SRC_BUF_SIZE: - vpi_failed_p2 += 1 - VLogger.error(f"[Phase A-pre-vpi] {ent_vp2.Path} | source file exceeds {SRC_BUF_SIZE} byte limit", "PhaseA") - f_vp2.close() - continue - src_buf_vp2: bytes = stdlib.malloc(SRC_BUF_SIZE) - if src_buf_vp2 is None: - f_vp2.close() - vpi_failed_p2 += 1 - VLogger.error(f"[Phase A-pre-vpi] {ent_vp2.Path} | source buffer allocation failed (OOM)", "PhaseA") - continue - bytes_read_vp2: LONG = f_vp2.read_all(src_buf_vp2, SRC_BUF_SIZE) - f_vp2.close() - if bytes_read_vp2 <= 0: - stdlib.free(src_buf_vp2) - vpi_failed_p2 += 1 - VLogger.error(f"[Phase A-pre-vpi] {ent_vp2.Path} | file read failed or empty", "PhaseA") - continue - if bytes_read_vp2 < SRC_BUF_SIZE: src_buf_vp2[bytes_read_vp2] = 0 - lx_vp2: ast.Lexer* = ast.new_lexer(mb) - if lx_vp2 is None: - stdlib.free(src_buf_vp2) - vpi_failed_p2 += 1 - VLogger.error(f"[Phase A-pre-vpi] {ent_vp2.Path} | lexer allocation failed (OOM)", "PhaseA") - continue - ast._lexer_init(lx_vp2, src_buf_vp2, mb) - tokens_vp2: ast.Token* = ast.tokenize(lx_vp2) - tree_vp2: ast.AST* = ast.parse_tokens(mb, tokens_vp2) - if tree_vp2 is None: - stdlib.free(src_buf_vp2) - vpi_failed_p2 += 1 - # Parse failure: inline the line/column + reason in the log line (same source as the full - # Python-style traceback below), since "AST parse failed" alone gives no location or cause. - VLogger.error(f"[Phase A-pre-vpi] {ent_vp2.Path} | AST parse failed (line {ast.get_parse_err_line()} col {ast.get_parse_err_col()}: {ast.get_parse_err_msg()})", "PhaseA") - GenApi.dump_parse_traceback(ent_vp2.Path) - continue - pyi_buf_vp2: bytes = stdlib.malloc(PYI_BUF_SIZE_P2) - if pyi_buf_vp2 is not None: - # 文件级名字窗口（FileFuncNames 等）由 _GeneratePyiFromAst 入口统一 reset+重建， - # 不再在此处重复调用；契约见 StructGenMixin.reset_file_name_window。 - pyi_pos_vp2: sizet = StubConverter._GeneratePyiFromAst(mb, tree_vp2, ent_vp2.Path, pyi_buf_vp2, PYI_BUF_SIZE_P2) - if pyi_pos_vp2 > 0: - pyi_path_vp2: str = StubMergerCollect._sliced_path(temp_dir, td_len_vpa, ent_vp2.Sha1, "vpi") - if pyi_path_vp2 is not None: - pf_vp2: fileio.File* = fileio.File(pyi_path_vp2, fileio.MODE.W) - if not pf_vp2.closed: - pf_vp2.write(pyi_buf_vp2, pyi_pos_vp2) - pf_vp2.close() - stdlib.free(pyi_path_vp2) - stdlib.free(pyi_buf_vp2) - else: - vpi_failed_p2 += 1 - VLogger.error(f"[Phase A-pre-vpi] {ent_vp2.Path} | pyi buffer allocation failed (OOM)", "PhaseA") - stdlib.free(src_buf_vp2) - # OOM cleanup: free this file's AST (the vpi is already on disk; persistent data is stdlib-allocated) - mb.reset_to(mark2) - if vpi_failed_p2 > 0: - VLogger.error(f"Phase A-pre-vpi: {vpi_failed_p2} files failed vpi pre-generation (see the per-file [Phase A-pre-vpi] logs above for failing files and causes)", "PhaseA") - return 1 - - - # === 2. Phase A: generate stub + text for each file === - # app_deps_buf collects App files' direct deps (space-separated) as Phase B+'s worklist (65536, matching StubMerger). - APP_DEPS_BUF_SIZE: sizet = 65536 - app_deps_buf: bytes = None - app_deps_len: sizet = 0 - if do_phase1 != 0: - log.banner("Phase A: generate stub + text") - # Initialize the Phase A convergence-failure diagnostic table (stdlib, survives reset) - _PhaseADiagInit(sizet(file_count) + StubMerger.MAX_INCLUDES) - - # === Phase A-pre-inc: pre-register includes structs + fill deps === - # Must run before App pre-registration (multi-pass resolves cross-file inheritance) and fills PopulateIncludesDeps. - if includes_dir is not None and includes_binary_dir is not None: - store_count_inc_pre: int = StubMergerStore.GetSha1StoreCount() - if store_count_inc_pre > 0: - store_sha1_inc_pre: bytes = StubMergerStore.GetSha1StoreArrPtr() - store_rel_inc_pre: bytes = StubMergerStore.GetSha1StoreRelArrPtr() - store_incdir_inc_pre: bytes = StubMergerStore.GetSha1StoreIncdirArrPtr() - deps_filled_pre: int = 0 - PHASE_A_INC_MAX_PASSES: int = 3 - # Track includes that fail pre-registration: after all passes, any failure aborts the build - # (no silent downgrade). App files must be skipped here; Phase A-pre handles them. - inc_fail_flags: bytes = stdlib.malloc(store_count_inc_pre) - if inc_fail_flags is not None: string.memset(inc_fail_flags, 1, store_count_inc_pre) - for pass_inc_i in range(PHASE_A_INC_MAX_PASSES): - struct_count_before_inc: int = GenApi.get_struct_count() - for si_inc in range(store_count_inc_pre): - inc_sha1_pre: str = store_sha1_inc_pre + sizet(si_inc) * 17 - inc_rel_pre: str = store_rel_inc_pre + sizet(si_inc) * StubMerger.MAX_REL_PATH_LEN - if inc_rel_pre is None or inc_rel_pre[0] == '\0': continue - # Skip App source files (main.vp etc.): pre-registered by Phase A-pre, not a failure - is_app_file_inc: int = 0 - for i_app_inc in range(file_count): - ea_app_inc: uint64 = uint64(entries) + i_app_inc * entry_size - ent_app_inc: SrcFileEntry* = (SrcFileEntry*)(ea_app_inc) - if ent_app_inc is not None and ent_app_inc.Sha1 is not None and string.strcmp(ent_app_inc.Sha1, inc_sha1_pre) == 0: - is_app_file_inc = 1 - break - if is_app_file_inc != 0 and inc_fail_flags is not None: - inc_fail_flags[si_inc] = 0 - continue - rel_len_pre: sizet = string.strlen(inc_rel_pre) - # The includes root dir (per-entry from the IncDir array; empty falls back to includes_dir) - incdir_pre: str = store_incdir_inc_pre + sizet(si_inc) * StubMerger.MAX_INC_DIR_LEN - if incdir_pre is None or incdir_pre[0] == '\0': incdir_pre = includes_dir - incdir_len_pre: sizet = string.strlen(incdir_pre) - full_path_pre: bytes = stdlib.malloc(incdir_len_pre + rel_len_pre + 2) - if full_path_pre is None: continue - viperlib.snprintf(full_path_pre, incdir_len_pre + rel_len_pre + 2, "%s/%s", incdir_pre, inc_rel_pre) - pkg_inc_pre: str = GenApi.compute_package_from_relpath(mb, inc_rel_pre) - tr_inc_pre: HandlesTranslator.Translator* = BuildPipeline.TranslateFileGetTrans(mb, full_path_pre, inc_sha1_pre, pkg_inc_pre, 1) - stdlib.free(full_path_pre) - if tr_inc_pre is not None: - # Record this file's deps (for convergence diagnostics; stdlib table survives reset) - _PhaseADiagRecord(inc_sha1_pre, tr_inc_pre._imported_modules, None) - # Pre-registration succeeded: clear the failure flag - if inc_fail_flags is not None: inc_fail_flags[si_inc] = 0 - # Fill deps only on the first pass (PopulateIncludesDeps dedupes internally). - # Must be called before reset_to(mark2) since it deep-copies into stdlib. - if pass_inc_i == 0 and tr_inc_pre._imported_modules is not None and StubMergerStore.PopulateIncludesDeps(inc_sha1_pre, tr_inc_pre._imported_modules) == 0: - deps_filled_pre = deps_filled_pre + 1 - # OOM cleanup: free this includes file's transient pre-registration data - # (AST/tokens/lexer/IR/Handler); stdlib-allocated struct entries are unaffected - mb.reset_to(mark2) - struct_count_after_inc: int = GenApi.get_struct_count() - # Convergence check: no new struct registered this pass -> all classes are registered - if struct_count_after_inc == struct_count_before_inc: break - # Includes still failing after all passes -> abort (no downgrade) - if inc_fail_flags is not None: - for si_inc_f in range(store_count_inc_pre): - if inc_fail_flags[si_inc_f] != 0: - rel_f: str = store_rel_inc_pre + sizet(si_inc_f) * StubMerger.MAX_REL_PATH_LEN - VLogger.error(f"includes pre-registration failed: {rel_f} (file cannot be translated; possible syntax/parse/compile error), aborting compilation", "prePhaseA") - stdlib.free(inc_fail_flags) - VLogger.info(f"includes pre-registration + dependency fill: {deps_filled_pre}/{store_count_inc_pre}", "prePhaseA") - - # === Phase A-pre: pre-register struct/enum/union for all sources === - # Registers declarations only (declare_only=1), no method bodies; multi-pass resolves cross-file ordering. - VLogger.info("pre-registering struct/enum/union...", "PhaseA") - src_dir_len_pre: sizet = string.strlen(source_dir) - PHASE_A_PRE_MAX_PASSES: int = 3 - for pass_i in range(PHASE_A_PRE_MAX_PASSES): - struct_count_before: int = GenApi.get_struct_count() - for i in range(file_count): - ea_pre: uint64 = uint64(entries) + i * entry_size - ent_pre: SrcFileEntry* = (SrcFileEntry*)(ea_pre) - if ent_pre is None or ent_pre.Path is None: continue - pkg_pre: str = None - if string.strlen(ent_pre.Path) > src_dir_len_pre + 1: - rel_path_pre: str = ent_pre.Path + src_dir_len_pre + 1 - pkg_pre = GenApi.compute_package_from_relpath(mb, rel_path_pre) - # App pre-registration passes pre_allow_dep_inject=1: imports and dep vpi injection happen at - # declaration time so field annotations share Phase A's context (else struct layouts diverge). - tr_pre: HandlesTranslator.Translator* = BuildPipeline.TranslateFileGetTrans(mb, ent_pre.Path, ent_pre.Sha1, pkg_pre, 1, None, 1) - if tr_pre is None: VLogger.error(f"pre-registration failed: {ent_pre.Path}", "PhaseA") - # OOM cleanup: free this App file's transient pre-registration data - mb.reset_to(mark2) - struct_count_after: int = GenApi.get_struct_count() - # Convergence check: no new struct registered this pass -> all classes are registered - if struct_count_after == struct_count_before: break - VLogger.info("pre-registration complete", "PhaseA") - - # Phase A IR buffer: 1MB is not enough, ast/parser.vp's text.ll exceeds 1MB and got truncated -> - # merged IR lost the tail (opt: "end of file in global variable name"). - PHASE_A_IR_SIZE: sizet = 8388608 - # FULL dump = stub + text (same module), with 2x headroom - PHASE_A_FULL_SIZE: sizet = 16777216 - td_len_pa: sizet = string.strlen(temp_dir) - - # Set the temp dir so HandlesExprCall can read dependency modules' text.ll for cross-module - # method return-type lookup (avoids pointer returns being truncated to i32) - GenApi.SetTempDir(temp_dir) - - # app_deps_buf collects App files' direct dependency modules (space-separated) as Phase B+'s initial - # worklist for _BuildReachableSha1Set; declared outside the if, only assigned here. - app_deps_buf = stdlib.malloc(APP_DEPS_BUF_SIZE) - app_deps_len = 0 - if app_deps_buf is not None: app_deps_buf[0] = '\0' - - # Phase A multi-pass retry: with several sources, main.vp may translate before its imports - # (filesystem order); failed files are retried next pass, converging in at most file_count passes. - done_flags: bytes = stdlib.malloc(file_count) - if done_flags is None: - VLogger.error("Phase A done_flags allocation failed", "PhaseA") - stdlib.free(entries) - return 1 - string.memset(done_flags, 0, file_count) - PHASE_A_MAX_PASSES: int = 32 - for pass_a in range(PHASE_A_MAX_PASSES): - completed_this_pass: int = 0 - for i in range(file_count): - if done_flags[i] != 0: continue - ea: uint64 = uint64(entries) + i * entry_size - ent: SrcFileEntry* = (SrcFileEntry*)(ea) - # Timing point 0: start processing this file (translate phase start) - t_pa0: uint64 = w32.win32base.GetTickCount64() - if ent is None or ent.Path is None: - done_flags[i] = 1 - completed_this_pass += 1 - continue - # Compute the package name (the directory part relative to source_dir) - src_dir_len_pa: sizet = string.strlen(source_dir) - pkg_pa: str = None - if string.strlen(ent.Path) > src_dir_len_pa + 1: - rel_path_pa: str = ent.Path + src_dir_len_pa + 1 - pkg_pa = GenApi.compute_package_from_relpath(mb, rel_path_pa) - tr_a: HandlesTranslator.Translator* = BuildPipeline.TranslateFileGetTrans(mb, ent.Path, ent.Sha1, pkg_pa, 0, temp_dir) - # Timing point 1: translate returned - t_pa1: uint64 = w32.win32base.GetTickCount64() - if tr_a is None: - # Translation failed (syntax/parse error): retry next pass; convergence check reports it. - # OOM cleanup: a failed translation also produced AST/tokens, freed immediately. - mb.reset_to(mark2) - continue - # Record this file's deps (for convergence diagnostics; stdlib table survives reset) - _PhaseADiagRecord(ent.Sha1, tr_a._imported_modules, None) - # Collect _imported_modules into app_deps_buf (for Phase B+ on-demand compilation). - # Dependencies were filled early during translation. - if tr_a._imported_modules is not None and app_deps_buf is not None: - # Dedupe per module: several App files importing the same module would otherwise inflate - # app_deps toward 64KB, filling the initial worklist and overflowing BFS. - im_len_a: sizet = string.strlen(tr_a._imported_modules) - im_pos_a: sizet = 0 - while im_pos_a < im_len_a: - im_start_a: sizet = im_pos_a - while im_pos_a < im_len_a and tr_a._imported_modules[im_pos_a] != ' ': im_pos_a += 1 - im_mod_len: sizet = im_pos_a - im_start_a - if im_pos_a < im_len_a: im_pos_a += 1 - if im_mod_len == 0: continue - im_mod_buf: bytes = stdlib.malloc(im_mod_len + 1) - if im_mod_buf is None: continue - string.strncpy(im_mod_buf, tr_a._imported_modules + im_start_a, im_mod_len) - im_mod_buf[im_mod_len] = '\0' - # Skip if already in app_deps (exact word-boundary match) - if app_deps_len > 0 and GenApi.is_module_imported(app_deps_buf, im_mod_buf) != 0: - stdlib.free(im_mod_buf) - continue - if app_deps_len + im_mod_len + 2 < APP_DEPS_BUF_SIZE: - if app_deps_len > 0: - app_deps_buf[app_deps_len] = ' ' - app_deps_len = app_deps_len + 1 - string.strcpy(app_deps_buf + app_deps_len, im_mod_buf) - app_deps_len = app_deps_len + im_mod_len - app_deps_buf[app_deps_len] = '\0' - stdlib.free(im_mod_buf) - # dump stub IR (declarations only) - stub_buf_a: bytes = stdlib.malloc(PHASE_A_IR_SIZE) - if stub_buf_a is None: VLogger.error("stub buffer allocation failed", "PhaseA") - tr_a.dump_ir(stub_buf_a, PHASE_A_IR_SIZE, mlirlite.OUTPUT_STUB) - stub_len_a: sizet = string.strlen(stub_buf_a) - - # Save stub.ll (sliced path: temp_dir/{sha1 prefix}/{sha1}.stub.ll) - stub_path_a: str = StubMergerCollect._sliced_path(temp_dir, td_len_pa, ent.Sha1, "stub.ll") - if stub_path_a is not None: - sf_a: fileio.File* = fileio.File(stub_path_a, fileio.MODE.W) - if not sf_a.closed: - sf_a.write(stub_buf_a, stub_len_a) - sf_a.close() - stdlib.free(stub_path_a) - stdlib.free(stub_buf_a) - - # dump text IR (definitions only) - text_buf_a: bytes = stdlib.malloc(PHASE_A_IR_SIZE) - if text_buf_a is None: VLogger.error("text buffer allocation failed", "PhaseA") - tr_a.dump_ir(text_buf_a, PHASE_A_IR_SIZE, mlirlite.OUTPUT_TEXT) - text_len_a: sizet = string.strlen(text_buf_a) - - # Save text.ll (sliced path) - text_path_a: str = StubMergerCollect._sliced_path(temp_dir, td_len_pa, ent.Sha1, "text.ll") - if text_path_a is not None: - tf_a: fileio.File* = fileio.File(text_path_a, fileio.MODE.W) - if not tf_a.closed: - tf_a.write(text_buf_a, text_len_a) - tf_a.close() - stdlib.free(text_path_a) - stdlib.free(text_buf_a) - - # Save {sha1}.meta (structured sidecar: function list / struct layout / vtable) - StubMergerIR.WriteModuleMeta(tr_a.Module, temp_dir, td_len_pa, ent.Sha1) - - # Save {sha1}.full.ll (in-memory FULL artifact: a self-contained single-module IR) - full_buf_a: bytes = stdlib.malloc(PHASE_A_FULL_SIZE) - if full_buf_a is None: VLogger.error("full.ll buffer allocation failed", "PhaseA") - tr_a.dump_ir(full_buf_a, PHASE_A_FULL_SIZE, mlirlite.OUTPUT_FULL) - full_len_a: sizet = string.strlen(full_buf_a) - full_path_a: str = StubMergerCollect._sliced_path(temp_dir, td_len_pa, ent.Sha1, "full.ll") - if full_path_a is not None: - ffa: fileio.File* = fileio.File(full_path_a, fileio.MODE.W) - if not ffa.closed: - ffa.write(full_buf_a, full_len_a) - ffa.close() - stdlib.free(full_path_a) - stdlib.free(full_buf_a) - - # --traceback: take this module's debug call-site records and write .vpcall. - if BuildPipeline._traceback_mode != 0: - vpcall_text_a: str = mlirlite.DebugTakeText() - if vpcall_text_a is not None: - BuildPipeline.WriteCallSidecar(temp_dir, ent.Sha1, vpcall_text_a) - stdlib.free(vpcall_text_a) - - # Save dependencies (_imported_modules) for Phase B (sliced path) - deps_path_a: str = StubMergerCollect._sliced_path(temp_dir, td_len_pa, ent.Sha1, "deps.txt") - if deps_path_a is not None: - df_a: fileio.File* = fileio.File(deps_path_a, fileio.MODE.W) - if not df_a.closed: - if tr_a._imported_modules is not None: - dl_a: sizet = string.strlen(tr_a._imported_modules) - df_a.write(tr_a._imported_modules, dl_a) - df_a.close() - stdlib.free(deps_path_a) - - # Timing point 2: stub+text dump and write done; print per-file progress (as in Phase1 1b) - t_pa2: uint64 = w32.win32base.GetTickCount64() - VLogger.info(f"[{i+1}/{file_count}] PhaseA: {ent.Path} (translate={t_pa1-t_pa0} ms, dump={t_pa2-t_pa1} ms, total={t_pa2-t_pa0} ms)", "PhaseA") - - done_flags[i] = 1 - completed_this_pass += 1 - # OOM cleanup: stub/text/deps are on disk, so free this file's transient translation data - # (AST/tokens/lexer/IR/Handler); app_deps_buf already deep-copied the deps. - mb.reset_to(mark2) - # Convergence check: stop when all files are done - all_done: int = 1 - for i_ck in range(file_count): - if done_flags[i_ck] == 0: - all_done = 0 - break - if all_done != 0: break - if completed_this_pass == 0: - # No progress: remaining files cannot finish (real compile errors). Abort and list them, each - # with a dependency-graph dump. VLogger.error calls sys.exit(1), so it must come after all output. - td_len_fail: sizet = string.strlen(temp_dir) - for i_err in range(file_count): - if done_flags[i_err] == 0: - ea_err: uint64 = uint64(entries) + i_err * entry_size - ent_err: SrcFileEntry* = (SrcFileEntry*)(ea_err) - if ent_err is not None and ent_err.Path is not None: - if ent_err.Sha1 is not None: _PhaseADumpFailGraph(temp_dir, td_len_fail, ent_err.Path, ent_err.Sha1) - stdlib.free(done_flags) - _PhaseADiagFree() - VLogger.error("Phase A multi-pass convergence failed, aborting compilation", "PhaseA") - stdlib.free(entries) - return 1 - stdlib.free(done_flags) - # Phase A converged: free the diagnostic table (needed only on the failure path) - _PhaseADiagFree() - - if do_phase2 == 0: - VLogger.info("Phase A complete (stub generation only)", "project") - if app_deps_buf is not None: stdlib.free(app_deps_buf) - stdlib.free(entries) - return 0 - - # === 3. Phase B: compile each file to .obj === - if do_phase2 != 0: - log.banner("Phase B: compile .obj") - - # Collect .obj paths (64KB to avoid includes .obj path overflow dropping link symbols). - # reachable_set/reachable_count: filled by Phase B+, used by Phase C for on-demand linking. - reachable_set: bytes = None - reachable_count: int = 0 - OBJ_PATHS_SIZE: sizet = 65536 - obj_paths: bytes = stdlib.malloc(OBJ_PATHS_SIZE) - if obj_paths is None: return 1 - obj_paths[0] = '\0' - obj_pos: sizet = 0 - # main_obj_path: the .obj that defines the user main (placed first at link time so the linker - # does not pick another module's wrapper main) - main_obj_path: bytes = stdlib.malloc(512) - if main_obj_path is None: return 1 - main_obj_path[0] = '\0' - compiled_count: int = 0 - - # IR buffer size (16MB, to hold the in-memory FULL artifact {sha1}.full.ll; FULL = stub + text) - COMBINED_IR_SIZE: sizet = 16777216 - - # llc task table (stdlib.malloc, outlives threads): a contiguous array of LlcTask records; the loop - # only writes .ll + builds tasks, then run_llc_parallel runs llc in parallel batches. - llc_tasks: bytes = stdlib.malloc(sizet(file_count) * BuildPipeline.LlcTask.__sizeof__()) - if llc_tasks is None: - VLogger.error("llc task table allocation failed", "PhaseB") - return 1 - llc_task_count: int = 0 - - for i in range(file_count): - ea: uint64 = uint64(entries) + i * entry_size - ent: SrcFileEntry* = (SrcFileEntry*)(ea) - if ent is None or ent.Path is None or ent.Sha1 is None: continue - # Start of this file's three-step timing (GetTickCount64, ms) - t_pb0: uint64 = w32.win32base.GetTickCount64() - # Read the in-memory FULL artifact {sha1}.full.ll -> complete IR (replaces merging stub + dep stub/text) - combined_ir: bytes = stdlib.malloc(COMBINED_IR_SIZE) - if combined_ir is None: - VLogger.error(f"combined_ir allocation failed: {ent.Path}", "PhaseB") - continue - combined_len: sizet = StubMergerBuild.LoadFullIR(temp_dir, ent.Sha1, combined_ir, COMBINED_IR_SIZE) - if combined_len == 0: - VLogger.error(f"failed to read full.ll: {ent.Path}", "PhaseB") - stdlib.free(combined_ir) - continue - # Timing point 1: full.ll read done - t_pb1: uint64 = w32.win32base.GetTickCount64() - - # Write .ll (llc threads only read this file); llc itself runs parallel in batches after the loop - ret_w: int = BuildPipeline.write_ir_to_file(combined_ir, combined_len, temp_dir, ent.Sha1) - stdlib.free(combined_ir) - if ret_w != 0: - VLogger.error(f"failed to write .ll: {ent.Path}", "PhaseB") - sys.exit(1) - # Timing point 2: .ll write done - t_pb2: uint64 = w32.win32base.GetTickCount64() - - # Build the llc task (command string is stdlib.malloc'd so threads' lifetimes are not reset-reclaimed) - llc_ir_path: str = BuildPipeline.build_sliced_path(temp_dir, ent.Sha1, "ll") - if llc_ir_path is None: - VLogger.error("failed to build llc input path", "PhaseB") - sys.exit(1) - tslot: BuildPipeline.LlcTask* = BuildPipeline.LlcTask*(uint64(llc_tasks) + sizet(llc_task_count) * BuildPipeline.LlcTask.__sizeof__()) - if BuildPipeline.init_llc_task(tslot, llc_ir_path, output_dir, ent.Sha1, cc_cmd, cc_flags) != 0: - VLogger.error(f"failed to build llc task: {ent.Path}", "PhaseB") - sys.exit(1) - # Timing point 3: llc task construction done - t_pb3: uint64 = w32.win32base.GetTickCount64() - llc_task_count += 1 - - compiled_count += 1 - # Show files submitted to llc (with the three timings: merge stub / write .ll / build task) - VLogger.info(f"[{i+1}/{file_count}] llc submitted: {ent.Path} (merge={t_pb1-t_pb0} ms, write={t_pb2-t_pb1} ms, task={t_pb3-t_pb2} ms, total={t_pb3-t_pb0} ms)", "PhaseB") - # Build the .obj path and detect the main module (test_main.vp or main.vp; .py no longer recognized) - od_len: sizet = string.strlen(output_dir) - is_main_mod: int = 0 - if string.strstr(ent.Path, "test_main.vp") is not None: is_main_mod = 1 - elif string.strstr(ent.Path, "main.vp") is not None: is_main_mod = 1 - - # Sliced path: output_dir/{sha1 prefix}/{sha1}.obj - obj_path_sliced: str = StubMergerCollect._sliced_path(output_dir, od_len, ent.Sha1, "obj") - if obj_path_sliced is not None: - op_sliced_len: sizet = string.strlen(obj_path_sliced) - if is_main_mod != 0: - # main module: copy to main_obj_path (so main is first at link time) - if op_sliced_len < 512: string.strcpy(main_obj_path, obj_path_sliced) - else: VLogger.error("main_obj_path buffer too small, aborting compilation", "PhaseB") - else: - # Other modules: append to obj_paths - if obj_pos + op_sliced_len + 2 < OBJ_PATHS_SIZE: - if obj_pos > 0: - obj_paths[obj_pos] = ' ' - obj_pos += 1 - string.strcpy(obj_paths + obj_pos, obj_path_sliced) - obj_pos += op_sliced_len - obj_paths[obj_pos] = '\0' - else: VLogger.error(".obj path buffer too small", "PhaseB") - stdlib.free(obj_path_sliced) - - # Run all llc jobs in parallel (batched at LLC_MAX_THREADS threads) - ret_p: int = BuildPipeline.run_llc_parallel(llc_tasks, llc_task_count, BuildPipeline.LLC_MAX_THREADS) - if ret_p != 0: - VLogger.error("llc compilation failed, aborting compilation", "PhaseB") - sys.exit(1) - - # Free the llc tasks (command strings + records) - ti: int = 0 - while ti < llc_task_count: - tfree: BuildPipeline.LlcTask* = BuildPipeline.LlcTask*(uint64(llc_tasks) + sizet(ti) * BuildPipeline.LlcTask.__sizeof__()) - BuildPipeline.free_llc_task(tfree) - ti += 1 - stdlib.free(llc_tasks) - - # Assemble .s files: llvm-mc -> .obj (isr.s etc., bare symbols matching the .vp CExport/State stubs). - # The obj is added to obj_paths for Phase C linking. - od_len_as: sizet = string.strlen(output_dir) - asm_compiled: int = 0 - for ai_as in range(asm_file_count): - ea_as: uint64 = uint64(asm_entries) + ai_as * entry_size - ent_as: SrcFileEntry* = (SrcFileEntry*)(ea_as) - if ent_as is None or ent_as.Path is None or ent_as.Sha1 is None: continue - aret_as: int = BuildPipeline.asm_to_obj(ent_as.Path, ent_as.Sha1, output_dir, Config.TargetTriple) - if aret_as != 0: - VLogger.error(f"llvm-mc assembly failed, aborting compilation: {ent_as.Path}", "PhaseB") - sys.exit(1) - obj_path_asm: str = StubMergerCollect._sliced_path(output_dir, od_len_as, ent_as.Sha1, "obj") - if obj_path_asm is not None: - op_len_asm: sizet = string.strlen(obj_path_asm) - if obj_pos + op_len_asm + 2 < OBJ_PATHS_SIZE: - if obj_pos > 0: - obj_paths[obj_pos] = ' ' - obj_pos += 1 - string.strcpy(obj_paths + obj_pos, obj_path_asm) - obj_pos += op_len_asm - obj_paths[obj_pos] = '\0' - stdlib.free(obj_path_asm) - asm_compiled += 1 - VLogger.info(f"[asm {asm_compiled}] assembly complete: {ent_as.Path}", "PhaseB") - - VLogger.success(f"compilation complete: {compiled_count}/{file_count}", "PhaseB") - - if compiled_count == 0: - VLogger.error("no files compiled successfully", "PhaseB") - stdlib.free(entries) - stdlib.free(asm_entries) - return 1 - - # === 3.5 Compile missing includes (on demand via the dependency graph) === - # Only compile includes reached directly/indirectly from App sources (via _BuildReachableSha1Set). - if includes_dir is not None and includes_binary_dir is not None: - inc_compiled: int = 0 - td_len_mi: sizet = string.strlen(temp_dir) - - # Diagnostic log: confirm the Phase B+ entry and Sha1Store state - VLogger.info(f"PhaseB+ entry: store_count={StubMergerStore.GetSha1StoreCount()} includes_binary_dir={includes_binary_dir}", "PhaseB+") - - # Create includes_binary_dir (so .obj files can be written and collect_obj_files can scan them) - BuildPipeline.ensure_dir(includes_binary_dir) - - # declare_only preprocessing was moved before Phase A (Phase A-pre-inc) so includes structs are - # registered when Phase A translates App sources. PopulateIncludesDeps is filled there too. - - # Build the reachable SHA1 set (dependency graph traversal). - # When app_deps_buf is None, fall back to scanning source_dir (non-recursive). - REACHABLE_SET_SIZE: sizet = sizet(StubMerger.MAX_INCLUDES_SHA1) * 17 - reachable_set: bytes = stdlib.malloc(REACHABLE_SET_SIZE) - reachable_count: int = 0 - if reachable_set is not None: - string.memset(reachable_set, 0, REACHABLE_SET_SIZE) - reachable_count = StubMergerBuild._BuildReachableSha1Set(mb, source_dir, temp_dir, reachable_set, app_deps_buf, includes_dir) - VLogger.info(f"reachable includes SHA1: {reachable_count} entries", "PhaseB+") - # fast-fail: reachable-set construction failed (deps not found); never fall back, abort - if reachable_count < 0: - VLogger.error("reachable set construction failed (fast-fail), aborting compilation", "PhaseB+") - return 1 - else: - VLogger.error("reachable_set allocation failed, aborting compilation", "PhaseB+") - return 1 - - # Free app_deps_buf (reachable_set is built, no longer needed) - if app_deps_buf is not None: - stdlib.free(app_deps_buf) - app_deps_buf = None - - # Get the SHA1/module-name arrays from the global store as local copies (bytes) to avoid address - # issues from passing the store's union return (bytes*) across functions; rel/incdir keep pointers. - td_len_pb: sizet = string.strlen(temp_dir) - store_sha1_arr_mi: bytes = stdlib.malloc(StubMerger.MAX_INCLUDES * 17) - store_mod_arr_mi: bytes = stdlib.malloc(StubMerger.MAX_INCLUDES * 64) - store_count_mi: int = 0 - if store_sha1_arr_mi is not None and store_mod_arr_mi is not None: - store_count_mi = StubMergerIR._BuildIncludesSha1Map(temp_dir, td_len_pb, store_sha1_arr_mi, store_mod_arr_mi) - store_rel_arr_mi: bytes = StubMergerStore.GetSha1StoreRelArrPtr() - store_incdir_arr_mi: bytes = StubMergerStore.GetSha1StoreIncdirArrPtr() - if store_count_mi > 0: - - # Topologically sort reachable includes: dependencies are fully translated first. Cross-module - # vtable generation needs dependency functions' TypeStr registered globally (e.g. astaux/base AST). - topo_idx: sizet* = (sizet*)(stdlib.malloc(8 * (store_count_mi + 1))) - topo_count: int = 0 - if topo_idx is None: - VLogger.error("topological sort index allocation failed, aborting compilation", "PhaseB+") - topo_count = 0 - else: - topo_count = StubMergerTopo.BuildTopoSortedIndices(store_count_mi, store_sha1_arr_mi, store_mod_arr_mi, reachable_set, reachable_count, topo_idx) - - for si_mi in range(topo_count): - # Current entry (topological index; reachable modules only) - ti: int = topo_idx[si_mi] - # Get the current entry's SHA1 and rel_path - inc_sha1_mi: str = store_sha1_arr_mi + sizet(ti) * 17 - inc_rel_mi: str = store_rel_arr_mi + sizet(ti) * StubMerger.MAX_REL_PATH_LEN - # Skip App sources: an App file may be scanned as an include and be reachable; compiling - # another .obj into includes_binary would duplicate the output/ .obj's symbols. - app_skip_mi: int = 0 - for ai_mi in range(file_count): - ea_ap: uint64 = uint64(entries) + ai_mi * entry_size - ent_ap: SrcFileEntry* = (SrcFileEntry*)(ea_ap) - if ent_ap is not None and ent_ap.Sha1 is not None: - if string.strcmp(ent_ap.Sha1, inc_sha1_mi) == 0: - app_skip_mi = 1 - break - if app_skip_mi != 0: continue - # Check whether the .obj already exists in includes.binary (sliced subdir) - ibd_len_mi: sizet = string.strlen(includes_binary_dir) - check_pat_mi: str = StubMergerCollect._sliced_path(includes_binary_dir, ibd_len_mi, inc_sha1_mi, "obj") - check_fd_mi: w32.win32file.WIN32_FIND_DATAA* = stdlib.malloc(w32.win32file.WIN32_FIND_DATAA.__sizeof__()) - obj_exists_mi: int = 0 - if check_pat_mi is not None and check_fd_mi is not None: - string.memset(check_fd_mi, 0, w32.win32file.WIN32_FIND_DATAA.__sizeof__()) - check_h_mi: w32.win32base.HANDLE = w32.win32file.FindFirstFileA(check_pat_mi, check_fd_mi) - if check_h_mi != w32.win32base.INVALID_HANDLE_VALUE: - w32.win32file.FindClose(check_h_mi) - obj_exists_mi = 1 - if obj_exists_mi != 0: - if check_pat_mi is not None: stdlib.free(check_pat_mi) - stdlib.free(check_fd_mi) - continue - - # The .obj does not exist; compile it. - # Build the source path: {includes root}/{rel_path} (per-entry IncDir; empty falls back to includes_dir) - rel_path_len_mi: sizet = string.strlen(inc_rel_mi) - incdir_mi: str = store_incdir_arr_mi + sizet(ti) * StubMerger.MAX_INC_DIR_LEN - if incdir_mi is None or incdir_mi[0] == '\0': incdir_mi = includes_dir - inc_dir_len_mi: sizet = string.strlen(incdir_mi) - src_fp_mi: bytes = stdlib.malloc(inc_dir_len_mi + 1 + rel_path_len_mi + 1) - if src_fp_mi is None: - stdlib.free(check_pat_mi) - if check_fd_mi is not None: stdlib.free(check_fd_mi) - VLogger.error("includes source path allocation failed, aborting compilation", "PhaseB+") - viperlib.snprintf(src_fp_mi, inc_dir_len_mi + 1 + rel_path_len_mi + 1, "%s/%s", incdir_mi, inc_rel_mi) - - # Check whether it is a declaration-only file (read {sha1}.meta: no FUNC ... DEF means pure declaration) - is_decl_mi: int = StubMergerIR.IsDeclOnlyModule(temp_dir, td_len_mi, inc_sha1_mi) - - # is_decl_mi: -1 = no meta (Phase1 did not translate it), 1 = declaration-only, 0 = impl. - # Skip only declaration-only (is_decl_mi == 1); -1 and 0 continue to the translation logic. - if is_decl_mi == 1: - stdlib.free(check_pat_mi) - if check_fd_mi is not None: stdlib.free(check_fd_mi) - stdlib.free(src_fp_mi) - continue - - # Try to read the in-memory FULL artifact {sha1}.full.ll (should be produced by Phase1/PhaseA) - inc_combined_mi: bytes = stdlib.malloc(COMBINED_IR_SIZE) - inc_combined_len_mi: sizet = 0 - if inc_combined_mi is not None: inc_combined_len_mi = StubMergerBuild.LoadFullIR(temp_dir, inc_sha1_mi, inc_combined_mi, COMBINED_IR_SIZE) - - # If full.ll is missing, translate the source, save stub/text/meta/full, then retry - if inc_combined_len_mi == 0 and inc_combined_mi is not None: - # Compute the includes file's package name (the directory part relative to includes_dir) - inc_pkg_mi: str = None - if string.strlen(src_fp_mi) > inc_dir_len_mi + 1: - inc_rel_mi2: str = src_fp_mi + inc_dir_len_mi + 1 - inc_pkg_mi = GenApi.compute_package_from_relpath(mb, inc_rel_mi2) - tr_mi: HandlesTranslator.Translator* = BuildPipeline.TranslateFileGetTrans(mb, src_fp_mi, inc_sha1_mi, inc_pkg_mi) - if tr_mi is not None: - # PBP IR buffer: same as PHASE_A_IR_SIZE (parser.vp's text.ll >1MB truncation root cause) - PBP_IR_SIZE: sizet = 8388608 - # FULL dump = stub + text, with 2x headroom - PBP_FULL_SIZE: sizet = 16777216 - # Save stub.ll (sliced path) - inc_stub_buf: bytes = stdlib.malloc(PBP_IR_SIZE) - if inc_stub_buf is not None: - tr_mi.dump_ir(inc_stub_buf, PBP_IR_SIZE, mlirlite.OUTPUT_STUB) - inc_stub_len: sizet = string.strlen(inc_stub_buf) - inc_stub_path: str = StubMergerCollect._sliced_path(temp_dir, td_len_mi, inc_sha1_mi, "stub.ll") - if inc_stub_path is not None: - isf: fileio.File* = fileio.File(inc_stub_path, fileio.MODE.W) - if not isf.closed: - isf.write(inc_stub_buf, inc_stub_len) - isf.close() - stdlib.free(inc_stub_path) - stdlib.free(inc_stub_buf) - # Save text.ll (sliced path) - inc_text_buf: bytes = stdlib.malloc(PBP_IR_SIZE) - if inc_text_buf is not None: - tr_mi.dump_ir(inc_text_buf, PBP_IR_SIZE, mlirlite.OUTPUT_TEXT) - inc_text_len: sizet = string.strlen(inc_text_buf) - inc_text_path: str = StubMergerCollect._sliced_path(temp_dir, td_len_mi, inc_sha1_mi, "text.ll") - if inc_text_path is not None: - itf: fileio.File* = fileio.File(inc_text_path, fileio.MODE.W) - if not itf.closed: - itf.write(inc_text_buf, inc_text_len) - itf.close() - stdlib.free(inc_text_path) - stdlib.free(inc_text_buf) - # Save {sha1}.meta (structured sidecar: function list / struct layout / vtable) - StubMergerIR.WriteModuleMeta(tr_mi.Module, temp_dir, td_len_mi, inc_sha1_mi) - # Save {sha1}.full.ll (in-memory FULL artifact) - inc_full_buf: bytes = stdlib.malloc(PBP_FULL_SIZE) - if inc_full_buf is None: VLogger.error("full.ll buffer allocation failed", "PhaseB+") - tr_mi.dump_ir(inc_full_buf, PBP_FULL_SIZE, mlirlite.OUTPUT_FULL) - inc_full_len: sizet = string.strlen(inc_full_buf) - inc_full_path: str = StubMergerCollect._sliced_path(temp_dir, td_len_mi, inc_sha1_mi, "full.ll") - if inc_full_path is not None: - ifff: fileio.File* = fileio.File(inc_full_path, fileio.MODE.W) - if not ifff.closed: - ifff.write(inc_full_buf, inc_full_len) - ifff.close() - stdlib.free(inc_full_path) - stdlib.free(inc_full_buf) - # --traceback: take this module's debug call-site records and write .vpcall. - if BuildPipeline._traceback_mode != 0: - vpcall_text_mi: str = mlirlite.DebugTakeText() - if vpcall_text_mi is not None: - BuildPipeline.WriteCallSidecar(temp_dir, inc_sha1_mi, vpcall_text_mi) - stdlib.free(vpcall_text_mi) - # Retry reading full.ll - inc_combined_len_mi = StubMergerBuild.LoadFullIR(temp_dir, inc_sha1_mi, inc_combined_mi, COMBINED_IR_SIZE) - # OOM cleanup: stub/text/meta/full are on disk, free this includes file's transient - # translation data (AST/tokens/lexer/IR/Handler); reads only touch files. - mb.reset_to(mark2) - - if inc_combined_len_mi == 0: - # Diagnostic: check whether full.ll/stub.ll/text.ll exist - diag_full: str = StubMergerCollect._sliced_path(temp_dir, td_len_mi, inc_sha1_mi, "full.ll") - diag_stub: str = StubMergerCollect._sliced_path(temp_dir, td_len_mi, inc_sha1_mi, "stub.ll") - diag_text: str = StubMergerCollect._sliced_path(temp_dir, td_len_mi, inc_sha1_mi, "text.ll") - diag_full_ex: int = 0 - diag_stub_ex: int = 0 - diag_text_ex: int = 0 - if diag_full is not None: - df_diag_f: fileio.File* = fileio.File(diag_full, fileio.MODE.R) - if not df_diag_f.closed: - diag_full_ex = 1 - df_diag_f.close() - stdlib.free(diag_full) - if diag_stub is not None: - df_diag_s: fileio.File* = fileio.File(diag_stub, fileio.MODE.R) - if not df_diag_s.closed: - diag_stub_ex = 1 - df_diag_s.close() - stdlib.free(diag_stub) - if diag_text is not None: - df_diag_t: fileio.File* = fileio.File(diag_text, fileio.MODE.R) - if not df_diag_t.closed: - diag_text_ex = 1 - df_diag_t.close() - stdlib.free(diag_text) - VLogger.error(f"failed to read full.ll: {src_fp_mi} (full={diag_full_ex} stub={diag_stub_ex} text={diag_text_ex} is_decl={is_decl_mi})", "PhaseB+") - if inc_combined_mi is not None: stdlib.free(inc_combined_mi) - stdlib.free(check_pat_mi) - if check_fd_mi is not None: stdlib.free(check_fd_mi) - stdlib.free(src_fp_mi) - continue - - # Compile to .obj (output to includes_binary_dir, consistent with the existence check and link collection) - inc_cret_mi: int = BuildPipeline.compile_module_to_obj(inc_combined_mi, inc_combined_len_mi, temp_dir, includes_binary_dir, inc_sha1_mi, cc_cmd, cc_flags) - stdlib.free(inc_combined_mi) - - if inc_cret_mi != 0: - VLogger.error(f"llc compilation failed, aborting compilation: {src_fp_mi}", "PhaseB+") - sys.exit(1) - - inc_compiled += 1 - # Show the compiled includes file - VLogger.info(f"[includes {inc_compiled}] compilation complete: {src_fp_mi}", "PhaseB+") - - # Add to obj_paths (sliced path, taken from includes_binary_dir) - ibd_len_mi2: sizet = string.strlen(includes_binary_dir) - inc_obj_sliced: str = StubMergerCollect._sliced_path(includes_binary_dir, ibd_len_mi2, inc_sha1_mi, "obj") - - if inc_obj_sliced is not None: - inc_obj_len: sizet = string.strlen(inc_obj_sliced) - if obj_pos + inc_obj_len + 2 < OBJ_PATHS_SIZE: - if obj_pos > 0: - obj_paths[obj_pos] = ' ' - obj_pos += 1 - string.strcpy(obj_paths + obj_pos, inc_obj_sliced) - obj_pos += inc_obj_len - obj_paths[obj_pos] = '\0' - else: VLogger.error("includes .obj path buffer overflow, aborting compilation", "PhaseB+") - stdlib.free(inc_obj_sliced) - - # Free this iteration's temporary memory - if check_pat_mi is not None: stdlib.free(check_pat_mi) - - if check_fd_mi is not None: stdlib.free(check_fd_mi) - - stdlib.free(src_fp_mi) - - # Free the topological indices and local map copies - if topo_idx is not None: stdlib.free(topo_idx) - if store_sha1_arr_mi is not None: stdlib.free(store_sha1_arr_mi) - if store_mod_arr_mi is not None: stdlib.free(store_mod_arr_mi) - - VLogger.info(f"compiled missing includes: {inc_compiled} entries", "PhaseB+") - - # Free app_deps_buf (fallback when the Phase B+ block did not run) - if app_deps_buf is not None: - stdlib.free(app_deps_buf) - app_deps_buf = None - - # === 4. Phase C: link all .obj -> .exe === - log.banner("Phase C: link") - - od_len2: sizet = string.strlen(output_dir) - lo_len: sizet = string.strlen(linker_output) - exe_path: bytes = stdlib.malloc(od_len2 + lo_len + 2) - if exe_path is not None: viperlib.snprintf(exe_path, od_len2 + lo_len + 2, "%s/%s", output_dir, linker_output) - else: - exe_path = linker_output - VLogger.error("exe_path allocation failed, aborting compilation", "PhaseC") - - # Build the final .obj path list: main_obj_path first, reachable .objs after - final_obj_paths: bytes = stdlib.malloc(OBJ_PATHS_SIZE + 512) - if final_obj_paths is None: - stdlib.free(entries) - return 1 - final_obj_paths[0] = '\0' - fop_pos: sizet = 0 - # main_obj_path goes first (so the linker prefers the user main) - if main_obj_path[0] != '\0': - mlen: sizet = string.strlen(main_obj_path) - string.strcpy(final_obj_paths, main_obj_path) - fop_pos = mlen - - # Merge the App non-main modules' .obj (obj_paths, space-separated) compiled by Phase B/B+. - # final_obj_paths must include main_obj + reachable includes + App deps, else linking hits undefined symbols. - if obj_pos > 0: - if fop_pos + obj_pos + 1 < OBJ_PATHS_SIZE + 512: - if fop_pos > 0: - final_obj_paths[fop_pos] = ' ' - fop_pos += 1 - string.strcpy(final_obj_paths + fop_pos, obj_paths) - fop_pos += obj_pos - final_obj_paths[fop_pos] = '\0' - - # Link by graph: iterate reachable_set and collect only reachable includes' .obj, - # no longer collecting every .obj under includes.binary. - if reachable_set is not None and reachable_count > 0: - ibd_len_pc: sizet = string.strlen(includes_binary_dir) - td_len_pc: sizet = string.strlen(temp_dir) - ri_pc: int - for ri_pc in range(reachable_count): - rsha1_pc: str = reachable_set + sizet(ri_pc) * 17 - if rsha1_pc[0] == '\0': continue - # Skip App sources: their .obj is already in final_obj_paths; collecting a same-sha1 .obj from includes_binary would duplicate symbols. - app_skip_pc: int = 0 - for ai_pc in range(file_count): - ea_pc: uint64 = uint64(entries) + ai_pc * entry_size - ent_pc: SrcFileEntry* = (SrcFileEntry*)(ea_pc) - if ent_pc is not None and ent_pc.Sha1 is not None: - if string.strcmp(ent_pc.Sha1, rsha1_pc) == 0: - app_skip_pc = 1 - break - if app_skip_pc != 0: continue - obj_path_pc: str = StubMergerCollect._sliced_path(includes_binary_dir, ibd_len_pc, rsha1_pc, "obj") - if obj_path_pc is not None: - # Check whether the .obj file exists - cf_pc: fileio.File* = fileio.File(obj_path_pc, fileio.MODE.R) - if not cf_pc.closed: - cf_pc.close() - # The .obj exists; append to final_obj_paths - op_len_pc: sizet = string.strlen(obj_path_pc) - if fop_pos + op_len_pc + 2 < OBJ_PATHS_SIZE + 512: - if fop_pos > 0: - final_obj_paths[fop_pos] = ' ' - fop_pos += 1 - string.strcpy(final_obj_paths + fop_pos, obj_path_pc) - fop_pos += op_len_pc - final_obj_paths[fop_pos] = '\0' - else: - # .obj missing: a declaration-only module (no define lines) needs no .obj, so skip silently; - # under phase2command mode the missing .obj is expected and the path is still added. - if BuildPipeline._phase2_cmd_path is not None: - op_len_p2: sizet = string.strlen(obj_path_pc) - if fop_pos + op_len_p2 + 2 < OBJ_PATHS_SIZE + 512: - if fop_pos > 0: - final_obj_paths[fop_pos] = ' ' - fop_pos += 1 - string.strcpy(final_obj_paths + fop_pos, obj_path_pc) - fop_pos += op_len_p2 - final_obj_paths[fop_pos] = '\0' - elif StubMergerIR.IsDeclOnlyModule(temp_dir, td_len_pc, rsha1_pc) != 1: VLogger.error(f"reachable module .obj missing: {obj_path_pc}", "PhaseC") - stdlib.free(obj_path_pc) - - final_obj_paths[fop_pos] = '\0' - obj_paths_len: sizet = fop_pos - - # Free reachable_set (Phase C is done with it) - if reachable_set is not None: - stdlib.free(reachable_set) - reachable_set = None - - # Link by graph: pass None for includes_binary_dir to avoid collecting everything - lret: int = BuildPipeline.link_objs_to_exe(final_obj_paths, obj_paths_len, linker_cmd, linker_flags, exe_path, None, linker_static_libs, linker_dynamic_libs) - - if lret == 0: - VLogger.success(f"output: {exe_path}", "PhaseC") - # Deploy declared runtime deps (project.vpj runtime.*) next to the exe; fail-fast on error. - # Skipped under --phase2command (the exe is not linked yet). - if BuildPipeline._phase2_cmd_path is None: - if BuildPipeline.deploy_runtime_deps(exe_path) != 0: - VLogger.error("runtime dependency deployment failed, aborting compilation", "runtime") - stdlib.free(entries) - stdlib.free(asm_entries) - return 1 - # --traceback: export .stackmap (skipped under --phase2command, - # where nothing was actually linked). - if BuildPipeline._traceback_mode != 0 and BuildPipeline._phase2_cmd_path is None: BuildPipeline.WriteStackMap(mb, exe_path) - if args.get_bool("run") and BuildPipeline._phase2_cmd_path is not None: - # phase2command mode: the exe is not linked yet (the command is in the bat); skip --run - VLogger.info("phase2command mode: skipping --run (exe not linked yet)", "phase2cmd") - elif args.get_bool("run"): - VLogger.info(f"run: {exe_path}", "run") - rp: subprocess.CompletedProcess* = subprocess.run(exe_path, False, False) - if rp is not None: VLogger.info(f"exit code: {rp.returncode}", "run") - else: - VLogger.error("link failed", "PhaseC") - stdlib.free(entries) - stdlib.free(asm_entries) - return 1 - - stdlib.free(entries) - stdlib.free(asm_entries) - return 0 +import lib.core.Gen.GenApi as GenApi +import t, c +from stdint import * +import stdio +import string +import stdlib +import memhub +import viperlib +import win32.fileio as fileio +import win32.win32file +import win32.win32base +import sys +import ast +import mlirlite +import subprocess +import argparse +import lib.core.VLogger as VLogger +import lib.core.Gen.Translator as HandlesTranslator +import lib.Phase.BuildPipeline as BuildPipeline +import lib.core.IncludesScanner as IncludesScanner +import lib.core.CompilerFactory as CompilerFactory +import lib.Stub.StubMerger as StubMerger +import lib.Projectrans.Utils as Utils +import lib.Projectrans.Config as Config +import lib.Stub.StubMerger.StubMergerBuild as StubMergerBuild +import lib.Stub.StubMerger.StubMergerCollect as StubMergerCollect +import lib.Stub.StubMerger.StubMergerIR as StubMergerIR +import lib.Stub.StubMerger.StubMergerStore as StubMergerStore +import lib.Stub.StubMerger.StubMergerTopo as StubMergerTopo +import lib.Stub.StubGen.Converter as StubConverter + +# Global mbuddy pointer +_mbuddy: memhub.MemBuddy* + +# Source buffer size (1MB) +define SRC_BUF_SIZE = 1048576 + +# Max source files (recursive scanning grows the count; too small truncates the list -> undefined symbol) +define MAX_SRC_FILES = 128 + +@t.NoVTable +class SrcFileEntry: + """Source file entry""" + Path: str # full path + Sha1: str # first 16 chars of SHA1 + +# _ScanDirForPyFiles - recursively scan a dir and collect .vp/.py files into entries, returning the new count. +# Args: mb pool; dir_path; entries SrcFileEntry array; entry_size; file_count; max_files. +def _ScanDirForPyFiles(mb: memhub.MemBuddy*, dir_path: str, entries: SrcFileEntry*, entry_size: sizet, file_count: int, max_files: int) -> int: + """Recursively scan a directory, collect .vp/.py files into entries, return the new file_count""" + if dir_path is None or entries is None: return file_count + if file_count >= max_files: return file_count + + dir_len: sizet = string.strlen(dir_path) + pattern: bytes = stdlib.malloc(dir_len + 16) + if pattern is None: return file_count + viperlib.snprintf(pattern, dir_len + 16, "%s/*", dir_path) + + find_data_size: sizet = win32.win32file.WIN32_FIND_DATAA.__sizeof__() + find_data: win32.win32file.WIN32_FIND_DATAA* = stdlib.malloc(find_data_size + 16) + if find_data is None: + stdlib.free(pattern) + return file_count + string.memset(find_data, 0, find_data_size + 16) + + handle: win32.win32base.HANDLE = win32.win32file.FindFirstFileA(pattern, find_data) + if handle == win32.win32base.INVALID_HANDLE_VALUE: + stdlib.free(pattern) + stdlib.free(find_data) + return file_count + + while True: + fname: str = find_data.cFileName + if fname is None: + if win32.win32file.FindNextFileA(handle, find_data) == 0: break + continue + fname_len: sizet = string.strlen(fname) + if fname_len <= 0: + if win32.win32file.FindNextFileA(handle, find_data) == 0: break + continue + # Skip . and .. + is_dot: int = 0 + if fname_len == 1 and fname[0] == '.': is_dot = 1 + elif fname_len == 2 and fname[0] == '.' and fname[1] == '.': is_dot = 1 + if is_dot != 0: + if win32.win32file.FindNextFileA(handle, find_data) == 0: break + continue + # Check whether it is a directory + attrs: ULONG = find_data.dwFileAttributes + is_dir: int = 0 + if (attrs & win32.win32file.FILE_ATTRIBUTE_DIRECTORY) != 0: is_dir = 1 + if is_dir != 0: + # Recurse into the subdirectory + sub_dir: bytes = Utils.JoinPath(dir_path, fname) + if sub_dir is not None: + file_count = _ScanDirForPyFiles(mb, sub_dir, entries, entry_size, file_count, max_files) + stdlib.free(sub_dir) + if win32.win32file.FindNextFileA(handle, find_data) == 0: break + continue + # Check for a .vp source file (Viper sources are only .vp now; .py is no longer recognized) + if fname_len > 3 and Utils.is_vp_at(fname, 0, fname_len) != 0 and file_count < max_files: + full_path: bytes = Utils.JoinPath(dir_path, fname) + if full_path is not None: + sbuf: bytes = Utils.read_file_to_buf(full_path, SRC_BUF_SIZE) + if sbuf is None: + if win32.win32file.FindNextFileA(handle, find_data) == 0: break + continue + # Use IncludesScanner.compute_file_sha1 (CRLF->LF normalized) as StubMerger does for the same + # App sources; raw-byte hashing makes stub/define symbol prefixes differ -> undefined reference. + sha1_val: str = IncludesScanner.compute_file_sha1(mb, full_path) + if sha1_val is not None: + ea: uint64 = uint64(entries) + file_count * entry_size + ent: SrcFileEntry* = (SrcFileEntry*)(ea) + if ent is not None: + ent.Path = full_path + ent.Sha1 = sha1_val + file_count += 1 + + VLogger.info(f"{fname} (sha1={sha1_val})", "project") + stdlib.free(sbuf) + # full_path is not freed: ent.Path references it + + if win32.win32file.FindNextFileA(handle, find_data) == 0: break + + win32.win32file.FindClose(handle) + stdlib.free(pattern) + stdlib.free(find_data) + return file_count + + +# _ScanDirForAsmFiles - recursively scan a dir and collect .s assembly files. +# Like _ScanDirForPyFiles; VPC does not translate .s but assembles it to .obj via llvm-mc for linking. +def _ScanDirForAsmFiles(mb: memhub.MemBuddy*, dir_path: str, entries: SrcFileEntry*, entry_size: sizet, file_count: int, max_files: int) -> int: + """Recursively scan a directory, collect .s assembly files into entries, return the new file_count""" + if dir_path is None or entries is None: return file_count + if file_count >= max_files: return file_count + + dir_len: sizet = string.strlen(dir_path) + pattern: bytes = stdlib.malloc(dir_len + 16) + if pattern is None: return file_count + viperlib.snprintf(pattern, dir_len + 16, "%s/*", dir_path) + + find_data_size: sizet = win32.win32file.WIN32_FIND_DATAA.__sizeof__() + find_data: win32.win32file.WIN32_FIND_DATAA* = stdlib.malloc(find_data_size + 16) + if find_data is None: + stdlib.free(pattern) + return file_count + string.memset(find_data, 0, find_data_size + 16) + + handle: win32.win32base.HANDLE = win32.win32file.FindFirstFileA(pattern, find_data) + if handle == win32.win32base.INVALID_HANDLE_VALUE: + stdlib.free(pattern) + stdlib.free(find_data) + return file_count + + while True: + fname: str = find_data.cFileName + if fname is None: + if win32.win32file.FindNextFileA(handle, find_data) == 0: break + continue + fname_len: sizet = string.strlen(fname) + if fname_len <= 0: + if win32.win32file.FindNextFileA(handle, find_data) == 0: break + continue + # Skip . and .. + is_dot: int = 0 + if fname_len == 1 and fname[0] == '.': is_dot = 1 + elif fname_len == 2 and fname[0] == '.' and fname[1] == '.': is_dot = 1 + if is_dot != 0: + if win32.win32file.FindNextFileA(handle, find_data) == 0: break + continue + # Check whether it is a directory + attrs: ULONG = find_data.dwFileAttributes + is_dir: int = 0 + if (attrs & win32.win32file.FILE_ATTRIBUTE_DIRECTORY) != 0: is_dir = 1 + if is_dir != 0: + sub_dir: bytes = Utils.JoinPath(dir_path, fname) + if sub_dir is not None: + file_count = _ScanDirForAsmFiles(mb, sub_dir, entries, entry_size, file_count, max_files) + stdlib.free(sub_dir) + if win32.win32file.FindNextFileA(handle, find_data) == 0: break + continue + # Check for the .s suffix (assembly source) + is_asm: int = 0 + if fname_len > 2 and fname[fname_len - 2] == '.' and fname[fname_len - 1] == 's': is_asm = 1 + if is_asm != 0 and file_count < max_files: + full_path: bytes = Utils.JoinPath(dir_path, fname) + if full_path is not None: + # Same SHA1 convention as .vp (CRLF->LF normalized), used for the sliced path + sha1_val: str = IncludesScanner.compute_file_sha1(mb, full_path) + if sha1_val is not None: + ea: uint64 = uint64(entries) + file_count * entry_size + ent: SrcFileEntry* = (SrcFileEntry*)(ea) + if ent is not None: + ent.Path = full_path + ent.Sha1 = sha1_val + file_count += 1 + + VLogger.info(f"{fname} (asm sha1={sha1_val})", "project") + # full_path is not freed: ent.Path references it + + if win32.win32file.FindNextFileA(handle, find_data) == 0: break + + win32.win32file.FindClose(handle) + stdlib.free(pattern) + stdlib.free(find_data) + return file_count + + +# _PreRegisterAppGenericTemplates - pre-register App generic class templates before mark2. +# The generic tables live in the pool; registering after mark2 would be reclaimed by reset_to, corrupting later passes. +def _PreRegisterAppGenericTemplates(mb: memhub.MemBuddy*, entries: SrcFileEntry*, file_count: int, entry_size: sizet) -> int: + """Pre-register App source generic class templates before mark2, return the processed count""" + if mb is None or entries is None or file_count <= 0: return 0 + handled: int = 0 + for i in range(int(file_count)): + ea_pa: uint64 = uint64(entries) + i * entry_size + ent_pa: SrcFileEntry* = (SrcFileEntry*)(ea_pa) + if ent_pa is None or ent_pa.Path is None or ent_pa.Sha1 is None: continue + # Read the file + f_pa: fileio.File* = fileio.File(ent_pa.Path, fileio.MODE.R) + if f_pa.closed: continue + # read_all's capacity includes the terminator (reads at most cap-1 bytes); longer files are truncated + if f_pa.size() >= SRC_BUF_SIZE: + VLogger.error(f"source file exceeds {SRC_BUF_SIZE} byte limit, refusing to read: {ent_pa.Path}", "PhaseA") + f_pa.close() + continue + src_buf_pa: bytes = stdlib.malloc(SRC_BUF_SIZE) + if src_buf_pa is None: + f_pa.close() + continue + bytes_read_pa: LONG = f_pa.read_all(src_buf_pa, SRC_BUF_SIZE) + f_pa.close() + if bytes_read_pa <= 0: + stdlib.free(src_buf_pa) + continue + if bytes_read_pa < SRC_BUF_SIZE: src_buf_pa[bytes_read_pa] = 0 + # Parse the AST only (perm pool keeps template nodes alive across the mark2 reset) + perm_pa: memhub.MemBuddy* = CompilerFactory.factory.PermPool + if perm_pa is None: perm_pa = mb + lx_pa: ast.Lexer* = ast.new_lexer(perm_pa) + if lx_pa is None: + stdlib.free(src_buf_pa) + continue + ast._lexer_init(lx_pa, src_buf_pa, perm_pa) + tokens_pa: ast.Token* = ast.tokenize(lx_pa) + tree_pa: ast.AST* = ast.parse_tokens(perm_pa, tokens_pa) + if tree_pa is None: + stdlib.free(src_buf_pa) + continue + # Walk top-level ClassDef/FunctionDef and register generic templates (type_params != None) + ch_pa: list[ast.AST*]* = tree_pa.children + if ch_pa is not None: + cn_pa: sizet = len(ch_pa) + for ci_pa in range(int(cn_pa)): + child_pa: ast.AST* = ch_pa.get(ci_pa) + if child_pa is None: continue + ck_pa: int = child_pa.kind() + if ck_pa == ast.ASTKind.ClassDef: + cd_pa: ast.ClassDef* = ast.ClassDef*(child_pa) + if cd_pa is None or cd_pa.name is None: continue + tp_pa: list[str]* = cd_pa.type_params + if tp_pa is None: continue + GenApi.set_current_file_type(ent_pa.Path) + GenApi._register_generic_template(perm_pa, cd_pa, ent_pa.Sha1) + handled += 1 + elif ck_pa == ast.ASTKind.FunctionDef: + fd_pa: ast.FunctionDef* = ast.FunctionDef*(child_pa) + if fd_pa is None or fd_pa.name is None: continue + tf_pa: list[str]* = fd_pa.type_params + if tf_pa is None: continue + GenApi.set_current_file_type(ent_pa.Path) + GenApi._register_generic_func_template(perm_pa, fd_pa, ent_pa.Sha1) + handled += 1 + stdlib.free(src_buf_pa) + if handled > 0: VLogger.info(f"pre-registered {handled} App generic templates before mark2 (classes + functions)", "PhaseA") + return handled + + +# Phase A convergence-failure dependency-graph diagnostics. +# Each file's deps and missing details are persisted to a stdlib table (survives reset) and dumped on failure. +PA_DEPS_MAX_STR: sizet = 1024 +PA_MISS_MAX_STR: sizet = 768 +PA_GRAPH_DEPTH_MAX: int = 3 # missing-chain drill-down depth (counts from layer 1) +PA_GRAPH_OK_PROBE: int = 2 # ready nodes are probed 1 more level (to find hidden misses) +_pa_diag_max: sizet = 0 +_pa_diag_count: int = 0 +_pa_diag_sha1: bytes = None +_pa_diag_deps: bytes = None +_pa_diag_miss: bytes = None + + +def _PhaseADiagInit(max_count: sizet) -> None: + """Allocate the persistent Phase A convergence-failure diagnostic table""" + global _pa_diag_max, _pa_diag_count, _pa_diag_sha1, _pa_diag_deps, _pa_diag_miss + _pa_diag_max = max_count + _pa_diag_count = 0 + _pa_diag_sha1 = stdlib.malloc(max_count * 17) + _pa_diag_deps = stdlib.malloc(max_count * PA_DEPS_MAX_STR) + _pa_diag_miss = stdlib.malloc(max_count * PA_MISS_MAX_STR) + if _pa_diag_sha1 is not None: string.memset(_pa_diag_sha1, 0, max_count * 17) + if _pa_diag_deps is not None: string.memset(_pa_diag_deps, 0, max_count * PA_DEPS_MAX_STR) + if _pa_diag_miss is not None: string.memset(_pa_diag_miss, 0, max_count * PA_MISS_MAX_STR) + + +def _PhaseADiagFree() -> None: + """Free the persistent diagnostic table""" + global _pa_diag_max, _pa_diag_count, _pa_diag_sha1, _pa_diag_deps, _pa_diag_miss + if _pa_diag_sha1 is not None: + stdlib.free(_pa_diag_sha1) + _pa_diag_sha1 = None + if _pa_diag_deps is not None: + stdlib.free(_pa_diag_deps) + _pa_diag_deps = None + if _pa_diag_miss is not None: + stdlib.free(_pa_diag_miss) + _pa_diag_miss = None + _pa_diag_count = 0 + _pa_diag_max = 0 + + +def _PhaseADiagRecord(sha1: str, deps: str, miss: str) -> None: + """Record or update one file's diagnostics: sha1 to deps plus missing detail""" + global _pa_diag_count + if _pa_diag_sha1 is None or _pa_diag_deps is None or sha1 is None: return + if string.strlen(sha1) < 16: return + for i in range(int(_pa_diag_count)): + es: bytes = _pa_diag_sha1 + sizet(i) * 17 + if string.strcmp(es, sha1) == 0: + if miss is not None and _pa_diag_miss is not None: + maddr: bytes = _pa_diag_miss + sizet(i) * PA_MISS_MAX_STR + string.strncpy(maddr, miss, PA_MISS_MAX_STR - 1) + maddr[PA_MISS_MAX_STR - 1] = 0 + return + if _pa_diag_count >= _pa_diag_max: return + saddr: bytes = _pa_diag_sha1 + sizet(_pa_diag_count) * 17 + string.strncpy(saddr, sha1, 16) + saddr[16] = 0 + daddr: bytes = _pa_diag_deps + sizet(_pa_diag_count) * PA_DEPS_MAX_STR + if deps is not None: string.strncpy(daddr, deps, PA_DEPS_MAX_STR - 1) + daddr[PA_DEPS_MAX_STR - 1] = 0 + if miss is not None and _pa_diag_miss is not None: + maddr2: bytes = _pa_diag_miss + sizet(_pa_diag_count) * PA_MISS_MAX_STR + string.strncpy(maddr2, miss, PA_MISS_MAX_STR - 1) + maddr2[PA_MISS_MAX_STR - 1] = 0 + _pa_diag_count += 1 + + +def _PhaseADiagDepsLookup(sha1: str) -> str: + """Look up the persisted deps string by sha1; caller frees the copy, None if absent""" + if _pa_diag_sha1 is None or _pa_diag_deps is None or sha1 is None: return None + for i in range(int(_pa_diag_count)): + es: bytes = _pa_diag_sha1 + sizet(i) * 17 + if string.strcmp(es, sha1) == 0: + src: bytes = _pa_diag_deps + sizet(i) * PA_DEPS_MAX_STR + ln: sizet = string.strlen(src) + out: bytes = stdlib.malloc(ln + 1) + if out is None: return None + string.strcpy(out, src) + return out + return None + + +def _PhaseADiagMissLookup(sha1: str) -> str: + """Look up the persisted missing detail by sha1; caller frees the copy, None if absent""" + if _pa_diag_sha1 is None or _pa_diag_miss is None or sha1 is None: return None + for i in range(int(_pa_diag_count)): + es: bytes = _pa_diag_sha1 + sizet(i) * 17 + if string.strcmp(es, sha1) == 0: + src: bytes = _pa_diag_miss + sizet(i) * PA_MISS_MAX_STR + ln: sizet = string.strlen(src) + if ln == 0: return None + out: bytes = stdlib.malloc(ln + 1) + if out is None: return None + string.strcpy(out, src) + return out + return None + + +def _PhaseATextLLExists(temp_dir: str, td_len: sizet, sha1: str) -> int: + """Probe whether {temp_dir}/{sha1 first 2 chars}/{sha1}.text.ll exists, 1=yes""" + if temp_dir is None or td_len == 0 or sha1 is None or string.strlen(sha1) < 2: return 0 + tpath: str = StubMergerCollect._sliced_path(temp_dir, td_len, sha1, "text.ll") + if tpath is None: return 0 + tf: fileio.File* = fileio.File(tpath, fileio.MODE.R) + exists: int = 0 + if not tf.closed: + tf.close() + exists = 1 + stdlib.free(tpath) + return exists + + +def _PhaseAResolveDepSha1(mod_name: str) -> str: + """Resolve a dependency module name to SHA1 (exact then suffix), None if absent""" + if mod_name is None: return None + s1: str = StubMergerStore.LookupSha1ByModName(mod_name) + if s1 is not None: return s1 + return StubMergerStore.LookupSha1ByModNameSuffix(mod_name) + + +def _PhaseADumpNode(temp_dir: str, td_len: sizet, sha1: str, depth: int, prefix: bytes) -> None: + """Recursively print the dependency subgraph; [MISS]=missing text.ll, [OK]=ready""" + deps: bytes = None + if sha1 is not None: deps = _PhaseADiagDepsLookup(sha1) + if deps is None and sha1 is not None: + # Fallback: read {temp_dir}/{prefix}/{sha1}.deps.txt (Phase1 already wrote it for includes) + dpath: str = StubMergerCollect._sliced_path(temp_dir, td_len, sha1, "deps.txt") + if dpath is not None: + df: fileio.File* = fileio.File(dpath, fileio.MODE.R) + if not df.closed: + deps = stdlib.malloc(PA_DEPS_MAX_STR) + if deps is not None: + rl: int64 = df.read_all(deps, PA_DEPS_MAX_STR) + if rl < 0: rl = 0 + # read_all's size arg is the buffer capacity (incl. terminator): it reads at most MAX-1 + deps[rl] = 0 + df.close() + stdlib.free(dpath) + if deps is None or deps[0] == 0: + if deps is not None: stdlib.free(deps) + return + dl: sizet = string.strlen(deps) + token_count: int = 0 + pos: sizet = 0 + while pos < dl: + while pos < dl and deps[pos] == ' ': pos += 1 + if pos >= dl: break + token_count += 1 + while pos < dl and deps[pos] != ' ': pos += 1 + pos = 0 + idx: int = 0 + while pos < dl: + while pos < dl and deps[pos] == ' ': pos += 1 + if pos >= dl: break + ts: sizet = pos + while pos < dl and deps[pos] != ' ': pos += 1 + tl: sizet = pos - ts + nm: bytes = stdlib.malloc(tl + 1) + if nm is not None: + string.strncpy(nm, deps + ts, tl) + nm[tl] = 0 + is_last: int = 0 + if idx + 1 >= token_count: is_last = 1 + builtin: int = 0 + if string.strcmp(nm, "t") == 0 or string.strcmp(nm, "c") == 0: builtin = 1 + dep_sha1: str = _PhaseAResolveDepSha1(nm) + txt_ok: int = 0 + if dep_sha1 is not None: txt_ok = _PhaseATextLLExists(temp_dir, td_len, dep_sha1) + # Branch characters are inlined (don't return string literals: VPC would return a dangling + # stack pointer for a literal -> printf %s crashes and the graph output is swallowed) + br_1: str = "|-- " + br_2: str = "`-- " + if builtin != 0: + if is_last != 0: stdio.printf("%s%s%s (builtin)\n", prefix, br_2, nm) + else: stdio.printf("%s%s%s (builtin)\n", prefix, br_1, nm) + elif dep_sha1 is None: + if is_last != 0: stdio.printf("%s%s%s (unresolved)\n", prefix, br_2, nm) + else: stdio.printf("%s%s%s (unresolved)\n", prefix, br_1, nm) + elif txt_ok != 0: + if is_last != 0: stdio.printf("%s%s%s [OK]\n", prefix, br_2, nm) + else: stdio.printf("%s%s%s [OK]\n", prefix, br_1, nm) + else: + if is_last != 0: stdio.printf("%s%s%s [MISS] text.ll=%s\n", prefix, br_2, nm, dep_sha1) + else: stdio.printf("%s%s%s [MISS] text.ll=%s\n", prefix, br_1, nm, dep_sha1) + # Drill down: MISS follows the whole chain (to PA_GRAPH_DEPTH_MAX); OK probes 1 level only + can_expand: int = 0 + if dep_sha1 is not None and builtin == 0: + if txt_ok == 0: + if depth + 1 < PA_GRAPH_DEPTH_MAX: can_expand = 1 + else: + if depth + 1 < PA_GRAPH_OK_PROBE: can_expand = 1 + if can_expand != 0: + # +5: strcat appends "| "/" " (4 chars) + NUL; strlen+4 would overflow 1 byte and + # corrupt heap metadata, crashing later malloc/free and aborting the graph output + npl: sizet = string.strlen(prefix) + 5 + new_prefix: bytes = stdlib.malloc(npl) + if new_prefix is not None: + string.strcpy(new_prefix, prefix) + if is_last != 0: string.strcat(new_prefix, " ") + else: string.strcat(new_prefix, "| ") + _PhaseADumpNode(temp_dir, td_len, dep_sha1, depth + 1, new_prefix) + stdlib.free(new_prefix) + stdlib.free(nm) + idx += 1 + stdlib.free(deps) + + +def _PhaseADumpFailGraph(temp_dir: str, td_len: sizet, path: str, sha1: str) -> None: + """Convergence-failure diagnostics: pending detail plus dependency tree. + Flush after every printf so the trace survives a later crash.""" + stdio.printf("[PhaseA-Fail] ==============================================================\n") + stdio.fflush(None) + stdio.printf("[PhaseA-Fail] FAIL TO TRANSLATE: %s [sha1=%s]\n", path, sha1) + stdio.fflush(None) + miss: bytes = _PhaseADiagMissLookup(sha1) + if miss is not None: + if miss[0] != 0: stdio.printf("[PhaseA-Fail] pending reason: %s\n", miss) + stdlib.free(miss) + stdio.fflush(None) + stdio.printf("[PhaseA-Fail] dep graph ([MISS]=missing text.ll, [OK]=ready, depth<=%d):\n", PA_GRAPH_DEPTH_MAX) + stdio.fflush(None) + root_prefix: bytes = stdlib.malloc(1) + if root_prefix is not None: + root_prefix[0] = 0 + _PhaseADumpNode(temp_dir, td_len, sha1, 0, root_prefix) + stdlib.free(root_prefix) + stdio.fflush(None) + + +def RunMultiFileProject(mb: memhub.MemBuddy*, source_dir: str, temp_dir: str, output_dir: str, cc_cmd: str, cc_flags: str, linker_cmd: str, linker_flags: str, linker_output: str, includes_binary_dir: str, includes_dir: str, do_phase1: int, do_phase2: int, log: VLogger.Logger*, args: argparse.ParsedArgs*, linker_static_libs: str = None, linker_dynamic_libs: str = None) -> int: + """Compile a multi-file project; return 0 on success, non-zero on failure""" + if source_dir is None: + VLogger.error("source_dir is empty", "project") + return 1 + + VLogger.info(f"multi-file project compilation: {source_dir}", "project") + + # === 1. Recursively scan .vp/.py files under source_dir (including subdirectories) === + entry_size: sizet = SrcFileEntry.__sizeof__() + entries: SrcFileEntry* = stdlib.malloc(MAX_SRC_FILES * entry_size) + if entries is None: return 1 + string.memset(entries, 0, MAX_SRC_FILES * entry_size) + + file_count: int = _ScanDirForPyFiles(mb, source_dir, entries, entry_size, 0, MAX_SRC_FILES) + VLogger.info(f"{file_count} source files in total", "project") + + # Scan .s assembly files (separate array; not translated in Phase A, assembled by llvm-mc after Phase B) + asm_entries: SrcFileEntry* = stdlib.malloc(MAX_SRC_FILES * entry_size) + if asm_entries is None: + stdlib.free(entries) + return 1 + string.memset(asm_entries, 0, MAX_SRC_FILES * entry_size) + asm_file_count: int = _ScanDirForAsmFiles(mb, source_dir, asm_entries, entry_size, 0, MAX_SRC_FILES) + if asm_file_count > 0: VLogger.info(f"{asm_file_count} assembly files (assembled by llvm-mc and linked)", "project") + + if file_count == 0: + stdlib.free(entries) + stdlib.free(asm_entries) + return 1 + + # Initialize the AST tables + ast._init_tables(mb) + + # Pre-register App generic class templates before mark2: the tables/nodes are pool-allocated, so they + # must be registered before mark2 to survive reset_to(mark2); otherwise later passes hit dangling memory. + _PreRegisterAppGenericTemplates(mb, entries, file_count, entry_size) + + # OOM cleanup (mirrors Phase1's mark/reset pattern): without reclamation the AST/IR of all includes + # pile up in the pool. mark2 lets each file's transient data be freed by reset_to after translation. + mark2: sizet = mb.mark() + # struct/enum entries registered after mark2 use stdlib allocation (survive reset, as in Phase1) + GenApi.set_stdlib_alloc(1) + + # Ensure the temp/output directories exist + BuildPipeline.ensure_dir(temp_dir) + BuildPipeline.ensure_dir(output_dir) + + # === 0.5 Fill Sha1Store on demand from the dependency graph (removes the Phase1 dependency) === + # If Sha1Store is empty, scan the dependency graph (App-first BFS) so later "compile missing includes" works. + if includes_dir is not None: + if StubMergerStore.GetSha1StoreCount() == 0: + rs_05: bytes = stdlib.malloc(StubMerger.MAX_INCLUDES_SHA1 * 17) + if rs_05 is None: VLogger.error("reachable_set allocation failed, cannot fill Sha1Store", "project") + else: + string.memset(rs_05, 0, StubMerger.MAX_INCLUDES_SHA1 * 17) + rc_05: int = StubMergerBuild._BuildReachableSha1Set(mb, source_dir, temp_dir, rs_05, None, includes_dir) + if rc_05 < 0: VLogger.error("dependency graph scan failed, cannot fill Sha1Store", "project") + elif rc_05 > 0: VLogger.info(f"Phase2 dependency graph filled Sha1Store on demand: {rc_05} entries", "project") + stdlib.free(rs_05) + + # Append App source SHA1s to _sha1_map.txt (human-readable only; format {sha1}:{rel_path}) and to the + # in-memory store (AppendToSha1MapStore) for cross-module CDefine lookups. + td_len_am: sizet = string.strlen(temp_dir) + src_dir_len_am: sizet = string.strlen(source_dir) + map_path_am: bytes = stdlib.malloc(td_len_am + 32) + if map_path_am is not None: + viperlib.snprintf(map_path_am, td_len_am + 32, "%s/_sha1_map.txt", temp_dir) + mf: fileio.File* = fileio.File(map_path_am, fileio.MODE.A) + line_am: bytes = stdlib.malloc(512) + # First fill the in-memory store (no file I/O), then write _sha1_map.txt + app_filled: int = 0 + for i in range(int(file_count)): + ea_am: uint64 = uint64(entries) + i * entry_size + ent_am: SrcFileEntry* = (SrcFileEntry*)(ea_am) + if ent_am is None or ent_am.Path is None or ent_am.Sha1 is None: continue + # Compute the relative path (strip the source_dir prefix, keep the directory structure) + p_str: str = ent_am.Path + p_len: sizet = string.strlen(p_str) + rel_path_am: str = p_str + if p_len > src_dir_len_am + 1: + if string.strncmp(p_str, source_dir, src_dir_len_am) == 0: rel_path_am = p_str + src_dir_len_am + 1 + # Also append to the in-memory store (for cross-module CDefine lookups) + # rel_path_am looks like "lib/core/StubMerger.vp", matching _sha1_map.txt + if StubMergerStore.AppendToSha1MapStore(ent_am.Sha1, rel_path_am) != 0: VLogger.error(f"AppendToSha1MapStore failed: {rel_path_am}", "project") + app_filled += 1 + # Write _sha1_map.txt (human-readable output) + if not mf.closed and line_am is not None: + viperlib.snprintf(line_am, 512, "%s:%s\n", ent_am.Sha1, rel_path_am) + ll_am: sizet = string.strlen(line_am) + mf.write(line_am, ll_am) + if line_am is not None: stdlib.free(line_am) + if not mf.closed: + mf.close() + VLogger.info(f"appended {app_filled} App files to _sha1_map.txt and the in-memory store", "project") + stdlib.free(map_path_am) + + # The module SHA1 map is filled into the StubMerger global store by PopulateSha1MapStore + + # AppendToSha1MapStore; HandlesExprCall._lookup_module_sha1 queries it directly, no local copy needed. + + + # === Phase A-pre-vpi: pre-generate .vpi stubs for all App sources === + # Deps' .vpi must exist before their dependents translate; pre-generating avoids missing dep declarations. + td_len_vpa: sizet = string.strlen(temp_dir) + PYI_BUF_SIZE_P2: sizet = 262144 + vpi_failed_p2: int = 0 + for i_vp2 in range(int(file_count)): + ea_vp2: uint64 = uint64(entries) + i_vp2 * entry_size + ent_vp2: SrcFileEntry* = (SrcFileEntry*)(ea_vp2) + if ent_vp2 is None or ent_vp2.Path is None or ent_vp2.Sha1 is None: + vpi_failed_p2 += 1 + VLogger.error("[Phase A-pre-vpi] invalid source entry (Path/Sha1 missing), skipping", "PhaseA") + continue + # Skip if the vpi already exists (sha1 is content-based, so a same-named vpi is valid; cross-run cache) + vpi_path_vp2: str = StubMergerCollect._sliced_path(temp_dir, td_len_vpa, ent_vp2.Sha1, "vpi") + if vpi_path_vp2 is None: + vpi_failed_p2 += 1 + VLogger.error(f"[Phase A-pre-vpi] {ent_vp2.Path} | failed to build vpi path", "PhaseA") + continue + vf_vp2: fileio.File* = fileio.File(vpi_path_vp2, fileio.MODE.R) + if not vf_vp2.closed: + vf_vp2.close() + stdlib.free(vpi_path_vp2) + continue + stdlib.free(vpi_path_vp2) + # Read the source -> parse the AST -> generate the vpi (rel_path = ent_vp2.Path: vpi content does + # not depend on it; it is only used for fatal_error attribution, pointing at this source file) + f_vp2: fileio.File* = fileio.File(ent_vp2.Path, fileio.MODE.R) + if f_vp2.closed: + vpi_failed_p2 += 1 + VLogger.error(f"[Phase A-pre-vpi] {ent_vp2.Path} | cannot open source file", "PhaseA") + continue + # read_all's capacity includes the terminator (reads at most cap-1 bytes); longer files are truncated + if f_vp2.size() >= SRC_BUF_SIZE: + vpi_failed_p2 += 1 + VLogger.error(f"[Phase A-pre-vpi] {ent_vp2.Path} | source file exceeds {SRC_BUF_SIZE} byte limit", "PhaseA") + f_vp2.close() + continue + src_buf_vp2: bytes = stdlib.malloc(SRC_BUF_SIZE) + if src_buf_vp2 is None: + f_vp2.close() + vpi_failed_p2 += 1 + VLogger.error(f"[Phase A-pre-vpi] {ent_vp2.Path} | source buffer allocation failed (OOM)", "PhaseA") + continue + bytes_read_vp2: LONG = f_vp2.read_all(src_buf_vp2, SRC_BUF_SIZE) + f_vp2.close() + if bytes_read_vp2 <= 0: + stdlib.free(src_buf_vp2) + vpi_failed_p2 += 1 + VLogger.error(f"[Phase A-pre-vpi] {ent_vp2.Path} | file read failed or empty", "PhaseA") + continue + if bytes_read_vp2 < SRC_BUF_SIZE: src_buf_vp2[bytes_read_vp2] = 0 + lx_vp2: ast.Lexer* = ast.new_lexer(mb) + if lx_vp2 is None: + stdlib.free(src_buf_vp2) + vpi_failed_p2 += 1 + VLogger.error(f"[Phase A-pre-vpi] {ent_vp2.Path} | lexer allocation failed (OOM)", "PhaseA") + continue + ast._lexer_init(lx_vp2, src_buf_vp2, mb) + tokens_vp2: ast.Token* = ast.tokenize(lx_vp2) + tree_vp2: ast.AST* = ast.parse_tokens(mb, tokens_vp2) + if tree_vp2 is None: + stdlib.free(src_buf_vp2) + vpi_failed_p2 += 1 + # Parse failure: inline the line/column + reason in the log line (same source as the full + # Python-style traceback below), since "AST parse failed" alone gives no location or cause. + VLogger.error(f"[Phase A-pre-vpi] {ent_vp2.Path} | AST parse failed (line {ast.get_parse_err_line()} col {ast.get_parse_err_col()}: {ast.get_parse_err_msg()})", "PhaseA") + GenApi.dump_parse_traceback(ent_vp2.Path) + continue + pyi_buf_vp2: bytes = stdlib.malloc(PYI_BUF_SIZE_P2) + if pyi_buf_vp2 is not None: + # 文件级名字窗口（FileFuncNames 等）由 _GeneratePyiFromAst 入口统一 reset+重建， + # 不再在此处重复调用；契约见 StructGenMixin.reset_file_name_window。 + pyi_pos_vp2: sizet = StubConverter._GeneratePyiFromAst(mb, tree_vp2, ent_vp2.Path, pyi_buf_vp2, PYI_BUF_SIZE_P2) + if pyi_pos_vp2 > 0: + pyi_path_vp2: str = StubMergerCollect._sliced_path(temp_dir, td_len_vpa, ent_vp2.Sha1, "vpi") + if pyi_path_vp2 is not None: + pf_vp2: fileio.File* = fileio.File(pyi_path_vp2, fileio.MODE.W) + if not pf_vp2.closed: + pf_vp2.write(pyi_buf_vp2, pyi_pos_vp2) + pf_vp2.close() + stdlib.free(pyi_path_vp2) + stdlib.free(pyi_buf_vp2) + else: + vpi_failed_p2 += 1 + VLogger.error(f"[Phase A-pre-vpi] {ent_vp2.Path} | pyi buffer allocation failed (OOM)", "PhaseA") + stdlib.free(src_buf_vp2) + # OOM cleanup: free this file's AST (the vpi is already on disk; persistent data is stdlib-allocated) + mb.reset_to(mark2) + if vpi_failed_p2 > 0: + VLogger.error(f"Phase A-pre-vpi: {vpi_failed_p2} files failed vpi pre-generation (see the per-file [Phase A-pre-vpi] logs above for failing files and causes)", "PhaseA") + return 1 + + + # === 2. Phase A: generate stub + text for each file === + # app_deps_buf collects App files' direct deps (space-separated) as Phase B+'s worklist (65536, matching StubMerger). + APP_DEPS_BUF_SIZE: sizet = 65536 + app_deps_buf: bytes = None + app_deps_len: sizet = 0 + if do_phase1 != 0: + log.banner("Phase A: generate stub + text") + # Initialize the Phase A convergence-failure diagnostic table (stdlib, survives reset) + _PhaseADiagInit(sizet(file_count) + StubMerger.MAX_INCLUDES) + + # === Phase A-pre-inc: pre-register includes structs + fill deps === + # Must run before App pre-registration (multi-pass resolves cross-file inheritance) and fills PopulateIncludesDeps. + if includes_dir is not None and includes_binary_dir is not None: + store_count_inc_pre: int = StubMergerStore.GetSha1StoreCount() + if store_count_inc_pre > 0: + store_sha1_inc_pre: bytes = StubMergerStore.GetSha1StoreArrPtr() + store_rel_inc_pre: bytes = StubMergerStore.GetSha1StoreRelArrPtr() + store_incdir_inc_pre: bytes = StubMergerStore.GetSha1StoreIncdirArrPtr() + deps_filled_pre: int = 0 + PHASE_A_INC_MAX_PASSES: int = 3 + # Track includes that fail pre-registration: after all passes, any failure aborts the build + # (no silent downgrade). App files must be skipped here; Phase A-pre handles them. + inc_fail_flags: bytes = stdlib.malloc(store_count_inc_pre) + if inc_fail_flags is not None: string.memset(inc_fail_flags, 1, store_count_inc_pre) + for pass_inc_i in range(int(PHASE_A_INC_MAX_PASSES)): + struct_count_before_inc: int = GenApi.get_struct_count() + for si_inc in range(int(store_count_inc_pre)): + inc_sha1_pre: str = store_sha1_inc_pre + sizet(si_inc) * 17 + inc_rel_pre: str = store_rel_inc_pre + sizet(si_inc) * StubMerger.MAX_REL_PATH_LEN + if inc_rel_pre is None or inc_rel_pre[0] == '\0': continue + # Skip App source files (main.vp etc.): pre-registered by Phase A-pre, not a failure + is_app_file_inc: int = 0 + for i_app_inc in range(int(file_count)): + ea_app_inc: uint64 = uint64(entries) + i_app_inc * entry_size + ent_app_inc: SrcFileEntry* = (SrcFileEntry*)(ea_app_inc) + if ent_app_inc is not None and ent_app_inc.Sha1 is not None and string.strcmp(ent_app_inc.Sha1, inc_sha1_pre) == 0: + is_app_file_inc = 1 + break + if is_app_file_inc != 0 and inc_fail_flags is not None: + inc_fail_flags[si_inc] = 0 + continue + rel_len_pre: sizet = string.strlen(inc_rel_pre) + # The includes root dir (per-entry from the IncDir array; empty falls back to includes_dir) + incdir_pre: str = store_incdir_inc_pre + sizet(si_inc) * StubMerger.MAX_INC_DIR_LEN + if incdir_pre is None or incdir_pre[0] == '\0': incdir_pre = includes_dir + incdir_len_pre: sizet = string.strlen(incdir_pre) + full_path_pre: bytes = stdlib.malloc(incdir_len_pre + rel_len_pre + 2) + if full_path_pre is None: continue + viperlib.snprintf(full_path_pre, incdir_len_pre + rel_len_pre + 2, "%s/%s", incdir_pre, inc_rel_pre) + pkg_inc_pre: str = GenApi.compute_package_from_relpath(mb, inc_rel_pre) + tr_inc_pre: HandlesTranslator.Translator* = BuildPipeline.TranslateFileGetTrans(mb, full_path_pre, inc_sha1_pre, pkg_inc_pre, 1) + stdlib.free(full_path_pre) + if tr_inc_pre is not None: + # Record this file's deps (for convergence diagnostics; stdlib table survives reset) + _PhaseADiagRecord(inc_sha1_pre, tr_inc_pre._imported_modules, None) + # Pre-registration succeeded: clear the failure flag + if inc_fail_flags is not None: inc_fail_flags[si_inc] = 0 + # Fill deps only on the first pass (PopulateIncludesDeps dedupes internally). + # Must be called before reset_to(mark2) since it deep-copies into stdlib. + if pass_inc_i == 0 and tr_inc_pre._imported_modules is not None and StubMergerStore.PopulateIncludesDeps(inc_sha1_pre, tr_inc_pre._imported_modules) == 0: + deps_filled_pre = deps_filled_pre + 1 + # OOM cleanup: free this includes file's transient pre-registration data + # (AST/tokens/lexer/IR/Handler); stdlib-allocated struct entries are unaffected + mb.reset_to(mark2) + struct_count_after_inc: int = GenApi.get_struct_count() + # Convergence check: no new struct registered this pass -> all classes are registered + if struct_count_after_inc == struct_count_before_inc: break + # Includes still failing after all passes -> abort (no downgrade) + if inc_fail_flags is not None: + for si_inc_f in range(int(store_count_inc_pre)): + if inc_fail_flags[si_inc_f] != 0: + rel_f: str = store_rel_inc_pre + sizet(si_inc_f) * StubMerger.MAX_REL_PATH_LEN + VLogger.error(f"includes pre-registration failed: {rel_f} (file cannot be translated; possible syntax/parse/compile error), aborting compilation", "prePhaseA") + stdlib.free(inc_fail_flags) + VLogger.info(f"includes pre-registration + dependency fill: {deps_filled_pre}/{store_count_inc_pre}", "prePhaseA") + + # === Phase A-pre: pre-register struct/enum/union for all sources === + # Registers declarations only (declare_only=1), no method bodies; multi-pass resolves cross-file ordering. + VLogger.info("pre-registering struct/enum/union...", "PhaseA") + src_dir_len_pre: sizet = string.strlen(source_dir) + PHASE_A_PRE_MAX_PASSES: int = 3 + for pass_i in range(int(PHASE_A_PRE_MAX_PASSES)): + struct_count_before: int = GenApi.get_struct_count() + for i in range(int(file_count)): + ea_pre: uint64 = uint64(entries) + i * entry_size + ent_pre: SrcFileEntry* = (SrcFileEntry*)(ea_pre) + if ent_pre is None or ent_pre.Path is None: continue + pkg_pre: str = None + if string.strlen(ent_pre.Path) > src_dir_len_pre + 1: + rel_path_pre: str = ent_pre.Path + src_dir_len_pre + 1 + pkg_pre = GenApi.compute_package_from_relpath(mb, rel_path_pre) + # App pre-registration passes pre_allow_dep_inject=1: imports and dep vpi injection happen at + # declaration time so field annotations share Phase A's context (else struct layouts diverge). + tr_pre: HandlesTranslator.Translator* = BuildPipeline.TranslateFileGetTrans(mb, ent_pre.Path, ent_pre.Sha1, pkg_pre, 1, None, 1) + if tr_pre is None: VLogger.error(f"pre-registration failed: {ent_pre.Path}", "PhaseA") + # OOM cleanup: free this App file's transient pre-registration data + mb.reset_to(mark2) + struct_count_after: int = GenApi.get_struct_count() + # Convergence check: no new struct registered this pass -> all classes are registered + if struct_count_after == struct_count_before: break + VLogger.info("pre-registration complete", "PhaseA") + + # Phase A IR buffer: 1MB is not enough, ast/parser.vp's text.ll exceeds 1MB and got truncated -> + # merged IR lost the tail (opt: "end of file in global variable name"). + PHASE_A_IR_SIZE: sizet = 8388608 + # FULL dump = stub + text (same module), with 2x headroom + PHASE_A_FULL_SIZE: sizet = 16777216 + td_len_pa: sizet = string.strlen(temp_dir) + + # Set the temp dir so HandlesExprCall can read dependency modules' text.ll for cross-module + # method return-type lookup (avoids pointer returns being truncated to i32) + GenApi.SetTempDir(temp_dir) + + # app_deps_buf collects App files' direct dependency modules (space-separated) as Phase B+'s initial + # worklist for _BuildReachableSha1Set; declared outside the if, only assigned here. + app_deps_buf = stdlib.malloc(APP_DEPS_BUF_SIZE) + app_deps_len = 0 + if app_deps_buf is not None: app_deps_buf[0] = '\0' + + # Phase A multi-pass retry: with several sources, main.vp may translate before its imports + # (filesystem order); failed files are retried next pass, converging in at most file_count passes. + done_flags: bytes = stdlib.malloc(file_count) + if done_flags is None: + VLogger.error("Phase A done_flags allocation failed", "PhaseA") + stdlib.free(entries) + return 1 + string.memset(done_flags, 0, file_count) + PHASE_A_MAX_PASSES: int = 32 + for pass_a in range(int(PHASE_A_MAX_PASSES)): + completed_this_pass: int = 0 + for i in range(int(file_count)): + if done_flags[i] != 0: continue + ea: uint64 = uint64(entries) + i * entry_size + ent: SrcFileEntry* = (SrcFileEntry*)(ea) + # Timing point 0: start processing this file (translate phase start) + t_pa0: uint64 = win32.win32base.GetTickCount64() + if ent is None or ent.Path is None: + done_flags[i] = 1 + completed_this_pass += 1 + continue + # Compute the package name (the directory part relative to source_dir) + src_dir_len_pa: sizet = string.strlen(source_dir) + pkg_pa: str = None + if string.strlen(ent.Path) > src_dir_len_pa + 1: + rel_path_pa: str = ent.Path + src_dir_len_pa + 1 + pkg_pa = GenApi.compute_package_from_relpath(mb, rel_path_pa) + tr_a: HandlesTranslator.Translator* = BuildPipeline.TranslateFileGetTrans(mb, ent.Path, ent.Sha1, pkg_pa, 0, temp_dir) + # Timing point 1: translate returned + t_pa1: uint64 = win32.win32base.GetTickCount64() + if tr_a is None: + # Translation failed (syntax/parse error): retry next pass; convergence check reports it. + # OOM cleanup: a failed translation also produced AST/tokens, freed immediately. + mb.reset_to(mark2) + continue + # Record this file's deps (for convergence diagnostics; stdlib table survives reset) + _PhaseADiagRecord(ent.Sha1, tr_a._imported_modules, None) + # Collect _imported_modules into app_deps_buf (for Phase B+ on-demand compilation). + # Dependencies were filled early during translation. + if tr_a._imported_modules is not None and app_deps_buf is not None: + # Dedupe per module: several App files importing the same module would otherwise inflate + # app_deps toward 64KB, filling the initial worklist and overflowing BFS. + im_len_a: sizet = string.strlen(tr_a._imported_modules) + im_pos_a: sizet = 0 + while im_pos_a < im_len_a: + im_start_a: sizet = im_pos_a + while im_pos_a < im_len_a and tr_a._imported_modules[im_pos_a] != ' ': im_pos_a += 1 + im_mod_len: sizet = im_pos_a - im_start_a + if im_pos_a < im_len_a: im_pos_a += 1 + if im_mod_len == 0: continue + im_mod_buf: bytes = stdlib.malloc(im_mod_len + 1) + if im_mod_buf is None: continue + string.strncpy(im_mod_buf, tr_a._imported_modules + im_start_a, im_mod_len) + im_mod_buf[im_mod_len] = '\0' + # Skip if already in app_deps (exact word-boundary match) + if app_deps_len > 0 and GenApi.is_module_imported(app_deps_buf, im_mod_buf) != 0: + stdlib.free(im_mod_buf) + continue + if app_deps_len + im_mod_len + 2 < APP_DEPS_BUF_SIZE: + if app_deps_len > 0: + app_deps_buf[app_deps_len] = ' ' + app_deps_len = app_deps_len + 1 + string.strcpy(app_deps_buf + app_deps_len, im_mod_buf) + app_deps_len = app_deps_len + im_mod_len + app_deps_buf[app_deps_len] = '\0' + stdlib.free(im_mod_buf) + # dump stub IR (declarations only) + stub_buf_a: bytes = stdlib.malloc(PHASE_A_IR_SIZE) + if stub_buf_a is None: VLogger.error("stub buffer allocation failed", "PhaseA") + tr_a.dump_ir(stub_buf_a, PHASE_A_IR_SIZE, mlirlite.OUTPUT_STUB) + stub_len_a: sizet = string.strlen(stub_buf_a) + + # Save stub.ll (sliced path: temp_dir/{sha1 prefix}/{sha1}.stub.ll) + stub_path_a: str = StubMergerCollect._sliced_path(temp_dir, td_len_pa, ent.Sha1, "stub.ll") + if stub_path_a is not None: + sf_a: fileio.File* = fileio.File(stub_path_a, fileio.MODE.W) + if not sf_a.closed: + sf_a.write(stub_buf_a, stub_len_a) + sf_a.close() + stdlib.free(stub_path_a) + stdlib.free(stub_buf_a) + + # dump text IR (definitions only) + text_buf_a: bytes = stdlib.malloc(PHASE_A_IR_SIZE) + if text_buf_a is None: VLogger.error("text buffer allocation failed", "PhaseA") + tr_a.dump_ir(text_buf_a, PHASE_A_IR_SIZE, mlirlite.OUTPUT_TEXT) + text_len_a: sizet = string.strlen(text_buf_a) + + # Save text.ll (sliced path) + text_path_a: str = StubMergerCollect._sliced_path(temp_dir, td_len_pa, ent.Sha1, "text.ll") + if text_path_a is not None: + tf_a: fileio.File* = fileio.File(text_path_a, fileio.MODE.W) + if not tf_a.closed: + tf_a.write(text_buf_a, text_len_a) + tf_a.close() + stdlib.free(text_path_a) + stdlib.free(text_buf_a) + + # Save {sha1}.meta (structured sidecar: function list / struct layout / vtable) + StubMergerIR.WriteModuleMeta(tr_a.Module, temp_dir, td_len_pa, ent.Sha1) + + # Save {sha1}.full.ll (in-memory FULL artifact: a self-contained single-module IR) + full_buf_a: bytes = stdlib.malloc(PHASE_A_FULL_SIZE) + if full_buf_a is None: VLogger.error("full.ll buffer allocation failed", "PhaseA") + tr_a.dump_ir(full_buf_a, PHASE_A_FULL_SIZE, mlirlite.OUTPUT_FULL) + full_len_a: sizet = string.strlen(full_buf_a) + full_path_a: str = StubMergerCollect._sliced_path(temp_dir, td_len_pa, ent.Sha1, "full.ll") + if full_path_a is not None: + ffa: fileio.File* = fileio.File(full_path_a, fileio.MODE.W) + if not ffa.closed: + ffa.write(full_buf_a, full_len_a) + ffa.close() + stdlib.free(full_path_a) + stdlib.free(full_buf_a) + + # --traceback: take this module's debug call-site records and write .vpcall. + if BuildPipeline._traceback_mode != 0: + vpcall_text_a: str = mlirlite.DebugTakeText() + if vpcall_text_a is not None: + BuildPipeline.WriteCallSidecar(temp_dir, ent.Sha1, vpcall_text_a) + stdlib.free(vpcall_text_a) + + # Save dependencies (_imported_modules) for Phase B (sliced path) + deps_path_a: str = StubMergerCollect._sliced_path(temp_dir, td_len_pa, ent.Sha1, "deps.txt") + if deps_path_a is not None: + df_a: fileio.File* = fileio.File(deps_path_a, fileio.MODE.W) + if not df_a.closed: + if tr_a._imported_modules is not None: + dl_a: sizet = string.strlen(tr_a._imported_modules) + df_a.write(tr_a._imported_modules, dl_a) + df_a.close() + stdlib.free(deps_path_a) + + # Timing point 2: stub+text dump and write done; print per-file progress (as in Phase1 1b) + t_pa2: uint64 = win32.win32base.GetTickCount64() + VLogger.info(f"[{i+1}/{file_count}] PhaseA: {ent.Path} (translate={t_pa1-t_pa0} ms, dump={t_pa2-t_pa1} ms, total={t_pa2-t_pa0} ms)", "PhaseA") + + done_flags[i] = 1 + completed_this_pass += 1 + # OOM cleanup: stub/text/deps are on disk, so free this file's transient translation data + # (AST/tokens/lexer/IR/Handler); app_deps_buf already deep-copied the deps. + mb.reset_to(mark2) + # Convergence check: stop when all files are done + all_done: int = 1 + for i_ck in range(int(file_count)): + if done_flags[i_ck] == 0: + all_done = 0 + break + if all_done != 0: break + if completed_this_pass == 0: + # No progress: remaining files cannot finish (real compile errors). Abort and list them, each + # with a dependency-graph dump. VLogger.error calls sys.exit(1), so it must come after all output. + td_len_fail: sizet = string.strlen(temp_dir) + for i_err in range(int(file_count)): + if done_flags[i_err] == 0: + ea_err: uint64 = uint64(entries) + i_err * entry_size + ent_err: SrcFileEntry* = (SrcFileEntry*)(ea_err) + if ent_err is not None and ent_err.Path is not None: + if ent_err.Sha1 is not None: _PhaseADumpFailGraph(temp_dir, td_len_fail, ent_err.Path, ent_err.Sha1) + stdlib.free(done_flags) + _PhaseADiagFree() + VLogger.error("Phase A multi-pass convergence failed, aborting compilation", "PhaseA") + stdlib.free(entries) + return 1 + stdlib.free(done_flags) + # Phase A converged: free the diagnostic table (needed only on the failure path) + _PhaseADiagFree() + + if do_phase2 == 0: + VLogger.info("Phase A complete (stub generation only)", "project") + if app_deps_buf is not None: stdlib.free(app_deps_buf) + stdlib.free(entries) + return 0 + + # === 3. Phase B: compile each file to .obj === + if do_phase2 != 0: + log.banner("Phase B: compile .obj") + + # Collect .obj paths (64KB to avoid includes .obj path overflow dropping link symbols). + # reachable_set/reachable_count: filled by Phase B+, used by Phase C for on-demand linking. + reachable_set: bytes = None + reachable_count: int = 0 + OBJ_PATHS_SIZE: sizet = 65536 + obj_paths: bytes = stdlib.malloc(OBJ_PATHS_SIZE) + if obj_paths is None: return 1 + obj_paths[0] = '\0' + obj_pos: sizet = 0 + # main_obj_path: the .obj that defines the user main (placed first at link time so the linker + # does not pick another module's wrapper main) + main_obj_path: bytes = stdlib.malloc(512) + if main_obj_path is None: return 1 + main_obj_path[0] = '\0' + compiled_count: int = 0 + + # IR buffer size (16MB, to hold the in-memory FULL artifact {sha1}.full.ll; FULL = stub + text) + COMBINED_IR_SIZE: sizet = 16777216 + + # llc task table (stdlib.malloc, outlives threads): a contiguous array of LlcTask records; the loop + # only writes .ll + builds tasks, then run_llc_parallel runs llc in parallel batches. + llc_tasks: bytes = stdlib.malloc(sizet(file_count) * BuildPipeline.LlcTask.__sizeof__()) + if llc_tasks is None: + VLogger.error("llc task table allocation failed", "PhaseB") + return 1 + llc_task_count: int = 0 + + for i in range(int(file_count)): + ea: uint64 = uint64(entries) + i * entry_size + ent: SrcFileEntry* = (SrcFileEntry*)(ea) + if ent is None or ent.Path is None or ent.Sha1 is None: continue + # Start of this file's three-step timing (GetTickCount64, ms) + t_pb0: uint64 = win32.win32base.GetTickCount64() + # Read the in-memory FULL artifact {sha1}.full.ll -> complete IR (replaces merging stub + dep stub/text) + combined_ir: bytes = stdlib.malloc(COMBINED_IR_SIZE) + if combined_ir is None: + VLogger.error(f"combined_ir allocation failed: {ent.Path}", "PhaseB") + continue + combined_len: sizet = StubMergerBuild.LoadFullIR(temp_dir, ent.Sha1, combined_ir, COMBINED_IR_SIZE) + if combined_len == 0: + VLogger.error(f"failed to read full.ll: {ent.Path}", "PhaseB") + stdlib.free(combined_ir) + continue + # Timing point 1: full.ll read done + t_pb1: uint64 = win32.win32base.GetTickCount64() + + # Write .ll (llc threads only read this file); llc itself runs parallel in batches after the loop + ret_w: int = BuildPipeline.write_ir_to_file(combined_ir, combined_len, temp_dir, ent.Sha1) + stdlib.free(combined_ir) + if ret_w != 0: + VLogger.error(f"failed to write .ll: {ent.Path}", "PhaseB") + sys.exit(1) + # Timing point 2: .ll write done + t_pb2: uint64 = win32.win32base.GetTickCount64() + + # Build the llc task (command string is stdlib.malloc'd so threads' lifetimes are not reset-reclaimed) + llc_ir_path: str = BuildPipeline.build_sliced_path(temp_dir, ent.Sha1, "ll") + if llc_ir_path is None: + VLogger.error("failed to build llc input path", "PhaseB") + sys.exit(1) + tslot: BuildPipeline.LlcTask* = BuildPipeline.LlcTask*(uint64(llc_tasks) + sizet(llc_task_count) * BuildPipeline.LlcTask.__sizeof__()) + if BuildPipeline.init_llc_task(tslot, llc_ir_path, output_dir, ent.Sha1, cc_cmd, cc_flags) != 0: + VLogger.error(f"failed to build llc task: {ent.Path}", "PhaseB") + sys.exit(1) + # Timing point 3: llc task construction done + t_pb3: uint64 = win32.win32base.GetTickCount64() + llc_task_count += 1 + + compiled_count += 1 + # Show files submitted to llc (with the three timings: merge stub / write .ll / build task) + VLogger.info(f"[{i+1}/{file_count}] llc submitted: {ent.Path} (merge={t_pb1-t_pb0} ms, write={t_pb2-t_pb1} ms, task={t_pb3-t_pb2} ms, total={t_pb3-t_pb0} ms)", "PhaseB") + # Build the .obj path and detect the main module (test_main.vp or main.vp; .py no longer recognized) + od_len: sizet = string.strlen(output_dir) + is_main_mod: int = 0 + if string.strstr(ent.Path, "test_main.vp") is not None: is_main_mod = 1 + elif string.strstr(ent.Path, "main.vp") is not None: is_main_mod = 1 + + # Sliced path: output_dir/{sha1 prefix}/{sha1}.obj + obj_path_sliced: str = StubMergerCollect._sliced_path(output_dir, od_len, ent.Sha1, "obj") + if obj_path_sliced is not None: + op_sliced_len: sizet = string.strlen(obj_path_sliced) + if is_main_mod != 0: + # main module: copy to main_obj_path (so main is first at link time) + if op_sliced_len < 512: string.strcpy(main_obj_path, obj_path_sliced) + else: VLogger.error("main_obj_path buffer too small, aborting compilation", "PhaseB") + else: + # Other modules: append to obj_paths + if obj_pos + op_sliced_len + 2 < OBJ_PATHS_SIZE: + if obj_pos > 0: + obj_paths[obj_pos] = ' ' + obj_pos += 1 + string.strcpy(obj_paths + obj_pos, obj_path_sliced) + obj_pos += op_sliced_len + obj_paths[obj_pos] = '\0' + else: VLogger.error(".obj path buffer too small", "PhaseB") + stdlib.free(obj_path_sliced) + + # Run all llc jobs in parallel (batched at LLC_MAX_THREADS threads) + ret_p: int = BuildPipeline.run_llc_parallel(llc_tasks, llc_task_count, BuildPipeline.LLC_MAX_THREADS) + if ret_p != 0: + VLogger.error("llc compilation failed, aborting compilation", "PhaseB") + sys.exit(1) + + # Free the llc tasks (command strings + records) + ti: int = 0 + while ti < llc_task_count: + tfree: BuildPipeline.LlcTask* = BuildPipeline.LlcTask*(uint64(llc_tasks) + sizet(ti) * BuildPipeline.LlcTask.__sizeof__()) + BuildPipeline.free_llc_task(tfree) + ti += 1 + stdlib.free(llc_tasks) + + # Assemble .s files: llvm-mc -> .obj (isr.s etc., bare symbols matching the .vp CExport/State stubs). + # The obj is added to obj_paths for Phase C linking. + od_len_as: sizet = string.strlen(output_dir) + asm_compiled: int = 0 + for ai_as in range(int(asm_file_count)): + ea_as: uint64 = uint64(asm_entries) + ai_as * entry_size + ent_as: SrcFileEntry* = (SrcFileEntry*)(ea_as) + if ent_as is None or ent_as.Path is None or ent_as.Sha1 is None: continue + aret_as: int = BuildPipeline.asm_to_obj(ent_as.Path, ent_as.Sha1, output_dir, Config.TargetTriple) + if aret_as != 0: + VLogger.error(f"llvm-mc assembly failed, aborting compilation: {ent_as.Path}", "PhaseB") + sys.exit(1) + obj_path_asm: str = StubMergerCollect._sliced_path(output_dir, od_len_as, ent_as.Sha1, "obj") + if obj_path_asm is not None: + op_len_asm: sizet = string.strlen(obj_path_asm) + if obj_pos + op_len_asm + 2 < OBJ_PATHS_SIZE: + if obj_pos > 0: + obj_paths[obj_pos] = ' ' + obj_pos += 1 + string.strcpy(obj_paths + obj_pos, obj_path_asm) + obj_pos += op_len_asm + obj_paths[obj_pos] = '\0' + stdlib.free(obj_path_asm) + asm_compiled += 1 + VLogger.info(f"[asm {asm_compiled}] assembly complete: {ent_as.Path}", "PhaseB") + + VLogger.success(f"compilation complete: {compiled_count}/{file_count}", "PhaseB") + + if compiled_count == 0: + VLogger.error("no files compiled successfully", "PhaseB") + stdlib.free(entries) + stdlib.free(asm_entries) + return 1 + + # === 3.5 Compile missing includes (on demand via the dependency graph) === + # Only compile includes reached directly/indirectly from App sources (via _BuildReachableSha1Set). + if includes_dir is not None and includes_binary_dir is not None: + inc_compiled: int = 0 + td_len_mi: sizet = string.strlen(temp_dir) + + # Diagnostic log: confirm the Phase B+ entry and Sha1Store state + VLogger.info(f"PhaseB+ entry: store_count={StubMergerStore.GetSha1StoreCount()} includes_binary_dir={includes_binary_dir}", "PhaseB+") + + # Create includes_binary_dir (so .obj files can be written and collect_obj_files can scan them) + BuildPipeline.ensure_dir(includes_binary_dir) + + # declare_only preprocessing was moved before Phase A (Phase A-pre-inc) so includes structs are + # registered when Phase A translates App sources. PopulateIncludesDeps is filled there too. + + # Build the reachable SHA1 set (dependency graph traversal). + # When app_deps_buf is None, fall back to scanning source_dir (non-recursive). + REACHABLE_SET_SIZE: sizet = sizet(StubMerger.MAX_INCLUDES_SHA1) * 17 + reachable_set: bytes = stdlib.malloc(REACHABLE_SET_SIZE) + reachable_count: int = 0 + if reachable_set is not None: + string.memset(reachable_set, 0, REACHABLE_SET_SIZE) + reachable_count = StubMergerBuild._BuildReachableSha1Set(mb, source_dir, temp_dir, reachable_set, app_deps_buf, includes_dir) + VLogger.info(f"reachable includes SHA1: {reachable_count} entries", "PhaseB+") + # fast-fail: reachable-set construction failed (deps not found); never fall back, abort + if reachable_count < 0: + VLogger.error("reachable set construction failed (fast-fail), aborting compilation", "PhaseB+") + return 1 + else: + VLogger.error("reachable_set allocation failed, aborting compilation", "PhaseB+") + return 1 + + # Free app_deps_buf (reachable_set is built, no longer needed) + if app_deps_buf is not None: + stdlib.free(app_deps_buf) + app_deps_buf = None + + # Get the SHA1/module-name arrays from the global store as local copies (bytes) to avoid address + # issues from passing the store's union return (bytes*) across functions; rel/incdir keep pointers. + td_len_pb: sizet = string.strlen(temp_dir) + store_sha1_arr_mi: bytes = stdlib.malloc(StubMerger.MAX_INCLUDES * 17) + store_mod_arr_mi: bytes = stdlib.malloc(StubMerger.MAX_INCLUDES * 64) + store_count_mi: int = 0 + if store_sha1_arr_mi is not None and store_mod_arr_mi is not None: + store_count_mi = StubMergerIR._BuildIncludesSha1Map(temp_dir, td_len_pb, store_sha1_arr_mi, store_mod_arr_mi) + store_rel_arr_mi: bytes = StubMergerStore.GetSha1StoreRelArrPtr() + store_incdir_arr_mi: bytes = StubMergerStore.GetSha1StoreIncdirArrPtr() + if store_count_mi > 0: + + # Topologically sort reachable includes: dependencies are fully translated first. Cross-module + # vtable generation needs dependency functions' TypeStr registered globally (e.g. astaux/base AST). + topo_idx: sizet* = (sizet*)(stdlib.malloc(8 * (store_count_mi + 1))) + topo_count: int = 0 + if topo_idx is None: + VLogger.error("topological sort index allocation failed, aborting compilation", "PhaseB+") + topo_count = 0 + else: + topo_count = StubMergerTopo.BuildTopoSortedIndices(store_count_mi, store_sha1_arr_mi, store_mod_arr_mi, reachable_set, reachable_count, topo_idx) + + for si_mi in range(int(topo_count)): + # Current entry (topological index; reachable modules only) + ti: int = int(topo_idx[si_mi]) + # Get the current entry's SHA1 and rel_path + inc_sha1_mi: str = store_sha1_arr_mi + sizet(ti) * 17 + inc_rel_mi: str = store_rel_arr_mi + sizet(ti) * StubMerger.MAX_REL_PATH_LEN + # Skip App sources: an App file may be scanned as an include and be reachable; compiling + # another .obj into includes_binary would duplicate the output/ .obj's symbols. + app_skip_mi: int = 0 + for ai_mi in range(int(file_count)): + ea_ap: uint64 = uint64(entries) + ai_mi * entry_size + ent_ap: SrcFileEntry* = (SrcFileEntry*)(ea_ap) + if ent_ap is not None and ent_ap.Sha1 is not None: + if string.strcmp(ent_ap.Sha1, inc_sha1_mi) == 0: + app_skip_mi = 1 + break + if app_skip_mi != 0: continue + # Check whether the .obj already exists in includes.binary (sliced subdir) + ibd_len_mi: sizet = string.strlen(includes_binary_dir) + check_pat_mi: str = StubMergerCollect._sliced_path(includes_binary_dir, ibd_len_mi, inc_sha1_mi, "obj") + check_fd_mi: win32.win32file.WIN32_FIND_DATAA* = stdlib.malloc(win32.win32file.WIN32_FIND_DATAA.__sizeof__()) + obj_exists_mi: int = 0 + if check_pat_mi is not None and check_fd_mi is not None: + string.memset(check_fd_mi, 0, win32.win32file.WIN32_FIND_DATAA.__sizeof__()) + check_h_mi: win32.win32base.HANDLE = win32.win32file.FindFirstFileA(check_pat_mi, check_fd_mi) + if check_h_mi != win32.win32base.INVALID_HANDLE_VALUE: + win32.win32file.FindClose(check_h_mi) + obj_exists_mi = 1 + if obj_exists_mi != 0: + if check_pat_mi is not None: stdlib.free(check_pat_mi) + stdlib.free(check_fd_mi) + continue + + # The .obj does not exist; compile it. + # Build the source path: {includes root}/{rel_path} (per-entry IncDir; empty falls back to includes_dir) + rel_path_len_mi: sizet = string.strlen(inc_rel_mi) + incdir_mi: str = store_incdir_arr_mi + sizet(ti) * StubMerger.MAX_INC_DIR_LEN + if incdir_mi is None or incdir_mi[0] == '\0': incdir_mi = includes_dir + inc_dir_len_mi: sizet = string.strlen(incdir_mi) + src_fp_mi: bytes = stdlib.malloc(inc_dir_len_mi + 1 + rel_path_len_mi + 1) + if src_fp_mi is None: + stdlib.free(check_pat_mi) + if check_fd_mi is not None: stdlib.free(check_fd_mi) + VLogger.error("includes source path allocation failed, aborting compilation", "PhaseB+") + viperlib.snprintf(src_fp_mi, inc_dir_len_mi + 1 + rel_path_len_mi + 1, "%s/%s", incdir_mi, inc_rel_mi) + + # Check whether it is a declaration-only file (read {sha1}.meta: no FUNC ... DEF means pure declaration) + is_decl_mi: int = StubMergerIR.IsDeclOnlyModule(temp_dir, td_len_mi, inc_sha1_mi) + + # is_decl_mi: -1 = no meta (Phase1 did not translate it), 1 = declaration-only, 0 = impl. + # Skip only declaration-only (is_decl_mi == 1); -1 and 0 continue to the translation logic. + if is_decl_mi == 1: + stdlib.free(check_pat_mi) + if check_fd_mi is not None: stdlib.free(check_fd_mi) + stdlib.free(src_fp_mi) + continue + + # Try to read the in-memory FULL artifact {sha1}.full.ll (should be produced by Phase1/PhaseA) + inc_combined_mi: bytes = stdlib.malloc(COMBINED_IR_SIZE) + inc_combined_len_mi: sizet = 0 + if inc_combined_mi is not None: inc_combined_len_mi = StubMergerBuild.LoadFullIR(temp_dir, inc_sha1_mi, inc_combined_mi, COMBINED_IR_SIZE) + + # If full.ll is missing, translate the source, save stub/text/meta/full, then retry + if inc_combined_len_mi == 0 and inc_combined_mi is not None: + # Compute the includes file's package name (the directory part relative to includes_dir) + inc_pkg_mi: str = None + if string.strlen(src_fp_mi) > inc_dir_len_mi + 1: + inc_rel_mi2: str = src_fp_mi + inc_dir_len_mi + 1 + inc_pkg_mi = GenApi.compute_package_from_relpath(mb, inc_rel_mi2) + tr_mi: HandlesTranslator.Translator* = BuildPipeline.TranslateFileGetTrans(mb, src_fp_mi, inc_sha1_mi, inc_pkg_mi) + if tr_mi is not None: + # PBP IR buffer: same as PHASE_A_IR_SIZE (parser.vp's text.ll >1MB truncation root cause) + PBP_IR_SIZE: sizet = 8388608 + # FULL dump = stub + text, with 2x headroom + PBP_FULL_SIZE: sizet = 16777216 + # Save stub.ll (sliced path) + inc_stub_buf: bytes = stdlib.malloc(PBP_IR_SIZE) + if inc_stub_buf is not None: + tr_mi.dump_ir(inc_stub_buf, PBP_IR_SIZE, mlirlite.OUTPUT_STUB) + inc_stub_len: sizet = string.strlen(inc_stub_buf) + inc_stub_path: str = StubMergerCollect._sliced_path(temp_dir, td_len_mi, inc_sha1_mi, "stub.ll") + if inc_stub_path is not None: + isf: fileio.File* = fileio.File(inc_stub_path, fileio.MODE.W) + if not isf.closed: + isf.write(inc_stub_buf, inc_stub_len) + isf.close() + stdlib.free(inc_stub_path) + stdlib.free(inc_stub_buf) + # Save text.ll (sliced path) + inc_text_buf: bytes = stdlib.malloc(PBP_IR_SIZE) + if inc_text_buf is not None: + tr_mi.dump_ir(inc_text_buf, PBP_IR_SIZE, mlirlite.OUTPUT_TEXT) + inc_text_len: sizet = string.strlen(inc_text_buf) + inc_text_path: str = StubMergerCollect._sliced_path(temp_dir, td_len_mi, inc_sha1_mi, "text.ll") + if inc_text_path is not None: + itf: fileio.File* = fileio.File(inc_text_path, fileio.MODE.W) + if not itf.closed: + itf.write(inc_text_buf, inc_text_len) + itf.close() + stdlib.free(inc_text_path) + stdlib.free(inc_text_buf) + # Save {sha1}.meta (structured sidecar: function list / struct layout / vtable) + StubMergerIR.WriteModuleMeta(tr_mi.Module, temp_dir, td_len_mi, inc_sha1_mi) + # Save {sha1}.full.ll (in-memory FULL artifact) + inc_full_buf: bytes = stdlib.malloc(PBP_FULL_SIZE) + if inc_full_buf is None: VLogger.error("full.ll buffer allocation failed", "PhaseB+") + tr_mi.dump_ir(inc_full_buf, PBP_FULL_SIZE, mlirlite.OUTPUT_FULL) + inc_full_len: sizet = string.strlen(inc_full_buf) + inc_full_path: str = StubMergerCollect._sliced_path(temp_dir, td_len_mi, inc_sha1_mi, "full.ll") + if inc_full_path is not None: + ifff: fileio.File* = fileio.File(inc_full_path, fileio.MODE.W) + if not ifff.closed: + ifff.write(inc_full_buf, inc_full_len) + ifff.close() + stdlib.free(inc_full_path) + stdlib.free(inc_full_buf) + # --traceback: take this module's debug call-site records and write .vpcall. + if BuildPipeline._traceback_mode != 0: + vpcall_text_mi: str = mlirlite.DebugTakeText() + if vpcall_text_mi is not None: + BuildPipeline.WriteCallSidecar(temp_dir, inc_sha1_mi, vpcall_text_mi) + stdlib.free(vpcall_text_mi) + # Retry reading full.ll + inc_combined_len_mi = StubMergerBuild.LoadFullIR(temp_dir, inc_sha1_mi, inc_combined_mi, COMBINED_IR_SIZE) + # OOM cleanup: stub/text/meta/full are on disk, free this includes file's transient + # translation data (AST/tokens/lexer/IR/Handler); reads only touch files. + mb.reset_to(mark2) + + if inc_combined_len_mi == 0: + # Diagnostic: check whether full.ll/stub.ll/text.ll exist + diag_full: str = StubMergerCollect._sliced_path(temp_dir, td_len_mi, inc_sha1_mi, "full.ll") + diag_stub: str = StubMergerCollect._sliced_path(temp_dir, td_len_mi, inc_sha1_mi, "stub.ll") + diag_text: str = StubMergerCollect._sliced_path(temp_dir, td_len_mi, inc_sha1_mi, "text.ll") + diag_full_ex: int = 0 + diag_stub_ex: int = 0 + diag_text_ex: int = 0 + if diag_full is not None: + df_diag_f: fileio.File* = fileio.File(diag_full, fileio.MODE.R) + if not df_diag_f.closed: + diag_full_ex = 1 + df_diag_f.close() + stdlib.free(diag_full) + if diag_stub is not None: + df_diag_s: fileio.File* = fileio.File(diag_stub, fileio.MODE.R) + if not df_diag_s.closed: + diag_stub_ex = 1 + df_diag_s.close() + stdlib.free(diag_stub) + if diag_text is not None: + df_diag_t: fileio.File* = fileio.File(diag_text, fileio.MODE.R) + if not df_diag_t.closed: + diag_text_ex = 1 + df_diag_t.close() + stdlib.free(diag_text) + VLogger.error(f"failed to read full.ll: {src_fp_mi} (full={diag_full_ex} stub={diag_stub_ex} text={diag_text_ex} is_decl={is_decl_mi})", "PhaseB+") + if inc_combined_mi is not None: stdlib.free(inc_combined_mi) + stdlib.free(check_pat_mi) + if check_fd_mi is not None: stdlib.free(check_fd_mi) + stdlib.free(src_fp_mi) + continue + + # Compile to .obj (output to includes_binary_dir, consistent with the existence check and link collection) + inc_cret_mi: int = BuildPipeline.compile_module_to_obj(inc_combined_mi, inc_combined_len_mi, temp_dir, includes_binary_dir, inc_sha1_mi, cc_cmd, cc_flags) + stdlib.free(inc_combined_mi) + + if inc_cret_mi != 0: + VLogger.error(f"llc compilation failed, aborting compilation: {src_fp_mi}", "PhaseB+") + sys.exit(1) + + inc_compiled += 1 + # Show the compiled includes file + VLogger.info(f"[includes {inc_compiled}] compilation complete: {src_fp_mi}", "PhaseB+") + + # Add to obj_paths (sliced path, taken from includes_binary_dir) + ibd_len_mi2: sizet = string.strlen(includes_binary_dir) + inc_obj_sliced: str = StubMergerCollect._sliced_path(includes_binary_dir, ibd_len_mi2, inc_sha1_mi, "obj") + + if inc_obj_sliced is not None: + inc_obj_len: sizet = string.strlen(inc_obj_sliced) + if obj_pos + inc_obj_len + 2 < OBJ_PATHS_SIZE: + if obj_pos > 0: + obj_paths[obj_pos] = ' ' + obj_pos += 1 + string.strcpy(obj_paths + obj_pos, inc_obj_sliced) + obj_pos += inc_obj_len + obj_paths[obj_pos] = '\0' + else: VLogger.error("includes .obj path buffer overflow, aborting compilation", "PhaseB+") + stdlib.free(inc_obj_sliced) + + # Free this iteration's temporary memory + if check_pat_mi is not None: stdlib.free(check_pat_mi) + + if check_fd_mi is not None: stdlib.free(check_fd_mi) + + stdlib.free(src_fp_mi) + + # Free the topological indices and local map copies + if topo_idx is not None: stdlib.free(topo_idx) + if store_sha1_arr_mi is not None: stdlib.free(store_sha1_arr_mi) + if store_mod_arr_mi is not None: stdlib.free(store_mod_arr_mi) + + VLogger.info(f"compiled missing includes: {inc_compiled} entries", "PhaseB+") + + # Free app_deps_buf (fallback when the Phase B+ block did not run) + if app_deps_buf is not None: + stdlib.free(app_deps_buf) + app_deps_buf = None + + # === 4. Phase C: link all .obj -> .exe === + log.banner("Phase C: link") + + od_len2: sizet = string.strlen(output_dir) + lo_len: sizet = string.strlen(linker_output) + exe_path: bytes = stdlib.malloc(od_len2 + lo_len + 2) + if exe_path is not None: viperlib.snprintf(exe_path, od_len2 + lo_len + 2, "%s/%s", output_dir, linker_output) + else: + exe_path = linker_output + VLogger.error("exe_path allocation failed, aborting compilation", "PhaseC") + + # Build the final .obj path list: main_obj_path first, reachable .objs after + final_obj_paths: bytes = stdlib.malloc(OBJ_PATHS_SIZE + 512) + if final_obj_paths is None: + stdlib.free(entries) + return 1 + final_obj_paths[0] = '\0' + fop_pos: sizet = 0 + # main_obj_path goes first (so the linker prefers the user main) + if main_obj_path[0] != '\0': + mlen: sizet = string.strlen(main_obj_path) + string.strcpy(final_obj_paths, main_obj_path) + fop_pos = mlen + + # Merge the App non-main modules' .obj (obj_paths, space-separated) compiled by Phase B/B+. + # final_obj_paths must include main_obj + reachable includes + App deps, else linking hits undefined symbols. + if obj_pos > 0: + if fop_pos + obj_pos + 1 < OBJ_PATHS_SIZE + 512: + if fop_pos > 0: + final_obj_paths[fop_pos] = ' ' + fop_pos += 1 + string.strcpy(final_obj_paths + fop_pos, obj_paths) + fop_pos += obj_pos + final_obj_paths[fop_pos] = '\0' + + # Link by graph: iterate reachable_set and collect only reachable includes' .obj, + # no longer collecting every .obj under includes.binary. + if reachable_set is not None and reachable_count > 0: + ibd_len_pc: sizet = string.strlen(includes_binary_dir) + td_len_pc: sizet = string.strlen(temp_dir) + ri_pc: int + for ri_pc in range(int(reachable_count)): + rsha1_pc: str = reachable_set + sizet(ri_pc) * 17 + if rsha1_pc[0] == '\0': continue + # Skip App sources: their .obj is already in final_obj_paths; collecting a same-sha1 .obj from includes_binary would duplicate symbols. + app_skip_pc: int = 0 + for ai_pc in range(int(file_count)): + ea_pc: uint64 = uint64(entries) + ai_pc * entry_size + ent_pc: SrcFileEntry* = (SrcFileEntry*)(ea_pc) + if ent_pc is not None and ent_pc.Sha1 is not None: + if string.strcmp(ent_pc.Sha1, rsha1_pc) == 0: + app_skip_pc = 1 + break + if app_skip_pc != 0: continue + obj_path_pc: str = StubMergerCollect._sliced_path(includes_binary_dir, ibd_len_pc, rsha1_pc, "obj") + if obj_path_pc is not None: + # Check whether the .obj file exists + cf_pc: fileio.File* = fileio.File(obj_path_pc, fileio.MODE.R) + if not cf_pc.closed: + cf_pc.close() + # The .obj exists; append to final_obj_paths + op_len_pc: sizet = string.strlen(obj_path_pc) + if fop_pos + op_len_pc + 2 < OBJ_PATHS_SIZE + 512: + if fop_pos > 0: + final_obj_paths[fop_pos] = ' ' + fop_pos += 1 + string.strcpy(final_obj_paths + fop_pos, obj_path_pc) + fop_pos += op_len_pc + final_obj_paths[fop_pos] = '\0' + else: + # .obj missing: a declaration-only module (no define lines) needs no .obj, so skip silently; + # under phase2command mode the missing .obj is expected and the path is still added. + if BuildPipeline._phase2_cmd_path is not None: + op_len_p2: sizet = string.strlen(obj_path_pc) + if fop_pos + op_len_p2 + 2 < OBJ_PATHS_SIZE + 512: + if fop_pos > 0: + final_obj_paths[fop_pos] = ' ' + fop_pos += 1 + string.strcpy(final_obj_paths + fop_pos, obj_path_pc) + fop_pos += op_len_p2 + final_obj_paths[fop_pos] = '\0' + elif StubMergerIR.IsDeclOnlyModule(temp_dir, td_len_pc, rsha1_pc) != 1: VLogger.error(f"reachable module .obj missing: {obj_path_pc}", "PhaseC") + stdlib.free(obj_path_pc) + + final_obj_paths[fop_pos] = '\0' + obj_paths_len: sizet = fop_pos + + # Free reachable_set (Phase C is done with it) + if reachable_set is not None: + stdlib.free(reachable_set) + reachable_set = None + + # Link by graph: pass None for includes_binary_dir to avoid collecting everything + lret: int = BuildPipeline.link_objs_to_exe(final_obj_paths, obj_paths_len, linker_cmd, linker_flags, exe_path, None, linker_static_libs, linker_dynamic_libs) + + if lret == 0: + VLogger.success(f"output: {exe_path}", "PhaseC") + # Deploy declared runtime deps (project.vpj runtime.*) next to the exe; fail-fast on error. + # Skipped under --phase2command (the exe is not linked yet). + if BuildPipeline._phase2_cmd_path is None: + if BuildPipeline.deploy_runtime_deps(exe_path) != 0: + VLogger.error("runtime dependency deployment failed, aborting compilation", "runtime") + stdlib.free(entries) + stdlib.free(asm_entries) + return 1 + # --traceback: export .stackmap (skipped under --phase2command, + # where nothing was actually linked). + if BuildPipeline._traceback_mode != 0 and BuildPipeline._phase2_cmd_path is None: BuildPipeline.WriteStackMap(mb, exe_path) + if args.get_bool("run") and BuildPipeline._phase2_cmd_path is not None: + # phase2command mode: the exe is not linked yet (the command is in the bat); skip --run + VLogger.info("phase2command mode: skipping --run (exe not linked yet)", "phase2cmd") + elif args.get_bool("run"): + VLogger.info(f"run: {exe_path}", "run") + rp: subprocess.CompletedProcess* = subprocess.run(exe_path, False, False) + if rp is not None: VLogger.info(f"exit code: {rp.returncode}", "run") + else: + VLogger.error("link failed", "PhaseC") + stdlib.free(entries) + stdlib.free(asm_entries) + return 1 + + stdlib.free(entries) + stdlib.free(asm_entries) + return 0 diff --git a/App/lib/Projectrans/Config.vp b/App/lib/Projectrans/Config.vp index ddb47a4..c8b3062 100644 --- a/App/lib/Projectrans/Config.vp +++ b/App/lib/Projectrans/Config.vp @@ -1,601 +1,601 @@ -import t, c -from stdint import * -import memhub -import string -import stdio -import stdlib -import viperlib -import lib.core.VLogger as VLogger -from json.__parser import parse as json_parse -from json import JsonValue, JSON_OBJECT, JSON_INT -import w32.fileio as fileio - - -# Config - project configuration - -# Global mbuddy pointer -_mbuddy: memhub.MemManager* - -# Config values (loaded from project.vpj); default None/0: callers must guard with `is not None` (else garbage pointer crash) -SourceDir: str = None -BuildDir: str = None -TempDir: str = None # = {BuildDir}/temp (auto-computed) -OutputDir: str = None # = {BuildDir}/output (auto-computed) -ProjectName: str = None -ProjectVersion: str = None -CompilerCmd: str = None -CompilerFlags: str = None # compiler flags (space-separated, joined from compiler.flags array) -LinkerCmd: str = None -LinkerFlags: str = None # linker flags (space-separated, joined from linker.flags array) -LinkerOutput: str = None -LinkerStaticLibs: str = None # explicit static libs (space-separated abs paths, from linker.static_libs array) -LinkerDynamicLibs: str = None # explicit dynamic libs (space-separated abs paths, from linker.dynamic_libs array, .dll/.so) -TargetTriple: str = None -TargetDataLayout: str = None -ProjectDir: str = None # directory containing project.vpj (set by resolve_paths) -Sha1SliceLevel: int = 0 # SHA1 slice level (directory dispersion depth) -StrictMode: int = 0 -IncludesDir: str = None # includes dir path (first element of includes array, single-dir compat) -IncludesDirs: str = None # all includes dirs (space-separated, multi-dir support) -Qt6Path: str = None # runtime.qt6_path: Qt6 install root (optional) -Qt6Dlls: str = None # runtime.qt6_dlls joined (space-separated DLL names; optional) -Qt6Plugins: str = None # runtime.qt6_plugins joined (space-separated paths relative to /plugins; optional) - -# includes prebuilt cache dir suffix (asan mode uses .binary_asan to isolate instrumented obj) -_inc_bin_suffix: str = ".binary" - -# Emission backend: single-track MLIR (since 2026-10-03). -# The tree now uses includes/mlirlite; BuildPipeline always runs mlir-opt -> mlir-translate to lower back to LLVM IR. - - -def Load_project_config(path: str) -> int: - """Load the project config file (project.vpj); 0=success, non-zero=failure.""" - global SourceDir, BuildDir, TempDir, OutputDir, ProjectName, ProjectVersion - global CompilerCmd, CompilerFlags, LinkerCmd, LinkerFlags, LinkerOutput, LinkerStaticLibs, LinkerDynamicLibs - global TargetTriple, TargetDataLayout, Sha1SliceLevel, StrictMode - global IncludesDir, IncludesDirs - global Qt6Path, Qt6Dlls, Qt6Plugins - - if path is None: return 1 - if _mbuddy is None: return 1 - - # Open file - f: fileio.File* = fileio.File(path, fileio.MODE.R) - if f is None: return 1 - if f.closed: return 1 - - # Allocate read buffer - CFG_BUF_SIZE: sizet = 8192 - buf: bytes = _mbuddy.alloc(CFG_BUF_SIZE) - if buf is None: - f.close() - return 1 - - bytes_read: int64 = f.read_all(buf, CFG_BUF_SIZE) - f.close() - - if bytes_read <= 0: return 1 - - # Parse JSON - root: JsonValue* = json_parse(_mbuddy, buf) - if root is None: return 1 - if not root.is_object(): return 1 - - # Read top-level fields (explicit __getitem__ calls for old compiler binaries) - sd_val: JsonValue* = root["source_dir"] - SourceDir = sd_val.as_string() if sd_val is not None else None - # build_dir: unified build dir (default ./.tpv_build); temp and output are subdirs - bd_val: JsonValue* = root["build_dir"] - if bd_val is not None and bd_val.is_string(): BuildDir = bd_val.as_string() - else: BuildDir = "./.tpv_build" - name_val: JsonValue* = root["name"] - ProjectName = name_val.as_string() if name_val is not None else None - ver_val: JsonValue* = root["version"] - ProjectVersion = ver_val.as_string() if ver_val is not None else None - - # Read compiler sub-object - compiler: JsonValue* = root["compiler"] - if compiler is not None and compiler.is_object(): - cc_cmd_val: JsonValue* = compiler["cmd"] - CompilerCmd = cc_cmd_val.as_string() if cc_cmd_val is not None else None - # Read compiler.flags array, join into a space-separated string - cc_flags_val: JsonValue* = compiler["flags"] - CompilerFlags = _join_flags_array(cc_flags_val) - - # Read linker sub-object - linker: JsonValue* = root["linker"] - if linker is not None and linker.is_object(): - ld_cmd_val: JsonValue* = linker["cmd"] - LinkerCmd = ld_cmd_val.as_string() if ld_cmd_val is not None else None - ld_out_val: JsonValue* = linker["output"] - LinkerOutput = ld_out_val.as_string() if ld_out_val is not None else None - # Read linker.flags array, join into a space-separated string - ld_flags_val: JsonValue* = linker["flags"] - LinkerFlags = _join_flags_array(ld_flags_val) - # Read linker.static_libs array: explicit static libs (.a/.obj); joined raw here, - # then resolved to abs paths against project_dir in resolve_paths. - sl_val: JsonValue* = linker["static_libs"] - LinkerStaticLibs = _join_flags_array(sl_val) - # Read linker.dynamic_libs array: explicit dynamic libs (.dll/.so); joined raw here, - # then resolved to abs paths against project_dir in resolve_paths. - dl_val: JsonValue* = linker["dynamic_libs"] - LinkerDynamicLibs = _join_flags_array(dl_val) - - # Read target sub-object - target: JsonValue* = root["target"] - if target is not None and target.is_object(): - tgt_triple_val: JsonValue* = target["triple"] - TargetTriple = tgt_triple_val.as_string() if tgt_triple_val is not None else None - tgt_dl_val: JsonValue* = target["datalayout"] - TargetDataLayout = tgt_dl_val.as_string() if tgt_dl_val is not None else None - - # Read options sub-object - # Sha1SliceLevel: SHA1 slice level (default 1; each level uses 2 SHA1 chars: 0=none, 1=b7/, 2=b7/90/) - options: JsonValue* = root["options"] - Sha1SliceLevel = 1 - StrictMode = 1 - if options is not None and options.is_object(): - sl: JsonValue* = options["sha1_slice_level"] - if sl is not None and sl.is_int(): Sha1SliceLevel = sl.as_int() - sm: JsonValue* = options["strict_mode"] - if sm is not None and sm.is_bool(): StrictMode = 1 if sm.as_bool() else 0 - - # Read includes array (multi-dir: first element -> IncludesDir, all joined -> IncludesDirs) - includes_val: JsonValue* = root["includes"] - if includes_val is not None and includes_val.is_array(): - arr_len: sizet = len(includes_val) - inc_buf: bytes = stdlib.malloc(2048) - if inc_buf is not None: - inc_buf[0] = '\0' - inc_pos: sizet = 0 - for inc_i in range(arr_len): - inc_item: JsonValue* = includes_val.get_item(inc_i) - if inc_item is None or not inc_item.is_string(): continue - inc_str: str = inc_item.as_string() - inc_slen: sizet = string.strlen(inc_str) - if inc_i == 0: IncludesDir = inc_str - # Append into IncludesDirs (space-separated) - if inc_pos + inc_slen + 1 < 2048: - if inc_pos > 0: - inc_buf[inc_pos] = ' ' - inc_pos += 1 - string.strcpy(inc_buf + inc_pos, inc_str) - inc_pos += inc_slen - inc_buf[inc_pos] = '\0' - IncludesDirs = inc_buf - if IncludesDir is None: IncludesDir = inc_buf - - # Read runtime sub-object (optional): declared runtime deps deployed next to the exe. - # qt6_dlls / qt6_plugins join via _join_flags_array (space-separated, same口径 as linker.static_libs). - runtime: JsonValue* = root["runtime"] - if runtime is not None and runtime.is_object(): - qp_val: JsonValue* = runtime["qt6_path"] - Qt6Path = qp_val.as_string() if qp_val is not None else None - qd_val: JsonValue* = runtime["qt6_dlls"] - Qt6Dlls = _join_flags_array(qd_val) - qpl_val: JsonValue* = runtime["qt6_plugins"] - Qt6Plugins = _join_flags_array(qpl_val) - - return 0 - - -def _join_flags_array(flags_val: JsonValue*) -> str: - """Join a JSON string array into a space-separated string; None if empty/invalid.""" - if flags_val is None: return None - if not flags_val.is_array(): return None - - arr_len: sizet = len(flags_val) - if arr_len == 0: return None - - # First pass: compute total length (all flags + space separators) - total_len: sizet = 0 - for i in range(arr_len): - item: JsonValue* = flags_val.get_item(i) - if item is not None and item.is_string(): - s: str = item.as_string() - if s is not None: - total_len += string.strlen(s) + 1 # +1 for space - - if total_len == 0: return None - - # Allocate buffer (total_len includes a trailing space per flag; last one becomes '\0') - buf: str = _mbuddy.alloc(total_len + 1) - if buf is None: return None - - # Second pass: join (flags separated by spaces) - pos: sizet = 0 - for i in range(arr_len): - item = flags_val.get_item(i) - if item is not None and item.is_string(): - s = item.as_string() - if s is not None: - slen: sizet = string.strlen(s) - if pos > 0: - buf[pos] = ' ' - pos += 1 - string.strcpy(buf + pos, s) - pos += slen - buf[pos] = '\0' - return buf - - -def _join_path(rel: str, project_dir: str) -> str: - """Join a relative path with project_dir, expanding env vars and normalizing it.""" - if rel is None: return None - # Env var expansion: only when '%' present (fileio.ExpandEnvironmentPath returns None on - # failure/no '%'); on failure keep the original string unchanged. - ep: str = fileio.ExpandEnvironmentPath(rel) - if ep is not None: rel = ep - # Absolute paths (starting with / or \) stay unchanged - if rel[0] == '/' or rel[0] == '\\': return rel - # Skip joining when rel already has the project_dir prefix (avoids double paths); boundary at separator or end - if project_dir is not None: - pdl: sizet = string.strlen(project_dir) - if pdl > 0 and string.strncmp(rel, project_dir, pdl) == 0: - if rel[pdl] == '/' or rel[pdl] == '\\' or rel[pdl] == '\0': return rel - - rel_len: sizet = string.strlen(rel) - dir_len: sizet = string.strlen(project_dir) - - # Handle ../ or ..\ prefix: join project_dir/../... - if rel_len >= 3 and rel[0] == '.' and rel[1] == '.': - if rel[2] == '/' or rel[2] == '\\': - if dir_len == 0: return rel - total_len: sizet = dir_len + 1 + rel_len + 1 - buf: str = _mbuddy.alloc(total_len) - if buf is None: return rel - viperlib.snprintf(buf, total_len, "%s/%s", project_dir, rel) - return _normalize_path(buf) - - # Handle ./ or .\ prefix: join project_dir + rel[2:] - rel_off: sizet = 0 - if rel_len >= 2 and rel[0] == '.': - if rel[1] == '/' or rel[1] == '\\': rel_off = 2 - else: - return rel # not starting with ./: keep unchanged - else: - return rel # not starting with .: keep unchanged - - if dir_len == 0: - # project_dir empty (current dir): just strip the ./ prefix - return rel + rel_off - # Join: project_dir + "/" + rel[rel_off:] - total_len = dir_len + 1 + (rel_len - rel_off) + 1 - buf = _mbuddy.alloc(total_len) - if buf is None: - return rel # allocation failed: return original path - viperlib.snprintf(buf, total_len, "%s/%s", project_dir, rel + rel_off) - return _normalize_path(buf) - - -def _normalize_path(path: str) -> str: - """Normalize a path by resolving . and .. segments; absolute paths pass through.""" - if path is None: return None - plen: sizet = string.strlen(path) - if plen == 0: return path - # Absolute paths (/ or \ start, or drive letter like C:) are not normalized - if path[0] == '/' or path[0] == '\\': return path - if plen >= 2 and path[1] == ':': return path - # Output buffer (worst case = input length + possible ./ prefix) - out: str = _mbuddy.alloc(plen + 3) - if out is None: return path - seg_start: sizet = 0 - out_len: sizet = 0 - has_seg: int = 0 - for i in range(plen + 1): - ch: char = '\0' - if i < plen: ch = path[i] - if ch == '/' or ch == '\\' or ch == '\0': - seg_len: sizet = i - seg_start - if seg_len > 0: - # Single "." segment: skip - if seg_len == 1 and path[seg_start] == '.': pass - # Single ".." segment: pop previous; leading ".." with nothing to pop must be kept (like Python normpath) - elif seg_len == 2 and path[seg_start] == '.' and path[seg_start + 1] == '.': - # Whether the last segment is already a leading ".." (standalone: at start or preceded by a separator) - last_is_dotdot: int = 0 - if out_len >= 2 and out[out_len - 1] == '.' and out[out_len - 2] == '.': - if out_len == 2: last_is_dotdot = 1 - elif out[out_len - 3] == '/' or out[out_len - 3] == '\\': last_is_dotdot = 1 - if out_len > 0 and last_is_dotdot == 0: - # Back up to the previous separator - k: sizet = out_len - while k > 0 and out[k - 1] != '/' and out[k - 1] != '\\': k -= 1 - if k > 0: out_len = k - 1 - else: out_len = 0 - # Keep separator logic when segments remain - if out_len > 0: has_seg = 1 - else: has_seg = 0 - else: - # No normal segment to pop: append ".." as a normal segment (keep leading ..) - if has_seg > 0: - out[out_len] = '/' - out_len += 1 - out[out_len] = '.' - out[out_len + 1] = '.' - out_len += 2 - has_seg = 1 - else: - # Normal segment: append separator + segment - if has_seg > 0: - out[out_len] = '/' - out_len += 1 - for jj in range(seg_len): - out[out_len] = path[seg_start + jj] - out_len += 1 - has_seg = 1 - seg_start = i + 1 - if has_seg == 0: - # All segments consumed (e.g. ./..): return current dir - return "./" - # Re-add ./ prefix for relative paths (matches original style; except those starting with ..) - if out[0] != '.' and out[0] != '/': - # Shift right by 2 to make room for "./" (copy back-to-front to avoid overwriting) - mv: sizet = out_len - while mv > 0: - mv -= 1 - out[mv + 2] = out[mv] - out[0] = '.' - out[1] = '/' - out_len += 2 - out[out_len] = '\0' - return out - - -def resolve_paths(project_dir: str) -> int: - """Resolve and normalize project paths (src/build/includes); 0=success.""" - global SourceDir, BuildDir, TempDir, OutputDir, IncludesDir, IncludesDirs - global ProjectDir, LinkerStaticLibs, LinkerDynamicLibs - if project_dir is None: return 1 - if _mbuddy is None: return 1 - - ProjectDir = project_dir - - SourceDir = _join_path(SourceDir, project_dir) - BuildDir = _join_path(BuildDir, project_dir) - - # TempDir = {BuildDir}/temp; OutputDir = {BuildDir}/output - if BuildDir is not None: - bd_len: sizet = string.strlen(BuildDir) - temp_buf: str = _mbuddy.alloc(bd_len + 8) - if temp_buf is not None: - viperlib.snprintf(temp_buf, bd_len + 8, "%s/temp", BuildDir) - TempDir = temp_buf - # OutputDir = {BuildDir}/output - output_buf: str = _mbuddy.alloc(bd_len + 8) - if output_buf is not None: - viperlib.snprintf(output_buf, bd_len + 8, "%s/output", BuildDir) - OutputDir = output_buf - - # includes paths may be of the form ../includes, resolved against project_dir. - # Normalize IncludesDir (first element) and IncludesDirs (all, space-separated) item by item. - if IncludesDirs is not None: - id_total: sizet = string.strlen(IncludesDirs) - id_out: str = _mbuddy.alloc(id_total * 3 + 64) - if id_out is not None: - id_out[0] = '\0' - id_pos: sizet = 0 - id_i: sizet = 0 - while id_i < id_total: - while id_i < id_total and IncludesDirs[id_i] == ' ': id_i += 1 - if id_i >= id_total: break - id_start: sizet = id_i - while id_i < id_total and IncludesDirs[id_i] != ' ': id_i += 1 - seg_len: sizet = id_i - id_start - if seg_len > 0: - seg_buf: str = _mbuddy.alloc(seg_len + 1) - if seg_buf is not None: - string.strncpy(seg_buf, IncludesDirs + id_start, seg_len) - seg_buf[seg_len] = '\0' - joined: str = _join_path(seg_buf, project_dir) - if joined is not None: - jl: sizet = string.strlen(joined) - if id_pos > 0: - id_out[id_pos] = ' ' - id_pos += 1 - string.strcpy(id_out + id_pos, joined) - id_pos += jl - id_out[id_pos] = '\0' - IncludesDirs = id_out - if IncludesDir is not None: IncludesDir = _join_path(IncludesDir, project_dir) - # linker.static_libs explicit paths resolved segment-by-segment to abs paths against project_dir - if LinkerStaticLibs is not None: - sl_total: sizet = string.strlen(LinkerStaticLibs) - sl_out: str = _mbuddy.alloc(sl_total * 3 + 64) - if sl_out is not None: - sl_out[0] = '\0' - sl_pos: sizet = 0 - sl_i: sizet = 0 - while sl_i < sl_total: - while sl_i < sl_total and LinkerStaticLibs[sl_i] == ' ': sl_i += 1 - if sl_i >= sl_total: break - sl_start: sizet = sl_i - while sl_i < sl_total and LinkerStaticLibs[sl_i] != ' ': sl_i += 1 - seg_len: sizet = sl_i - sl_start - if seg_len > 0: - seg_buf: str = _mbuddy.alloc(seg_len + 1) - if seg_buf is not None: - string.strncpy(seg_buf, LinkerStaticLibs + sl_start, seg_len) - seg_buf[seg_len] = '\0' - joined: str = _join_path(seg_buf, project_dir) - if joined is not None: - jl: sizet = string.strlen(joined) - if sl_pos > 0: - sl_out[sl_pos] = ' ' - sl_pos += 1 - string.strcpy(sl_out + sl_pos, joined) - sl_pos += jl - sl_out[sl_pos] = '\0' - LinkerStaticLibs = sl_out - # linker.dynamic_libs explicit files (.dll/.so) likewise resolved to abs paths against project_dir. - if LinkerDynamicLibs is not None: - dl_total: sizet = string.strlen(LinkerDynamicLibs) - dl_out: str = _mbuddy.alloc(dl_total * 3 + 64) - if dl_out is not None: - dl_out[0] = '\0' - dl_pos: sizet = 0 - dl_i: sizet = 0 - while dl_i < dl_total: - while dl_i < dl_total and LinkerDynamicLibs[dl_i] == ' ': dl_i += 1 - if dl_i >= dl_total: break - dl_start: sizet = dl_i - while dl_i < dl_total and LinkerDynamicLibs[dl_i] != ' ': dl_i += 1 - seg_len: sizet = dl_i - dl_start - if seg_len > 0: - seg_buf: str = _mbuddy.alloc(seg_len + 1) - if seg_buf is not None: - string.strncpy(seg_buf, LinkerDynamicLibs + dl_start, seg_len) - seg_buf[seg_len] = '\0' - joined: str = _join_path(seg_buf, project_dir) - if joined is not None: - jl: sizet = string.strlen(joined) - if dl_pos > 0: - dl_out[dl_pos] = ' ' - dl_pos += 1 - string.strcpy(dl_out + dl_pos, joined) - dl_pos += jl - dl_out[dl_pos] = '\0' - LinkerDynamicLibs = dl_out - # -T