1051 lines
53 KiB
Plaintext
1051 lines
53 KiB
Plaintext
from stdint import *
|
||
import string
|
||
import stdlib
|
||
import memhub
|
||
import viperlib
|
||
import win32.fileio as fileio
|
||
import win32.win32base
|
||
import ast
|
||
import mlirlite
|
||
import lib.core.VLogger as VLogger
|
||
import lib.core.Gen.Translator as HandlesTranslator
|
||
import lib.core.IncludesScanner as IncludesScanner
|
||
import lib.Stub.StubMerger as StubMerger
|
||
import lib.core.Gen.Generator as GenGenerator
|
||
import lib.core.CompilerFactory as CompilerFactory
|
||
import lib.Phase.BuildPipeline as BuildPipeline
|
||
import lib.Projectrans.Config as Config
|
||
import lib.Stub.StubGen.Converter as StubConverter
|
||
import lib.Stub.StubMerger.StubMergerBuild as StubMergerBuild
|
||
import lib.Stub.StubMerger.StubMergerCollect as StubMergerCollect
|
||
import lib.Stub.StubMerger.StubMergerIR as StubMergerIR
|
||
import lib.Stub.StubMerger.StubMergerStore as StubMergerStore
|
||
|
||
import lib.Phase.Phase1.Phase1Scan as Phase1Scan
|
||
import lib.Phase.Phase1.Phase1Topo as Phase1Topo
|
||
import lib.Phase.Phase1 as Phase1
|
||
|
||
import lib.core.Gen.GenApi as GenApi
|
||
import lib.core.Gen.DebugInfoGen as DebugInfoGen
|
||
|
||
|
||
# _InjectedCreatesCycle - cycle check before auto-injecting a container module
|
||
# (_list/_dict); a hit (e.g. memhub<->_list) skips injection to avoid a Phase A deadlock.
|
||
def _InjectedCreatesCycle(entry: IncludesScanner.FileEntry*, mod_name: str) -> int:
|
||
"""Return 1 if injecting mod_name into the current module would create a cycle"""
|
||
if entry is None or entry.RelPath is None or mod_name is None: return 0
|
||
inj_sha1: str = StubMergerStore.LookupSha1ByModName(mod_name)
|
||
if inj_sha1 is None: return 0
|
||
inj_deps: str = StubMergerStore.LookupIncludesDeps(inj_sha1)
|
||
if inj_deps is None: return 0
|
||
rp: str = entry.RelPath
|
||
# Current module name = basename of RelPath (after the last '/' or '\'), without extension
|
||
base: str = string.strrchr(rp, '/')
|
||
if base is None: base = string.strrchr(rp, '\\')
|
||
if base is not None: base = base + 1
|
||
else: base = rp
|
||
dot: str = string.strchr(base, '.')
|
||
nm_buf: bytes = stdlib.malloc(128)
|
||
if nm_buf is None: return 0
|
||
if dot is not None:
|
||
nm_len: sizet = sizet(uint64(dot) - uint64(base))
|
||
if nm_len > 127: nm_len = 127
|
||
string.strncpy(nm_buf, base, nm_len)
|
||
nm_buf[nm_len] = '\0'
|
||
else:
|
||
string.strncpy(nm_buf, base, 127)
|
||
nm_buf[127] = '\0'
|
||
nm_len2: sizet = string.strlen(nm_buf)
|
||
d_len: sizet = string.strlen(inj_deps)
|
||
dp: sizet = 0
|
||
while dp < d_len:
|
||
ts: sizet = dp
|
||
while dp < d_len and inj_deps[dp] != ' ': dp += 1
|
||
tl: sizet = dp - ts
|
||
if dp < d_len: dp += 1
|
||
if tl > 0 and tl == nm_len2 and string.strncmp(inj_deps + ts, nm_buf, tl) == 0:
|
||
stdlib.free(nm_buf)
|
||
return 1
|
||
stdlib.free(nm_buf)
|
||
return 0
|
||
|
||
|
||
def _DepsContainsName(deps: str, name: str) -> int:
|
||
"""Return 1 if the space-separated deps string contains the name token exactly"""
|
||
if deps is None or name is None: return 0
|
||
nl: sizet = string.strlen(name)
|
||
dl: sizet = string.strlen(deps)
|
||
dp: sizet = 0
|
||
while dp < dl:
|
||
ts: sizet = dp
|
||
while dp < dl and deps[dp] != ' ': dp += 1
|
||
tl: sizet = dp - ts
|
||
if dp < dl: dp += 1
|
||
if tl == nl and string.strncmp(deps + ts, name, nl) == 0: return 1
|
||
return 0
|
||
|
||
|
||
def _DepsRemoveName(deps: str, name: str) -> int:
|
||
"""Remove the name token from the deps string in place; returns 1 if changed"""
|
||
if deps is None or name is None: return 0
|
||
nl: sizet = string.strlen(name)
|
||
dl: sizet = string.strlen(deps)
|
||
buf: bytes = stdlib.malloc(StubMerger.MAX_DEPS_STR_LEN)
|
||
if buf is None: return 0
|
||
buf[0] = '\0'
|
||
bl: sizet = 0
|
||
dp: sizet = 0
|
||
while dp < dl:
|
||
ts: sizet = dp
|
||
while dp < dl and deps[dp] != ' ': dp += 1
|
||
tl: sizet = dp - ts
|
||
if dp < dl: dp += 1
|
||
if tl == 0: continue
|
||
if tl == nl and string.strncmp(deps + ts, name, nl) == 0: continue
|
||
if bl > 0:
|
||
buf[bl] = ' '
|
||
bl += 1
|
||
string.strncpy(buf + bl, deps + ts, tl)
|
||
bl += tl
|
||
buf[bl] = '\0'
|
||
string.strcpy(deps, buf)
|
||
stdlib.free(buf)
|
||
return 1
|
||
|
||
|
||
def _RewriteDepsTxt(temp_dir: str, sha1: str, deps_str: str) -> None:
|
||
"""Rewrite the on-disk deps.txt file to match the in-memory deps after breaking a cycle"""
|
||
if temp_dir is None or sha1 is None or deps_str is None: return
|
||
td_len: sizet = string.strlen(temp_dir)
|
||
dpath: str = StubMergerCollect._sliced_path(temp_dir, td_len, sha1, "deps.txt")
|
||
if dpath is None: return
|
||
df: fileio.File* = fileio.File(dpath, fileio.MODE.W)
|
||
if not df.closed:
|
||
dl: sizet = string.strlen(deps_str)
|
||
if dl > 0: df.write(deps_str, dl)
|
||
df.close()
|
||
stdlib.free(dpath)
|
||
|
||
|
||
def _BreakContainerCyclesInDeps(temp_dir: str) -> int:
|
||
"""Remove _list/_dict/testcheck injection cycle edges after scanning and sync deps.txt"""
|
||
if StubMerger._includes_deps_sha1 is None or StubMerger._includes_deps_str is None: return 0
|
||
cnt: sizet = StubMerger._includes_deps_count
|
||
ci: sizet
|
||
for ci in range(int(cnt)):
|
||
sha1_m: str = StubMerger._includes_deps_sha1 + sizet(ci) * 17
|
||
deps_m: str = StubMerger._includes_deps_str + sizet(ci) * StubMerger.MAX_DEPS_STR_LEN
|
||
if sha1_m is None or deps_m is None: continue
|
||
if string.strstr(deps_m, "_list") is None and string.strstr(deps_m, "_dict") is None and string.strstr(deps_m, "testcheck") is None: continue
|
||
# Module name (basename of RelPath without extension)
|
||
rel_m: str = StubMergerStore.LookupSha1RelPath(sha1_m)
|
||
if rel_m is None: continue
|
||
base_m: str = string.strrchr(rel_m, '/')
|
||
if base_m is None: base_m = string.strrchr(rel_m, '\\')
|
||
if base_m is not None: base_m = base_m + 1
|
||
else: base_m = rel_m
|
||
dot_m: str = string.strchr(base_m, '.')
|
||
if dot_m is not None:
|
||
nm_len_m: sizet = sizet(uint64(dot_m) - uint64(base_m))
|
||
nm_m: bytes = stdlib.malloc(nm_len_m + 1)
|
||
if nm_m is None: continue
|
||
string.strncpy(nm_m, base_m, nm_len_m)
|
||
nm_m[nm_len_m] = '\0'
|
||
else:
|
||
nm_m: bytes = stdlib.malloc(129)
|
||
if nm_m is None: continue
|
||
string.strncpy(nm_m, base_m, 128)
|
||
nm_m[128] = '\0'
|
||
# _list <-> M cycle
|
||
if string.strstr(deps_m, "_list") is not None:
|
||
lst_sha1: str = StubMergerStore.LookupSha1ByModName("_list")
|
||
if lst_sha1 is not None:
|
||
lst_deps: str = StubMergerStore.LookupIncludesDeps(lst_sha1)
|
||
if lst_deps is not None and _DepsContainsName(lst_deps, nm_m) != 0:
|
||
if _DepsRemoveName(deps_m, "_list") != 0:
|
||
# Rewrite deps.txt after breaking the cycle in memory (diagnostic fallback reads the file)
|
||
_RewriteDepsTxt(temp_dir, sha1_m, deps_m)
|
||
# _dict <-> M cycle
|
||
if string.strstr(deps_m, "_dict") is not None:
|
||
dct_sha1: str = StubMergerStore.LookupSha1ByModName("_dict")
|
||
if dct_sha1 is not None:
|
||
dct_deps: str = StubMergerStore.LookupIncludesDeps(dct_sha1)
|
||
if dct_deps is not None and _DepsContainsName(dct_deps, nm_m) != 0:
|
||
if _DepsRemoveName(deps_m, "_dict") != 0: _RewriteDepsTxt(temp_dir, sha1_m, deps_m)
|
||
# testcheck <-> M cycle (assert auto-import)
|
||
if string.strstr(deps_m, "testcheck") is not None:
|
||
tc_sha1: str = StubMergerStore.LookupSha1ByModName("testcheck")
|
||
if tc_sha1 is not None:
|
||
tc_deps: str = StubMergerStore.LookupIncludesDeps(tc_sha1)
|
||
if tc_deps is not None and _DepsContainsName(tc_deps, nm_m) != 0:
|
||
if _DepsRemoveName(deps_m, "testcheck") != 0: _RewriteDepsTxt(temp_dir, sha1_m, deps_m)
|
||
stdlib.free(nm_m)
|
||
return 0
|
||
|
||
|
||
# _BuildReachableResult - rebuild the reachable FileEntry list from the Sha1Store.
|
||
# Only incdir-nonempty (includes) entries enter result; App entries would cause duplicate symbols.
|
||
def _BuildReachableResult(mb: memhub.MemBuddy*, includes_dir: str) -> IncludesScanner.ScanResult*:
|
||
"""Build the reachable FileEntry list from the Sha1Store (reachable includes only)"""
|
||
if mb is None: return None
|
||
store_count: sizet = sizet(StubMerger._sha1_store_count)
|
||
if store_count <= 0 or StubMerger._sha1_store_arr is None: return None
|
||
result: IncludesScanner.ScanResult* = IncludesScanner.create_scan_result(mb)
|
||
if result is None: return None
|
||
entry_size: sizet = IncludesScanner.FileEntry.__sizeof__()
|
||
i: sizet
|
||
for i in range(int(store_count)):
|
||
sidx: sizet = sizet(i) * 17
|
||
if StubMerger._sha1_store_arr[sidx] == '\0': continue
|
||
# Only includes entries (non-empty incdir); App entries do not enter result
|
||
iidx: sizet = sizet(i) * StubMerger.MAX_INC_DIR_LEN
|
||
if StubMerger._sha1_store_incdir[iidx] == '\0': continue
|
||
eaddr: uint64 = uint64(result.Entries) + uint64(result.Count) * entry_size
|
||
entry: IncludesScanner.FileEntry* = IncludesScanner.FileEntry*(eaddr)
|
||
if entry is None: continue
|
||
# Deep-copy SHA1 (store slot is 17 bytes)
|
||
sha1_buf: bytes = stdlib.malloc(17)
|
||
if sha1_buf is None: continue
|
||
string.strcpy(sha1_buf, StubMerger._sha1_store_arr + sidx)
|
||
entry.Sha1 = sha1_buf
|
||
# Deep-copy RelPath / IncDir
|
||
ridx: sizet = sizet(i) * StubMerger.MAX_REL_PATH_LEN
|
||
rel_len: sizet = string.strlen(StubMerger._sha1_store_rel + ridx)
|
||
inc_len: sizet = string.strlen(StubMerger._sha1_store_incdir + iidx)
|
||
rel_buf: bytes = stdlib.malloc(rel_len + 1)
|
||
inc_buf: bytes = stdlib.malloc(inc_len + 1)
|
||
if rel_buf is None or inc_buf is None:
|
||
if rel_buf is not None: stdlib.free(rel_buf)
|
||
if inc_buf is not None: stdlib.free(inc_buf)
|
||
stdlib.free(sha1_buf)
|
||
continue
|
||
string.strcpy(rel_buf, StubMerger._sha1_store_rel + ridx)
|
||
string.strcpy(inc_buf, StubMerger._sha1_store_incdir + iidx)
|
||
entry.RelPath = rel_buf
|
||
entry.IncDir = inc_buf
|
||
# Absolute path = inc_dir + '/' + rel_path (matches _TryResolveModuleFile)
|
||
abs_buf: bytes = stdlib.malloc(inc_len + 1 + rel_len + 1)
|
||
if abs_buf is None:
|
||
stdlib.free(sha1_buf)
|
||
stdlib.free(rel_buf)
|
||
stdlib.free(inc_buf)
|
||
continue
|
||
string.strcpy(abs_buf, inc_buf)
|
||
abs_buf[inc_len] = '/'
|
||
string.strcpy(abs_buf + inc_len + 1, rel_buf)
|
||
entry.Path = abs_buf
|
||
# Module name (absolute module name saved when registering in the store)
|
||
midx: sizet = sizet(i) * 64
|
||
mod_len: sizet = string.strlen(StubMerger._sha1_store_mod + midx)
|
||
if mod_len > 0:
|
||
mod_buf: bytes = stdlib.malloc(mod_len + 1)
|
||
if mod_buf is not None:
|
||
string.strcpy(mod_buf, StubMerger._sha1_store_mod + midx)
|
||
entry.ModuleName = mod_buf
|
||
result.Count += 1
|
||
return result
|
||
|
||
# ============================================================
|
||
# Phase1Main - Phase1 main flow (scan -> pre-register -> 1a/1b translate -> generate stub)
|
||
|
||
# ============================================================
|
||
# RunPhase1 - scan the includes dir; translate and split stubs only when missing.
|
||
def RunPhase1(mb: memhub.MemBuddy*, includes_dir: str, temp_dir: str, log: VLogger.Logger*) -> int:
|
||
"""Phase1: scan the includes directory and translate/generate stubs on demand"""
|
||
if includes_dir is None or temp_dir is None:
|
||
VLogger.error("includes_dir or temp_dir is empty; cannot run Phase1", "Phase1")
|
||
return 1
|
||
|
||
# Set the global temp_dir (used for cross-module CDefine lookup)
|
||
GenApi.set_temp_dir(temp_dir)
|
||
# Also set HandlesExprCall's temp_dir (for the on-demand vpi return-annotation cache);
|
||
# must run before includes translation since Phase2's SetTempDir is too late.
|
||
GenApi.SetTempDir(temp_dir)
|
||
|
||
# Ensure the temp dir exists (build_dir/temp may not exist yet)
|
||
BuildPipeline.ensure_dir(temp_dir)
|
||
|
||
log.banner("Phase1: scanning includes (on-demand translation)")
|
||
|
||
# Init the AST tables once, before the dependency-graph BFS: if _init_tables first runs
|
||
# after the BFS file-level mark, reset_to frees the tables and later tokenize crashes.
|
||
ast._init_tables(mb)
|
||
|
||
# On-demand dependency scan (App-first): seed the queue from App imports, then BFS-resolve
|
||
# reachable includes only (read + SHA1 + import parse); unreferenced ones are never read.
|
||
t_scan0: uint64 = win32.win32base.GetTickCount64()
|
||
reachable_set: str = None
|
||
reachable_count: int = 0
|
||
use_reachable: int = 0
|
||
if Config.SourceDir is not None:
|
||
reachable_set = stdlib.malloc(StubMerger.MAX_INCLUDES_SHA1 * 17)
|
||
if reachable_set is not None:
|
||
string.memset(reachable_set, 0, StubMerger.MAX_INCLUDES_SHA1 * 17)
|
||
reachable_count = StubMergerBuild._BuildReachableSha1Set(mb, Config.SourceDir, temp_dir, reachable_set, None, includes_dir)
|
||
if reachable_count < 0:
|
||
VLogger.error("failed to build the reachable set (dependency-graph scan failed); aborting compilation", "Phase1")
|
||
return 1
|
||
if reachable_count <= 0:
|
||
VLogger.warning("App has no reachable includes dependencies; skipping Phase1", "Phase1")
|
||
stdlib.free(reachable_set)
|
||
reachable_set = None
|
||
return 0
|
||
use_reachable = 1
|
||
else:
|
||
VLogger.error("failed to allocate reachable_set; aborting compilation", "Phase1")
|
||
return 1
|
||
else:
|
||
VLogger.warning("SourceDir unavailable; cannot build the dependency graph; skipping Phase1", "Phase1")
|
||
return 0
|
||
t_scan1: uint64 = win32.win32base.GetTickCount64()
|
||
VLogger.info(f"dependency-graph scan total time: {t_scan1 - t_scan0} ms (reachable {reachable_count} includes)", "Phase1")
|
||
|
||
# Build the reachable FileEntry list from the Sha1Store; 1a/1b-pre/1b iterate result
|
||
# (non-empty incdir = includes). use_reachable filtering is idempotent.
|
||
result: IncludesScanner.ScanResult* = _BuildReachableResult(mb, includes_dir)
|
||
if result is None:
|
||
VLogger.error("failed to build the reachable FileEntry list; aborting compilation", "Phase1")
|
||
return 1
|
||
VLogger.info(f"reachable includes entries: {result.Count}", "Phase1")
|
||
|
||
# After scanning, clear _list/_dict injection cycle edges (e.g. _list<->memhub) so the
|
||
# graph has none and Phase A converges; deps.txt is rewritten in sync.
|
||
_BreakContainerCyclesInDeps(temp_dir)
|
||
|
||
# Write _sha1_map.txt (reachable includes only, human-readable; analysis uses the in-memory store)
|
||
StubMergerCollect.WriteIncludesSha1Map(mb, temp_dir, result, reachable_set, reachable_count)
|
||
|
||
# Legacy compat: sha1_set/set_count alias the reachable set (unused when use_reachable=1)
|
||
sha1_set: str = reachable_set
|
||
set_count: int = reachable_count
|
||
# Module SHA1 map is registered into StubMerger's global store by the dependency scan;
|
||
# HandlesExprCall._lookup_module_sha1 queries that store directly.
|
||
|
||
# FileEntry struct size (shared by Phase 1a and 1b)
|
||
entry_size: sizet = IncludesScanner.FileEntry.__sizeof__()
|
||
|
||
|
||
# ============================================================
|
||
# Phase 1a: register struct/enum/union for reachable includes (fixes alphabetical-order deps).
|
||
p1a_reg: int = 0
|
||
p1a_skp: int = 0
|
||
p1a_fl: int = 0
|
||
|
||
# ============================================================
|
||
# [temp diagnostic] Snapshot mb state if the early Phase1 flat buffer allocation fails.
|
||
if mb is not None:
|
||
# Run a cycle-detecting self_check first: a corrupted free list would loop forever in
|
||
# dump_stats, so skip dump_stats when self_check returns 4 (cycle).
|
||
p1_sc: int = mb.self_check()
|
||
if p1_sc != 4: mb.dump_stats("phase1-pre")
|
||
else: VLogger.info("[EVIDENCE] self_check=4 (free list cycle); skipping dump_stats to prevent an infinite loop", "Phase1")
|
||
p1_mo: int = mb.max_order
|
||
if p1_mo < 0 or p1_mo > 32:
|
||
p1_mo = 32 # Clamp: prevents this probe from out-of-bounds reading free_lists if max_order is corrupted
|
||
p1_fl_head: uint64 = uint64(mb.free_lists[p1_mo])
|
||
p1_mb_addr: uint64 = uint64(mb)
|
||
p1_base_addr: uint64 = uint64(mb.base)
|
||
p1_usable_addr: uint64 = uint64(mb.usable)
|
||
p1_chkbase_addr: uint64 = uint64(mb.chk_base)
|
||
VLogger.info(f"[EVIDENCE] mb={p1_mb_addr} base={p1_base_addr} usable={p1_usable_addr} " f"usable_size={mb.usable_size} max_order={mb.max_order} self_check={p1_sc} " f"detection={mb.detection}", "Phase1")
|
||
VLogger.info(f"[EVIDENCE] chk_base={p1_chkbase_addr} chk_depth={mb.chk_depth} " f"chk_cap={mb.chk_cap} free_lists[clamp={p1_mo}]={p1_fl_head}", "Phase1")
|
||
p1_sz_cn: sizet = sizet(Phase1.MAX_CLASSES) * Phase1.CLASS_NAME_LEN
|
||
p1_sz_ds: sizet = sizet(Phase1.MAX_CLASSES * Phase1.SHA1_LEN)
|
||
p1_sz_tp: sizet = sizet(4 * Phase1.MAX_FILE_DEPS)
|
||
p1_sz_bx: sizet = sizet(4)
|
||
p1_o_cn: int = int(mb._order_for_size(p1_sz_cn + 16))
|
||
p1_o_ds: int = int(mb._order_for_size(p1_sz_ds + 16))
|
||
p1_o_tp: int = int(mb._order_for_size(p1_sz_tp + 16))
|
||
p1_o_bx: int = int(mb._order_for_size(p1_sz_bx + 16))
|
||
p1_fl_cn: uint64 = uint64(mb.free_lists[p1_o_cn])
|
||
p1_fl_ds: uint64 = uint64(mb.free_lists[p1_o_ds])
|
||
VLogger.info(f"[EVIDENCE] order(class_names {p1_sz_cn}B+16)={p1_o_cn} fl={p1_fl_cn} " f"order(def_sha1s {p1_sz_ds}B+16)={p1_o_ds} fl={p1_fl_ds} " f"order(topo {p1_sz_tp}B+16)={p1_o_tp} order(box {p1_sz_bx}B+16)={p1_o_bx} " f"(fl==0 only means that order is empty and can still be split by a higher order; if alloc fails while dump_stats shows free>0 that is the definitive proof of metadata corruption; " f"any order>max_order={mb.max_order} means an abnormal size)", "Phase1")
|
||
|
||
# Pre-scan class inheritance and build the topo order (flat byte arrays avoid str-pointer NULL crashes)
|
||
# 取消整数提升后，模块属性 CDefine 按最小位宽物化为 i16/i8；512*64=32768 恰好越出 i16 有符号上限而回绕成负值，
|
||
# 首乘数显式 sizet() 加宽到 64 位。下面 def_sha1s_buf 的 512*17=8704 未溢出，无需改。
|
||
class_names_buf: bytes = mb.alloc(sizet(Phase1.MAX_CLASSES) * Phase1.CLASS_NAME_LEN)
|
||
parent_names_buf: bytes = mb.alloc(sizet(Phase1.MAX_CLASSES) * Phase1.CLASS_NAME_LEN)
|
||
def_sha1s_buf: bytes = mb.alloc(Phase1.MAX_CLASSES * Phase1.SHA1_LEN)
|
||
topo_order: rawptr = mb.alloc(4 * Phase1.MAX_FILE_DEPS)
|
||
topo_count_box: rawptr = mb.alloc(4)
|
||
generic_file_indices: rawptr = mb.alloc(4 * Phase1.MAX_FILE_DEPS)
|
||
generic_file_count_box: rawptr = mb.alloc(4)
|
||
if generic_file_indices is not None: string.memset(generic_file_indices, 0, 4 * Phase1.MAX_FILE_DEPS)
|
||
if generic_file_count_box is not None: string.memset(generic_file_count_box, 0, 4)
|
||
topo_count: int = 0
|
||
|
||
# These three flat buffers are Phase1Scan._ScanClassInheritance's contract pre-buffers; it
|
||
# fails fast on NULL, so report and abort per buffer on allocation failure (never fall back).
|
||
if class_names_buf is None: VLogger.error("class-inheritance pre-scan flat buffer allocation failed: class_names_buf is NULL; aborting compilation", "Phase1")
|
||
if parent_names_buf is None: VLogger.error("class-inheritance pre-scan flat buffer allocation failed: parent_names_buf is NULL; aborting compilation", "Phase1")
|
||
if def_sha1s_buf is None: VLogger.error("class-inheritance pre-scan flat buffer allocation failed: def_sha1s_buf is NULL; aborting compilation", "Phase1")
|
||
|
||
string.memset(class_names_buf, 0, sizet(Phase1.MAX_CLASSES) * Phase1.CLASS_NAME_LEN)
|
||
string.memset(parent_names_buf, 0, sizet(Phase1.MAX_CLASSES) * Phase1.CLASS_NAME_LEN)
|
||
string.memset(def_sha1s_buf, 0, Phase1.MAX_CLASSES * Phase1.SHA1_LEN)
|
||
if topo_order is not None: string.memset(topo_order, 0, 4 * Phase1.MAX_FILE_DEPS)
|
||
if topo_count_box is not None: string.memset(topo_count_box, 0, 4)
|
||
if generic_file_indices is not None: string.memset(generic_file_indices, 0, 4 * Phase1.MAX_FILE_DEPS)
|
||
if generic_file_count_box is not None: string.memset(generic_file_count_box, 0, 4)
|
||
|
||
class_count_scanned: int = Phase1Scan._ScanClassInheritance(mb, result, reachable_set, reachable_count, use_reachable, sha1_set, set_count, class_names_buf, parent_names_buf, def_sha1s_buf, generic_file_indices, generic_file_count_box)
|
||
|
||
if class_count_scanned > 0:
|
||
td_len_ts_call: sizet = string.strlen(temp_dir)
|
||
ret_topo: int = Phase1Topo._TopoSortFiles(mb, result, reachable_set, reachable_count, use_reachable, sha1_set, set_count, class_names_buf, parent_names_buf, def_sha1s_buf, class_count_scanned, topo_order, topo_count_box, temp_dir, td_len_ts_call)
|
||
if ret_topo == 0:
|
||
tc_ptr: int* = (int*)(topo_count_box)
|
||
if tc_ptr is not None: topo_count = tc_ptr[0]
|
||
VLogger.info(f"topological sort: {class_count_scanned} classes, {topo_count} files processed in dependency order", "Phase1")
|
||
else:
|
||
VLogger.error("topological sort failed (class-inheritance dependencies could not be resolved); aborting compilation", "Phase1")
|
||
return 1
|
||
else:
|
||
VLogger.warning("class-inheritance pre-scan produced no result; falling back to alphabetical order", "Phase1")
|
||
topo_count = 0
|
||
|
||
# ============================================================
|
||
# Phase 1a-pre-generic-all: pre-register all includes' generic templates before mark1.
|
||
full_scan_g: IncludesScanner.ScanResult* = IncludesScanner.scan_includes(mb, includes_dir)
|
||
if full_scan_g is not None: Phase1Scan._PreRegisterAllGenericTemplates(mb, full_scan_g)
|
||
else: Phase1Scan._PreRegisterAllGenericTemplates(mb, result)
|
||
|
||
# ============================================================
|
||
# Phase 1a: register struct/enum/union (before mark1, all includes) with OOM control.
|
||
GenApi.set_stdlib_alloc(1)
|
||
|
||
# Iterate files: topo order preferred, falling back to alphabetical
|
||
iter_count: int = int(topo_count if topo_count > 0 else result.Count)
|
||
# 1a total-time start (struct/enum/union registration)
|
||
t_a0: uint64 = win32.win32base.GetTickCount64()
|
||
for i in range(int(iter_count)):
|
||
# Per-file mark: free this file's AST after translating (registered data is in perm_pool)
|
||
p1a_mark: sizet = mb.mark()
|
||
# Get the file index: topo order or alphabetical
|
||
file_idx: int = int(i)
|
||
if topo_count > 0:
|
||
od_addr: uint64 = uint64(topo_order) + i * 4
|
||
od_ptr: int* = int*(od_addr)
|
||
if od_ptr is not None: file_idx = od_ptr[0]
|
||
|
||
entry_addr_r: uint64 = uint64(result.Entries) + file_idx * entry_size
|
||
entry_r: IncludesScanner.FileEntry* = IncludesScanner.FileEntry*(entry_addr_r)
|
||
if entry_r is None:
|
||
p1a_fl += 1
|
||
continue
|
||
|
||
sha1_r: str = entry_r.Sha1
|
||
if sha1_r is None:
|
||
p1a_fl += 1
|
||
continue
|
||
|
||
# On-demand filter (topo already filtered; the alphabetical fallback must check)
|
||
if topo_count == 0:
|
||
if use_reachable != 0:
|
||
if StubMergerCollect._is_in_sha1_set(sha1_r, reachable_set, reachable_count) == 0:
|
||
p1a_skp += 1
|
||
continue
|
||
else:
|
||
if StubMergerCollect._is_in_sha1_set(sha1_r, sha1_set, set_count) == 0:
|
||
p1a_skp += 1
|
||
continue
|
||
|
||
# Per-file timer start (read + parse, to locate 1a stalls)
|
||
t_fa0: uint64 = win32.win32base.GetTickCount64()
|
||
|
||
# Read file contents
|
||
file_path_r: str = entry_r.Path
|
||
f_r: fileio.File* = fileio.File(file_path_r, fileio.MODE.R)
|
||
if f_r.closed:
|
||
p1a_fl += 1
|
||
continue
|
||
|
||
# read_all's capacity includes the terminator (reads at most cap-1 bytes); longer files are truncated
|
||
if f_r.size() >= Phase1.SRC_BUF_SIZE:
|
||
VLogger.error(f"source file exceeds the {Phase1.SRC_BUF_SIZE}-byte limit, refusing to read: {file_path_r}", "Phase1")
|
||
f_r.close()
|
||
p1a_fl += 1
|
||
continue
|
||
|
||
src_buf_r: bytes = stdlib.malloc(Phase1.SRC_BUF_SIZE)
|
||
if src_buf_r is None:
|
||
f_r.close()
|
||
p1a_fl += 1
|
||
continue
|
||
|
||
bytes_read_r: LONG = f_r.read_all(src_buf_r, Phase1.SRC_BUF_SIZE)
|
||
f_r.close()
|
||
if bytes_read_r <= 0:
|
||
stdlib.free(src_buf_r)
|
||
p1a_fl += 1
|
||
continue
|
||
if bytes_read_r < Phase1.SRC_BUF_SIZE: src_buf_r[bytes_read_r] = 0
|
||
|
||
# Parse AST
|
||
lx_r: ast.Lexer* = ast.new_lexer(mb)
|
||
if lx_r is None:
|
||
stdlib.free(src_buf_r)
|
||
p1a_fl += 1
|
||
continue
|
||
ast._lexer_init(lx_r, src_buf_r, mb)
|
||
tokens_r: ast.Token* = ast.tokenize(lx_r)
|
||
tree_r: ast.AST* = ast.parse_tokens(mb, tokens_r)
|
||
if tree_r is None:
|
||
stdlib.free(src_buf_r)
|
||
p1a_fl += 1
|
||
continue
|
||
|
||
# Timer 1: read + parse done
|
||
t_fa1: uint64 = win32.win32base.GetTickCount64()
|
||
|
||
# Create Translator with _declare_only=1 (register struct/enum/union)
|
||
tr_r: HandlesTranslator.Translator* = HandlesTranslator.Translator()
|
||
if tr_r is None:
|
||
stdlib.free(src_buf_r)
|
||
p1a_fl += 1
|
||
continue
|
||
tr_r.ModuleSha1 = sha1_r
|
||
tr_r._declare_only = 1
|
||
tr_r.CurrentPackage = GenApi.compute_package_from_relpath(mb, entry_r.RelPath)
|
||
GenApi.set_current_file_type(file_path_r)
|
||
GenApi.set_current_file_imports(file_path_r)
|
||
GenApi.set_current_module_sha1(sha1_r)
|
||
GenApi.clear_cdefine_constants()
|
||
GenApi.clear_ctypedefs()
|
||
GenApi.reset_visible_structs(mb, 0)
|
||
ret_r: int = tr_r.translate(tree_r)
|
||
# Timer 2: translate(declare=1) returned
|
||
t_fa2: uint64 = win32.win32base.GetTickCount64()
|
||
if ret_r != 0: p1a_fl += 1
|
||
else: p1a_reg += 1
|
||
|
||
# Per-file progress log (parse/translate segments, to locate 1a stalls)
|
||
VLogger.info(f"[{i+1}/{iter_count}] 1a: {file_path_r} (parse={t_fa1-t_fa0} ms, translate={t_fa2-t_fa1} ms, total={t_fa2-t_fa0} ms)", "Phase1")
|
||
|
||
# Release the Translator's C malloc resources
|
||
if tr_r._global_names is not None: stdlib.free(tr_r._global_names)
|
||
if tr_r._nonlocal_names is not None: stdlib.free(tr_r._nonlocal_names)
|
||
stdlib.free(src_buf_r)
|
||
|
||
# [TEMP-DIAG] Per-file sentinel: check the tail canary of live blocks in [p1a_mark,
|
||
# chk_depth). Must run before reset_to; only under --detection-deep (mb.deep_detect != 0).
|
||
if mb.deep_detect != 0:
|
||
chk_rc_1a: int = mb.check_alive_blocks(p1a_mark)
|
||
if chk_rc_1a != 0:
|
||
VLogger.error(f"memory corruption (1a per-file post-check, rc={chk_rc_1a}): {file_path_r}", "Phase1")
|
||
stdlib.exit(1)
|
||
|
||
# Reclaim this file's transient AST/lexer/token/IR (registered data is in perm_pool, so
|
||
# reset does not affect it; IncludesScanner result predates p1a_mark and is kept too).
|
||
mb.reset_to(p1a_mark)
|
||
|
||
# 1a total time (struct/enum/union registration, incl. per-file read+parse+declaration)
|
||
t_a1: uint64 = win32.win32base.GetTickCount64()
|
||
VLogger.info(f"1a total time: {t_a1-t_a0} ms (registering struct/enum, reg={p1a_reg} skp={p1a_skp})", "Phase1")
|
||
|
||
if p1a_fl > 0:
|
||
VLogger.error(f"Phase 1a: {p1a_fl} files failed to register struct", "Phase1")
|
||
return 1
|
||
|
||
# Release topo-sort resources (topo_order/topo_count_box stay until Phase 1b ends;
|
||
# the class/def arrays are no longer used by 1b).
|
||
if class_names_buf is not None: mb.free(class_names_buf)
|
||
if parent_names_buf is not None: mb.free(parent_names_buf)
|
||
if def_sha1s_buf is not None: mb.free(def_sha1s_buf)
|
||
if generic_file_indices is not None: mb.free(generic_file_indices)
|
||
if generic_file_count_box is not None: mb.free(generic_file_count_box)
|
||
|
||
# Phase 1b: full translation (structs already registered; translate method bodies).
|
||
# mark1 precedes all cross-file persistent tables; reset per file to avoid pool exhaustion.
|
||
mark1: sizet = mb.mark()
|
||
|
||
# Struct/enum entries registered in 1b use stdlib allocation (permanent, not reset)
|
||
GenApi.set_stdlib_alloc(1)
|
||
|
||
# ============================================================
|
||
# Phase 1b-pre: pre-generate .vpi stubs for all files to translate (deps must exist first).
|
||
p1bpre_failed: int = 0
|
||
dir_len_pv: sizet = string.strlen(temp_dir)
|
||
# 1b-pre total-time start (vpi pre-generation)
|
||
t_pv0: uint64 = win32.win32base.GetTickCount64()
|
||
for i_pv in range(int(result.Count)):
|
||
entry_addr_pv: uint64 = uint64(result.Entries) + i_pv * entry_size
|
||
entry_pv: IncludesScanner.FileEntry* = IncludesScanner.FileEntry*(entry_addr_pv)
|
||
if entry_pv is None or entry_pv.Sha1 is None:
|
||
p1bpre_failed += 1
|
||
continue
|
||
sha1_pv: str = entry_pv.Sha1
|
||
in_set_pv: int = 0
|
||
if use_reachable != 0: in_set_pv = StubMergerCollect._is_in_sha1_set(sha1_pv, reachable_set, reachable_count)
|
||
else: in_set_pv = StubMergerCollect._is_in_sha1_set(sha1_pv, sha1_set, set_count)
|
||
if in_set_pv == 0: continue
|
||
# Skip if the vpi already exists
|
||
vpi_path_pv: str = StubMergerCollect._sliced_path(temp_dir, dir_len_pv, sha1_pv, "vpi")
|
||
if vpi_path_pv is None:
|
||
p1bpre_failed += 1
|
||
VLogger.error(f"[Phase 1b-pre] {entry_pv.Path} | failed to generate the vpi path", "Phase1")
|
||
continue
|
||
vf_pv: fileio.File* = fileio.File(vpi_path_pv, fileio.MODE.R)
|
||
if not vf_pv.closed:
|
||
vf_pv.close()
|
||
stdlib.free(vpi_path_pv)
|
||
continue
|
||
stdlib.free(vpi_path_pv)
|
||
# Read the file, parse the AST, generate the vpi
|
||
file_path_pv: str = entry_pv.Path
|
||
f_pv: fileio.File* = fileio.File(file_path_pv, fileio.MODE.R)
|
||
if f_pv.closed:
|
||
p1bpre_failed += 1
|
||
VLogger.error(f"[Phase 1b-pre] {file_path_pv} | cannot open the source file", "Phase1")
|
||
continue
|
||
# read_all's capacity includes the terminator (reads at most cap-1 bytes); longer files are truncated
|
||
if f_pv.size() >= Phase1.SRC_BUF_SIZE:
|
||
p1bpre_failed += 1
|
||
VLogger.error(f"[Phase 1b-pre] {file_path_pv} | source file exceeds the {Phase1.SRC_BUF_SIZE}-byte limit", "Phase1")
|
||
f_pv.close()
|
||
continue
|
||
src_buf_pv: bytes = stdlib.malloc(Phase1.SRC_BUF_SIZE)
|
||
if src_buf_pv is None:
|
||
f_pv.close()
|
||
p1bpre_failed += 1
|
||
VLogger.error(f"[Phase 1b-pre] {file_path_pv} | source buffer allocation failed (OOM)", "Phase1")
|
||
continue
|
||
bytes_read_pv: LONG = f_pv.read_all(src_buf_pv, Phase1.SRC_BUF_SIZE)
|
||
f_pv.close()
|
||
if bytes_read_pv <= 0:
|
||
stdlib.free(src_buf_pv)
|
||
p1bpre_failed += 1
|
||
VLogger.error(f"[Phase 1b-pre] {file_path_pv} | file read failed or is empty", "Phase1")
|
||
continue
|
||
if bytes_read_pv < Phase1.SRC_BUF_SIZE: src_buf_pv[bytes_read_pv] = 0
|
||
lx_pv: ast.Lexer* = ast.new_lexer(mb)
|
||
if lx_pv is None:
|
||
stdlib.free(src_buf_pv)
|
||
p1bpre_failed += 1
|
||
VLogger.error(f"[Phase 1b-pre] {file_path_pv} | lexer allocation failed (OOM)", "Phase1")
|
||
continue
|
||
ast._lexer_init(lx_pv, src_buf_pv, mb)
|
||
tokens_pv: ast.Token* = ast.tokenize(lx_pv)
|
||
tree_pv: ast.AST* = ast.parse_tokens(mb, tokens_pv)
|
||
if tree_pv is None:
|
||
stdlib.free(src_buf_pv)
|
||
p1bpre_failed += 1
|
||
# Parse failure: the log line inlines line/col + reason, plus a full Python-style
|
||
# traceback (same reporting path as the single-file BuildPipeline).
|
||
VLogger.error(f"[Phase 1b-pre] {file_path_pv} | AST parse failed (line {ast.get_parse_err_line()} col {ast.get_parse_err_col()}: {ast.get_parse_err_msg()})", "Phase1")
|
||
GenApi.dump_parse_traceback(file_path_pv)
|
||
continue
|
||
pyi_buf_pv: bytes = stdlib.malloc(Phase1.PYI_BUF_SIZE)
|
||
if pyi_buf_pv is not None:
|
||
pyi_pos_pv: sizet = StubConverter._GeneratePyiFromAst(mb, tree_pv, entry_pv.RelPath, pyi_buf_pv, Phase1.PYI_BUF_SIZE)
|
||
if pyi_pos_pv > 0:
|
||
pyi_path_pv: str = StubMergerCollect._sliced_path(temp_dir, dir_len_pv, sha1_pv, "vpi")
|
||
if pyi_path_pv is not None:
|
||
pf_pv: fileio.File* = fileio.File(pyi_path_pv, fileio.MODE.W)
|
||
if not pf_pv.closed:
|
||
pf_pv.write(pyi_buf_pv, pyi_pos_pv)
|
||
pf_pv.close()
|
||
stdlib.free(pyi_path_pv)
|
||
stdlib.free(pyi_buf_pv)
|
||
else:
|
||
p1bpre_failed += 1
|
||
VLogger.error(f"[Phase 1b-pre] {file_path_pv} | pyi buffer allocation failed (OOM)", "Phase1")
|
||
stdlib.free(src_buf_pv)
|
||
# [TEMP-DIAG] 1b-pre per-file sentinel (as above): check the tail canary of live blocks
|
||
# in [mark1, chk_depth). Must run before reset_to(mark1); only under --detection-deep.
|
||
if mb.deep_detect != 0:
|
||
chk_rc_pv: int = mb.check_alive_blocks(mark1)
|
||
if chk_rc_pv != 0:
|
||
VLogger.error(f"memory corruption (1b-pre per-file post-check, rc={chk_rc_pv}): {file_path_pv}", "Phase1")
|
||
stdlib.exit(1)
|
||
|
||
# Reclaim this file's AST (the vpi is written; nothing else persists after mark1)
|
||
mb.reset_to(mark1)
|
||
|
||
# 1b-pre total time (vpi pre-generation)
|
||
t_pv1: uint64 = win32.win32base.GetTickCount64()
|
||
VLogger.info(f"1b-pre total time: {t_pv1-t_pv0} ms (vpi pre-generation)", "Phase1")
|
||
|
||
if p1bpre_failed > 0:
|
||
VLogger.error(f"Phase 1b-pre: {p1bpre_failed} files failed vpi pre-generation (see the per-file [Phase 1b-pre] log lines above for failures and causes)", "Phase1")
|
||
return 1
|
||
|
||
# Iterate each file (topo order preferred so base-class files translate first and their
|
||
# TypeStr is registered for cross-module vtables; fall back to scan order when topo_count==0).
|
||
translated: int = 0
|
||
skipped: int = 0
|
||
failed: int = 0
|
||
# 1b total-time start (full translation)
|
||
t_b0: uint64 = win32.win32base.GetTickCount64()
|
||
|
||
loop_n_b: int = int(result.Count)
|
||
if topo_count > 0: loop_n_b = topo_count
|
||
for i in range(int(loop_n_b)):
|
||
# Get the entry (topo: file_idx from topo_order, else the scan index)
|
||
file_idx_b: int = int(i)
|
||
if topo_count > 0:
|
||
od_addr_b: uint64 = uint64(topo_order) + sizet(i) * 4
|
||
od_ptr_b: int* = (int*)(od_addr_b)
|
||
if od_ptr_b is not None: file_idx_b = od_ptr_b[0]
|
||
entry_addr: uint64 = uint64(result.Entries) + sizet(file_idx_b) * entry_size
|
||
entry: IncludesScanner.FileEntry* = IncludesScanner.FileEntry*(entry_addr)
|
||
if entry is None:
|
||
failed += 1
|
||
continue
|
||
|
||
sha1: str = entry.Sha1
|
||
if sha1 is None:
|
||
failed += 1
|
||
continue
|
||
|
||
# Check whether the SHA1 is in the translate set (topo already filtered; only the fallback checks)
|
||
in_set: int = 1
|
||
if topo_count == 0:
|
||
in_set = 0
|
||
if use_reachable != 0: in_set = StubMergerCollect._is_in_sha1_set(sha1, reachable_set, reachable_count)
|
||
else: in_set = StubMergerCollect._is_in_sha1_set(sha1, sha1_set, set_count)
|
||
if in_set == 0:
|
||
skipped += 1
|
||
continue
|
||
|
||
# Build the stub path: {temp_dir}/{sha1}.stub.ll
|
||
dir_len: sizet = string.strlen(temp_dir)
|
||
sha1_len: sizet = string.strlen(sha1)
|
||
stub_path: str = StubMergerCollect._sliced_path(temp_dir, dir_len, sha1, "stub.ll")
|
||
if stub_path is None:
|
||
failed += 1
|
||
continue
|
||
|
||
# Skip already-existing stubs, except modules with class definitions: class registration
|
||
# only happens at translation, so skipping breaks cross-module find_struct_by_module.
|
||
force_translate: int = 0
|
||
vpi_path_1b: str = StubMergerCollect._sliced_path(temp_dir, dir_len, sha1, "vpi")
|
||
if vpi_path_1b is not None:
|
||
vf_1b: fileio.File* = fileio.File(vpi_path_1b, fileio.MODE.R)
|
||
if not vf_1b.closed:
|
||
vbuf_1b: bytes = stdlib.malloc(1048576)
|
||
if vbuf_1b is not None:
|
||
vrd_1b: LONG = vf_1b.read_all(vbuf_1b, 1048576)
|
||
vf_1b.close()
|
||
if vrd_1b > 0:
|
||
if vrd_1b < 1048576: vbuf_1b[vrd_1b] = 0
|
||
vtotal_1b: sizet = string.strlen(vbuf_1b)
|
||
vpos_1b: sizet = 0
|
||
while vpos_1b < vtotal_1b:
|
||
vls_1b: sizet = vpos_1b
|
||
while vpos_1b < vtotal_1b and vbuf_1b[vpos_1b] != '\n': vpos_1b += 1
|
||
vll_1b: sizet = vpos_1b - vls_1b
|
||
if vpos_1b < vtotal_1b: vpos_1b += 1
|
||
vi2_1b: sizet = vls_1b
|
||
while vi2_1b < vls_1b + vll_1b and vbuf_1b[vi2_1b] == ' ': vi2_1b += 1
|
||
if vll_1b >= 6 and string.strncmp(vbuf_1b + vi2_1b, "class ", 6) == 0:
|
||
force_translate = 1
|
||
break
|
||
stdlib.free(vbuf_1b)
|
||
else: vf_1b.close()
|
||
stdlib.free(vpi_path_1b)
|
||
if force_translate == 1:
|
||
# Has class: force translation (overwrite stub/text, re-register classes)
|
||
rp: str = entry.RelPath
|
||
VLogger.info(f"contains a class definition, forcing translation: {rp}(sha1={sha1})", "Phase1")
|
||
else:
|
||
# Skip already-existing stubs (on-demand translation)
|
||
sf: fileio.File* = fileio.File(stub_path, fileio.MODE.R)
|
||
if not sf.closed:
|
||
sf.close()
|
||
# Check that text.ll exists too (vtable scan needs it)
|
||
text_path: str = StubMergerCollect._sliced_path(temp_dir, dir_len, sha1, "text.ll")
|
||
if text_path is not None:
|
||
tf: fileio.File* = fileio.File(text_path, fileio.MODE.R)
|
||
if not tf.closed:
|
||
tf.close()
|
||
# The sidecar ({sha1}.meta) and online FULL artifact ({sha1}.full.ll) must also exist;
|
||
# otherwise re-translate to regenerate them for Phase B+/C.
|
||
meta_path_1b: str = StubMergerCollect._sliced_path(temp_dir, dir_len, sha1, "meta")
|
||
meta_ok_1b: int = 0
|
||
if meta_path_1b is not None:
|
||
mf_chk: fileio.File* = fileio.File(meta_path_1b, fileio.MODE.R)
|
||
if not mf_chk.closed:
|
||
mf_chk.close()
|
||
meta_ok_1b = 1
|
||
stdlib.free(meta_path_1b)
|
||
full_path_1b: str = StubMergerCollect._sliced_path(temp_dir, dir_len, sha1, "full.ll")
|
||
full_ok_1b: int = 0
|
||
if full_path_1b is not None:
|
||
ff_chk: fileio.File* = fileio.File(full_path_1b, fileio.MODE.R)
|
||
if not ff_chk.closed:
|
||
ff_chk.close()
|
||
full_ok_1b = 1
|
||
stdlib.free(full_path_1b)
|
||
if meta_ok_1b != 0 and full_ok_1b != 0:
|
||
stdlib.free(text_path)
|
||
stdlib.free(stub_path)
|
||
continue
|
||
stdlib.free(text_path)
|
||
# text.ll missing, re-translate
|
||
rp: str = entry.RelPath
|
||
VLogger.info(f"text.ll not found, re-translating: {rp}(sha1={sha1})", "Phase1")
|
||
else:
|
||
# stub missing, translate
|
||
rp: str = entry.RelPath
|
||
VLogger.info(f"translating: {rp}(sha1={sha1})", "Phase1")
|
||
|
||
stdlib.free(stub_path)
|
||
|
||
# Per-file timer start: read (parse/translate/dump segments, matching Phase2 instrumentation)
|
||
t_f0: uint64 = win32.win32base.GetTickCount64()
|
||
|
||
# Read file contents
|
||
file_path: str = entry.Path
|
||
f: fileio.File* = fileio.File(file_path, fileio.MODE.R)
|
||
if f is None:
|
||
VLogger.error(f"cannot open (f=None): {file_path}", "Phase1")
|
||
failed += 1
|
||
continue
|
||
if f.closed:
|
||
VLogger.error(f"cannot open: {file_path}", "Phase1")
|
||
failed += 1
|
||
continue
|
||
|
||
# read_all's capacity includes the terminator (reads at most cap-1 bytes); longer files are truncated
|
||
if f.size() >= Phase1.SRC_BUF_SIZE:
|
||
VLogger.error(f"source file exceeds the {Phase1.SRC_BUF_SIZE}-byte limit, refusing to read: {file_path}", "Phase1")
|
||
f.close()
|
||
failed += 1
|
||
continue
|
||
|
||
src_buf: bytes = stdlib.malloc(Phase1.SRC_BUF_SIZE)
|
||
if src_buf is None:
|
||
f.close()
|
||
failed += 1
|
||
continue
|
||
|
||
bytes_read: LONG = f.read_all(src_buf, Phase1.SRC_BUF_SIZE)
|
||
f.close()
|
||
if bytes_read <= 0:
|
||
VLogger.error(f"read failed: {file_path}", "Phase1")
|
||
stdlib.free(src_buf)
|
||
failed += 1
|
||
continue
|
||
if bytes_read < Phase1.SRC_BUF_SIZE: src_buf[bytes_read] = 0
|
||
|
||
# Parse AST
|
||
lx: ast.Lexer* = ast.new_lexer(mb)
|
||
if lx is None:
|
||
stdlib.free(src_buf)
|
||
failed += 1
|
||
continue
|
||
ast._lexer_init(lx, src_buf, mb)
|
||
tokens: ast.Token* = ast.tokenize(lx)
|
||
tree: ast.AST* = ast.parse_tokens(mb, tokens)
|
||
if tree is None:
|
||
# Parse failure: line/col + reason go into the log line (reason supplied by parser.vp's
|
||
# _record_* / parse_tokens fallback).
|
||
VLogger.error(f"AST parse failed: {file_path} (line {ast.get_parse_err_line()} col {ast.get_parse_err_col()}: {ast.get_parse_err_msg()})", "Phase1")
|
||
stdlib.free(src_buf)
|
||
failed += 1
|
||
continue
|
||
|
||
# Timer 1: read + lexer/tokenize/parse done
|
||
t_f1: uint64 = win32.win32base.GetTickCount64()
|
||
|
||
# Register CTypedef by scanning source text: AST string fields (attr/id) point into src_buf
|
||
# and dangle across phases (reading attr faults), so match "NAME: t.CTypedef = TYPE" lines.
|
||
GenApi.RegisterCTypedefFromSource(mb, src_buf)
|
||
|
||
# Generate the .vpi stub (walk the AST directly). 1b-pre already generated one, so this
|
||
# is only a fallback when the vpi is missing.
|
||
vpi_path_1b: str = StubMergerCollect._sliced_path(temp_dir, dir_len, sha1, "vpi")
|
||
if vpi_path_1b is not None:
|
||
vf_1b: fileio.File* = fileio.File(vpi_path_1b, fileio.MODE.R)
|
||
vpi_exists_1b: int = 0
|
||
if not vf_1b.closed:
|
||
vf_1b.close()
|
||
vpi_exists_1b = 1
|
||
if vpi_exists_1b == 0:
|
||
pyi_buf: bytes = stdlib.malloc(Phase1.PYI_BUF_SIZE)
|
||
if pyi_buf is not None:
|
||
pyi_pos: sizet = StubConverter._GeneratePyiFromAst(mb, tree, entry.Path, pyi_buf, Phase1.PYI_BUF_SIZE)
|
||
if pyi_pos > 0:
|
||
pf_1b: fileio.File* = fileio.File(vpi_path_1b, fileio.MODE.W)
|
||
if not pf_1b.closed:
|
||
pf_1b.write(pyi_buf, pyi_pos)
|
||
pf_1b.close()
|
||
stdlib.free(pyi_buf)
|
||
stdlib.free(vpi_path_1b)
|
||
|
||
# Translate AST -> LLVM IR
|
||
tr: HandlesTranslator.Translator* = HandlesTranslator.Translator()
|
||
if tr is None:
|
||
stdlib.free(src_buf)
|
||
failed += 1
|
||
continue
|
||
tr.ModuleSha1 = sha1
|
||
tr.CurrentPackage = GenApi.compute_package_from_relpath(mb, entry.RelPath)
|
||
GenApi.set_current_file_type(file_path)
|
||
# --traceback: mirror the DWARF switch into the Gen-layer emitter (same
|
||
# BuildPipeline._traceback_mode that gates -frame-pointer=all / dbghelp / Install()).
|
||
DebugInfoGen.SetEnabled(BuildPipeline._traceback_mode)
|
||
GenApi.set_current_file_imports(file_path)
|
||
GenApi.set_current_module_sha1(sha1)
|
||
GenApi.clear_cdefine_constants()
|
||
GenApi.clear_ctypedefs()
|
||
GenApi.reset_visible_structs(mb, 0)
|
||
|
||
# Pre-translate sentinel: parsing also allocates from the pool, so a parser overrun that
|
||
# corrupts a live block header is pinned to the current file (deterministic crash location).
|
||
pre_rc: int = mb.check_alive_blocks(mark1)
|
||
if pre_rc != 0:
|
||
VLogger.error(f"memory corruption (live blocks before translate, rc={pre_rc}): {file_path}", "Phase1")
|
||
stdlib.exit(1)
|
||
|
||
ret: int = tr.translate(tree)
|
||
# Timer 2: translate returned
|
||
t_f2: uint64 = win32.win32base.GetTickCount64()
|
||
# Diagnostic: translate returned normally (if it hangs inside, this line is not printed,
|
||
# distinguishing an internal hang from a later dump hang).
|
||
if ret == 0: VLogger.info(f"translation complete (translate returned): {file_path}", "Phase1")
|
||
if ret != 0:
|
||
VLogger.error(f"translation failed: {file_path}", "Phase1") # error terminates
|
||
stdlib.free(src_buf)
|
||
if tr._global_names is not None: stdlib.free(tr._global_names)
|
||
if tr._nonlocal_names is not None: stdlib.free(tr._nonlocal_names)
|
||
failed += 1
|
||
continue
|
||
|
||
# --traceback: inject the startup Install() ctor when translating traceback.vp;
|
||
# the injector lives in the Gen layer, adding no cross-module mlirlite dependency.
|
||
if BuildPipeline._traceback_mode != 0 and BuildPipeline._tb_basename_is_traceback(file_path) != 0: HandlesTranslator.InjectTracebackInstallCtor(tr)
|
||
|
||
# Diagnostic: the pre-translate sentinel misses write-throughs during translate, so check
|
||
# live block headers/canaries before dump (rc=6 means a tail canary was overwritten).
|
||
post_rc: int = mb.check_alive_blocks(mark1)
|
||
if post_rc != 0: VLogger.error(f"memory corruption (live blocks after translate, rc={post_rc}): {file_path}", "Phase1")
|
||
|
||
# Dump stub IR (declarations only). 1MB is not enough: ast/parser.vp's text.ll exceeds it
|
||
# and is truncated (opt: "end of file in global variable name"), so use 8MB.
|
||
PHASE1_IR_SIZE: sizet = 8388608
|
||
# FULL dump = stub + text (same module), with 2x headroom
|
||
PHASE1_FULL_SIZE: sizet = 16777216
|
||
stub_buf: bytes = stdlib.malloc(PHASE1_IR_SIZE)
|
||
if stub_buf is None:
|
||
stdlib.free(src_buf)
|
||
if tr._global_names is not None: stdlib.free(tr._global_names)
|
||
if tr._nonlocal_names is not None: stdlib.free(tr._nonlocal_names)
|
||
failed += 1
|
||
continue
|
||
tr.dump_ir(stub_buf, PHASE1_IR_SIZE, mlirlite.OUTPUT_STUB)
|
||
stub_len: sizet = string.strlen(stub_buf)
|
||
|
||
# save stub.ll
|
||
dir_len_p1: sizet = string.strlen(temp_dir)
|
||
stub_path_p1: str = StubMergerCollect._sliced_path(temp_dir, dir_len_p1, sha1, "stub.ll")
|
||
if stub_path_p1 is not None:
|
||
sf_p1: fileio.File* = fileio.File(stub_path_p1, fileio.MODE.W)
|
||
if not sf_p1.closed:
|
||
sf_p1.write(stub_buf, stub_len)
|
||
sf_p1.close()
|
||
stdlib.free(stub_path_p1)
|
||
stdlib.free(stub_buf)
|
||
|
||
# dump text IR(definitions only)
|
||
text_buf: bytes = stdlib.malloc(PHASE1_IR_SIZE)
|
||
if text_buf is None:
|
||
stdlib.free(src_buf)
|
||
if tr._global_names is not None: stdlib.free(tr._global_names)
|
||
if tr._nonlocal_names is not None: stdlib.free(tr._nonlocal_names)
|
||
failed += 1
|
||
continue
|
||
tr.dump_ir(text_buf, PHASE1_IR_SIZE, mlirlite.OUTPUT_TEXT)
|
||
text_len: sizet = string.strlen(text_buf)
|
||
|
||
# save text.ll
|
||
text_path_p1: str = StubMergerCollect._sliced_path(temp_dir, dir_len_p1, sha1, "text.ll")
|
||
if text_path_p1 is not None:
|
||
tf_p1: fileio.File* = fileio.File(text_path_p1, fileio.MODE.W)
|
||
if not tf_p1.closed:
|
||
tf_p1.write(text_buf, text_len)
|
||
tf_p1.close()
|
||
stdlib.free(text_path_p1)
|
||
stdlib.free(text_buf)
|
||
|
||
# save {sha1}.meta (structured sidecar: function list/struct layout/vtables, no IR-text reverse-engineering)
|
||
StubMergerIR.WriteModuleMeta(tr.Module, temp_dir, dir_len_p1, sha1)
|
||
|
||
# save {sha1}.full.ll (the online FULL artifact: a self-contained single-module IR for
|
||
# Phase B to compile directly, replacing the stub + dependency text merge).
|
||
full_buf_p1: bytes = stdlib.malloc(PHASE1_FULL_SIZE)
|
||
if full_buf_p1 is None: VLogger.error("full.ll buffer allocation failed", "Phase1")
|
||
tr.dump_ir(full_buf_p1, PHASE1_FULL_SIZE, mlirlite.OUTPUT_FULL)
|
||
full_len_p1: sizet = string.strlen(full_buf_p1)
|
||
full_path_p1: str = StubMergerCollect._sliced_path(temp_dir, dir_len_p1, sha1, "full.ll")
|
||
if full_path_p1 is not None:
|
||
ff_p1: fileio.File* = fileio.File(full_path_p1, fileio.MODE.W)
|
||
if not ff_p1.closed:
|
||
ff_p1.write(full_buf_p1, full_len_p1)
|
||
ff_p1.close()
|
||
stdlib.free(full_path_p1)
|
||
stdlib.free(full_buf_p1)
|
||
|
||
# --traceback: take this module's debug call-site records (accumulated in mlirlite) and
|
||
# write <sha1>.vpcall for WriteStackMap to reconstruct call-site lines. No file when off.
|
||
if BuildPipeline._traceback_mode != 0:
|
||
vpcall_text_p1: str = mlirlite.DebugTakeText()
|
||
if vpcall_text_p1 is not None:
|
||
BuildPipeline.WriteCallSidecar(temp_dir, sha1, vpcall_text_p1)
|
||
stdlib.free(vpcall_text_p1)
|
||
|
||
# Timer 3: dump (stub+text) done; print the per-file three-segment timing
|
||
t_f3: uint64 = win32.win32base.GetTickCount64()
|
||
VLogger.info(f"[{i+1}/{loop_n_b}] translating: {file_path} (parse={t_f1-t_f0} ms, translate={t_f2-t_f1} ms, dump={t_f3-t_f2} ms, total={t_f3-t_f0} ms)", "Phase1")
|
||
|
||
# save dependencies(_imported_modules) for Phase B
|
||
deps_path_p1: str = StubMergerCollect._sliced_path(temp_dir, dir_len_p1, sha1, "deps.txt")
|
||
if deps_path_p1 is not None:
|
||
df_p1: fileio.File* = fileio.File(deps_path_p1, fileio.MODE.W)
|
||
if not df_p1.closed:
|
||
if tr._imported_modules is not None:
|
||
dl: sizet = string.strlen(tr._imported_modules)
|
||
df_p1.write(tr._imported_modules, dl)
|
||
df_p1.close()
|
||
stdlib.free(deps_path_p1)
|
||
|
||
# Release the Translator's C malloc resources
|
||
if tr._global_names is not None: stdlib.free(tr._global_names)
|
||
if tr._nonlocal_names is not None: stdlib.free(tr._nonlocal_names)
|
||
|
||
# All dump_ir calls done, src_buf no longer needed
|
||
stdlib.free(src_buf)
|
||
|
||
translated += 1
|
||
|
||
# OOM reclamation: free this file's transient AST/tokens/lexer/IR/dump data (persistent
|
||
# tables moved to stdlib). Memory sentinels locate overruns deterministically.
|
||
chk_rc: int = mb.check_alive_blocks(mark1)
|
||
if chk_rc != 0: VLogger.error(f"memory corruption (live block header, rc={chk_rc}): {file_path}", "Phase1")
|
||
# Out-of-bounds writes are the crash root cause (unrelated to reclamation/UAF, confirmed
|
||
# by A/B tests); reset_to plus the tail canary pins the offending file.
|
||
mb.reset_to(mark1)
|
||
sc_rc: int = mb.self_check()
|
||
if sc_rc != 0: VLogger.error(f"memory corruption (free list, rc={sc_rc}): {file_path}", "Phase1")
|
||
|
||
# 1b total time (full translation + dump)
|
||
t_b1: uint64 = win32.win32base.GetTickCount64()
|
||
VLogger.info(f"1b total time: {t_b1-t_b0} ms (full translation, trans={translated} skip={skipped} fail={failed})", "Phase1")
|
||
|
||
# Release the topo-sort resources used by Phase 1b
|
||
if topo_order is not None: mb.free(topo_order)
|
||
if topo_count_box is not None: mb.free(topo_count_box)
|
||
|
||
VLogger.info(f"done: translated={translated} skipped={skipped} failed={failed}", "Phase1")
|
||
|
||
if failed > 0:
|
||
VLogger.error(f"Phase1: {failed} files failed to translate", "Phase1")
|
||
return 1
|
||
|
||
# Release the reachable SHA1 set (if allocated)
|
||
if reachable_set is not None: stdlib.free(reachable_set)
|
||
|
||
return 0
|