Files
ViperC/App/lib/Phase/Phase2/__init__.vp
T

1538 lines
85 KiB
Plaintext

import lib.core.Gen.GenApi as GenApi
import t, c
from stdint import *
import stdio
import string
import stdlib
import memhub
import viperlib
import win32.fileio as fileio
import win32.win32file
import win32.win32base
import sys
import ast
import mlirlite
import subprocess
import argparse
import lib.core.VLogger as VLogger
import lib.core.Gen.Translator as HandlesTranslator
import lib.Phase.BuildPipeline as BuildPipeline
import lib.core.IncludesScanner as IncludesScanner
import lib.core.CompilerFactory as CompilerFactory
import lib.Stub.StubMerger as StubMerger
import lib.Projectrans.Utils as Utils
import lib.Projectrans.Config as Config
import lib.Stub.StubMerger.StubMergerBuild as StubMergerBuild
import lib.Stub.StubMerger.StubMergerCollect as StubMergerCollect
import lib.Stub.StubMerger.StubMergerIR as StubMergerIR
import lib.Stub.StubMerger.StubMergerStore as StubMergerStore
import lib.Stub.StubMerger.StubMergerTopo as StubMergerTopo
import lib.Stub.StubGen.Converter as StubConverter
# Global mbuddy pointer
_mbuddy: memhub.MemBuddy*
# Source buffer size (1MB)
define SRC_BUF_SIZE = 1048576
# Max source files (recursive scanning grows the count; too small truncates the list -> undefined symbol)
define MAX_SRC_FILES = 128
@t.NoVTable
class SrcFileEntry:
"""Source file entry"""
Path: str # full path
Sha1: str # first 16 chars of SHA1
# _ScanDirForPyFiles - recursively scan a dir and collect .vp/.py files into entries, returning the new count.
# Args: mb pool; dir_path; entries SrcFileEntry array; entry_size; file_count; max_files.
def _ScanDirForPyFiles(mb: memhub.MemBuddy*, dir_path: str, entries: SrcFileEntry*, entry_size: sizet, file_count: int, max_files: int) -> int:
"""Recursively scan a directory, collect .vp/.py files into entries, return the new file_count"""
if dir_path is None or entries is None: return file_count
if file_count >= max_files: return file_count
dir_len: sizet = string.strlen(dir_path)
pattern: bytes = stdlib.malloc(dir_len + 16)
if pattern is None: return file_count
viperlib.snprintf(pattern, dir_len + 16, "%s/*", dir_path)
find_data_size: sizet = win32.win32file.WIN32_FIND_DATAA.__sizeof__()
find_data: win32.win32file.WIN32_FIND_DATAA* = stdlib.malloc(find_data_size + 16)
if find_data is None:
stdlib.free(pattern)
return file_count
string.memset(find_data, 0, find_data_size + 16)
handle: win32.win32base.HANDLE = win32.win32file.FindFirstFileA(pattern, find_data)
if handle == win32.win32base.INVALID_HANDLE_VALUE:
stdlib.free(pattern)
stdlib.free(find_data)
return file_count
while True:
fname: str = find_data.cFileName
if fname is None:
if win32.win32file.FindNextFileA(handle, find_data) == 0: break
continue
fname_len: sizet = string.strlen(fname)
if fname_len <= 0:
if win32.win32file.FindNextFileA(handle, find_data) == 0: break
continue
# Skip . and ..
is_dot: int = 0
if fname_len == 1 and fname[0] == '.': is_dot = 1
elif fname_len == 2 and fname[0] == '.' and fname[1] == '.': is_dot = 1
if is_dot != 0:
if win32.win32file.FindNextFileA(handle, find_data) == 0: break
continue
# Check whether it is a directory
attrs: ULONG = find_data.dwFileAttributes
is_dir: int = 0
if (attrs & win32.win32file.FILE_ATTRIBUTE_DIRECTORY) != 0: is_dir = 1
if is_dir != 0:
# Recurse into the subdirectory
sub_dir: bytes = Utils.JoinPath(dir_path, fname)
if sub_dir is not None:
file_count = _ScanDirForPyFiles(mb, sub_dir, entries, entry_size, file_count, max_files)
stdlib.free(sub_dir)
if win32.win32file.FindNextFileA(handle, find_data) == 0: break
continue
# Check for a .vp source file (Viper sources are only .vp now; .py is no longer recognized)
if fname_len > 3 and Utils.is_vp_at(fname, 0, fname_len) != 0 and file_count < max_files:
full_path: bytes = Utils.JoinPath(dir_path, fname)
if full_path is not None:
sbuf: bytes = Utils.read_file_to_buf(full_path, SRC_BUF_SIZE)
if sbuf is None:
if win32.win32file.FindNextFileA(handle, find_data) == 0: break
continue
# Use IncludesScanner.compute_file_sha1 (CRLF->LF normalized) as StubMerger does for the same
# App sources; raw-byte hashing makes stub/define symbol prefixes differ -> undefined reference.
sha1_val: str = IncludesScanner.compute_file_sha1(mb, full_path)
if sha1_val is not None:
ea: uint64 = uint64(entries) + file_count * entry_size
ent: SrcFileEntry* = (SrcFileEntry*)(ea)
if ent is not None:
ent.Path = full_path
ent.Sha1 = sha1_val
file_count += 1
VLogger.info(f"{fname} (sha1={sha1_val})", "project")
stdlib.free(sbuf)
# full_path is not freed: ent.Path references it
if win32.win32file.FindNextFileA(handle, find_data) == 0: break
win32.win32file.FindClose(handle)
stdlib.free(pattern)
stdlib.free(find_data)
return file_count
# _ScanDirForAsmFiles - recursively scan a dir and collect .s assembly files.
# Like _ScanDirForPyFiles; VPC does not translate .s but assembles it to .obj via llvm-mc for linking.
def _ScanDirForAsmFiles(mb: memhub.MemBuddy*, dir_path: str, entries: SrcFileEntry*, entry_size: sizet, file_count: int, max_files: int) -> int:
"""Recursively scan a directory, collect .s assembly files into entries, return the new file_count"""
if dir_path is None or entries is None: return file_count
if file_count >= max_files: return file_count
dir_len: sizet = string.strlen(dir_path)
pattern: bytes = stdlib.malloc(dir_len + 16)
if pattern is None: return file_count
viperlib.snprintf(pattern, dir_len + 16, "%s/*", dir_path)
find_data_size: sizet = win32.win32file.WIN32_FIND_DATAA.__sizeof__()
find_data: win32.win32file.WIN32_FIND_DATAA* = stdlib.malloc(find_data_size + 16)
if find_data is None:
stdlib.free(pattern)
return file_count
string.memset(find_data, 0, find_data_size + 16)
handle: win32.win32base.HANDLE = win32.win32file.FindFirstFileA(pattern, find_data)
if handle == win32.win32base.INVALID_HANDLE_VALUE:
stdlib.free(pattern)
stdlib.free(find_data)
return file_count
while True:
fname: str = find_data.cFileName
if fname is None:
if win32.win32file.FindNextFileA(handle, find_data) == 0: break
continue
fname_len: sizet = string.strlen(fname)
if fname_len <= 0:
if win32.win32file.FindNextFileA(handle, find_data) == 0: break
continue
# Skip . and ..
is_dot: int = 0
if fname_len == 1 and fname[0] == '.': is_dot = 1
elif fname_len == 2 and fname[0] == '.' and fname[1] == '.': is_dot = 1
if is_dot != 0:
if win32.win32file.FindNextFileA(handle, find_data) == 0: break
continue
# Check whether it is a directory
attrs: ULONG = find_data.dwFileAttributes
is_dir: int = 0
if (attrs & win32.win32file.FILE_ATTRIBUTE_DIRECTORY) != 0: is_dir = 1
if is_dir != 0:
sub_dir: bytes = Utils.JoinPath(dir_path, fname)
if sub_dir is not None:
file_count = _ScanDirForAsmFiles(mb, sub_dir, entries, entry_size, file_count, max_files)
stdlib.free(sub_dir)
if win32.win32file.FindNextFileA(handle, find_data) == 0: break
continue
# Check for the .s suffix (assembly source)
is_asm: int = 0
if fname_len > 2 and fname[fname_len - 2] == '.' and fname[fname_len - 1] == 's': is_asm = 1
if is_asm != 0 and file_count < max_files:
full_path: bytes = Utils.JoinPath(dir_path, fname)
if full_path is not None:
# Same SHA1 convention as .vp (CRLF->LF normalized), used for the sliced path
sha1_val: str = IncludesScanner.compute_file_sha1(mb, full_path)
if sha1_val is not None:
ea: uint64 = uint64(entries) + file_count * entry_size
ent: SrcFileEntry* = (SrcFileEntry*)(ea)
if ent is not None:
ent.Path = full_path
ent.Sha1 = sha1_val
file_count += 1
VLogger.info(f"{fname} (asm sha1={sha1_val})", "project")
# full_path is not freed: ent.Path references it
if win32.win32file.FindNextFileA(handle, find_data) == 0: break
win32.win32file.FindClose(handle)
stdlib.free(pattern)
stdlib.free(find_data)
return file_count
# _PreRegisterAppGenericTemplates - pre-register App generic class templates before mark2.
# The generic tables live in the pool; registering after mark2 would be reclaimed by reset_to, corrupting later passes.
def _PreRegisterAppGenericTemplates(mb: memhub.MemBuddy*, entries: SrcFileEntry*, file_count: int, entry_size: sizet) -> int:
"""Pre-register App source generic class templates before mark2, return the processed count"""
if mb is None or entries is None or file_count <= 0: return 0
handled: int = 0
for i in range(int(file_count)):
ea_pa: uint64 = uint64(entries) + i * entry_size
ent_pa: SrcFileEntry* = (SrcFileEntry*)(ea_pa)
if ent_pa is None or ent_pa.Path is None or ent_pa.Sha1 is None: continue
# Read the file
f_pa: fileio.File* = fileio.File(ent_pa.Path, fileio.MODE.R)
if f_pa.closed: continue
# read_all's capacity includes the terminator (reads at most cap-1 bytes); longer files are truncated
if f_pa.size() >= SRC_BUF_SIZE:
VLogger.error(f"source file exceeds {SRC_BUF_SIZE} byte limit, refusing to read: {ent_pa.Path}", "PhaseA")
f_pa.close()
continue
src_buf_pa: bytes = stdlib.malloc(SRC_BUF_SIZE)
if src_buf_pa is None:
f_pa.close()
continue
bytes_read_pa: LONG = f_pa.read_all(src_buf_pa, SRC_BUF_SIZE)
f_pa.close()
if bytes_read_pa <= 0:
stdlib.free(src_buf_pa)
continue
if bytes_read_pa < SRC_BUF_SIZE: src_buf_pa[bytes_read_pa] = 0
# Parse the AST only (perm pool keeps template nodes alive across the mark2 reset)
perm_pa: memhub.MemBuddy* = CompilerFactory.factory.PermPool
if perm_pa is None: perm_pa = mb
lx_pa: ast.Lexer* = ast.new_lexer(perm_pa)
if lx_pa is None:
stdlib.free(src_buf_pa)
continue
ast._lexer_init(lx_pa, src_buf_pa, perm_pa)
tokens_pa: ast.Token* = ast.tokenize(lx_pa)
tree_pa: ast.AST* = ast.parse_tokens(perm_pa, tokens_pa)
if tree_pa is None:
stdlib.free(src_buf_pa)
continue
# Walk top-level ClassDef/FunctionDef and register generic templates (type_params != None)
ch_pa: list[ast.AST*]* = tree_pa.children
if ch_pa is not None:
cn_pa: sizet = len(ch_pa)
for ci_pa in range(int(cn_pa)):
child_pa: ast.AST* = ch_pa.get(ci_pa)
if child_pa is None: continue
ck_pa: int = child_pa.kind()
if ck_pa == ast.ASTKind.ClassDef:
cd_pa: ast.ClassDef* = ast.ClassDef*(child_pa)
if cd_pa is None or cd_pa.name is None: continue
tp_pa: list[str]* = cd_pa.type_params
if tp_pa is None: continue
GenApi.set_current_file_type(ent_pa.Path)
GenApi._register_generic_template(perm_pa, cd_pa, ent_pa.Sha1)
handled += 1
elif ck_pa == ast.ASTKind.FunctionDef:
fd_pa: ast.FunctionDef* = ast.FunctionDef*(child_pa)
if fd_pa is None or fd_pa.name is None: continue
tf_pa: list[str]* = fd_pa.type_params
if tf_pa is None: continue
GenApi.set_current_file_type(ent_pa.Path)
GenApi._register_generic_func_template(perm_pa, fd_pa, ent_pa.Sha1)
handled += 1
stdlib.free(src_buf_pa)
if handled > 0: VLogger.info(f"pre-registered {handled} App generic templates before mark2 (classes + functions)", "PhaseA")
return handled
# Phase A convergence-failure dependency-graph diagnostics.
# Each file's deps and missing details are persisted to a stdlib table (survives reset) and dumped on failure.
PA_DEPS_MAX_STR: sizet = 1024
PA_MISS_MAX_STR: sizet = 768
PA_GRAPH_DEPTH_MAX: int = 3 # missing-chain drill-down depth (counts from layer 1)
PA_GRAPH_OK_PROBE: int = 2 # ready nodes are probed 1 more level (to find hidden misses)
_pa_diag_max: sizet = 0
_pa_diag_count: int = 0
_pa_diag_sha1: bytes = None
_pa_diag_deps: bytes = None
_pa_diag_miss: bytes = None
def _PhaseADiagInit(max_count: sizet) -> None:
"""Allocate the persistent Phase A convergence-failure diagnostic table"""
global _pa_diag_max, _pa_diag_count, _pa_diag_sha1, _pa_diag_deps, _pa_diag_miss
_pa_diag_max = max_count
_pa_diag_count = 0
_pa_diag_sha1 = stdlib.malloc(max_count * 17)
_pa_diag_deps = stdlib.malloc(max_count * PA_DEPS_MAX_STR)
_pa_diag_miss = stdlib.malloc(max_count * PA_MISS_MAX_STR)
if _pa_diag_sha1 is not None: string.memset(_pa_diag_sha1, 0, max_count * 17)
if _pa_diag_deps is not None: string.memset(_pa_diag_deps, 0, max_count * PA_DEPS_MAX_STR)
if _pa_diag_miss is not None: string.memset(_pa_diag_miss, 0, max_count * PA_MISS_MAX_STR)
def _PhaseADiagFree() -> None:
"""Free the persistent diagnostic table"""
global _pa_diag_max, _pa_diag_count, _pa_diag_sha1, _pa_diag_deps, _pa_diag_miss
if _pa_diag_sha1 is not None:
stdlib.free(_pa_diag_sha1)
_pa_diag_sha1 = None
if _pa_diag_deps is not None:
stdlib.free(_pa_diag_deps)
_pa_diag_deps = None
if _pa_diag_miss is not None:
stdlib.free(_pa_diag_miss)
_pa_diag_miss = None
_pa_diag_count = 0
_pa_diag_max = 0
def _PhaseADiagRecord(sha1: str, deps: str, miss: str) -> None:
"""Record or update one file's diagnostics: sha1 to deps plus missing detail"""
global _pa_diag_count
if _pa_diag_sha1 is None or _pa_diag_deps is None or sha1 is None: return
if string.strlen(sha1) < 16: return
for i in range(int(_pa_diag_count)):
es: bytes = _pa_diag_sha1 + sizet(i) * 17
if string.strcmp(es, sha1) == 0:
if miss is not None and _pa_diag_miss is not None:
maddr: bytes = _pa_diag_miss + sizet(i) * PA_MISS_MAX_STR
string.strncpy(maddr, miss, PA_MISS_MAX_STR - 1)
maddr[PA_MISS_MAX_STR - 1] = 0
return
if _pa_diag_count >= _pa_diag_max: return
saddr: bytes = _pa_diag_sha1 + sizet(_pa_diag_count) * 17
string.strncpy(saddr, sha1, 16)
saddr[16] = 0
daddr: bytes = _pa_diag_deps + sizet(_pa_diag_count) * PA_DEPS_MAX_STR
if deps is not None: string.strncpy(daddr, deps, PA_DEPS_MAX_STR - 1)
daddr[PA_DEPS_MAX_STR - 1] = 0
if miss is not None and _pa_diag_miss is not None:
maddr2: bytes = _pa_diag_miss + sizet(_pa_diag_count) * PA_MISS_MAX_STR
string.strncpy(maddr2, miss, PA_MISS_MAX_STR - 1)
maddr2[PA_MISS_MAX_STR - 1] = 0
_pa_diag_count += 1
def _PhaseADiagDepsLookup(sha1: str) -> str:
"""Look up the persisted deps string by sha1; caller frees the copy, None if absent"""
if _pa_diag_sha1 is None or _pa_diag_deps is None or sha1 is None: return None
for i in range(int(_pa_diag_count)):
es: bytes = _pa_diag_sha1 + sizet(i) * 17
if string.strcmp(es, sha1) == 0:
src: bytes = _pa_diag_deps + sizet(i) * PA_DEPS_MAX_STR
ln: sizet = string.strlen(src)
out: bytes = stdlib.malloc(ln + 1)
if out is None: return None
string.strcpy(out, src)
return out
return None
def _PhaseADiagMissLookup(sha1: str) -> str:
"""Look up the persisted missing detail by sha1; caller frees the copy, None if absent"""
if _pa_diag_sha1 is None or _pa_diag_miss is None or sha1 is None: return None
for i in range(int(_pa_diag_count)):
es: bytes = _pa_diag_sha1 + sizet(i) * 17
if string.strcmp(es, sha1) == 0:
src: bytes = _pa_diag_miss + sizet(i) * PA_MISS_MAX_STR
ln: sizet = string.strlen(src)
if ln == 0: return None
out: bytes = stdlib.malloc(ln + 1)
if out is None: return None
string.strcpy(out, src)
return out
return None
def _PhaseATextLLExists(temp_dir: str, td_len: sizet, sha1: str) -> int:
"""Probe whether {temp_dir}/{sha1 first 2 chars}/{sha1}.text.ll exists, 1=yes"""
if temp_dir is None or td_len == 0 or sha1 is None or string.strlen(sha1) < 2: return 0
tpath: str = StubMergerCollect._sliced_path(temp_dir, td_len, sha1, "text.ll")
if tpath is None: return 0
tf: fileio.File* = fileio.File(tpath, fileio.MODE.R)
exists: int = 0
if not tf.closed:
tf.close()
exists = 1
stdlib.free(tpath)
return exists
def _PhaseAResolveDepSha1(mod_name: str) -> str:
"""Resolve a dependency module name to SHA1 (exact then suffix), None if absent"""
if mod_name is None: return None
s1: str = StubMergerStore.LookupSha1ByModName(mod_name)
if s1 is not None: return s1
return StubMergerStore.LookupSha1ByModNameSuffix(mod_name)
def _PhaseADumpNode(temp_dir: str, td_len: sizet, sha1: str, depth: int, prefix: bytes) -> None:
"""Recursively print the dependency subgraph; [MISS]=missing text.ll, [OK]=ready"""
deps: bytes = None
if sha1 is not None: deps = _PhaseADiagDepsLookup(sha1)
if deps is None and sha1 is not None:
# Fallback: read {temp_dir}/{prefix}/{sha1}.deps.txt (Phase1 already wrote it for includes)
dpath: str = StubMergerCollect._sliced_path(temp_dir, td_len, sha1, "deps.txt")
if dpath is not None:
df: fileio.File* = fileio.File(dpath, fileio.MODE.R)
if not df.closed:
deps = stdlib.malloc(PA_DEPS_MAX_STR)
if deps is not None:
rl: int64 = df.read_all(deps, PA_DEPS_MAX_STR)
if rl < 0: rl = 0
# read_all's size arg is the buffer capacity (incl. terminator): it reads at most MAX-1
deps[rl] = 0
df.close()
stdlib.free(dpath)
if deps is None or deps[0] == 0:
if deps is not None: stdlib.free(deps)
return
dl: sizet = string.strlen(deps)
token_count: int = 0
pos: sizet = 0
while pos < dl:
while pos < dl and deps[pos] == ' ': pos += 1
if pos >= dl: break
token_count += 1
while pos < dl and deps[pos] != ' ': pos += 1
pos = 0
idx: int = 0
while pos < dl:
while pos < dl and deps[pos] == ' ': pos += 1
if pos >= dl: break
ts: sizet = pos
while pos < dl and deps[pos] != ' ': pos += 1
tl: sizet = pos - ts
nm: bytes = stdlib.malloc(tl + 1)
if nm is not None:
string.strncpy(nm, deps + ts, tl)
nm[tl] = 0
is_last: int = 0
if idx + 1 >= token_count: is_last = 1
builtin: int = 0
if string.strcmp(nm, "t") == 0 or string.strcmp(nm, "c") == 0: builtin = 1
dep_sha1: str = _PhaseAResolveDepSha1(nm)
txt_ok: int = 0
if dep_sha1 is not None: txt_ok = _PhaseATextLLExists(temp_dir, td_len, dep_sha1)
# Branch characters are inlined (don't return string literals: VPC would return a dangling
# stack pointer for a literal -> printf %s crashes and the graph output is swallowed)
br_1: str = "|-- "
br_2: str = "`-- "
if builtin != 0:
if is_last != 0: stdio.printf("%s%s%s (builtin)\n", prefix, br_2, nm)
else: stdio.printf("%s%s%s (builtin)\n", prefix, br_1, nm)
elif dep_sha1 is None:
if is_last != 0: stdio.printf("%s%s%s (unresolved)\n", prefix, br_2, nm)
else: stdio.printf("%s%s%s (unresolved)\n", prefix, br_1, nm)
elif txt_ok != 0:
if is_last != 0: stdio.printf("%s%s%s [OK]\n", prefix, br_2, nm)
else: stdio.printf("%s%s%s [OK]\n", prefix, br_1, nm)
else:
if is_last != 0: stdio.printf("%s%s%s [MISS] text.ll=%s\n", prefix, br_2, nm, dep_sha1)
else: stdio.printf("%s%s%s [MISS] text.ll=%s\n", prefix, br_1, nm, dep_sha1)
# Drill down: MISS follows the whole chain (to PA_GRAPH_DEPTH_MAX); OK probes 1 level only
can_expand: int = 0
if dep_sha1 is not None and builtin == 0:
if txt_ok == 0:
if depth + 1 < PA_GRAPH_DEPTH_MAX: can_expand = 1
else:
if depth + 1 < PA_GRAPH_OK_PROBE: can_expand = 1
if can_expand != 0:
# +5: strcat appends "| "/" " (4 chars) + NUL; strlen+4 would overflow 1 byte and
# corrupt heap metadata, crashing later malloc/free and aborting the graph output
npl: sizet = string.strlen(prefix) + 5
new_prefix: bytes = stdlib.malloc(npl)
if new_prefix is not None:
string.strcpy(new_prefix, prefix)
if is_last != 0: string.strcat(new_prefix, " ")
else: string.strcat(new_prefix, "| ")
_PhaseADumpNode(temp_dir, td_len, dep_sha1, depth + 1, new_prefix)
stdlib.free(new_prefix)
stdlib.free(nm)
idx += 1
stdlib.free(deps)
def _PhaseADumpFailGraph(temp_dir: str, td_len: sizet, path: str, sha1: str) -> None:
"""Convergence-failure diagnostics: pending detail plus dependency tree.
Flush after every printf so the trace survives a later crash."""
stdio.printf("[PhaseA-Fail] ==============================================================\n")
stdio.fflush(None)
stdio.printf("[PhaseA-Fail] FAIL TO TRANSLATE: %s [sha1=%s]\n", path, sha1)
stdio.fflush(None)
miss: bytes = _PhaseADiagMissLookup(sha1)
if miss is not None:
if miss[0] != 0: stdio.printf("[PhaseA-Fail] pending reason: %s\n", miss)
stdlib.free(miss)
stdio.fflush(None)
stdio.printf("[PhaseA-Fail] dep graph ([MISS]=missing text.ll, [OK]=ready, depth<=%d):\n", PA_GRAPH_DEPTH_MAX)
stdio.fflush(None)
root_prefix: bytes = stdlib.malloc(1)
if root_prefix is not None:
root_prefix[0] = 0
_PhaseADumpNode(temp_dir, td_len, sha1, 0, root_prefix)
stdlib.free(root_prefix)
stdio.fflush(None)
def RunMultiFileProject(mb: memhub.MemBuddy*, source_dir: str, temp_dir: str, output_dir: str, cc_cmd: str, cc_flags: str, linker_cmd: str, linker_flags: str, linker_output: str, includes_binary_dir: str, includes_dir: str, do_phase1: int, do_phase2: int, log: VLogger.Logger*, args: argparse.ParsedArgs*, linker_static_libs: str = None, linker_dynamic_libs: str = None) -> int:
"""Compile a multi-file project; return 0 on success, non-zero on failure"""
if source_dir is None:
VLogger.error("source_dir is empty", "project")
return 1
VLogger.info(f"multi-file project compilation: {source_dir}", "project")
# === 1. Recursively scan .vp/.py files under source_dir (including subdirectories) ===
entry_size: sizet = SrcFileEntry.__sizeof__()
entries: SrcFileEntry* = stdlib.malloc(MAX_SRC_FILES * entry_size)
if entries is None: return 1
string.memset(entries, 0, MAX_SRC_FILES * entry_size)
file_count: int = _ScanDirForPyFiles(mb, source_dir, entries, entry_size, 0, MAX_SRC_FILES)
VLogger.info(f"{file_count} source files in total", "project")
# Scan .s assembly files (separate array; not translated in Phase A, assembled by llvm-mc after Phase B)
asm_entries: SrcFileEntry* = stdlib.malloc(MAX_SRC_FILES * entry_size)
if asm_entries is None:
stdlib.free(entries)
return 1
string.memset(asm_entries, 0, MAX_SRC_FILES * entry_size)
asm_file_count: int = _ScanDirForAsmFiles(mb, source_dir, asm_entries, entry_size, 0, MAX_SRC_FILES)
if asm_file_count > 0: VLogger.info(f"{asm_file_count} assembly files (assembled by llvm-mc and linked)", "project")
if file_count == 0:
stdlib.free(entries)
stdlib.free(asm_entries)
return 1
# Initialize the AST tables
ast._init_tables(mb)
# Pre-register App generic class templates before mark2: the tables/nodes are pool-allocated, so they
# must be registered before mark2 to survive reset_to(mark2); otherwise later passes hit dangling memory.
_PreRegisterAppGenericTemplates(mb, entries, file_count, entry_size)
# OOM cleanup (mirrors Phase1's mark/reset pattern): without reclamation the AST/IR of all includes
# pile up in the pool. mark2 lets each file's transient data be freed by reset_to after translation.
mark2: sizet = mb.mark()
# struct/enum entries registered after mark2 use stdlib allocation (survive reset, as in Phase1)
GenApi.set_stdlib_alloc(1)
# Ensure the temp/output directories exist
BuildPipeline.ensure_dir(temp_dir)
BuildPipeline.ensure_dir(output_dir)
# === 0.5 Fill Sha1Store on demand from the dependency graph (removes the Phase1 dependency) ===
# If Sha1Store is empty, scan the dependency graph (App-first BFS) so later "compile missing includes" works.
if includes_dir is not None:
if StubMergerStore.GetSha1StoreCount() == 0:
rs_05: bytes = stdlib.malloc(StubMerger.MAX_INCLUDES_SHA1 * 17)
if rs_05 is None: VLogger.error("reachable_set allocation failed, cannot fill Sha1Store", "project")
else:
string.memset(rs_05, 0, StubMerger.MAX_INCLUDES_SHA1 * 17)
rc_05: int = StubMergerBuild._BuildReachableSha1Set(mb, source_dir, temp_dir, rs_05, None, includes_dir)
if rc_05 < 0: VLogger.error("dependency graph scan failed, cannot fill Sha1Store", "project")
elif rc_05 > 0: VLogger.info(f"Phase2 dependency graph filled Sha1Store on demand: {rc_05} entries", "project")
stdlib.free(rs_05)
# Append App source SHA1s to _sha1_map.txt (human-readable only; format {sha1}:{rel_path}) and to the
# in-memory store (AppendToSha1MapStore) for cross-module CDefine lookups.
td_len_am: sizet = string.strlen(temp_dir)
src_dir_len_am: sizet = string.strlen(source_dir)
map_path_am: bytes = stdlib.malloc(td_len_am + 32)
if map_path_am is not None:
viperlib.snprintf(map_path_am, td_len_am + 32, "%s/_sha1_map.txt", temp_dir)
mf: fileio.File* = fileio.File(map_path_am, fileio.MODE.A)
line_am: bytes = stdlib.malloc(512)
# First fill the in-memory store (no file I/O), then write _sha1_map.txt
app_filled: int = 0
for i in range(int(file_count)):
ea_am: uint64 = uint64(entries) + i * entry_size
ent_am: SrcFileEntry* = (SrcFileEntry*)(ea_am)
if ent_am is None or ent_am.Path is None or ent_am.Sha1 is None: continue
# Compute the relative path (strip the source_dir prefix, keep the directory structure)
p_str: str = ent_am.Path
p_len: sizet = string.strlen(p_str)
rel_path_am: str = p_str
if p_len > src_dir_len_am + 1:
if string.strncmp(p_str, source_dir, src_dir_len_am) == 0: rel_path_am = p_str + src_dir_len_am + 1
# Also append to the in-memory store (for cross-module CDefine lookups)
# rel_path_am looks like "lib/core/StubMerger.vp", matching _sha1_map.txt
if StubMergerStore.AppendToSha1MapStore(ent_am.Sha1, rel_path_am) != 0: VLogger.error(f"AppendToSha1MapStore failed: {rel_path_am}", "project")
app_filled += 1
# Write _sha1_map.txt (human-readable output)
if not mf.closed and line_am is not None:
viperlib.snprintf(line_am, 512, "%s:%s\n", ent_am.Sha1, rel_path_am)
ll_am: sizet = string.strlen(line_am)
mf.write(line_am, ll_am)
if line_am is not None: stdlib.free(line_am)
if not mf.closed:
mf.close()
VLogger.info(f"appended {app_filled} App files to _sha1_map.txt and the in-memory store", "project")
stdlib.free(map_path_am)
# The module SHA1 map is filled into the StubMerger global store by PopulateSha1MapStore +
# AppendToSha1MapStore; HandlesExprCall._lookup_module_sha1 queries it directly, no local copy needed.
# === Phase A-pre-vpi: pre-generate .vpi stubs for all App sources ===
# Deps' .vpi must exist before their dependents translate; pre-generating avoids missing dep declarations.
td_len_vpa: sizet = string.strlen(temp_dir)
PYI_BUF_SIZE_P2: sizet = 262144
vpi_failed_p2: int = 0
for i_vp2 in range(int(file_count)):
ea_vp2: uint64 = uint64(entries) + i_vp2 * entry_size
ent_vp2: SrcFileEntry* = (SrcFileEntry*)(ea_vp2)
if ent_vp2 is None or ent_vp2.Path is None or ent_vp2.Sha1 is None:
vpi_failed_p2 += 1
VLogger.error("[Phase A-pre-vpi] invalid source entry (Path/Sha1 missing), skipping", "PhaseA")
continue
# Skip if the vpi already exists (sha1 is content-based, so a same-named vpi is valid; cross-run cache)
vpi_path_vp2: str = StubMergerCollect._sliced_path(temp_dir, td_len_vpa, ent_vp2.Sha1, "vpi")
if vpi_path_vp2 is None:
vpi_failed_p2 += 1
VLogger.error(f"[Phase A-pre-vpi] {ent_vp2.Path} | failed to build vpi path", "PhaseA")
continue
vf_vp2: fileio.File* = fileio.File(vpi_path_vp2, fileio.MODE.R)
if not vf_vp2.closed:
vf_vp2.close()
stdlib.free(vpi_path_vp2)
continue
stdlib.free(vpi_path_vp2)
# Read the source -> parse the AST -> generate the vpi (rel_path = ent_vp2.Path: vpi content does
# not depend on it; it is only used for fatal_error attribution, pointing at this source file)
f_vp2: fileio.File* = fileio.File(ent_vp2.Path, fileio.MODE.R)
if f_vp2.closed:
vpi_failed_p2 += 1
VLogger.error(f"[Phase A-pre-vpi] {ent_vp2.Path} | cannot open source file", "PhaseA")
continue
# read_all's capacity includes the terminator (reads at most cap-1 bytes); longer files are truncated
if f_vp2.size() >= SRC_BUF_SIZE:
vpi_failed_p2 += 1
VLogger.error(f"[Phase A-pre-vpi] {ent_vp2.Path} | source file exceeds {SRC_BUF_SIZE} byte limit", "PhaseA")
f_vp2.close()
continue
src_buf_vp2: bytes = stdlib.malloc(SRC_BUF_SIZE)
if src_buf_vp2 is None:
f_vp2.close()
vpi_failed_p2 += 1
VLogger.error(f"[Phase A-pre-vpi] {ent_vp2.Path} | source buffer allocation failed (OOM)", "PhaseA")
continue
bytes_read_vp2: LONG = f_vp2.read_all(src_buf_vp2, SRC_BUF_SIZE)
f_vp2.close()
if bytes_read_vp2 <= 0:
stdlib.free(src_buf_vp2)
vpi_failed_p2 += 1
VLogger.error(f"[Phase A-pre-vpi] {ent_vp2.Path} | file read failed or empty", "PhaseA")
continue
if bytes_read_vp2 < SRC_BUF_SIZE: src_buf_vp2[bytes_read_vp2] = 0
lx_vp2: ast.Lexer* = ast.new_lexer(mb)
if lx_vp2 is None:
stdlib.free(src_buf_vp2)
vpi_failed_p2 += 1
VLogger.error(f"[Phase A-pre-vpi] {ent_vp2.Path} | lexer allocation failed (OOM)", "PhaseA")
continue
ast._lexer_init(lx_vp2, src_buf_vp2, mb)
tokens_vp2: ast.Token* = ast.tokenize(lx_vp2)
tree_vp2: ast.AST* = ast.parse_tokens(mb, tokens_vp2)
if tree_vp2 is None:
stdlib.free(src_buf_vp2)
vpi_failed_p2 += 1
# Parse failure: inline the line/column + reason in the log line (same source as the full
# Python-style traceback below), since "AST parse failed" alone gives no location or cause.
VLogger.error(f"[Phase A-pre-vpi] {ent_vp2.Path} | AST parse failed (line {ast.get_parse_err_line()} col {ast.get_parse_err_col()}: {ast.get_parse_err_msg()})", "PhaseA")
GenApi.dump_parse_traceback(ent_vp2.Path)
continue
pyi_buf_vp2: bytes = stdlib.malloc(PYI_BUF_SIZE_P2)
if pyi_buf_vp2 is not None:
# 文件级名字窗口(FileFuncNames 等)由 _GeneratePyiFromAst 入口统一 reset+重建,
# 不再在此处重复调用;契约见 StructGenMixin.reset_file_name_window。
pyi_pos_vp2: sizet = StubConverter._GeneratePyiFromAst(mb, tree_vp2, ent_vp2.Path, pyi_buf_vp2, PYI_BUF_SIZE_P2)
if pyi_pos_vp2 > 0:
pyi_path_vp2: str = StubMergerCollect._sliced_path(temp_dir, td_len_vpa, ent_vp2.Sha1, "vpi")
if pyi_path_vp2 is not None:
pf_vp2: fileio.File* = fileio.File(pyi_path_vp2, fileio.MODE.W)
if not pf_vp2.closed:
pf_vp2.write(pyi_buf_vp2, pyi_pos_vp2)
pf_vp2.close()
stdlib.free(pyi_path_vp2)
stdlib.free(pyi_buf_vp2)
else:
vpi_failed_p2 += 1
VLogger.error(f"[Phase A-pre-vpi] {ent_vp2.Path} | pyi buffer allocation failed (OOM)", "PhaseA")
stdlib.free(src_buf_vp2)
# OOM cleanup: free this file's AST (the vpi is already on disk; persistent data is stdlib-allocated)
mb.reset_to(mark2)
if vpi_failed_p2 > 0:
VLogger.error(f"Phase A-pre-vpi: {vpi_failed_p2} files failed vpi pre-generation (see the per-file [Phase A-pre-vpi] logs above for failing files and causes)", "PhaseA")
return 1
# === 2. Phase A: generate stub + text for each file ===
# app_deps_buf collects App files' direct deps (space-separated) as Phase B+'s worklist (65536, matching StubMerger).
APP_DEPS_BUF_SIZE: sizet = 65536
app_deps_buf: bytes = None
app_deps_len: sizet = 0
if do_phase1 != 0:
log.banner("Phase A: generate stub + text")
# Initialize the Phase A convergence-failure diagnostic table (stdlib, survives reset)
_PhaseADiagInit(sizet(file_count) + StubMerger.MAX_INCLUDES)
# === Phase A-pre-inc: pre-register includes structs + fill deps ===
# Must run before App pre-registration (multi-pass resolves cross-file inheritance) and fills PopulateIncludesDeps.
if includes_dir is not None and includes_binary_dir is not None:
store_count_inc_pre: int = StubMergerStore.GetSha1StoreCount()
if store_count_inc_pre > 0:
store_sha1_inc_pre: bytes = StubMergerStore.GetSha1StoreArrPtr()
store_rel_inc_pre: bytes = StubMergerStore.GetSha1StoreRelArrPtr()
store_incdir_inc_pre: bytes = StubMergerStore.GetSha1StoreIncdirArrPtr()
deps_filled_pre: int = 0
PHASE_A_INC_MAX_PASSES: int = 3
# Track includes that fail pre-registration: after all passes, any failure aborts the build
# (no silent downgrade). App files must be skipped here; Phase A-pre handles them.
inc_fail_flags: bytes = stdlib.malloc(store_count_inc_pre)
if inc_fail_flags is not None: string.memset(inc_fail_flags, 1, store_count_inc_pre)
for pass_inc_i in range(int(PHASE_A_INC_MAX_PASSES)):
struct_count_before_inc: int = GenApi.get_struct_count()
for si_inc in range(int(store_count_inc_pre)):
inc_sha1_pre: str = store_sha1_inc_pre + sizet(si_inc) * 17
inc_rel_pre: str = store_rel_inc_pre + sizet(si_inc) * StubMerger.MAX_REL_PATH_LEN
if inc_rel_pre is None or inc_rel_pre[0] == '\0': continue
# Skip App source files (main.vp etc.): pre-registered by Phase A-pre, not a failure
is_app_file_inc: int = 0
for i_app_inc in range(int(file_count)):
ea_app_inc: uint64 = uint64(entries) + i_app_inc * entry_size
ent_app_inc: SrcFileEntry* = (SrcFileEntry*)(ea_app_inc)
if ent_app_inc is not None and ent_app_inc.Sha1 is not None and string.strcmp(ent_app_inc.Sha1, inc_sha1_pre) == 0:
is_app_file_inc = 1
break
if is_app_file_inc != 0 and inc_fail_flags is not None:
inc_fail_flags[si_inc] = 0
continue
rel_len_pre: sizet = string.strlen(inc_rel_pre)
# The includes root dir (per-entry from the IncDir array; empty falls back to includes_dir)
incdir_pre: str = store_incdir_inc_pre + sizet(si_inc) * StubMerger.MAX_INC_DIR_LEN
if incdir_pre is None or incdir_pre[0] == '\0': incdir_pre = includes_dir
incdir_len_pre: sizet = string.strlen(incdir_pre)
full_path_pre: bytes = stdlib.malloc(incdir_len_pre + rel_len_pre + 2)
if full_path_pre is None: continue
viperlib.snprintf(full_path_pre, incdir_len_pre + rel_len_pre + 2, "%s/%s", incdir_pre, inc_rel_pre)
pkg_inc_pre: str = GenApi.compute_package_from_relpath(mb, inc_rel_pre)
tr_inc_pre: HandlesTranslator.Translator* = BuildPipeline.TranslateFileGetTrans(mb, full_path_pre, inc_sha1_pre, pkg_inc_pre, 1)
stdlib.free(full_path_pre)
if tr_inc_pre is not None:
# Record this file's deps (for convergence diagnostics; stdlib table survives reset)
_PhaseADiagRecord(inc_sha1_pre, tr_inc_pre._imported_modules, None)
# Pre-registration succeeded: clear the failure flag
if inc_fail_flags is not None: inc_fail_flags[si_inc] = 0
# Fill deps only on the first pass (PopulateIncludesDeps dedupes internally).
# Must be called before reset_to(mark2) since it deep-copies into stdlib.
if pass_inc_i == 0 and tr_inc_pre._imported_modules is not None and StubMergerStore.PopulateIncludesDeps(inc_sha1_pre, tr_inc_pre._imported_modules) == 0:
deps_filled_pre = deps_filled_pre + 1
# OOM cleanup: free this includes file's transient pre-registration data
# (AST/tokens/lexer/IR/Handler); stdlib-allocated struct entries are unaffected
mb.reset_to(mark2)
struct_count_after_inc: int = GenApi.get_struct_count()
# Convergence check: no new struct registered this pass -> all classes are registered
if struct_count_after_inc == struct_count_before_inc: break
# Includes still failing after all passes -> abort (no downgrade)
if inc_fail_flags is not None:
for si_inc_f in range(int(store_count_inc_pre)):
if inc_fail_flags[si_inc_f] != 0:
rel_f: str = store_rel_inc_pre + sizet(si_inc_f) * StubMerger.MAX_REL_PATH_LEN
VLogger.error(f"includes pre-registration failed: {rel_f} (file cannot be translated; possible syntax/parse/compile error), aborting compilation", "prePhaseA")
stdlib.free(inc_fail_flags)
VLogger.info(f"includes pre-registration + dependency fill: {deps_filled_pre}/{store_count_inc_pre}", "prePhaseA")
# === Phase A-pre: pre-register struct/enum/union for all sources ===
# Registers declarations only (declare_only=1), no method bodies; multi-pass resolves cross-file ordering.
VLogger.info("pre-registering struct/enum/union...", "PhaseA")
src_dir_len_pre: sizet = string.strlen(source_dir)
PHASE_A_PRE_MAX_PASSES: int = 3
for pass_i in range(int(PHASE_A_PRE_MAX_PASSES)):
struct_count_before: int = GenApi.get_struct_count()
for i in range(int(file_count)):
ea_pre: uint64 = uint64(entries) + i * entry_size
ent_pre: SrcFileEntry* = (SrcFileEntry*)(ea_pre)
if ent_pre is None or ent_pre.Path is None: continue
pkg_pre: str = None
if string.strlen(ent_pre.Path) > src_dir_len_pre + 1:
rel_path_pre: str = ent_pre.Path + src_dir_len_pre + 1
pkg_pre = GenApi.compute_package_from_relpath(mb, rel_path_pre)
# App pre-registration passes pre_allow_dep_inject=1: imports and dep vpi injection happen at
# declaration time so field annotations share Phase A's context (else struct layouts diverge).
tr_pre: HandlesTranslator.Translator* = BuildPipeline.TranslateFileGetTrans(mb, ent_pre.Path, ent_pre.Sha1, pkg_pre, 1, None, 1)
if tr_pre is None: VLogger.error(f"pre-registration failed: {ent_pre.Path}", "PhaseA")
# OOM cleanup: free this App file's transient pre-registration data
mb.reset_to(mark2)
struct_count_after: int = GenApi.get_struct_count()
# Convergence check: no new struct registered this pass -> all classes are registered
if struct_count_after == struct_count_before: break
VLogger.info("pre-registration complete", "PhaseA")
# Phase A IR buffer: 1MB is not enough, ast/parser.vp's text.ll exceeds 1MB and got truncated ->
# merged IR lost the tail (opt: "end of file in global variable name").
PHASE_A_IR_SIZE: sizet = 8388608
# FULL dump = stub + text (same module), with 2x headroom
PHASE_A_FULL_SIZE: sizet = 16777216
td_len_pa: sizet = string.strlen(temp_dir)
# Set the temp dir so HandlesExprCall can read dependency modules' text.ll for cross-module
# method return-type lookup (avoids pointer returns being truncated to i32)
GenApi.SetTempDir(temp_dir)
# app_deps_buf collects App files' direct dependency modules (space-separated) as Phase B+'s initial
# worklist for _BuildReachableSha1Set; declared outside the if, only assigned here.
app_deps_buf = stdlib.malloc(APP_DEPS_BUF_SIZE)
app_deps_len = 0
if app_deps_buf is not None: app_deps_buf[0] = '\0'
# Phase A multi-pass retry: with several sources, main.vp may translate before its imports
# (filesystem order); failed files are retried next pass, converging in at most file_count passes.
done_flags: bytes = stdlib.malloc(file_count)
if done_flags is None:
VLogger.error("Phase A done_flags allocation failed", "PhaseA")
stdlib.free(entries)
return 1
string.memset(done_flags, 0, file_count)
PHASE_A_MAX_PASSES: int = 32
for pass_a in range(int(PHASE_A_MAX_PASSES)):
completed_this_pass: int = 0
for i in range(int(file_count)):
if done_flags[i] != 0: continue
ea: uint64 = uint64(entries) + i * entry_size
ent: SrcFileEntry* = (SrcFileEntry*)(ea)
# Timing point 0: start processing this file (translate phase start)
t_pa0: uint64 = win32.win32base.GetTickCount64()
if ent is None or ent.Path is None:
done_flags[i] = 1
completed_this_pass += 1
continue
# Compute the package name (the directory part relative to source_dir)
src_dir_len_pa: sizet = string.strlen(source_dir)
pkg_pa: str = None
if string.strlen(ent.Path) > src_dir_len_pa + 1:
rel_path_pa: str = ent.Path + src_dir_len_pa + 1
pkg_pa = GenApi.compute_package_from_relpath(mb, rel_path_pa)
tr_a: HandlesTranslator.Translator* = BuildPipeline.TranslateFileGetTrans(mb, ent.Path, ent.Sha1, pkg_pa, 0, temp_dir)
# Timing point 1: translate returned
t_pa1: uint64 = win32.win32base.GetTickCount64()
if tr_a is None:
# Translation failed (syntax/parse error): retry next pass; convergence check reports it.
# OOM cleanup: a failed translation also produced AST/tokens, freed immediately.
mb.reset_to(mark2)
continue
# Record this file's deps (for convergence diagnostics; stdlib table survives reset)
_PhaseADiagRecord(ent.Sha1, tr_a._imported_modules, None)
# Collect _imported_modules into app_deps_buf (for Phase B+ on-demand compilation).
# Dependencies were filled early during translation.
if tr_a._imported_modules is not None and app_deps_buf is not None:
# Dedupe per module: several App files importing the same module would otherwise inflate
# app_deps toward 64KB, filling the initial worklist and overflowing BFS.
im_len_a: sizet = string.strlen(tr_a._imported_modules)
im_pos_a: sizet = 0
while im_pos_a < im_len_a:
im_start_a: sizet = im_pos_a
while im_pos_a < im_len_a and tr_a._imported_modules[im_pos_a] != ' ': im_pos_a += 1
im_mod_len: sizet = im_pos_a - im_start_a
if im_pos_a < im_len_a: im_pos_a += 1
if im_mod_len == 0: continue
im_mod_buf: bytes = stdlib.malloc(im_mod_len + 1)
if im_mod_buf is None: continue
string.strncpy(im_mod_buf, tr_a._imported_modules + im_start_a, im_mod_len)
im_mod_buf[im_mod_len] = '\0'
# Skip if already in app_deps (exact word-boundary match)
if app_deps_len > 0 and GenApi.is_module_imported(app_deps_buf, im_mod_buf) != 0:
stdlib.free(im_mod_buf)
continue
if app_deps_len + im_mod_len + 2 < APP_DEPS_BUF_SIZE:
if app_deps_len > 0:
app_deps_buf[app_deps_len] = ' '
app_deps_len = app_deps_len + 1
string.strcpy(app_deps_buf + app_deps_len, im_mod_buf)
app_deps_len = app_deps_len + im_mod_len
app_deps_buf[app_deps_len] = '\0'
stdlib.free(im_mod_buf)
# dump stub IR (declarations only)
stub_buf_a: bytes = stdlib.malloc(PHASE_A_IR_SIZE)
if stub_buf_a is None: VLogger.error("stub buffer allocation failed", "PhaseA")
tr_a.dump_ir(stub_buf_a, PHASE_A_IR_SIZE, mlirlite.OUTPUT_STUB)
stub_len_a: sizet = string.strlen(stub_buf_a)
# Save stub.ll (sliced path: temp_dir/{sha1 prefix}/{sha1}.stub.ll)
stub_path_a: str = StubMergerCollect._sliced_path(temp_dir, td_len_pa, ent.Sha1, "stub.ll")
if stub_path_a is not None:
sf_a: fileio.File* = fileio.File(stub_path_a, fileio.MODE.W)
if not sf_a.closed:
sf_a.write(stub_buf_a, stub_len_a)
sf_a.close()
stdlib.free(stub_path_a)
stdlib.free(stub_buf_a)
# dump text IR (definitions only)
text_buf_a: bytes = stdlib.malloc(PHASE_A_IR_SIZE)
if text_buf_a is None: VLogger.error("text buffer allocation failed", "PhaseA")
tr_a.dump_ir(text_buf_a, PHASE_A_IR_SIZE, mlirlite.OUTPUT_TEXT)
text_len_a: sizet = string.strlen(text_buf_a)
# Save text.ll (sliced path)
text_path_a: str = StubMergerCollect._sliced_path(temp_dir, td_len_pa, ent.Sha1, "text.ll")
if text_path_a is not None:
tf_a: fileio.File* = fileio.File(text_path_a, fileio.MODE.W)
if not tf_a.closed:
tf_a.write(text_buf_a, text_len_a)
tf_a.close()
stdlib.free(text_path_a)
stdlib.free(text_buf_a)
# Save {sha1}.meta (structured sidecar: function list / struct layout / vtable)
StubMergerIR.WriteModuleMeta(tr_a.Module, temp_dir, td_len_pa, ent.Sha1)
# Save {sha1}.full.ll (in-memory FULL artifact: a self-contained single-module IR)
full_buf_a: bytes = stdlib.malloc(PHASE_A_FULL_SIZE)
if full_buf_a is None: VLogger.error("full.ll buffer allocation failed", "PhaseA")
tr_a.dump_ir(full_buf_a, PHASE_A_FULL_SIZE, mlirlite.OUTPUT_FULL)
full_len_a: sizet = string.strlen(full_buf_a)
full_path_a: str = StubMergerCollect._sliced_path(temp_dir, td_len_pa, ent.Sha1, "full.ll")
if full_path_a is not None:
ffa: fileio.File* = fileio.File(full_path_a, fileio.MODE.W)
if not ffa.closed:
ffa.write(full_buf_a, full_len_a)
ffa.close()
stdlib.free(full_path_a)
stdlib.free(full_buf_a)
# --traceback: take this module's debug call-site records and write <sha1>.vpcall.
if BuildPipeline._traceback_mode != 0:
vpcall_text_a: str = mlirlite.DebugTakeText()
if vpcall_text_a is not None:
BuildPipeline.WriteCallSidecar(temp_dir, ent.Sha1, vpcall_text_a)
stdlib.free(vpcall_text_a)
# Save dependencies (_imported_modules) for Phase B (sliced path)
deps_path_a: str = StubMergerCollect._sliced_path(temp_dir, td_len_pa, ent.Sha1, "deps.txt")
if deps_path_a is not None:
df_a: fileio.File* = fileio.File(deps_path_a, fileio.MODE.W)
if not df_a.closed:
if tr_a._imported_modules is not None:
dl_a: sizet = string.strlen(tr_a._imported_modules)
df_a.write(tr_a._imported_modules, dl_a)
df_a.close()
stdlib.free(deps_path_a)
# Timing point 2: stub+text dump and write done; print per-file progress (as in Phase1 1b)
t_pa2: uint64 = win32.win32base.GetTickCount64()
VLogger.info(f"[{i+1}/{file_count}] PhaseA: {ent.Path} (translate={t_pa1-t_pa0} ms, dump={t_pa2-t_pa1} ms, total={t_pa2-t_pa0} ms)", "PhaseA")
done_flags[i] = 1
completed_this_pass += 1
# OOM cleanup: stub/text/deps are on disk, so free this file's transient translation data
# (AST/tokens/lexer/IR/Handler); app_deps_buf already deep-copied the deps.
mb.reset_to(mark2)
# Convergence check: stop when all files are done
all_done: int = 1
for i_ck in range(int(file_count)):
if done_flags[i_ck] == 0:
all_done = 0
break
if all_done != 0: break
if completed_this_pass == 0:
# No progress: remaining files cannot finish (real compile errors). Abort and list them, each
# with a dependency-graph dump. VLogger.error calls sys.exit(1), so it must come after all output.
td_len_fail: sizet = string.strlen(temp_dir)
for i_err in range(int(file_count)):
if done_flags[i_err] == 0:
ea_err: uint64 = uint64(entries) + i_err * entry_size
ent_err: SrcFileEntry* = (SrcFileEntry*)(ea_err)
if ent_err is not None and ent_err.Path is not None:
if ent_err.Sha1 is not None: _PhaseADumpFailGraph(temp_dir, td_len_fail, ent_err.Path, ent_err.Sha1)
stdlib.free(done_flags)
_PhaseADiagFree()
VLogger.error("Phase A multi-pass convergence failed, aborting compilation", "PhaseA")
stdlib.free(entries)
return 1
stdlib.free(done_flags)
# Phase A converged: free the diagnostic table (needed only on the failure path)
_PhaseADiagFree()
if do_phase2 == 0:
VLogger.info("Phase A complete (stub generation only)", "project")
if app_deps_buf is not None: stdlib.free(app_deps_buf)
stdlib.free(entries)
return 0
# === 3. Phase B: compile each file to .obj ===
if do_phase2 != 0:
log.banner("Phase B: compile .obj")
# Collect .obj paths (64KB to avoid includes .obj path overflow dropping link symbols).
# reachable_set/reachable_count: filled by Phase B+, used by Phase C for on-demand linking.
reachable_set: bytes = None
reachable_count: int = 0
OBJ_PATHS_SIZE: sizet = 65536
obj_paths: bytes = stdlib.malloc(OBJ_PATHS_SIZE)
if obj_paths is None: return 1
obj_paths[0] = '\0'
obj_pos: sizet = 0
# main_obj_path: the .obj that defines the user main (placed first at link time so the linker
# does not pick another module's wrapper main)
main_obj_path: bytes = stdlib.malloc(512)
if main_obj_path is None: return 1
main_obj_path[0] = '\0'
compiled_count: int = 0
# IR buffer size (16MB, to hold the in-memory FULL artifact {sha1}.full.ll; FULL = stub + text)
COMBINED_IR_SIZE: sizet = 16777216
# llc task table (stdlib.malloc, outlives threads): a contiguous array of LlcTask records; the loop
# only writes .ll + builds tasks, then run_llc_parallel runs llc in parallel batches.
llc_tasks: bytes = stdlib.malloc(sizet(file_count) * BuildPipeline.LlcTask.__sizeof__())
if llc_tasks is None:
VLogger.error("llc task table allocation failed", "PhaseB")
return 1
llc_task_count: int = 0
for i in range(int(file_count)):
ea: uint64 = uint64(entries) + i * entry_size
ent: SrcFileEntry* = (SrcFileEntry*)(ea)
if ent is None or ent.Path is None or ent.Sha1 is None: continue
# Start of this file's three-step timing (GetTickCount64, ms)
t_pb0: uint64 = win32.win32base.GetTickCount64()
# Read the in-memory FULL artifact {sha1}.full.ll -> complete IR (replaces merging stub + dep stub/text)
combined_ir: bytes = stdlib.malloc(COMBINED_IR_SIZE)
if combined_ir is None:
VLogger.error(f"combined_ir allocation failed: {ent.Path}", "PhaseB")
continue
combined_len: sizet = StubMergerBuild.LoadFullIR(temp_dir, ent.Sha1, combined_ir, COMBINED_IR_SIZE)
if combined_len == 0:
VLogger.error(f"failed to read full.ll: {ent.Path}", "PhaseB")
stdlib.free(combined_ir)
continue
# Timing point 1: full.ll read done
t_pb1: uint64 = win32.win32base.GetTickCount64()
# Write .ll (llc threads only read this file); llc itself runs parallel in batches after the loop
ret_w: int = BuildPipeline.write_ir_to_file(combined_ir, combined_len, temp_dir, ent.Sha1)
stdlib.free(combined_ir)
if ret_w != 0:
VLogger.error(f"failed to write .ll: {ent.Path}", "PhaseB")
sys.exit(1)
# Timing point 2: .ll write done
t_pb2: uint64 = win32.win32base.GetTickCount64()
# Build the llc task (command string is stdlib.malloc'd so threads' lifetimes are not reset-reclaimed)
llc_ir_path: str = BuildPipeline.build_sliced_path(temp_dir, ent.Sha1, "ll")
if llc_ir_path is None:
VLogger.error("failed to build llc input path", "PhaseB")
sys.exit(1)
tslot: BuildPipeline.LlcTask* = BuildPipeline.LlcTask*(uint64(llc_tasks) + sizet(llc_task_count) * BuildPipeline.LlcTask.__sizeof__())
if BuildPipeline.init_llc_task(tslot, llc_ir_path, output_dir, ent.Sha1, cc_cmd, cc_flags) != 0:
VLogger.error(f"failed to build llc task: {ent.Path}", "PhaseB")
sys.exit(1)
# Timing point 3: llc task construction done
t_pb3: uint64 = win32.win32base.GetTickCount64()
llc_task_count += 1
compiled_count += 1
# Show files submitted to llc (with the three timings: merge stub / write .ll / build task)
VLogger.info(f"[{i+1}/{file_count}] llc submitted: {ent.Path} (merge={t_pb1-t_pb0} ms, write={t_pb2-t_pb1} ms, task={t_pb3-t_pb2} ms, total={t_pb3-t_pb0} ms)", "PhaseB")
# Build the .obj path and detect the main module (test_main.vp or main.vp; .py no longer recognized)
od_len: sizet = string.strlen(output_dir)
is_main_mod: int = 0
if string.strstr(ent.Path, "test_main.vp") is not None: is_main_mod = 1
elif string.strstr(ent.Path, "main.vp") is not None: is_main_mod = 1
# Sliced path: output_dir/{sha1 prefix}/{sha1}.obj
obj_path_sliced: str = StubMergerCollect._sliced_path(output_dir, od_len, ent.Sha1, "obj")
if obj_path_sliced is not None:
op_sliced_len: sizet = string.strlen(obj_path_sliced)
if is_main_mod != 0:
# main module: copy to main_obj_path (so main is first at link time)
if op_sliced_len < 512: string.strcpy(main_obj_path, obj_path_sliced)
else: VLogger.error("main_obj_path buffer too small, aborting compilation", "PhaseB")
else:
# Other modules: append to obj_paths
if obj_pos + op_sliced_len + 2 < OBJ_PATHS_SIZE:
if obj_pos > 0:
obj_paths[obj_pos] = ' '
obj_pos += 1
string.strcpy(obj_paths + obj_pos, obj_path_sliced)
obj_pos += op_sliced_len
obj_paths[obj_pos] = '\0'
else: VLogger.error(".obj path buffer too small", "PhaseB")
stdlib.free(obj_path_sliced)
# Run all llc jobs in parallel (batched at LLC_MAX_THREADS threads)
ret_p: int = BuildPipeline.run_llc_parallel(llc_tasks, llc_task_count, BuildPipeline.LLC_MAX_THREADS)
if ret_p != 0:
VLogger.error("llc compilation failed, aborting compilation", "PhaseB")
sys.exit(1)
# Free the llc tasks (command strings + records)
ti: int = 0
while ti < llc_task_count:
tfree: BuildPipeline.LlcTask* = BuildPipeline.LlcTask*(uint64(llc_tasks) + sizet(ti) * BuildPipeline.LlcTask.__sizeof__())
BuildPipeline.free_llc_task(tfree)
ti += 1
stdlib.free(llc_tasks)
# Assemble .s files: llvm-mc -> .obj (isr.s etc., bare symbols matching the .vp CExport/State stubs).
# The obj is added to obj_paths for Phase C linking.
od_len_as: sizet = string.strlen(output_dir)
asm_compiled: int = 0
for ai_as in range(int(asm_file_count)):
ea_as: uint64 = uint64(asm_entries) + ai_as * entry_size
ent_as: SrcFileEntry* = (SrcFileEntry*)(ea_as)
if ent_as is None or ent_as.Path is None or ent_as.Sha1 is None: continue
aret_as: int = BuildPipeline.asm_to_obj(ent_as.Path, ent_as.Sha1, output_dir, Config.TargetTriple)
if aret_as != 0:
VLogger.error(f"llvm-mc assembly failed, aborting compilation: {ent_as.Path}", "PhaseB")
sys.exit(1)
obj_path_asm: str = StubMergerCollect._sliced_path(output_dir, od_len_as, ent_as.Sha1, "obj")
if obj_path_asm is not None:
op_len_asm: sizet = string.strlen(obj_path_asm)
if obj_pos + op_len_asm + 2 < OBJ_PATHS_SIZE:
if obj_pos > 0:
obj_paths[obj_pos] = ' '
obj_pos += 1
string.strcpy(obj_paths + obj_pos, obj_path_asm)
obj_pos += op_len_asm
obj_paths[obj_pos] = '\0'
stdlib.free(obj_path_asm)
asm_compiled += 1
VLogger.info(f"[asm {asm_compiled}] assembly complete: {ent_as.Path}", "PhaseB")
VLogger.success(f"compilation complete: {compiled_count}/{file_count}", "PhaseB")
if compiled_count == 0:
VLogger.error("no files compiled successfully", "PhaseB")
stdlib.free(entries)
stdlib.free(asm_entries)
return 1
# === 3.5 Compile missing includes (on demand via the dependency graph) ===
# Only compile includes reached directly/indirectly from App sources (via _BuildReachableSha1Set).
if includes_dir is not None and includes_binary_dir is not None:
inc_compiled: int = 0
td_len_mi: sizet = string.strlen(temp_dir)
# Diagnostic log: confirm the Phase B+ entry and Sha1Store state
VLogger.info(f"PhaseB+ entry: store_count={StubMergerStore.GetSha1StoreCount()} includes_binary_dir={includes_binary_dir}", "PhaseB+")
# Create includes_binary_dir (so .obj files can be written and collect_obj_files can scan them)
BuildPipeline.ensure_dir(includes_binary_dir)
# declare_only preprocessing was moved before Phase A (Phase A-pre-inc) so includes structs are
# registered when Phase A translates App sources. PopulateIncludesDeps is filled there too.
# Build the reachable SHA1 set (dependency graph traversal).
# When app_deps_buf is None, fall back to scanning source_dir (non-recursive).
REACHABLE_SET_SIZE: sizet = sizet(StubMerger.MAX_INCLUDES_SHA1) * 17
reachable_set: bytes = stdlib.malloc(REACHABLE_SET_SIZE)
reachable_count: int = 0
if reachable_set is not None:
string.memset(reachable_set, 0, REACHABLE_SET_SIZE)
reachable_count = StubMergerBuild._BuildReachableSha1Set(mb, source_dir, temp_dir, reachable_set, app_deps_buf, includes_dir)
VLogger.info(f"reachable includes SHA1: {reachable_count} entries", "PhaseB+")
# fast-fail: reachable-set construction failed (deps not found); never fall back, abort
if reachable_count < 0:
VLogger.error("reachable set construction failed (fast-fail), aborting compilation", "PhaseB+")
return 1
else:
VLogger.error("reachable_set allocation failed, aborting compilation", "PhaseB+")
return 1
# Free app_deps_buf (reachable_set is built, no longer needed)
if app_deps_buf is not None:
stdlib.free(app_deps_buf)
app_deps_buf = None
# Get the SHA1/module-name arrays from the global store as local copies (bytes) to avoid address
# issues from passing the store's union return (bytes*) across functions; rel/incdir keep pointers.
td_len_pb: sizet = string.strlen(temp_dir)
store_sha1_arr_mi: bytes = stdlib.malloc(StubMerger.MAX_INCLUDES * 17)
store_mod_arr_mi: bytes = stdlib.malloc(StubMerger.MAX_INCLUDES * 64)
store_count_mi: int = 0
if store_sha1_arr_mi is not None and store_mod_arr_mi is not None:
store_count_mi = StubMergerIR._BuildIncludesSha1Map(temp_dir, td_len_pb, store_sha1_arr_mi, store_mod_arr_mi)
store_rel_arr_mi: bytes = StubMergerStore.GetSha1StoreRelArrPtr()
store_incdir_arr_mi: bytes = StubMergerStore.GetSha1StoreIncdirArrPtr()
if store_count_mi > 0:
# Topologically sort reachable includes: dependencies are fully translated first. Cross-module
# vtable generation needs dependency functions' TypeStr registered globally (e.g. astaux/base AST).
topo_idx: sizet* = (sizet*)(stdlib.malloc(8 * (store_count_mi + 1)))
topo_count: int = 0
if topo_idx is None:
VLogger.error("topological sort index allocation failed, aborting compilation", "PhaseB+")
topo_count = 0
else:
topo_count = StubMergerTopo.BuildTopoSortedIndices(store_count_mi, store_sha1_arr_mi, store_mod_arr_mi, reachable_set, reachable_count, topo_idx)
for si_mi in range(int(topo_count)):
# Current entry (topological index; reachable modules only)
ti: int = int(topo_idx[si_mi])
# Get the current entry's SHA1 and rel_path
inc_sha1_mi: str = store_sha1_arr_mi + sizet(ti) * 17
inc_rel_mi: str = store_rel_arr_mi + sizet(ti) * StubMerger.MAX_REL_PATH_LEN
# Skip App sources: an App file may be scanned as an include and be reachable; compiling
# another .obj into includes_binary would duplicate the output/ .obj's symbols.
app_skip_mi: int = 0
for ai_mi in range(int(file_count)):
ea_ap: uint64 = uint64(entries) + ai_mi * entry_size
ent_ap: SrcFileEntry* = (SrcFileEntry*)(ea_ap)
if ent_ap is not None and ent_ap.Sha1 is not None:
if string.strcmp(ent_ap.Sha1, inc_sha1_mi) == 0:
app_skip_mi = 1
break
if app_skip_mi != 0: continue
# Check whether the .obj already exists in includes.binary (sliced subdir)
ibd_len_mi: sizet = string.strlen(includes_binary_dir)
check_pat_mi: str = StubMergerCollect._sliced_path(includes_binary_dir, ibd_len_mi, inc_sha1_mi, "obj")
check_fd_mi: win32.win32file.WIN32_FIND_DATAA* = stdlib.malloc(win32.win32file.WIN32_FIND_DATAA.__sizeof__())
obj_exists_mi: int = 0
if check_pat_mi is not None and check_fd_mi is not None:
string.memset(check_fd_mi, 0, win32.win32file.WIN32_FIND_DATAA.__sizeof__())
check_h_mi: win32.win32base.HANDLE = win32.win32file.FindFirstFileA(check_pat_mi, check_fd_mi)
if check_h_mi != win32.win32base.INVALID_HANDLE_VALUE:
win32.win32file.FindClose(check_h_mi)
obj_exists_mi = 1
if obj_exists_mi != 0:
if check_pat_mi is not None: stdlib.free(check_pat_mi)
stdlib.free(check_fd_mi)
continue
# The .obj does not exist; compile it.
# Build the source path: {includes root}/{rel_path} (per-entry IncDir; empty falls back to includes_dir)
rel_path_len_mi: sizet = string.strlen(inc_rel_mi)
incdir_mi: str = store_incdir_arr_mi + sizet(ti) * StubMerger.MAX_INC_DIR_LEN
if incdir_mi is None or incdir_mi[0] == '\0': incdir_mi = includes_dir
inc_dir_len_mi: sizet = string.strlen(incdir_mi)
src_fp_mi: bytes = stdlib.malloc(inc_dir_len_mi + 1 + rel_path_len_mi + 1)
if src_fp_mi is None:
stdlib.free(check_pat_mi)
if check_fd_mi is not None: stdlib.free(check_fd_mi)
VLogger.error("includes source path allocation failed, aborting compilation", "PhaseB+")
viperlib.snprintf(src_fp_mi, inc_dir_len_mi + 1 + rel_path_len_mi + 1, "%s/%s", incdir_mi, inc_rel_mi)
# Check whether it is a declaration-only file (read {sha1}.meta: no FUNC ... DEF means pure declaration)
is_decl_mi: int = StubMergerIR.IsDeclOnlyModule(temp_dir, td_len_mi, inc_sha1_mi)
# is_decl_mi: -1 = no meta (Phase1 did not translate it), 1 = declaration-only, 0 = impl.
# Skip only declaration-only (is_decl_mi == 1); -1 and 0 continue to the translation logic.
if is_decl_mi == 1:
stdlib.free(check_pat_mi)
if check_fd_mi is not None: stdlib.free(check_fd_mi)
stdlib.free(src_fp_mi)
continue
# Try to read the in-memory FULL artifact {sha1}.full.ll (should be produced by Phase1/PhaseA)
inc_combined_mi: bytes = stdlib.malloc(COMBINED_IR_SIZE)
inc_combined_len_mi: sizet = 0
if inc_combined_mi is not None: inc_combined_len_mi = StubMergerBuild.LoadFullIR(temp_dir, inc_sha1_mi, inc_combined_mi, COMBINED_IR_SIZE)
# If full.ll is missing, translate the source, save stub/text/meta/full, then retry
if inc_combined_len_mi == 0 and inc_combined_mi is not None:
# Compute the includes file's package name (the directory part relative to includes_dir)
inc_pkg_mi: str = None
if string.strlen(src_fp_mi) > inc_dir_len_mi + 1:
inc_rel_mi2: str = src_fp_mi + inc_dir_len_mi + 1
inc_pkg_mi = GenApi.compute_package_from_relpath(mb, inc_rel_mi2)
tr_mi: HandlesTranslator.Translator* = BuildPipeline.TranslateFileGetTrans(mb, src_fp_mi, inc_sha1_mi, inc_pkg_mi)
if tr_mi is not None:
# PBP IR buffer: same as PHASE_A_IR_SIZE (parser.vp's text.ll >1MB truncation root cause)
PBP_IR_SIZE: sizet = 8388608
# FULL dump = stub + text, with 2x headroom
PBP_FULL_SIZE: sizet = 16777216
# Save stub.ll (sliced path)
inc_stub_buf: bytes = stdlib.malloc(PBP_IR_SIZE)
if inc_stub_buf is not None:
tr_mi.dump_ir(inc_stub_buf, PBP_IR_SIZE, mlirlite.OUTPUT_STUB)
inc_stub_len: sizet = string.strlen(inc_stub_buf)
inc_stub_path: str = StubMergerCollect._sliced_path(temp_dir, td_len_mi, inc_sha1_mi, "stub.ll")
if inc_stub_path is not None:
isf: fileio.File* = fileio.File(inc_stub_path, fileio.MODE.W)
if not isf.closed:
isf.write(inc_stub_buf, inc_stub_len)
isf.close()
stdlib.free(inc_stub_path)
stdlib.free(inc_stub_buf)
# Save text.ll (sliced path)
inc_text_buf: bytes = stdlib.malloc(PBP_IR_SIZE)
if inc_text_buf is not None:
tr_mi.dump_ir(inc_text_buf, PBP_IR_SIZE, mlirlite.OUTPUT_TEXT)
inc_text_len: sizet = string.strlen(inc_text_buf)
inc_text_path: str = StubMergerCollect._sliced_path(temp_dir, td_len_mi, inc_sha1_mi, "text.ll")
if inc_text_path is not None:
itf: fileio.File* = fileio.File(inc_text_path, fileio.MODE.W)
if not itf.closed:
itf.write(inc_text_buf, inc_text_len)
itf.close()
stdlib.free(inc_text_path)
stdlib.free(inc_text_buf)
# Save {sha1}.meta (structured sidecar: function list / struct layout / vtable)
StubMergerIR.WriteModuleMeta(tr_mi.Module, temp_dir, td_len_mi, inc_sha1_mi)
# Save {sha1}.full.ll (in-memory FULL artifact)
inc_full_buf: bytes = stdlib.malloc(PBP_FULL_SIZE)
if inc_full_buf is None: VLogger.error("full.ll buffer allocation failed", "PhaseB+")
tr_mi.dump_ir(inc_full_buf, PBP_FULL_SIZE, mlirlite.OUTPUT_FULL)
inc_full_len: sizet = string.strlen(inc_full_buf)
inc_full_path: str = StubMergerCollect._sliced_path(temp_dir, td_len_mi, inc_sha1_mi, "full.ll")
if inc_full_path is not None:
ifff: fileio.File* = fileio.File(inc_full_path, fileio.MODE.W)
if not ifff.closed:
ifff.write(inc_full_buf, inc_full_len)
ifff.close()
stdlib.free(inc_full_path)
stdlib.free(inc_full_buf)
# --traceback: take this module's debug call-site records and write <sha1>.vpcall.
if BuildPipeline._traceback_mode != 0:
vpcall_text_mi: str = mlirlite.DebugTakeText()
if vpcall_text_mi is not None:
BuildPipeline.WriteCallSidecar(temp_dir, inc_sha1_mi, vpcall_text_mi)
stdlib.free(vpcall_text_mi)
# Retry reading full.ll
inc_combined_len_mi = StubMergerBuild.LoadFullIR(temp_dir, inc_sha1_mi, inc_combined_mi, COMBINED_IR_SIZE)
# OOM cleanup: stub/text/meta/full are on disk, free this includes file's transient
# translation data (AST/tokens/lexer/IR/Handler); reads only touch files.
mb.reset_to(mark2)
if inc_combined_len_mi == 0:
# Diagnostic: check whether full.ll/stub.ll/text.ll exist
diag_full: str = StubMergerCollect._sliced_path(temp_dir, td_len_mi, inc_sha1_mi, "full.ll")
diag_stub: str = StubMergerCollect._sliced_path(temp_dir, td_len_mi, inc_sha1_mi, "stub.ll")
diag_text: str = StubMergerCollect._sliced_path(temp_dir, td_len_mi, inc_sha1_mi, "text.ll")
diag_full_ex: int = 0
diag_stub_ex: int = 0
diag_text_ex: int = 0
if diag_full is not None:
df_diag_f: fileio.File* = fileio.File(diag_full, fileio.MODE.R)
if not df_diag_f.closed:
diag_full_ex = 1
df_diag_f.close()
stdlib.free(diag_full)
if diag_stub is not None:
df_diag_s: fileio.File* = fileio.File(diag_stub, fileio.MODE.R)
if not df_diag_s.closed:
diag_stub_ex = 1
df_diag_s.close()
stdlib.free(diag_stub)
if diag_text is not None:
df_diag_t: fileio.File* = fileio.File(diag_text, fileio.MODE.R)
if not df_diag_t.closed:
diag_text_ex = 1
df_diag_t.close()
stdlib.free(diag_text)
VLogger.error(f"failed to read full.ll: {src_fp_mi} (full={diag_full_ex} stub={diag_stub_ex} text={diag_text_ex} is_decl={is_decl_mi})", "PhaseB+")
if inc_combined_mi is not None: stdlib.free(inc_combined_mi)
stdlib.free(check_pat_mi)
if check_fd_mi is not None: stdlib.free(check_fd_mi)
stdlib.free(src_fp_mi)
continue
# Compile to .obj (output to includes_binary_dir, consistent with the existence check and link collection)
inc_cret_mi: int = BuildPipeline.compile_module_to_obj(inc_combined_mi, inc_combined_len_mi, temp_dir, includes_binary_dir, inc_sha1_mi, cc_cmd, cc_flags)
stdlib.free(inc_combined_mi)
if inc_cret_mi != 0:
VLogger.error(f"llc compilation failed, aborting compilation: {src_fp_mi}", "PhaseB+")
sys.exit(1)
inc_compiled += 1
# Show the compiled includes file
VLogger.info(f"[includes {inc_compiled}] compilation complete: {src_fp_mi}", "PhaseB+")
# Add to obj_paths (sliced path, taken from includes_binary_dir)
ibd_len_mi2: sizet = string.strlen(includes_binary_dir)
inc_obj_sliced: str = StubMergerCollect._sliced_path(includes_binary_dir, ibd_len_mi2, inc_sha1_mi, "obj")
if inc_obj_sliced is not None:
inc_obj_len: sizet = string.strlen(inc_obj_sliced)
if obj_pos + inc_obj_len + 2 < OBJ_PATHS_SIZE:
if obj_pos > 0:
obj_paths[obj_pos] = ' '
obj_pos += 1
string.strcpy(obj_paths + obj_pos, inc_obj_sliced)
obj_pos += inc_obj_len
obj_paths[obj_pos] = '\0'
else: VLogger.error("includes .obj path buffer overflow, aborting compilation", "PhaseB+")
stdlib.free(inc_obj_sliced)
# Free this iteration's temporary memory
if check_pat_mi is not None: stdlib.free(check_pat_mi)
if check_fd_mi is not None: stdlib.free(check_fd_mi)
stdlib.free(src_fp_mi)
# Free the topological indices and local map copies
if topo_idx is not None: stdlib.free(topo_idx)
if store_sha1_arr_mi is not None: stdlib.free(store_sha1_arr_mi)
if store_mod_arr_mi is not None: stdlib.free(store_mod_arr_mi)
VLogger.info(f"compiled missing includes: {inc_compiled} entries", "PhaseB+")
# Free app_deps_buf (fallback when the Phase B+ block did not run)
if app_deps_buf is not None:
stdlib.free(app_deps_buf)
app_deps_buf = None
# === 4. Phase C: link all .obj -> .exe ===
log.banner("Phase C: link")
od_len2: sizet = string.strlen(output_dir)
lo_len: sizet = string.strlen(linker_output)
exe_path: bytes = stdlib.malloc(od_len2 + lo_len + 2)
if exe_path is not None: viperlib.snprintf(exe_path, od_len2 + lo_len + 2, "%s/%s", output_dir, linker_output)
else:
exe_path = linker_output
VLogger.error("exe_path allocation failed, aborting compilation", "PhaseC")
# Build the final .obj path list: main_obj_path first, reachable .objs after
final_obj_paths: bytes = stdlib.malloc(OBJ_PATHS_SIZE + 512)
if final_obj_paths is None:
stdlib.free(entries)
return 1
final_obj_paths[0] = '\0'
fop_pos: sizet = 0
# main_obj_path goes first (so the linker prefers the user main)
if main_obj_path[0] != '\0':
mlen: sizet = string.strlen(main_obj_path)
string.strcpy(final_obj_paths, main_obj_path)
fop_pos = mlen
# Merge the App non-main modules' .obj (obj_paths, space-separated) compiled by Phase B/B+.
# final_obj_paths must include main_obj + reachable includes + App deps, else linking hits undefined symbols.
if obj_pos > 0:
if fop_pos + obj_pos + 1 < OBJ_PATHS_SIZE + 512:
if fop_pos > 0:
final_obj_paths[fop_pos] = ' '
fop_pos += 1
string.strcpy(final_obj_paths + fop_pos, obj_paths)
fop_pos += obj_pos
final_obj_paths[fop_pos] = '\0'
# Link by graph: iterate reachable_set and collect only reachable includes' .obj,
# no longer collecting every .obj under includes.binary.
if reachable_set is not None and reachable_count > 0:
ibd_len_pc: sizet = string.strlen(includes_binary_dir)
td_len_pc: sizet = string.strlen(temp_dir)
ri_pc: int
for ri_pc in range(int(reachable_count)):
rsha1_pc: str = reachable_set + sizet(ri_pc) * 17
if rsha1_pc[0] == '\0': continue
# Skip App sources: their .obj is already in final_obj_paths; collecting a same-sha1 .obj from includes_binary would duplicate symbols.
app_skip_pc: int = 0
for ai_pc in range(int(file_count)):
ea_pc: uint64 = uint64(entries) + ai_pc * entry_size
ent_pc: SrcFileEntry* = (SrcFileEntry*)(ea_pc)
if ent_pc is not None and ent_pc.Sha1 is not None:
if string.strcmp(ent_pc.Sha1, rsha1_pc) == 0:
app_skip_pc = 1
break
if app_skip_pc != 0: continue
obj_path_pc: str = StubMergerCollect._sliced_path(includes_binary_dir, ibd_len_pc, rsha1_pc, "obj")
if obj_path_pc is not None:
# Check whether the .obj file exists
cf_pc: fileio.File* = fileio.File(obj_path_pc, fileio.MODE.R)
if not cf_pc.closed:
cf_pc.close()
# The .obj exists; append to final_obj_paths
op_len_pc: sizet = string.strlen(obj_path_pc)
if fop_pos + op_len_pc + 2 < OBJ_PATHS_SIZE + 512:
if fop_pos > 0:
final_obj_paths[fop_pos] = ' '
fop_pos += 1
string.strcpy(final_obj_paths + fop_pos, obj_path_pc)
fop_pos += op_len_pc
final_obj_paths[fop_pos] = '\0'
else:
# .obj missing: a declaration-only module (no define lines) needs no .obj, so skip silently;
# under phase2command mode the missing .obj is expected and the path is still added.
if BuildPipeline._phase2_cmd_path is not None:
op_len_p2: sizet = string.strlen(obj_path_pc)
if fop_pos + op_len_p2 + 2 < OBJ_PATHS_SIZE + 512:
if fop_pos > 0:
final_obj_paths[fop_pos] = ' '
fop_pos += 1
string.strcpy(final_obj_paths + fop_pos, obj_path_pc)
fop_pos += op_len_p2
final_obj_paths[fop_pos] = '\0'
elif StubMergerIR.IsDeclOnlyModule(temp_dir, td_len_pc, rsha1_pc) != 1: VLogger.error(f"reachable module .obj missing: {obj_path_pc}", "PhaseC")
stdlib.free(obj_path_pc)
final_obj_paths[fop_pos] = '\0'
obj_paths_len: sizet = fop_pos
# Free reachable_set (Phase C is done with it)
if reachable_set is not None:
stdlib.free(reachable_set)
reachable_set = None
# Link by graph: pass None for includes_binary_dir to avoid collecting everything
lret: int = BuildPipeline.link_objs_to_exe(final_obj_paths, obj_paths_len, linker_cmd, linker_flags, exe_path, None, linker_static_libs, linker_dynamic_libs)
if lret == 0:
VLogger.success(f"output: {exe_path}", "PhaseC")
# Deploy declared runtime deps (project.vpj runtime.*) next to the exe; fail-fast on error.
# Skipped under --phase2command (the exe is not linked yet).
if BuildPipeline._phase2_cmd_path is None:
if BuildPipeline.deploy_runtime_deps(exe_path) != 0:
VLogger.error("runtime dependency deployment failed, aborting compilation", "runtime")
stdlib.free(entries)
stdlib.free(asm_entries)
return 1
# --traceback: export <exe>.stackmap (skipped under --phase2command,
# where nothing was actually linked).
if BuildPipeline._traceback_mode != 0 and BuildPipeline._phase2_cmd_path is None: BuildPipeline.WriteStackMap(mb, exe_path)
if args.get_bool("run") and BuildPipeline._phase2_cmd_path is not None:
# phase2command mode: the exe is not linked yet (the command is in the bat); skip --run
VLogger.info("phase2command mode: skipping --run (exe not linked yet)", "phase2cmd")
elif args.get_bool("run"):
VLogger.info(f"run: {exe_path}", "run")
rp: subprocess.CompletedProcess* = subprocess.run(exe_path, False, False)
if rp is not None: VLogger.info(f"exit code: {rp.returncode}", "run")
else:
VLogger.error("link failed", "PhaseC")
stdlib.free(entries)
stdlib.free(asm_entries)
return 1
stdlib.free(entries)
stdlib.free(asm_entries)
return 0