1538 lines
85 KiB
Plaintext
1538 lines
85 KiB
Plaintext
import lib.core.Gen.GenApi as GenApi
|
|
import t, c
|
|
from stdint import *
|
|
import stdio
|
|
import string
|
|
import stdlib
|
|
import memhub
|
|
import viperlib
|
|
import win32.fileio as fileio
|
|
import win32.win32file
|
|
import win32.win32base
|
|
import sys
|
|
import ast
|
|
import mlirlite
|
|
import subprocess
|
|
import argparse
|
|
import lib.core.VLogger as VLogger
|
|
import lib.core.Gen.Translator as HandlesTranslator
|
|
import lib.Phase.BuildPipeline as BuildPipeline
|
|
import lib.core.IncludesScanner as IncludesScanner
|
|
import lib.core.CompilerFactory as CompilerFactory
|
|
import lib.Stub.StubMerger as StubMerger
|
|
import lib.Projectrans.Utils as Utils
|
|
import lib.Projectrans.Config as Config
|
|
import lib.Stub.StubMerger.StubMergerBuild as StubMergerBuild
|
|
import lib.Stub.StubMerger.StubMergerCollect as StubMergerCollect
|
|
import lib.Stub.StubMerger.StubMergerIR as StubMergerIR
|
|
import lib.Stub.StubMerger.StubMergerStore as StubMergerStore
|
|
import lib.Stub.StubMerger.StubMergerTopo as StubMergerTopo
|
|
import lib.Stub.StubGen.Converter as StubConverter
|
|
|
|
# Global mbuddy pointer
|
|
_mbuddy: memhub.MemBuddy*
|
|
|
|
# Source buffer size (1MB)
|
|
define SRC_BUF_SIZE = 1048576
|
|
|
|
# Max source files (recursive scanning grows the count; too small truncates the list -> undefined symbol)
|
|
define MAX_SRC_FILES = 128
|
|
|
|
@t.NoVTable
|
|
class SrcFileEntry:
|
|
"""Source file entry"""
|
|
Path: str # full path
|
|
Sha1: str # first 16 chars of SHA1
|
|
|
|
# _ScanDirForPyFiles - recursively scan a dir and collect .vp/.py files into entries, returning the new count.
|
|
# Args: mb pool; dir_path; entries SrcFileEntry array; entry_size; file_count; max_files.
|
|
def _ScanDirForPyFiles(mb: memhub.MemBuddy*, dir_path: str, entries: SrcFileEntry*, entry_size: sizet, file_count: int, max_files: int) -> int:
|
|
"""Recursively scan a directory, collect .vp/.py files into entries, return the new file_count"""
|
|
if dir_path is None or entries is None: return file_count
|
|
if file_count >= max_files: return file_count
|
|
|
|
dir_len: sizet = string.strlen(dir_path)
|
|
pattern: bytes = stdlib.malloc(dir_len + 16)
|
|
if pattern is None: return file_count
|
|
viperlib.snprintf(pattern, dir_len + 16, "%s/*", dir_path)
|
|
|
|
find_data_size: sizet = win32.win32file.WIN32_FIND_DATAA.__sizeof__()
|
|
find_data: win32.win32file.WIN32_FIND_DATAA* = stdlib.malloc(find_data_size + 16)
|
|
if find_data is None:
|
|
stdlib.free(pattern)
|
|
return file_count
|
|
string.memset(find_data, 0, find_data_size + 16)
|
|
|
|
handle: win32.win32base.HANDLE = win32.win32file.FindFirstFileA(pattern, find_data)
|
|
if handle == win32.win32base.INVALID_HANDLE_VALUE:
|
|
stdlib.free(pattern)
|
|
stdlib.free(find_data)
|
|
return file_count
|
|
|
|
while True:
|
|
fname: str = find_data.cFileName
|
|
if fname is None:
|
|
if win32.win32file.FindNextFileA(handle, find_data) == 0: break
|
|
continue
|
|
fname_len: sizet = string.strlen(fname)
|
|
if fname_len <= 0:
|
|
if win32.win32file.FindNextFileA(handle, find_data) == 0: break
|
|
continue
|
|
# Skip . and ..
|
|
is_dot: int = 0
|
|
if fname_len == 1 and fname[0] == '.': is_dot = 1
|
|
elif fname_len == 2 and fname[0] == '.' and fname[1] == '.': is_dot = 1
|
|
if is_dot != 0:
|
|
if win32.win32file.FindNextFileA(handle, find_data) == 0: break
|
|
continue
|
|
# Check whether it is a directory
|
|
attrs: ULONG = find_data.dwFileAttributes
|
|
is_dir: int = 0
|
|
if (attrs & win32.win32file.FILE_ATTRIBUTE_DIRECTORY) != 0: is_dir = 1
|
|
if is_dir != 0:
|
|
# Recurse into the subdirectory
|
|
sub_dir: bytes = Utils.JoinPath(dir_path, fname)
|
|
if sub_dir is not None:
|
|
file_count = _ScanDirForPyFiles(mb, sub_dir, entries, entry_size, file_count, max_files)
|
|
stdlib.free(sub_dir)
|
|
if win32.win32file.FindNextFileA(handle, find_data) == 0: break
|
|
continue
|
|
# Check for a .vp source file (Viper sources are only .vp now; .py is no longer recognized)
|
|
if fname_len > 3 and Utils.is_vp_at(fname, 0, fname_len) != 0 and file_count < max_files:
|
|
full_path: bytes = Utils.JoinPath(dir_path, fname)
|
|
if full_path is not None:
|
|
sbuf: bytes = Utils.read_file_to_buf(full_path, SRC_BUF_SIZE)
|
|
if sbuf is None:
|
|
if win32.win32file.FindNextFileA(handle, find_data) == 0: break
|
|
continue
|
|
# Use IncludesScanner.compute_file_sha1 (CRLF->LF normalized) as StubMerger does for the same
|
|
# App sources; raw-byte hashing makes stub/define symbol prefixes differ -> undefined reference.
|
|
sha1_val: str = IncludesScanner.compute_file_sha1(mb, full_path)
|
|
if sha1_val is not None:
|
|
ea: uint64 = uint64(entries) + file_count * entry_size
|
|
ent: SrcFileEntry* = (SrcFileEntry*)(ea)
|
|
if ent is not None:
|
|
ent.Path = full_path
|
|
ent.Sha1 = sha1_val
|
|
file_count += 1
|
|
|
|
VLogger.info(f"{fname} (sha1={sha1_val})", "project")
|
|
stdlib.free(sbuf)
|
|
# full_path is not freed: ent.Path references it
|
|
|
|
if win32.win32file.FindNextFileA(handle, find_data) == 0: break
|
|
|
|
win32.win32file.FindClose(handle)
|
|
stdlib.free(pattern)
|
|
stdlib.free(find_data)
|
|
return file_count
|
|
|
|
|
|
# _ScanDirForAsmFiles - recursively scan a dir and collect .s assembly files.
|
|
# Like _ScanDirForPyFiles; VPC does not translate .s but assembles it to .obj via llvm-mc for linking.
|
|
def _ScanDirForAsmFiles(mb: memhub.MemBuddy*, dir_path: str, entries: SrcFileEntry*, entry_size: sizet, file_count: int, max_files: int) -> int:
|
|
"""Recursively scan a directory, collect .s assembly files into entries, return the new file_count"""
|
|
if dir_path is None or entries is None: return file_count
|
|
if file_count >= max_files: return file_count
|
|
|
|
dir_len: sizet = string.strlen(dir_path)
|
|
pattern: bytes = stdlib.malloc(dir_len + 16)
|
|
if pattern is None: return file_count
|
|
viperlib.snprintf(pattern, dir_len + 16, "%s/*", dir_path)
|
|
|
|
find_data_size: sizet = win32.win32file.WIN32_FIND_DATAA.__sizeof__()
|
|
find_data: win32.win32file.WIN32_FIND_DATAA* = stdlib.malloc(find_data_size + 16)
|
|
if find_data is None:
|
|
stdlib.free(pattern)
|
|
return file_count
|
|
string.memset(find_data, 0, find_data_size + 16)
|
|
|
|
handle: win32.win32base.HANDLE = win32.win32file.FindFirstFileA(pattern, find_data)
|
|
if handle == win32.win32base.INVALID_HANDLE_VALUE:
|
|
stdlib.free(pattern)
|
|
stdlib.free(find_data)
|
|
return file_count
|
|
|
|
while True:
|
|
fname: str = find_data.cFileName
|
|
if fname is None:
|
|
if win32.win32file.FindNextFileA(handle, find_data) == 0: break
|
|
continue
|
|
fname_len: sizet = string.strlen(fname)
|
|
if fname_len <= 0:
|
|
if win32.win32file.FindNextFileA(handle, find_data) == 0: break
|
|
continue
|
|
# Skip . and ..
|
|
is_dot: int = 0
|
|
if fname_len == 1 and fname[0] == '.': is_dot = 1
|
|
elif fname_len == 2 and fname[0] == '.' and fname[1] == '.': is_dot = 1
|
|
if is_dot != 0:
|
|
if win32.win32file.FindNextFileA(handle, find_data) == 0: break
|
|
continue
|
|
# Check whether it is a directory
|
|
attrs: ULONG = find_data.dwFileAttributes
|
|
is_dir: int = 0
|
|
if (attrs & win32.win32file.FILE_ATTRIBUTE_DIRECTORY) != 0: is_dir = 1
|
|
if is_dir != 0:
|
|
sub_dir: bytes = Utils.JoinPath(dir_path, fname)
|
|
if sub_dir is not None:
|
|
file_count = _ScanDirForAsmFiles(mb, sub_dir, entries, entry_size, file_count, max_files)
|
|
stdlib.free(sub_dir)
|
|
if win32.win32file.FindNextFileA(handle, find_data) == 0: break
|
|
continue
|
|
# Check for the .s suffix (assembly source)
|
|
is_asm: int = 0
|
|
if fname_len > 2 and fname[fname_len - 2] == '.' and fname[fname_len - 1] == 's': is_asm = 1
|
|
if is_asm != 0 and file_count < max_files:
|
|
full_path: bytes = Utils.JoinPath(dir_path, fname)
|
|
if full_path is not None:
|
|
# Same SHA1 convention as .vp (CRLF->LF normalized), used for the sliced path
|
|
sha1_val: str = IncludesScanner.compute_file_sha1(mb, full_path)
|
|
if sha1_val is not None:
|
|
ea: uint64 = uint64(entries) + file_count * entry_size
|
|
ent: SrcFileEntry* = (SrcFileEntry*)(ea)
|
|
if ent is not None:
|
|
ent.Path = full_path
|
|
ent.Sha1 = sha1_val
|
|
file_count += 1
|
|
|
|
VLogger.info(f"{fname} (asm sha1={sha1_val})", "project")
|
|
# full_path is not freed: ent.Path references it
|
|
|
|
if win32.win32file.FindNextFileA(handle, find_data) == 0: break
|
|
|
|
win32.win32file.FindClose(handle)
|
|
stdlib.free(pattern)
|
|
stdlib.free(find_data)
|
|
return file_count
|
|
|
|
|
|
# _PreRegisterAppGenericTemplates - pre-register App generic class templates before mark2.
|
|
# The generic tables live in the pool; registering after mark2 would be reclaimed by reset_to, corrupting later passes.
|
|
def _PreRegisterAppGenericTemplates(mb: memhub.MemBuddy*, entries: SrcFileEntry*, file_count: int, entry_size: sizet) -> int:
|
|
"""Pre-register App source generic class templates before mark2, return the processed count"""
|
|
if mb is None or entries is None or file_count <= 0: return 0
|
|
handled: int = 0
|
|
for i in range(int(file_count)):
|
|
ea_pa: uint64 = uint64(entries) + i * entry_size
|
|
ent_pa: SrcFileEntry* = (SrcFileEntry*)(ea_pa)
|
|
if ent_pa is None or ent_pa.Path is None or ent_pa.Sha1 is None: continue
|
|
# Read the file
|
|
f_pa: fileio.File* = fileio.File(ent_pa.Path, fileio.MODE.R)
|
|
if f_pa.closed: continue
|
|
# read_all's capacity includes the terminator (reads at most cap-1 bytes); longer files are truncated
|
|
if f_pa.size() >= SRC_BUF_SIZE:
|
|
VLogger.error(f"source file exceeds {SRC_BUF_SIZE} byte limit, refusing to read: {ent_pa.Path}", "PhaseA")
|
|
f_pa.close()
|
|
continue
|
|
src_buf_pa: bytes = stdlib.malloc(SRC_BUF_SIZE)
|
|
if src_buf_pa is None:
|
|
f_pa.close()
|
|
continue
|
|
bytes_read_pa: LONG = f_pa.read_all(src_buf_pa, SRC_BUF_SIZE)
|
|
f_pa.close()
|
|
if bytes_read_pa <= 0:
|
|
stdlib.free(src_buf_pa)
|
|
continue
|
|
if bytes_read_pa < SRC_BUF_SIZE: src_buf_pa[bytes_read_pa] = 0
|
|
# Parse the AST only (perm pool keeps template nodes alive across the mark2 reset)
|
|
perm_pa: memhub.MemBuddy* = CompilerFactory.factory.PermPool
|
|
if perm_pa is None: perm_pa = mb
|
|
lx_pa: ast.Lexer* = ast.new_lexer(perm_pa)
|
|
if lx_pa is None:
|
|
stdlib.free(src_buf_pa)
|
|
continue
|
|
ast._lexer_init(lx_pa, src_buf_pa, perm_pa)
|
|
tokens_pa: ast.Token* = ast.tokenize(lx_pa)
|
|
tree_pa: ast.AST* = ast.parse_tokens(perm_pa, tokens_pa)
|
|
if tree_pa is None:
|
|
stdlib.free(src_buf_pa)
|
|
continue
|
|
# Walk top-level ClassDef/FunctionDef and register generic templates (type_params != None)
|
|
ch_pa: list[ast.AST*]* = tree_pa.children
|
|
if ch_pa is not None:
|
|
cn_pa: sizet = len(ch_pa)
|
|
for ci_pa in range(int(cn_pa)):
|
|
child_pa: ast.AST* = ch_pa.get(ci_pa)
|
|
if child_pa is None: continue
|
|
ck_pa: int = child_pa.kind()
|
|
if ck_pa == ast.ASTKind.ClassDef:
|
|
cd_pa: ast.ClassDef* = ast.ClassDef*(child_pa)
|
|
if cd_pa is None or cd_pa.name is None: continue
|
|
tp_pa: list[str]* = cd_pa.type_params
|
|
if tp_pa is None: continue
|
|
GenApi.set_current_file_type(ent_pa.Path)
|
|
GenApi._register_generic_template(perm_pa, cd_pa, ent_pa.Sha1)
|
|
handled += 1
|
|
elif ck_pa == ast.ASTKind.FunctionDef:
|
|
fd_pa: ast.FunctionDef* = ast.FunctionDef*(child_pa)
|
|
if fd_pa is None or fd_pa.name is None: continue
|
|
tf_pa: list[str]* = fd_pa.type_params
|
|
if tf_pa is None: continue
|
|
GenApi.set_current_file_type(ent_pa.Path)
|
|
GenApi._register_generic_func_template(perm_pa, fd_pa, ent_pa.Sha1)
|
|
handled += 1
|
|
stdlib.free(src_buf_pa)
|
|
if handled > 0: VLogger.info(f"pre-registered {handled} App generic templates before mark2 (classes + functions)", "PhaseA")
|
|
return handled
|
|
|
|
|
|
# Phase A convergence-failure dependency-graph diagnostics.
|
|
# Each file's deps and missing details are persisted to a stdlib table (survives reset) and dumped on failure.
|
|
PA_DEPS_MAX_STR: sizet = 1024
|
|
PA_MISS_MAX_STR: sizet = 768
|
|
PA_GRAPH_DEPTH_MAX: int = 3 # missing-chain drill-down depth (counts from layer 1)
|
|
PA_GRAPH_OK_PROBE: int = 2 # ready nodes are probed 1 more level (to find hidden misses)
|
|
_pa_diag_max: sizet = 0
|
|
_pa_diag_count: int = 0
|
|
_pa_diag_sha1: bytes = None
|
|
_pa_diag_deps: bytes = None
|
|
_pa_diag_miss: bytes = None
|
|
|
|
|
|
def _PhaseADiagInit(max_count: sizet) -> None:
|
|
"""Allocate the persistent Phase A convergence-failure diagnostic table"""
|
|
global _pa_diag_max, _pa_diag_count, _pa_diag_sha1, _pa_diag_deps, _pa_diag_miss
|
|
_pa_diag_max = max_count
|
|
_pa_diag_count = 0
|
|
_pa_diag_sha1 = stdlib.malloc(max_count * 17)
|
|
_pa_diag_deps = stdlib.malloc(max_count * PA_DEPS_MAX_STR)
|
|
_pa_diag_miss = stdlib.malloc(max_count * PA_MISS_MAX_STR)
|
|
if _pa_diag_sha1 is not None: string.memset(_pa_diag_sha1, 0, max_count * 17)
|
|
if _pa_diag_deps is not None: string.memset(_pa_diag_deps, 0, max_count * PA_DEPS_MAX_STR)
|
|
if _pa_diag_miss is not None: string.memset(_pa_diag_miss, 0, max_count * PA_MISS_MAX_STR)
|
|
|
|
|
|
def _PhaseADiagFree() -> None:
|
|
"""Free the persistent diagnostic table"""
|
|
global _pa_diag_max, _pa_diag_count, _pa_diag_sha1, _pa_diag_deps, _pa_diag_miss
|
|
if _pa_diag_sha1 is not None:
|
|
stdlib.free(_pa_diag_sha1)
|
|
_pa_diag_sha1 = None
|
|
if _pa_diag_deps is not None:
|
|
stdlib.free(_pa_diag_deps)
|
|
_pa_diag_deps = None
|
|
if _pa_diag_miss is not None:
|
|
stdlib.free(_pa_diag_miss)
|
|
_pa_diag_miss = None
|
|
_pa_diag_count = 0
|
|
_pa_diag_max = 0
|
|
|
|
|
|
def _PhaseADiagRecord(sha1: str, deps: str, miss: str) -> None:
|
|
"""Record or update one file's diagnostics: sha1 to deps plus missing detail"""
|
|
global _pa_diag_count
|
|
if _pa_diag_sha1 is None or _pa_diag_deps is None or sha1 is None: return
|
|
if string.strlen(sha1) < 16: return
|
|
for i in range(int(_pa_diag_count)):
|
|
es: bytes = _pa_diag_sha1 + sizet(i) * 17
|
|
if string.strcmp(es, sha1) == 0:
|
|
if miss is not None and _pa_diag_miss is not None:
|
|
maddr: bytes = _pa_diag_miss + sizet(i) * PA_MISS_MAX_STR
|
|
string.strncpy(maddr, miss, PA_MISS_MAX_STR - 1)
|
|
maddr[PA_MISS_MAX_STR - 1] = 0
|
|
return
|
|
if _pa_diag_count >= _pa_diag_max: return
|
|
saddr: bytes = _pa_diag_sha1 + sizet(_pa_diag_count) * 17
|
|
string.strncpy(saddr, sha1, 16)
|
|
saddr[16] = 0
|
|
daddr: bytes = _pa_diag_deps + sizet(_pa_diag_count) * PA_DEPS_MAX_STR
|
|
if deps is not None: string.strncpy(daddr, deps, PA_DEPS_MAX_STR - 1)
|
|
daddr[PA_DEPS_MAX_STR - 1] = 0
|
|
if miss is not None and _pa_diag_miss is not None:
|
|
maddr2: bytes = _pa_diag_miss + sizet(_pa_diag_count) * PA_MISS_MAX_STR
|
|
string.strncpy(maddr2, miss, PA_MISS_MAX_STR - 1)
|
|
maddr2[PA_MISS_MAX_STR - 1] = 0
|
|
_pa_diag_count += 1
|
|
|
|
|
|
def _PhaseADiagDepsLookup(sha1: str) -> str:
|
|
"""Look up the persisted deps string by sha1; caller frees the copy, None if absent"""
|
|
if _pa_diag_sha1 is None or _pa_diag_deps is None or sha1 is None: return None
|
|
for i in range(int(_pa_diag_count)):
|
|
es: bytes = _pa_diag_sha1 + sizet(i) * 17
|
|
if string.strcmp(es, sha1) == 0:
|
|
src: bytes = _pa_diag_deps + sizet(i) * PA_DEPS_MAX_STR
|
|
ln: sizet = string.strlen(src)
|
|
out: bytes = stdlib.malloc(ln + 1)
|
|
if out is None: return None
|
|
string.strcpy(out, src)
|
|
return out
|
|
return None
|
|
|
|
|
|
def _PhaseADiagMissLookup(sha1: str) -> str:
|
|
"""Look up the persisted missing detail by sha1; caller frees the copy, None if absent"""
|
|
if _pa_diag_sha1 is None or _pa_diag_miss is None or sha1 is None: return None
|
|
for i in range(int(_pa_diag_count)):
|
|
es: bytes = _pa_diag_sha1 + sizet(i) * 17
|
|
if string.strcmp(es, sha1) == 0:
|
|
src: bytes = _pa_diag_miss + sizet(i) * PA_MISS_MAX_STR
|
|
ln: sizet = string.strlen(src)
|
|
if ln == 0: return None
|
|
out: bytes = stdlib.malloc(ln + 1)
|
|
if out is None: return None
|
|
string.strcpy(out, src)
|
|
return out
|
|
return None
|
|
|
|
|
|
def _PhaseATextLLExists(temp_dir: str, td_len: sizet, sha1: str) -> int:
|
|
"""Probe whether {temp_dir}/{sha1 first 2 chars}/{sha1}.text.ll exists, 1=yes"""
|
|
if temp_dir is None or td_len == 0 or sha1 is None or string.strlen(sha1) < 2: return 0
|
|
tpath: str = StubMergerCollect._sliced_path(temp_dir, td_len, sha1, "text.ll")
|
|
if tpath is None: return 0
|
|
tf: fileio.File* = fileio.File(tpath, fileio.MODE.R)
|
|
exists: int = 0
|
|
if not tf.closed:
|
|
tf.close()
|
|
exists = 1
|
|
stdlib.free(tpath)
|
|
return exists
|
|
|
|
|
|
def _PhaseAResolveDepSha1(mod_name: str) -> str:
|
|
"""Resolve a dependency module name to SHA1 (exact then suffix), None if absent"""
|
|
if mod_name is None: return None
|
|
s1: str = StubMergerStore.LookupSha1ByModName(mod_name)
|
|
if s1 is not None: return s1
|
|
return StubMergerStore.LookupSha1ByModNameSuffix(mod_name)
|
|
|
|
|
|
def _PhaseADumpNode(temp_dir: str, td_len: sizet, sha1: str, depth: int, prefix: bytes) -> None:
|
|
"""Recursively print the dependency subgraph; [MISS]=missing text.ll, [OK]=ready"""
|
|
deps: bytes = None
|
|
if sha1 is not None: deps = _PhaseADiagDepsLookup(sha1)
|
|
if deps is None and sha1 is not None:
|
|
# Fallback: read {temp_dir}/{prefix}/{sha1}.deps.txt (Phase1 already wrote it for includes)
|
|
dpath: str = StubMergerCollect._sliced_path(temp_dir, td_len, sha1, "deps.txt")
|
|
if dpath is not None:
|
|
df: fileio.File* = fileio.File(dpath, fileio.MODE.R)
|
|
if not df.closed:
|
|
deps = stdlib.malloc(PA_DEPS_MAX_STR)
|
|
if deps is not None:
|
|
rl: int64 = df.read_all(deps, PA_DEPS_MAX_STR)
|
|
if rl < 0: rl = 0
|
|
# read_all's size arg is the buffer capacity (incl. terminator): it reads at most MAX-1
|
|
deps[rl] = 0
|
|
df.close()
|
|
stdlib.free(dpath)
|
|
if deps is None or deps[0] == 0:
|
|
if deps is not None: stdlib.free(deps)
|
|
return
|
|
dl: sizet = string.strlen(deps)
|
|
token_count: int = 0
|
|
pos: sizet = 0
|
|
while pos < dl:
|
|
while pos < dl and deps[pos] == ' ': pos += 1
|
|
if pos >= dl: break
|
|
token_count += 1
|
|
while pos < dl and deps[pos] != ' ': pos += 1
|
|
pos = 0
|
|
idx: int = 0
|
|
while pos < dl:
|
|
while pos < dl and deps[pos] == ' ': pos += 1
|
|
if pos >= dl: break
|
|
ts: sizet = pos
|
|
while pos < dl and deps[pos] != ' ': pos += 1
|
|
tl: sizet = pos - ts
|
|
nm: bytes = stdlib.malloc(tl + 1)
|
|
if nm is not None:
|
|
string.strncpy(nm, deps + ts, tl)
|
|
nm[tl] = 0
|
|
is_last: int = 0
|
|
if idx + 1 >= token_count: is_last = 1
|
|
builtin: int = 0
|
|
if string.strcmp(nm, "t") == 0 or string.strcmp(nm, "c") == 0: builtin = 1
|
|
dep_sha1: str = _PhaseAResolveDepSha1(nm)
|
|
txt_ok: int = 0
|
|
if dep_sha1 is not None: txt_ok = _PhaseATextLLExists(temp_dir, td_len, dep_sha1)
|
|
# Branch characters are inlined (don't return string literals: VPC would return a dangling
|
|
# stack pointer for a literal -> printf %s crashes and the graph output is swallowed)
|
|
br_1: str = "|-- "
|
|
br_2: str = "`-- "
|
|
if builtin != 0:
|
|
if is_last != 0: stdio.printf("%s%s%s (builtin)\n", prefix, br_2, nm)
|
|
else: stdio.printf("%s%s%s (builtin)\n", prefix, br_1, nm)
|
|
elif dep_sha1 is None:
|
|
if is_last != 0: stdio.printf("%s%s%s (unresolved)\n", prefix, br_2, nm)
|
|
else: stdio.printf("%s%s%s (unresolved)\n", prefix, br_1, nm)
|
|
elif txt_ok != 0:
|
|
if is_last != 0: stdio.printf("%s%s%s [OK]\n", prefix, br_2, nm)
|
|
else: stdio.printf("%s%s%s [OK]\n", prefix, br_1, nm)
|
|
else:
|
|
if is_last != 0: stdio.printf("%s%s%s [MISS] text.ll=%s\n", prefix, br_2, nm, dep_sha1)
|
|
else: stdio.printf("%s%s%s [MISS] text.ll=%s\n", prefix, br_1, nm, dep_sha1)
|
|
# Drill down: MISS follows the whole chain (to PA_GRAPH_DEPTH_MAX); OK probes 1 level only
|
|
can_expand: int = 0
|
|
if dep_sha1 is not None and builtin == 0:
|
|
if txt_ok == 0:
|
|
if depth + 1 < PA_GRAPH_DEPTH_MAX: can_expand = 1
|
|
else:
|
|
if depth + 1 < PA_GRAPH_OK_PROBE: can_expand = 1
|
|
if can_expand != 0:
|
|
# +5: strcat appends "| "/" " (4 chars) + NUL; strlen+4 would overflow 1 byte and
|
|
# corrupt heap metadata, crashing later malloc/free and aborting the graph output
|
|
npl: sizet = string.strlen(prefix) + 5
|
|
new_prefix: bytes = stdlib.malloc(npl)
|
|
if new_prefix is not None:
|
|
string.strcpy(new_prefix, prefix)
|
|
if is_last != 0: string.strcat(new_prefix, " ")
|
|
else: string.strcat(new_prefix, "| ")
|
|
_PhaseADumpNode(temp_dir, td_len, dep_sha1, depth + 1, new_prefix)
|
|
stdlib.free(new_prefix)
|
|
stdlib.free(nm)
|
|
idx += 1
|
|
stdlib.free(deps)
|
|
|
|
|
|
def _PhaseADumpFailGraph(temp_dir: str, td_len: sizet, path: str, sha1: str) -> None:
|
|
"""Convergence-failure diagnostics: pending detail plus dependency tree.
|
|
Flush after every printf so the trace survives a later crash."""
|
|
stdio.printf("[PhaseA-Fail] ==============================================================\n")
|
|
stdio.fflush(None)
|
|
stdio.printf("[PhaseA-Fail] FAIL TO TRANSLATE: %s [sha1=%s]\n", path, sha1)
|
|
stdio.fflush(None)
|
|
miss: bytes = _PhaseADiagMissLookup(sha1)
|
|
if miss is not None:
|
|
if miss[0] != 0: stdio.printf("[PhaseA-Fail] pending reason: %s\n", miss)
|
|
stdlib.free(miss)
|
|
stdio.fflush(None)
|
|
stdio.printf("[PhaseA-Fail] dep graph ([MISS]=missing text.ll, [OK]=ready, depth<=%d):\n", PA_GRAPH_DEPTH_MAX)
|
|
stdio.fflush(None)
|
|
root_prefix: bytes = stdlib.malloc(1)
|
|
if root_prefix is not None:
|
|
root_prefix[0] = 0
|
|
_PhaseADumpNode(temp_dir, td_len, sha1, 0, root_prefix)
|
|
stdlib.free(root_prefix)
|
|
stdio.fflush(None)
|
|
|
|
|
|
def RunMultiFileProject(mb: memhub.MemBuddy*, source_dir: str, temp_dir: str, output_dir: str, cc_cmd: str, cc_flags: str, linker_cmd: str, linker_flags: str, linker_output: str, includes_binary_dir: str, includes_dir: str, do_phase1: int, do_phase2: int, log: VLogger.Logger*, args: argparse.ParsedArgs*, linker_static_libs: str = None, linker_dynamic_libs: str = None) -> int:
|
|
"""Compile a multi-file project; return 0 on success, non-zero on failure"""
|
|
if source_dir is None:
|
|
VLogger.error("source_dir is empty", "project")
|
|
return 1
|
|
|
|
VLogger.info(f"multi-file project compilation: {source_dir}", "project")
|
|
|
|
# === 1. Recursively scan .vp/.py files under source_dir (including subdirectories) ===
|
|
entry_size: sizet = SrcFileEntry.__sizeof__()
|
|
entries: SrcFileEntry* = stdlib.malloc(MAX_SRC_FILES * entry_size)
|
|
if entries is None: return 1
|
|
string.memset(entries, 0, MAX_SRC_FILES * entry_size)
|
|
|
|
file_count: int = _ScanDirForPyFiles(mb, source_dir, entries, entry_size, 0, MAX_SRC_FILES)
|
|
VLogger.info(f"{file_count} source files in total", "project")
|
|
|
|
# Scan .s assembly files (separate array; not translated in Phase A, assembled by llvm-mc after Phase B)
|
|
asm_entries: SrcFileEntry* = stdlib.malloc(MAX_SRC_FILES * entry_size)
|
|
if asm_entries is None:
|
|
stdlib.free(entries)
|
|
return 1
|
|
string.memset(asm_entries, 0, MAX_SRC_FILES * entry_size)
|
|
asm_file_count: int = _ScanDirForAsmFiles(mb, source_dir, asm_entries, entry_size, 0, MAX_SRC_FILES)
|
|
if asm_file_count > 0: VLogger.info(f"{asm_file_count} assembly files (assembled by llvm-mc and linked)", "project")
|
|
|
|
if file_count == 0:
|
|
stdlib.free(entries)
|
|
stdlib.free(asm_entries)
|
|
return 1
|
|
|
|
# Initialize the AST tables
|
|
ast._init_tables(mb)
|
|
|
|
# Pre-register App generic class templates before mark2: the tables/nodes are pool-allocated, so they
|
|
# must be registered before mark2 to survive reset_to(mark2); otherwise later passes hit dangling memory.
|
|
_PreRegisterAppGenericTemplates(mb, entries, file_count, entry_size)
|
|
|
|
# OOM cleanup (mirrors Phase1's mark/reset pattern): without reclamation the AST/IR of all includes
|
|
# pile up in the pool. mark2 lets each file's transient data be freed by reset_to after translation.
|
|
mark2: sizet = mb.mark()
|
|
# struct/enum entries registered after mark2 use stdlib allocation (survive reset, as in Phase1)
|
|
GenApi.set_stdlib_alloc(1)
|
|
|
|
# Ensure the temp/output directories exist
|
|
BuildPipeline.ensure_dir(temp_dir)
|
|
BuildPipeline.ensure_dir(output_dir)
|
|
|
|
# === 0.5 Fill Sha1Store on demand from the dependency graph (removes the Phase1 dependency) ===
|
|
# If Sha1Store is empty, scan the dependency graph (App-first BFS) so later "compile missing includes" works.
|
|
if includes_dir is not None:
|
|
if StubMergerStore.GetSha1StoreCount() == 0:
|
|
rs_05: bytes = stdlib.malloc(StubMerger.MAX_INCLUDES_SHA1 * 17)
|
|
if rs_05 is None: VLogger.error("reachable_set allocation failed, cannot fill Sha1Store", "project")
|
|
else:
|
|
string.memset(rs_05, 0, StubMerger.MAX_INCLUDES_SHA1 * 17)
|
|
rc_05: int = StubMergerBuild._BuildReachableSha1Set(mb, source_dir, temp_dir, rs_05, None, includes_dir)
|
|
if rc_05 < 0: VLogger.error("dependency graph scan failed, cannot fill Sha1Store", "project")
|
|
elif rc_05 > 0: VLogger.info(f"Phase2 dependency graph filled Sha1Store on demand: {rc_05} entries", "project")
|
|
stdlib.free(rs_05)
|
|
|
|
# Append App source SHA1s to _sha1_map.txt (human-readable only; format {sha1}:{rel_path}) and to the
|
|
# in-memory store (AppendToSha1MapStore) for cross-module CDefine lookups.
|
|
td_len_am: sizet = string.strlen(temp_dir)
|
|
src_dir_len_am: sizet = string.strlen(source_dir)
|
|
map_path_am: bytes = stdlib.malloc(td_len_am + 32)
|
|
if map_path_am is not None:
|
|
viperlib.snprintf(map_path_am, td_len_am + 32, "%s/_sha1_map.txt", temp_dir)
|
|
mf: fileio.File* = fileio.File(map_path_am, fileio.MODE.A)
|
|
line_am: bytes = stdlib.malloc(512)
|
|
# First fill the in-memory store (no file I/O), then write _sha1_map.txt
|
|
app_filled: int = 0
|
|
for i in range(int(file_count)):
|
|
ea_am: uint64 = uint64(entries) + i * entry_size
|
|
ent_am: SrcFileEntry* = (SrcFileEntry*)(ea_am)
|
|
if ent_am is None or ent_am.Path is None or ent_am.Sha1 is None: continue
|
|
# Compute the relative path (strip the source_dir prefix, keep the directory structure)
|
|
p_str: str = ent_am.Path
|
|
p_len: sizet = string.strlen(p_str)
|
|
rel_path_am: str = p_str
|
|
if p_len > src_dir_len_am + 1:
|
|
if string.strncmp(p_str, source_dir, src_dir_len_am) == 0: rel_path_am = p_str + src_dir_len_am + 1
|
|
# Also append to the in-memory store (for cross-module CDefine lookups)
|
|
# rel_path_am looks like "lib/core/StubMerger.vp", matching _sha1_map.txt
|
|
if StubMergerStore.AppendToSha1MapStore(ent_am.Sha1, rel_path_am) != 0: VLogger.error(f"AppendToSha1MapStore failed: {rel_path_am}", "project")
|
|
app_filled += 1
|
|
# Write _sha1_map.txt (human-readable output)
|
|
if not mf.closed and line_am is not None:
|
|
viperlib.snprintf(line_am, 512, "%s:%s\n", ent_am.Sha1, rel_path_am)
|
|
ll_am: sizet = string.strlen(line_am)
|
|
mf.write(line_am, ll_am)
|
|
if line_am is not None: stdlib.free(line_am)
|
|
if not mf.closed:
|
|
mf.close()
|
|
VLogger.info(f"appended {app_filled} App files to _sha1_map.txt and the in-memory store", "project")
|
|
stdlib.free(map_path_am)
|
|
|
|
# The module SHA1 map is filled into the StubMerger global store by PopulateSha1MapStore +
|
|
# AppendToSha1MapStore; HandlesExprCall._lookup_module_sha1 queries it directly, no local copy needed.
|
|
|
|
|
|
# === Phase A-pre-vpi: pre-generate .vpi stubs for all App sources ===
|
|
# Deps' .vpi must exist before their dependents translate; pre-generating avoids missing dep declarations.
|
|
td_len_vpa: sizet = string.strlen(temp_dir)
|
|
PYI_BUF_SIZE_P2: sizet = 262144
|
|
vpi_failed_p2: int = 0
|
|
for i_vp2 in range(int(file_count)):
|
|
ea_vp2: uint64 = uint64(entries) + i_vp2 * entry_size
|
|
ent_vp2: SrcFileEntry* = (SrcFileEntry*)(ea_vp2)
|
|
if ent_vp2 is None or ent_vp2.Path is None or ent_vp2.Sha1 is None:
|
|
vpi_failed_p2 += 1
|
|
VLogger.error("[Phase A-pre-vpi] invalid source entry (Path/Sha1 missing), skipping", "PhaseA")
|
|
continue
|
|
# Skip if the vpi already exists (sha1 is content-based, so a same-named vpi is valid; cross-run cache)
|
|
vpi_path_vp2: str = StubMergerCollect._sliced_path(temp_dir, td_len_vpa, ent_vp2.Sha1, "vpi")
|
|
if vpi_path_vp2 is None:
|
|
vpi_failed_p2 += 1
|
|
VLogger.error(f"[Phase A-pre-vpi] {ent_vp2.Path} | failed to build vpi path", "PhaseA")
|
|
continue
|
|
vf_vp2: fileio.File* = fileio.File(vpi_path_vp2, fileio.MODE.R)
|
|
if not vf_vp2.closed:
|
|
vf_vp2.close()
|
|
stdlib.free(vpi_path_vp2)
|
|
continue
|
|
stdlib.free(vpi_path_vp2)
|
|
# Read the source -> parse the AST -> generate the vpi (rel_path = ent_vp2.Path: vpi content does
|
|
# not depend on it; it is only used for fatal_error attribution, pointing at this source file)
|
|
f_vp2: fileio.File* = fileio.File(ent_vp2.Path, fileio.MODE.R)
|
|
if f_vp2.closed:
|
|
vpi_failed_p2 += 1
|
|
VLogger.error(f"[Phase A-pre-vpi] {ent_vp2.Path} | cannot open source file", "PhaseA")
|
|
continue
|
|
# read_all's capacity includes the terminator (reads at most cap-1 bytes); longer files are truncated
|
|
if f_vp2.size() >= SRC_BUF_SIZE:
|
|
vpi_failed_p2 += 1
|
|
VLogger.error(f"[Phase A-pre-vpi] {ent_vp2.Path} | source file exceeds {SRC_BUF_SIZE} byte limit", "PhaseA")
|
|
f_vp2.close()
|
|
continue
|
|
src_buf_vp2: bytes = stdlib.malloc(SRC_BUF_SIZE)
|
|
if src_buf_vp2 is None:
|
|
f_vp2.close()
|
|
vpi_failed_p2 += 1
|
|
VLogger.error(f"[Phase A-pre-vpi] {ent_vp2.Path} | source buffer allocation failed (OOM)", "PhaseA")
|
|
continue
|
|
bytes_read_vp2: LONG = f_vp2.read_all(src_buf_vp2, SRC_BUF_SIZE)
|
|
f_vp2.close()
|
|
if bytes_read_vp2 <= 0:
|
|
stdlib.free(src_buf_vp2)
|
|
vpi_failed_p2 += 1
|
|
VLogger.error(f"[Phase A-pre-vpi] {ent_vp2.Path} | file read failed or empty", "PhaseA")
|
|
continue
|
|
if bytes_read_vp2 < SRC_BUF_SIZE: src_buf_vp2[bytes_read_vp2] = 0
|
|
lx_vp2: ast.Lexer* = ast.new_lexer(mb)
|
|
if lx_vp2 is None:
|
|
stdlib.free(src_buf_vp2)
|
|
vpi_failed_p2 += 1
|
|
VLogger.error(f"[Phase A-pre-vpi] {ent_vp2.Path} | lexer allocation failed (OOM)", "PhaseA")
|
|
continue
|
|
ast._lexer_init(lx_vp2, src_buf_vp2, mb)
|
|
tokens_vp2: ast.Token* = ast.tokenize(lx_vp2)
|
|
tree_vp2: ast.AST* = ast.parse_tokens(mb, tokens_vp2)
|
|
if tree_vp2 is None:
|
|
stdlib.free(src_buf_vp2)
|
|
vpi_failed_p2 += 1
|
|
# Parse failure: inline the line/column + reason in the log line (same source as the full
|
|
# Python-style traceback below), since "AST parse failed" alone gives no location or cause.
|
|
VLogger.error(f"[Phase A-pre-vpi] {ent_vp2.Path} | AST parse failed (line {ast.get_parse_err_line()} col {ast.get_parse_err_col()}: {ast.get_parse_err_msg()})", "PhaseA")
|
|
GenApi.dump_parse_traceback(ent_vp2.Path)
|
|
continue
|
|
pyi_buf_vp2: bytes = stdlib.malloc(PYI_BUF_SIZE_P2)
|
|
if pyi_buf_vp2 is not None:
|
|
# 文件级名字窗口(FileFuncNames 等)由 _GeneratePyiFromAst 入口统一 reset+重建,
|
|
# 不再在此处重复调用;契约见 StructGenMixin.reset_file_name_window。
|
|
pyi_pos_vp2: sizet = StubConverter._GeneratePyiFromAst(mb, tree_vp2, ent_vp2.Path, pyi_buf_vp2, PYI_BUF_SIZE_P2)
|
|
if pyi_pos_vp2 > 0:
|
|
pyi_path_vp2: str = StubMergerCollect._sliced_path(temp_dir, td_len_vpa, ent_vp2.Sha1, "vpi")
|
|
if pyi_path_vp2 is not None:
|
|
pf_vp2: fileio.File* = fileio.File(pyi_path_vp2, fileio.MODE.W)
|
|
if not pf_vp2.closed:
|
|
pf_vp2.write(pyi_buf_vp2, pyi_pos_vp2)
|
|
pf_vp2.close()
|
|
stdlib.free(pyi_path_vp2)
|
|
stdlib.free(pyi_buf_vp2)
|
|
else:
|
|
vpi_failed_p2 += 1
|
|
VLogger.error(f"[Phase A-pre-vpi] {ent_vp2.Path} | pyi buffer allocation failed (OOM)", "PhaseA")
|
|
stdlib.free(src_buf_vp2)
|
|
# OOM cleanup: free this file's AST (the vpi is already on disk; persistent data is stdlib-allocated)
|
|
mb.reset_to(mark2)
|
|
if vpi_failed_p2 > 0:
|
|
VLogger.error(f"Phase A-pre-vpi: {vpi_failed_p2} files failed vpi pre-generation (see the per-file [Phase A-pre-vpi] logs above for failing files and causes)", "PhaseA")
|
|
return 1
|
|
|
|
|
|
# === 2. Phase A: generate stub + text for each file ===
|
|
# app_deps_buf collects App files' direct deps (space-separated) as Phase B+'s worklist (65536, matching StubMerger).
|
|
APP_DEPS_BUF_SIZE: sizet = 65536
|
|
app_deps_buf: bytes = None
|
|
app_deps_len: sizet = 0
|
|
if do_phase1 != 0:
|
|
log.banner("Phase A: generate stub + text")
|
|
# Initialize the Phase A convergence-failure diagnostic table (stdlib, survives reset)
|
|
_PhaseADiagInit(sizet(file_count) + StubMerger.MAX_INCLUDES)
|
|
|
|
# === Phase A-pre-inc: pre-register includes structs + fill deps ===
|
|
# Must run before App pre-registration (multi-pass resolves cross-file inheritance) and fills PopulateIncludesDeps.
|
|
if includes_dir is not None and includes_binary_dir is not None:
|
|
store_count_inc_pre: int = StubMergerStore.GetSha1StoreCount()
|
|
if store_count_inc_pre > 0:
|
|
store_sha1_inc_pre: bytes = StubMergerStore.GetSha1StoreArrPtr()
|
|
store_rel_inc_pre: bytes = StubMergerStore.GetSha1StoreRelArrPtr()
|
|
store_incdir_inc_pre: bytes = StubMergerStore.GetSha1StoreIncdirArrPtr()
|
|
deps_filled_pre: int = 0
|
|
PHASE_A_INC_MAX_PASSES: int = 3
|
|
# Track includes that fail pre-registration: after all passes, any failure aborts the build
|
|
# (no silent downgrade). App files must be skipped here; Phase A-pre handles them.
|
|
inc_fail_flags: bytes = stdlib.malloc(store_count_inc_pre)
|
|
if inc_fail_flags is not None: string.memset(inc_fail_flags, 1, store_count_inc_pre)
|
|
for pass_inc_i in range(int(PHASE_A_INC_MAX_PASSES)):
|
|
struct_count_before_inc: int = GenApi.get_struct_count()
|
|
for si_inc in range(int(store_count_inc_pre)):
|
|
inc_sha1_pre: str = store_sha1_inc_pre + sizet(si_inc) * 17
|
|
inc_rel_pre: str = store_rel_inc_pre + sizet(si_inc) * StubMerger.MAX_REL_PATH_LEN
|
|
if inc_rel_pre is None or inc_rel_pre[0] == '\0': continue
|
|
# Skip App source files (main.vp etc.): pre-registered by Phase A-pre, not a failure
|
|
is_app_file_inc: int = 0
|
|
for i_app_inc in range(int(file_count)):
|
|
ea_app_inc: uint64 = uint64(entries) + i_app_inc * entry_size
|
|
ent_app_inc: SrcFileEntry* = (SrcFileEntry*)(ea_app_inc)
|
|
if ent_app_inc is not None and ent_app_inc.Sha1 is not None and string.strcmp(ent_app_inc.Sha1, inc_sha1_pre) == 0:
|
|
is_app_file_inc = 1
|
|
break
|
|
if is_app_file_inc != 0 and inc_fail_flags is not None:
|
|
inc_fail_flags[si_inc] = 0
|
|
continue
|
|
rel_len_pre: sizet = string.strlen(inc_rel_pre)
|
|
# The includes root dir (per-entry from the IncDir array; empty falls back to includes_dir)
|
|
incdir_pre: str = store_incdir_inc_pre + sizet(si_inc) * StubMerger.MAX_INC_DIR_LEN
|
|
if incdir_pre is None or incdir_pre[0] == '\0': incdir_pre = includes_dir
|
|
incdir_len_pre: sizet = string.strlen(incdir_pre)
|
|
full_path_pre: bytes = stdlib.malloc(incdir_len_pre + rel_len_pre + 2)
|
|
if full_path_pre is None: continue
|
|
viperlib.snprintf(full_path_pre, incdir_len_pre + rel_len_pre + 2, "%s/%s", incdir_pre, inc_rel_pre)
|
|
pkg_inc_pre: str = GenApi.compute_package_from_relpath(mb, inc_rel_pre)
|
|
tr_inc_pre: HandlesTranslator.Translator* = BuildPipeline.TranslateFileGetTrans(mb, full_path_pre, inc_sha1_pre, pkg_inc_pre, 1)
|
|
stdlib.free(full_path_pre)
|
|
if tr_inc_pre is not None:
|
|
# Record this file's deps (for convergence diagnostics; stdlib table survives reset)
|
|
_PhaseADiagRecord(inc_sha1_pre, tr_inc_pre._imported_modules, None)
|
|
# Pre-registration succeeded: clear the failure flag
|
|
if inc_fail_flags is not None: inc_fail_flags[si_inc] = 0
|
|
# Fill deps only on the first pass (PopulateIncludesDeps dedupes internally).
|
|
# Must be called before reset_to(mark2) since it deep-copies into stdlib.
|
|
if pass_inc_i == 0 and tr_inc_pre._imported_modules is not None and StubMergerStore.PopulateIncludesDeps(inc_sha1_pre, tr_inc_pre._imported_modules) == 0:
|
|
deps_filled_pre = deps_filled_pre + 1
|
|
# OOM cleanup: free this includes file's transient pre-registration data
|
|
# (AST/tokens/lexer/IR/Handler); stdlib-allocated struct entries are unaffected
|
|
mb.reset_to(mark2)
|
|
struct_count_after_inc: int = GenApi.get_struct_count()
|
|
# Convergence check: no new struct registered this pass -> all classes are registered
|
|
if struct_count_after_inc == struct_count_before_inc: break
|
|
# Includes still failing after all passes -> abort (no downgrade)
|
|
if inc_fail_flags is not None:
|
|
for si_inc_f in range(int(store_count_inc_pre)):
|
|
if inc_fail_flags[si_inc_f] != 0:
|
|
rel_f: str = store_rel_inc_pre + sizet(si_inc_f) * StubMerger.MAX_REL_PATH_LEN
|
|
VLogger.error(f"includes pre-registration failed: {rel_f} (file cannot be translated; possible syntax/parse/compile error), aborting compilation", "prePhaseA")
|
|
stdlib.free(inc_fail_flags)
|
|
VLogger.info(f"includes pre-registration + dependency fill: {deps_filled_pre}/{store_count_inc_pre}", "prePhaseA")
|
|
|
|
# === Phase A-pre: pre-register struct/enum/union for all sources ===
|
|
# Registers declarations only (declare_only=1), no method bodies; multi-pass resolves cross-file ordering.
|
|
VLogger.info("pre-registering struct/enum/union...", "PhaseA")
|
|
src_dir_len_pre: sizet = string.strlen(source_dir)
|
|
PHASE_A_PRE_MAX_PASSES: int = 3
|
|
for pass_i in range(int(PHASE_A_PRE_MAX_PASSES)):
|
|
struct_count_before: int = GenApi.get_struct_count()
|
|
for i in range(int(file_count)):
|
|
ea_pre: uint64 = uint64(entries) + i * entry_size
|
|
ent_pre: SrcFileEntry* = (SrcFileEntry*)(ea_pre)
|
|
if ent_pre is None or ent_pre.Path is None: continue
|
|
pkg_pre: str = None
|
|
if string.strlen(ent_pre.Path) > src_dir_len_pre + 1:
|
|
rel_path_pre: str = ent_pre.Path + src_dir_len_pre + 1
|
|
pkg_pre = GenApi.compute_package_from_relpath(mb, rel_path_pre)
|
|
# App pre-registration passes pre_allow_dep_inject=1: imports and dep vpi injection happen at
|
|
# declaration time so field annotations share Phase A's context (else struct layouts diverge).
|
|
tr_pre: HandlesTranslator.Translator* = BuildPipeline.TranslateFileGetTrans(mb, ent_pre.Path, ent_pre.Sha1, pkg_pre, 1, None, 1)
|
|
if tr_pre is None: VLogger.error(f"pre-registration failed: {ent_pre.Path}", "PhaseA")
|
|
# OOM cleanup: free this App file's transient pre-registration data
|
|
mb.reset_to(mark2)
|
|
struct_count_after: int = GenApi.get_struct_count()
|
|
# Convergence check: no new struct registered this pass -> all classes are registered
|
|
if struct_count_after == struct_count_before: break
|
|
VLogger.info("pre-registration complete", "PhaseA")
|
|
|
|
# Phase A IR buffer: 1MB is not enough, ast/parser.vp's text.ll exceeds 1MB and got truncated ->
|
|
# merged IR lost the tail (opt: "end of file in global variable name").
|
|
PHASE_A_IR_SIZE: sizet = 8388608
|
|
# FULL dump = stub + text (same module), with 2x headroom
|
|
PHASE_A_FULL_SIZE: sizet = 16777216
|
|
td_len_pa: sizet = string.strlen(temp_dir)
|
|
|
|
# Set the temp dir so HandlesExprCall can read dependency modules' text.ll for cross-module
|
|
# method return-type lookup (avoids pointer returns being truncated to i32)
|
|
GenApi.SetTempDir(temp_dir)
|
|
|
|
# app_deps_buf collects App files' direct dependency modules (space-separated) as Phase B+'s initial
|
|
# worklist for _BuildReachableSha1Set; declared outside the if, only assigned here.
|
|
app_deps_buf = stdlib.malloc(APP_DEPS_BUF_SIZE)
|
|
app_deps_len = 0
|
|
if app_deps_buf is not None: app_deps_buf[0] = '\0'
|
|
|
|
# Phase A multi-pass retry: with several sources, main.vp may translate before its imports
|
|
# (filesystem order); failed files are retried next pass, converging in at most file_count passes.
|
|
done_flags: bytes = stdlib.malloc(file_count)
|
|
if done_flags is None:
|
|
VLogger.error("Phase A done_flags allocation failed", "PhaseA")
|
|
stdlib.free(entries)
|
|
return 1
|
|
string.memset(done_flags, 0, file_count)
|
|
PHASE_A_MAX_PASSES: int = 32
|
|
for pass_a in range(int(PHASE_A_MAX_PASSES)):
|
|
completed_this_pass: int = 0
|
|
for i in range(int(file_count)):
|
|
if done_flags[i] != 0: continue
|
|
ea: uint64 = uint64(entries) + i * entry_size
|
|
ent: SrcFileEntry* = (SrcFileEntry*)(ea)
|
|
# Timing point 0: start processing this file (translate phase start)
|
|
t_pa0: uint64 = win32.win32base.GetTickCount64()
|
|
if ent is None or ent.Path is None:
|
|
done_flags[i] = 1
|
|
completed_this_pass += 1
|
|
continue
|
|
# Compute the package name (the directory part relative to source_dir)
|
|
src_dir_len_pa: sizet = string.strlen(source_dir)
|
|
pkg_pa: str = None
|
|
if string.strlen(ent.Path) > src_dir_len_pa + 1:
|
|
rel_path_pa: str = ent.Path + src_dir_len_pa + 1
|
|
pkg_pa = GenApi.compute_package_from_relpath(mb, rel_path_pa)
|
|
tr_a: HandlesTranslator.Translator* = BuildPipeline.TranslateFileGetTrans(mb, ent.Path, ent.Sha1, pkg_pa, 0, temp_dir)
|
|
# Timing point 1: translate returned
|
|
t_pa1: uint64 = win32.win32base.GetTickCount64()
|
|
if tr_a is None:
|
|
# Translation failed (syntax/parse error): retry next pass; convergence check reports it.
|
|
# OOM cleanup: a failed translation also produced AST/tokens, freed immediately.
|
|
mb.reset_to(mark2)
|
|
continue
|
|
# Record this file's deps (for convergence diagnostics; stdlib table survives reset)
|
|
_PhaseADiagRecord(ent.Sha1, tr_a._imported_modules, None)
|
|
# Collect _imported_modules into app_deps_buf (for Phase B+ on-demand compilation).
|
|
# Dependencies were filled early during translation.
|
|
if tr_a._imported_modules is not None and app_deps_buf is not None:
|
|
# Dedupe per module: several App files importing the same module would otherwise inflate
|
|
# app_deps toward 64KB, filling the initial worklist and overflowing BFS.
|
|
im_len_a: sizet = string.strlen(tr_a._imported_modules)
|
|
im_pos_a: sizet = 0
|
|
while im_pos_a < im_len_a:
|
|
im_start_a: sizet = im_pos_a
|
|
while im_pos_a < im_len_a and tr_a._imported_modules[im_pos_a] != ' ': im_pos_a += 1
|
|
im_mod_len: sizet = im_pos_a - im_start_a
|
|
if im_pos_a < im_len_a: im_pos_a += 1
|
|
if im_mod_len == 0: continue
|
|
im_mod_buf: bytes = stdlib.malloc(im_mod_len + 1)
|
|
if im_mod_buf is None: continue
|
|
string.strncpy(im_mod_buf, tr_a._imported_modules + im_start_a, im_mod_len)
|
|
im_mod_buf[im_mod_len] = '\0'
|
|
# Skip if already in app_deps (exact word-boundary match)
|
|
if app_deps_len > 0 and GenApi.is_module_imported(app_deps_buf, im_mod_buf) != 0:
|
|
stdlib.free(im_mod_buf)
|
|
continue
|
|
if app_deps_len + im_mod_len + 2 < APP_DEPS_BUF_SIZE:
|
|
if app_deps_len > 0:
|
|
app_deps_buf[app_deps_len] = ' '
|
|
app_deps_len = app_deps_len + 1
|
|
string.strcpy(app_deps_buf + app_deps_len, im_mod_buf)
|
|
app_deps_len = app_deps_len + im_mod_len
|
|
app_deps_buf[app_deps_len] = '\0'
|
|
stdlib.free(im_mod_buf)
|
|
# dump stub IR (declarations only)
|
|
stub_buf_a: bytes = stdlib.malloc(PHASE_A_IR_SIZE)
|
|
if stub_buf_a is None: VLogger.error("stub buffer allocation failed", "PhaseA")
|
|
tr_a.dump_ir(stub_buf_a, PHASE_A_IR_SIZE, mlirlite.OUTPUT_STUB)
|
|
stub_len_a: sizet = string.strlen(stub_buf_a)
|
|
|
|
# Save stub.ll (sliced path: temp_dir/{sha1 prefix}/{sha1}.stub.ll)
|
|
stub_path_a: str = StubMergerCollect._sliced_path(temp_dir, td_len_pa, ent.Sha1, "stub.ll")
|
|
if stub_path_a is not None:
|
|
sf_a: fileio.File* = fileio.File(stub_path_a, fileio.MODE.W)
|
|
if not sf_a.closed:
|
|
sf_a.write(stub_buf_a, stub_len_a)
|
|
sf_a.close()
|
|
stdlib.free(stub_path_a)
|
|
stdlib.free(stub_buf_a)
|
|
|
|
# dump text IR (definitions only)
|
|
text_buf_a: bytes = stdlib.malloc(PHASE_A_IR_SIZE)
|
|
if text_buf_a is None: VLogger.error("text buffer allocation failed", "PhaseA")
|
|
tr_a.dump_ir(text_buf_a, PHASE_A_IR_SIZE, mlirlite.OUTPUT_TEXT)
|
|
text_len_a: sizet = string.strlen(text_buf_a)
|
|
|
|
# Save text.ll (sliced path)
|
|
text_path_a: str = StubMergerCollect._sliced_path(temp_dir, td_len_pa, ent.Sha1, "text.ll")
|
|
if text_path_a is not None:
|
|
tf_a: fileio.File* = fileio.File(text_path_a, fileio.MODE.W)
|
|
if not tf_a.closed:
|
|
tf_a.write(text_buf_a, text_len_a)
|
|
tf_a.close()
|
|
stdlib.free(text_path_a)
|
|
stdlib.free(text_buf_a)
|
|
|
|
# Save {sha1}.meta (structured sidecar: function list / struct layout / vtable)
|
|
StubMergerIR.WriteModuleMeta(tr_a.Module, temp_dir, td_len_pa, ent.Sha1)
|
|
|
|
# Save {sha1}.full.ll (in-memory FULL artifact: a self-contained single-module IR)
|
|
full_buf_a: bytes = stdlib.malloc(PHASE_A_FULL_SIZE)
|
|
if full_buf_a is None: VLogger.error("full.ll buffer allocation failed", "PhaseA")
|
|
tr_a.dump_ir(full_buf_a, PHASE_A_FULL_SIZE, mlirlite.OUTPUT_FULL)
|
|
full_len_a: sizet = string.strlen(full_buf_a)
|
|
full_path_a: str = StubMergerCollect._sliced_path(temp_dir, td_len_pa, ent.Sha1, "full.ll")
|
|
if full_path_a is not None:
|
|
ffa: fileio.File* = fileio.File(full_path_a, fileio.MODE.W)
|
|
if not ffa.closed:
|
|
ffa.write(full_buf_a, full_len_a)
|
|
ffa.close()
|
|
stdlib.free(full_path_a)
|
|
stdlib.free(full_buf_a)
|
|
|
|
# --traceback: take this module's debug call-site records and write <sha1>.vpcall.
|
|
if BuildPipeline._traceback_mode != 0:
|
|
vpcall_text_a: str = mlirlite.DebugTakeText()
|
|
if vpcall_text_a is not None:
|
|
BuildPipeline.WriteCallSidecar(temp_dir, ent.Sha1, vpcall_text_a)
|
|
stdlib.free(vpcall_text_a)
|
|
|
|
# Save dependencies (_imported_modules) for Phase B (sliced path)
|
|
deps_path_a: str = StubMergerCollect._sliced_path(temp_dir, td_len_pa, ent.Sha1, "deps.txt")
|
|
if deps_path_a is not None:
|
|
df_a: fileio.File* = fileio.File(deps_path_a, fileio.MODE.W)
|
|
if not df_a.closed:
|
|
if tr_a._imported_modules is not None:
|
|
dl_a: sizet = string.strlen(tr_a._imported_modules)
|
|
df_a.write(tr_a._imported_modules, dl_a)
|
|
df_a.close()
|
|
stdlib.free(deps_path_a)
|
|
|
|
# Timing point 2: stub+text dump and write done; print per-file progress (as in Phase1 1b)
|
|
t_pa2: uint64 = win32.win32base.GetTickCount64()
|
|
VLogger.info(f"[{i+1}/{file_count}] PhaseA: {ent.Path} (translate={t_pa1-t_pa0} ms, dump={t_pa2-t_pa1} ms, total={t_pa2-t_pa0} ms)", "PhaseA")
|
|
|
|
done_flags[i] = 1
|
|
completed_this_pass += 1
|
|
# OOM cleanup: stub/text/deps are on disk, so free this file's transient translation data
|
|
# (AST/tokens/lexer/IR/Handler); app_deps_buf already deep-copied the deps.
|
|
mb.reset_to(mark2)
|
|
# Convergence check: stop when all files are done
|
|
all_done: int = 1
|
|
for i_ck in range(int(file_count)):
|
|
if done_flags[i_ck] == 0:
|
|
all_done = 0
|
|
break
|
|
if all_done != 0: break
|
|
if completed_this_pass == 0:
|
|
# No progress: remaining files cannot finish (real compile errors). Abort and list them, each
|
|
# with a dependency-graph dump. VLogger.error calls sys.exit(1), so it must come after all output.
|
|
td_len_fail: sizet = string.strlen(temp_dir)
|
|
for i_err in range(int(file_count)):
|
|
if done_flags[i_err] == 0:
|
|
ea_err: uint64 = uint64(entries) + i_err * entry_size
|
|
ent_err: SrcFileEntry* = (SrcFileEntry*)(ea_err)
|
|
if ent_err is not None and ent_err.Path is not None:
|
|
if ent_err.Sha1 is not None: _PhaseADumpFailGraph(temp_dir, td_len_fail, ent_err.Path, ent_err.Sha1)
|
|
stdlib.free(done_flags)
|
|
_PhaseADiagFree()
|
|
VLogger.error("Phase A multi-pass convergence failed, aborting compilation", "PhaseA")
|
|
stdlib.free(entries)
|
|
return 1
|
|
stdlib.free(done_flags)
|
|
# Phase A converged: free the diagnostic table (needed only on the failure path)
|
|
_PhaseADiagFree()
|
|
|
|
if do_phase2 == 0:
|
|
VLogger.info("Phase A complete (stub generation only)", "project")
|
|
if app_deps_buf is not None: stdlib.free(app_deps_buf)
|
|
stdlib.free(entries)
|
|
return 0
|
|
|
|
# === 3. Phase B: compile each file to .obj ===
|
|
if do_phase2 != 0:
|
|
log.banner("Phase B: compile .obj")
|
|
|
|
# Collect .obj paths (64KB to avoid includes .obj path overflow dropping link symbols).
|
|
# reachable_set/reachable_count: filled by Phase B+, used by Phase C for on-demand linking.
|
|
reachable_set: bytes = None
|
|
reachable_count: int = 0
|
|
OBJ_PATHS_SIZE: sizet = 65536
|
|
obj_paths: bytes = stdlib.malloc(OBJ_PATHS_SIZE)
|
|
if obj_paths is None: return 1
|
|
obj_paths[0] = '\0'
|
|
obj_pos: sizet = 0
|
|
# main_obj_path: the .obj that defines the user main (placed first at link time so the linker
|
|
# does not pick another module's wrapper main)
|
|
main_obj_path: bytes = stdlib.malloc(512)
|
|
if main_obj_path is None: return 1
|
|
main_obj_path[0] = '\0'
|
|
compiled_count: int = 0
|
|
|
|
# IR buffer size (16MB, to hold the in-memory FULL artifact {sha1}.full.ll; FULL = stub + text)
|
|
COMBINED_IR_SIZE: sizet = 16777216
|
|
|
|
# llc task table (stdlib.malloc, outlives threads): a contiguous array of LlcTask records; the loop
|
|
# only writes .ll + builds tasks, then run_llc_parallel runs llc in parallel batches.
|
|
llc_tasks: bytes = stdlib.malloc(sizet(file_count) * BuildPipeline.LlcTask.__sizeof__())
|
|
if llc_tasks is None:
|
|
VLogger.error("llc task table allocation failed", "PhaseB")
|
|
return 1
|
|
llc_task_count: int = 0
|
|
|
|
for i in range(int(file_count)):
|
|
ea: uint64 = uint64(entries) + i * entry_size
|
|
ent: SrcFileEntry* = (SrcFileEntry*)(ea)
|
|
if ent is None or ent.Path is None or ent.Sha1 is None: continue
|
|
# Start of this file's three-step timing (GetTickCount64, ms)
|
|
t_pb0: uint64 = win32.win32base.GetTickCount64()
|
|
# Read the in-memory FULL artifact {sha1}.full.ll -> complete IR (replaces merging stub + dep stub/text)
|
|
combined_ir: bytes = stdlib.malloc(COMBINED_IR_SIZE)
|
|
if combined_ir is None:
|
|
VLogger.error(f"combined_ir allocation failed: {ent.Path}", "PhaseB")
|
|
continue
|
|
combined_len: sizet = StubMergerBuild.LoadFullIR(temp_dir, ent.Sha1, combined_ir, COMBINED_IR_SIZE)
|
|
if combined_len == 0:
|
|
VLogger.error(f"failed to read full.ll: {ent.Path}", "PhaseB")
|
|
stdlib.free(combined_ir)
|
|
continue
|
|
# Timing point 1: full.ll read done
|
|
t_pb1: uint64 = win32.win32base.GetTickCount64()
|
|
|
|
# Write .ll (llc threads only read this file); llc itself runs parallel in batches after the loop
|
|
ret_w: int = BuildPipeline.write_ir_to_file(combined_ir, combined_len, temp_dir, ent.Sha1)
|
|
stdlib.free(combined_ir)
|
|
if ret_w != 0:
|
|
VLogger.error(f"failed to write .ll: {ent.Path}", "PhaseB")
|
|
sys.exit(1)
|
|
# Timing point 2: .ll write done
|
|
t_pb2: uint64 = win32.win32base.GetTickCount64()
|
|
|
|
# Build the llc task (command string is stdlib.malloc'd so threads' lifetimes are not reset-reclaimed)
|
|
llc_ir_path: str = BuildPipeline.build_sliced_path(temp_dir, ent.Sha1, "ll")
|
|
if llc_ir_path is None:
|
|
VLogger.error("failed to build llc input path", "PhaseB")
|
|
sys.exit(1)
|
|
tslot: BuildPipeline.LlcTask* = BuildPipeline.LlcTask*(uint64(llc_tasks) + sizet(llc_task_count) * BuildPipeline.LlcTask.__sizeof__())
|
|
if BuildPipeline.init_llc_task(tslot, llc_ir_path, output_dir, ent.Sha1, cc_cmd, cc_flags) != 0:
|
|
VLogger.error(f"failed to build llc task: {ent.Path}", "PhaseB")
|
|
sys.exit(1)
|
|
# Timing point 3: llc task construction done
|
|
t_pb3: uint64 = win32.win32base.GetTickCount64()
|
|
llc_task_count += 1
|
|
|
|
compiled_count += 1
|
|
# Show files submitted to llc (with the three timings: merge stub / write .ll / build task)
|
|
VLogger.info(f"[{i+1}/{file_count}] llc submitted: {ent.Path} (merge={t_pb1-t_pb0} ms, write={t_pb2-t_pb1} ms, task={t_pb3-t_pb2} ms, total={t_pb3-t_pb0} ms)", "PhaseB")
|
|
# Build the .obj path and detect the main module (test_main.vp or main.vp; .py no longer recognized)
|
|
od_len: sizet = string.strlen(output_dir)
|
|
is_main_mod: int = 0
|
|
if string.strstr(ent.Path, "test_main.vp") is not None: is_main_mod = 1
|
|
elif string.strstr(ent.Path, "main.vp") is not None: is_main_mod = 1
|
|
|
|
# Sliced path: output_dir/{sha1 prefix}/{sha1}.obj
|
|
obj_path_sliced: str = StubMergerCollect._sliced_path(output_dir, od_len, ent.Sha1, "obj")
|
|
if obj_path_sliced is not None:
|
|
op_sliced_len: sizet = string.strlen(obj_path_sliced)
|
|
if is_main_mod != 0:
|
|
# main module: copy to main_obj_path (so main is first at link time)
|
|
if op_sliced_len < 512: string.strcpy(main_obj_path, obj_path_sliced)
|
|
else: VLogger.error("main_obj_path buffer too small, aborting compilation", "PhaseB")
|
|
else:
|
|
# Other modules: append to obj_paths
|
|
if obj_pos + op_sliced_len + 2 < OBJ_PATHS_SIZE:
|
|
if obj_pos > 0:
|
|
obj_paths[obj_pos] = ' '
|
|
obj_pos += 1
|
|
string.strcpy(obj_paths + obj_pos, obj_path_sliced)
|
|
obj_pos += op_sliced_len
|
|
obj_paths[obj_pos] = '\0'
|
|
else: VLogger.error(".obj path buffer too small", "PhaseB")
|
|
stdlib.free(obj_path_sliced)
|
|
|
|
# Run all llc jobs in parallel (batched at LLC_MAX_THREADS threads)
|
|
ret_p: int = BuildPipeline.run_llc_parallel(llc_tasks, llc_task_count, BuildPipeline.LLC_MAX_THREADS)
|
|
if ret_p != 0:
|
|
VLogger.error("llc compilation failed, aborting compilation", "PhaseB")
|
|
sys.exit(1)
|
|
|
|
# Free the llc tasks (command strings + records)
|
|
ti: int = 0
|
|
while ti < llc_task_count:
|
|
tfree: BuildPipeline.LlcTask* = BuildPipeline.LlcTask*(uint64(llc_tasks) + sizet(ti) * BuildPipeline.LlcTask.__sizeof__())
|
|
BuildPipeline.free_llc_task(tfree)
|
|
ti += 1
|
|
stdlib.free(llc_tasks)
|
|
|
|
# Assemble .s files: llvm-mc -> .obj (isr.s etc., bare symbols matching the .vp CExport/State stubs).
|
|
# The obj is added to obj_paths for Phase C linking.
|
|
od_len_as: sizet = string.strlen(output_dir)
|
|
asm_compiled: int = 0
|
|
for ai_as in range(int(asm_file_count)):
|
|
ea_as: uint64 = uint64(asm_entries) + ai_as * entry_size
|
|
ent_as: SrcFileEntry* = (SrcFileEntry*)(ea_as)
|
|
if ent_as is None or ent_as.Path is None or ent_as.Sha1 is None: continue
|
|
aret_as: int = BuildPipeline.asm_to_obj(ent_as.Path, ent_as.Sha1, output_dir, Config.TargetTriple)
|
|
if aret_as != 0:
|
|
VLogger.error(f"llvm-mc assembly failed, aborting compilation: {ent_as.Path}", "PhaseB")
|
|
sys.exit(1)
|
|
obj_path_asm: str = StubMergerCollect._sliced_path(output_dir, od_len_as, ent_as.Sha1, "obj")
|
|
if obj_path_asm is not None:
|
|
op_len_asm: sizet = string.strlen(obj_path_asm)
|
|
if obj_pos + op_len_asm + 2 < OBJ_PATHS_SIZE:
|
|
if obj_pos > 0:
|
|
obj_paths[obj_pos] = ' '
|
|
obj_pos += 1
|
|
string.strcpy(obj_paths + obj_pos, obj_path_asm)
|
|
obj_pos += op_len_asm
|
|
obj_paths[obj_pos] = '\0'
|
|
stdlib.free(obj_path_asm)
|
|
asm_compiled += 1
|
|
VLogger.info(f"[asm {asm_compiled}] assembly complete: {ent_as.Path}", "PhaseB")
|
|
|
|
VLogger.success(f"compilation complete: {compiled_count}/{file_count}", "PhaseB")
|
|
|
|
if compiled_count == 0:
|
|
VLogger.error("no files compiled successfully", "PhaseB")
|
|
stdlib.free(entries)
|
|
stdlib.free(asm_entries)
|
|
return 1
|
|
|
|
# === 3.5 Compile missing includes (on demand via the dependency graph) ===
|
|
# Only compile includes reached directly/indirectly from App sources (via _BuildReachableSha1Set).
|
|
if includes_dir is not None and includes_binary_dir is not None:
|
|
inc_compiled: int = 0
|
|
td_len_mi: sizet = string.strlen(temp_dir)
|
|
|
|
# Diagnostic log: confirm the Phase B+ entry and Sha1Store state
|
|
VLogger.info(f"PhaseB+ entry: store_count={StubMergerStore.GetSha1StoreCount()} includes_binary_dir={includes_binary_dir}", "PhaseB+")
|
|
|
|
# Create includes_binary_dir (so .obj files can be written and collect_obj_files can scan them)
|
|
BuildPipeline.ensure_dir(includes_binary_dir)
|
|
|
|
# declare_only preprocessing was moved before Phase A (Phase A-pre-inc) so includes structs are
|
|
# registered when Phase A translates App sources. PopulateIncludesDeps is filled there too.
|
|
|
|
# Build the reachable SHA1 set (dependency graph traversal).
|
|
# When app_deps_buf is None, fall back to scanning source_dir (non-recursive).
|
|
REACHABLE_SET_SIZE: sizet = sizet(StubMerger.MAX_INCLUDES_SHA1) * 17
|
|
reachable_set: bytes = stdlib.malloc(REACHABLE_SET_SIZE)
|
|
reachable_count: int = 0
|
|
if reachable_set is not None:
|
|
string.memset(reachable_set, 0, REACHABLE_SET_SIZE)
|
|
reachable_count = StubMergerBuild._BuildReachableSha1Set(mb, source_dir, temp_dir, reachable_set, app_deps_buf, includes_dir)
|
|
VLogger.info(f"reachable includes SHA1: {reachable_count} entries", "PhaseB+")
|
|
# fast-fail: reachable-set construction failed (deps not found); never fall back, abort
|
|
if reachable_count < 0:
|
|
VLogger.error("reachable set construction failed (fast-fail), aborting compilation", "PhaseB+")
|
|
return 1
|
|
else:
|
|
VLogger.error("reachable_set allocation failed, aborting compilation", "PhaseB+")
|
|
return 1
|
|
|
|
# Free app_deps_buf (reachable_set is built, no longer needed)
|
|
if app_deps_buf is not None:
|
|
stdlib.free(app_deps_buf)
|
|
app_deps_buf = None
|
|
|
|
# Get the SHA1/module-name arrays from the global store as local copies (bytes) to avoid address
|
|
# issues from passing the store's union return (bytes*) across functions; rel/incdir keep pointers.
|
|
td_len_pb: sizet = string.strlen(temp_dir)
|
|
store_sha1_arr_mi: bytes = stdlib.malloc(StubMerger.MAX_INCLUDES * 17)
|
|
store_mod_arr_mi: bytes = stdlib.malloc(StubMerger.MAX_INCLUDES * 64)
|
|
store_count_mi: int = 0
|
|
if store_sha1_arr_mi is not None and store_mod_arr_mi is not None:
|
|
store_count_mi = StubMergerIR._BuildIncludesSha1Map(temp_dir, td_len_pb, store_sha1_arr_mi, store_mod_arr_mi)
|
|
store_rel_arr_mi: bytes = StubMergerStore.GetSha1StoreRelArrPtr()
|
|
store_incdir_arr_mi: bytes = StubMergerStore.GetSha1StoreIncdirArrPtr()
|
|
if store_count_mi > 0:
|
|
|
|
# Topologically sort reachable includes: dependencies are fully translated first. Cross-module
|
|
# vtable generation needs dependency functions' TypeStr registered globally (e.g. astaux/base AST).
|
|
topo_idx: sizet* = (sizet*)(stdlib.malloc(8 * (store_count_mi + 1)))
|
|
topo_count: int = 0
|
|
if topo_idx is None:
|
|
VLogger.error("topological sort index allocation failed, aborting compilation", "PhaseB+")
|
|
topo_count = 0
|
|
else:
|
|
topo_count = StubMergerTopo.BuildTopoSortedIndices(store_count_mi, store_sha1_arr_mi, store_mod_arr_mi, reachable_set, reachable_count, topo_idx)
|
|
|
|
for si_mi in range(int(topo_count)):
|
|
# Current entry (topological index; reachable modules only)
|
|
ti: int = int(topo_idx[si_mi])
|
|
# Get the current entry's SHA1 and rel_path
|
|
inc_sha1_mi: str = store_sha1_arr_mi + sizet(ti) * 17
|
|
inc_rel_mi: str = store_rel_arr_mi + sizet(ti) * StubMerger.MAX_REL_PATH_LEN
|
|
# Skip App sources: an App file may be scanned as an include and be reachable; compiling
|
|
# another .obj into includes_binary would duplicate the output/ .obj's symbols.
|
|
app_skip_mi: int = 0
|
|
for ai_mi in range(int(file_count)):
|
|
ea_ap: uint64 = uint64(entries) + ai_mi * entry_size
|
|
ent_ap: SrcFileEntry* = (SrcFileEntry*)(ea_ap)
|
|
if ent_ap is not None and ent_ap.Sha1 is not None:
|
|
if string.strcmp(ent_ap.Sha1, inc_sha1_mi) == 0:
|
|
app_skip_mi = 1
|
|
break
|
|
if app_skip_mi != 0: continue
|
|
# Check whether the .obj already exists in includes.binary (sliced subdir)
|
|
ibd_len_mi: sizet = string.strlen(includes_binary_dir)
|
|
check_pat_mi: str = StubMergerCollect._sliced_path(includes_binary_dir, ibd_len_mi, inc_sha1_mi, "obj")
|
|
check_fd_mi: win32.win32file.WIN32_FIND_DATAA* = stdlib.malloc(win32.win32file.WIN32_FIND_DATAA.__sizeof__())
|
|
obj_exists_mi: int = 0
|
|
if check_pat_mi is not None and check_fd_mi is not None:
|
|
string.memset(check_fd_mi, 0, win32.win32file.WIN32_FIND_DATAA.__sizeof__())
|
|
check_h_mi: win32.win32base.HANDLE = win32.win32file.FindFirstFileA(check_pat_mi, check_fd_mi)
|
|
if check_h_mi != win32.win32base.INVALID_HANDLE_VALUE:
|
|
win32.win32file.FindClose(check_h_mi)
|
|
obj_exists_mi = 1
|
|
if obj_exists_mi != 0:
|
|
if check_pat_mi is not None: stdlib.free(check_pat_mi)
|
|
stdlib.free(check_fd_mi)
|
|
continue
|
|
|
|
# The .obj does not exist; compile it.
|
|
# Build the source path: {includes root}/{rel_path} (per-entry IncDir; empty falls back to includes_dir)
|
|
rel_path_len_mi: sizet = string.strlen(inc_rel_mi)
|
|
incdir_mi: str = store_incdir_arr_mi + sizet(ti) * StubMerger.MAX_INC_DIR_LEN
|
|
if incdir_mi is None or incdir_mi[0] == '\0': incdir_mi = includes_dir
|
|
inc_dir_len_mi: sizet = string.strlen(incdir_mi)
|
|
src_fp_mi: bytes = stdlib.malloc(inc_dir_len_mi + 1 + rel_path_len_mi + 1)
|
|
if src_fp_mi is None:
|
|
stdlib.free(check_pat_mi)
|
|
if check_fd_mi is not None: stdlib.free(check_fd_mi)
|
|
VLogger.error("includes source path allocation failed, aborting compilation", "PhaseB+")
|
|
viperlib.snprintf(src_fp_mi, inc_dir_len_mi + 1 + rel_path_len_mi + 1, "%s/%s", incdir_mi, inc_rel_mi)
|
|
|
|
# Check whether it is a declaration-only file (read {sha1}.meta: no FUNC ... DEF means pure declaration)
|
|
is_decl_mi: int = StubMergerIR.IsDeclOnlyModule(temp_dir, td_len_mi, inc_sha1_mi)
|
|
|
|
# is_decl_mi: -1 = no meta (Phase1 did not translate it), 1 = declaration-only, 0 = impl.
|
|
# Skip only declaration-only (is_decl_mi == 1); -1 and 0 continue to the translation logic.
|
|
if is_decl_mi == 1:
|
|
stdlib.free(check_pat_mi)
|
|
if check_fd_mi is not None: stdlib.free(check_fd_mi)
|
|
stdlib.free(src_fp_mi)
|
|
continue
|
|
|
|
# Try to read the in-memory FULL artifact {sha1}.full.ll (should be produced by Phase1/PhaseA)
|
|
inc_combined_mi: bytes = stdlib.malloc(COMBINED_IR_SIZE)
|
|
inc_combined_len_mi: sizet = 0
|
|
if inc_combined_mi is not None: inc_combined_len_mi = StubMergerBuild.LoadFullIR(temp_dir, inc_sha1_mi, inc_combined_mi, COMBINED_IR_SIZE)
|
|
|
|
# If full.ll is missing, translate the source, save stub/text/meta/full, then retry
|
|
if inc_combined_len_mi == 0 and inc_combined_mi is not None:
|
|
# Compute the includes file's package name (the directory part relative to includes_dir)
|
|
inc_pkg_mi: str = None
|
|
if string.strlen(src_fp_mi) > inc_dir_len_mi + 1:
|
|
inc_rel_mi2: str = src_fp_mi + inc_dir_len_mi + 1
|
|
inc_pkg_mi = GenApi.compute_package_from_relpath(mb, inc_rel_mi2)
|
|
tr_mi: HandlesTranslator.Translator* = BuildPipeline.TranslateFileGetTrans(mb, src_fp_mi, inc_sha1_mi, inc_pkg_mi)
|
|
if tr_mi is not None:
|
|
# PBP IR buffer: same as PHASE_A_IR_SIZE (parser.vp's text.ll >1MB truncation root cause)
|
|
PBP_IR_SIZE: sizet = 8388608
|
|
# FULL dump = stub + text, with 2x headroom
|
|
PBP_FULL_SIZE: sizet = 16777216
|
|
# Save stub.ll (sliced path)
|
|
inc_stub_buf: bytes = stdlib.malloc(PBP_IR_SIZE)
|
|
if inc_stub_buf is not None:
|
|
tr_mi.dump_ir(inc_stub_buf, PBP_IR_SIZE, mlirlite.OUTPUT_STUB)
|
|
inc_stub_len: sizet = string.strlen(inc_stub_buf)
|
|
inc_stub_path: str = StubMergerCollect._sliced_path(temp_dir, td_len_mi, inc_sha1_mi, "stub.ll")
|
|
if inc_stub_path is not None:
|
|
isf: fileio.File* = fileio.File(inc_stub_path, fileio.MODE.W)
|
|
if not isf.closed:
|
|
isf.write(inc_stub_buf, inc_stub_len)
|
|
isf.close()
|
|
stdlib.free(inc_stub_path)
|
|
stdlib.free(inc_stub_buf)
|
|
# Save text.ll (sliced path)
|
|
inc_text_buf: bytes = stdlib.malloc(PBP_IR_SIZE)
|
|
if inc_text_buf is not None:
|
|
tr_mi.dump_ir(inc_text_buf, PBP_IR_SIZE, mlirlite.OUTPUT_TEXT)
|
|
inc_text_len: sizet = string.strlen(inc_text_buf)
|
|
inc_text_path: str = StubMergerCollect._sliced_path(temp_dir, td_len_mi, inc_sha1_mi, "text.ll")
|
|
if inc_text_path is not None:
|
|
itf: fileio.File* = fileio.File(inc_text_path, fileio.MODE.W)
|
|
if not itf.closed:
|
|
itf.write(inc_text_buf, inc_text_len)
|
|
itf.close()
|
|
stdlib.free(inc_text_path)
|
|
stdlib.free(inc_text_buf)
|
|
# Save {sha1}.meta (structured sidecar: function list / struct layout / vtable)
|
|
StubMergerIR.WriteModuleMeta(tr_mi.Module, temp_dir, td_len_mi, inc_sha1_mi)
|
|
# Save {sha1}.full.ll (in-memory FULL artifact)
|
|
inc_full_buf: bytes = stdlib.malloc(PBP_FULL_SIZE)
|
|
if inc_full_buf is None: VLogger.error("full.ll buffer allocation failed", "PhaseB+")
|
|
tr_mi.dump_ir(inc_full_buf, PBP_FULL_SIZE, mlirlite.OUTPUT_FULL)
|
|
inc_full_len: sizet = string.strlen(inc_full_buf)
|
|
inc_full_path: str = StubMergerCollect._sliced_path(temp_dir, td_len_mi, inc_sha1_mi, "full.ll")
|
|
if inc_full_path is not None:
|
|
ifff: fileio.File* = fileio.File(inc_full_path, fileio.MODE.W)
|
|
if not ifff.closed:
|
|
ifff.write(inc_full_buf, inc_full_len)
|
|
ifff.close()
|
|
stdlib.free(inc_full_path)
|
|
stdlib.free(inc_full_buf)
|
|
# --traceback: take this module's debug call-site records and write <sha1>.vpcall.
|
|
if BuildPipeline._traceback_mode != 0:
|
|
vpcall_text_mi: str = mlirlite.DebugTakeText()
|
|
if vpcall_text_mi is not None:
|
|
BuildPipeline.WriteCallSidecar(temp_dir, inc_sha1_mi, vpcall_text_mi)
|
|
stdlib.free(vpcall_text_mi)
|
|
# Retry reading full.ll
|
|
inc_combined_len_mi = StubMergerBuild.LoadFullIR(temp_dir, inc_sha1_mi, inc_combined_mi, COMBINED_IR_SIZE)
|
|
# OOM cleanup: stub/text/meta/full are on disk, free this includes file's transient
|
|
# translation data (AST/tokens/lexer/IR/Handler); reads only touch files.
|
|
mb.reset_to(mark2)
|
|
|
|
if inc_combined_len_mi == 0:
|
|
# Diagnostic: check whether full.ll/stub.ll/text.ll exist
|
|
diag_full: str = StubMergerCollect._sliced_path(temp_dir, td_len_mi, inc_sha1_mi, "full.ll")
|
|
diag_stub: str = StubMergerCollect._sliced_path(temp_dir, td_len_mi, inc_sha1_mi, "stub.ll")
|
|
diag_text: str = StubMergerCollect._sliced_path(temp_dir, td_len_mi, inc_sha1_mi, "text.ll")
|
|
diag_full_ex: int = 0
|
|
diag_stub_ex: int = 0
|
|
diag_text_ex: int = 0
|
|
if diag_full is not None:
|
|
df_diag_f: fileio.File* = fileio.File(diag_full, fileio.MODE.R)
|
|
if not df_diag_f.closed:
|
|
diag_full_ex = 1
|
|
df_diag_f.close()
|
|
stdlib.free(diag_full)
|
|
if diag_stub is not None:
|
|
df_diag_s: fileio.File* = fileio.File(diag_stub, fileio.MODE.R)
|
|
if not df_diag_s.closed:
|
|
diag_stub_ex = 1
|
|
df_diag_s.close()
|
|
stdlib.free(diag_stub)
|
|
if diag_text is not None:
|
|
df_diag_t: fileio.File* = fileio.File(diag_text, fileio.MODE.R)
|
|
if not df_diag_t.closed:
|
|
diag_text_ex = 1
|
|
df_diag_t.close()
|
|
stdlib.free(diag_text)
|
|
VLogger.error(f"failed to read full.ll: {src_fp_mi} (full={diag_full_ex} stub={diag_stub_ex} text={diag_text_ex} is_decl={is_decl_mi})", "PhaseB+")
|
|
if inc_combined_mi is not None: stdlib.free(inc_combined_mi)
|
|
stdlib.free(check_pat_mi)
|
|
if check_fd_mi is not None: stdlib.free(check_fd_mi)
|
|
stdlib.free(src_fp_mi)
|
|
continue
|
|
|
|
# Compile to .obj (output to includes_binary_dir, consistent with the existence check and link collection)
|
|
inc_cret_mi: int = BuildPipeline.compile_module_to_obj(inc_combined_mi, inc_combined_len_mi, temp_dir, includes_binary_dir, inc_sha1_mi, cc_cmd, cc_flags)
|
|
stdlib.free(inc_combined_mi)
|
|
|
|
if inc_cret_mi != 0:
|
|
VLogger.error(f"llc compilation failed, aborting compilation: {src_fp_mi}", "PhaseB+")
|
|
sys.exit(1)
|
|
|
|
inc_compiled += 1
|
|
# Show the compiled includes file
|
|
VLogger.info(f"[includes {inc_compiled}] compilation complete: {src_fp_mi}", "PhaseB+")
|
|
|
|
# Add to obj_paths (sliced path, taken from includes_binary_dir)
|
|
ibd_len_mi2: sizet = string.strlen(includes_binary_dir)
|
|
inc_obj_sliced: str = StubMergerCollect._sliced_path(includes_binary_dir, ibd_len_mi2, inc_sha1_mi, "obj")
|
|
|
|
if inc_obj_sliced is not None:
|
|
inc_obj_len: sizet = string.strlen(inc_obj_sliced)
|
|
if obj_pos + inc_obj_len + 2 < OBJ_PATHS_SIZE:
|
|
if obj_pos > 0:
|
|
obj_paths[obj_pos] = ' '
|
|
obj_pos += 1
|
|
string.strcpy(obj_paths + obj_pos, inc_obj_sliced)
|
|
obj_pos += inc_obj_len
|
|
obj_paths[obj_pos] = '\0'
|
|
else: VLogger.error("includes .obj path buffer overflow, aborting compilation", "PhaseB+")
|
|
stdlib.free(inc_obj_sliced)
|
|
|
|
# Free this iteration's temporary memory
|
|
if check_pat_mi is not None: stdlib.free(check_pat_mi)
|
|
|
|
if check_fd_mi is not None: stdlib.free(check_fd_mi)
|
|
|
|
stdlib.free(src_fp_mi)
|
|
|
|
# Free the topological indices and local map copies
|
|
if topo_idx is not None: stdlib.free(topo_idx)
|
|
if store_sha1_arr_mi is not None: stdlib.free(store_sha1_arr_mi)
|
|
if store_mod_arr_mi is not None: stdlib.free(store_mod_arr_mi)
|
|
|
|
VLogger.info(f"compiled missing includes: {inc_compiled} entries", "PhaseB+")
|
|
|
|
# Free app_deps_buf (fallback when the Phase B+ block did not run)
|
|
if app_deps_buf is not None:
|
|
stdlib.free(app_deps_buf)
|
|
app_deps_buf = None
|
|
|
|
# === 4. Phase C: link all .obj -> .exe ===
|
|
log.banner("Phase C: link")
|
|
|
|
od_len2: sizet = string.strlen(output_dir)
|
|
lo_len: sizet = string.strlen(linker_output)
|
|
exe_path: bytes = stdlib.malloc(od_len2 + lo_len + 2)
|
|
if exe_path is not None: viperlib.snprintf(exe_path, od_len2 + lo_len + 2, "%s/%s", output_dir, linker_output)
|
|
else:
|
|
exe_path = linker_output
|
|
VLogger.error("exe_path allocation failed, aborting compilation", "PhaseC")
|
|
|
|
# Build the final .obj path list: main_obj_path first, reachable .objs after
|
|
final_obj_paths: bytes = stdlib.malloc(OBJ_PATHS_SIZE + 512)
|
|
if final_obj_paths is None:
|
|
stdlib.free(entries)
|
|
return 1
|
|
final_obj_paths[0] = '\0'
|
|
fop_pos: sizet = 0
|
|
# main_obj_path goes first (so the linker prefers the user main)
|
|
if main_obj_path[0] != '\0':
|
|
mlen: sizet = string.strlen(main_obj_path)
|
|
string.strcpy(final_obj_paths, main_obj_path)
|
|
fop_pos = mlen
|
|
|
|
# Merge the App non-main modules' .obj (obj_paths, space-separated) compiled by Phase B/B+.
|
|
# final_obj_paths must include main_obj + reachable includes + App deps, else linking hits undefined symbols.
|
|
if obj_pos > 0:
|
|
if fop_pos + obj_pos + 1 < OBJ_PATHS_SIZE + 512:
|
|
if fop_pos > 0:
|
|
final_obj_paths[fop_pos] = ' '
|
|
fop_pos += 1
|
|
string.strcpy(final_obj_paths + fop_pos, obj_paths)
|
|
fop_pos += obj_pos
|
|
final_obj_paths[fop_pos] = '\0'
|
|
|
|
# Link by graph: iterate reachable_set and collect only reachable includes' .obj,
|
|
# no longer collecting every .obj under includes.binary.
|
|
if reachable_set is not None and reachable_count > 0:
|
|
ibd_len_pc: sizet = string.strlen(includes_binary_dir)
|
|
td_len_pc: sizet = string.strlen(temp_dir)
|
|
ri_pc: int
|
|
for ri_pc in range(int(reachable_count)):
|
|
rsha1_pc: str = reachable_set + sizet(ri_pc) * 17
|
|
if rsha1_pc[0] == '\0': continue
|
|
# Skip App sources: their .obj is already in final_obj_paths; collecting a same-sha1 .obj from includes_binary would duplicate symbols.
|
|
app_skip_pc: int = 0
|
|
for ai_pc in range(int(file_count)):
|
|
ea_pc: uint64 = uint64(entries) + ai_pc * entry_size
|
|
ent_pc: SrcFileEntry* = (SrcFileEntry*)(ea_pc)
|
|
if ent_pc is not None and ent_pc.Sha1 is not None:
|
|
if string.strcmp(ent_pc.Sha1, rsha1_pc) == 0:
|
|
app_skip_pc = 1
|
|
break
|
|
if app_skip_pc != 0: continue
|
|
obj_path_pc: str = StubMergerCollect._sliced_path(includes_binary_dir, ibd_len_pc, rsha1_pc, "obj")
|
|
if obj_path_pc is not None:
|
|
# Check whether the .obj file exists
|
|
cf_pc: fileio.File* = fileio.File(obj_path_pc, fileio.MODE.R)
|
|
if not cf_pc.closed:
|
|
cf_pc.close()
|
|
# The .obj exists; append to final_obj_paths
|
|
op_len_pc: sizet = string.strlen(obj_path_pc)
|
|
if fop_pos + op_len_pc + 2 < OBJ_PATHS_SIZE + 512:
|
|
if fop_pos > 0:
|
|
final_obj_paths[fop_pos] = ' '
|
|
fop_pos += 1
|
|
string.strcpy(final_obj_paths + fop_pos, obj_path_pc)
|
|
fop_pos += op_len_pc
|
|
final_obj_paths[fop_pos] = '\0'
|
|
else:
|
|
# .obj missing: a declaration-only module (no define lines) needs no .obj, so skip silently;
|
|
# under phase2command mode the missing .obj is expected and the path is still added.
|
|
if BuildPipeline._phase2_cmd_path is not None:
|
|
op_len_p2: sizet = string.strlen(obj_path_pc)
|
|
if fop_pos + op_len_p2 + 2 < OBJ_PATHS_SIZE + 512:
|
|
if fop_pos > 0:
|
|
final_obj_paths[fop_pos] = ' '
|
|
fop_pos += 1
|
|
string.strcpy(final_obj_paths + fop_pos, obj_path_pc)
|
|
fop_pos += op_len_p2
|
|
final_obj_paths[fop_pos] = '\0'
|
|
elif StubMergerIR.IsDeclOnlyModule(temp_dir, td_len_pc, rsha1_pc) != 1: VLogger.error(f"reachable module .obj missing: {obj_path_pc}", "PhaseC")
|
|
stdlib.free(obj_path_pc)
|
|
|
|
final_obj_paths[fop_pos] = '\0'
|
|
obj_paths_len: sizet = fop_pos
|
|
|
|
# Free reachable_set (Phase C is done with it)
|
|
if reachable_set is not None:
|
|
stdlib.free(reachable_set)
|
|
reachable_set = None
|
|
|
|
# Link by graph: pass None for includes_binary_dir to avoid collecting everything
|
|
lret: int = BuildPipeline.link_objs_to_exe(final_obj_paths, obj_paths_len, linker_cmd, linker_flags, exe_path, None, linker_static_libs, linker_dynamic_libs)
|
|
|
|
if lret == 0:
|
|
VLogger.success(f"output: {exe_path}", "PhaseC")
|
|
# Deploy declared runtime deps (project.vpj runtime.*) next to the exe; fail-fast on error.
|
|
# Skipped under --phase2command (the exe is not linked yet).
|
|
if BuildPipeline._phase2_cmd_path is None:
|
|
if BuildPipeline.deploy_runtime_deps(exe_path) != 0:
|
|
VLogger.error("runtime dependency deployment failed, aborting compilation", "runtime")
|
|
stdlib.free(entries)
|
|
stdlib.free(asm_entries)
|
|
return 1
|
|
# --traceback: export <exe>.stackmap (skipped under --phase2command,
|
|
# where nothing was actually linked).
|
|
if BuildPipeline._traceback_mode != 0 and BuildPipeline._phase2_cmd_path is None: BuildPipeline.WriteStackMap(mb, exe_path)
|
|
if args.get_bool("run") and BuildPipeline._phase2_cmd_path is not None:
|
|
# phase2command mode: the exe is not linked yet (the command is in the bat); skip --run
|
|
VLogger.info("phase2command mode: skipping --run (exe not linked yet)", "phase2cmd")
|
|
elif args.get_bool("run"):
|
|
VLogger.info(f"run: {exe_path}", "run")
|
|
rp: subprocess.CompletedProcess* = subprocess.run(exe_path, False, False)
|
|
if rp is not None: VLogger.info(f"exit code: {rp.returncode}", "run")
|
|
else:
|
|
VLogger.error("link failed", "PhaseC")
|
|
stdlib.free(entries)
|
|
stdlib.free(asm_entries)
|
|
return 1
|
|
|
|
stdlib.free(entries)
|
|
stdlib.free(asm_entries)
|
|
return 0
|