ViperC/App/lib/core/IncludesScanner.vp

from stdint import *
import memhub
import string
import stdlib
import win32.win32file as win32file
import win32.win32base as win32base
import win32.fileio as fileio
import hashlib
import viperlib
import lib.Projectrans.Utils as Utils
import lib.core.VLogger as VLogger


# IncludesScanner - recursive scan of the includes directory (Win32 FindFirstFileA/
# FindNextFileA); collects .vp paths and SHA1s for Phase1 stub gen / Phase2 merge.

# global mbuddy pointer
_mbuddy: memhub.MemBuddy*

# max file path length
define MAX_PATH_LEN = 512
# max files per scan
define MAX_FILES = 1024


# FileEntry - file entry (path + SHA1)
novirtual class FileEntry:
    """A scanned file entry."""
    Path: str           # absolute file path
    Sha1: str           # first 16 SHA1 chars (16 bytes + null)
    RelPath: str        # path relative to the includes root
    ModuleName: str     # module name (e.g. "ast.parser")
    IncDir: str         # owning includes root dir (multi-root support)


# ScanResult - scan result
novirtual class ScanResult:
    """A collection of scan results."""
    Entries: FileEntry*   # FileEntry array
    Count: int
    Capacity: int


# create_scan_result - create the scan result container
def create_scan_result(pool: memhub.MemBuddy*) -> ScanResult*:
    """Create the scan result container with MAX_FILES preallocated slots."""
    if pool is None: return None
    size: sizet = MAX_FILES * FileEntry.__sizeof__()
    # use stdlib.malloc to avoid exhausting the mbuddy pool (consistent with StubMerger)
    entries: FileEntry* = stdlib.malloc(size)
    if entries is None: return None
    string.memset(entries, 0, size)

    result: ScanResult* = stdlib.malloc(ScanResult.__sizeof__())
    if result is None: return None
    string.memset(result, 0, ScanResult.__sizeof__())
    result.Entries = entries
    result.Count = 0
    result.Capacity = MAX_FILES
    return result


# add_file_entry - append a file entry to the scan result
def add_file_entry(result: ScanResult*, pool: memhub.MemBuddy*, abs_path: str, rel_path: str, sha1: str, inc_dir: str) -> int:
    """Append a file entry to the scan result; inc_dir is the owning includes root."""
    if result is None or pool is None:  return 1
    if result.Count >= result.Capacity: return 1

    # compute module name: '/' -> '.', strip .vp/.py (Utils handles __init__.vp)
    mod_buf: str = Utils.PathToModuleName(rel_path, 0)
    if mod_buf is None: return 1

    # get the entry address
    entry_size: sizet = FileEntry.__sizeof__()
    entry_addr: uint64 = uint64(result.Entries) + result.Count * entry_size
    entry: FileEntry* = (FileEntry*)(entry_addr)
    if entry is None: return 1

    # copy the path string
    abs_len: sizet = string.strlen(abs_path)
    abs_buf: str = stdlib.malloc(abs_len + 1)
    if abs_buf is None: return 1
    string.strcpy(abs_buf, abs_path)
    entry.Path = abs_buf

    # copy the SHA1
    sha1_buf: str = stdlib.malloc(17)
    if sha1_buf is None: return 1
    string.strcpy(sha1_buf, sha1)
    entry.Sha1 = sha1_buf

    # copy the relative path
    rel_buf: str = stdlib.malloc(string.strlen(rel_path) + 1)
    if rel_buf is None: return 1
    string.strcpy(rel_buf, rel_path)
    entry.RelPath = rel_buf

    # copy the module name
    entry.ModuleName = mod_buf

    # copy the owning includes root dir (used by Phase2 path joining)
    if inc_dir is not None:
        inc_len: sizet = string.strlen(inc_dir)
        inc_buf: str = stdlib.malloc(inc_len + 1)
        if inc_buf is None: return 1
        string.strcpy(inc_buf, inc_dir)
        entry.IncDir = inc_buf

    result.Count += 1
    return 0


# compute_file_sha1 - read file content and compute SHA1 (CRLF->LF, matching Projectrans.py)
def compute_file_sha1(pool: memhub.MemBuddy*, file_path: str) -> str:
    """Read a file, normalize CRLF to LF, and return the first 16 SHA1 hex chars."""
    if pool is None or file_path is None: return None

    # read file content; BUF_SIZE must match SRC_BUF_SIZE (1MB) or trailing edits won't change SHA1
    BUF_SIZE: sizet = 1048576
    # Refuse larger files: hashing only the first 1 MiB gives two files that differ later the same
    # SHA1, and the SHA1 keys the stub/cache files (colliding modules would reuse each other's IR).
    sf_sz: fileio.File* = fileio.File(file_path, fileio.MODE.R)
    if sf_sz.closed: return None
    too_big: int = 0
    if sf_sz.size() >= BUF_SIZE: too_big = 1
    sf_sz.close()
    if too_big != 0:
        VLogger.error(f"source file exceeds the {BUF_SIZE}-byte limit and cannot be hashed completely: {file_path}", "IncludesScanner")
        return None
    buf: bytes = Utils.read_file_to_buf(file_path, BUF_SIZE)
    if buf is None: return None

    # strip \r in place (CRLF -> LF), matching Projectrans.py text-mode reads
    buf_len: sizet = string.strlen(buf)
    write_pos: sizet = 0
    for ch in buf:
        if ch != '\r':
            buf[write_pos] = ch
            write_pos += 1

    # add a null terminator
    buf[write_pos] = 0

    # compute SHA1
    ctx: hashlib.sha1* = hashlib.sha1()
    if ctx is None:
        stdlib.free(buf)
        return None
    ctx.update(buf)

    digest: bytes = stdlib.malloc(hashlib.SHA1_DIGEST_LEN)
    if digest is None:
        stdlib.free(buf)
        return None
    ctx.final(digest)

    # convert to hex (first 8 bytes = 16 hex chars; stdlib allocation unchanged)
    hex_buf: str = Utils.bytes_to_hex(pool, digest, 8, 0, 1)
    stdlib.free(buf)
    stdlib.free(digest)
    return hex_buf


# scan_directory_recursive - recursively scan a directory (FindFirstFileA/FindNextFileA)
def scan_directory_recursive(pool: memhub.MemBuddy*, root_dir: str, rel_prefix: str, result: ScanResult*, inc_dir: str) -> int:
    """Recursively collect .vp/.py files under a directory; inc_dir is the includes root."""
    if pool is None or root_dir is None or result is None: return 1
    # subdir recursion failure count (not ignored; fail-fast at the summary)
    subdir_failed: int = 0

    # build search pattern: root_dir/*
    root_len: sizet = string.strlen(root_dir)
    pattern: bytes = stdlib.malloc(root_len + 4)
    if pattern is None: return 1
    viperlib.snprintf(pattern, root_len + 4, "%s/*", root_dir)

    # find_data buffer +16 slack: VPC's WIN32_FIND_DATAA.__sizeof__() undercounts Windows writes
    fd_size_s: sizet = win32file.WIN32_FIND_DATAA.__sizeof__()
    find_data: win32file.WIN32_FIND_DATAA* = stdlib.malloc(fd_size_s + 16)
    if find_data is None: return 1
    string.memset(find_data, 0, fd_size_s + 16)

    handle: win32base.HANDLE = win32file.FindFirstFileA(pattern, find_data)
    if handle == win32base.INVALID_HANDLE_VALUE: return 1

    # iterate all files and subdirectories
    while True:
        # skip . and ..
        fname: str = find_data.cFileName
        if fname is not None:
            fname0: char = fname[0]
            if fname0 == '.':
                fname1: char = fname[1]
                if fname1 == '\0':
                    # "."
                    if win32file.FindNextFileA(handle, find_data) == 0: break
                    continue
                elif fname1 == '.':
                    fname2: char = fname[2]
                    if fname2 == '\0':
                        # ".."
                        if win32file.FindNextFileA(handle, find_data) == 0: break
                        continue

        # check whether it is a directory
        is_dir: int = find_data.dwFileAttributes & win32file.FILE_ATTRIBUTE_DIRECTORY

        # build the full path
        fname_len: sizet = string.strlen(fname)
        full_path: bytes = Utils.JoinPath(root_dir, fname)
        if full_path is None: break

        # build the relative path
        prefix_len: sizet = 0
        if rel_prefix is not None: prefix_len = string.strlen(rel_prefix)
        rel_path: bytes = None
        if rel_prefix is not None and prefix_len > 0: rel_path = Utils.JoinPath(rel_prefix, fname)
        else:
            rel_path = stdlib.malloc(fname_len + 1)
            if rel_path is None: break
            string.strcpy(rel_path, fname)
        if rel_path is None: break

        if is_dir != 0:
            # recurse into the subdirectory
            if scan_directory_recursive(pool, full_path, rel_path, result, inc_dir) != 0: subdir_failed += 1
        else:
            # check for a .vp source file (Viper uses .vp only; no longer .py)
            is_py: int = 0
            if Utils.is_vp_at(fname, 0, fname_len) != 0: is_py = 1

            if is_py != 0:
                # skip files under __pycache__
                is_pycache: int = 0
                if rel_prefix is not None:
                    if string.strcmp(rel_prefix, "__pycache__") == 0: is_pycache = 1

                if is_pycache == 0:
                    # compute SHA1
                    sha1: str = compute_file_sha1(pool, full_path)
                    if sha1 is not None:
                        # fail-fast: a full table used to drop the file silently, so it never got a
                        # SHA1 entry / dependency edge and was therefore never compiled.
                        if add_file_entry(result, pool, full_path, rel_path, sha1, inc_dir) != 0:
                            VLogger.error(f"includes scan could not register '{full_path}' (the {MAX_FILES}-file capacity or an allocation failed); raise MAX_FILES", "scan")
                        VLogger.debug(f"scan: {rel_path} -> {sha1}", "scan")
                    else: VLogger.warning(f"SHA1 computation failed: {full_path}", "scan")

        # continue to the next file
        if win32file.FindNextFileA(handle, find_data) == 0: break

    win32file.FindClose(handle)
    if subdir_failed > 0: VLogger.error(f"subdirectory scan failed for {subdir_failed} entries", "scan")

    return 0


# scan_includes - scan entry point: return all .vp file paths and SHA1s
def scan_includes(pool: memhub.MemBuddy*, includes_dirs: str) -> ScanResult*:
    """Scan space-separated includes dirs and return all scanned .vp/.py files."""
    if pool is None or includes_dirs is None: return None

    result: ScanResult* = create_scan_result(pool)
    if result is None: return None

    # parse space-separated includes dirs, recursively scanning and merging each
    dirs_total: sizet = string.strlen(includes_dirs)
    d_i: sizet = 0
    while d_i < dirs_total:
        while d_i < dirs_total and includes_dirs[d_i] == ' ': d_i += 1
        if d_i >= dirs_total: break
        d_start: sizet = d_i
        while d_i < dirs_total and includes_dirs[d_i] != ' ': d_i += 1
        d_len: sizet = d_i - d_start
        if d_len == 0: continue
        dir_buf: bytes = stdlib.malloc(d_len + 1)
        if dir_buf is None: VLogger.error("failed to allocate the includes directory; aborting scan", "scan")
        string.strncpy(dir_buf, includes_dirs + d_start, d_len)
        dir_buf[d_len] = '\0'
        scan_directory_recursive(pool, dir_buf, None, result, dir_buf)
        stdlib.free(dir_buf)

    return result


# find_entry_by_module - find a file entry by module name
def find_entry_by_module(result: ScanResult*, module_name: str) -> FileEntry*:
    """Find a file entry by module name."""
    if result is None or module_name is None: return None
    entry_size: sizet = FileEntry.__sizeof__()
    for i in range(int(result.Count)):
        entry_addr: uint64 = uint64(result.Entries) + i * entry_size
        entry: FileEntry* = (FileEntry*)(entry_addr)
        if entry is not None:
            if entry.ModuleName is not None:
                if string.strcmp(entry.ModuleName, module_name) == 0: return entry
    return None


# _PathExistsA - path existence check (GetFileAttributesA)
def _PathExistsA(path: str) -> int:
    """Return 1 if the path exists, else 0, using GetFileAttributesA."""
    if path is None: return 0
    attr: ULONG = win32file.GetFileAttributesA(path)
    if attr == 0xFFFFFFFF:  # INVALID_FILE_ATTRIBUTES
        return 0
    return 1


# _TryResolveModuleFile - try one candidate path and write outputs (1=hit, 0=miss);
# out_abs_path/out_rel_path/out_inc_dir are rawptr output slots.
def _TryResolveModuleFile(inc_dir: str, rel_seg: bytes, rel_len: sizet, suffix: str, out_abs_path: rawptr, out_rel_path: rawptr, out_inc_dir: rawptr) -> int:
    if inc_dir is None or rel_seg is None or suffix is None: return 0
    inc_len: sizet = string.strlen(inc_dir)
    suf_len: sizet = string.strlen(suffix)
    # candidate absolute path: {inc}/{rel}{suffix} (hand-built to avoid %. *s)
    cand_len: sizet = inc_len + 1 + rel_len + suf_len
    cand: bytes = stdlib.malloc(cand_len + 1)
    if cand is None: return 0
    string.strcpy(cand, inc_dir)
    cand[inc_len] = '/'
    string.strncpy(cand + inc_len + 1, rel_seg, rel_len)
    string.strcpy(cand + inc_len + 1 + rel_len, suffix)
    if _PathExistsA(cand) == 0:
        stdlib.free(cand)
        return 0
    # path relative to the includes root: {rel}{suffix}
    rel_path: bytes = stdlib.malloc(rel_len + suf_len + 1)
    if rel_path is None:
        stdlib.free(cand)
        return 0
    string.strncpy(rel_path, rel_seg, rel_len)
    rel_path[rel_len] = '\0'
    string.strcat(rel_path, suffix)
    # Deep-copy the owning includes root dir; allocate everything before writing
    # slots to avoid a half-written state when inc allocation fails.
    inc_buf: bytes = None
    if out_inc_dir is not None:
        inc_buf = stdlib.malloc(inc_len + 1)
        if inc_buf is None:
            stdlib.free(cand)
            stdlib.free(rel_path)
            return 0
        string.strcpy(inc_buf, inc_dir)
    # Write output slots (stdlib deep copies; caller frees). Slots must be cast to
    # (sizet*): rawptr* folds to rawptr and [0] becomes a 1-byte access.
    if out_abs_path is not None:
        abs_slot: sizet* = (sizet*)(out_abs_path)
        abs_slot[0] = sizet(cand)
    else: stdlib.free(cand)
    if out_rel_path is not None:
        rel_slot: sizet* = (sizet*)(out_rel_path)
        rel_slot[0] = sizet(rel_path)
    else: stdlib.free(rel_path)
    if out_inc_dir is not None and inc_buf is not None:
        inc_slot: sizet* = (sizet*)(out_inc_dir)
        inc_slot[0] = sizet(inc_buf)
    return 1


# ResolveModuleToPath - module name -> file path, resolved on demand (no full scan).
# Tries each includes dir, preferring file modules over packages. Returns 0 found, 1 not.
def ResolveModuleToPath(includes_dirs: str, mod_name: str, out_abs_path: rawptr, out_rel_path: rawptr, out_inc_dir: rawptr) -> int:
    if includes_dirs is None or mod_name is None: return 1
    mn_len: sizet = string.strlen(mod_name)
    if mn_len == 0: return 1
    # mod_name -> relative path segment ('.' -> '/'): ast.parser -> ast/parser
    rel_seg: bytes = stdlib.malloc(mn_len + 1)
    if rel_seg is None: return 1
    rel_len: sizet = 0
    for mi in range(int(mn_len)):
        ch_m: char = mod_name[mi]
        if ch_m == '.': rel_seg[rel_len] = '/'
        else: rel_seg[rel_len] = ch_m
        rel_len += 1
    rel_seg[rel_len] = '\0'

    found: int = 0
    d_total: sizet = string.strlen(includes_dirs)
    d_i: sizet = 0
    while d_i < d_total:
        while d_i < d_total and includes_dirs[d_i] == ' ': d_i += 1
        if d_i >= d_total: break
        d_start: sizet = d_i
        while d_i < d_total and includes_dirs[d_i] != ' ': d_i += 1
        d_len: sizet = d_i - d_start
        if d_len == 0: continue
        inc_dir_buf: bytes = stdlib.malloc(d_len + 1)
        if inc_dir_buf is None: continue
        string.strncpy(inc_dir_buf, includes_dirs + d_start, d_len)
        inc_dir_buf[d_len] = '\0'
        # candidate 1: {inc}/{rel}.vp (file module)
        if _TryResolveModuleFile(inc_dir_buf, rel_seg, rel_len, ".vp", out_abs_path, out_rel_path, out_inc_dir) != 0:
            found = 1
            stdlib.free(inc_dir_buf)
            break
        # candidate 2: {inc}/{rel}/__init__.vp (package module)
        if _TryResolveModuleFile(inc_dir_buf, rel_seg, rel_len, "/__init__.vp", out_abs_path, out_rel_path, out_inc_dir) != 0:
            found = 1
        stdlib.free(inc_dir_buf)
        if found != 0: break
    stdlib.free(rel_seg)
    if found != 0: return 0
    return 1