ViperC/App/lib/Phase/Phase1/Phase1Scan.vp

from stdint import *
import string
import stdlib
import memhub
import win32.fileio as fileio
import ast
import lib.core.VLogger as VLogger
import lib.core.IncludesScanner as IncludesScanner
import lib.core.CompilerFactory as CompilerFactory
import lib.Stub.StubMerger.StubMergerCollect as StubMergerCollect

import lib.Phase.Phase1 as Phase1

import lib.core.Gen.GenApi as GenApi


# Phase1Scan - scan class inheritance, generic-class markers and container usage.
# Functions live in this module; globals are accessed cross-module via Phase1._xxx.

# Topological sort stores class info/deps in flat byte arrays (names in
# class_names_buf/parent_names_buf, SHA1s in def_sha1s_buf) to avoid NULL str fields.


# _ScanClassInheritance - pre-scan includes files, parse ASTs and collect ClassDef
# names/parent names into flat byte arrays; returns the class count (-1 on failure).
def _ScanClassInheritance(mb: memhub.MemBuddy*, result: IncludesScanner.ScanResult*, reachable_set: str, reachable_count: int, use_reachable: int, sha1_set: str, set_count: int, class_names_buf: bytes, parent_names_buf: bytes, def_sha1s_buf: bytes, generic_file_indices: rawptr, generic_file_count_box: rawptr) -> int:
    """Pre-scan includes files and collect class inheritance info."""
    if result is None: return -1
    # The three flat buffers are mb.alloc'ed by the caller; a NULL buffer would make
    # strcpy write the zero page, so fail-fast abort instead of returning -1 silently.
    if class_names_buf is None or parent_names_buf is None or def_sha1s_buf is None: VLogger.error("class-inheritance pre-scan flat buffer allocation failed (class_names_buf/parent_names_buf/def_sha1s_buf is NULL)", "Phase1")

    entry_size: sizet = IncludesScanner.FileEntry.__sizeof__()
    class_count: int = 0

    for i in range(int(result.Count)):
        # Per-file mark: reclaim this file's AST right after collecting inheritance
        # info (keeping all ASTs alive until mark1 would OOM the 1GB main pool).
        scan_mark: sizet = mb.mark()
        entry_addr: uint64 = uint64(result.Entries) + i * entry_size
        entry: IncludesScanner.FileEntry* = IncludesScanner.FileEntry*(entry_addr)
        if entry is None or entry.Sha1 is None: continue

        sha1_e: str = entry.Sha1
        file_has_generic: int = 0

        # on-demand translation filter
        in_set: int = 0
        if use_reachable != 0: in_set = StubMergerCollect._is_in_sha1_set(sha1_e, reachable_set, reachable_count)
        else: in_set = StubMergerCollect._is_in_sha1_set(sha1_e, sha1_set, set_count)
        if in_set == 0: continue

        # read the file
        file_path: str = entry.Path
        f: fileio.File* = fileio.File(file_path, fileio.MODE.R)
        if f.closed: continue

        # read_all's capacity includes the terminator (reads at most cap-1 bytes); longer files are truncated
        if f.size() >= Phase1.SRC_BUF_SIZE:
            VLogger.error(f"source file exceeds the {Phase1.SRC_BUF_SIZE}-byte limit, refusing to read: {file_path}", "Phase1")
            f.close()
            continue

        src_buf: bytes = stdlib.malloc(Phase1.SRC_BUF_SIZE)
        if src_buf is None:
            f.close()
            continue

        bytes_read: LONG = f.read_all(src_buf, Phase1.SRC_BUF_SIZE)
        f.close()
        if bytes_read <= 0:
            stdlib.free(src_buf)
            continue
        if bytes_read < Phase1.SRC_BUF_SIZE: src_buf[bytes_read] = 0

        # parse the AST
        lx: ast.Lexer* = ast.new_lexer(mb)
        if lx is None:
            stdlib.free(src_buf)
            continue
        ast._lexer_init(lx, src_buf, mb)
        tokens: ast.Token* = ast.tokenize(lx)
        tree: ast.AST* = ast.parse_tokens(mb, tokens)
        if tree is None:
            stdlib.free(src_buf)
            continue

        # walk top-level AST nodes, collect ClassDef
        ch: list[ast.AST*]* = tree.children
        if ch is not None:
            cn: sizet = len(ch)
            for ci in range(int(cn)):
                child: ast.AST* = ch.get(ci)
                if child is None: continue
                kd: int = child.kind()
                if kd != ast.ASTKind.ClassDef: continue

                cd: ast.ClassDef* = ast.ClassDef*(child)
                if cd is None or cd.name is None: continue

                # detect generic classes (type_params); record the file index for Phase 1a
                if file_has_generic == 0:
                    tp_scan: list[str]* = cd.type_params
                    if tp_scan is not None:
                        file_has_generic = 1
                        if generic_file_count_box is not None:
                            gfc_ptr_scan: int* = (int*)(generic_file_count_box)
                            if gfc_ptr_scan is not None:
                                gfc_scan: int = gfc_ptr_scan[0]
                                if gfc_scan < Phase1.MAX_FILE_DEPS and generic_file_indices is not None:
                                    gi_addr_scan: uint64 = uint64(generic_file_indices) + gfc_scan * 4
                                    gi_ptr_scan: int* = (int*)(gi_addr_scan)
                                    if gi_ptr_scan is not None: gi_ptr_scan[0] = i
                                    gfc_ptr_scan[0] = gfc_scan + 1

                if class_count >= Phase1.MAX_CLASSES: break

                # fill flat array: class name (offset = class_count * Phase1.CLASS_NAME_LEN)
                name_dst: str = class_names_buf + class_count * Phase1.CLASS_NAME_LEN
                nm_len: sizet = string.strlen(cd.name)
                if nm_len >= Phase1.CLASS_NAME_LEN:
                    string.strncpy(name_dst, cd.name, Phase1.CLASS_NAME_LEN - 1)
                    name_dst[Phase1.CLASS_NAME_LEN - 1] = '\0'
                else: string.strcpy(name_dst, cd.name)

                # fill flat array: parent name (no parent -> '\0')
                parent_dst: str = parent_names_buf + class_count * Phase1.CLASS_NAME_LEN
                parent_dst[0] = '\0'
                parent_nm: str = GenApi._get_parent_class(cd, None)
                # parent_nm is an internal AST pointer; no need to free
                if parent_nm is not None:
                    pnm_len: sizet = string.strlen(parent_nm)
                    if pnm_len >= Phase1.CLASS_NAME_LEN:
                        string.strncpy(parent_dst, parent_nm, Phase1.CLASS_NAME_LEN - 1)
                        parent_dst[Phase1.CLASS_NAME_LEN - 1] = '\0'
                    else: string.strcpy(parent_dst, parent_nm)

                # fill flat array: defining SHA1 (offset = class_count * Phase1.SHA1_LEN)
                sha1_dst: str = def_sha1s_buf + class_count * Phase1.SHA1_LEN
                string.strcpy(sha1_dst, sha1_e)

                class_count += 1

        stdlib.free(src_buf)
        # reclaim this file's AST/lexer/token (names/SHA1 already copied into flat arrays)
        mb.reset_to(scan_mark)

    return class_count
# _BufHasGenericClassMarker - text scan for a generic-class marker (PEP 695 "class
# Name["). Used to pre-filter files before registering generic templates.
def _BufHasGenericClassMarker(buf: bytes, blen: sizet) -> int:
    """Return 1 if buf contains a 'class identifier[' generic-class marker, else 0."""
    if buf is None or blen <= 6: return 0
    pos: sizet = 0
    while pos + 5 < blen:
        if buf[pos] == 'c' and buf[pos + 1] == 'l' and buf[pos + 2] == 'a' and buf[pos + 3] == 's' and buf[pos + 4] == 's':
            # 'class' preceded by a word boundary (line start or non-identifier)
            prev_ok: int = 1
            if pos > 0:
                pc: char = buf[pos - 1]
                if(pc >= 'a' and pc <= 'z') or(pc >= 'A' and pc <= 'Z') or(pc >= '0' and pc <= '9') or pc == '_': prev_ok = 0
            if prev_ok != 0:
                p: sizet = pos + 5
                # skip whitespace
                while p < blen and(buf[p] == ' ' or buf[p] == '\t'): p += 1
                # read the class-name identifier
                if p < blen and((buf[p] >= 'a' and buf[p] <= 'z') or(buf[p] >= 'A' and buf[p] <= 'Z') or buf[p] == '_'):
                    while p < blen and((buf[p] >= 'a' and buf[p] <= 'z') or(buf[p] >= 'A' and buf[p] <= 'Z') or(buf[p] >= '0' and buf[p] <= '9') or buf[p] == '_'):
                        p += 1
                    # non-space after the name is '[' -> generic class
                    while p < blen and(buf[p] == ' ' or buf[p] == '\t'): p += 1
                    if p < blen and buf[p] == '[': return 1
            pos += 1
        else: pos += 1
    return 0
# _BufHasGenericFuncMarker - text scan for a generic-function marker (PEP 695
# "def name["); cheap pre-filter before parsing for generic function templates.
def _BufHasGenericFuncMarker(buf: bytes, blen: sizet) -> int:
    """Return 1 if buf contains a 'def identifier[' generic-function marker, else 0."""
    if buf is None or blen <= 4: return 0
    pos: sizet = 0
    while pos + 3 < blen:
        if buf[pos] == 'd' and buf[pos + 1] == 'e' and buf[pos + 2] == 'f':
            # 'def' preceded by a word boundary (line start or non-identifier)
            prev_ok: int = 1
            if pos > 0:
                pc: char = buf[pos - 1]
                if(pc >= 'a' and pc <= 'z') or(pc >= 'A' and pc <= 'Z') or(pc >= '0' and pc <= '9') or pc == '_': prev_ok = 0
            if prev_ok != 0:
                p: sizet = pos + 3
                # skip whitespace
                while p < blen and(buf[p] == ' ' or buf[p] == '\t'): p += 1
                # read the function-name identifier
                if p < blen and((buf[p] >= 'a' and buf[p] <= 'z') or(buf[p] >= 'A' and buf[p] <= 'Z') or buf[p] == '_'):
                    while p < blen and((buf[p] >= 'a' and buf[p] <= 'z') or(buf[p] >= 'A' and buf[p] <= 'Z') or(buf[p] >= '0' and buf[p] <= '9') or buf[p] == '_'):
                        p += 1
                    # non-space after the name is '[' -> generic function
                    while p < blen and(buf[p] == ' ' or buf[p] == '\t'): p += 1
                    if p < blen and buf[p] == '[': return 1
            pos += 1
        else: pos += 1
    return 0
# _PreRegisterAllGenericTemplates - pre-register includes generic templates before
# mark1 so the lazy tables survive later resets; parses only marker-matched files.
def _PreRegisterAllGenericTemplates(mb: memhub.MemBuddy*, result: IncludesScanner.ScanResult*) -> int:
    """Pre-register generic templates from all includes files and return the count."""
    if result is None or mb is None: return 0
    perm: memhub.MemBuddy* = CompilerFactory.factory.PermPool
    if perm is None: return 0
    p1e_size: sizet = IncludesScanner.FileEntry.__sizeof__()
    handled_count: int = 0
    for i in range(int(result.Count)):
        entry_addr_pg: uint64 = uint64(result.Entries) + i * p1e_size
        entry_pg: IncludesScanner.FileEntry* = (IncludesScanner.FileEntry*)(entry_addr_pg)
        if entry_pg is None or entry_pg.Sha1 is None: continue
        # read the file
        file_path_pg: str = entry_pg.Path
        f_pg: fileio.File* = fileio.File(file_path_pg, fileio.MODE.R)
        if f_pg.closed: continue
        # read_all's capacity includes the terminator (reads at most cap-1 bytes); longer files are truncated
        if f_pg.size() >= Phase1.SRC_BUF_SIZE:
            VLogger.error(f"source file exceeds the {Phase1.SRC_BUF_SIZE}-byte limit, refusing to read: {file_path_pg}", "Phase1")
            f_pg.close()
            continue
        src_buf_pg: bytes = stdlib.malloc(Phase1.SRC_BUF_SIZE)
        if src_buf_pg is None:
            f_pg.close()
            continue
        bytes_read_pg: LONG = f_pg.read_all(src_buf_pg, Phase1.SRC_BUF_SIZE)
        f_pg.close()
        if bytes_read_pg <= 0:
            stdlib.free(src_buf_pg)
            continue
        if bytes_read_pg < Phase1.SRC_BUF_SIZE: src_buf_pg[bytes_read_pg] = 0
        # Text-scan filter: skip non-generic files (no parse, no residency, avoids
        # OOM). A class or function marker hit selects the file for parsing.
        if _BufHasGenericClassMarker(src_buf_pg, bytes_read_pg) == 0 and _BufHasGenericFuncMarker(src_buf_pg, bytes_read_pg) == 0:
            stdlib.free(src_buf_pg)
            continue
        # generic file: parse in perm_pool and register templates (survives reset)
        lx_pg: ast.Lexer* = ast.new_lexer(perm)
        if lx_pg is None:
            stdlib.free(src_buf_pg)
            continue
        ast._lexer_init(lx_pg, src_buf_pg, perm)
        tokens_pg: ast.Token* = ast.tokenize(lx_pg)
        tree_pg: ast.AST* = ast.parse_tokens(perm, tokens_pg)
        if tree_pg is None:
            stdlib.free(src_buf_pg)
            continue
        # walk top-level ClassDef/FunctionDef, register generic templates (type_params != None)
        ch_pg: list[ast.AST*]* = tree_pg.children
        if ch_pg is not None:
            cn_pg: sizet = len(ch_pg)
            for ci_pg in range(int(cn_pg)):
                child_pg: ast.AST* = ch_pg.get(ci_pg)
                if child_pg is None: continue
                ck_pg: int = child_pg.kind()
                if ck_pg == ast.ASTKind.ClassDef:
                    cd_pg: ast.ClassDef* = ast.ClassDef*(child_pg)
                    if cd_pg is None or cd_pg.name is None: continue
                    tp_pg: list[str]* = cd_pg.type_params
                    if tp_pg is None: continue
                    GenApi.set_current_file_type(file_path_pg)
                    GenApi._register_generic_template(perm, cd_pg, entry_pg.Sha1)
                    handled_count += 1
                elif ck_pg == ast.ASTKind.FunctionDef:
                    fd_pg: ast.FunctionDef* = ast.FunctionDef*(child_pg)
                    if fd_pg is None or fd_pg.name is None: continue
                    tf_pg: list[str]* = fd_pg.type_params
                    if tf_pg is None: continue
                    GenApi.set_current_file_type(file_path_pg)
                    GenApi._register_generic_func_template(perm, fd_pg, entry_pg.Sha1)
                    handled_count += 1
        stdlib.free(src_buf_pg)
    if handled_count > 0: VLogger.info(f"pre-registered {handled_count} includes generic templates (classes and functions) in the permanent pool before mark1", "Phase1")
    else: VLogger.info(f"pre-registered 0 generic templates in the permanent pool before mark1 (scanned {result.Count} files; the marker may not have matched or the full scan failed)", "Phase1")
    return handled_count
# _FindClassDefSha1 - look up a class name's defining file SHA1 in the flat arrays;
# returns a pointer into def_sha1s_buf, or None if not found.
def _FindClassDefSha1(class_names_buf: bytes, def_sha1s_buf: bytes, class_count: int, class_name: str) -> str:
    """Find a class name's defining file SHA1 in the flat arrays."""
    if class_names_buf is None or def_sha1s_buf is None or class_name is None: return None
    for i in range(int(class_count)):
        name_ptr: str = class_names_buf + i * Phase1.CLASS_NAME_LEN
        if string.strcmp(name_ptr, class_name) == 0: return def_sha1s_buf + i * Phase1.SHA1_LEN
    return None
# _ScanTreeForContainers - recursively scan the AST for list/dict container usage:
# recurse expression/annotation struct fields first, then statement-block children.

def _AnnIsContainer(ann: ast.AST*) -> int:
    """Return 1 if the annotation is a list/dict container, else 0."""
    while ann is not None:
        ak_c: int = ann.kind()
        if ak_c == ast.ASTKind.Name:
            nm_c: ast.Name* = ast.Name*(ann)
            if nm_c.id is not None:
                if string.strcmp(nm_c.id, "list") == 0: return 1
                if string.strcmp(nm_c.id, "dict") == 0: return 1
            return 0
        elif ak_c == ast.ASTKind.Subscript:
            sub_c: ast.Subscript* = ast.Subscript*(ann)
            ann = sub_c.value
        else: return 0
    return 0


def _ScanTreeForContainers(node: ast.AST*, need_list_box: rawptr, need_dict_box: rawptr, depth: int = 0):
    """Recursively scan the AST for list/dict container usage."""
    # Recursion depth guard: a self-referential AST cycle would recurse until the
    # stack is exhausted; fail-fast abort.
    if depth > 128: VLogger.error("container scan recursion too deep (AST cycle or nesting over 128 levels)", "Phase1")
    if node is None: return
    k_s: int = node.kind()
    if k_s == ast.ASTKind.Name:
        nm_s: ast.Name* = ast.Name*(node)
        if nm_s.id is not None:
            if string.strcmp(nm_s.id, "list") == 0: need_list_box[0] = 1
            elif string.strcmp(nm_s.id, "dict") == 0: need_dict_box[0] = 1
    elif k_s == ast.ASTKind.Subscript:
        sub_s: ast.Subscript* = ast.Subscript*(node)
        _ScanTreeForContainers(sub_s.value, need_list_box, need_dict_box, depth + 1)
        _ScanTreeForContainers(sub_s.slice, need_list_box, need_dict_box, depth + 1)
    elif k_s == ast.ASTKind.BinOp:
        bop_s: ast.BinOp* = ast.BinOp*(node)
        _ScanTreeForContainers(bop_s.left, need_list_box, need_dict_box, depth + 1)
        _ScanTreeForContainers(bop_s.right, need_list_box, need_dict_box, depth + 1)
    elif k_s == ast.ASTKind.Call:
        cl_s2: ast.Call* = ast.Call*(node)
        _ScanTreeForContainers(cl_s2.func, need_list_box, need_dict_box, depth + 1)
        cl_args_s: list[ast.AST*]* = cl_s2.args
        if cl_args_s is not None:
            na_s: sizet = len(cl_args_s)
            i_a2: sizet
            for i_a2 in range(int(na_s)): _ScanTreeForContainers(cl_args_s.get(i_a2), need_list_box, need_dict_box, depth + 1)
        cl_kws_s: list[ast.AST*]* = cl_s2.keywords
        if cl_kws_s is not None:
            nk_s: sizet = len(cl_kws_s)
            i_k2: sizet
            for i_k2 in range(int(nk_s)):
                kw_s: ast.Keyword* = ast.Keyword*(cl_kws_s.get(i_k2))
                if kw_s is not None: _ScanTreeForContainers(kw_s.value, need_list_box, need_dict_box, depth + 1)
    elif k_s == ast.ASTKind.Assign:
        asn_s: ast.Assign* = ast.Assign*(node)
        _ScanTreeForContainers(asn_s.value, need_list_box, need_dict_box, depth + 1)
        tgts_s: list[ast.AST*]* = asn_s.targets
        if tgts_s is not None:
            nt_s: sizet = len(tgts_s)
            i_t2: sizet
            for i_t2 in range(int(nt_s)): _ScanTreeForContainers(tgts_s.get(i_t2), need_list_box, need_dict_box, depth + 1)
    elif k_s == ast.ASTKind.AnnAssign:
        aa_s2: ast.AnnAssign* = ast.AnnAssign*(node)
        _ScanTreeForContainers(aa_s2.annotation, need_list_box, need_dict_box, depth + 1)
        _ScanTreeForContainers(aa_s2.value, need_list_box, need_dict_box, depth + 1)
    elif k_s == ast.ASTKind.FunctionDef:
        fn_s: ast.FunctionDef* = ast.FunctionDef*(node)
        _ScanTreeForContainers(fn_s.returns, need_list_box, need_dict_box, depth + 1)
        fn_args_s: ast.Arguments* = ast.Arguments*(fn_s.args)
        if fn_args_s is not None:
            fa_s: list[ast.AST*]* = fn_args_s.args
            if fa_s is not None:
                nf_s: sizet = len(fa_s)
                i_f2: sizet
                for i_f2 in range(int(nf_s)):
                    arg_s: ast.Arg* = ast.Arg*(fa_s.get(i_f2))
                    if arg_s is not None: _ScanTreeForContainers(arg_s.annotation, need_list_box, need_dict_box, depth + 1)
    elif k_s == ast.ASTKind.ClassDef:
        cd_s: ast.ClassDef* = ast.ClassDef*(node)
        bases_s: list[ast.AST*]* = cd_s.bases
        if bases_s is not None:
            nb_s: sizet = len(bases_s)
            i_b2: sizet
            for i_b2 in range(int(nb_s)): _ScanTreeForContainers(bases_s.get(i_b2), need_list_box, need_dict_box, depth + 1)
    elif k_s == ast.ASTKind.Attribute:
        at_s: ast.Attribute* = ast.Attribute*(node)
        _ScanTreeForContainers(at_s.value, need_list_box, need_dict_box, depth + 1)
    elif k_s == ast.ASTKind.Tuple:
        tu_s: ast.Tuple* = ast.Tuple*(node)
        elts_s: list[ast.AST*]* = tu_s.elts
        if elts_s is not None:
            ne_s: sizet = len(elts_s)
            i_e2: sizet
            for i_e2 in range(int(ne_s)): _ScanTreeForContainers(elts_s.get(i_e2), need_list_box, need_dict_box, depth + 1)
    elif k_s == ast.ASTKind.UnaryOp:
        uop_s: ast.UnaryOp* = ast.UnaryOp*(node)
        _ScanTreeForContainers(uop_s.operand, need_list_box, need_dict_box, depth + 1)
    elif k_s == ast.ASTKind.Expr:
        ex_s: ast.Expr* = ast.Expr*(node)
        _ScanTreeForContainers(ex_s.value, need_list_box, need_dict_box, depth + 1)
    elif k_s == ast.ASTKind.Return:
        rt_s: ast.Return* = ast.Return*(node)
        _ScanTreeForContainers(rt_s.value, need_list_box, need_dict_box, depth + 1)
    elif k_s == ast.ASTKind.Compare:
        cp_s: ast.Compare* = ast.Compare*(node)
        _ScanTreeForContainers(cp_s.left, need_list_box, need_dict_box, depth + 1)
        cps_s: list[ast.AST*]* = cp_s.comparators
        if cps_s is not None:
            nc_s: sizet = len(cps_s)
            i_c2: sizet
            for i_c2 in range(int(nc_s)): _ScanTreeForContainers(cps_s.get(i_c2), need_list_box, need_dict_box, depth + 1)
    elif k_s == ast.ASTKind.BoolOp:
        bo_s: ast.BoolOp* = ast.BoolOp*(node)
        bvs_s: list[ast.AST*]* = bo_s.values
        if bvs_s is not None:
            nv_s: sizet = len(bvs_s)
            i_v2: sizet
            for i_v2 in range(int(nv_s)): _ScanTreeForContainers(bvs_s.get(i_v2), need_list_box, need_dict_box, depth + 1)
    # `[...]`/`{}` literals do not trigger _list/_dict imports (compile-time constant lists, array init);
    # recurse into statement-block children.
    if node.children is not None:
        ch_s: list[ast.AST*]* = node.children
        n_s: sizet = len(ch_s)
        ci_s: sizet
        for ci_s in range(int(n_s)): _ScanTreeForContainers(ch_s.get(ci_s), need_list_box, need_dict_box, depth + 1)