import t, c from stdint import * import memhub import string import stdio import stdlib import win32.win32file as win32file import win32.win32base as win32base import hashlib import viperlib import lib.Projectrans.Utils as Utils import lib.core.VLogger as VLogger import lib.Projectrans.Config as Config # IncludesScanner - recursive scan of the includes directory (Win32 FindFirstFileA/ # FindNextFileA); collects .vp paths and SHA1s for Phase1 stub gen / Phase2 merge. # global mbuddy pointer _mbuddy: memhub.MemBuddy* # max file path length define MAX_PATH_LEN = 512 # max files per scan define MAX_FILES = 1024 # FileEntry - file entry (path + SHA1) novirtual class FileEntry: """A scanned file entry.""" Path: str # absolute file path Sha1: str # first 16 SHA1 chars (16 bytes + null) RelPath: str # path relative to the includes root ModuleName: str # module name (e.g. "ast.parser") IncDir: str # owning includes root dir (multi-root support) # ScanResult - scan result novirtual class ScanResult: """A collection of scan results.""" Entries: FileEntry* # FileEntry array Count: int Capacity: int # create_scan_result - create the scan result container def create_scan_result(pool: memhub.MemBuddy*) -> ScanResult*: """Create the scan result container with MAX_FILES preallocated slots.""" if pool is None: return None size: sizet = MAX_FILES * FileEntry.__sizeof__() # use stdlib.malloc to avoid exhausting the mbuddy pool (consistent with StubMerger) entries: FileEntry* = stdlib.malloc(size) if entries is None: return None string.memset(entries, 0, size) result: ScanResult* = stdlib.malloc(ScanResult.__sizeof__()) if result is None: return None string.memset(result, 0, ScanResult.__sizeof__()) result.Entries = entries result.Count = 0 result.Capacity = MAX_FILES return result # add_file_entry - append a file entry to the scan result def add_file_entry(result: ScanResult*, pool: memhub.MemBuddy*, abs_path: str, rel_path: str, sha1: str, inc_dir: str) -> int: """Append a file entry to the scan result; inc_dir is the owning includes root.""" if result is None or pool is None: return 1 if result.Count >= result.Capacity: return 1 # compute module name: '/' -> '.', strip .vp/.py (Utils handles __init__.vp) mod_buf: str = Utils.PathToModuleName(rel_path, 0) if mod_buf is None: return 1 # get the entry address entry_size: sizet = FileEntry.__sizeof__() entry_addr: uint64 = uint64(result.Entries) + result.Count * entry_size entry: FileEntry* = (FileEntry*)(entry_addr) if entry is None: return 1 # copy the path string abs_len: sizet = string.strlen(abs_path) abs_buf: str = stdlib.malloc(abs_len + 1) if abs_buf is None: return 1 string.strcpy(abs_buf, abs_path) entry.Path = abs_buf # copy the SHA1 sha1_buf: str = stdlib.malloc(17) if sha1_buf is None: return 1 string.strcpy(sha1_buf, sha1) entry.Sha1 = sha1_buf # copy the relative path rel_buf: str = stdlib.malloc(string.strlen(rel_path) + 1) if rel_buf is None: return 1 string.strcpy(rel_buf, rel_path) entry.RelPath = rel_buf # copy the module name entry.ModuleName = mod_buf # copy the owning includes root dir (used by Phase2 path joining) if inc_dir is not None: inc_len: sizet = string.strlen(inc_dir) inc_buf: str = stdlib.malloc(inc_len + 1) if inc_buf is None: return 1 string.strcpy(inc_buf, inc_dir) entry.IncDir = inc_buf result.Count += 1 return 0 # compute_file_sha1 - read file content and compute SHA1 (CRLF->LF, matching Projectrans.py) def compute_file_sha1(pool: memhub.MemBuddy*, file_path: str) -> str: """Read a file, normalize CRLF to LF, and return the first 16 SHA1 hex chars.""" if pool is None or file_path is None: return None # read file content; BUF_SIZE must match SRC_BUF_SIZE (1MB) or trailing edits won't change SHA1 BUF_SIZE: sizet = 1048576 buf: bytes = Utils.read_file_to_buf(file_path, BUF_SIZE) if buf is None: return None # strip \r in place (CRLF -> LF), matching Projectrans.py text-mode reads buf_len: sizet = string.strlen(buf) write_pos: sizet = 0 for ch in buf: if ch != '\r': buf[write_pos] = ch write_pos += 1 # add a null terminator buf[write_pos] = 0 # compute SHA1 ctx: hashlib.sha1* = hashlib.sha1() if ctx is None: stdlib.free(buf) return None ctx.update(buf) digest: bytes = stdlib.malloc(hashlib.SHA1_DIGEST_LEN) if digest is None: stdlib.free(buf) return None ctx.final(digest) # convert to hex (first 8 bytes = 16 hex chars; stdlib allocation unchanged) hex_buf: str = Utils.bytes_to_hex(pool, digest, 8, 0, 1) stdlib.free(buf) stdlib.free(digest) return hex_buf # scan_directory_recursive - recursively scan a directory (FindFirstFileA/FindNextFileA) def scan_directory_recursive(pool: memhub.MemBuddy*, root_dir: str, rel_prefix: str, result: ScanResult*, inc_dir: str) -> int: """Recursively collect .vp/.py files under a directory; inc_dir is the includes root.""" if pool is None or root_dir is None or result is None: return 1 # subdir recursion failure count (not ignored; fail-fast at the summary) subdir_failed: int = 0 # build search pattern: root_dir/* root_len: sizet = string.strlen(root_dir) pattern: bytes = stdlib.malloc(root_len + 4) if pattern is None: return 1 viperlib.snprintf(pattern, root_len + 4, "%s/*", root_dir) # find_data buffer +16 slack: VPC's WIN32_FIND_DATAA.__sizeof__() undercounts Windows writes fd_size_s: sizet = win32file.WIN32_FIND_DATAA.__sizeof__() find_data: win32file.WIN32_FIND_DATAA* = stdlib.malloc(fd_size_s + 16) if find_data is None: return 1 string.memset(find_data, 0, fd_size_s + 16) handle: win32base.HANDLE = win32file.FindFirstFileA(pattern, find_data) if handle == win32base.INVALID_HANDLE_VALUE: return 1 # iterate all files and subdirectories while True: # skip . and .. fname: str = find_data.cFileName if fname is not None: fname0: char = fname[0] if fname0 == '.': fname1: char = fname[1] if fname1 == '\0': # "." if win32file.FindNextFileA(handle, find_data) == 0: break continue elif fname1 == '.': fname2: char = fname[2] if fname2 == '\0': # ".." if win32file.FindNextFileA(handle, find_data) == 0: break continue # check whether it is a directory is_dir: int = find_data.dwFileAttributes & win32file.FILE_ATTRIBUTE_DIRECTORY # build the full path fname_len: sizet = string.strlen(fname) full_path: bytes = Utils.JoinPath(root_dir, fname) if full_path is None: break # build the relative path prefix_len: sizet = 0 if rel_prefix is not None: prefix_len = string.strlen(rel_prefix) rel_path: bytes = None if rel_prefix is not None and prefix_len > 0: rel_path = Utils.JoinPath(rel_prefix, fname) else: rel_path = stdlib.malloc(fname_len + 1) if rel_path is None: break string.strcpy(rel_path, fname) if rel_path is None: break if is_dir != 0: # recurse into the subdirectory if scan_directory_recursive(pool, full_path, rel_path, result, inc_dir) != 0: subdir_failed += 1 else: # check for a .vp source file (Viper uses .vp only; no longer .py) is_py: int = 0 if Utils.is_vp_at(fname, 0, fname_len) != 0: is_py = 1 if is_py != 0: # skip files under __pycache__ is_pycache: int = 0 if rel_prefix is not None: if string.strcmp(rel_prefix, "__pycache__") == 0: is_pycache = 1 if is_pycache == 0: # compute SHA1 sha1: str = compute_file_sha1(pool, full_path) if sha1 is not None: add_file_entry(result, pool, full_path, rel_path, sha1, inc_dir) VLogger.debug(f"scan: {rel_path} -> {sha1}", "scan") else: VLogger.warning(f"SHA1 computation failed: {full_path}", "scan") # continue to the next file if win32file.FindNextFileA(handle, find_data) == 0: break win32file.FindClose(handle) if subdir_failed > 0: VLogger.error(f"subdirectory scan failed for {subdir_failed} entries", "scan") return 0 # scan_includes - scan entry point: return all .vp file paths and SHA1s def scan_includes(pool: memhub.MemBuddy*, includes_dirs: str) -> ScanResult*: """Scan space-separated includes dirs and return all scanned .vp/.py files.""" if pool is None or includes_dirs is None: return None result: ScanResult* = create_scan_result(pool) if result is None: return None # parse space-separated includes dirs, recursively scanning and merging each dirs_total: sizet = string.strlen(includes_dirs) d_i: sizet = 0 while d_i < dirs_total: while d_i < dirs_total and includes_dirs[d_i] == ' ': d_i += 1 if d_i >= dirs_total: break d_start: sizet = d_i while d_i < dirs_total and includes_dirs[d_i] != ' ': d_i += 1 d_len: sizet = d_i - d_start if d_len == 0: continue dir_buf: bytes = stdlib.malloc(d_len + 1) if dir_buf is None: VLogger.error("failed to allocate the includes directory; aborting scan", "scan") string.strncpy(dir_buf, includes_dirs + d_start, d_len) dir_buf[d_len] = '\0' scan_directory_recursive(pool, dir_buf, None, result, dir_buf) stdlib.free(dir_buf) return result # find_entry_by_module - find a file entry by module name def find_entry_by_module(result: ScanResult*, module_name: str) -> FileEntry*: """Find a file entry by module name.""" if result is None or module_name is None: return None entry_size: sizet = FileEntry.__sizeof__() for i in range(int(result.Count)): entry_addr: uint64 = uint64(result.Entries) + i * entry_size entry: FileEntry* = (FileEntry*)(entry_addr) if entry is not None: if entry.ModuleName is not None: if string.strcmp(entry.ModuleName, module_name) == 0: return entry return None # _PathExistsA - path existence check (GetFileAttributesA) def _PathExistsA(path: str) -> int: """Return 1 if the path exists, else 0, using GetFileAttributesA.""" if path is None: return 0 attr: ULONG = win32file.GetFileAttributesA(path) if attr == 0xFFFFFFFF: # INVALID_FILE_ATTRIBUTES return 0 return 1 # _TryResolveModuleFile - try one candidate path and write outputs (1=hit, 0=miss); # out_abs_path/out_rel_path/out_inc_dir are t.CPtr output slots. def _TryResolveModuleFile(inc_dir: str, rel_seg: bytes, rel_len: sizet, suffix: str, out_abs_path: t.CPtr, out_rel_path: t.CPtr, out_inc_dir: t.CPtr) -> int: if inc_dir is None or rel_seg is None or suffix is None: return 0 inc_len: sizet = string.strlen(inc_dir) suf_len: sizet = string.strlen(suffix) # candidate absolute path: {inc}/{rel}{suffix} (hand-built to avoid %. *s) cand_len: sizet = inc_len + 1 + rel_len + suf_len cand: bytes = stdlib.malloc(cand_len + 1) if cand is None: return 0 string.strcpy(cand, inc_dir) cand[inc_len] = '/' string.strncpy(cand + inc_len + 1, rel_seg, rel_len) string.strcpy(cand + inc_len + 1 + rel_len, suffix) if _PathExistsA(cand) == 0: stdlib.free(cand) return 0 # path relative to the includes root: {rel}{suffix} rel_path: bytes = stdlib.malloc(rel_len + suf_len + 1) if rel_path is None: stdlib.free(cand) return 0 string.strncpy(rel_path, rel_seg, rel_len) rel_path[rel_len] = '\0' string.strcat(rel_path, suffix) # Deep-copy the owning includes root dir; allocate everything before writing # slots to avoid a half-written state when inc allocation fails. inc_buf: bytes = None if out_inc_dir is not None: inc_buf = stdlib.malloc(inc_len + 1) if inc_buf is None: stdlib.free(cand) stdlib.free(rel_path) return 0 string.strcpy(inc_buf, inc_dir) # Write output slots (stdlib deep copies; caller frees). Slots must be cast to # (sizet*): t.CPtr* folds to t.CPtr and [0] becomes a 1-byte access. if out_abs_path is not None: abs_slot: sizet* = (sizet*)(out_abs_path) abs_slot[0] = sizet(cand) else: stdlib.free(cand) if out_rel_path is not None: rel_slot: sizet* = (sizet*)(out_rel_path) rel_slot[0] = sizet(rel_path) else: stdlib.free(rel_path) if out_inc_dir is not None and inc_buf is not None: inc_slot: sizet* = (sizet*)(out_inc_dir) inc_slot[0] = sizet(inc_buf) return 1 # ResolveModuleToPath - module name -> file path, resolved on demand (no full scan). # Tries each includes dir, preferring file modules over packages. Returns 0 found, 1 not. def ResolveModuleToPath(includes_dirs: str, mod_name: str, out_abs_path: t.CPtr, out_rel_path: t.CPtr, out_inc_dir: t.CPtr) -> int: if includes_dirs is None or mod_name is None: return 1 mn_len: sizet = string.strlen(mod_name) if mn_len == 0: return 1 # mod_name -> relative path segment ('.' -> '/'): ast.parser -> ast/parser rel_seg: bytes = stdlib.malloc(mn_len + 1) if rel_seg is None: return 1 rel_len: sizet = 0 for mi in range(int(mn_len)): ch_m: char = mod_name[mi] if ch_m == '.': rel_seg[rel_len] = '/' else: rel_seg[rel_len] = ch_m rel_len += 1 rel_seg[rel_len] = '\0' found: int = 0 d_total: sizet = string.strlen(includes_dirs) d_i: sizet = 0 while d_i < d_total: while d_i < d_total and includes_dirs[d_i] == ' ': d_i += 1 if d_i >= d_total: break d_start: sizet = d_i while d_i < d_total and includes_dirs[d_i] != ' ': d_i += 1 d_len: sizet = d_i - d_start if d_len == 0: continue inc_dir_buf: bytes = stdlib.malloc(d_len + 1) if inc_dir_buf is None: continue string.strncpy(inc_dir_buf, includes_dirs + d_start, d_len) inc_dir_buf[d_len] = '\0' # candidate 1: {inc}/{rel}.vp (file module) if _TryResolveModuleFile(inc_dir_buf, rel_seg, rel_len, ".vp", out_abs_path, out_rel_path, out_inc_dir) != 0: found = 1 stdlib.free(inc_dir_buf) break # candidate 2: {inc}/{rel}/__init__.vp (package module) if _TryResolveModuleFile(inc_dir_buf, rel_seg, rel_len, "/__init__.vp", out_abs_path, out_rel_path, out_inc_dir) != 0: found = 1 stdlib.free(inc_dir_buf) if found != 0: break stdlib.free(rel_seg) if found != 0: return 0 return 1