import t, c from stdint import * import memhub import string import stdio import stdlib import w32.win32file as win32file import w32.win32base as win32base import hashlib import viperlib import lib.Projectrans.Utils as Utils import lib.core.VLogger as VLogger # ============================================================ # IncludesScanner - includes 目录递归扫描(Win32 FindFirstFileA/FindNextFileA) # 递归收集 .vp 文件路径与 SHA1,供 Phase1 stub 生成 / Phase2 stub 合并使用 # ============================================================ # 全局 mbuddy 指针 _mbuddy: memhub.MemBuddy* # 文件路径最大长度 MAX_PATH_LEN: t.CDefine = 512 # 单次扫描最大文件数 MAX_FILES: t.CDefine = 1024 # ============================================================ # FileEntry - 文件条目(路径 + SHA1) # ============================================================ @t.NoVTable class FileEntry: """扫描到的文件条目""" Path: str # 文件绝对路径 Sha1: str # SHA1 前16字符(16字节+null) RelPath: str # 相对于 includes 根目录的路径 ModuleName: str # 模块名(如 "ast.parser") IncDir: str # 所属 includes 根目录(多 includes 目录支持) # ============================================================ # ScanResult - 扫描结果 # ============================================================ @t.NoVTable class ScanResult: """扫描结果集合""" Entries: FileEntry | t.CPtr # FileEntry 数组 Count: int Capacity: int # ============================================================ # create_scan_result - 创建扫描结果容器 # ============================================================ def create_scan_result(pool: memhub.MemBuddy*) -> ScanResult | t.CPtr: """创建扫描结果容器,预分配 MAX_FILES 个槽位""" if pool is None: return None size: sizet = MAX_FILES * FileEntry.__sizeof__() # 使用 stdlib.malloc 避免 mbuddy 池耗尽(与 StubMerger 保持一致) entries: FileEntry | t.CPtr = stdlib.malloc(size) if entries is None: return None string.memset(entries, 0, size) result: ScanResult | t.CPtr = stdlib.malloc(ScanResult.__sizeof__()) if result is None: return None string.memset(result, 0, ScanResult.__sizeof__()) result.Entries = entries result.Count = 0 result.Capacity = MAX_FILES return result # ============================================================ # add_file_entry - 向扫描结果添加文件条目 # ============================================================ def add_file_entry(result: ScanResult | t.CPtr, pool: memhub.MemBuddy*, abs_path: str, rel_path: str, sha1: str, inc_dir: str) -> int: """添加文件条目到扫描结果,返回 0 成功(inc_dir 为所属 includes 根目录)""" if result is None or pool is None: return 1 if result.Count >= result.Capacity: return 1 # 计算模块名:rel_path 中 / 替换为 .,去掉 .vp/.py 扩展名(__init__.vp 特判由 Utils 统一处理) mod_buf: str = Utils.PathToModuleName(rel_path, 0) if mod_buf is None: return 1 # 获取条目地址 entry_size: sizet = FileEntry.__sizeof__() entry_addr: uint64 = uint64(result.Entries) + result.Count * entry_size entry: FileEntry | t.CPtr = (FileEntry | t.CPtr)(entry_addr) if entry is None: return 1 # 复制路径字符串 abs_len: sizet = string.strlen(abs_path) abs_buf: str = stdlib.malloc(abs_len + 1) if abs_buf is None: return 1 string.strcpy(abs_buf, abs_path) entry.Path = abs_buf # 复制 SHA1 sha1_buf: str = stdlib.malloc(17) if sha1_buf is None: return 1 string.strcpy(sha1_buf, sha1) entry.Sha1 = sha1_buf # 复制相对路径 rel_buf: str = stdlib.malloc(string.strlen(rel_path) + 1) if rel_buf is None: return 1 string.strcpy(rel_buf, rel_path) entry.RelPath = rel_buf # 复制模块名 entry.ModuleName = mod_buf # 复制所属 includes 根目录(多 includes 目录支持,Phase2 拼路径用) if inc_dir is not None: inc_len: sizet = string.strlen(inc_dir) inc_buf: str = stdlib.malloc(inc_len + 1) if inc_buf is None: return 1 string.strcpy(inc_buf, inc_dir) entry.IncDir = inc_buf result.Count += 1 return 0 # ============================================================ # compute_file_sha1 - 读取文件内容并计算 SHA1(CRLF→LF 后计算,与 Projectrans.py 一致) # ============================================================ def compute_file_sha1(pool: memhub.MemBuddy*, file_path: str) -> str: """读取文件内容并计算 SHA1 前16字符(CRLF→LF 后计算,与 Projectrans.py 一致)""" if pool is None or file_path is None: return None # 读取文件内容(stdlib.malloc 分配,避免 mbuddy 池耗尽);BUF_SIZE 须与翻译侧 SRC_BUF_SIZE(1MB)一致,否则尾部变更不改 SHA1 导致缓存永不失效 BUF_SIZE: sizet = 1048576 buf: bytes = Utils.read_file_to_buf(file_path, BUF_SIZE) if buf is None: return None # 原地去除 \r(CRLF → LF),与 Projectrans.py 文本模式读取一致 buf_len: sizet = string.strlen(buf) write_pos: sizet = 0 for ch in buf: if ch != '\r': buf[write_pos] = ch write_pos += 1 # 添加 null 终止符 buf[write_pos] = 0 # 计算 SHA1 ctx: rawptr[hashlib.sha1] = hashlib.sha1() if ctx is None: stdlib.free(buf) return None ctx.update(buf) digest: bytes = stdlib.malloc(hashlib.SHA1_DIGEST_LEN) if digest is None: stdlib.free(buf) return None ctx.final(digest) # 转为十六进制字符串(取前 8 字节 = 16 个十六进制字符,stdlib 分配保持原行为) hex_buf: str = Utils.bytes_to_hex(pool, digest, 8, 0, 1) stdlib.free(buf) stdlib.free(digest) return hex_buf # ============================================================ # scan_directory_recursive - 递归扫描目录(FindFirstFileA/FindNextFileA 遍历目录树) # ============================================================ def scan_directory_recursive(pool: memhub.MemBuddy*, root_dir: str, rel_prefix: str, result: ScanResult | t.CPtr, inc_dir: str) -> int: """递归扫描目录,收集 .vp/.py 文件(inc_dir 为所属 includes 根目录)""" if pool is None or root_dir is None or result is None: return 1 # 子目录递归失败计数(不静默忽略,汇总处 fail-fast 终止) subdir_failed: int = 0 # 构造搜索模式: root_dir/* root_len: sizet = string.strlen(root_dir) pattern: bytes = stdlib.malloc(root_len + 4) if pattern is None: return 1 viperlib.snprintf(pattern, root_len + 4, "%s/*", root_dir) # find_data 缓冲 +16 余量:TPV 的 WIN32_FIND_DATAA.__sizeof__() 小于 Windows 实际写入,越界写会破坏相邻堆块 fd_size_s: sizet = win32file.WIN32_FIND_DATAA.__sizeof__() find_data: rawptr[win32file.WIN32_FIND_DATAA] = stdlib.malloc(fd_size_s + 16) if find_data is None: return 1 string.memset(find_data, 0, fd_size_s + 16) handle: win32base.HANDLE = win32file.FindFirstFileA(pattern, find_data) if handle == win32base.INVALID_HANDLE_VALUE: return 1 # 遍历所有文件和子目录 while True: # 跳过 . 和 .. fname: str = find_data.cFileName if fname is not None: fname0: char = fname[0] if fname0 == '.': fname1: char = fname[1] if fname1 == '\0': # "." if win32file.FindNextFileA(handle, find_data) == 0: break continue elif fname1 == '.': fname2: char = fname[2] if fname2 == '\0': # ".." if win32file.FindNextFileA(handle, find_data) == 0: break continue # 检查是否为目录 is_dir: int = find_data.dwFileAttributes & win32file.FILE_ATTRIBUTE_DIRECTORY # 构造完整路径 fname_len: sizet = string.strlen(fname) full_path: bytes = Utils.JoinPath(root_dir, fname) if full_path is None: break # 构造相对路径 prefix_len: sizet = 0 if rel_prefix is not None: prefix_len = string.strlen(rel_prefix) rel_path: bytes = None if rel_prefix is not None and prefix_len > 0: rel_path = Utils.JoinPath(rel_prefix, fname) else: rel_path = stdlib.malloc(fname_len + 1) if rel_path is None: break string.strcpy(rel_path, fname) if rel_path is None: break if is_dir != 0: # 递归扫描子目录 if scan_directory_recursive(pool, full_path, rel_path, result, inc_dir) != 0: subdir_failed += 1 else: # 检查是否为 .vp 源文件(Viper 源仅 .vp,不再识别 .py) is_py: int = 0 if Utils.is_vp_at(fname, 0, fname_len) != 0: is_py = 1 if is_py != 0: # 跳过 __pycache__ 目录下的文件 is_pycache: int = 0 if rel_prefix is not None: if string.strcmp(rel_prefix, "__pycache__") == 0: is_pycache = 1 if is_pycache == 0: # 计算 SHA1 sha1: str = compute_file_sha1(pool, full_path) if sha1 is not None: add_file_entry(result, pool, full_path, rel_path, sha1, inc_dir) VLogger.debug(f"扫描: {rel_path} -> {sha1}", "scan") else: VLogger.warning(f"SHA1 计算失败: {full_path}", "scan") # 继续搜索下一个文件 if win32file.FindNextFileA(handle, find_data) == 0: break win32file.FindClose(handle) if subdir_failed > 0: VLogger.error(f"子目录扫描失败 {subdir_failed} 个", "scan") return 0 # ============================================================ # scan_includes - 扫描 includes 目录入口(返回所有 .vp 文件路径与 SHA1) # ============================================================ def scan_includes(pool: memhub.MemBuddy*, includes_dirs: str) -> ScanResult | t.CPtr: """扫描 includes 目录(支持空格分隔的多目录),返回所有 .vp/.py 文件的扫描结果""" if pool is None or includes_dirs is None: return None result: ScanResult | t.CPtr = create_scan_result(pool) if result is None: return None # 解析空格分隔的多 includes 目录(如 "../../includes ./includes"),逐目录递归扫描合并 dirs_total: sizet = string.strlen(includes_dirs) d_i: sizet = 0 while d_i < dirs_total: while d_i < dirs_total and includes_dirs[d_i] == ' ': d_i += 1 if d_i >= dirs_total: break d_start: sizet = d_i while d_i < dirs_total and includes_dirs[d_i] != ' ': d_i += 1 d_len: sizet = d_i - d_start if d_len == 0: continue dir_buf: bytes = stdlib.malloc(d_len + 1) if dir_buf is None: VLogger.error("includes 目录分配失败,终止扫描", "scan") string.strncpy(dir_buf, includes_dirs + d_start, d_len) dir_buf[d_len] = '\0' scan_directory_recursive(pool, dir_buf, None, result, dir_buf) stdlib.free(dir_buf) return result # ============================================================ # find_entry_by_module - 按模块名查找文件条目 # ============================================================ def find_entry_by_module(result: ScanResult | t.CPtr, module_name: str) -> FileEntry | t.CPtr: """按模块名查找文件条目""" if result is None or module_name is None: return None entry_size: sizet = FileEntry.__sizeof__() for i in range(result.Count): entry_addr: uint64 = uint64(result.Entries) + i * entry_size entry: FileEntry | t.CPtr = (FileEntry | t.CPtr)(entry_addr) if entry is not None: if entry.ModuleName is not None: if string.strcmp(entry.ModuleName, module_name) == 0: return entry return None # ============================================================ # _PathExistsA - 路径存在性检查(GetFileAttributesA) # ============================================================ def _PathExistsA(path: str) -> int: """路径是否存在(1=存在,0=不存在/参数错误) 用 GetFileAttributesA(无输出缓冲)替代 FindFirstFileA:后者写 WIN32_FIND_DATAA 结构,而 TPV 该结构 sizeof 小于 Windows 实际写入 (约 592 字节,差距 >270 字节),会越界写坏相邻堆块(BFS 输出槽 0x2e 崩溃根因)。GetFileAttributesA 只返回属性值,无越界风险。 """ if path is None: return 0 attr: ULONG = w32.win32file.GetFileAttributesA(path) if attr == 0xFFFFFFFF: # INVALID_FILE_ATTRIBUTES return 0 return 1 # ============================================================ # _TryResolveModuleFile - 尝试单个候选路径并写输出 # # Args: # inc_dir: includes 根目录 # rel_seg: 模块名转出的相对路径段(ast/parser) # rel_len: rel_seg 长度 # suffix: 候选后缀(".vp" 或 "/__init__.vp") # out_abs_path/out_rel_path/out_inc_dir: 输出槽(t.CPtr 指针) # # Returns: # 1 命中(已写输出),0 未命中 # ============================================================ def _TryResolveModuleFile(inc_dir: str, rel_seg: bytes, rel_len: sizet, suffix: str, out_abs_path: t.CPtr, out_rel_path: t.CPtr, out_inc_dir: t.CPtr) -> int: if inc_dir is None or rel_seg is None or suffix is None: return 0 inc_len: sizet = string.strlen(inc_dir) suf_len: sizet = string.strlen(suffix) # 候选绝对路径: {inc}/{rel}{suffix}(手写拼接,避免 %. *s 精度修饰符) cand_len: sizet = inc_len + 1 + rel_len + suf_len cand: bytes = stdlib.malloc(cand_len + 1) if cand is None: return 0 string.strcpy(cand, inc_dir) cand[inc_len] = '/' string.strncpy(cand + inc_len + 1, rel_seg, rel_len) string.strcpy(cand + inc_len + 1 + rel_len, suffix) if _PathExistsA(cand) == 0: stdlib.free(cand) return 0 # 相对 includes 根目录的路径: {rel}{suffix} rel_path: bytes = stdlib.malloc(rel_len + suf_len + 1) if rel_path is None: stdlib.free(cand) return 0 string.strncpy(rel_path, rel_seg, rel_len) rel_path[rel_len] = '\0' string.strcat(rel_path, suffix) # 所属 includes 根目录深拷贝:先全部分配就绪再统一写槽, # 避免 inc 分配失败时 abs/rel 已转移所有权造成的半写状态 inc_buf: bytes = None if out_inc_dir is not None: inc_buf = stdlib.malloc(inc_len + 1) if inc_buf is None: stdlib.free(cand) stdlib.free(rel_path) return 0 string.strcpy(inc_buf, inc_dir) # 写输出槽(stdlib 深拷贝,调用方负责 free) # 槽联合必须用 (sizet*):t.CPtr|t.CPtr 会被折叠成 t.CPtr, # [0] 变成 1 字节访问 → 指针被截断(0x2e 崩溃根因)。CSizeT 成员为 # 8 字节,写入用 sizet(ptr) 直接转换;读取用 t\.CVoid(u64, t.CPtr) # 转回指针(t\.CVoid 仅配合 t.CPtr 表示 void* 强转)。 if out_abs_path is not None: abs_slot: sizet* = (sizet*)(out_abs_path) abs_slot[0] = sizet(cand) else: stdlib.free(cand) if out_rel_path is not None: rel_slot: sizet* = (sizet*)(out_rel_path) rel_slot[0] = sizet(rel_path) else: stdlib.free(rel_path) if out_inc_dir is not None and inc_buf is not None: inc_slot: sizet* = (sizet*)(out_inc_dir) inc_slot[0] = sizet(inc_buf) return 1 # ============================================================ # ResolveModuleToPath - 模块名→文件路径(按文件系统按需解析,不做全量目录遍历) # # 解析规则(多 includes 目录逐个尝试,模块优先于包): # {inc_dir}/{mod.replace('.','/')}.vp (文件模块) # {inc_dir}/{mod.replace('.','/')}/__init__.vp (包模块) # # Args: # includes_dirs: 空格分隔的多个 includes 根目录 # mod_name: 绝对模块名 # out_abs_path: 输出绝对路径(stdlib.malloc,调用方负责 free) # out_rel_path: 输出相对 includes 根目录的路径(同上,如 ast/parser.vp) # out_inc_dir: 输出所属 includes 根目录(同上) # # Returns: # 0 找到;1 未找到/参数错误 # ============================================================ def ResolveModuleToPath(includes_dirs: str, mod_name: str, out_abs_path: t.CPtr, out_rel_path: t.CPtr, out_inc_dir: t.CPtr) -> int: if includes_dirs is None or mod_name is None: return 1 mn_len: sizet = string.strlen(mod_name) if mn_len == 0: return 1 # mod_name → 相对路径段(. → /):ast.parser → ast/parser rel_seg: bytes = stdlib.malloc(mn_len + 1) if rel_seg is None: return 1 rel_len: sizet = 0 for mi in range(mn_len): ch_m: char = mod_name[mi] if ch_m == '.': rel_seg[rel_len] = '/' else: rel_seg[rel_len] = ch_m rel_len += 1 rel_seg[rel_len] = '\0' found: int = 0 d_total: sizet = string.strlen(includes_dirs) d_i: sizet = 0 while d_i < d_total: while d_i < d_total and includes_dirs[d_i] == ' ': d_i += 1 if d_i >= d_total: break d_start: sizet = d_i while d_i < d_total and includes_dirs[d_i] != ' ': d_i += 1 d_len: sizet = d_i - d_start if d_len == 0: continue inc_dir_buf: bytes = stdlib.malloc(d_len + 1) if inc_dir_buf is None: continue string.strncpy(inc_dir_buf, includes_dirs + d_start, d_len) inc_dir_buf[d_len] = '\0' # 候选 1: {inc}/{rel}.vp(文件模块) if _TryResolveModuleFile(inc_dir_buf, rel_seg, rel_len, ".vp", out_abs_path, out_rel_path, out_inc_dir) != 0: found = 1 stdlib.free(inc_dir_buf) break # 候选 2: {inc}/{rel}/__init__.vp(包模块) if _TryResolveModuleFile(inc_dir_buf, rel_seg, rel_len, "/__init__.vp", out_abs_path, out_rel_path, out_inc_dir) != 0: found = 1 stdlib.free(inc_dir_buf) if found != 0: break stdlib.free(rel_seg) if found != 0: return 0 return 1