import t, c from stdint import * import stdio import string import stdlib import memhub import viperlib import w32.fileio as fileio import w32.win32file import w32.win32base import ast import llvmlite import lib.core.VLogger as VLogger import lib.core.Handles.HandlesTranslator as HandlesTranslator import lib.core.Handles.HandlesStruct as HandlesStruct import lib.core.Handles.HandlesType as HandlesType import lib.core.Handles.HandlesExprCall as HandlesExprCall import lib.core.Handles.HandlesImports as HandlesImports import lib.core.Handles.HandlesClassDef as HandlesClassDef import lib.core.IncludesScanner as IncludesScanner import lib.core.StubMerger as StubMerger import lib.core.BuildPipeline as BuildPipeline import lib.Projectrans.Config as Config import lib.StubGen.Converter as StubConverter # 全局 mbuddy 指针 _mbuddy: memhub.MemManager | t.CPtr # 源代码缓冲区大小(1MB) SRC_BUF_SIZE: t.CDefine = 1048576 # pyi 缓冲区大小(256KB) PYI_BUF_SIZE: t.CSizeT = 262144 # 拓扑排序相关常量 MAX_CLASSES: t.CDefine = 512 MAX_FILE_DEPS: t.CDefine = 256 MAX_DEP_EDGES: t.CDefine = 16 SHA1_LEN: t.CDefine = 17 CLASS_NAME_LEN: t.CDefine = 64 # 拓扑排序使用扁平字节数组存储类信息和文件依赖, # 避免结构体中的 str 指针字段在 mb.alloc 零填充时为 NULL 导致崩溃。 # 类名/父类名存储在 class_names_buf/parent_names_buf(每个 CLASS_NAME_LEN 字节), # 定义 SHA1 存储在 def_sha1s_buf(每个 SHA1_LEN 字节)。 # ============================================================ # _ScanClassInheritance - 预扫描 includes 文件,收集类继承关系 # # 遍历所有可达 includes 文件,解析 AST 收集 ClassDef 的类名和父类名。 # 结果存入扁平字节数组,返回收集到的类数量。 # # Args: # mb: 内存池 # result: IncludesScanner 扫描结果 # reachable_set: 可达 SHA1 集合 # reachable_count: 可达 SHA1 数量 # use_reachable: 是否使用可达集合过滤 # sha1_set: SHA1 集合(use_reachable=0 时使用) # set_count: SHA1 集合数量 # class_names_buf: 类名扁平数组(每个 CLASS_NAME_LEN 字节,连续内存) # parent_names_buf: 父类名扁平数组(每个 CLASS_NAME_LEN 字节,无父类填 '\0') # def_sha1s_buf: 定义 SHA1 扁平数组(每个 SHA1_LEN 字节) # # Returns: # 收集到的类数量,-1 失败 # ============================================================ def _ScanClassInheritance(mb: memhub.MemBuddy | t.CPtr, result: IncludesScanner.ScanResult | t.CPtr, reachable_set: str, reachable_count: int, use_reachable: int, sha1_set: str, set_count: int, class_names_buf: bytes, parent_names_buf: bytes, def_sha1s_buf: bytes) -> int: """预扫描 includes 文件,收集类继承关系""" if result is None or class_names_buf is None or parent_names_buf is None or def_sha1s_buf is None: return -1 entry_size: t.CSizeT = IncludesScanner.FileEntry.__sizeof__() class_count: int = 0 for i in range(result.Count): entry_addr: t.CUInt64T = t.CUInt64T(result.Entries) + i * entry_size entry: IncludesScanner.FileEntry | t.CPtr = (IncludesScanner.FileEntry | t.CPtr)(t.CVoid(entry_addr, t.CPtr)) if entry is None or entry.Sha1 is None: continue sha1_e: str = entry.Sha1 # 按需翻译过滤 in_set: int = 0 if use_reachable != 0: in_set = StubMerger._is_in_sha1_set(sha1_e, reachable_set, reachable_count) else: in_set = StubMerger._is_in_sha1_set(sha1_e, sha1_set, set_count) if in_set == 0: continue # 读取文件 file_path: str = entry.Path f: fileio.File | t.CPtr = fileio.File(file_path, fileio.MODE.R) if f.closed: continue src_buf: bytes = stdlib.malloc(SRC_BUF_SIZE) if src_buf is None: f.close() continue bytes_read: LONG = f.read_all(src_buf, SRC_BUF_SIZE) f.close() if bytes_read <= 0: stdlib.free(src_buf) continue if bytes_read < SRC_BUF_SIZE: src_buf[bytes_read] = 0 else: src_buf[SRC_BUF_SIZE - 1] = 0 # 解析 AST lx: ast.Lexer | t.CPtr = ast.new_lexer(mb) if lx is None: stdlib.free(src_buf) continue ast._lexer_init(lx, src_buf, mb) tokens: ast.Token | t.CPtr = ast.tokenize(lx) tree: ast.AST | t.CPtr = ast.parse_tokens(mb, tokens) if tree is None: stdlib.free(src_buf) continue # 遍历 AST 顶层节点,收集 ClassDef ch: list[ast.AST | t.CPtr] | t.CPtr = tree.children if ch is not None: cn: t.CSizeT = ch.__len__() for ci in range(cn): child: ast.AST | t.CPtr = ch.get(ci) if child is None: continue kd: int = child.kind() if kd != ast.ASTKind.ClassDef: continue cd: ast.ClassDef | t.CPtr = (ast.ClassDef | t.CPtr)(child) if cd is None or cd.name is None: continue if class_count >= MAX_CLASSES: break # 填充扁平数组:类名(偏移 = class_count * CLASS_NAME_LEN) name_dst: str = class_names_buf + class_count * CLASS_NAME_LEN nm_len: t.CSizeT = string.strlen(cd.name) if nm_len >= CLASS_NAME_LEN: string.strncpy(name_dst, cd.name, CLASS_NAME_LEN - 1) name_dst[CLASS_NAME_LEN - 1] = '\0' else: string.strcpy(name_dst, cd.name) # 填充扁平数组:父类名(无父类填 '\0') parent_dst: str = parent_names_buf + class_count * CLASS_NAME_LEN parent_dst[0] = '\0' parent_nm: str = HandlesClassDef._get_parent_class(cd, None) # parent_nm 是 AST 节点内部指针,不需要释放 if parent_nm is not None: pnm_len: t.CSizeT = string.strlen(parent_nm) if pnm_len >= CLASS_NAME_LEN: string.strncpy(parent_dst, parent_nm, CLASS_NAME_LEN - 1) parent_dst[CLASS_NAME_LEN - 1] = '\0' else: string.strcpy(parent_dst, parent_nm) # 填充扁平数组:定义 SHA1(偏移 = class_count * SHA1_LEN) sha1_dst: str = def_sha1s_buf + class_count * SHA1_LEN string.strcpy(sha1_dst, sha1_e) class_count += 1 stdlib.free(src_buf) return class_count # ============================================================ # _FindClassDefSha1 - 在扁平数组中查找类名对应的定义文件 SHA1 # # Args: # class_names_buf: 类名扁平数组 # def_sha1s_buf: 定义 SHA1 扁平数组 # class_count: 类数量 # class_name: 要查找的类名 # # Returns: # SHA1 字符串指针(指向 def_sha1s_buf 内部偏移),None 未找到 # ============================================================ def _FindClassDefSha1(class_names_buf: bytes, def_sha1s_buf: bytes, class_count: int, class_name: str) -> str: """查找类名对应的定义文件 SHA1""" if class_names_buf is None or def_sha1s_buf is None or class_name is None: return None for i in range(class_count): name_ptr: str = class_names_buf + i * CLASS_NAME_LEN if string.strcmp(name_ptr, class_name) == 0: return def_sha1s_buf + i * SHA1_LEN return None # ============================================================ # _TopoSortFiles - 根据类继承关系对文件进行拓扑排序 # # 算法(Kahn): # 1. 从扁平类信息数组构建"文件 SHA1 → 依赖文件 SHA1 集合"映射 # 2. 计算每个文件的入度(依赖数量) # 3. 入度为 0 的文件先处理,处理后减少依赖它的文件入度 # 4. 重复直到所有文件处理完成 # # 注意:依赖方向是"父类所在文件 → 子类所在文件", # 即父类文件必须先处理。Kahn 用 DepCount 作为剩余依赖计数, # 依赖为 0 的先处理,初始化 InDegree = DepCount。 # # Args: # mb: 内存池 # result: IncludesScanner 扫描结果 # reachable_set: 可达 SHA1 集合 # reachable_count: 可达 SHA1 数量 # use_reachable: 是否使用可达集合过滤 # sha1_set: SHA1 集合 # set_count: SHA1 集合数量 # class_names_buf: 类名扁平数组 # parent_names_buf: 父类名扁平数组 # def_sha1s_buf: 定义 SHA1 扁平数组 # class_count: 类数量 # out_order: 输出拓扑顺序的文件索引数组(调用者分配,int 数组) # out_count: 输出文件数量 # # Returns: # 0 成功,-1 失败 # ============================================================ def _TopoSortFiles(mb: memhub.MemBuddy | t.CPtr, result: IncludesScanner.ScanResult | t.CPtr, reachable_set: str, reachable_count: int, use_reachable: int, sha1_set: str, set_count: int, class_names_buf: bytes, parent_names_buf: bytes, def_sha1s_buf: bytes, class_count: int, out_order: t.CPtr, out_count: t.CPtr) -> int: """根据类继承关系对文件进行拓扑排序""" if result is None or out_order is None or out_count is None: return -1 entry_size: t.CSizeT = IncludesScanner.FileEntry.__sizeof__() # 1. 分配扁平数组:文件 SHA1、依赖 SHA1 列表、依赖数量、入度、处理状态 # 用扁平数组代替 FileDepEntry 结构体,避免 str 指针字段 NULL 崩溃 file_sha1s_buf: bytes = mb.alloc(MAX_FILE_DEPS * SHA1_LEN) file_deps_buf: bytes = mb.alloc(MAX_FILE_DEPS * MAX_DEP_EDGES * SHA1_LEN) file_dep_counts: t.CPtr = mb.alloc(MAX_FILE_DEPS * 4) file_indegrees: t.CPtr = mb.alloc(MAX_FILE_DEPS * 4) file_processed: t.CPtr = mb.alloc(MAX_FILE_DEPS * 4) if file_sha1s_buf is None or file_deps_buf is None or file_dep_counts is None or file_indegrees is None or file_processed is None: if file_sha1s_buf is not None: mb.free(file_sha1s_buf) if file_deps_buf is not None: mb.free(file_deps_buf) if file_dep_counts is not None: mb.free(file_dep_counts) if file_indegrees is not None: mb.free(file_indegrees) if file_processed is not None: mb.free(file_processed) return -1 string.memset(file_sha1s_buf, 0, MAX_FILE_DEPS * SHA1_LEN) string.memset(file_deps_buf, 0, MAX_FILE_DEPS * MAX_DEP_EDGES * SHA1_LEN) string.memset(file_dep_counts, 0, MAX_FILE_DEPS * 4) string.memset(file_indegrees, 0, MAX_FILE_DEPS * 4) string.memset(file_processed, 0, MAX_FILE_DEPS * 4) file_count: int = 0 # sha1→file_index 映射(简单线性查找,文件数不多) for i in range(result.Count): entry_addr: t.CUInt64T = t.CUInt64T(result.Entries) + i * entry_size entry: IncludesScanner.FileEntry | t.CPtr = (IncludesScanner.FileEntry | t.CPtr)(t.CVoid(entry_addr, t.CPtr)) if entry is None or entry.Sha1 is None: continue sha1_e: str = entry.Sha1 in_set: int = 0 if use_reachable != 0: in_set = StubMerger._is_in_sha1_set(sha1_e, reachable_set, reachable_count) else: in_set = StubMerger._is_in_sha1_set(sha1_e, sha1_set, set_count) if in_set == 0: continue if file_count >= MAX_FILE_DEPS: break # 初始化扁平数组:文件 SHA1(偏移 = file_count * SHA1_LEN) # DepCount/InDegree/Processed 已由 memset 清零 sha1_dst: str = file_sha1s_buf + file_count * SHA1_LEN string.strcpy(sha1_dst, sha1_e) file_count += 1 # 2. 构建依赖边:对每个类,找到父类定义所在文件,建立 子文件→父文件 依赖 for ci in range(class_count): # 读取父类名(偏移 = ci * CLASS_NAME_LEN) parent_ptr: str = parent_names_buf + ci * CLASS_NAME_LEN if parent_ptr[0] == '\0': continue # 读取子类定义 SHA1(偏移 = ci * SHA1_LEN) child_def_sha1: str = def_sha1s_buf + ci * SHA1_LEN # 查找父类定义所在文件的 SHA1 parent_sha1: str = _FindClassDefSha1(class_names_buf, def_sha1s_buf, class_count, parent_ptr) if parent_sha1 is None: continue # 父类不在 includes 中(如 Object、CEnum 等标记基类) # 父类文件 SHA1 不能等于子类文件 SHA1(同文件内继承不需要依赖) if string.strcmp(parent_sha1, child_def_sha1) == 0: continue # 找到子类文件对应的索引,添加父文件 SHA1 到依赖列表 for fi in range(file_count): fsha1: str = file_sha1s_buf + fi * SHA1_LEN if string.strcmp(fsha1, child_def_sha1) != 0: continue # 读取当前依赖数量 cnt_addr: t.CUInt64T = t.CUInt64T(file_dep_counts) + fi * 4 cnt_ptr: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(cnt_addr, t.CPtr)) if cnt_ptr is None: break cur_cnt: int = cnt_ptr[0] # 检查依赖是否已添加(去重) already: int = 0 for di in range(cur_cnt): dep_slot: str = file_deps_buf + fi * MAX_DEP_EDGES * SHA1_LEN + di * SHA1_LEN if string.strcmp(dep_slot, parent_sha1) == 0: already = 1 break if already != 0: break if cur_cnt < MAX_DEP_EDGES: dep_dst: str = file_deps_buf + fi * MAX_DEP_EDGES * SHA1_LEN + cur_cnt * SHA1_LEN string.strcpy(dep_dst, parent_sha1) cnt_ptr[0] = cur_cnt + 1 break # 3. Kahn 算法:InDegree = DepCount(依赖多少个父文件) # 依赖为 0 的先处理,处理后减少依赖它的文件入度 for fi in range(file_count): cnt_addr3: t.CUInt64T = t.CUInt64T(file_dep_counts) + fi * 4 cnt_ptr3: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(cnt_addr3, t.CPtr)) ind_addr3: t.CUInt64T = t.CUInt64T(file_indegrees) + fi * 4 ind_ptr3: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(ind_addr3, t.CPtr)) if cnt_ptr3 is not None and ind_ptr3 is not None: ind_ptr3[0] = cnt_ptr3[0] # 4. 拓扑排序输出 out_idx: int = 0 changed: int = 1 while changed != 0: changed = 0 for fi in range(file_count): # 检查处理状态 proc_addr4: t.CUInt64T = t.CUInt64T(file_processed) + fi * 4 proc_ptr4: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(proc_addr4, t.CPtr)) if proc_ptr4 is None or proc_ptr4[0] != 0: continue # 检查入度 ind_addr4: t.CUInt64T = t.CUInt64T(file_indegrees) + fi * 4 ind_ptr4: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(ind_addr4, t.CPtr)) if ind_ptr4 is None or ind_ptr4[0] > 0: continue # 入度为 0,可以处理 # 找到对应的 result.Entries 索引 target_sha1: str = file_sha1s_buf + fi * SHA1_LEN for ri in range(result.Count): ea_r: t.CUInt64T = t.CUInt64T(result.Entries) + ri * entry_size ent_r: IncludesScanner.FileEntry | t.CPtr = (IncludesScanner.FileEntry | t.CPtr)(t.CVoid(ea_r, t.CPtr)) if ent_r is not None and ent_r.Sha1 is not None: if string.strcmp(ent_r.Sha1, target_sha1) == 0: # 写入输出索引数组 out_idx_addr: t.CUInt64T = t.CUInt64T(out_order) + out_idx * 4 out_int_ptr: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(out_idx_addr, t.CPtr)) if out_int_ptr is not None: out_int_ptr[0] = ri out_idx += 1 break # 标记已处理 proc_ptr4[0] = 1 changed = 1 # 减少依赖该文件的其他文件的入度 for fj in range(file_count): proc_addr5: t.CUInt64T = t.CUInt64T(file_processed) + fj * 4 proc_ptr5: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(proc_addr5, t.CPtr)) if proc_ptr5 is None or proc_ptr5[0] != 0: continue # 读取 fj 的依赖数量 cnt_addr5: t.CUInt64T = t.CUInt64T(file_dep_counts) + fj * 4 cnt_ptr5: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(cnt_addr5, t.CPtr)) if cnt_ptr5 is None: continue fj_dep_cnt: int = cnt_ptr5[0] for di in range(fj_dep_cnt): dep_slot5: str = file_deps_buf + fj * MAX_DEP_EDGES * SHA1_LEN + di * SHA1_LEN if string.strcmp(dep_slot5, target_sha1) == 0: ind_addr5: t.CUInt64T = t.CUInt64T(file_indegrees) + fj * 4 ind_ptr5: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(ind_addr5, t.CPtr)) if ind_ptr5 is not None: ind_ptr5[0] = ind_ptr5[0] - 1 break # 处理循环依赖(未处理的文件按原顺序追加) if out_idx < file_count: for fi in range(file_count): proc_addr6: t.CUInt64T = t.CUInt64T(file_processed) + fi * 4 proc_ptr6: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(proc_addr6, t.CPtr)) if proc_ptr6 is None or proc_ptr6[0] != 0: continue target_sha1_2: str = file_sha1s_buf + fi * SHA1_LEN for ri in range(result.Count): ea_r2: t.CUInt64T = t.CUInt64T(result.Entries) + ri * entry_size ent_r2: IncludesScanner.FileEntry | t.CPtr = (IncludesScanner.FileEntry | t.CPtr)(t.CVoid(ea_r2, t.CPtr)) if ent_r2 is not None and ent_r2.Sha1 is not None: if string.strcmp(ent_r2.Sha1, target_sha1_2) == 0: out_idx_addr2: t.CUInt64T = t.CUInt64T(out_order) + out_idx * 4 out_int_ptr2: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(out_idx_addr2, t.CPtr)) if out_int_ptr2 is not None: out_int_ptr2[0] = ri out_idx += 1 break proc_ptr6[0] = 1 # 写入输出数量 out_count_ptr: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(out_count, t.CPtr)) if out_count_ptr is not None: out_count_ptr[0] = out_idx # 释放扁平数组 mb.free(file_sha1s_buf) mb.free(file_deps_buf) mb.free(file_dep_counts) mb.free(file_indegrees) mb.free(file_processed) return 0 # ============================================================ # RunPhase1 - Phase1: 扫描 includes 目录,按需翻译并生成 stub # # 对每个 .py 文件检查 stub 是否已存在,若不存在则翻译并分离 stub。 # "按需翻译":仅对 stub 不存在的文件执行翻译,避免重复工作。 # # Args: # mb: 内存池 # includes_dir: includes 目录路径 # temp_dir: 临时目录(保存 stub) # log: 日志器 # # Returns: # 0 成功,非 0 失败 # ============================================================ def RunPhase1(mb: memhub.MemBuddy | t.CPtr, includes_dir: str, temp_dir: str, log: VLogger.Logger | t.CPtr) -> int: """Phase1: 扫描 includes 目录,按需翻译并生成 stub""" if includes_dir is None or temp_dir is None: VLogger.warning("includes_dir 或 temp_dir 为空,跳过", "Phase1") return 1 # 设置全局 temp_dir(供跨模块 CDefine 查找使用) HandlesType.set_temp_dir(temp_dir) # 确保 temp 目录存在(build_dir/temp 可能尚未创建) BuildPipeline.ensure_dir(temp_dir) if log is not None: log.banner("Phase1: 扫描 includes(按需翻译)") # 扫描 includes 目录 result: IncludesScanner.ScanResult | t.CPtr = IncludesScanner.scan_includes(mb, includes_dir) if result is None: VLogger.error("扫描失败", "Phase1") return 1 # 从扫描结果直接构建 SHA1 集合(不依赖 _sha1_map.txt 文件) # Phase1 作为编译器自身,应从内存中的扫描结果获取 SHA1,避免外部文件依赖 sha1_set: str = stdlib.malloc(StubMerger.MAX_INCLUDES_SHA1 * 17) if sha1_set is None: VLogger.error("sha1_set 分配失败", "Phase1") return 1 string.memset(sha1_set, 0, StubMerger.MAX_INCLUDES_SHA1 * 17) set_count: int = 0 p1_entry_size_init: t.CSizeT = IncludesScanner.FileEntry.__sizeof__() for se_i in range(result.Count): if set_count >= StubMerger.MAX_INCLUDES_SHA1: break se_addr: t.CUInt64T = t.CUInt64T(result.Entries) + se_i * p1_entry_size_init se_ent: IncludesScanner.FileEntry | t.CPtr = (IncludesScanner.FileEntry | t.CPtr)(t.CVoid(se_addr, t.CPtr)) if se_ent is None or se_ent.Sha1 is None: continue string.strcpy(sha1_set + set_count * 17, se_ent.Sha1) set_count += 1 if set_count <= 0: VLogger.warning("扫描结果无有效 SHA1,跳过 Phase1(项目可能无 includes 依赖)", "Phase1") stdlib.free(sha1_set) return 0 # 写入 _sha1_map.txt(人类可读输出,程序内部不读取;机器分析使用 PopulateSha1MapStore 内存存储器) StubMerger.WriteIncludesSha1Map(mb, temp_dir, result, None, 0) # 填充全局 SHA1 映射存储器(内存中,不依赖 _sha1_map.txt 文件) StubMerger.PopulateSha1MapStore(result) # 构建模块 SHA1 映射(供跨模块函数调用名混淆使用) td_len_p1map: t.CSizeT = string.strlen(temp_dir) p1_sha1_arr: bytes = stdlib.malloc(StubMerger.MAX_INCLUDES * 17) p1_mod_arr: bytes = stdlib.malloc(StubMerger.MAX_INCLUDES * 64) p1_inc_count: int = 0 if p1_sha1_arr is not None and p1_mod_arr is not None: p1_inc_count = StubMerger._BuildIncludesSha1Map(temp_dir, td_len_p1map, p1_sha1_arr, p1_mod_arr) HandlesExprCall.set_module_sha1_map(p1_sha1_arr, p1_mod_arr, p1_inc_count) # 初始化 AST 表(只需一次) ast._init_tables(mb) # FileEntry 结构体大小(Phase 1a 和 1b 共用) entry_size: t.CSizeT = IncludesScanner.FileEntry.__sizeof__() # ============================================================ # Phase 1a-pre: 扫描所有 includes 文件的 import 依赖 # # 只处理 Import/ImportFrom 语句,跳过 ClassDef/FunctionDef 等, # 不调用 resolve_annotation_type,避免 list[...] 等不支持的语法触发 crash。 # 生成 .deps.txt 供依赖图按需翻译使用。 # ============================================================ p1a_registered: int = 0 p1a_skipped: int = 0 p1a_failed: int = 0 for i in range(result.Count): entry_addr_a: t.CUInt64T = t.CUInt64T(result.Entries) + i * entry_size entry_a: IncludesScanner.FileEntry | t.CPtr = (IncludesScanner.FileEntry | t.CPtr)(t.CVoid(entry_addr_a, t.CPtr)) if entry_a is None: p1a_failed += 1 continue sha1_a: str = entry_a.Sha1 if sha1_a is None: p1a_failed += 1 continue if StubMerger._is_in_sha1_set(sha1_a, sha1_set, set_count) == 0: p1a_skipped += 1 continue # 读取文件内容 file_path_a: str = entry_a.Path f_a: fileio.File | t.CPtr = fileio.File(file_path_a, fileio.MODE.R) if f_a.closed: p1a_failed += 1 continue src_buf_a: bytes = stdlib.malloc(SRC_BUF_SIZE) if src_buf_a is None: f_a.close() p1a_failed += 1 continue bytes_read_a: LONG = f_a.read_all(src_buf_a, SRC_BUF_SIZE) f_a.close() if bytes_read_a <= 0: stdlib.free(src_buf_a) p1a_failed += 1 continue if bytes_read_a < SRC_BUF_SIZE: src_buf_a[bytes_read_a] = 0 else: src_buf_a[SRC_BUF_SIZE - 1] = 0 # 解析 AST lx_a: ast.Lexer | t.CPtr = ast.new_lexer(mb) if lx_a is None: stdlib.free(src_buf_a) p1a_failed += 1 continue ast._lexer_init(lx_a, src_buf_a, mb) tokens_a: ast.Token | t.CPtr = ast.tokenize(lx_a) tree_a: ast.AST | t.CPtr = ast.parse_tokens(mb, tokens_a) if tree_a is None: stdlib.free(src_buf_a) p1a_failed += 1 continue # 创建 Translator,设置 _declare_only=2(只扫描 import,跳过 ClassDef) tr_a: HandlesTranslator.Translator | t.CPtr = HandlesTranslator.Translator() if tr_a is None: stdlib.free(src_buf_a) p1a_failed += 1 continue tr_a.ModuleSha1 = sha1_a tr_a._declare_only = 2 tr_a.CurrentPackage = HandlesImports.compute_package_from_relpath(mb, entry_a.RelPath) HandlesType.set_current_file(file_path_a) HandlesType.set_current_module_sha1(sha1_a) HandlesType.clear_cdefine_constants() HandlesStruct.reset_visible_structs(mb, 0) ret_a: int = tr_a.translate(tree_a) if ret_a != 0: p1a_failed += 1 else: p1a_registered += 1 # 生成 .deps.txt(记录依赖模块名,供依赖图按需翻译使用) td_len_a: t.CSizeT = string.strlen(temp_dir) deps_path_a: str = StubMerger._sliced_path(temp_dir, td_len_a, sha1_a, "deps.txt") if deps_path_a is not None: df_a: fileio.File | t.CPtr = fileio.File(deps_path_a, fileio.MODE.W) if not df_a.closed: if tr_a._imported_modules is not None: dl_a: t.CSizeT = string.strlen(tr_a._imported_modules) df_a.write(tr_a._imported_modules, dl_a) df_a.close() stdlib.free(deps_path_a) # 释放 Translator 的 C malloc 资源 if tr_a._global_names is not None: stdlib.free(tr_a._global_names) if tr_a._nonlocal_names is not None: stdlib.free(tr_a._nonlocal_names) stdlib.free(src_buf_a) # ============================================================ # 依赖图按需翻译:构建可达 SHA1 集合 # # 从 Config.SourceDir 的源文件开始,解析 import 语句,递归收集 # 可达的 includes 文件 SHA1。Phase 1b 只翻译可达集合中的文件, # 避免翻译不需要的 includes(如 Test 不依赖 llvmlite,则不翻译)。 # 如果 Config.SourceDir 不可用(仅运行 Phase1),回退到 sha1_set。 # ============================================================ reachable_set: str = None reachable_count: int = 0 use_reachable: int = 0 if Config.SourceDir is not None: reachable_set = stdlib.malloc(StubMerger.MAX_INCLUDES_SHA1 * 17) if reachable_set is not None: string.memset(reachable_set, 0, StubMerger.MAX_INCLUDES_SHA1 * 17) reachable_count = StubMerger._BuildReachableSha1Set(mb, Config.SourceDir, temp_dir, reachable_set) if reachable_count > 0: use_reachable = 1 # 用可达集合重新生成 _sha1_map.txt(按图求索的最终产物) # Phase B+ 只遍历这些条目,避免编译不需要的 includes(如 asm.py) StubMerger.WriteIncludesSha1Map(mb, temp_dir, result, reachable_set, reachable_count) # 重新加载 sha1_set,使后续 Phase 1a-pre/1a/1b 的过滤也使用可达集合 string.memset(sha1_set, 0, StubMerger.MAX_INCLUDES_SHA1 * 17) set_count = StubMerger._load_includes_sha1_set(mb, temp_dir, sha1_set) else: stdlib.free(reachable_set) reachable_set = None # ============================================================ # Phase 1a: 注册可达 includes 文件的 struct/enum/union # # 解决字母序依赖问题:按字母顺序翻译时,后面的 struct 未注册 # 前面就需要用。Phase 1a 先注册所有 struct/enum/union 到全局表, # Phase 1b 全量翻译时 struct 已注册,走 existing 路径只翻译方法体。 # 只处理可达文件,避免翻译不需要的 includes(如 Test 不依赖 ast 模块)。 # # 拓扑排序:预扫描所有文件收集类继承关系,按依赖顺序处理文件, # 确保父类所在文件先于子类所在文件处理(如 base.py 先于 astaux.py)。 # ============================================================ p1a_reg: int = 0 p1a_skp: int = 0 p1a_fl: int = 0 # 预扫描类继承关系,构建拓扑顺序(用扁平字节数组,避免结构体 str 指针 NULL 崩溃) class_names_buf: bytes = mb.alloc(MAX_CLASSES * CLASS_NAME_LEN) parent_names_buf: bytes = mb.alloc(MAX_CLASSES * CLASS_NAME_LEN) def_sha1s_buf: bytes = mb.alloc(MAX_CLASSES * SHA1_LEN) topo_order: t.CPtr = mb.alloc(4 * MAX_FILE_DEPS) topo_count_box: t.CPtr = mb.alloc(4) topo_count: int = 0 if class_names_buf is not None and parent_names_buf is not None and def_sha1s_buf is not None and topo_order is not None and topo_count_box is not None: string.memset(class_names_buf, 0, MAX_CLASSES * CLASS_NAME_LEN) string.memset(parent_names_buf, 0, MAX_CLASSES * CLASS_NAME_LEN) string.memset(def_sha1s_buf, 0, MAX_CLASSES * SHA1_LEN) string.memset(topo_order, 0, 4 * MAX_FILE_DEPS) string.memset(topo_count_box, 0, 4) class_count_scanned: int = _ScanClassInheritance( mb, result, reachable_set, reachable_count, use_reachable, sha1_set, set_count, class_names_buf, parent_names_buf, def_sha1s_buf) if class_count_scanned > 0: ret_topo: int = _TopoSortFiles( mb, result, reachable_set, reachable_count, use_reachable, sha1_set, set_count, class_names_buf, parent_names_buf, def_sha1s_buf, class_count_scanned, topo_order, topo_count_box) if ret_topo == 0: tc_ptr: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(topo_count_box, t.CPtr)) if tc_ptr is not None: topo_count = tc_ptr[0] fb_topo: t.CChar | t.CPtr = VLogger.fmt_buf() if fb_topo is not None: viperlib.snprintf(fb_topo, 1024, "拓扑排序: %d 个类, %d 个文件按依赖顺序处理", class_count_scanned, topo_count) VLogger.info(fb_topo, "Phase1") else: VLogger.warning("拓扑排序失败,回退到字母序", "Phase1") topo_count = 0 else: VLogger.warning("类继承预扫描无结果,回退到字母序", "Phase1") topo_count = 0 # 遍历文件:优先使用拓扑顺序,回退到字母序 iter_count: int = topo_count if topo_count > 0 else result.Count for i in range(iter_count): # 获取文件索引:拓扑顺序或字母序 file_idx: int = i if topo_count > 0: od_addr: t.CUInt64T = t.CUInt64T(topo_order) + i * 4 od_ptr: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(od_addr, t.CPtr)) if od_ptr is not None: file_idx = od_ptr[0] entry_addr_r: t.CUInt64T = t.CUInt64T(result.Entries) + file_idx * entry_size entry_r: IncludesScanner.FileEntry | t.CPtr = (IncludesScanner.FileEntry | t.CPtr)(t.CVoid(entry_addr_r, t.CPtr)) if entry_r is None: p1a_fl += 1 continue sha1_r: str = entry_r.Sha1 if sha1_r is None: p1a_fl += 1 continue # 按需翻译过滤(拓扑排序已过滤,但字母序回退需要检查) if topo_count == 0: if use_reachable != 0: if StubMerger._is_in_sha1_set(sha1_r, reachable_set, reachable_count) == 0: p1a_skp += 1 continue else: if StubMerger._is_in_sha1_set(sha1_r, sha1_set, set_count) == 0: p1a_skp += 1 continue # 读取文件内容 file_path_r: str = entry_r.Path f_r: fileio.File | t.CPtr = fileio.File(file_path_r, fileio.MODE.R) if f_r.closed: p1a_fl += 1 continue src_buf_r: bytes = stdlib.malloc(SRC_BUF_SIZE) if src_buf_r is None: f_r.close() p1a_fl += 1 continue bytes_read_r: LONG = f_r.read_all(src_buf_r, SRC_BUF_SIZE) f_r.close() if bytes_read_r <= 0: stdlib.free(src_buf_r) p1a_fl += 1 continue if bytes_read_r < SRC_BUF_SIZE: src_buf_r[bytes_read_r] = 0 else: src_buf_r[SRC_BUF_SIZE - 1] = 0 # 解析 AST lx_r: ast.Lexer | t.CPtr = ast.new_lexer(mb) if lx_r is None: stdlib.free(src_buf_r) p1a_fl += 1 continue ast._lexer_init(lx_r, src_buf_r, mb) tokens_r: ast.Token | t.CPtr = ast.tokenize(lx_r) tree_r: ast.AST | t.CPtr = ast.parse_tokens(mb, tokens_r) if tree_r is None: stdlib.free(src_buf_r) p1a_fl += 1 continue # 创建 Translator,设置 _declare_only=1(注册 struct/enum/union) tr_r: HandlesTranslator.Translator | t.CPtr = HandlesTranslator.Translator() if tr_r is None: stdlib.free(src_buf_r) p1a_fl += 1 continue tr_r.ModuleSha1 = sha1_r tr_r._declare_only = 1 tr_r.CurrentPackage = HandlesImports.compute_package_from_relpath(mb, entry_r.RelPath) HandlesType.set_current_file(file_path_r) HandlesType.set_current_module_sha1(sha1_r) HandlesType.clear_cdefine_constants() HandlesStruct.reset_visible_structs(mb, 0) ret_r: int = tr_r.translate(tree_r) if ret_r != 0: p1a_fl += 1 else: p1a_reg += 1 # 释放 Translator 的 C malloc 资源 if tr_r._global_names is not None: stdlib.free(tr_r._global_names) if tr_r._nonlocal_names is not None: stdlib.free(tr_r._nonlocal_names) stdlib.free(src_buf_r) # 释放拓扑排序资源 if class_names_buf is not None: mb.free(class_names_buf) if parent_names_buf is not None: mb.free(parent_names_buf) if def_sha1s_buf is not None: mb.free(def_sha1s_buf) if topo_order is not None: mb.free(topo_order) if topo_count_box is not None: mb.free(topo_count_box) # ============================================================ # Phase 1b: 全量翻译(struct 已注册,走 existing 路径翻译方法体) # ============================================================ # 遍历每个文件 translated: int = 0 skipped: int = 0 failed: int = 0 for i in range(result.Count): # 获取 entry entry_addr: t.CUInt64T = t.CUInt64T(result.Entries) + i * entry_size entry: IncludesScanner.FileEntry | t.CPtr = (IncludesScanner.FileEntry | t.CPtr)(t.CVoid(entry_addr, t.CPtr)) if entry is None: failed += 1 continue sha1: str = entry.Sha1 if sha1 is None: failed += 1 continue # 检查 SHA1 是否在翻译集合中(按需翻译:只翻译可达的文件) in_set: int = 0 if use_reachable != 0: in_set = StubMerger._is_in_sha1_set(sha1, reachable_set, reachable_count) else: in_set = StubMerger._is_in_sha1_set(sha1, sha1_set, set_count) if in_set == 0: skipped += 1 continue # 构造 stub 路径: {temp_dir}/{sha1}.stub.ll dir_len: t.CSizeT = string.strlen(temp_dir) sha1_len: t.CSizeT = string.strlen(sha1) stub_path: str = StubMerger._sliced_path(temp_dir, dir_len, sha1, "stub.ll") if stub_path is None: failed += 1 continue # 检查 stub 是否已存在(按需翻译:跳过已存在的) sf: fileio.File | t.CPtr = fileio.File(stub_path, fileio.MODE.R) if not sf.closed: sf.close() # 检查 text.ll 是否也存在(虚表扫描需要 text.ll) text_path: str = StubMerger._sliced_path(temp_dir, dir_len, sha1, "text.ll") if text_path is not None: tf: fileio.File | t.CPtr = fileio.File(text_path, fileio.MODE.R) if not tf.closed: tf.close() stdlib.free(text_path) stdlib.free(stub_path) continue stdlib.free(text_path) # text.ll 不存在,需要重新翻译 rp: str = entry.RelPath fb: t.CChar | t.CPtr = VLogger.fmt_buf() if fb is not None: viperlib.snprintf(fb, 1024, "text.ll 不存在,重新翻译: %s (sha1=%s)", rp, sha1) VLogger.info(fb, "Phase1") else: # stub 不存在,需要翻译 rp: str = entry.RelPath fb: t.CChar | t.CPtr = VLogger.fmt_buf() if fb is not None: viperlib.snprintf(fb, 1024, "翻译: %s (sha1=%s)", rp, sha1) VLogger.info(fb, "Phase1") stdlib.free(stub_path) # 读取文件内容 file_path: str = entry.Path f: fileio.File | t.CPtr = fileio.File(file_path, fileio.MODE.R) if f is None: fb: t.CChar | t.CPtr = VLogger.fmt_buf() if fb is not None: viperlib.snprintf(fb, 1024, "无法打开(None): %s", file_path) VLogger.error(fb, "Phase1") failed += 1 continue if f.closed: fb: t.CChar | t.CPtr = VLogger.fmt_buf() if fb is not None: viperlib.snprintf(fb, 1024, "无法打开: %s", file_path) VLogger.error(fb, "Phase1") failed += 1 continue src_buf: bytes = stdlib.malloc(SRC_BUF_SIZE) if src_buf is None: f.close() failed += 1 continue bytes_read: LONG = f.read_all(src_buf, SRC_BUF_SIZE) f.close() if bytes_read <= 0: fb: t.CChar | t.CPtr = VLogger.fmt_buf() if fb is not None: viperlib.snprintf(fb, 1024, "读取失败: %s", file_path) VLogger.error(fb, "Phase1") stdlib.free(src_buf) failed += 1 continue if bytes_read < SRC_BUF_SIZE: src_buf[bytes_read] = 0 else: src_buf[SRC_BUF_SIZE - 1] = 0 # 解析 AST stdio.printf("[P1B-1] parse %s\n", entry.RelPath) lx: ast.Lexer | t.CPtr = ast.new_lexer(mb) if lx is None: stdlib.free(src_buf) failed += 1 continue ast._lexer_init(lx, src_buf, mb) tokens: ast.Token | t.CPtr = ast.tokenize(lx) tree: ast.AST | t.CPtr = ast.parse_tokens(mb, tokens) stdio.printf("[P1B-1a] parse-done %s\n", entry.RelPath) if tree is None: fb: t.CChar | t.CPtr = VLogger.fmt_buf() if fb is not None: viperlib.snprintf(fb, 1024, "AST 解析失败: %s", file_path) VLogger.error(fb, "Phase1") stdlib.free(src_buf) failed += 1 continue # 生成 .pyi 存根文件(直接遍历 AST,不依赖 PythonToStubConverter) stdio.printf("[P1B-2] pyi %s\n", entry.RelPath) pyi_buf: bytes = stdlib.malloc(PYI_BUF_SIZE) if pyi_buf is not None: pyi_pos: t.CSizeT = StubConverter._GeneratePyiFromAst(mb, tree, entry.RelPath, pyi_buf, PYI_BUF_SIZE) if pyi_pos > 0: pyi_path: str = StubMerger._sliced_path(temp_dir, dir_len, sha1, "pyi") if pyi_path is not None: pf: fileio.File | t.CPtr = fileio.File(pyi_path, fileio.MODE.W) if not pf.closed: pf.write(pyi_buf, pyi_pos) pf.close() stdlib.free(pyi_path) stdlib.free(pyi_buf) # 翻译 AST → LLVM IR stdio.printf("[P1B-3] translate %s\n", entry.RelPath) tr: HandlesTranslator.Translator | t.CPtr = HandlesTranslator.Translator() if tr is None: stdlib.free(src_buf) failed += 1 continue tr.ModuleSha1 = sha1 tr.CurrentPackage = HandlesImports.compute_package_from_relpath(mb, entry.RelPath) HandlesType.set_current_file(file_path) HandlesType.set_current_module_sha1(sha1) HandlesType.clear_cdefine_constants() HandlesStruct.reset_visible_structs(mb, 0) ret: int = tr.translate(tree) stdio.printf("[P1B-3a] translate-done %s ret=%d\n", entry.RelPath, ret) if ret != 0: fb: t.CChar | t.CPtr = VLogger.fmt_buf() if fb is not None: viperlib.snprintf(fb, 1024, "翻译失败: %s", file_path) VLogger.error(fb, "Phase1") stdlib.free(src_buf) if tr._global_names is not None: stdlib.free(tr._global_names) if tr._nonlocal_names is not None: stdlib.free(tr._nonlocal_names) failed += 1 continue # dump stub IR (declarations only) stdio.printf("[P1B-4] stub %s\n", entry.RelPath) PHASE1_IR_SIZE: t.CSizeT = 1048576 stub_buf: bytes = stdlib.malloc(PHASE1_IR_SIZE) if stub_buf is None: stdlib.free(src_buf) if tr._global_names is not None: stdlib.free(tr._global_names) if tr._nonlocal_names is not None: stdlib.free(tr._nonlocal_names) failed += 1 continue tr.dump_ir(stub_buf, PHASE1_IR_SIZE, llvmlite.OUTPUT_STUB) stub_len: t.CSizeT = string.strlen(stub_buf) # save stub.ll dir_len_p1: t.CSizeT = string.strlen(temp_dir) stub_path_p1: str = StubMerger._sliced_path(temp_dir, dir_len_p1, sha1, "stub.ll") if stub_path_p1 is not None: sf_p1: fileio.File | t.CPtr = fileio.File(stub_path_p1, fileio.MODE.W) if not sf_p1.closed: sf_p1.write(stub_buf, stub_len) sf_p1.close() stdlib.free(stub_path_p1) stdlib.free(stub_buf) # dump text IR (definitions only) stdio.printf("[P1B-5] text %s\n", entry.RelPath) text_buf: bytes = stdlib.malloc(PHASE1_IR_SIZE) if text_buf is None: stdlib.free(src_buf) if tr._global_names is not None: stdlib.free(tr._global_names) if tr._nonlocal_names is not None: stdlib.free(tr._nonlocal_names) failed += 1 continue tr.dump_ir(text_buf, PHASE1_IR_SIZE, llvmlite.OUTPUT_TEXT) text_len: t.CSizeT = string.strlen(text_buf) # save text.ll text_path_p1: str = StubMerger._sliced_path(temp_dir, dir_len_p1, sha1, "text.ll") if text_path_p1 is not None: tf_p1: fileio.File | t.CPtr = fileio.File(text_path_p1, fileio.MODE.W) if not tf_p1.closed: tf_p1.write(text_buf, text_len) tf_p1.close() stdlib.free(text_path_p1) stdlib.free(text_buf) # save dependencies (_imported_modules) for Phase B deps_path_p1: str = StubMerger._sliced_path(temp_dir, dir_len_p1, sha1, "deps.txt") if deps_path_p1 is not None: df_p1: fileio.File | t.CPtr = fileio.File(deps_path_p1, fileio.MODE.W) if not df_p1.closed: if tr._imported_modules is not None: dl: t.CSizeT = string.strlen(tr._imported_modules) df_p1.write(tr._imported_modules, dl) df_p1.close() stdlib.free(deps_path_p1) # 释放 Translator 的 C malloc 资源 if tr._global_names is not None: stdlib.free(tr._global_names) if tr._nonlocal_names is not None: stdlib.free(tr._nonlocal_names) # 所有 dump_ir 完成,src_buf 不再需要 stdlib.free(src_buf) translated += 1 fb: t.CChar | t.CPtr = VLogger.fmt_buf() if fb is not None: viperlib.snprintf(fb, 1024, "完成: 翻译=%d 跳过=%d 失败=%d", translated, skipped, failed) VLogger.info(fb, "Phase1") # 释放可达 SHA1 集合(如果分配了) if reachable_set is not None: stdlib.free(reachable_set) return 0