diff --git a/App/lib/core/Phase1.vp b/App/lib/core/Phase1.vp index c401e0e..483cad0 100644 --- a/App/lib/core/Phase1.vp +++ b/App/lib/core/Phase1.vp @@ -44,1763 +44,3 @@ MAX_FILE_DEPS: t.CDefine = 256 MAX_DEP_EDGES: t.CDefine = 16 SHA1_LEN: t.CDefine = 17 CLASS_NAME_LEN: t.CDefine = 64 - -# 拓扑排序使用扁平字节数组存储类信息和文件依赖， -# 避免结构体中的 str 指针字段在 mb.alloc 零填充时为 NULL 导致崩溃。 -# 类名/父类名存储在 class_names_buf/parent_names_buf（每个 CLASS_NAME_LEN 字节）， -# 定义 SHA1 存储在 def_sha1s_buf（每个 SHA1_LEN 字节）。 - - -# ============================================================ -# _ScanClassInheritance - 预扫描 includes 文件，收集类继承关系 -# -# 遍历所有可达 includes 文件，解析 AST 收集 ClassDef 的类名和父类名。 -# 结果存入扁平字节数组，返回收集到的类数量。 -# -# Args: -# mb: 内存池 -# result: IncludesScanner 扫描结果 -# reachable_set: 可达 SHA1 集合 -# reachable_count: 可达 SHA1 数量 -# use_reachable: 是否使用可达集合过滤 -# sha1_set: SHA1 集合（use_reachable=0 时使用） -# set_count: SHA1 集合数量 -# class_names_buf: 类名扁平数组（每个 CLASS_NAME_LEN 字节，连续内存） -# parent_names_buf: 父类名扁平数组（每个 CLASS_NAME_LEN 字节，无父类填 '\0'） -# def_sha1s_buf: 定义 SHA1 扁平数组（每个 SHA1_LEN 字节） -# -# Returns: -# 收集到的类数量，-1 失败 -# ============================================================ -def _ScanClassInheritance(mb: memhub.MemBuddy | t.CPtr, - result: IncludesScanner.ScanResult | t.CPtr, - reachable_set: str, reachable_count: int, - use_reachable: int, - sha1_set: str, set_count: int, - class_names_buf: bytes, - parent_names_buf: bytes, - def_sha1s_buf: bytes, - generic_file_indices: t.CPtr, - generic_file_count_box: t.CPtr) -> int: - """预扫描 includes 文件，收集类继承关系""" - if result is None or class_names_buf is None or parent_names_buf is None or def_sha1s_buf is None: - return -1 - - entry_size: t.CSizeT = IncludesScanner.FileEntry.__sizeof__() - class_count: int = 0 - - for i in range(result.Count): - # 文件级 mark：收集完类继承信息后立即回收本文件 AST（可达 includes - # 全量 parse 的 AST 若驻留到 mark1 → 1GB 主池耗尽 OOM 根因； - # class_names_buf/parent_names_buf/def_sha1s_buf 在调用前分配， - # 位于本 mark 之前，reset 不影响；generic_file_indices 同理） - scan_mark: t.CSizeT = mb.mark() - entry_addr: t.CUInt64T = t.CUInt64T(result.Entries) + i * entry_size - entry: IncludesScanner.FileEntry | t.CPtr = (IncludesScanner.FileEntry | t.CPtr)(t.CVoid(entry_addr, t.CPtr)) - if entry is None or entry.Sha1 is None: - continue - - sha1_e: str = entry.Sha1 - file_has_generic: int = 0 - - # 按需翻译过滤 - in_set: int = 0 - if use_reachable != 0: - in_set = StubMergerCollect._is_in_sha1_set(sha1_e, reachable_set, reachable_count) - else: - in_set = StubMergerCollect._is_in_sha1_set(sha1_e, sha1_set, set_count) - if in_set == 0: - continue - - # 读取文件 - file_path: str = entry.Path - f: fileio.File | t.CPtr = fileio.File(file_path, fileio.MODE.R) - if f.closed: - continue - - src_buf: bytes = stdlib.malloc(SRC_BUF_SIZE) - if src_buf is None: - f.close() - continue - - bytes_read: LONG = f.read_all(src_buf, SRC_BUF_SIZE) - f.close() - if bytes_read <= 0: - stdlib.free(src_buf) - continue - if bytes_read < SRC_BUF_SIZE: - src_buf[bytes_read] = 0 - else: - src_buf[SRC_BUF_SIZE - 1] = 0 - - # 解析 AST - lx: ast.Lexer | t.CPtr = ast.new_lexer(mb) - if lx is None: - stdlib.free(src_buf) - continue - ast._lexer_init(lx, src_buf, mb) - tokens: ast.Token | t.CPtr = ast.tokenize(lx) - tree: ast.AST | t.CPtr = ast.parse_tokens(mb, tokens) - if tree is None: - stdlib.free(src_buf) - continue - - # 遍历 AST 顶层节点，收集 ClassDef - ch: list[ast.AST | t.CPtr] | t.CPtr = tree.children - if ch is not None: - cn: t.CSizeT = len(ch) - for ci in range(cn): - child: ast.AST | t.CPtr = ch.get(ci) - if child is None: - continue - kd: int = child.kind() - if kd != ast.ASTKind.ClassDef: - continue - - cd: ast.ClassDef | t.CPtr = (ast.ClassDef | t.CPtr)(child) - if cd is None or cd.name is None: - continue - - # 检测泛型类（有 type_params），记录文件索引以供 Phase 1a 预注册 - if file_has_generic == 0: - tp_scan: list[str] | t.CPtr = cd.type_params - if tp_scan is not None: - file_has_generic = 1 - if generic_file_count_box is not None: - gfc_ptr_scan: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(generic_file_count_box, t.CPtr)) - if gfc_ptr_scan is not None: - gfc_scan: int = gfc_ptr_scan[0] - if gfc_scan < MAX_FILE_DEPS and generic_file_indices is not None: - gi_addr_scan: t.CUInt64T = t.CUInt64T(generic_file_indices) + gfc_scan * 4 - gi_ptr_scan: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(gi_addr_scan, t.CPtr)) - if gi_ptr_scan is not None: - gi_ptr_scan[0] = i - gfc_ptr_scan[0] = gfc_scan + 1 - - if class_count >= MAX_CLASSES: - break - - # 填充扁平数组：类名（偏移 = class_count * CLASS_NAME_LEN） - name_dst: str = class_names_buf + class_count * CLASS_NAME_LEN - nm_len: t.CSizeT = string.strlen(cd.name) - if nm_len >= CLASS_NAME_LEN: - string.strncpy(name_dst, cd.name, CLASS_NAME_LEN - 1) - name_dst[CLASS_NAME_LEN - 1] = '\0' - else: - string.strcpy(name_dst, cd.name) - - # 填充扁平数组：父类名（无父类填 '\0'） - parent_dst: str = parent_names_buf + class_count * CLASS_NAME_LEN - parent_dst[0] = '\0' - parent_nm: str = HandlesClassDef._get_parent_class(cd, None) - # parent_nm 是 AST 节点内部指针，不需要释放 - if parent_nm is not None: - pnm_len: t.CSizeT = string.strlen(parent_nm) - if pnm_len >= CLASS_NAME_LEN: - string.strncpy(parent_dst, parent_nm, CLASS_NAME_LEN - 1) - parent_dst[CLASS_NAME_LEN - 1] = '\0' - else: - string.strcpy(parent_dst, parent_nm) - - # 填充扁平数组：定义 SHA1（偏移 = class_count * SHA1_LEN） - sha1_dst: str = def_sha1s_buf + class_count * SHA1_LEN - string.strcpy(sha1_dst, sha1_e) - - class_count += 1 - - stdlib.free(src_buf) - # 回收本文件 AST/lexer/token（类名/父类/SHA1 已拷入扁平数组） - mb.reset_to(scan_mark) - - return class_count - - -# ============================================================ -# _BufHasGenericClassMarker - 文本扫描检测文件是否含泛型类标记 -# -# 泛型类（PEP 695）语法: class Name[T]:（类名后紧跟 '['）。 -# 逐字节扫描找词边界 'class' 关键字，其后标识符后非空白是 '[' → 命中。 -# 用于 _PreRegisterAllGenericTemplates 预注册前筛选，避免 parse 全部 -# 98 个 includes 导致 AST 驻留 mark1 前把 pool 推到 OOM -# （NewAstTest [52/68] pool 耗尽根因：1a-pre 已 parse 全部驻留， -# 预注册再全量 parse 等于翻倍）。 -# 误报（字符串/注释里的 "class X["）无害（多 parse 一个文件）； -# 漏报有害（泛型模板未预注册 → 悬空复发），模式保守匹配行内单行形式 -# （TPV 源码泛型类均为单行 `class X[T]:`，grep 确认）。 -# ============================================================ -def _BufHasGenericClassMarker(buf: bytes, blen: t.CSizeT) -> int: - """文本扫描：buf 中是否有 'class 标识符[' 泛型类标记，返回 1/0""" - if buf is None or blen <= 6: - return 0 - pos: t.CSizeT = 0 - while pos + 5 < blen: - if buf[pos] == 'c' and buf[pos + 1] == 'l' and buf[pos + 2] == 'a' \ - and buf[pos + 3] == 's' and buf[pos + 4] == 's': - # class 前是词边界（行首或非标识符字符） - prev_ok: int = 1 - if pos > 0: - pc: t.CChar = buf[pos - 1] - if (pc >= 'a' and pc <= 'z') or (pc >= 'A' and pc <= 'Z') \ - or (pc >= '0' and pc <= '9') or pc == '_': - prev_ok = 0 - if prev_ok != 0: - p: t.CSizeT = pos + 5 - # 跳过空白 - while p < blen and (buf[p] == ' ' or buf[p] == '\t'): - p += 1 - # 读类名标识符 - if p < blen and ((buf[p] >= 'a' and buf[p] <= 'z') - or (buf[p] >= 'A' and buf[p] <= 'Z') or buf[p] == '_'): - while p < blen and ((buf[p] >= 'a' and buf[p] <= 'z') - or (buf[p] >= 'A' and buf[p] <= 'Z') - or (buf[p] >= '0' and buf[p] <= '9') - or buf[p] == '_'): - p += 1 - # 类名后非空白是 '[' → 泛型类 - while p < blen and (buf[p] == ' ' or buf[p] == '\t'): - p += 1 - if p < blen and buf[p] == '[': - return 1 - pos += 1 - else: - pos += 1 - return 0 - - -# ============================================================ -# _PreRegisterAllGenericTemplates - 预注册 includes 泛型类模板（mark1 前） -# -# 背景：泛型模板表（GenericClassNames/GenericClassNodes/GenericClassSha1s/ -# GenericClassFiles + 模板节点）是懒分配的跨文件持久表，首次注册必须落在 -# mark1 之前才能跨 Phase1 1b（reset_to(mark1)）与 Phase2（reset_to(mark2)， -# mark2 > mark1）存活。现有 1a-pre-generic 只注册"可达"泛型文件；FuncTest -# 这类可达集合无泛型的项目，模板要等 Phase2 Phase A-pre-inc（mark2 之后） -# 翻译全部 includes 时才注册 → reset 回收表与节点 → _find_generic_template -# 遍历悬空内存 strcmp 崩溃。 -# 本函数先文本扫描（_BufHasGenericClassMarker，不 parse、不驻留）筛选出 -# 含泛型类的文件（全量 parse 会把 98 个 includes 的 AST 驻留 mark1 前 → -# NewAstTest [52/68] pool 耗尽 OOM），仅对命中的文件用**永久池（perm_pool）** -# 解析 → 模板节点与表直接进 perm_pool，跨 mark 永久存活、不占主 pool。 -# 重复注册由 _register_generic_template 内部查重跳过。 -# ============================================================ -def _PreRegisterAllGenericTemplates(mb: memhub.MemBuddy | t.CPtr, - result: IncludesScanner.ScanResult | t.CPtr) -> int: - """预注册全部 includes 文件的泛型类模板，返回处理到的泛型类数量""" - if result is None or mb is None: - return 0 - perm: memhub.MemBuddy | t.CPtr = CompilerFactory.get_perm_pool() - if perm is None: - return 0 - p1e_size: t.CSizeT = IncludesScanner.FileEntry.__sizeof__() - handled_count: int = 0 - for i in range(result.Count): - entry_addr_pg: t.CUInt64T = t.CUInt64T(result.Entries) + i * p1e_size - entry_pg: IncludesScanner.FileEntry | t.CPtr = ( - IncludesScanner.FileEntry | t.CPtr)(t.CVoid(entry_addr_pg, t.CPtr)) - if entry_pg is None or entry_pg.Sha1 is None: - continue - # 读取文件 - file_path_pg: str = entry_pg.Path - f_pg: fileio.File | t.CPtr = fileio.File(file_path_pg, fileio.MODE.R) - if f_pg.closed: - continue - src_buf_pg: bytes = stdlib.malloc(SRC_BUF_SIZE) - if src_buf_pg is None: - f_pg.close() - continue - bytes_read_pg: LONG = f_pg.read_all(src_buf_pg, SRC_BUF_SIZE) - f_pg.close() - if bytes_read_pg <= 0: - stdlib.free(src_buf_pg) - continue - if bytes_read_pg < SRC_BUF_SIZE: - src_buf_pg[bytes_read_pg] = 0 - else: - src_buf_pg[SRC_BUF_SIZE - 1] = 0 - # 文本扫描筛选：非泛型文件跳过（不 parse、不驻留，防 OOM） - if _BufHasGenericClassMarker(src_buf_pg, bytes_read_pg) == 0: - stdlib.free(src_buf_pg) - continue - # 泛型文件：永久池解析 → 注册模板（节点/表进 perm_pool，跨 reset 存活） - lx_pg: ast.Lexer | t.CPtr = ast.new_lexer(perm) - if lx_pg is None: - stdlib.free(src_buf_pg) - continue - ast._lexer_init(lx_pg, src_buf_pg, perm) - tokens_pg: ast.Token | t.CPtr = ast.tokenize(lx_pg) - tree_pg: ast.AST | t.CPtr = ast.parse_tokens(perm, tokens_pg) - if tree_pg is None: - stdlib.free(src_buf_pg) - continue - # 遍历顶层 ClassDef，注册泛型模板（type_params 非 None） - ch_pg: list[ast.AST | t.CPtr] | t.CPtr = tree_pg.children - if ch_pg is not None: - cn_pg: t.CSizeT = len(ch_pg) - for ci_pg in range(cn_pg): - child_pg: ast.AST | t.CPtr = ch_pg.get(ci_pg) - if child_pg is None or child_pg.kind() != ast.ASTKind.ClassDef: - continue - cd_pg: ast.ClassDef | t.CPtr = (ast.ClassDef | t.CPtr)(child_pg) - if cd_pg is None or cd_pg.name is None: - continue - tp_pg: list[str] | t.CPtr = cd_pg.type_params - if tp_pg is None: - continue - HandlesType.set_current_file(file_path_pg) - HandlesClassDef._register_generic_template(perm, cd_pg, entry_pg.Sha1) - handled_count += 1 - stdlib.free(src_buf_pg) - if handled_count > 0: - fb_gpre2: str = VLogger.fmt_buf() - if fb_gpre2 is not None: - viperlib.snprintf(fb_gpre2, 1024, "mark1 前永久池预注册 %d 个 includes 泛型类模板", handled_count) - VLogger.info(fb_gpre2, "Phase1") - return handled_count - - -# ============================================================ -# _FindClassDefSha1 - 在扁平数组中查找类名对应的定义文件 SHA1 -# -# Args: -# class_names_buf: 类名扁平数组 -# def_sha1s_buf: 定义 SHA1 扁平数组 -# class_count: 类数量 -# class_name: 要查找的类名 -# -# Returns: -# SHA1 字符串指针（指向 def_sha1s_buf 内部偏移），None 未找到 -# ============================================================ -def _FindClassDefSha1(class_names_buf: bytes, def_sha1s_buf: bytes, - class_count: int, class_name: str) -> str: - """查找类名对应的定义文件 SHA1""" - if class_names_buf is None or def_sha1s_buf is None or class_name is None: - return None - for i in range(class_count): - name_ptr: str = class_names_buf + i * CLASS_NAME_LEN - if string.strcmp(name_ptr, class_name) == 0: - return def_sha1s_buf + i * SHA1_LEN - return None - - -# ============================================================ -# _ScanTreeForContainers - 递归扫描 AST 检测 list/dict 容器使用 -# -# TPV 的 AST 中，语句块挂在 children，而表达式/注解树全部以结构体 -# 字段挂载（value/slice/func/args/annotation/targets/returns/bases/ -# left/right 等）。只递归 children 会漏掉 list[...] | t.CPtr（BinOp -# 包裹）、list[...](...)（Call.func）、字段注解等一切非语句块位置。 -# 因此每个节点先递归其表达式承载字段，再递归 children 语句块。 -# 命中即置 need_list/need_dict；误报无害（最多多编译两个空壳模块）。 -# ============================================================ -def _ScanTreeForContainers(node: ast.AST | t.CPtr, - need_list_box: t.CPtr, - need_dict_box: t.CPtr): - """递归扫描 AST（字段 + 语句块），检测 list/dict 容器使用""" - if node is None: - return - k_s: int = node.kind() - if k_s == ast.ASTKind.Name: - nm_s: ast.Name | t.CPtr = (ast.Name | t.CPtr)(node) - if nm_s.id is not None: - if string.strcmp(nm_s.id, "list") == 0: - need_list_box[0] = 1 - elif string.strcmp(nm_s.id, "dict") == 0: - need_dict_box[0] = 1 - elif k_s == ast.ASTKind.Subscript: - sub_s: ast.Subscript | t.CPtr = (ast.Subscript | t.CPtr)(node) - _ScanTreeForContainers(sub_s.value, need_list_box, need_dict_box) - _ScanTreeForContainers(sub_s.slice, need_list_box, need_dict_box) - elif k_s == ast.ASTKind.BinOp: - bop_s: ast.BinOp | t.CPtr = (ast.BinOp | t.CPtr)(node) - _ScanTreeForContainers(bop_s.left, need_list_box, need_dict_box) - _ScanTreeForContainers(bop_s.right, need_list_box, need_dict_box) - elif k_s == ast.ASTKind.Call: - cl_s2: ast.Call | t.CPtr = (ast.Call | t.CPtr)(node) - _ScanTreeForContainers(cl_s2.func, need_list_box, need_dict_box) - cl_args_s: list[ast.AST | t.CPtr] | t.CPtr = cl_s2.args - if cl_args_s is not None: - na_s: t.CSizeT = len(cl_args_s) - i_a2: t.CSizeT - for i_a2 in range(na_s): - _ScanTreeForContainers(cl_args_s.get(i_a2), need_list_box, need_dict_box) - cl_kws_s: list[ast.AST | t.CPtr] | t.CPtr = cl_s2.keywords - if cl_kws_s is not None: - nk_s: t.CSizeT = len(cl_kws_s) - i_k2: t.CSizeT - for i_k2 in range(nk_s): - kw_s: ast.Keyword | t.CPtr = (ast.Keyword | t.CPtr)(cl_kws_s.get(i_k2)) - if kw_s is not None: - _ScanTreeForContainers(kw_s.value, need_list_box, need_dict_box) - elif k_s == ast.ASTKind.Assign: - asn_s: ast.Assign | t.CPtr = (ast.Assign | t.CPtr)(node) - _ScanTreeForContainers(asn_s.value, need_list_box, need_dict_box) - tgts_s: list[ast.AST | t.CPtr] | t.CPtr = asn_s.targets - if tgts_s is not None: - nt_s: t.CSizeT = len(tgts_s) - i_t2: t.CSizeT - for i_t2 in range(nt_s): - _ScanTreeForContainers(tgts_s.get(i_t2), need_list_box, need_dict_box) - elif k_s == ast.ASTKind.AnnAssign: - aa_s2: ast.AnnAssign | t.CPtr = (ast.AnnAssign | t.CPtr)(node) - _ScanTreeForContainers(aa_s2.annotation, need_list_box, need_dict_box) - _ScanTreeForContainers(aa_s2.value, need_list_box, need_dict_box) - elif k_s == ast.ASTKind.FunctionDef: - fn_s: ast.FunctionDef | t.CPtr = (ast.FunctionDef | t.CPtr)(node) - _ScanTreeForContainers(fn_s.returns, need_list_box, need_dict_box) - fn_args_s: ast.Arguments | t.CPtr = fn_s.args - if fn_args_s is not None: - fa_s: list[ast.AST | t.CPtr] | t.CPtr = fn_args_s.args - if fa_s is not None: - nf_s: t.CSizeT = len(fa_s) - i_f2: t.CSizeT - for i_f2 in range(nf_s): - arg_s: ast.Arg | t.CPtr = (ast.Arg | t.CPtr)(fa_s.get(i_f2)) - if arg_s is not None: - _ScanTreeForContainers(arg_s.annotation, need_list_box, need_dict_box) - elif k_s == ast.ASTKind.ClassDef: - cd_s: ast.ClassDef | t.CPtr = (ast.ClassDef | t.CPtr)(node) - bases_s: list[ast.AST | t.CPtr] | t.CPtr = cd_s.bases - if bases_s is not None: - nb_s: t.CSizeT = len(bases_s) - i_b2: t.CSizeT - for i_b2 in range(nb_s): - _ScanTreeForContainers(bases_s.get(i_b2), need_list_box, need_dict_box) - elif k_s == ast.ASTKind.Attribute: - at_s: ast.Attribute | t.CPtr = (ast.Attribute | t.CPtr)(node) - _ScanTreeForContainers(at_s.value, need_list_box, need_dict_box) - elif k_s == ast.ASTKind.Tuple: - tu_s: ast.Tuple | t.CPtr = (ast.Tuple | t.CPtr)(node) - elts_s: list[ast.AST | t.CPtr] | t.CPtr = tu_s.elts - if elts_s is not None: - ne_s: t.CSizeT = len(elts_s) - i_e2: t.CSizeT - for i_e2 in range(ne_s): - _ScanTreeForContainers(elts_s.get(i_e2), need_list_box, need_dict_box) - elif k_s == ast.ASTKind.UnaryOp: - uop_s: ast.UnaryOp | t.CPtr = (ast.UnaryOp | t.CPtr)(node) - _ScanTreeForContainers(uop_s.operand, need_list_box, need_dict_box) - elif k_s == ast.ASTKind.Expr: - ex_s: ast.Expr | t.CPtr = (ast.Expr | t.CPtr)(node) - _ScanTreeForContainers(ex_s.value, need_list_box, need_dict_box) - elif k_s == ast.ASTKind.Return: - rt_s: ast.Return | t.CPtr = (ast.Return | t.CPtr)(node) - _ScanTreeForContainers(rt_s.value, need_list_box, need_dict_box) - elif k_s == ast.ASTKind.Compare: - cp_s: ast.Compare | t.CPtr = (ast.Compare | t.CPtr)(node) - _ScanTreeForContainers(cp_s.left, need_list_box, need_dict_box) - cps_s: list[ast.AST | t.CPtr] | t.CPtr = cp_s.comparators - if cps_s is not None: - nc_s: t.CSizeT = len(cps_s) - i_c2: t.CSizeT - for i_c2 in range(nc_s): - _ScanTreeForContainers(cps_s.get(i_c2), need_list_box, need_dict_box) - elif k_s == ast.ASTKind.BoolOp: - bo_s: ast.BoolOp | t.CPtr = (ast.BoolOp | t.CPtr)(node) - bvs_s: list[ast.AST | t.CPtr] | t.CPtr = bo_s.values - if bvs_s is not None: - nv_s: t.CSizeT = len(bvs_s) - i_v2: t.CSizeT - for i_v2 in range(nv_s): - _ScanTreeForContainers(bvs_s.get(i_v2), need_list_box, need_dict_box) - elif k_s == ast.ASTKind.List: - # [] 字面量 - need_list_box[0] = 1 - elif k_s == ast.ASTKind.Dict: - # {} 字面量 - need_dict_box[0] = 1 - # 递归语句块子节点 - if node.children is not None: - ch_s: list[ast.AST | t.CPtr] | t.CPtr = node.children - n_s: t.CSizeT = len(ch_s) - ci_s: t.CSizeT - for ci_s in range(n_s): - _ScanTreeForContainers(ch_s.get(ci_s), need_list_box, need_dict_box) - - -# ============================================================ -# _TopoSortFiles - 根据类继承关系对文件进行拓扑排序 -# -# 算法（Kahn）： -# 1. 从扁平类信息数组构建"文件 SHA1 → 依赖文件 SHA1 集合"映射 -# 2. 计算每个文件的入度（依赖数量） -# 3. 入度为 0 的文件先处理，处理后减少依赖它的文件入度 -# 4. 重复直到所有文件处理完成 -# -# 注意：依赖方向是"父类所在文件 → 子类所在文件"， -# 即父类文件必须先处理。Kahn 用 DepCount 作为剩余依赖计数， -# 依赖为 0 的先处理，初始化 InDegree = DepCount。 -# -# Args: -# mb: 内存池 -# result: IncludesScanner 扫描结果 -# reachable_set: 可达 SHA1 集合 -# reachable_count: 可达 SHA1 数量 -# use_reachable: 是否使用可达集合过滤 -# sha1_set: SHA1 集合 -# set_count: SHA1 集合数量 -# class_names_buf: 类名扁平数组 -# parent_names_buf: 父类名扁平数组 -# def_sha1s_buf: 定义 SHA1 扁平数组 -# class_count: 类数量 -# out_order: 输出拓扑顺序的文件索引数组（调用者分配，int 数组） -# out_count: 输出文件数量 -# -# Returns: -# 0 成功，-1 失败 -# ============================================================ -def _TopoSortFiles(mb: memhub.MemBuddy | t.CPtr, - result: IncludesScanner.ScanResult | t.CPtr, - reachable_set: str, reachable_count: int, - use_reachable: int, - sha1_set: str, set_count: int, - class_names_buf: bytes, parent_names_buf: bytes, - def_sha1s_buf: bytes, class_count: int, - out_order: t.CPtr, out_count: t.CPtr, - temp_dir: str, td_len_ts: t.CSizeT) -> int: - """根据类继承关系对文件进行拓扑排序""" - if result is None or out_order is None or out_count is None: - return -1 - - entry_size: t.CSizeT = IncludesScanner.FileEntry.__sizeof__() - - # 1. 分配扁平数组：文件 SHA1、依赖 SHA1 列表、依赖数量、入度、处理状态 - # 用扁平数组代替 FileDepEntry 结构体，避免 str 指针字段 NULL 崩溃 - file_sha1s_buf: bytes = mb.alloc(MAX_FILE_DEPS * SHA1_LEN) - file_deps_buf: bytes = mb.alloc(MAX_FILE_DEPS * MAX_DEP_EDGES * SHA1_LEN) - file_dep_counts: t.CPtr = mb.alloc(MAX_FILE_DEPS * 4) - file_indegrees: t.CPtr = mb.alloc(MAX_FILE_DEPS * 4) - file_processed: t.CPtr = mb.alloc(MAX_FILE_DEPS * 4) - - if file_sha1s_buf is None or file_deps_buf is None or file_dep_counts is None or file_indegrees is None or file_processed is None: - if file_sha1s_buf is not None: - mb.free(file_sha1s_buf) - if file_deps_buf is not None: - mb.free(file_deps_buf) - if file_dep_counts is not None: - mb.free(file_dep_counts) - if file_indegrees is not None: - mb.free(file_indegrees) - if file_processed is not None: - mb.free(file_processed) - return -1 - - string.memset(file_sha1s_buf, 0, MAX_FILE_DEPS * SHA1_LEN) - string.memset(file_deps_buf, 0, MAX_FILE_DEPS * MAX_DEP_EDGES * SHA1_LEN) - string.memset(file_dep_counts, 0, MAX_FILE_DEPS * 4) - string.memset(file_indegrees, 0, MAX_FILE_DEPS * 4) - string.memset(file_processed, 0, MAX_FILE_DEPS * 4) - - file_count: int = 0 - # sha1→file_index 映射（简单线性查找，文件数不多） - for i in range(result.Count): - entry_addr: t.CUInt64T = t.CUInt64T(result.Entries) + i * entry_size - entry: IncludesScanner.FileEntry | t.CPtr = (IncludesScanner.FileEntry | t.CPtr)(t.CVoid(entry_addr, t.CPtr)) - if entry is None or entry.Sha1 is None: - continue - - sha1_e: str = entry.Sha1 - in_set: int = 0 - if use_reachable != 0: - in_set = StubMergerCollect._is_in_sha1_set(sha1_e, reachable_set, reachable_count) - else: - in_set = StubMergerCollect._is_in_sha1_set(sha1_e, sha1_set, set_count) - if in_set == 0: - continue - - if file_count >= MAX_FILE_DEPS: - break - - # 初始化扁平数组：文件 SHA1（偏移 = file_count * SHA1_LEN） - # DepCount/InDegree/Processed 已由 memset 清零 - sha1_dst: str = file_sha1s_buf + file_count * SHA1_LEN - string.strcpy(sha1_dst, sha1_e) - - file_count += 1 - - # 2. 构建依赖边：对每个类，找到父类定义所在文件，建立 子文件→父文件 依赖 - for ci in range(class_count): - # 读取父类名（偏移 = ci * CLASS_NAME_LEN） - parent_ptr: str = parent_names_buf + ci * CLASS_NAME_LEN - if parent_ptr[0] == '\0': - continue - - # 读取子类定义 SHA1（偏移 = ci * SHA1_LEN） - child_def_sha1: str = def_sha1s_buf + ci * SHA1_LEN - - # 查找父类定义所在文件的 SHA1 - parent_sha1: str = _FindClassDefSha1(class_names_buf, def_sha1s_buf, class_count, parent_ptr) - if parent_sha1 is None: - continue # 父类不在 includes 中（如 Object、CEnum 等标记基类） - - # 父类文件 SHA1 不能等于子类文件 SHA1（同文件内继承不需要依赖） - if string.strcmp(parent_sha1, child_def_sha1) == 0: - continue - - # 找到子类文件对应的索引，添加父文件 SHA1 到依赖列表 - for fi in range(file_count): - fsha1: str = file_sha1s_buf + fi * SHA1_LEN - if string.strcmp(fsha1, child_def_sha1) != 0: - continue - - # 读取当前依赖数量 - cnt_addr: t.CUInt64T = t.CUInt64T(file_dep_counts) + fi * 4 - cnt_ptr: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(cnt_addr, t.CPtr)) - if cnt_ptr is None: - break - cur_cnt: int = cnt_ptr[0] - - # 检查依赖是否已添加（去重） - already: int = 0 - for di in range(cur_cnt): - dep_slot: str = file_deps_buf + fi * MAX_DEP_EDGES * SHA1_LEN + di * SHA1_LEN - if string.strcmp(dep_slot, parent_sha1) == 0: - already = 1 - break - if already != 0: - break - - if cur_cnt < MAX_DEP_EDGES: - dep_dst: str = file_deps_buf + fi * MAX_DEP_EDGES * SHA1_LEN + cur_cnt * SHA1_LEN - string.strcpy(dep_dst, parent_sha1) - cnt_ptr[0] = cur_cnt + 1 - break - - # 2.5 补充 import 依赖边：被 import 的模块先完整翻译。 - # - # 仅按类继承建边不够：跨模块全局变量 stub（lexer.vp from-import - # tokens.vp 的 _op_head）、跨模块 vtable TypeStr（astaux 继承 base 的 - # AST）、泛型模板（base.vp 用 list[...]）都要求依赖模块先翻译，而它们 - # 之间可能无类继承关系。deps（_includes_deps_*）由 Phase 1a-pre 填充 - # （含容器按需注入的 _list/_dict）。 - # - # 用 _BuildIncludesSha1Map 复制局部 sha1→模块名 数组（与 - # _BuildReachableSha1Set 的 BFS 一致），避免直接使用全局 store 返回值 - # 跨函数传参（bytes|t.CPtr 联合转 str 参数）的地址异常。 - sha1_arr_ts: bytes = stdlib.malloc(StubMerger.MAX_INCLUDES * 17) - mod_arr_ts: bytes = stdlib.malloc(StubMerger.MAX_INCLUDES * 64) - map_count_ts: int = 0 - if sha1_arr_ts is not None and mod_arr_ts is not None: - map_count_ts = StubMergerIR._BuildIncludesSha1Map( - temp_dir, td_len_ts, sha1_arr_ts, mod_arr_ts) - for fi_imp in range(file_count): - if map_count_ts <= 0: - break - fsha1_imp: str = file_sha1s_buf + fi_imp * SHA1_LEN - deps_imp: str = StubMergerStore.LookupIncludesDeps(fsha1_imp) - if deps_imp is None: - continue - dl_imp: t.CSizeT = string.strlen(deps_imp) - dpos_imp: t.CSizeT = 0 - while dpos_imp < dl_imp: - dstart_imp: t.CSizeT = dpos_imp - while dpos_imp < dl_imp and deps_imp[dpos_imp] != ' ': - dpos_imp += 1 - dlen_imp: t.CSizeT = dpos_imp - dstart_imp - if dpos_imp < dl_imp: - dpos_imp += 1 - if dlen_imp == 0: - continue - dname_imp: bytes = stdlib.malloc(dlen_imp + 1) - if dname_imp is None: - continue - string.strncpy(dname_imp, deps_imp + dstart_imp, dlen_imp) - dname_imp[dlen_imp] = '\0' - dsha1_imp: str = StubMergerIR._FindSha1ByModName( - sha1_arr_ts, mod_arr_ts, map_count_ts, dname_imp) - stdlib.free(dname_imp) - if dsha1_imp is None: - continue - # 找依赖文件在列表中的索引（跳过自身） - for fj_imp in range(file_count): - if string.strcmp(file_sha1s_buf + fj_imp * SHA1_LEN, dsha1_imp) != 0: - continue - if fj_imp == fi_imp: - break - # 去重 + 加边：fi_imp 依赖 fj_imp（InDegree[fi_imp]++） - cnt_addr_imp: t.CUInt64T = t.CUInt64T(file_dep_counts) + fi_imp * 4 - cnt_ptr_imp: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(cnt_addr_imp, t.CPtr)) - if cnt_ptr_imp is None: - break - cur_cnt_imp: int = cnt_ptr_imp[0] - already_imp: int = 0 - for di_imp in range(cur_cnt_imp): - dep_slot_imp: str = file_deps_buf + fi_imp * MAX_DEP_EDGES * SHA1_LEN + di_imp * SHA1_LEN - if string.strcmp(dep_slot_imp, dsha1_imp) == 0: - already_imp = 1 - break - if already_imp == 0 and cur_cnt_imp < MAX_DEP_EDGES: - dep_dst_imp: str = file_deps_buf + fi_imp * MAX_DEP_EDGES * SHA1_LEN + cur_cnt_imp * SHA1_LEN - string.strcpy(dep_dst_imp, dsha1_imp) - cnt_ptr_imp[0] = cur_cnt_imp + 1 - break - - # 释放 import 边用的局部映射数组 - if sha1_arr_ts is not None: - stdlib.free(sha1_arr_ts) - if mod_arr_ts is not None: - stdlib.free(mod_arr_ts) - - # 3. Kahn 算法：InDegree = DepCount（依赖多少个父文件） - # 依赖为 0 的先处理，处理后减少依赖它的文件入度 - for fi in range(file_count): - cnt_addr3: t.CUInt64T = t.CUInt64T(file_dep_counts) + fi * 4 - cnt_ptr3: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(cnt_addr3, t.CPtr)) - ind_addr3: t.CUInt64T = t.CUInt64T(file_indegrees) + fi * 4 - ind_ptr3: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(ind_addr3, t.CPtr)) - if cnt_ptr3 is not None and ind_ptr3 is not None: - ind_ptr3[0] = cnt_ptr3[0] - - # 4. 拓扑排序输出 - out_idx: int = 0 - changed: int = 1 - while changed != 0: - changed = 0 - for fi in range(file_count): - # 检查处理状态 - proc_addr4: t.CUInt64T = t.CUInt64T(file_processed) + fi * 4 - proc_ptr4: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(proc_addr4, t.CPtr)) - if proc_ptr4 is None or proc_ptr4[0] != 0: - continue - # 检查入度 - ind_addr4: t.CUInt64T = t.CUInt64T(file_indegrees) + fi * 4 - ind_ptr4: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(ind_addr4, t.CPtr)) - if ind_ptr4 is None or ind_ptr4[0] > 0: - continue - - # 入度为 0，可以处理 - # 找到对应的 result.Entries 索引 - target_sha1: str = file_sha1s_buf + fi * SHA1_LEN - for ri in range(result.Count): - ea_r: t.CUInt64T = t.CUInt64T(result.Entries) + ri * entry_size - ent_r: IncludesScanner.FileEntry | t.CPtr = (IncludesScanner.FileEntry | t.CPtr)(t.CVoid(ea_r, t.CPtr)) - if ent_r is not None and ent_r.Sha1 is not None: - if string.strcmp(ent_r.Sha1, target_sha1) == 0: - # 写入输出索引数组 - out_idx_addr: t.CUInt64T = t.CUInt64T(out_order) + out_idx * 4 - out_int_ptr: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(out_idx_addr, t.CPtr)) - if out_int_ptr is not None: - out_int_ptr[0] = ri - out_idx += 1 - break - - # 标记已处理 - proc_ptr4[0] = 1 - changed = 1 - - # 减少依赖该文件的其他文件的入度 - for fj in range(file_count): - proc_addr5: t.CUInt64T = t.CUInt64T(file_processed) + fj * 4 - proc_ptr5: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(proc_addr5, t.CPtr)) - if proc_ptr5 is None or proc_ptr5[0] != 0: - continue - # 读取 fj 的依赖数量 - cnt_addr5: t.CUInt64T = t.CUInt64T(file_dep_counts) + fj * 4 - cnt_ptr5: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(cnt_addr5, t.CPtr)) - if cnt_ptr5 is None: - continue - fj_dep_cnt: int = cnt_ptr5[0] - for di in range(fj_dep_cnt): - dep_slot5: str = file_deps_buf + fj * MAX_DEP_EDGES * SHA1_LEN + di * SHA1_LEN - if string.strcmp(dep_slot5, target_sha1) == 0: - ind_addr5: t.CUInt64T = t.CUInt64T(file_indegrees) + fj * 4 - ind_ptr5: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(ind_addr5, t.CPtr)) - if ind_ptr5 is not None: - ind_ptr5[0] = ind_ptr5[0] - 1 - break - - # 处理循环依赖：环内优先输出"被依赖者"（被其余剩余文件依赖数量多者先）。 - # - # 背景：容器按需注入（string 用 list 泛型 → 依赖 _list）与真实 import - # （_list import string）形成伪环，Kahn 无法完全归零。若按扫描顺序追加， - # 依赖者（astaux）会先于被依赖者（base）翻译 → 跨模块 vtable TypeStr - # 未注册 → 误杀。被依赖者先翻译，其 stub/TypeStr/模板先就绪，依赖者 - # 翻译时可引用（函数/全局声明由 1a-pre 的 vpi 兜底）。 - if out_idx < file_count: - rem_idx: t.CPtr = stdlib.malloc(4 * file_count) - rem_depcnt: t.CPtr = stdlib.malloc(4 * file_count) - rem_count: int = 0 - if rem_idx is not None and rem_depcnt is not None: - for fi in range(file_count): - proc_addr7: t.CUInt64T = t.CUInt64T(file_processed) + fi * 4 - proc_ptr7: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(proc_addr7, t.CPtr)) - if proc_ptr7 is not None and proc_ptr7[0] == 0: - rem_idx[rem_count] = fi - rem_depcnt[rem_count] = 0 - rem_count += 1 - # 计算每个剩余文件的被依赖数（被其余剩余文件依赖） - for i_a in range(rem_count): - fi_a: int = rem_idx[i_a] - target_sha1_a: str = file_sha1s_buf + fi_a * SHA1_LEN - for i_b in range(rem_count): - fi_b: int = rem_idx[i_b] - if fi_b == fi_a: - continue - cnt_addr_b: t.CUInt64T = t.CUInt64T(file_dep_counts) + fi_b * 4 - cnt_ptr_b: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(cnt_addr_b, t.CPtr)) - if cnt_ptr_b is None: - continue - cnt_b: int = cnt_ptr_b[0] - for di_b in range(cnt_b): - dep_slot_b: str = file_deps_buf + fi_b * MAX_DEP_EDGES * SHA1_LEN + di_b * SHA1_LEN - if string.strcmp(dep_slot_b, target_sha1_a) == 0: - rem_depcnt[i_a] = rem_depcnt[i_a] + 1 - break - # 冒泡降序（被依赖数多者在前） - for i_a in range(rem_count): - for j_a in range(rem_count - 1): - if rem_depcnt[j_a] < rem_depcnt[j_a + 1]: - tmp_i: int = rem_idx[j_a] - rem_idx[j_a] = rem_idx[j_a + 1] - rem_idx[j_a + 1] = tmp_i - tmp_d: int = rem_depcnt[j_a] - rem_depcnt[j_a] = rem_depcnt[j_a + 1] - rem_depcnt[j_a + 1] = tmp_d - # 按序输出剩余文件 - for i_a in range(rem_count): - fi_a2: int = rem_idx[i_a] - proc_addr8: t.CUInt64T = t.CUInt64T(file_processed) + fi_a2 * 4 - proc_ptr8: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(proc_addr8, t.CPtr)) - if proc_ptr8 is None or proc_ptr8[0] != 0: - continue - target_sha1_2: str = file_sha1s_buf + fi_a2 * SHA1_LEN - for ri in range(result.Count): - ea_r2: t.CUInt64T = t.CUInt64T(result.Entries) + ri * entry_size - ent_r2: IncludesScanner.FileEntry | t.CPtr = (IncludesScanner.FileEntry | t.CPtr)(t.CVoid(ea_r2, t.CPtr)) - if ent_r2 is not None and ent_r2.Sha1 is not None: - if string.strcmp(ent_r2.Sha1, target_sha1_2) == 0: - out_idx_addr2: t.CUInt64T = t.CUInt64T(out_order) + out_idx * 4 - out_int_ptr2: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(out_idx_addr2, t.CPtr)) - if out_int_ptr2 is not None: - out_int_ptr2[0] = ri - out_idx += 1 - break - proc_ptr8[0] = 1 - stdlib.free(rem_idx) - stdlib.free(rem_depcnt) - - # 写入输出数量 - out_count_ptr: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(out_count, t.CPtr)) - if out_count_ptr is not None: - out_count_ptr[0] = out_idx - - # 释放扁平数组 - mb.free(file_sha1s_buf) - mb.free(file_deps_buf) - mb.free(file_dep_counts) - mb.free(file_indegrees) - mb.free(file_processed) - - return 0 - - -# ============================================================ -# RunPhase1 - Phase1: 扫描 includes 目录，按需翻译并生成 stub -# -# 对每个 .vp/.py 文件检查 stub 是否已存在，若不存在则翻译并分离 stub。 -# "按需翻译"：仅对 stub 不存在的文件执行翻译，避免重复工作。 -# -# Args: -# mb: 内存池 -# includes_dir: includes 目录路径 -# temp_dir: 临时目录（保存 stub） -# log: 日志器 -# -# Returns: -# 0 成功，非 0 失败 -# ============================================================ -def RunPhase1(mb: memhub.MemBuddy | t.CPtr, includes_dir: str, temp_dir: str, - log: VLogger.Logger | t.CPtr) -> int: - """Phase1: 扫描 includes 目录，按需翻译并生成 stub""" - if includes_dir is None or temp_dir is None: - VLogger.error("includes_dir 或 temp_dir 为空，无法运行 Phase1", "Phase1") - return 1 - - # 设置全局 temp_dir（供跨模块 CDefine 查找使用） - HandlesType.set_temp_dir(temp_dir) - # 同步设置 HandlesExprCall 的 temp_dir（供 vpi stub 返回注解缓存按需加载）。 - # 必须在 includes 翻译前设置：Phase1 翻译 json 等 includes 时，"同模块方法 - # 调用（定义在后）"（如 json.__setitem__ 调 _pool_alloc）需要读 - # {temp_dir}/{sha1[0:2]}/{sha1}.vpi 兜底，而 Phase2 的 SetTempDir 太晚。 - ExprCallCtor.SetTempDir(temp_dir) - - # 确保 temp 目录存在（build_dir/temp 可能尚未创建） - BuildPipeline.ensure_dir(temp_dir) - - if log is not None: - log.banner("Phase1: 扫描 includes（按需翻译）") - - # 扫描 includes 目录 - result: IncludesScanner.ScanResult | t.CPtr = IncludesScanner.scan_includes(mb, includes_dir) - if result is None: - VLogger.error("扫描失败", "Phase1") - return 1 - - # 从扫描结果直接构建 SHA1 集合（不依赖 _sha1_map.txt 文件） - # Phase1 作为编译器自身，应从内存中的扫描结果获取 SHA1，避免外部文件依赖 - sha1_set: str = stdlib.malloc(StubMerger.MAX_INCLUDES_SHA1 * 17) - if sha1_set is None: - VLogger.error("sha1_set 分配失败", "Phase1") - return 1 - string.memset(sha1_set, 0, StubMerger.MAX_INCLUDES_SHA1 * 17) - set_count: int = 0 - p1_entry_size_init: t.CSizeT = IncludesScanner.FileEntry.__sizeof__() - for se_i in range(result.Count): - if set_count >= StubMerger.MAX_INCLUDES_SHA1: - break - se_addr: t.CUInt64T = t.CUInt64T(result.Entries) + se_i * p1_entry_size_init - se_ent: IncludesScanner.FileEntry | t.CPtr = (IncludesScanner.FileEntry | t.CPtr)(t.CVoid(se_addr, t.CPtr)) - if se_ent is None or se_ent.Sha1 is None: - continue - string.strcpy(sha1_set + set_count * 17, se_ent.Sha1) - set_count += 1 - # 先填充全局 SHA1 映射存储器（即使 set_count==0 也填充，确保 Phase2 能获取 includes 条目） - # 注意：PopulateSha1MapStore 会先 free 旧存储器再重新分配 - StubMergerStore.PopulateSha1MapStore(result) - # 写入 _sha1_map.txt（人类可读输出，程序内部不读取；机器分析使用 PopulateSha1MapStore 内存存储器） - StubMergerCollect.WriteIncludesSha1Map(mb, temp_dir, result, None, 0) - if set_count <= 0: - VLogger.warning("扫描结果无有效 SHA1，跳过 Phase1（项目可能无 includes 依赖）", "Phase1") - stdlib.free(sha1_set) - return 0 - # 模块 SHA1 映射已由 PopulateSha1MapStore 填充到 StubMerger 全局存储器， - # HandlesExprCall._lookup_module_sha1 直接查全局存储器，不再需要独立副本。 - - # 初始化 AST 表（只需一次） - ast._init_tables(mb) - - # FileEntry 结构体大小（Phase 1a 和 1b 共用） - entry_size: t.CSizeT = IncludesScanner.FileEntry.__sizeof__() - - # ============================================================ - # Phase 1a-pre: 扫描所有 includes 文件的 import 依赖 - # - # 只处理 Import/ImportFrom 语句，跳过 ClassDef/FunctionDef 等， - # 不调用 resolve_annotation_type，避免 list[...] 等不支持的语法触发 crash。 - # 生成 .deps.txt 供依赖图按需翻译使用。 - # ============================================================ - p1a_registered: int = 0 - p1a_skipped: int = 0 - p1a_failed: int = 0 - - for i in range(result.Count): - # 文件级 mark：import 依赖收集完（deps 写盘 + PopulateIncludesDeps - # 内部 stdlib 深拷贝）后立即回收本文件 AST——98 includes 全量 parse - # 的 AST 若驻留到 mark1 → 1GB 主池耗尽 OOM 根因之一 - scan_mark_a: t.CSizeT = mb.mark() - entry_addr_a: t.CUInt64T = t.CUInt64T(result.Entries) + i * entry_size - entry_a: IncludesScanner.FileEntry | t.CPtr = (IncludesScanner.FileEntry | t.CPtr)(t.CVoid(entry_addr_a, t.CPtr)) - if entry_a is None: - p1a_failed += 1 - continue - - sha1_a: str = entry_a.Sha1 - if sha1_a is None: - p1a_failed += 1 - continue - - if StubMergerCollect._is_in_sha1_set(sha1_a, sha1_set, set_count) == 0: - p1a_skipped += 1 - continue - - # 读取文件内容 - file_path_a: str = entry_a.Path - f_a: fileio.File | t.CPtr = fileio.File(file_path_a, fileio.MODE.R) - if f_a.closed: - p1a_failed += 1 - continue - - src_buf_a: bytes = stdlib.malloc(SRC_BUF_SIZE) - if src_buf_a is None: - f_a.close() - p1a_failed += 1 - continue - - bytes_read_a: LONG = f_a.read_all(src_buf_a, SRC_BUF_SIZE) - f_a.close() - if bytes_read_a <= 0: - stdlib.free(src_buf_a) - p1a_failed += 1 - continue - if bytes_read_a < SRC_BUF_SIZE: - src_buf_a[bytes_read_a] = 0 - else: - src_buf_a[SRC_BUF_SIZE - 1] = 0 - - # 解析 AST - lx_a: ast.Lexer | t.CPtr = ast.new_lexer(mb) - if lx_a is None: - stdlib.free(src_buf_a) - p1a_failed += 1 - continue - ast._lexer_init(lx_a, src_buf_a, mb) - tokens_a: ast.Token | t.CPtr = ast.tokenize(lx_a) - tree_a: ast.AST | t.CPtr = ast.parse_tokens(mb, tokens_a) - if tree_a is None: - stdlib.free(src_buf_a) - p1a_failed += 1 - continue - - # 创建 Translator，设置 _declare_only=2（只扫描 import，跳过 ClassDef） - tr_a: HandlesTranslator.Translator | t.CPtr = HandlesTranslator.Translator() - if tr_a is None: - stdlib.free(src_buf_a) - p1a_failed += 1 - continue - tr_a.ModuleSha1 = sha1_a - tr_a._declare_only = 2 - tr_a.CurrentPackage = HandlesImports.compute_package_from_relpath(mb, entry_a.RelPath) - HandlesType.set_current_file(file_path_a) - HandlesImports.set_current_file(file_path_a) - HandlesType.set_current_module_sha1(sha1_a) - HandlesType.clear_cdefine_constants() - HandlesType.clear_ctypedefs() - HandlesStruct.reset_visible_structs(mb, 0) - ret_a: int = tr_a.translate(tree_a) - if ret_a != 0: - p1a_failed += 1 - else: - p1a_registered += 1 - - # 容器模板按需纳入依赖：includes 使用 list/dict 泛型/构造/字面量 - # （如 base.vp 的 list[AST | CPtr]）时追加 _list/_dict 到 - # _imported_modules → 进 deps → _BuildReachableSha1Set 的可达集合 - # 自动包含（Phase1 与 Phase2 都受益），否则泛型模板未注册 → - # "泛型类 'list' 未找到模板"。对齐 App 层 _inject_auto_container_visibility。 - cl_box: t.CPtr = stdlib.malloc(4) - cd_box: t.CPtr = stdlib.malloc(4) - if cl_box is not None and cd_box is not None: - cl_box[0] = 0 - cd_box[0] = 0 - _ScanTreeForContainers(tree_a, cl_box, cd_box) - if cl_box[0] != 0: - if tr_a._imported_modules is None or \ - HandlesImports.is_module_imported(tr_a._imported_modules, "_list") == 0: - tr_a._imported_modules = HandlesImports.add_imported_module( - mb, tr_a._imported_modules, "_list") - if cd_box[0] != 0: - if tr_a._imported_modules is None or \ - HandlesImports.is_module_imported(tr_a._imported_modules, "_dict") == 0: - tr_a._imported_modules = HandlesImports.add_imported_module( - mb, tr_a._imported_modules, "_dict") - if cl_box is not None: - stdlib.free(cl_box) - if cd_box is not None: - stdlib.free(cd_box) - - # 生成 .deps.txt（记录依赖模块名，供依赖图按需翻译使用） - # 同时填充内存依赖存储器（PopulateIncludesDeps），供 _BuildReachableSha1Set - # 直接从内存读取，不依赖 .deps.txt 过程文件 - td_len_a: t.CSizeT = string.strlen(temp_dir) - deps_path_a: str = StubMergerCollect._sliced_path(temp_dir, td_len_a, sha1_a, "deps.txt") - if deps_path_a is not None: - df_a: fileio.File | t.CPtr = fileio.File(deps_path_a, fileio.MODE.W) - if not df_a.closed: - if tr_a._imported_modules is not None: - dl_a: t.CSizeT = string.strlen(tr_a._imported_modules) - df_a.write(tr_a._imported_modules, dl_a) - df_a.close() - stdlib.free(deps_path_a) - # 填充内存依赖存储器（供 _BuildReachableSha1Set 使用，不读 .deps.txt） - if tr_a._imported_modules is not None: - StubMergerStore.PopulateIncludesDeps(sha1_a, tr_a._imported_modules) - - # 释放 Translator 的 C malloc 资源 - if tr_a._global_names is not None: - stdlib.free(tr_a._global_names) - if tr_a._nonlocal_names is not None: - stdlib.free(tr_a._nonlocal_names) - - stdlib.free(src_buf_a) - # 回收本文件 AST/lexer/token（deps 已写盘、依赖存储器已深拷贝） - mb.reset_to(scan_mark_a) - - if p1a_failed > 0: - fb_p1apre: str = VLogger.fmt_buf() - if fb_p1apre is not None: - viperlib.snprintf(fb_p1apre, 1024, "Phase 1a-pre 有 %d 个文件扫描失败", p1a_failed) - VLogger.error(fb_p1apre, "Phase1") - return 1 - - # ============================================================ - # 依赖图按需翻译：构建可达 SHA1 集合 - # - # 从 Config.SourceDir 的源文件开始，解析 import 语句，递归收集 - # 可达的 includes 文件 SHA1。Phase 1b 只翻译可达集合中的文件， - # 避免翻译不需要的 includes（如 Test 不依赖 llvmlite，则不翻译）。 - # 如果 Config.SourceDir 不可用（仅运行 Phase1），回退到 sha1_set。 - # ============================================================ - reachable_set: str = None - reachable_count: int = 0 - use_reachable: int = 0 - if Config.SourceDir is not None: - reachable_set = stdlib.malloc(StubMerger.MAX_INCLUDES_SHA1 * 17) - if reachable_set is not None: - string.memset(reachable_set, 0, StubMerger.MAX_INCLUDES_SHA1 * 17) - reachable_count = StubMergerBuild._BuildReachableSha1Set(mb, Config.SourceDir, temp_dir, reachable_set) - if reachable_count < 0: - VLogger.error("可达集合构建失败，终止", "Phase1") - return 1 - if reachable_count > 0: - use_reachable = 1 - # 用可达集合重新生成 _sha1_map.txt（按图求索的最终产物） - # Phase B+ 只遍历这些条目，避免编译不需要的 includes（如 asm.vp） - StubMergerCollect.WriteIncludesSha1Map(mb, temp_dir, result, reachable_set, reachable_count) - # 重新加载 sha1_set，使后续 Phase 1a-pre/1a/1b 的过滤也使用可达集合 - string.memset(sha1_set, 0, StubMerger.MAX_INCLUDES_SHA1 * 17) - set_count = StubMergerCollect._load_includes_sha1_set(mb, temp_dir, sha1_set) - else: - VLogger.warning("可达集合为空，回退全量扫描", "Phase1") - stdlib.free(reachable_set) - reachable_set = None - - # ============================================================ - # Phase 1a: 注册可达 includes 文件的 struct/enum/union - # - # 解决字母序依赖问题：按字母顺序翻译时，后面的 struct 未注册 - # 前面就需要用。Phase 1a 先注册所有 struct/enum/union 到全局表， - # Phase 1b 全量翻译时 struct 已注册，走 existing 路径只翻译方法体。 - # 只处理可达文件，避免翻译不需要的 includes（如 Test 不依赖 ast 模块）。 - # - # 拓扑排序：预扫描所有文件收集类继承关系，按依赖顺序处理文件， - # 确保父类所在文件先于子类所在文件处理（如 base.vp 先于 astaux.vp）。 - # ============================================================ - p1a_reg: int = 0 - p1a_skp: int = 0 - p1a_fl: int = 0 - - # 预扫描类继承关系，构建拓扑顺序（用扁平字节数组，避免结构体 str 指针 NULL 崩溃） - class_names_buf: bytes = mb.alloc(MAX_CLASSES * CLASS_NAME_LEN) - parent_names_buf: bytes = mb.alloc(MAX_CLASSES * CLASS_NAME_LEN) - def_sha1s_buf: bytes = mb.alloc(MAX_CLASSES * SHA1_LEN) - topo_order: t.CPtr = mb.alloc(4 * MAX_FILE_DEPS) - topo_count_box: t.CPtr = mb.alloc(4) - generic_file_indices: t.CPtr = mb.alloc(4 * MAX_FILE_DEPS) - generic_file_count_box: t.CPtr = mb.alloc(4) - if generic_file_indices is not None: - string.memset(generic_file_indices, 0, 4 * MAX_FILE_DEPS) - if generic_file_count_box is not None: - string.memset(generic_file_count_box, 0, 4) - topo_count: int = 0 - - if class_names_buf is not None and parent_names_buf is not None and def_sha1s_buf is not None and topo_order is not None and topo_count_box is not None: - string.memset(class_names_buf, 0, MAX_CLASSES * CLASS_NAME_LEN) - string.memset(parent_names_buf, 0, MAX_CLASSES * CLASS_NAME_LEN) - string.memset(def_sha1s_buf, 0, MAX_CLASSES * SHA1_LEN) - string.memset(topo_order, 0, 4 * MAX_FILE_DEPS) - string.memset(topo_count_box, 0, 4) - if generic_file_indices is not None: - string.memset(generic_file_indices, 0, 4 * MAX_FILE_DEPS) - if generic_file_count_box is not None: - string.memset(generic_file_count_box, 0, 4) - - class_count_scanned: int = _ScanClassInheritance( - mb, result, reachable_set, reachable_count, - use_reachable, sha1_set, set_count, - class_names_buf, parent_names_buf, def_sha1s_buf, - generic_file_indices, generic_file_count_box) - - if class_count_scanned > 0: - td_len_ts_call: t.CSizeT = string.strlen(temp_dir) - ret_topo: int = _TopoSortFiles( - mb, result, reachable_set, reachable_count, - use_reachable, sha1_set, set_count, - class_names_buf, parent_names_buf, def_sha1s_buf, class_count_scanned, - topo_order, topo_count_box, - temp_dir, td_len_ts_call) - if ret_topo == 0: - tc_ptr: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(topo_count_box, t.CPtr)) - if tc_ptr is not None: - topo_count = tc_ptr[0] - fb_topo: str = VLogger.fmt_buf() - if fb_topo is not None: - viperlib.snprintf(fb_topo, 1024, - "拓扑排序: %d 个类, %d 个文件按依赖顺序处理", - class_count_scanned, topo_count) - VLogger.info(fb_topo, "Phase1") - else: - VLogger.error("拓扑排序失败（类继承依赖无法解析），终止编译", "Phase1") - return 1 - else: - VLogger.warning("类继承预扫描无结果，回退到字母序", "Phase1") - topo_count = 0 - - # ============================================================ - # Phase 1a-pre-generic-all: mark1 前全量预注册 includes 泛型模板 - # - # 1a-pre-generic 只覆盖"可达"泛型文件；非可达泛型文件（如 FuncTest - # 场景的 linkedlist/llvmlite/_list）的模板要等 Phase2 Phase A-pre-inc - # （mark2 之后）才注册 → reset 回收表/节点 → _find_generic_template - # strcmp 悬空崩溃。此处对全部 includes 轻量预注册，使模板表与节点 - # 都落在 mark1 之前（mark2 > mark1，reset_to(mark2) 不回收）。 - # ============================================================ - _PreRegisterAllGenericTemplates(mb, result) - - # ============================================================ - # Phase 1a: 注册 struct/enum/union（mark1 前，全量 includes） - # - # OOM 治理：1a 对全部 includes parse 完整 AST（主 pool 分配），若 - # 驻留到 mark1（循环后）会累积 98 文件 × 每文件 AST 树 → 1GB 主池 - # 耗尽（NewAstTest alloc 失败 0xBE 根因）。现改为： - # 1) StdlibAlloc=1：1a 注册的跨文件持久数据（StructTable 表、 - # Fields/Name/ModuleSha1、struct_ty 深拷贝、枚举/Renum/泛型 - # 模板等）全部落在 perm_pool（永不 reset），不占主 pool； - # 2) 每文件翻译完 reset_to(文件级 mark)：仅回收该文件的 AST/ - # lexer/token/IR 等瞬时数据，注册数据不受影响。 - # mark1 仍在 1a 结束后打，1b 每文件 reset_to(mark1) 语义不变。 - # ============================================================ - HandlesStruct.set_stdlib_alloc(1) - - # 遍历文件：优先使用拓扑顺序，回退到字母序 - iter_count: int = topo_count if topo_count > 0 else result.Count - for i in range(iter_count): - # 每文件级 mark：翻译完回收该文件 AST（注册数据在 perm_pool） - p1a_mark: t.CSizeT = mb.mark() - # 获取文件索引：拓扑顺序或字母序 - file_idx: int = i - if topo_count > 0: - od_addr: t.CUInt64T = t.CUInt64T(topo_order) + i * 4 - od_ptr: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(od_addr, t.CPtr)) - if od_ptr is not None: - file_idx = od_ptr[0] - - entry_addr_r: t.CUInt64T = t.CUInt64T(result.Entries) + file_idx * entry_size - entry_r: IncludesScanner.FileEntry | t.CPtr = (IncludesScanner.FileEntry | t.CPtr)(t.CVoid(entry_addr_r, t.CPtr)) - if entry_r is None: - p1a_fl += 1 - continue - - sha1_r: str = entry_r.Sha1 - if sha1_r is None: - p1a_fl += 1 - continue - - # 按需翻译过滤（拓扑排序已过滤，但字母序回退需要检查） - if topo_count == 0: - if use_reachable != 0: - if StubMergerCollect._is_in_sha1_set(sha1_r, reachable_set, reachable_count) == 0: - p1a_skp += 1 - continue - else: - if StubMergerCollect._is_in_sha1_set(sha1_r, sha1_set, set_count) == 0: - p1a_skp += 1 - continue - - # 读取文件内容 - file_path_r: str = entry_r.Path - f_r: fileio.File | t.CPtr = fileio.File(file_path_r, fileio.MODE.R) - if f_r.closed: - p1a_fl += 1 - continue - - src_buf_r: bytes = stdlib.malloc(SRC_BUF_SIZE) - if src_buf_r is None: - f_r.close() - p1a_fl += 1 - continue - - bytes_read_r: LONG = f_r.read_all(src_buf_r, SRC_BUF_SIZE) - f_r.close() - if bytes_read_r <= 0: - stdlib.free(src_buf_r) - p1a_fl += 1 - continue - if bytes_read_r < SRC_BUF_SIZE: - src_buf_r[bytes_read_r] = 0 - else: - src_buf_r[SRC_BUF_SIZE - 1] = 0 - - # 解析 AST - lx_r: ast.Lexer | t.CPtr = ast.new_lexer(mb) - if lx_r is None: - stdlib.free(src_buf_r) - p1a_fl += 1 - continue - ast._lexer_init(lx_r, src_buf_r, mb) - tokens_r: ast.Token | t.CPtr = ast.tokenize(lx_r) - tree_r: ast.AST | t.CPtr = ast.parse_tokens(mb, tokens_r) - if tree_r is None: - stdlib.free(src_buf_r) - p1a_fl += 1 - continue - - # 创建 Translator，设置 _declare_only=1（注册 struct/enum/union） - tr_r: HandlesTranslator.Translator | t.CPtr = HandlesTranslator.Translator() - if tr_r is None: - stdlib.free(src_buf_r) - p1a_fl += 1 - continue - tr_r.ModuleSha1 = sha1_r - tr_r._declare_only = 1 - tr_r.CurrentPackage = HandlesImports.compute_package_from_relpath(mb, entry_r.RelPath) - HandlesType.set_current_file(file_path_r) - HandlesImports.set_current_file(file_path_r) - HandlesType.set_current_module_sha1(sha1_r) - HandlesType.clear_cdefine_constants() - HandlesType.clear_ctypedefs() - HandlesStruct.reset_visible_structs(mb, 0) - ret_r: int = tr_r.translate(tree_r) - if ret_r != 0: - p1a_fl += 1 - else: - p1a_reg += 1 - - # 释放 Translator 的 C malloc 资源 - if tr_r._global_names is not None: - stdlib.free(tr_r._global_names) - if tr_r._nonlocal_names is not None: - stdlib.free(tr_r._nonlocal_names) - stdlib.free(src_buf_r) - - # 回收本文件 AST/lexer/token/IR 等瞬时数据（注册数据已在 perm_pool， - # reset 不影响；IncludesScanner result 等在 p1a_mark 前分配同样保留） - mb.reset_to(p1a_mark) - - if p1a_fl > 0: - fb_p1ar: str = VLogger.fmt_buf() - if fb_p1ar is not None: - viperlib.snprintf(fb_p1ar, 1024, "Phase 1a 有 %d 个文件 struct 注册失败", p1a_fl) - VLogger.error(fb_p1ar, "Phase1") - return 1 - - # 释放拓扑排序资源（topo_order/topo_count_box 保留到 Phase 1b 结束， - # 1b 按拓扑序翻译需要它们；class/def 数组 1b 不再使用） - if class_names_buf is not None: - mb.free(class_names_buf) - if parent_names_buf is not None: - mb.free(parent_names_buf) - if def_sha1s_buf is not None: - mb.free(def_sha1s_buf) - if generic_file_indices is not None: - mb.free(generic_file_indices) - if generic_file_count_box is not None: - mb.free(generic_file_count_box) - - # ============================================================ - # Phase 1b: 全量翻译（struct 已注册，走 existing 路径翻译方法体） - # ============================================================ - # OOM 回收：1a-pre/泛型预注册/1a 全部结束后打 mark1， - # 所有跨文件持久表（_struct_table/_enum_table/_generic_class_nodes/ - # _renum_*/关键字表等）都落在 mark1 之前。 - # 1b 每文件翻译完后 reset_to(mark1) 回收该文件的 AST/IR/Handler/dump 临时， - # 避免 90 文件累积导致 1GB 池 OOM。 - mark1: t.CSizeT = mb.mark() - - # 1b 阶段新注册的 struct/enum 条目改 stdlib 分配（跨文件永久存活，不被 reset 回收） - HandlesStruct.set_stdlib_alloc(1) - - # ============================================================ - # Phase 1b-pre: 预生成所有待翻译文件的 .vpi stub - # - # 依赖模块的 .vpi 必须在翻译前可用：1b 主循环按扫描顺序翻译（非拓扑）， - # 若 argparse 先于 hashtable 翻译，hashtable.vpi 尚未生成， - # LoadDepVpiIntoModule 读不到 → declare 未注入 → 跨模块调用 - # 返回类型推断错误（如 HashTable.__getitem__ 被推断为 Ptr(HashTable) - # 而 define 返回 i8*）→ 运行时返回值截断崩溃。 - # ============================================================ - p1bpre_failed: int = 0 - dir_len_pv: t.CSizeT = string.strlen(temp_dir) - for i_pv in range(result.Count): - entry_addr_pv: t.CUInt64T = t.CUInt64T(result.Entries) + i_pv * entry_size - entry_pv: IncludesScanner.FileEntry | t.CPtr = (IncludesScanner.FileEntry | t.CPtr)(t.CVoid(entry_addr_pv, t.CPtr)) - if entry_pv is None or entry_pv.Sha1 is None: - p1bpre_failed += 1 - continue - sha1_pv: str = entry_pv.Sha1 - in_set_pv: int = 0 - if use_reachable != 0: - in_set_pv = StubMergerCollect._is_in_sha1_set(sha1_pv, reachable_set, reachable_count) - else: - in_set_pv = StubMergerCollect._is_in_sha1_set(sha1_pv, sha1_set, set_count) - if in_set_pv == 0: - continue - # vpi 已存在则跳过 - vpi_path_pv: str = StubMergerCollect._sliced_path(temp_dir, dir_len_pv, sha1_pv, "vpi") - if vpi_path_pv is None: - p1bpre_failed += 1 - continue - vf_pv: fileio.File | t.CPtr = fileio.File(vpi_path_pv, fileio.MODE.R) - if not vf_pv.closed: - vf_pv.close() - stdlib.free(vpi_path_pv) - continue - stdlib.free(vpi_path_pv) - # 读取文件、解析 AST、生成 vpi - file_path_pv: str = entry_pv.Path - f_pv: fileio.File | t.CPtr = fileio.File(file_path_pv, fileio.MODE.R) - if f_pv.closed: - p1bpre_failed += 1 - continue - src_buf_pv: bytes = stdlib.malloc(SRC_BUF_SIZE) - if src_buf_pv is None: - f_pv.close() - p1bpre_failed += 1 - continue - bytes_read_pv: LONG = f_pv.read_all(src_buf_pv, SRC_BUF_SIZE) - f_pv.close() - if bytes_read_pv <= 0: - stdlib.free(src_buf_pv) - p1bpre_failed += 1 - continue - if bytes_read_pv < SRC_BUF_SIZE: - src_buf_pv[bytes_read_pv] = 0 - else: - src_buf_pv[SRC_BUF_SIZE - 1] = 0 - lx_pv: ast.Lexer | t.CPtr = ast.new_lexer(mb) - if lx_pv is None: - stdlib.free(src_buf_pv) - p1bpre_failed += 1 - continue - ast._lexer_init(lx_pv, src_buf_pv, mb) - tokens_pv: ast.Token | t.CPtr = ast.tokenize(lx_pv) - tree_pv: ast.AST | t.CPtr = ast.parse_tokens(mb, tokens_pv) - if tree_pv is None: - stdlib.free(src_buf_pv) - p1bpre_failed += 1 - continue - pyi_buf_pv: bytes = stdlib.malloc(PYI_BUF_SIZE) - if pyi_buf_pv is not None: - pyi_pos_pv: t.CSizeT = StubConverter._GeneratePyiFromAst(mb, tree_pv, entry_pv.RelPath, pyi_buf_pv, PYI_BUF_SIZE) - if pyi_pos_pv > 0: - pyi_path_pv: str = StubMergerCollect._sliced_path(temp_dir, dir_len_pv, sha1_pv, "vpi") - if pyi_path_pv is not None: - pf_pv: fileio.File | t.CPtr = fileio.File(pyi_path_pv, fileio.MODE.W) - if not pf_pv.closed: - pf_pv.write(pyi_buf_pv, pyi_pos_pv) - pf_pv.close() - stdlib.free(pyi_path_pv) - stdlib.free(pyi_buf_pv) - else: - p1bpre_failed += 1 - stdlib.free(src_buf_pv) - # 回收本文件 AST（vpi 已写盘，AST 不再需要；mark1 后无其他持久数据） - mb.reset_to(mark1) - - if p1bpre_failed > 0: - fb_p1bpre: str = VLogger.fmt_buf() - if fb_p1bpre is not None: - viperlib.snprintf(fb_p1bpre, 1024, "Phase 1b-pre 有 %d 个文件 vpi 预生成失败", p1bpre_failed) - VLogger.error(fb_p1bpre, "Phase1") - return 1 - - # 遍历每个文件（拓扑序优先：父类所在文件先翻译，确保跨模块 vtable - # 生成时依赖模块的函数 TypeStr 已注册；topo_count==0 回退扫描顺序） - translated: int = 0 - skipped: int = 0 - failed: int = 0 - - loop_n_b: int = result.Count - if topo_count > 0: - loop_n_b = topo_count - for i in range(loop_n_b): - # 获取 entry（拓扑序：file_idx 从 topo_order 取，否则用扫描索引） - file_idx_b: int = i - if topo_count > 0: - od_addr_b: t.CUInt64T = t.CUInt64T(topo_order) + t.CSizeT(i) * 4 - od_ptr_b: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(od_addr_b, t.CPtr)) - if od_ptr_b is not None: - file_idx_b = od_ptr_b[0] - entry_addr: t.CUInt64T = t.CUInt64T(result.Entries) + t.CSizeT(file_idx_b) * entry_size - entry: IncludesScanner.FileEntry | t.CPtr = (IncludesScanner.FileEntry | t.CPtr)(t.CVoid(entry_addr, t.CPtr)) - if entry is None: - failed += 1 - continue - - sha1: str = entry.Sha1 - if sha1 is None: - failed += 1 - continue - - # 检查 SHA1 是否在翻译集合中（拓扑序已过滤可达；仅回退扫描顺序时检查） - in_set: int = 1 - if topo_count == 0: - in_set = 0 - if use_reachable != 0: - in_set = StubMergerCollect._is_in_sha1_set(sha1, reachable_set, reachable_count) - else: - in_set = StubMergerCollect._is_in_sha1_set(sha1, sha1_set, set_count) - if in_set == 0: - skipped += 1 - continue - - # 构造 stub 路径: {temp_dir}/{sha1}.stub.ll - dir_len: t.CSizeT = string.strlen(temp_dir) - sha1_len: t.CSizeT = string.strlen(sha1) - stub_path: str = StubMergerCollect._sliced_path(temp_dir, dir_len, sha1, "stub.ll") - if stub_path is None: - failed += 1 - continue - - # 检查 stub 是否已存在（按需翻译：跳过已存在的） - sf: fileio.File | t.CPtr = fileio.File(stub_path, fileio.MODE.R) - if not sf.closed: - sf.close() - # 检查 text.ll 是否也存在（虚表扫描需要 text.ll） - text_path: str = StubMergerCollect._sliced_path(temp_dir, dir_len, sha1, "text.ll") - if text_path is not None: - tf: fileio.File | t.CPtr = fileio.File(text_path, fileio.MODE.R) - if not tf.closed: - tf.close() - stdlib.free(text_path) - stdlib.free(stub_path) - continue - stdlib.free(text_path) - # text.ll 不存在，需要重新翻译 - rp: str = entry.RelPath - fb: str = VLogger.fmt_buf() - if fb is not None: - viperlib.snprintf(fb, 1024, "text.ll 不存在，重新翻译: %s (sha1=%s)", rp, sha1) - VLogger.info(fb, "Phase1") - else: - # stub 不存在，需要翻译 - rp: str = entry.RelPath - fb: str = VLogger.fmt_buf() - if fb is not None: - viperlib.snprintf(fb, 1024, "翻译: %s (sha1=%s)", rp, sha1) - VLogger.info(fb, "Phase1") - - stdlib.free(stub_path) - - # 读取文件内容 - file_path: str = entry.Path - f: fileio.File | t.CPtr = fileio.File(file_path, fileio.MODE.R) - if f is None: - fb: str = VLogger.fmt_buf() - if fb is not None: - viperlib.snprintf(fb, 1024, "无法打开(None): %s", file_path) - VLogger.error(fb, "Phase1") - failed += 1 - continue - if f.closed: - fb: str = VLogger.fmt_buf() - if fb is not None: - viperlib.snprintf(fb, 1024, "无法打开: %s", file_path) - VLogger.error(fb, "Phase1") - failed += 1 - continue - - src_buf: bytes = stdlib.malloc(SRC_BUF_SIZE) - if src_buf is None: - f.close() - failed += 1 - continue - - bytes_read: LONG = f.read_all(src_buf, SRC_BUF_SIZE) - f.close() - if bytes_read <= 0: - fb: str = VLogger.fmt_buf() - if fb is not None: - viperlib.snprintf(fb, 1024, "读取失败: %s", file_path) - VLogger.error(fb, "Phase1") - stdlib.free(src_buf) - failed += 1 - continue - if bytes_read < SRC_BUF_SIZE: - src_buf[bytes_read] = 0 - else: - src_buf[SRC_BUF_SIZE - 1] = 0 - - # 解析 AST - lx: ast.Lexer | t.CPtr = ast.new_lexer(mb) - if lx is None: - stdlib.free(src_buf) - failed += 1 - continue - ast._lexer_init(lx, src_buf, mb) - tokens: ast.Token | t.CPtr = ast.tokenize(lx) - tree: ast.AST | t.CPtr = ast.parse_tokens(mb, tokens) - if tree is None: - fb: str = VLogger.fmt_buf() - if fb is not None: - viperlib.snprintf(fb, 1024, "AST 解析失败: %s", file_path) - VLogger.error(fb, "Phase1") - stdlib.free(src_buf) - failed += 1 - continue - - # 生成 .vpi 存根文件（直接遍历 AST，不依赖 PythonToStubConverter） - pyi_buf: bytes = stdlib.malloc(PYI_BUF_SIZE) - if pyi_buf is not None: - pyi_pos: t.CSizeT = StubConverter._GeneratePyiFromAst(mb, tree, entry.RelPath, pyi_buf, PYI_BUF_SIZE) - if pyi_pos > 0: - pyi_path: str = StubMergerCollect._sliced_path(temp_dir, dir_len, sha1, "vpi") - if pyi_path is not None: - pf: fileio.File | t.CPtr = fileio.File(pyi_path, fileio.MODE.W) - if not pf.closed: - pf.write(pyi_buf, pyi_pos) - pf.close() - stdlib.free(pyi_path) - stdlib.free(pyi_buf) - - # 翻译 AST → LLVM IR - tr: HandlesTranslator.Translator | t.CPtr = HandlesTranslator.Translator() - if tr is None: - stdlib.free(src_buf) - failed += 1 - continue - tr.ModuleSha1 = sha1 - tr.CurrentPackage = HandlesImports.compute_package_from_relpath(mb, entry.RelPath) - HandlesType.set_current_file(file_path) - HandlesImports.set_current_file(file_path) - HandlesType.set_current_module_sha1(sha1) - HandlesType.clear_cdefine_constants() - HandlesType.clear_ctypedefs() - HandlesStruct.reset_visible_structs(mb, 0) - - # 翻译前哨兵：AST 解析（lexer/tokenize/parse_tokens）同样在 pool 分配， - # 若解析器越界写破坏了存活块 header，此处立即定位到当前文件 - # （崩溃点漂移排查：把"翻译过程中随机崩"变成确定性定位） - pre_rc: t.CInt = mb.check_alive_blocks(mark1) - if pre_rc != 0: - fb: str = VLogger.fmt_buf() - if fb is not None: - viperlib.snprintf(fb, 1024, - "内存损坏(翻译前存活块, rc=%d): %s", pre_rc, file_path) - VLogger.error(fb, "Phase1") - stdlib.exit(1) - - ret: int = tr.translate(tree) - # 诊断：translate 正常返回（若挂起在 translate 内部，此行不会打印， - # 用于区分"translate 内部挂起"与"后续 dump 挂起"） - if ret == 0: - fb_done: str = VLogger.fmt_buf() - if fb_done is not None: - viperlib.snprintf(fb_done, 1024, "翻译完成(translate返回): %s", file_path) - VLogger.info(fb_done, "Phase1") - if ret != 0: - fb: str = VLogger.fmt_buf() - if fb is not None: - viperlib.snprintf(fb, 1024, "翻译失败: %s", file_path) - VLogger.error(fb, "Phase1") - stdlib.free(src_buf) - if tr._global_names is not None: - stdlib.free(tr._global_names) - if tr._nonlocal_names is not None: - stdlib.free(tr._nonlocal_names) - failed += 1 - continue - - # dump stub IR (declarations only) - PHASE1_IR_SIZE: t.CSizeT = 1048576 - stub_buf: bytes = stdlib.malloc(PHASE1_IR_SIZE) - if stub_buf is None: - stdlib.free(src_buf) - if tr._global_names is not None: - stdlib.free(tr._global_names) - if tr._nonlocal_names is not None: - stdlib.free(tr._nonlocal_names) - failed += 1 - continue - tr.dump_ir(stub_buf, PHASE1_IR_SIZE, llvmlite.OUTPUT_STUB) - stub_len: t.CSizeT = string.strlen(stub_buf) - - # save stub.ll - dir_len_p1: t.CSizeT = string.strlen(temp_dir) - stub_path_p1: str = StubMergerCollect._sliced_path(temp_dir, dir_len_p1, sha1, "stub.ll") - if stub_path_p1 is not None: - sf_p1: fileio.File | t.CPtr = fileio.File(stub_path_p1, fileio.MODE.W) - if not sf_p1.closed: - sf_p1.write(stub_buf, stub_len) - sf_p1.close() - stdlib.free(stub_path_p1) - stdlib.free(stub_buf) - - # dump text IR (definitions only) - text_buf: bytes = stdlib.malloc(PHASE1_IR_SIZE) - if text_buf is None: - stdlib.free(src_buf) - if tr._global_names is not None: - stdlib.free(tr._global_names) - if tr._nonlocal_names is not None: - stdlib.free(tr._nonlocal_names) - failed += 1 - continue - tr.dump_ir(text_buf, PHASE1_IR_SIZE, llvmlite.OUTPUT_TEXT) - text_len: t.CSizeT = string.strlen(text_buf) - - # save text.ll - text_path_p1: str = StubMergerCollect._sliced_path(temp_dir, dir_len_p1, sha1, "text.ll") - if text_path_p1 is not None: - tf_p1: fileio.File | t.CPtr = fileio.File(text_path_p1, fileio.MODE.W) - if not tf_p1.closed: - tf_p1.write(text_buf, text_len) - tf_p1.close() - stdlib.free(text_path_p1) - stdlib.free(text_buf) - - # save dependencies (_imported_modules) for Phase B - deps_path_p1: str = StubMergerCollect._sliced_path(temp_dir, dir_len_p1, sha1, "deps.txt") - if deps_path_p1 is not None: - df_p1: fileio.File | t.CPtr = fileio.File(deps_path_p1, fileio.MODE.W) - if not df_p1.closed: - if tr._imported_modules is not None: - dl: t.CSizeT = string.strlen(tr._imported_modules) - df_p1.write(tr._imported_modules, dl) - df_p1.close() - stdlib.free(deps_path_p1) - - # 释放 Translator 的 C malloc 资源 - if tr._global_names is not None: - stdlib.free(tr._global_names) - if tr._nonlocal_names is not None: - stdlib.free(tr._nonlocal_names) - - # 所有 dump_ir 完成，src_buf 不再需要 - stdlib.free(src_buf) - - translated += 1 - - # OOM 回收：回收本文件翻译产生的瞬时数据（AST/tokens/lexer/IR/Handler/dump 临时） - # 持久表（struct/enum/REnum/spec 缓存/全局函数名等）已迁出 reset 区间（stdlib）， - # reset_to(mark1) 只回收 mark1 之后分配、未被单独 free 的块。 - # - # 内存哨兵（崩溃点漂移排查）：reset 前检查存活块 header（抓越界写）， - # reset 后检查 free list 完整性（抓链表损坏）。任一损坏立即终止， - # 把"随机崩溃漂移"变成"确定性定位到当前文件"。 - # rc 语义（check_alive_blocks）：1=块地址低于 usable，2=高于 usable+size， - # 3=order 为负，4=order 超 max_order，5=块未对齐到 order 大小， - # 6=块尾 canary 损坏（越界写穿数据区——事发地定位） - # rc 语义（self_check）：1/2=越界，3=对齐错，4=free list 成环/超步数 - chk_rc: t.CInt = mb.check_alive_blocks(mark1) - if chk_rc != 0: - fb: str = VLogger.fmt_buf() - if fb is not None: - viperlib.snprintf(fb, 1024, - "内存损坏(存活块 header, rc=%d): %s", chk_rc, file_path) - VLogger.error(fb, "Phase1") - stdlib.exit(1) - # A/B 实验结论（已确认）：禁用 reset_to 后 stdarg 仍崩 → 越界写是根因， - # 与回收/UAF 无关。恢复 reset_to，改用块尾 canary（check_alive_blocks 校验） - # 定位越界写事发地到具体文件。 - mb.reset_to(mark1) - sc_rc: t.CInt = mb.self_check() - if sc_rc != 0: - fb: str = VLogger.fmt_buf() - if fb is not None: - viperlib.snprintf(fb, 1024, - "内存损坏(free list, rc=%d): %s", sc_rc, file_path) - VLogger.error(fb, "Phase1") - stdlib.exit(1) - - # 释放 Phase 1b 使用的拓扑排序资源 - if topo_order is not None: - mb.free(topo_order) - if topo_count_box is not None: - mb.free(topo_count_box) - - fb: str = VLogger.fmt_buf() - if fb is not None: - viperlib.snprintf(fb, 1024, "完成: 翻译=%d 跳过=%d 失败=%d", translated, skipped, failed) - VLogger.info(fb, "Phase1") - - if failed > 0: - fb_fail: str = VLogger.fmt_buf() - if fb_fail is not None: - viperlib.snprintf(fb_fail, 1024, "Phase1 有 %d 个文件翻译失败", failed) - VLogger.error(fb_fail, "Phase1") - return 1 - - # 释放可达 SHA1 集合（如果分配了） - if reachable_set is not None: - stdlib.free(reachable_set) - - return 0 diff --git a/App/lib/core/Phase1Main.vp b/App/lib/core/Phase1Main.vp new file mode 100644 index 0000000..23a3cbe --- /dev/null +++ b/App/lib/core/Phase1Main.vp @@ -0,0 +1,954 @@ +import t, c +from stdint import * +import string +import stdlib +import memhub +import viperlib +import w32.fileio as fileio +import w32.win32file +import w32.win32base +import ast +import llvmlite +import lib.core.VLogger as VLogger +import lib.core.Handles.HandlesTranslator as HandlesTranslator +import lib.core.Handles.HandlesStruct as HandlesStruct +import lib.core.Handles.HandlesType as HandlesType +import lib.core.Handles.HandlesExprCall as HandlesExprCall +import lib.core.Handles.HandlesImports as HandlesImports +import lib.core.Handles.HandlesClassDef as HandlesClassDef +import lib.core.IncludesScanner as IncludesScanner +import lib.core.StubMerger as StubMerger +import lib.core.Gen.Generator as GenGenerator +import lib.core.CompilerFactory as CompilerFactory +import lib.core.BuildPipeline as BuildPipeline +import lib.Projectrans.Config as Config +import lib.StubGen.Converter as StubConverter +import lib.core.StubMergerBuild as StubMergerBuild +import lib.core.StubMergerCollect as StubMergerCollect +import lib.core.StubMergerIR as StubMergerIR +import lib.core.StubMergerStore as StubMergerStore +import lib.core.Handles.ExprCallCtor as ExprCallCtor + + +import lib.core.Phase1Scan as Phase1Scan +import lib.core.Phase1Topo as Phase1Topo +import lib.core.Phase1 as Phase1 + +# ============================================================ +# Phase1Main - Phase1 拆分（阶段 E）：Phase1Main +# +# 直接移动模式：函数迁至此模块，原 Phase1.vp 仅保留常量/_mbuddy； +# 调用方经本模块前缀访问。全局符号经 Phase1._xxx 跨模块访问。 +# ============================================================ + +# ============================================================ +# RunPhase1 - Phase1: 扫描 includes 目录，按需翻译并生成 stub +# +# 对每个 .vp/.py 文件检查 stub 是否已存在，若不存在则翻译并分离 stub。 +# "按需翻译"：仅对 stub 不存在的文件执行翻译，避免重复工作。 +# +# Args: +# mb: 内存池 +# includes_dir: includes 目录路径 +# temp_dir: 临时目录（保存 stub） +# log: 日志器 +# +# Returns: +# 0 成功，非 0 失败 +# ============================================================ +def RunPhase1(mb: memhub.MemBuddy | t.CPtr, includes_dir: str, temp_dir: str, + log: VLogger.Logger | t.CPtr) -> int: + """Phase1: 扫描 includes 目录，按需翻译并生成 stub""" + if includes_dir is None or temp_dir is None: + VLogger.error("includes_dir 或 temp_dir 为空，无法运行 Phase1", "Phase1") + return 1 + + # 设置全局 temp_dir（供跨模块 CDefine 查找使用） + HandlesType.set_temp_dir(temp_dir) + # 同步设置 HandlesExprCall 的 temp_dir（供 vpi stub 返回注解缓存按需加载）。 + # 必须在 includes 翻译前设置：Phase1 翻译 json 等 includes 时，"同模块方法 + # 调用（定义在后）"（如 json.__setitem__ 调 _pool_alloc）需要读 + # {temp_dir}/{sha1[0:2]}/{sha1}.vpi 兜底，而 Phase2 的 SetTempDir 太晚。 + ExprCallCtor.SetTempDir(temp_dir) + + # 确保 temp 目录存在（build_dir/temp 可能尚未创建） + BuildPipeline.ensure_dir(temp_dir) + + if log is not None: + log.banner("Phase1: 扫描 includes（按需翻译）") + + # 扫描 includes 目录 + result: IncludesScanner.ScanResult | t.CPtr = IncludesScanner.scan_includes(mb, includes_dir) + if result is None: + VLogger.error("扫描失败", "Phase1") + return 1 + + # 从扫描结果直接构建 SHA1 集合（不依赖 _sha1_map.txt 文件） + # Phase1 作为编译器自身，应从内存中的扫描结果获取 SHA1，避免外部文件依赖 + sha1_set: str = stdlib.malloc(StubMerger.MAX_INCLUDES_SHA1 * 17) + if sha1_set is None: + VLogger.error("sha1_set 分配失败", "Phase1") + return 1 + string.memset(sha1_set, 0, StubMerger.MAX_INCLUDES_SHA1 * 17) + set_count: int = 0 + p1_entry_size_init: t.CSizeT = IncludesScanner.FileEntry.__sizeof__() + for se_i in range(result.Count): + if set_count >= StubMerger.MAX_INCLUDES_SHA1: + break + se_addr: t.CUInt64T = t.CUInt64T(result.Entries) + se_i * p1_entry_size_init + se_ent: IncludesScanner.FileEntry | t.CPtr = (IncludesScanner.FileEntry | t.CPtr)(t.CVoid(se_addr, t.CPtr)) + if se_ent is None or se_ent.Sha1 is None: + continue + string.strcpy(sha1_set + set_count * 17, se_ent.Sha1) + set_count += 1 + # 先填充全局 SHA1 映射存储器（即使 set_count==0 也填充，确保 Phase2 能获取 includes 条目） + # 注意：PopulateSha1MapStore 会先 free 旧存储器再重新分配 + StubMergerStore.PopulateSha1MapStore(result) + # 写入 _sha1_map.txt（人类可读输出，程序内部不读取；机器分析使用 PopulateSha1MapStore 内存存储器） + StubMergerCollect.WriteIncludesSha1Map(mb, temp_dir, result, None, 0) + if set_count <= 0: + VLogger.warning("扫描结果无有效 SHA1，跳过 Phase1（项目可能无 includes 依赖）", "Phase1") + stdlib.free(sha1_set) + return 0 + # 模块 SHA1 映射已由 PopulateSha1MapStore 填充到 StubMerger 全局存储器， + # HandlesExprCall._lookup_module_sha1 直接查全局存储器，不再需要独立副本。 + + # 初始化 AST 表（只需一次） + ast._init_tables(mb) + + # FileEntry 结构体大小（Phase 1a 和 1b 共用） + entry_size: t.CSizeT = IncludesScanner.FileEntry.__sizeof__() + + # ============================================================ + # Phase 1a-pre: 扫描所有 includes 文件的 import 依赖 + # + # 只处理 Import/ImportFrom 语句，跳过 ClassDef/FunctionDef 等， + # 不调用 resolve_annotation_type，避免 list[...] 等不支持的语法触发 crash。 + # 生成 .deps.txt 供依赖图按需翻译使用。 + # ============================================================ + p1a_registered: int = 0 + p1a_skipped: int = 0 + p1a_failed: int = 0 + + for i in range(result.Count): + # 文件级 mark：import 依赖收集完（deps 写盘 + PopulateIncludesDeps + # 内部 stdlib 深拷贝）后立即回收本文件 AST——98 includes 全量 parse + # 的 AST 若驻留到 mark1 → 1GB 主池耗尽 OOM 根因之一 + scan_mark_a: t.CSizeT = mb.mark() + entry_addr_a: t.CUInt64T = t.CUInt64T(result.Entries) + i * entry_size + entry_a: IncludesScanner.FileEntry | t.CPtr = (IncludesScanner.FileEntry | t.CPtr)(t.CVoid(entry_addr_a, t.CPtr)) + if entry_a is None: + p1a_failed += 1 + continue + + sha1_a: str = entry_a.Sha1 + if sha1_a is None: + p1a_failed += 1 + continue + + if StubMergerCollect._is_in_sha1_set(sha1_a, sha1_set, set_count) == 0: + p1a_skipped += 1 + continue + + # 读取文件内容 + file_path_a: str = entry_a.Path + f_a: fileio.File | t.CPtr = fileio.File(file_path_a, fileio.MODE.R) + if f_a.closed: + p1a_failed += 1 + continue + + src_buf_a: bytes = stdlib.malloc(Phase1.SRC_BUF_SIZE) + if src_buf_a is None: + f_a.close() + p1a_failed += 1 + continue + + bytes_read_a: LONG = f_a.read_all(src_buf_a, Phase1.SRC_BUF_SIZE) + f_a.close() + if bytes_read_a <= 0: + stdlib.free(src_buf_a) + p1a_failed += 1 + continue + if bytes_read_a < Phase1.SRC_BUF_SIZE: + src_buf_a[bytes_read_a] = 0 + else: + src_buf_a[Phase1.SRC_BUF_SIZE - 1] = 0 + + # 解析 AST + lx_a: ast.Lexer | t.CPtr = ast.new_lexer(mb) + if lx_a is None: + stdlib.free(src_buf_a) + p1a_failed += 1 + continue + ast._lexer_init(lx_a, src_buf_a, mb) + tokens_a: ast.Token | t.CPtr = ast.tokenize(lx_a) + tree_a: ast.AST | t.CPtr = ast.parse_tokens(mb, tokens_a) + if tree_a is None: + stdlib.free(src_buf_a) + p1a_failed += 1 + continue + + # 创建 Translator，设置 _declare_only=2（只扫描 import，跳过 ClassDef） + tr_a: HandlesTranslator.Translator | t.CPtr = HandlesTranslator.Translator() + if tr_a is None: + stdlib.free(src_buf_a) + p1a_failed += 1 + continue + tr_a.ModuleSha1 = sha1_a + tr_a._declare_only = 2 + tr_a.CurrentPackage = HandlesImports.compute_package_from_relpath(mb, entry_a.RelPath) + HandlesType.set_current_file(file_path_a) + HandlesImports.set_current_file(file_path_a) + HandlesType.set_current_module_sha1(sha1_a) + HandlesType.clear_cdefine_constants() + HandlesType.clear_ctypedefs() + HandlesStruct.reset_visible_structs(mb, 0) + ret_a: int = tr_a.translate(tree_a) + if ret_a != 0: + p1a_failed += 1 + else: + p1a_registered += 1 + + # 容器模板按需纳入依赖：includes 使用 list/dict 泛型/构造/字面量 + # （如 base.vp 的 list[AST | CPtr]）时追加 _list/_dict 到 + # _imported_modules → 进 deps → _BuildReachableSha1Set 的可达集合 + # 自动包含（Phase1 与 Phase2 都受益），否则泛型模板未注册 → + # "泛型类 'list' 未找到模板"。对齐 App 层 _inject_auto_container_visibility。 + cl_box: t.CPtr = stdlib.malloc(4) + cd_box: t.CPtr = stdlib.malloc(4) + if cl_box is not None and cd_box is not None: + cl_box[0] = 0 + cd_box[0] = 0 + Phase1Scan._ScanTreeForContainers(tree_a, cl_box, cd_box) + if cl_box[0] != 0: + if tr_a._imported_modules is None or \ + HandlesImports.is_module_imported(tr_a._imported_modules, "_list") == 0: + tr_a._imported_modules = HandlesImports.add_imported_module( + mb, tr_a._imported_modules, "_list") + if cd_box[0] != 0: + if tr_a._imported_modules is None or \ + HandlesImports.is_module_imported(tr_a._imported_modules, "_dict") == 0: + tr_a._imported_modules = HandlesImports.add_imported_module( + mb, tr_a._imported_modules, "_dict") + if cl_box is not None: + stdlib.free(cl_box) + if cd_box is not None: + stdlib.free(cd_box) + + # 生成 .deps.txt（记录依赖模块名，供依赖图按需翻译使用） + # 同时填充内存依赖存储器（PopulateIncludesDeps），供 _BuildReachableSha1Set + # 直接从内存读取，不依赖 .deps.txt 过程文件 + td_len_a: t.CSizeT = string.strlen(temp_dir) + deps_path_a: str = StubMergerCollect._sliced_path(temp_dir, td_len_a, sha1_a, "deps.txt") + if deps_path_a is not None: + df_a: fileio.File | t.CPtr = fileio.File(deps_path_a, fileio.MODE.W) + if not df_a.closed: + if tr_a._imported_modules is not None: + dl_a: t.CSizeT = string.strlen(tr_a._imported_modules) + df_a.write(tr_a._imported_modules, dl_a) + df_a.close() + stdlib.free(deps_path_a) + # 填充内存依赖存储器（供 _BuildReachableSha1Set 使用，不读 .deps.txt） + if tr_a._imported_modules is not None: + StubMergerStore.PopulateIncludesDeps(sha1_a, tr_a._imported_modules) + + # 释放 Translator 的 C malloc 资源 + if tr_a._global_names is not None: + stdlib.free(tr_a._global_names) + if tr_a._nonlocal_names is not None: + stdlib.free(tr_a._nonlocal_names) + + stdlib.free(src_buf_a) + # 回收本文件 AST/lexer/token（deps 已写盘、依赖存储器已深拷贝） + mb.reset_to(scan_mark_a) + + if p1a_failed > 0: + fb_p1apre: str = VLogger.fmt_buf() + if fb_p1apre is not None: + viperlib.snprintf(fb_p1apre, 1024, "Phase 1a-pre 有 %d 个文件扫描失败", p1a_failed) + VLogger.error(fb_p1apre, "Phase1") + return 1 + + # ============================================================ + # 依赖图按需翻译：构建可达 SHA1 集合 + # + # 从 Config.SourceDir 的源文件开始，解析 import 语句，递归收集 + # 可达的 includes 文件 SHA1。Phase 1b 只翻译可达集合中的文件， + # 避免翻译不需要的 includes（如 Test 不依赖 llvmlite，则不翻译）。 + # 如果 Config.SourceDir 不可用（仅运行 Phase1），回退到 sha1_set。 + # ============================================================ + reachable_set: str = None + reachable_count: int = 0 + use_reachable: int = 0 + if Config.SourceDir is not None: + reachable_set = stdlib.malloc(StubMerger.MAX_INCLUDES_SHA1 * 17) + if reachable_set is not None: + string.memset(reachable_set, 0, StubMerger.MAX_INCLUDES_SHA1 * 17) + reachable_count = StubMergerBuild._BuildReachableSha1Set(mb, Config.SourceDir, temp_dir, reachable_set) + if reachable_count < 0: + VLogger.error("可达集合构建失败，终止", "Phase1") + return 1 + if reachable_count > 0: + use_reachable = 1 + # 用可达集合重新生成 _sha1_map.txt（按图求索的最终产物） + # Phase B+ 只遍历这些条目，避免编译不需要的 includes（如 asm.vp） + StubMergerCollect.WriteIncludesSha1Map(mb, temp_dir, result, reachable_set, reachable_count) + # 重新加载 sha1_set，使后续 Phase 1a-pre/1a/1b 的过滤也使用可达集合 + string.memset(sha1_set, 0, StubMerger.MAX_INCLUDES_SHA1 * 17) + set_count = StubMergerCollect._load_includes_sha1_set(mb, temp_dir, sha1_set) + else: + VLogger.warning("可达集合为空，回退全量扫描", "Phase1") + stdlib.free(reachable_set) + reachable_set = None + + # ============================================================ + # Phase 1a: 注册可达 includes 文件的 struct/enum/union + # + # 解决字母序依赖问题：按字母顺序翻译时，后面的 struct 未注册 + # 前面就需要用。Phase 1a 先注册所有 struct/enum/union 到全局表， + # Phase 1b 全量翻译时 struct 已注册，走 existing 路径只翻译方法体。 + # 只处理可达文件，避免翻译不需要的 includes（如 Test 不依赖 ast 模块）。 + # + # 拓扑排序：预扫描所有文件收集类继承关系，按依赖顺序处理文件， + # 确保父类所在文件先于子类所在文件处理（如 base.vp 先于 astaux.vp）。 + # ============================================================ + p1a_reg: int = 0 + p1a_skp: int = 0 + p1a_fl: int = 0 + + # 预扫描类继承关系，构建拓扑顺序（用扁平字节数组，避免结构体 str 指针 NULL 崩溃） + class_names_buf: bytes = mb.alloc(Phase1.MAX_CLASSES * Phase1.CLASS_NAME_LEN) + parent_names_buf: bytes = mb.alloc(Phase1.MAX_CLASSES * Phase1.CLASS_NAME_LEN) + def_sha1s_buf: bytes = mb.alloc(Phase1.MAX_CLASSES * Phase1.SHA1_LEN) + topo_order: t.CPtr = mb.alloc(4 * Phase1.MAX_FILE_DEPS) + topo_count_box: t.CPtr = mb.alloc(4) + generic_file_indices: t.CPtr = mb.alloc(4 * Phase1.MAX_FILE_DEPS) + generic_file_count_box: t.CPtr = mb.alloc(4) + if generic_file_indices is not None: + string.memset(generic_file_indices, 0, 4 * Phase1.MAX_FILE_DEPS) + if generic_file_count_box is not None: + string.memset(generic_file_count_box, 0, 4) + topo_count: int = 0 + + if class_names_buf is not None and parent_names_buf is not None and def_sha1s_buf is not None and topo_order is not None and topo_count_box is not None: + string.memset(class_names_buf, 0, Phase1.MAX_CLASSES * Phase1.CLASS_NAME_LEN) + string.memset(parent_names_buf, 0, Phase1.MAX_CLASSES * Phase1.CLASS_NAME_LEN) + string.memset(def_sha1s_buf, 0, Phase1.MAX_CLASSES * Phase1.SHA1_LEN) + string.memset(topo_order, 0, 4 * Phase1.MAX_FILE_DEPS) + string.memset(topo_count_box, 0, 4) + if generic_file_indices is not None: + string.memset(generic_file_indices, 0, 4 * Phase1.MAX_FILE_DEPS) + if generic_file_count_box is not None: + string.memset(generic_file_count_box, 0, 4) + + class_count_scanned: int = Phase1Scan._ScanClassInheritance( + mb, result, reachable_set, reachable_count, + use_reachable, sha1_set, set_count, + class_names_buf, parent_names_buf, def_sha1s_buf, + generic_file_indices, generic_file_count_box) + + if class_count_scanned > 0: + td_len_ts_call: t.CSizeT = string.strlen(temp_dir) + ret_topo: int = Phase1Topo._TopoSortFiles( + mb, result, reachable_set, reachable_count, + use_reachable, sha1_set, set_count, + class_names_buf, parent_names_buf, def_sha1s_buf, class_count_scanned, + topo_order, topo_count_box, + temp_dir, td_len_ts_call) + if ret_topo == 0: + tc_ptr: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(topo_count_box, t.CPtr)) + if tc_ptr is not None: + topo_count = tc_ptr[0] + fb_topo: str = VLogger.fmt_buf() + if fb_topo is not None: + viperlib.snprintf(fb_topo, 1024, + "拓扑排序: %d 个类, %d 个文件按依赖顺序处理", + class_count_scanned, topo_count) + VLogger.info(fb_topo, "Phase1") + else: + VLogger.error("拓扑排序失败（类继承依赖无法解析），终止编译", "Phase1") + return 1 + else: + VLogger.warning("类继承预扫描无结果，回退到字母序", "Phase1") + topo_count = 0 + + # ============================================================ + # Phase 1a-pre-generic-all: mark1 前全量预注册 includes 泛型模板 + # + # 1a-pre-generic 只覆盖"可达"泛型文件；非可达泛型文件（如 FuncTest + # 场景的 linkedlist/llvmlite/_list）的模板要等 Phase2 Phase A-pre-inc + # （mark2 之后）才注册 → reset 回收表/节点 → _find_generic_template + # strcmp 悬空崩溃。此处对全部 includes 轻量预注册，使模板表与节点 + # 都落在 mark1 之前（mark2 > mark1，reset_to(mark2) 不回收）。 + # ============================================================ + Phase1Scan._PreRegisterAllGenericTemplates(mb, result) + + # ============================================================ + # Phase 1a: 注册 struct/enum/union（mark1 前，全量 includes） + # + # OOM 治理：1a 对全部 includes parse 完整 AST（主 pool 分配），若 + # 驻留到 mark1（循环后）会累积 98 文件 × 每文件 AST 树 → 1GB 主池 + # 耗尽（NewAstTest alloc 失败 0xBE 根因）。现改为： + # 1) StdlibAlloc=1：1a 注册的跨文件持久数据（StructTable 表、 + # Fields/Name/ModuleSha1、struct_ty 深拷贝、枚举/Renum/泛型 + # 模板等）全部落在 perm_pool（永不 reset），不占主 pool； + # 2) 每文件翻译完 reset_to(文件级 mark)：仅回收该文件的 AST/ + # lexer/token/IR 等瞬时数据，注册数据不受影响。 + # mark1 仍在 1a 结束后打，1b 每文件 reset_to(mark1) 语义不变。 + # ============================================================ + HandlesStruct.set_stdlib_alloc(1) + + # 遍历文件：优先使用拓扑顺序，回退到字母序 + iter_count: int = topo_count if topo_count > 0 else result.Count + for i in range(iter_count): + # 每文件级 mark：翻译完回收该文件 AST（注册数据在 perm_pool） + p1a_mark: t.CSizeT = mb.mark() + # 获取文件索引：拓扑顺序或字母序 + file_idx: int = i + if topo_count > 0: + od_addr: t.CUInt64T = t.CUInt64T(topo_order) + i * 4 + od_ptr: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(od_addr, t.CPtr)) + if od_ptr is not None: + file_idx = od_ptr[0] + + entry_addr_r: t.CUInt64T = t.CUInt64T(result.Entries) + file_idx * entry_size + entry_r: IncludesScanner.FileEntry | t.CPtr = (IncludesScanner.FileEntry | t.CPtr)(t.CVoid(entry_addr_r, t.CPtr)) + if entry_r is None: + p1a_fl += 1 + continue + + sha1_r: str = entry_r.Sha1 + if sha1_r is None: + p1a_fl += 1 + continue + + # 按需翻译过滤（拓扑排序已过滤，但字母序回退需要检查） + if topo_count == 0: + if use_reachable != 0: + if StubMergerCollect._is_in_sha1_set(sha1_r, reachable_set, reachable_count) == 0: + p1a_skp += 1 + continue + else: + if StubMergerCollect._is_in_sha1_set(sha1_r, sha1_set, set_count) == 0: + p1a_skp += 1 + continue + + # 读取文件内容 + file_path_r: str = entry_r.Path + f_r: fileio.File | t.CPtr = fileio.File(file_path_r, fileio.MODE.R) + if f_r.closed: + p1a_fl += 1 + continue + + src_buf_r: bytes = stdlib.malloc(Phase1.SRC_BUF_SIZE) + if src_buf_r is None: + f_r.close() + p1a_fl += 1 + continue + + bytes_read_r: LONG = f_r.read_all(src_buf_r, Phase1.SRC_BUF_SIZE) + f_r.close() + if bytes_read_r <= 0: + stdlib.free(src_buf_r) + p1a_fl += 1 + continue + if bytes_read_r < Phase1.SRC_BUF_SIZE: + src_buf_r[bytes_read_r] = 0 + else: + src_buf_r[Phase1.SRC_BUF_SIZE - 1] = 0 + + # 解析 AST + lx_r: ast.Lexer | t.CPtr = ast.new_lexer(mb) + if lx_r is None: + stdlib.free(src_buf_r) + p1a_fl += 1 + continue + ast._lexer_init(lx_r, src_buf_r, mb) + tokens_r: ast.Token | t.CPtr = ast.tokenize(lx_r) + tree_r: ast.AST | t.CPtr = ast.parse_tokens(mb, tokens_r) + if tree_r is None: + stdlib.free(src_buf_r) + p1a_fl += 1 + continue + + # 创建 Translator，设置 _declare_only=1（注册 struct/enum/union） + tr_r: HandlesTranslator.Translator | t.CPtr = HandlesTranslator.Translator() + if tr_r is None: + stdlib.free(src_buf_r) + p1a_fl += 1 + continue + tr_r.ModuleSha1 = sha1_r + tr_r._declare_only = 1 + tr_r.CurrentPackage = HandlesImports.compute_package_from_relpath(mb, entry_r.RelPath) + HandlesType.set_current_file(file_path_r) + HandlesImports.set_current_file(file_path_r) + HandlesType.set_current_module_sha1(sha1_r) + HandlesType.clear_cdefine_constants() + HandlesType.clear_ctypedefs() + HandlesStruct.reset_visible_structs(mb, 0) + ret_r: int = tr_r.translate(tree_r) + if ret_r != 0: + p1a_fl += 1 + else: + p1a_reg += 1 + + # 释放 Translator 的 C malloc 资源 + if tr_r._global_names is not None: + stdlib.free(tr_r._global_names) + if tr_r._nonlocal_names is not None: + stdlib.free(tr_r._nonlocal_names) + stdlib.free(src_buf_r) + + # 回收本文件 AST/lexer/token/IR 等瞬时数据（注册数据已在 perm_pool， + # reset 不影响；IncludesScanner result 等在 p1a_mark 前分配同样保留） + mb.reset_to(p1a_mark) + + if p1a_fl > 0: + fb_p1ar: str = VLogger.fmt_buf() + if fb_p1ar is not None: + viperlib.snprintf(fb_p1ar, 1024, "Phase 1a 有 %d 个文件 struct 注册失败", p1a_fl) + VLogger.error(fb_p1ar, "Phase1") + return 1 + + # 释放拓扑排序资源（topo_order/topo_count_box 保留到 Phase 1b 结束， + # 1b 按拓扑序翻译需要它们；class/def 数组 1b 不再使用） + if class_names_buf is not None: + mb.free(class_names_buf) + if parent_names_buf is not None: + mb.free(parent_names_buf) + if def_sha1s_buf is not None: + mb.free(def_sha1s_buf) + if generic_file_indices is not None: + mb.free(generic_file_indices) + if generic_file_count_box is not None: + mb.free(generic_file_count_box) + + # ============================================================ + # Phase 1b: 全量翻译（struct 已注册，走 existing 路径翻译方法体） + # ============================================================ + # OOM 回收：1a-pre/泛型预注册/1a 全部结束后打 mark1， + # 所有跨文件持久表（_struct_table/_enum_table/_generic_class_nodes/ + # _renum_*/关键字表等）都落在 mark1 之前。 + # 1b 每文件翻译完后 reset_to(mark1) 回收该文件的 AST/IR/Handler/dump 临时， + # 避免 90 文件累积导致 1GB 池 OOM。 + mark1: t.CSizeT = mb.mark() + + # 1b 阶段新注册的 struct/enum 条目改 stdlib 分配（跨文件永久存活，不被 reset 回收） + HandlesStruct.set_stdlib_alloc(1) + + # ============================================================ + # Phase 1b-pre: 预生成所有待翻译文件的 .vpi stub + # + # 依赖模块的 .vpi 必须在翻译前可用：1b 主循环按扫描顺序翻译（非拓扑）， + # 若 argparse 先于 hashtable 翻译，hashtable.vpi 尚未生成， + # LoadDepVpiIntoModule 读不到 → declare 未注入 → 跨模块调用 + # 返回类型推断错误（如 HashTable.__getitem__ 被推断为 Ptr(HashTable) + # 而 define 返回 i8*）→ 运行时返回值截断崩溃。 + # ============================================================ + p1bpre_failed: int = 0 + dir_len_pv: t.CSizeT = string.strlen(temp_dir) + for i_pv in range(result.Count): + entry_addr_pv: t.CUInt64T = t.CUInt64T(result.Entries) + i_pv * entry_size + entry_pv: IncludesScanner.FileEntry | t.CPtr = (IncludesScanner.FileEntry | t.CPtr)(t.CVoid(entry_addr_pv, t.CPtr)) + if entry_pv is None or entry_pv.Sha1 is None: + p1bpre_failed += 1 + continue + sha1_pv: str = entry_pv.Sha1 + in_set_pv: int = 0 + if use_reachable != 0: + in_set_pv = StubMergerCollect._is_in_sha1_set(sha1_pv, reachable_set, reachable_count) + else: + in_set_pv = StubMergerCollect._is_in_sha1_set(sha1_pv, sha1_set, set_count) + if in_set_pv == 0: + continue + # vpi 已存在则跳过 + vpi_path_pv: str = StubMergerCollect._sliced_path(temp_dir, dir_len_pv, sha1_pv, "vpi") + if vpi_path_pv is None: + p1bpre_failed += 1 + continue + vf_pv: fileio.File | t.CPtr = fileio.File(vpi_path_pv, fileio.MODE.R) + if not vf_pv.closed: + vf_pv.close() + stdlib.free(vpi_path_pv) + continue + stdlib.free(vpi_path_pv) + # 读取文件、解析 AST、生成 vpi + file_path_pv: str = entry_pv.Path + f_pv: fileio.File | t.CPtr = fileio.File(file_path_pv, fileio.MODE.R) + if f_pv.closed: + p1bpre_failed += 1 + continue + src_buf_pv: bytes = stdlib.malloc(Phase1.SRC_BUF_SIZE) + if src_buf_pv is None: + f_pv.close() + p1bpre_failed += 1 + continue + bytes_read_pv: LONG = f_pv.read_all(src_buf_pv, Phase1.SRC_BUF_SIZE) + f_pv.close() + if bytes_read_pv <= 0: + stdlib.free(src_buf_pv) + p1bpre_failed += 1 + continue + if bytes_read_pv < Phase1.SRC_BUF_SIZE: + src_buf_pv[bytes_read_pv] = 0 + else: + src_buf_pv[Phase1.SRC_BUF_SIZE - 1] = 0 + lx_pv: ast.Lexer | t.CPtr = ast.new_lexer(mb) + if lx_pv is None: + stdlib.free(src_buf_pv) + p1bpre_failed += 1 + continue + ast._lexer_init(lx_pv, src_buf_pv, mb) + tokens_pv: ast.Token | t.CPtr = ast.tokenize(lx_pv) + tree_pv: ast.AST | t.CPtr = ast.parse_tokens(mb, tokens_pv) + if tree_pv is None: + stdlib.free(src_buf_pv) + p1bpre_failed += 1 + continue + pyi_buf_pv: bytes = stdlib.malloc(Phase1.PYI_BUF_SIZE) + if pyi_buf_pv is not None: + pyi_pos_pv: t.CSizeT = StubConverter._GeneratePyiFromAst(mb, tree_pv, entry_pv.RelPath, pyi_buf_pv, Phase1.PYI_BUF_SIZE) + if pyi_pos_pv > 0: + pyi_path_pv: str = StubMergerCollect._sliced_path(temp_dir, dir_len_pv, sha1_pv, "vpi") + if pyi_path_pv is not None: + pf_pv: fileio.File | t.CPtr = fileio.File(pyi_path_pv, fileio.MODE.W) + if not pf_pv.closed: + pf_pv.write(pyi_buf_pv, pyi_pos_pv) + pf_pv.close() + stdlib.free(pyi_path_pv) + stdlib.free(pyi_buf_pv) + else: + p1bpre_failed += 1 + stdlib.free(src_buf_pv) + # 回收本文件 AST（vpi 已写盘，AST 不再需要；mark1 后无其他持久数据） + mb.reset_to(mark1) + + if p1bpre_failed > 0: + fb_p1bpre: str = VLogger.fmt_buf() + if fb_p1bpre is not None: + viperlib.snprintf(fb_p1bpre, 1024, "Phase 1b-pre 有 %d 个文件 vpi 预生成失败", p1bpre_failed) + VLogger.error(fb_p1bpre, "Phase1") + return 1 + + # 遍历每个文件（拓扑序优先：父类所在文件先翻译，确保跨模块 vtable + # 生成时依赖模块的函数 TypeStr 已注册；topo_count==0 回退扫描顺序） + translated: int = 0 + skipped: int = 0 + failed: int = 0 + + loop_n_b: int = result.Count + if topo_count > 0: + loop_n_b = topo_count + for i in range(loop_n_b): + # 获取 entry（拓扑序：file_idx 从 topo_order 取，否则用扫描索引） + file_idx_b: int = i + if topo_count > 0: + od_addr_b: t.CUInt64T = t.CUInt64T(topo_order) + t.CSizeT(i) * 4 + od_ptr_b: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(od_addr_b, t.CPtr)) + if od_ptr_b is not None: + file_idx_b = od_ptr_b[0] + entry_addr: t.CUInt64T = t.CUInt64T(result.Entries) + t.CSizeT(file_idx_b) * entry_size + entry: IncludesScanner.FileEntry | t.CPtr = (IncludesScanner.FileEntry | t.CPtr)(t.CVoid(entry_addr, t.CPtr)) + if entry is None: + failed += 1 + continue + + sha1: str = entry.Sha1 + if sha1 is None: + failed += 1 + continue + + # 检查 SHA1 是否在翻译集合中（拓扑序已过滤可达；仅回退扫描顺序时检查） + in_set: int = 1 + if topo_count == 0: + in_set = 0 + if use_reachable != 0: + in_set = StubMergerCollect._is_in_sha1_set(sha1, reachable_set, reachable_count) + else: + in_set = StubMergerCollect._is_in_sha1_set(sha1, sha1_set, set_count) + if in_set == 0: + skipped += 1 + continue + + # 构造 stub 路径: {temp_dir}/{sha1}.stub.ll + dir_len: t.CSizeT = string.strlen(temp_dir) + sha1_len: t.CSizeT = string.strlen(sha1) + stub_path: str = StubMergerCollect._sliced_path(temp_dir, dir_len, sha1, "stub.ll") + if stub_path is None: + failed += 1 + continue + + # 检查 stub 是否已存在（按需翻译：跳过已存在的） + sf: fileio.File | t.CPtr = fileio.File(stub_path, fileio.MODE.R) + if not sf.closed: + sf.close() + # 检查 text.ll 是否也存在（虚表扫描需要 text.ll） + text_path: str = StubMergerCollect._sliced_path(temp_dir, dir_len, sha1, "text.ll") + if text_path is not None: + tf: fileio.File | t.CPtr = fileio.File(text_path, fileio.MODE.R) + if not tf.closed: + tf.close() + stdlib.free(text_path) + stdlib.free(stub_path) + continue + stdlib.free(text_path) + # text.ll 不存在，需要重新翻译 + rp: str = entry.RelPath + fb: str = VLogger.fmt_buf() + if fb is not None: + viperlib.snprintf(fb, 1024, "text.ll 不存在，重新翻译: %s(sha1=%s)", rp, sha1) + VLogger.info(fb, "Phase1") + else: + # stub 不存在，需要翻译 + rp: str = entry.RelPath + fb: str = VLogger.fmt_buf() + if fb is not None: + viperlib.snprintf(fb, 1024, "翻译: %s(sha1=%s)", rp, sha1) + VLogger.info(fb, "Phase1") + + stdlib.free(stub_path) + + # 读取文件内容 + file_path: str = entry.Path + f: fileio.File | t.CPtr = fileio.File(file_path, fileio.MODE.R) + if f is None: + fb: str = VLogger.fmt_buf() + if fb is not None: + viperlib.snprintf(fb, 1024, "无法打开(None): %s", file_path) + VLogger.error(fb, "Phase1") + failed += 1 + continue + if f.closed: + fb: str = VLogger.fmt_buf() + if fb is not None: + viperlib.snprintf(fb, 1024, "无法打开: %s", file_path) + VLogger.error(fb, "Phase1") + failed += 1 + continue + + src_buf: bytes = stdlib.malloc(Phase1.SRC_BUF_SIZE) + if src_buf is None: + f.close() + failed += 1 + continue + + bytes_read: LONG = f.read_all(src_buf, Phase1.SRC_BUF_SIZE) + f.close() + if bytes_read <= 0: + fb: str = VLogger.fmt_buf() + if fb is not None: + viperlib.snprintf(fb, 1024, "读取失败: %s", file_path) + VLogger.error(fb, "Phase1") + stdlib.free(src_buf) + failed += 1 + continue + if bytes_read < Phase1.SRC_BUF_SIZE: + src_buf[bytes_read] = 0 + else: + src_buf[Phase1.SRC_BUF_SIZE - 1] = 0 + + # 解析 AST + lx: ast.Lexer | t.CPtr = ast.new_lexer(mb) + if lx is None: + stdlib.free(src_buf) + failed += 1 + continue + ast._lexer_init(lx, src_buf, mb) + tokens: ast.Token | t.CPtr = ast.tokenize(lx) + tree: ast.AST | t.CPtr = ast.parse_tokens(mb, tokens) + if tree is None: + fb: str = VLogger.fmt_buf() + if fb is not None: + viperlib.snprintf(fb, 1024, "AST 解析失败: %s", file_path) + VLogger.error(fb, "Phase1") + stdlib.free(src_buf) + failed += 1 + continue + + # 生成 .vpi 存根文件（直接遍历 AST，不依赖 PythonToStubConverter） + pyi_buf: bytes = stdlib.malloc(Phase1.PYI_BUF_SIZE) + if pyi_buf is not None: + pyi_pos: t.CSizeT = StubConverter._GeneratePyiFromAst(mb, tree, entry.RelPath, pyi_buf, Phase1.PYI_BUF_SIZE) + if pyi_pos > 0: + pyi_path: str = StubMergerCollect._sliced_path(temp_dir, dir_len, sha1, "vpi") + if pyi_path is not None: + pf: fileio.File | t.CPtr = fileio.File(pyi_path, fileio.MODE.W) + if not pf.closed: + pf.write(pyi_buf, pyi_pos) + pf.close() + stdlib.free(pyi_path) + stdlib.free(pyi_buf) + + # 翻译 AST → LLVM IR + tr: HandlesTranslator.Translator | t.CPtr = HandlesTranslator.Translator() + if tr is None: + stdlib.free(src_buf) + failed += 1 + continue + tr.ModuleSha1 = sha1 + tr.CurrentPackage = HandlesImports.compute_package_from_relpath(mb, entry.RelPath) + HandlesType.set_current_file(file_path) + HandlesImports.set_current_file(file_path) + HandlesType.set_current_module_sha1(sha1) + HandlesType.clear_cdefine_constants() + HandlesType.clear_ctypedefs() + HandlesStruct.reset_visible_structs(mb, 0) + + # 翻译前哨兵：AST 解析（lexer/tokenize/parse_tokens）同样在 pool 分配， + # 若解析器越界写破坏了存活块 header，此处立即定位到当前文件 + # （崩溃点漂移排查：把"翻译过程中随机崩"变成确定性定位） + pre_rc: t.CInt = mb.check_alive_blocks(mark1) + if pre_rc != 0: + fb: str = VLogger.fmt_buf() + if fb is not None: + viperlib.snprintf(fb, 1024, + "内存损坏(翻译前存活块, rc=%d): %s", pre_rc, file_path) + VLogger.error(fb, "Phase1") + stdlib.exit(1) + + ret: int = tr.translate(tree) + # 诊断：translate 正常返回（若挂起在 translate 内部，此行不会打印， + # 用于区分"translate 内部挂起"与"后续 dump 挂起"） + if ret == 0: + fb_done: str = VLogger.fmt_buf() + if fb_done is not None: + viperlib.snprintf(fb_done, 1024, "翻译完成(translate返回): %s", file_path) + VLogger.info(fb_done, "Phase1") + if ret != 0: + fb: str = VLogger.fmt_buf() + if fb is not None: + viperlib.snprintf(fb, 1024, "翻译失败: %s", file_path) + VLogger.error(fb, "Phase1") + stdlib.free(src_buf) + if tr._global_names is not None: + stdlib.free(tr._global_names) + if tr._nonlocal_names is not None: + stdlib.free(tr._nonlocal_names) + failed += 1 + continue + + # dump stub IR(declarations only) + PHASE1_IR_SIZE: t.CSizeT = 1048576 + stub_buf: bytes = stdlib.malloc(PHASE1_IR_SIZE) + if stub_buf is None: + stdlib.free(src_buf) + if tr._global_names is not None: + stdlib.free(tr._global_names) + if tr._nonlocal_names is not None: + stdlib.free(tr._nonlocal_names) + failed += 1 + continue + tr.dump_ir(stub_buf, PHASE1_IR_SIZE, llvmlite.OUTPUT_STUB) + stub_len: t.CSizeT = string.strlen(stub_buf) + + # save stub.ll + dir_len_p1: t.CSizeT = string.strlen(temp_dir) + stub_path_p1: str = StubMergerCollect._sliced_path(temp_dir, dir_len_p1, sha1, "stub.ll") + if stub_path_p1 is not None: + sf_p1: fileio.File | t.CPtr = fileio.File(stub_path_p1, fileio.MODE.W) + if not sf_p1.closed: + sf_p1.write(stub_buf, stub_len) + sf_p1.close() + stdlib.free(stub_path_p1) + stdlib.free(stub_buf) + + # dump text IR(definitions only) + text_buf: bytes = stdlib.malloc(PHASE1_IR_SIZE) + if text_buf is None: + stdlib.free(src_buf) + if tr._global_names is not None: + stdlib.free(tr._global_names) + if tr._nonlocal_names is not None: + stdlib.free(tr._nonlocal_names) + failed += 1 + continue + tr.dump_ir(text_buf, PHASE1_IR_SIZE, llvmlite.OUTPUT_TEXT) + text_len: t.CSizeT = string.strlen(text_buf) + + # save text.ll + text_path_p1: str = StubMergerCollect._sliced_path(temp_dir, dir_len_p1, sha1, "text.ll") + if text_path_p1 is not None: + tf_p1: fileio.File | t.CPtr = fileio.File(text_path_p1, fileio.MODE.W) + if not tf_p1.closed: + tf_p1.write(text_buf, text_len) + tf_p1.close() + stdlib.free(text_path_p1) + stdlib.free(text_buf) + + # save dependencies(_imported_modules) for Phase B + deps_path_p1: str = StubMergerCollect._sliced_path(temp_dir, dir_len_p1, sha1, "deps.txt") + if deps_path_p1 is not None: + df_p1: fileio.File | t.CPtr = fileio.File(deps_path_p1, fileio.MODE.W) + if not df_p1.closed: + if tr._imported_modules is not None: + dl: t.CSizeT = string.strlen(tr._imported_modules) + df_p1.write(tr._imported_modules, dl) + df_p1.close() + stdlib.free(deps_path_p1) + + # 释放 Translator 的 C malloc 资源 + if tr._global_names is not None: + stdlib.free(tr._global_names) + if tr._nonlocal_names is not None: + stdlib.free(tr._nonlocal_names) + + # 所有 dump_ir 完成，src_buf 不再需要 + stdlib.free(src_buf) + + translated += 1 + + # OOM 回收：回收本文件翻译产生的瞬时数据（AST/tokens/lexer/IR/Handler/dump 临时） + # 持久表（struct/enum/REnum/spec 缓存/全局函数名等）已迁出 reset 区间（stdlib）， + # reset_to(mark1) 只回收 mark1 之后分配、未被单独 free 的块。 + # + # 内存哨兵（崩溃点漂移排查）：reset 前检查存活块 header（抓越界写）， + # reset 后检查 free list 完整性（抓链表损坏）。任一损坏立即终止， + # 把"随机崩溃漂移"变成"确定性定位到当前文件"。 + # rc 语义（check_alive_blocks）：1=块地址低于 usable，2=高于 usable+size， + # 3=order 为负，4=order 超 max_order，5=块未对齐到 order 大小， + # 6=块尾 canary 损坏（越界写穿数据区——事发地定位） + # rc 语义（self_check）：1/2=越界，3=对齐错，4=free list 成环/超步数 + chk_rc: t.CInt = mb.check_alive_blocks(mark1) + if chk_rc != 0: + fb: str = VLogger.fmt_buf() + if fb is not None: + viperlib.snprintf(fb, 1024, + "内存损坏(存活块 header, rc=%d): %s", chk_rc, file_path) + VLogger.error(fb, "Phase1") + stdlib.exit(1) + # A/B 实验结论（已确认）：禁用 reset_to 后 stdarg 仍崩 → 越界写是根因， + # 与回收/UAF 无关。恢复 reset_to，改用块尾 canary（check_alive_blocks 校验） + # 定位越界写事发地到具体文件。 + mb.reset_to(mark1) + sc_rc: t.CInt = mb.self_check() + if sc_rc != 0: + fb: str = VLogger.fmt_buf() + if fb is not None: + viperlib.snprintf(fb, 1024, + "内存损坏(free list, rc=%d): %s", sc_rc, file_path) + VLogger.error(fb, "Phase1") + stdlib.exit(1) + + # 释放 Phase 1b 使用的拓扑排序资源 + if topo_order is not None: + mb.free(topo_order) + if topo_count_box is not None: + mb.free(topo_count_box) + + fb: str = VLogger.fmt_buf() + if fb is not None: + viperlib.snprintf(fb, 1024, "完成: 翻译=%d 跳过=%d 失败=%d", translated, skipped, failed) + VLogger.info(fb, "Phase1") + + if failed > 0: + fb_fail: str = VLogger.fmt_buf() + if fb_fail is not None: + viperlib.snprintf(fb_fail, 1024, "Phase1 有 %d 个文件翻译失败", failed) + VLogger.error(fb_fail, "Phase1") + return 1 + + # 释放可达 SHA1 集合（如果分配了） + if reachable_set is not None: + stdlib.free(reachable_set) + + return 0 diff --git a/App/lib/core/Phase1Scan.vp b/App/lib/core/Phase1Scan.vp new file mode 100644 index 0000000..7e728a9 --- /dev/null +++ b/App/lib/core/Phase1Scan.vp @@ -0,0 +1,505 @@ +import t, c +from stdint import * +import string +import stdlib +import memhub +import viperlib +import w32.fileio as fileio +import w32.win32file +import w32.win32base +import ast +import llvmlite +import lib.core.VLogger as VLogger +import lib.core.Handles.HandlesTranslator as HandlesTranslator +import lib.core.Handles.HandlesStruct as HandlesStruct +import lib.core.Handles.HandlesType as HandlesType +import lib.core.Handles.HandlesExprCall as HandlesExprCall +import lib.core.Handles.HandlesImports as HandlesImports +import lib.core.Handles.HandlesClassDef as HandlesClassDef +import lib.core.IncludesScanner as IncludesScanner +import lib.core.StubMerger as StubMerger +import lib.core.Gen.Generator as GenGenerator +import lib.core.CompilerFactory as CompilerFactory +import lib.core.BuildPipeline as BuildPipeline +import lib.Projectrans.Config as Config +import lib.StubGen.Converter as StubConverter +import lib.core.StubMergerBuild as StubMergerBuild +import lib.core.StubMergerCollect as StubMergerCollect +import lib.core.StubMergerIR as StubMergerIR +import lib.core.StubMergerStore as StubMergerStore +import lib.core.Handles.ExprCallCtor as ExprCallCtor + + +import lib.core.Phase1Topo as Phase1Topo +import lib.core.Phase1Main as Phase1Main +import lib.core.Phase1 as Phase1 + +# ============================================================ +# Phase1Scan - Phase1 拆分（阶段 E）：Phase1Scan +# +# 直接移动模式：函数迁至此模块，原 Phase1.vp 仅保留常量/_mbuddy； +# 调用方经本模块前缀访问。全局符号经 Phase1._xxx 跨模块访问。 +# ============================================================ + +# 拓扑排序使用扁平字节数组存储类信息和文件依赖， +# 避免结构体中的 str 指针字段在 mb.alloc 零填充时为 NULL 导致崩溃。 +# 类名/父类名存储在 class_names_buf/parent_names_buf（每个 Phase1.CLASS_NAME_LEN 字节）， +# 定义 SHA1 存储在 def_sha1s_buf（每个 Phase1.SHA1_LEN 字节）。 + + +# ============================================================ +# _ScanClassInheritance - 预扫描 includes 文件，收集类继承关系 +# +# 遍历所有可达 includes 文件，解析 AST 收集 ClassDef 的类名和父类名。 +# 结果存入扁平字节数组，返回收集到的类数量。 +# +# Args: +# mb: 内存池 +# result: IncludesScanner 扫描结果 +# reachable_set: 可达 SHA1 集合 +# reachable_count: 可达 SHA1 数量 +# use_reachable: 是否使用可达集合过滤 +# sha1_set: SHA1 集合（use_reachable=0 时使用） +# set_count: SHA1 集合数量 +# class_names_buf: 类名扁平数组（每个 Phase1.CLASS_NAME_LEN 字节，连续内存） +# parent_names_buf: 父类名扁平数组（每个 Phase1.CLASS_NAME_LEN 字节，无父类填 '\0'） +# def_sha1s_buf: 定义 SHA1 扁平数组（每个 Phase1.SHA1_LEN 字节） +# +# Returns: +# 收集到的类数量，-1 失败 +# ============================================================ +def _ScanClassInheritance(mb: memhub.MemBuddy | t.CPtr, + result: IncludesScanner.ScanResult | t.CPtr, + reachable_set: str, reachable_count: int, + use_reachable: int, + sha1_set: str, set_count: int, + class_names_buf: bytes, + parent_names_buf: bytes, + def_sha1s_buf: bytes, + generic_file_indices: t.CPtr, + generic_file_count_box: t.CPtr) -> int: + """预扫描 includes 文件，收集类继承关系""" + if result is None or class_names_buf is None or parent_names_buf is None or def_sha1s_buf is None: + return -1 + + entry_size: t.CSizeT = IncludesScanner.FileEntry.__sizeof__() + class_count: int = 0 + + for i in range(result.Count): + # 文件级 mark：收集完类继承信息后立即回收本文件 AST（可达 includes + # 全量 parse 的 AST 若驻留到 mark1 → 1GB 主池耗尽 OOM 根因； + # class_names_buf/parent_names_buf/def_sha1s_buf 在调用前分配， + # 位于本 mark 之前，reset 不影响；generic_file_indices 同理） + scan_mark: t.CSizeT = mb.mark() + entry_addr: t.CUInt64T = t.CUInt64T(result.Entries) + i * entry_size + entry: IncludesScanner.FileEntry | t.CPtr = (IncludesScanner.FileEntry | t.CPtr)(t.CVoid(entry_addr, t.CPtr)) + if entry is None or entry.Sha1 is None: + continue + + sha1_e: str = entry.Sha1 + file_has_generic: int = 0 + + # 按需翻译过滤 + in_set: int = 0 + if use_reachable != 0: + in_set = StubMergerCollect._is_in_sha1_set(sha1_e, reachable_set, reachable_count) + else: + in_set = StubMergerCollect._is_in_sha1_set(sha1_e, sha1_set, set_count) + if in_set == 0: + continue + + # 读取文件 + file_path: str = entry.Path + f: fileio.File | t.CPtr = fileio.File(file_path, fileio.MODE.R) + if f.closed: + continue + + src_buf: bytes = stdlib.malloc(Phase1.SRC_BUF_SIZE) + if src_buf is None: + f.close() + continue + + bytes_read: LONG = f.read_all(src_buf, Phase1.SRC_BUF_SIZE) + f.close() + if bytes_read <= 0: + stdlib.free(src_buf) + continue + if bytes_read < Phase1.SRC_BUF_SIZE: + src_buf[bytes_read] = 0 + else: + src_buf[Phase1.SRC_BUF_SIZE - 1] = 0 + + # 解析 AST + lx: ast.Lexer | t.CPtr = ast.new_lexer(mb) + if lx is None: + stdlib.free(src_buf) + continue + ast._lexer_init(lx, src_buf, mb) + tokens: ast.Token | t.CPtr = ast.tokenize(lx) + tree: ast.AST | t.CPtr = ast.parse_tokens(mb, tokens) + if tree is None: + stdlib.free(src_buf) + continue + + # 遍历 AST 顶层节点，收集 ClassDef + ch: list[ast.AST | t.CPtr] | t.CPtr = tree.children + if ch is not None: + cn: t.CSizeT = len(ch) + for ci in range(cn): + child: ast.AST | t.CPtr = ch.get(ci) + if child is None: + continue + kd: int = child.kind() + if kd != ast.ASTKind.ClassDef: + continue + + cd: ast.ClassDef | t.CPtr = (ast.ClassDef | t.CPtr)(child) + if cd is None or cd.name is None: + continue + + # 检测泛型类（有 type_params），记录文件索引以供 Phase 1a 预注册 + if file_has_generic == 0: + tp_scan: list[str] | t.CPtr = cd.type_params + if tp_scan is not None: + file_has_generic = 1 + if generic_file_count_box is not None: + gfc_ptr_scan: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(generic_file_count_box, t.CPtr)) + if gfc_ptr_scan is not None: + gfc_scan: int = gfc_ptr_scan[0] + if gfc_scan < Phase1.MAX_FILE_DEPS and generic_file_indices is not None: + gi_addr_scan: t.CUInt64T = t.CUInt64T(generic_file_indices) + gfc_scan * 4 + gi_ptr_scan: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(gi_addr_scan, t.CPtr)) + if gi_ptr_scan is not None: + gi_ptr_scan[0] = i + gfc_ptr_scan[0] = gfc_scan + 1 + + if class_count >= Phase1.MAX_CLASSES: + break + + # 填充扁平数组：类名（偏移 = class_count * Phase1.CLASS_NAME_LEN） + name_dst: str = class_names_buf + class_count * Phase1.CLASS_NAME_LEN + nm_len: t.CSizeT = string.strlen(cd.name) + if nm_len >= Phase1.CLASS_NAME_LEN: + string.strncpy(name_dst, cd.name, Phase1.CLASS_NAME_LEN - 1) + name_dst[Phase1.CLASS_NAME_LEN - 1] = '\0' + else: + string.strcpy(name_dst, cd.name) + + # 填充扁平数组：父类名（无父类填 '\0'） + parent_dst: str = parent_names_buf + class_count * Phase1.CLASS_NAME_LEN + parent_dst[0] = '\0' + parent_nm: str = HandlesClassDef._get_parent_class(cd, None) + # parent_nm 是 AST 节点内部指针，不需要释放 + if parent_nm is not None: + pnm_len: t.CSizeT = string.strlen(parent_nm) + if pnm_len >= Phase1.CLASS_NAME_LEN: + string.strncpy(parent_dst, parent_nm, Phase1.CLASS_NAME_LEN - 1) + parent_dst[Phase1.CLASS_NAME_LEN - 1] = '\0' + else: + string.strcpy(parent_dst, parent_nm) + + # 填充扁平数组：定义 SHA1（偏移 = class_count * Phase1.SHA1_LEN） + sha1_dst: str = def_sha1s_buf + class_count * Phase1.SHA1_LEN + string.strcpy(sha1_dst, sha1_e) + + class_count += 1 + + stdlib.free(src_buf) + # 回收本文件 AST/lexer/token（类名/父类/SHA1 已拷入扁平数组） + mb.reset_to(scan_mark) + + return class_count +# ============================================================ +# _BufHasGenericClassMarker - 文本扫描检测文件是否含泛型类标记 +# +# 泛型类（PEP 695）语法: class Name[T]:（类名后紧跟 '['）。 +# 逐字节扫描找词边界 'class' 关键字，其后标识符后非空白是 '[' → 命中。 +# 用于 _PreRegisterAllGenericTemplates 预注册前筛选，避免 parse 全部 +# 98 个 includes 导致 AST 驻留 mark1 前把 pool 推到 OOM +# （NewAstTest [52/68] pool 耗尽根因：1a-pre 已 parse 全部驻留， +# 预注册再全量 parse 等于翻倍）。 +# 误报（字符串/注释里的 "class X["）无害（多 parse 一个文件）； +# 漏报有害（泛型模板未预注册 → 悬空复发），模式保守匹配行内单行形式 +# （TPV 源码泛型类均为单行 `class X[T]:`，grep 确认）。 +# ============================================================ +def _BufHasGenericClassMarker(buf: bytes, blen: t.CSizeT) -> int: + """文本扫描：buf 中是否有 'class 标识符[' 泛型类标记，返回 1/0""" + if buf is None or blen <= 6: + return 0 + pos: t.CSizeT = 0 + while pos + 5 < blen: + if buf[pos] == 'c' and buf[pos + 1] == 'l' and buf[pos + 2] == 'a' \ + and buf[pos + 3] == 's' and buf[pos + 4] == 's': + # class 前是词边界（行首或非标识符字符） + prev_ok: int = 1 + if pos > 0: + pc: t.CChar = buf[pos - 1] + if(pc >= 'a' and pc <= 'z') or(pc >= 'A' and pc <= 'Z') \ + or(pc >= '0' and pc <= '9') or pc == '_': + prev_ok = 0 + if prev_ok != 0: + p: t.CSizeT = pos + 5 + # 跳过空白 + while p < blen and(buf[p] == ' ' or buf[p] == '\t'): + p += 1 + # 读类名标识符 + if p < blen and((buf[p] >= 'a' and buf[p] <= 'z') + or(buf[p] >= 'A' and buf[p] <= 'Z') or buf[p] == '_'): + while p < blen and((buf[p] >= 'a' and buf[p] <= 'z') + or(buf[p] >= 'A' and buf[p] <= 'Z') + or(buf[p] >= '0' and buf[p] <= '9') + or buf[p] == '_'): + p += 1 + # 类名后非空白是 '[' → 泛型类 + while p < blen and(buf[p] == ' ' or buf[p] == '\t'): + p += 1 + if p < blen and buf[p] == '[': + return 1 + pos += 1 + else: + pos += 1 + return 0 +# ============================================================ +# _PreRegisterAllGenericTemplates - 预注册 includes 泛型类模板（mark1 前） +# +# 背景：泛型模板表（GenericClassNames/GenericClassNodes/GenericClassSha1s/ +# GenericClassFiles + 模板节点）是懒分配的跨文件持久表，首次注册必须落在 +# mark1 之前才能跨 Phase1 1b（reset_to(mark1)）与 Phase2（reset_to(mark2)， +# mark2 > mark1）存活。现有 1a-pre-generic 只注册"可达"泛型文件；FuncTest +# 这类可达集合无泛型的项目，模板要等 Phase2 Phase A-pre-inc（mark2 之后） +# 翻译全部 includes 时才注册 → reset 回收表与节点 → _find_generic_template +# 遍历悬空内存 strcmp 崩溃。 +# 本函数先文本扫描（_BufHasGenericClassMarker，不 parse、不驻留）筛选出 +# 含泛型类的文件（全量 parse 会把 98 个 includes 的 AST 驻留 mark1 前 → +# NewAstTest [52/68] pool 耗尽 OOM），仅对命中的文件用**永久池（perm_pool）** +# 解析 → 模板节点与表直接进 perm_pool，跨 mark 永久存活、不占主 pool。 +# 重复注册由 _register_generic_template 内部查重跳过。 +# ============================================================ +def _PreRegisterAllGenericTemplates(mb: memhub.MemBuddy | t.CPtr, + result: IncludesScanner.ScanResult | t.CPtr) -> int: + """预注册全部 includes 文件的泛型类模板，返回处理到的泛型类数量""" + if result is None or mb is None: + return 0 + perm: memhub.MemBuddy | t.CPtr = CompilerFactory.get_perm_pool() + if perm is None: + return 0 + p1e_size: t.CSizeT = IncludesScanner.FileEntry.__sizeof__() + handled_count: int = 0 + for i in range(result.Count): + entry_addr_pg: t.CUInt64T = t.CUInt64T(result.Entries) + i * p1e_size + entry_pg: IncludesScanner.FileEntry | t.CPtr = ( + IncludesScanner.FileEntry | t.CPtr)(t.CVoid(entry_addr_pg, t.CPtr)) + if entry_pg is None or entry_pg.Sha1 is None: + continue + # 读取文件 + file_path_pg: str = entry_pg.Path + f_pg: fileio.File | t.CPtr = fileio.File(file_path_pg, fileio.MODE.R) + if f_pg.closed: + continue + src_buf_pg: bytes = stdlib.malloc(Phase1.SRC_BUF_SIZE) + if src_buf_pg is None: + f_pg.close() + continue + bytes_read_pg: LONG = f_pg.read_all(src_buf_pg, Phase1.SRC_BUF_SIZE) + f_pg.close() + if bytes_read_pg <= 0: + stdlib.free(src_buf_pg) + continue + if bytes_read_pg < Phase1.SRC_BUF_SIZE: + src_buf_pg[bytes_read_pg] = 0 + else: + src_buf_pg[Phase1.SRC_BUF_SIZE - 1] = 0 + # 文本扫描筛选：非泛型文件跳过（不 parse、不驻留，防 OOM） + if _BufHasGenericClassMarker(src_buf_pg, bytes_read_pg) == 0: + stdlib.free(src_buf_pg) + continue + # 泛型文件：永久池解析 → 注册模板（节点/表进 perm_pool，跨 reset 存活） + lx_pg: ast.Lexer | t.CPtr = ast.new_lexer(perm) + if lx_pg is None: + stdlib.free(src_buf_pg) + continue + ast._lexer_init(lx_pg, src_buf_pg, perm) + tokens_pg: ast.Token | t.CPtr = ast.tokenize(lx_pg) + tree_pg: ast.AST | t.CPtr = ast.parse_tokens(perm, tokens_pg) + if tree_pg is None: + stdlib.free(src_buf_pg) + continue + # 遍历顶层 ClassDef，注册泛型模板（type_params 非 None） + ch_pg: list[ast.AST | t.CPtr] | t.CPtr = tree_pg.children + if ch_pg is not None: + cn_pg: t.CSizeT = len(ch_pg) + for ci_pg in range(cn_pg): + child_pg: ast.AST | t.CPtr = ch_pg.get(ci_pg) + if child_pg is None or child_pg.kind() != ast.ASTKind.ClassDef: + continue + cd_pg: ast.ClassDef | t.CPtr = (ast.ClassDef | t.CPtr)(child_pg) + if cd_pg is None or cd_pg.name is None: + continue + tp_pg: list[str] | t.CPtr = cd_pg.type_params + if tp_pg is None: + continue + HandlesType.set_current_file(file_path_pg) + HandlesClassDef._register_generic_template(perm, cd_pg, entry_pg.Sha1) + handled_count += 1 + stdlib.free(src_buf_pg) + if handled_count > 0: + fb_gpre2: str = VLogger.fmt_buf() + if fb_gpre2 is not None: + viperlib.snprintf(fb_gpre2, 1024, "mark1 前永久池预注册 %d 个 includes 泛型类模板", handled_count) + VLogger.info(fb_gpre2, "Phase1") + return handled_count +# ============================================================ +# _FindClassDefSha1 - 在扁平数组中查找类名对应的定义文件 SHA1 +# +# Args: +# class_names_buf: 类名扁平数组 +# def_sha1s_buf: 定义 SHA1 扁平数组 +# class_count: 类数量 +# class_name: 要查找的类名 +# +# Returns: +# SHA1 字符串指针（指向 def_sha1s_buf 内部偏移），None 未找到 +# ============================================================ +def _FindClassDefSha1(class_names_buf: bytes, def_sha1s_buf: bytes, + class_count: int, class_name: str) -> str: + """查找类名对应的定义文件 SHA1""" + if class_names_buf is None or def_sha1s_buf is None or class_name is None: + return None + for i in range(class_count): + name_ptr: str = class_names_buf + i * Phase1.CLASS_NAME_LEN + if string.strcmp(name_ptr, class_name) == 0: + return def_sha1s_buf + i * Phase1.SHA1_LEN + return None +# ============================================================ +# _ScanTreeForContainers - 递归扫描 AST 检测 list/dict 容器使用 +# +# TPV 的 AST 中，语句块挂在 children，而表达式/注解树全部以结构体 +# 字段挂载（value/slice/func/args/annotation/targets/returns/bases/ +# left/right 等）。只递归 children 会漏掉 list[...] | t.CPtr（BinOp +# 包裹）、list[...](...)（Call.func）、字段注解等一切非语句块位置。 +# 因此每个节点先递归其表达式承载字段，再递归 children 语句块。 +# 命中即置 need_list/need_dict；误报无害（最多多编译两个空壳模块）。 +# ============================================================ +def _ScanTreeForContainers(node: ast.AST | t.CPtr, + need_list_box: t.CPtr, + need_dict_box: t.CPtr): + """递归扫描 AST（字段 + 语句块），检测 list/dict 容器使用""" + if node is None: + return + k_s: int = node.kind() + if k_s == ast.ASTKind.Name: + nm_s: ast.Name | t.CPtr = (ast.Name | t.CPtr)(node) + if nm_s.id is not None: + if string.strcmp(nm_s.id, "list") == 0: + need_list_box[0] = 1 + elif string.strcmp(nm_s.id, "dict") == 0: + need_dict_box[0] = 1 + elif k_s == ast.ASTKind.Subscript: + sub_s: ast.Subscript | t.CPtr = (ast.Subscript | t.CPtr)(node) + _ScanTreeForContainers(sub_s.value, need_list_box, need_dict_box) + _ScanTreeForContainers(sub_s.slice, need_list_box, need_dict_box) + elif k_s == ast.ASTKind.BinOp: + bop_s: ast.BinOp | t.CPtr = (ast.BinOp | t.CPtr)(node) + _ScanTreeForContainers(bop_s.left, need_list_box, need_dict_box) + _ScanTreeForContainers(bop_s.right, need_list_box, need_dict_box) + elif k_s == ast.ASTKind.Call: + cl_s2: ast.Call | t.CPtr = (ast.Call | t.CPtr)(node) + _ScanTreeForContainers(cl_s2.func, need_list_box, need_dict_box) + cl_args_s: list[ast.AST | t.CPtr] | t.CPtr = cl_s2.args + if cl_args_s is not None: + na_s: t.CSizeT = len(cl_args_s) + i_a2: t.CSizeT + for i_a2 in range(na_s): + _ScanTreeForContainers(cl_args_s.get(i_a2), need_list_box, need_dict_box) + cl_kws_s: list[ast.AST | t.CPtr] | t.CPtr = cl_s2.keywords + if cl_kws_s is not None: + nk_s: t.CSizeT = len(cl_kws_s) + i_k2: t.CSizeT + for i_k2 in range(nk_s): + kw_s: ast.Keyword | t.CPtr = (ast.Keyword | t.CPtr)(cl_kws_s.get(i_k2)) + if kw_s is not None: + _ScanTreeForContainers(kw_s.value, need_list_box, need_dict_box) + elif k_s == ast.ASTKind.Assign: + asn_s: ast.Assign | t.CPtr = (ast.Assign | t.CPtr)(node) + _ScanTreeForContainers(asn_s.value, need_list_box, need_dict_box) + tgts_s: list[ast.AST | t.CPtr] | t.CPtr = asn_s.targets + if tgts_s is not None: + nt_s: t.CSizeT = len(tgts_s) + i_t2: t.CSizeT + for i_t2 in range(nt_s): + _ScanTreeForContainers(tgts_s.get(i_t2), need_list_box, need_dict_box) + elif k_s == ast.ASTKind.AnnAssign: + aa_s2: ast.AnnAssign | t.CPtr = (ast.AnnAssign | t.CPtr)(node) + _ScanTreeForContainers(aa_s2.annotation, need_list_box, need_dict_box) + _ScanTreeForContainers(aa_s2.value, need_list_box, need_dict_box) + elif k_s == ast.ASTKind.FunctionDef: + fn_s: ast.FunctionDef | t.CPtr = (ast.FunctionDef | t.CPtr)(node) + _ScanTreeForContainers(fn_s.returns, need_list_box, need_dict_box) + fn_args_s: ast.Arguments | t.CPtr = fn_s.args + if fn_args_s is not None: + fa_s: list[ast.AST | t.CPtr] | t.CPtr = fn_args_s.args + if fa_s is not None: + nf_s: t.CSizeT = len(fa_s) + i_f2: t.CSizeT + for i_f2 in range(nf_s): + arg_s: ast.Arg | t.CPtr = (ast.Arg | t.CPtr)(fa_s.get(i_f2)) + if arg_s is not None: + _ScanTreeForContainers(arg_s.annotation, need_list_box, need_dict_box) + elif k_s == ast.ASTKind.ClassDef: + cd_s: ast.ClassDef | t.CPtr = (ast.ClassDef | t.CPtr)(node) + bases_s: list[ast.AST | t.CPtr] | t.CPtr = cd_s.bases + if bases_s is not None: + nb_s: t.CSizeT = len(bases_s) + i_b2: t.CSizeT + for i_b2 in range(nb_s): + _ScanTreeForContainers(bases_s.get(i_b2), need_list_box, need_dict_box) + elif k_s == ast.ASTKind.Attribute: + at_s: ast.Attribute | t.CPtr = (ast.Attribute | t.CPtr)(node) + _ScanTreeForContainers(at_s.value, need_list_box, need_dict_box) + elif k_s == ast.ASTKind.Tuple: + tu_s: ast.Tuple | t.CPtr = (ast.Tuple | t.CPtr)(node) + elts_s: list[ast.AST | t.CPtr] | t.CPtr = tu_s.elts + if elts_s is not None: + ne_s: t.CSizeT = len(elts_s) + i_e2: t.CSizeT + for i_e2 in range(ne_s): + _ScanTreeForContainers(elts_s.get(i_e2), need_list_box, need_dict_box) + elif k_s == ast.ASTKind.UnaryOp: + uop_s: ast.UnaryOp | t.CPtr = (ast.UnaryOp | t.CPtr)(node) + _ScanTreeForContainers(uop_s.operand, need_list_box, need_dict_box) + elif k_s == ast.ASTKind.Expr: + ex_s: ast.Expr | t.CPtr = (ast.Expr | t.CPtr)(node) + _ScanTreeForContainers(ex_s.value, need_list_box, need_dict_box) + elif k_s == ast.ASTKind.Return: + rt_s: ast.Return | t.CPtr = (ast.Return | t.CPtr)(node) + _ScanTreeForContainers(rt_s.value, need_list_box, need_dict_box) + elif k_s == ast.ASTKind.Compare: + cp_s: ast.Compare | t.CPtr = (ast.Compare | t.CPtr)(node) + _ScanTreeForContainers(cp_s.left, need_list_box, need_dict_box) + cps_s: list[ast.AST | t.CPtr] | t.CPtr = cp_s.comparators + if cps_s is not None: + nc_s: t.CSizeT = len(cps_s) + i_c2: t.CSizeT + for i_c2 in range(nc_s): + _ScanTreeForContainers(cps_s.get(i_c2), need_list_box, need_dict_box) + elif k_s == ast.ASTKind.BoolOp: + bo_s: ast.BoolOp | t.CPtr = (ast.BoolOp | t.CPtr)(node) + bvs_s: list[ast.AST | t.CPtr] | t.CPtr = bo_s.values + if bvs_s is not None: + nv_s: t.CSizeT = len(bvs_s) + i_v2: t.CSizeT + for i_v2 in range(nv_s): + _ScanTreeForContainers(bvs_s.get(i_v2), need_list_box, need_dict_box) + elif k_s == ast.ASTKind.List: + # [] 字面量 + need_list_box[0] = 1 + elif k_s == ast.ASTKind.Dict: + # {} 字面量 + need_dict_box[0] = 1 + # 递归语句块子节点 + if node.children is not None: + ch_s: list[ast.AST | t.CPtr] | t.CPtr = node.children + n_s: t.CSizeT = len(ch_s) + ci_s: t.CSizeT + for ci_s in range(n_s): + _ScanTreeForContainers(ch_s.get(ci_s), need_list_box, need_dict_box) diff --git a/App/lib/core/Phase1Topo.vp b/App/lib/core/Phase1Topo.vp new file mode 100644 index 0000000..fdf6b29 --- /dev/null +++ b/App/lib/core/Phase1Topo.vp @@ -0,0 +1,417 @@ +import t, c +from stdint import * +import string +import stdlib +import memhub +import viperlib +import w32.fileio as fileio +import w32.win32file +import w32.win32base +import ast +import llvmlite +import lib.core.VLogger as VLogger +import lib.core.Handles.HandlesTranslator as HandlesTranslator +import lib.core.Handles.HandlesStruct as HandlesStruct +import lib.core.Handles.HandlesType as HandlesType +import lib.core.Handles.HandlesExprCall as HandlesExprCall +import lib.core.Handles.HandlesImports as HandlesImports +import lib.core.Handles.HandlesClassDef as HandlesClassDef +import lib.core.IncludesScanner as IncludesScanner +import lib.core.StubMerger as StubMerger +import lib.core.Gen.Generator as GenGenerator +import lib.core.CompilerFactory as CompilerFactory +import lib.core.BuildPipeline as BuildPipeline +import lib.Projectrans.Config as Config +import lib.StubGen.Converter as StubConverter +import lib.core.StubMergerBuild as StubMergerBuild +import lib.core.StubMergerCollect as StubMergerCollect +import lib.core.StubMergerIR as StubMergerIR +import lib.core.StubMergerStore as StubMergerStore +import lib.core.Handles.ExprCallCtor as ExprCallCtor + + +import lib.core.Phase1Scan as Phase1Scan +import lib.core.Phase1Main as Phase1Main +import lib.core.Phase1 as Phase1 + +# ============================================================ +# Phase1Topo - Phase1 拆分（阶段 E）：Phase1Topo +# +# 直接移动模式：函数迁至此模块，原 Phase1.vp 仅保留常量/_mbuddy； +# 调用方经本模块前缀访问。全局符号经 Phase1._xxx 跨模块访问。 +# ============================================================ + +# ============================================================ +# _TopoSortFiles - 根据类继承关系对文件进行拓扑排序 +# +# 算法（Kahn）： +# 1. 从扁平类信息数组构建"文件 SHA1 → 依赖文件 SHA1 集合"映射 +# 2. 计算每个文件的入度（依赖数量） +# 3. 入度为 0 的文件先处理，处理后减少依赖它的文件入度 +# 4. 重复直到所有文件处理完成 +# +# 注意：依赖方向是"父类所在文件 → 子类所在文件"， +# 即父类文件必须先处理。Kahn 用 DepCount 作为剩余依赖计数， +# 依赖为 0 的先处理，初始化 InDegree = DepCount。 +# +# Args: +# mb: 内存池 +# result: IncludesScanner 扫描结果 +# reachable_set: 可达 SHA1 集合 +# reachable_count: 可达 SHA1 数量 +# use_reachable: 是否使用可达集合过滤 +# sha1_set: SHA1 集合 +# set_count: SHA1 集合数量 +# class_names_buf: 类名扁平数组 +# parent_names_buf: 父类名扁平数组 +# def_sha1s_buf: 定义 SHA1 扁平数组 +# class_count: 类数量 +# out_order: 输出拓扑顺序的文件索引数组（调用者分配，int 数组） +# out_count: 输出文件数量 +# +# Returns: +# 0 成功，-1 失败 +# ============================================================ +def _TopoSortFiles(mb: memhub.MemBuddy | t.CPtr, + result: IncludesScanner.ScanResult | t.CPtr, + reachable_set: str, reachable_count: int, + use_reachable: int, + sha1_set: str, set_count: int, + class_names_buf: bytes, parent_names_buf: bytes, + def_sha1s_buf: bytes, class_count: int, + out_order: t.CPtr, out_count: t.CPtr, + temp_dir: str, td_len_ts: t.CSizeT) -> int: + """根据类继承关系对文件进行拓扑排序""" + if result is None or out_order is None or out_count is None: + return -1 + + entry_size: t.CSizeT = IncludesScanner.FileEntry.__sizeof__() + + # 1. 分配扁平数组：文件 SHA1、依赖 SHA1 列表、依赖数量、入度、处理状态 + # 用扁平数组代替 FileDepEntry 结构体，避免 str 指针字段 NULL 崩溃 + file_sha1s_buf: bytes = mb.alloc(Phase1.MAX_FILE_DEPS * Phase1.SHA1_LEN) + file_deps_buf: bytes = mb.alloc(Phase1.MAX_FILE_DEPS * Phase1.MAX_DEP_EDGES * Phase1.SHA1_LEN) + file_dep_counts: t.CPtr = mb.alloc(Phase1.MAX_FILE_DEPS * 4) + file_indegrees: t.CPtr = mb.alloc(Phase1.MAX_FILE_DEPS * 4) + file_processed: t.CPtr = mb.alloc(Phase1.MAX_FILE_DEPS * 4) + + if file_sha1s_buf is None or file_deps_buf is None or file_dep_counts is None or file_indegrees is None or file_processed is None: + if file_sha1s_buf is not None: + mb.free(file_sha1s_buf) + if file_deps_buf is not None: + mb.free(file_deps_buf) + if file_dep_counts is not None: + mb.free(file_dep_counts) + if file_indegrees is not None: + mb.free(file_indegrees) + if file_processed is not None: + mb.free(file_processed) + return -1 + + string.memset(file_sha1s_buf, 0, Phase1.MAX_FILE_DEPS * Phase1.SHA1_LEN) + string.memset(file_deps_buf, 0, Phase1.MAX_FILE_DEPS * Phase1.MAX_DEP_EDGES * Phase1.SHA1_LEN) + string.memset(file_dep_counts, 0, Phase1.MAX_FILE_DEPS * 4) + string.memset(file_indegrees, 0, Phase1.MAX_FILE_DEPS * 4) + string.memset(file_processed, 0, Phase1.MAX_FILE_DEPS * 4) + + file_count: int = 0 + # sha1→file_index 映射（简单线性查找，文件数不多） + for i in range(result.Count): + entry_addr: t.CUInt64T = t.CUInt64T(result.Entries) + i * entry_size + entry: IncludesScanner.FileEntry | t.CPtr = (IncludesScanner.FileEntry | t.CPtr)(t.CVoid(entry_addr, t.CPtr)) + if entry is None or entry.Sha1 is None: + continue + + sha1_e: str = entry.Sha1 + in_set: int = 0 + if use_reachable != 0: + in_set = StubMergerCollect._is_in_sha1_set(sha1_e, reachable_set, reachable_count) + else: + in_set = StubMergerCollect._is_in_sha1_set(sha1_e, sha1_set, set_count) + if in_set == 0: + continue + + if file_count >= Phase1.MAX_FILE_DEPS: + break + + # 初始化扁平数组：文件 SHA1（偏移 = file_count * Phase1.SHA1_LEN） + # DepCount/InDegree/Processed 已由 memset 清零 + sha1_dst: str = file_sha1s_buf + file_count * Phase1.SHA1_LEN + string.strcpy(sha1_dst, sha1_e) + + file_count += 1 + + # 2. 构建依赖边：对每个类，找到父类定义所在文件，建立 子文件→父文件 依赖 + for ci in range(class_count): + # 读取父类名（偏移 = ci * Phase1.CLASS_NAME_LEN） + parent_ptr: str = parent_names_buf + ci * Phase1.CLASS_NAME_LEN + if parent_ptr[0] == '\0': + continue + + # 读取子类定义 SHA1（偏移 = ci * Phase1.SHA1_LEN） + child_def_sha1: str = def_sha1s_buf + ci * Phase1.SHA1_LEN + + # 查找父类定义所在文件的 SHA1 + parent_sha1: str = Phase1Scan._FindClassDefSha1(class_names_buf, def_sha1s_buf, class_count, parent_ptr) + if parent_sha1 is None: + continue # 父类不在 includes 中（如 Object、CEnum 等标记基类） + + # 父类文件 SHA1 不能等于子类文件 SHA1（同文件内继承不需要依赖） + if string.strcmp(parent_sha1, child_def_sha1) == 0: + continue + + # 找到子类文件对应的索引，添加父文件 SHA1 到依赖列表 + for fi in range(file_count): + fsha1: str = file_sha1s_buf + fi * Phase1.SHA1_LEN + if string.strcmp(fsha1, child_def_sha1) != 0: + continue + + # 读取当前依赖数量 + cnt_addr: t.CUInt64T = t.CUInt64T(file_dep_counts) + fi * 4 + cnt_ptr: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(cnt_addr, t.CPtr)) + if cnt_ptr is None: + break + cur_cnt: int = cnt_ptr[0] + + # 检查依赖是否已添加（去重） + already: int = 0 + for di in range(cur_cnt): + dep_slot: str = file_deps_buf + fi * Phase1.MAX_DEP_EDGES * Phase1.SHA1_LEN + di * Phase1.SHA1_LEN + if string.strcmp(dep_slot, parent_sha1) == 0: + already = 1 + break + if already != 0: + break + + if cur_cnt < Phase1.MAX_DEP_EDGES: + dep_dst: str = file_deps_buf + fi * Phase1.MAX_DEP_EDGES * Phase1.SHA1_LEN + cur_cnt * Phase1.SHA1_LEN + string.strcpy(dep_dst, parent_sha1) + cnt_ptr[0] = cur_cnt + 1 + break + + # 2.5 补充 import 依赖边：被 import 的模块先完整翻译。 + # + # 仅按类继承建边不够：跨模块全局变量 stub（lexer.vp from-import + # tokens.vp 的 _op_head）、跨模块 vtable TypeStr（astaux 继承 base 的 + # AST）、泛型模板（base.vp 用 list[...]）都要求依赖模块先翻译，而它们 + # 之间可能无类继承关系。deps（_includes_deps_*）由 Phase 1a-pre 填充 + # （含容器按需注入的 _list/_dict）。 + # + # 用 _BuildIncludesSha1Map 复制局部 sha1→模块名 数组（与 + # _BuildReachableSha1Set 的 BFS 一致），避免直接使用全局 store 返回值 + # 跨函数传参（bytes|t.CPtr 联合转 str 参数）的地址异常。 + sha1_arr_ts: bytes = stdlib.malloc(StubMerger.MAX_INCLUDES * 17) + mod_arr_ts: bytes = stdlib.malloc(StubMerger.MAX_INCLUDES * 64) + map_count_ts: int = 0 + if sha1_arr_ts is not None and mod_arr_ts is not None: + map_count_ts = StubMergerIR._BuildIncludesSha1Map( + temp_dir, td_len_ts, sha1_arr_ts, mod_arr_ts) + for fi_imp in range(file_count): + if map_count_ts <= 0: + break + fsha1_imp: str = file_sha1s_buf + fi_imp * Phase1.SHA1_LEN + deps_imp: str = StubMergerStore.LookupIncludesDeps(fsha1_imp) + if deps_imp is None: + continue + dl_imp: t.CSizeT = string.strlen(deps_imp) + dpos_imp: t.CSizeT = 0 + while dpos_imp < dl_imp: + dstart_imp: t.CSizeT = dpos_imp + while dpos_imp < dl_imp and deps_imp[dpos_imp] != ' ': + dpos_imp += 1 + dlen_imp: t.CSizeT = dpos_imp - dstart_imp + if dpos_imp < dl_imp: + dpos_imp += 1 + if dlen_imp == 0: + continue + dname_imp: bytes = stdlib.malloc(dlen_imp + 1) + if dname_imp is None: + continue + string.strncpy(dname_imp, deps_imp + dstart_imp, dlen_imp) + dname_imp[dlen_imp] = '\0' + dsha1_imp: str = StubMergerIR._FindSha1ByModName( + sha1_arr_ts, mod_arr_ts, map_count_ts, dname_imp) + stdlib.free(dname_imp) + if dsha1_imp is None: + continue + # 找依赖文件在列表中的索引（跳过自身） + for fj_imp in range(file_count): + if string.strcmp(file_sha1s_buf + fj_imp * Phase1.SHA1_LEN, dsha1_imp) != 0: + continue + if fj_imp == fi_imp: + break + # 去重 + 加边：fi_imp 依赖 fj_imp（InDegree[fi_imp]++） + cnt_addr_imp: t.CUInt64T = t.CUInt64T(file_dep_counts) + fi_imp * 4 + cnt_ptr_imp: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(cnt_addr_imp, t.CPtr)) + if cnt_ptr_imp is None: + break + cur_cnt_imp: int = cnt_ptr_imp[0] + already_imp: int = 0 + for di_imp in range(cur_cnt_imp): + dep_slot_imp: str = file_deps_buf + fi_imp * Phase1.MAX_DEP_EDGES * Phase1.SHA1_LEN + di_imp * Phase1.SHA1_LEN + if string.strcmp(dep_slot_imp, dsha1_imp) == 0: + already_imp = 1 + break + if already_imp == 0 and cur_cnt_imp < Phase1.MAX_DEP_EDGES: + dep_dst_imp: str = file_deps_buf + fi_imp * Phase1.MAX_DEP_EDGES * Phase1.SHA1_LEN + cur_cnt_imp * Phase1.SHA1_LEN + string.strcpy(dep_dst_imp, dsha1_imp) + cnt_ptr_imp[0] = cur_cnt_imp + 1 + break + + # 释放 import 边用的局部映射数组 + if sha1_arr_ts is not None: + stdlib.free(sha1_arr_ts) + if mod_arr_ts is not None: + stdlib.free(mod_arr_ts) + + # 3. Kahn 算法：InDegree = DepCount（依赖多少个父文件） + # 依赖为 0 的先处理，处理后减少依赖它的文件入度 + for fi in range(file_count): + cnt_addr3: t.CUInt64T = t.CUInt64T(file_dep_counts) + fi * 4 + cnt_ptr3: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(cnt_addr3, t.CPtr)) + ind_addr3: t.CUInt64T = t.CUInt64T(file_indegrees) + fi * 4 + ind_ptr3: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(ind_addr3, t.CPtr)) + if cnt_ptr3 is not None and ind_ptr3 is not None: + ind_ptr3[0] = cnt_ptr3[0] + + # 4. 拓扑排序输出 + out_idx: int = 0 + changed: int = 1 + while changed != 0: + changed = 0 + for fi in range(file_count): + # 检查处理状态 + proc_addr4: t.CUInt64T = t.CUInt64T(file_processed) + fi * 4 + proc_ptr4: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(proc_addr4, t.CPtr)) + if proc_ptr4 is None or proc_ptr4[0] != 0: + continue + # 检查入度 + ind_addr4: t.CUInt64T = t.CUInt64T(file_indegrees) + fi * 4 + ind_ptr4: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(ind_addr4, t.CPtr)) + if ind_ptr4 is None or ind_ptr4[0] > 0: + continue + + # 入度为 0，可以处理 + # 找到对应的 result.Entries 索引 + target_sha1: str = file_sha1s_buf + fi * Phase1.SHA1_LEN + for ri in range(result.Count): + ea_r: t.CUInt64T = t.CUInt64T(result.Entries) + ri * entry_size + ent_r: IncludesScanner.FileEntry | t.CPtr = (IncludesScanner.FileEntry | t.CPtr)(t.CVoid(ea_r, t.CPtr)) + if ent_r is not None and ent_r.Sha1 is not None: + if string.strcmp(ent_r.Sha1, target_sha1) == 0: + # 写入输出索引数组 + out_idx_addr: t.CUInt64T = t.CUInt64T(out_order) + out_idx * 4 + out_int_ptr: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(out_idx_addr, t.CPtr)) + if out_int_ptr is not None: + out_int_ptr[0] = ri + out_idx += 1 + break + + # 标记已处理 + proc_ptr4[0] = 1 + changed = 1 + + # 减少依赖该文件的其他文件的入度 + for fj in range(file_count): + proc_addr5: t.CUInt64T = t.CUInt64T(file_processed) + fj * 4 + proc_ptr5: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(proc_addr5, t.CPtr)) + if proc_ptr5 is None or proc_ptr5[0] != 0: + continue + # 读取 fj 的依赖数量 + cnt_addr5: t.CUInt64T = t.CUInt64T(file_dep_counts) + fj * 4 + cnt_ptr5: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(cnt_addr5, t.CPtr)) + if cnt_ptr5 is None: + continue + fj_dep_cnt: int = cnt_ptr5[0] + for di in range(fj_dep_cnt): + dep_slot5: str = file_deps_buf + fj * Phase1.MAX_DEP_EDGES * Phase1.SHA1_LEN + di * Phase1.SHA1_LEN + if string.strcmp(dep_slot5, target_sha1) == 0: + ind_addr5: t.CUInt64T = t.CUInt64T(file_indegrees) + fj * 4 + ind_ptr5: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(ind_addr5, t.CPtr)) + if ind_ptr5 is not None: + ind_ptr5[0] = ind_ptr5[0] - 1 + break + + # 处理循环依赖：环内优先输出"被依赖者"（被其余剩余文件依赖数量多者先）。 + # + # 背景：容器按需注入（string 用 list 泛型 → 依赖 _list）与真实 import + # （_list import string）形成伪环，Kahn 无法完全归零。若按扫描顺序追加， + # 依赖者（astaux）会先于被依赖者（base）翻译 → 跨模块 vtable TypeStr + # 未注册 → 误杀。被依赖者先翻译，其 stub/TypeStr/模板先就绪，依赖者 + # 翻译时可引用（函数/全局声明由 1a-pre 的 vpi 兜底）。 + if out_idx < file_count: + rem_idx: t.CPtr = stdlib.malloc(4 * file_count) + rem_depcnt: t.CPtr = stdlib.malloc(4 * file_count) + rem_count: int = 0 + if rem_idx is not None and rem_depcnt is not None: + for fi in range(file_count): + proc_addr7: t.CUInt64T = t.CUInt64T(file_processed) + fi * 4 + proc_ptr7: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(proc_addr7, t.CPtr)) + if proc_ptr7 is not None and proc_ptr7[0] == 0: + rem_idx[rem_count] = fi + rem_depcnt[rem_count] = 0 + rem_count += 1 + # 计算每个剩余文件的被依赖数（被其余剩余文件依赖） + for i_a in range(rem_count): + fi_a: int = rem_idx[i_a] + target_sha1_a: str = file_sha1s_buf + fi_a * Phase1.SHA1_LEN + for i_b in range(rem_count): + fi_b: int = rem_idx[i_b] + if fi_b == fi_a: + continue + cnt_addr_b: t.CUInt64T = t.CUInt64T(file_dep_counts) + fi_b * 4 + cnt_ptr_b: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(cnt_addr_b, t.CPtr)) + if cnt_ptr_b is None: + continue + cnt_b: int = cnt_ptr_b[0] + for di_b in range(cnt_b): + dep_slot_b: str = file_deps_buf + fi_b * Phase1.MAX_DEP_EDGES * Phase1.SHA1_LEN + di_b * Phase1.SHA1_LEN + if string.strcmp(dep_slot_b, target_sha1_a) == 0: + rem_depcnt[i_a] = rem_depcnt[i_a] + 1 + break + # 冒泡降序（被依赖数多者在前） + for i_a in range(rem_count): + for j_a in range(rem_count - 1): + if rem_depcnt[j_a] < rem_depcnt[j_a + 1]: + tmp_i: int = rem_idx[j_a] + rem_idx[j_a] = rem_idx[j_a + 1] + rem_idx[j_a + 1] = tmp_i + tmp_d: int = rem_depcnt[j_a] + rem_depcnt[j_a] = rem_depcnt[j_a + 1] + rem_depcnt[j_a + 1] = tmp_d + # 按序输出剩余文件 + for i_a in range(rem_count): + fi_a2: int = rem_idx[i_a] + proc_addr8: t.CUInt64T = t.CUInt64T(file_processed) + fi_a2 * 4 + proc_ptr8: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(proc_addr8, t.CPtr)) + if proc_ptr8 is None or proc_ptr8[0] != 0: + continue + target_sha1_2: str = file_sha1s_buf + fi_a2 * Phase1.SHA1_LEN + for ri in range(result.Count): + ea_r2: t.CUInt64T = t.CUInt64T(result.Entries) + ri * entry_size + ent_r2: IncludesScanner.FileEntry | t.CPtr = (IncludesScanner.FileEntry | t.CPtr)(t.CVoid(ea_r2, t.CPtr)) + if ent_r2 is not None and ent_r2.Sha1 is not None: + if string.strcmp(ent_r2.Sha1, target_sha1_2) == 0: + out_idx_addr2: t.CUInt64T = t.CUInt64T(out_order) + out_idx * 4 + out_int_ptr2: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(out_idx_addr2, t.CPtr)) + if out_int_ptr2 is not None: + out_int_ptr2[0] = ri + out_idx += 1 + break + proc_ptr8[0] = 1 + stdlib.free(rem_idx) + stdlib.free(rem_depcnt) + + # 写入输出数量 + out_count_ptr: t.CPtr = (t.CInt | t.CPtr)(t.CVoid(out_count, t.CPtr)) + if out_count_ptr is not None: + out_count_ptr[0] = out_idx + + # 释放扁平数组 + mb.free(file_sha1s_buf) + mb.free(file_deps_buf) + mb.free(file_dep_counts) + mb.free(file_indegrees) + mb.free(file_processed) + + return 0 diff --git a/App/main.vp b/App/main.vp index db2e504..b23e2f0 100644 --- a/App/main.vp +++ b/App/main.vp @@ -49,6 +49,7 @@ import hashlib import t, c import lib.core.StubMergerBuild as StubMergerBuild import lib.core.StubMergerCollect as StubMergerCollect +import lib.core.Phase1Main as Phase1Main # Main Pool 1 GiB @@ -392,7 +393,7 @@ TermiNexus Age %sY\n\n""", VIPER_VERSION, RUNTIME_VERSION, TPV_VERSION, TERMINEX ph1_temp: str = Config.TempDir if ph1_temp is None: ph1_temp = "." - if Phase1.RunPhase1(mb, Config.IncludesDirs, ph1_temp, log) != 0: + if Phase1Main.RunPhase1(mb, Config.IncludesDirs, ph1_temp, log) != 0: VLogger.error("Phase1 失败，终止编译", "Phase1") return 1 # 如果仅 Phase1（不执行 Phase2），直接退出