GVSDS/TransPyC
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed

Browse Source
This commit is contained in:
TermiNexus
2026-07-30 13:34:26 +08:00
parent a2cc28a6ab
commit f79c8ca643
43 changed files with 1690 additions and 1016 deletions
Download Patch File Download Diff File Expand all files Collapse all files

407
TransPyV/App/lib/core/StubMerger.py
View File

@@ -51,6 +51,19 @@ _sha1_store_mod: bytes # 模块名数组（每个 64 字节，MAX_INCLUDES 个
_sha1_store_rel: bytes # 相对路径数组（每个 256 字节，MAX_INCLUDES 个）
_sha1_store_count: int # 条目数
# 可达集合是否不完整（1=有 includes deps.txt 缺失，Phase C 应回退全量链接）
_g_reachable_incomplete: int
# includes 依赖内存存储（不依赖 deps.txt 过程文件）
# Phase B+ 前用 declare_only 翻译所有 includes 文件，提取 _imported_modules 存入此处
# _BuildReachableSha1Set 从此处读取依赖，不读 deps.txt
_includes_deps_sha1: bytes # SHA1 数组（每个 17 字节，MAX_INCLUDES 个）
_includes_deps_str: bytes # 依赖字符串数组（每个 512 字节，MAX_INCLUDES 个）
_includes_deps_count: int # 条目数
# 单条依赖字符串最大长度
MAX_DEPS_STR_LEN: t.CDefine = 512
# rel_path 最大长度
MAX_REL_PATH_LEN: t.CDefine = 256
@@ -493,6 +506,85 @@ def GetSha1StoreCount() -> int:
return _sha1_store_count
def IsReachableIncomplete() -> int:
"""返回可达集合是否不完整（1=有 deps.txt 缺失，应回退全量链接）"""
return _g_reachable_incomplete
# ============================================================
# PopulateIncludesDeps - 将 includes 文件的依赖存入内存
#
# Phase B+ 前用 declare_only 翻译 includes 文件，提取 _imported_modules
# 后调用此函数存入内存。_BuildReachableSha1Set 从内存读取，不读 deps.txt。
#
# Args:
# sha1: includes 文件的 SHA1（16字符）
# deps_str: 依赖模块名字符串（空格分隔，如 "t c stdint string"）
#
# Returns:
# 0 成功，非 0 失败
# ============================================================
def PopulateIncludesDeps(sha1: str, deps_str: str) -> int:
"""将 includes 文件的依赖存入内存"""
global _includes_deps_sha1, _includes_deps_str, _includes_deps_count
if sha1 is None:
return 1
# 延迟初始化
if _includes_deps_sha1 is None:
_includes_deps_sha1 = stdlib.malloc(MAX_INCLUDES * 17)
_includes_deps_str = stdlib.malloc(MAX_INCLUDES * MAX_DEPS_STR_LEN)
if _includes_deps_sha1 is None or _includes_deps_str is None:
return 1
string.memset(_includes_deps_sha1, 0, MAX_INCLUDES * 17)
string.memset(_includes_deps_str, 0, MAX_INCLUDES * MAX_DEPS_STR_LEN)
_includes_deps_count = 0
if _includes_deps_count >= MAX_INCLUDES:
return 1
# 去重：若 SHA1 已存在则跳过
for i in range(_includes_deps_count):
sidx: t.CSizeT = t.CSizeT(i) * 17
if string.strcmp(_includes_deps_sha1 + sidx, sha1) == 0:
return 0
# 写入 SHA1
idx: t.CSizeT = t.CSizeT(_includes_deps_count) * 17
string.strcpy(_includes_deps_sha1 + idx, sha1)
# 写入依赖字符串
idx2: t.CSizeT = t.CSizeT(_includes_deps_count) * MAX_DEPS_STR_LEN
if deps_str is not None:
dl: t.CSizeT = string.strlen(deps_str)
if dl < MAX_DEPS_STR_LEN:
string.strcpy(_includes_deps_str + idx2, deps_str)
else:
string.strncpy(_includes_deps_str + idx2, deps_str, MAX_DEPS_STR_LEN - 1)
else:
_includes_deps_str[idx2] = '\0'
_includes_deps_count = _includes_deps_count + 1
return 0
# ============================================================
# LookupIncludesDeps - 从内存查找 includes 文件的依赖
#
# Args:
# sha1: includes 文件的 SHA1（16字符）
#
# Returns:
# 依赖模块名字符串指针（空格分隔），None 表示未找到
# ============================================================
def LookupIncludesDeps(sha1: str) -> str:
"""从内存查找 includes 文件的依赖"""
if sha1 is None or _includes_deps_sha1 is None:
return None
if _includes_deps_count <= 0:
return None
for i in range(_includes_deps_count):
sidx: t.CSizeT = t.CSizeT(i) * 17
if string.strcmp(_includes_deps_sha1 + sidx, sha1) == 0:
idx2: t.CSizeT = t.CSizeT(i) * MAX_DEPS_STR_LEN
return _includes_deps_str + idx2
return None
def GetSha1StoreArrPtr() -> bytes | t.CPtr:
"""返回 SHA1 数组指针（每个条目 17 字节）"""
return _sha1_store_arr
@@ -538,6 +630,32 @@ def GetSha1StoreArr(out_sha1_arr: t.CPtr, out_mod_arr: t.CPtr, out_rel_arr: t.CP
return _sha1_store_count
# ============================================================
# LookupSha1RelPath - 通过 SHA1 查找相对路径
#
# 遍历全局存储器，返回 sha1 对应的相对路径指针（只读，不要 free）。
# 用于错误提示：将 sha1 转换为人类可读的文件路径。
#
# Args:
# sha1: SHA1 字符串（16 字符）
#
# Returns:
# 相对路径字符串指针（只读），None=未找到
# ============================================================
def LookupSha1RelPath(sha1: str) -> str:
"""通过 SHA1 查找相对路径，返回只读指针，None=未找到"""
if sha1 is None:
return None
if _sha1_store_arr is None or _sha1_store_rel is None:
return None
for i in range(_sha1_store_count):
sidx: t.CSizeT = t.CSizeT(i) * 17
if string.strcmp(_sha1_store_arr + sidx, sha1) == 0:
ridx: t.CSizeT = t.CSizeT(i) * MAX_REL_PATH_LEN
return _sha1_store_rel + ridx
return None
# ============================================================
# _IsFuncDeclaredOrDefined - 检查 out_buf 中是否已有函数的 declare/define
#
@@ -1265,11 +1383,19 @@ def _AddSha1ToSet(set_buf: t.CChar | t.CPtr, count: int, sha1: str) -> int:
# ============================================================
def _BuildReachableSha1Set(mb: memhub.MemBuddy | t.CPtr, source_dir: str,
temp_dir: str,
reachable_set: t.CChar | t.CPtr) -> int:
"""构建可达 SHA1 集合（依赖图按需翻译）"""
reachable_set: t.CChar | t.CPtr,
app_deps: str = None) -> int:
"""构建可达 SHA1 集合（依赖图按需翻译）
app_deps: App 源文件的直接依赖模块名（空格分隔），如果非空则跳过自己的扫描步骤
"""
global _g_reachable_incomplete
if source_dir is None or temp_dir is None or reachable_set is None:
return -1
# 重置不完整标志
_g_reachable_incomplete = 0
SRC_BUF_SIZE_R: t.CSizeT = 1048576
# 1. 构建 SHA1→module_name 映射
@@ -1293,29 +1419,9 @@ def _BuildReachableSha1Set(mb: memhub.MemBuddy | t.CPtr, source_dir: str,
viperlib.snprintf(fb, 1024, "SHA1 映射: %d 个", map_count)
VLogger.debug(fb, "Reachable")
# 2. 扫描 source_dir 下的 .py 文件，收集直接依赖
dir_len: t.CSizeT = string.strlen(source_dir)
pattern: bytes = stdlib.malloc(dir_len + 8)
if pattern is None:
stdlib.free(sha1_arr)
stdlib.free(mod_arr)
return -1
viperlib.snprintf(pattern, dir_len + 8, "%s/*.py", source_dir)
find_data_size: t.CSizeT = win32file.WIN32_FIND_DATAA.__sizeof__()
find_data: win32file.WIN32_FIND_DATAA | t.CPtr = stdlib.malloc(find_data_size + 16)
if find_data is None:
stdlib.free(pattern)
stdlib.free(sha1_arr)
stdlib.free(mod_arr)
return -1
string.memset(find_data, 0, find_data_size + 16)
# 工作列表（空格分隔的模块名）
worklist: bytes = stdlib.malloc(8192)
if worklist is None:
stdlib.free(pattern)
stdlib.free(find_data)
stdlib.free(sha1_arr)
stdlib.free(mod_arr)
return -1
@@ -1324,70 +1430,104 @@ def _BuildReachableSha1Set(mb: memhub.MemBuddy | t.CPtr, source_dir: str,
reachable_count: int = 0
handle: win32base.HANDLE = win32file.FindFirstFileA(pattern, find_data)
if handle == win32base.INVALID_HANDLE_VALUE:
fb: t.CChar | t.CPtr = VLogger.fmt_buf()
if fb is not None:
viperlib.snprintf(fb, 1024, "未找到 .py 文件: %s", pattern)
VLogger.warning(fb, "Reachable")
stdlib.free(pattern)
stdlib.free(find_data)
stdlib.free(sha1_arr)
stdlib.free(mod_arr)
stdlib.free(worklist)
return -1
# 2. 如果 app_deps 非空，直接使用它作为初始 worklist（跳过自己的非递归扫描）
# 否则，回退到自己的非递归扫描（source_dir/*.py，只扫描顶层）
if app_deps is not None and app_deps[0] != '\0':
ad_len: t.CSizeT = string.strlen(app_deps)
if ad_len > 0 and ad_len < 8192:
string.strcpy(worklist, app_deps)
wl_len = ad_len
else:
# 回退到自己的非递归扫描
dir_len: t.CSizeT = string.strlen(source_dir)
pattern: bytes = stdlib.malloc(dir_len + 8)
if pattern is None:
stdlib.free(sha1_arr)
stdlib.free(mod_arr)
stdlib.free(worklist)
return -1
viperlib.snprintf(pattern, dir_len + 8, "%s/*.py", source_dir)
while True:
fname: str = find_data.cFileName
if fname is not None:
fname_len: t.CSizeT = string.strlen(fname)
if fname_len > 3:
is_py: int = 0
if fname[fname_len - 3] == '.' and fname[fname_len - 2] == 'p' and fname[fname_len - 1] == 'y':
is_py = 1
if is_py != 0:
full_path: bytes = stdlib.malloc(dir_len + fname_len + 2)
if full_path is not None:
viperlib.snprintf(full_path, dir_len + fname_len + 2, "%s/%s", source_dir, fname)
sf: fileio.File | t.CPtr = fileio.File(full_path, fileio.MODE.R)
if not sf.closed:
sbuf: bytes = stdlib.malloc(SRC_BUF_SIZE_R)
if sbuf is not None:
br: LONG = sf.read_all(sbuf, SRC_BUF_SIZE_R)
sf.close()
if br > 0:
if br < SRC_BUF_SIZE_R:
sbuf[br] = 0
else:
sbuf[SRC_BUF_SIZE_R - 1] = 0
# 解析 AST 获取 _imported_modules
lx: ast.Lexer | t.CPtr = ast.new_lexer(mb)
if lx is not None:
ast._lexer_init(lx, sbuf, mb)
tokens: ast.Token | t.CPtr = ast.tokenize(lx)
tree: ast.AST | t.CPtr = ast.parse_tokens(mb, tokens)
if tree is not None:
tr: HandlesTranslator.Translator | t.CPtr = HandlesTranslator.Translator()
if tr is not None:
tr._declare_only = 2
# 源文件包名：fname 是顶级文件名（无目录分隔符），包为 None
tr.CurrentPackage = HandlesImports.compute_package_from_relpath(mb, fname)
HandlesStruct.reset_visible_structs(mb, 0)
tr.translate(tree)
# 获取 _imported_modules
if tr._imported_modules is not None:
im_len: t.CSizeT = string.strlen(tr._imported_modules)
if im_len > 0 and wl_len + im_len + 1 < 8192:
string.strcpy(worklist + wl_len, tr._imported_modules)
wl_len += im_len
worklist[wl_len] = ' '
wl_len += 1
worklist[wl_len] = '\0'
# 释放 Translator 资源
if tr._global_names is not None:
stdlib.free(tr._global_names)
if tr._nonlocal_names is not None:
stdlib.free(tr._nonlocal_names)
find_data_size: t.CSizeT = win32file.WIN32_FIND_DATAA.__sizeof__()
find_data: win32file.WIN32_FIND_DATAA | t.CPtr = stdlib.malloc(find_data_size + 16)
if find_data is None:
stdlib.free(pattern)
stdlib.free(sha1_arr)
stdlib.free(mod_arr)
stdlib.free(worklist)
return -1
string.memset(find_data, 0, find_data_size + 16)
handle: win32base.HANDLE = win32file.FindFirstFileA(pattern, find_data)
if handle == win32base.INVALID_HANDLE_VALUE:
fb_fd: t.CChar | t.CPtr = VLogger.fmt_buf()
if fb_fd is not None:
viperlib.snprintf(fb_fd, 1024, "未找到 .py 文件: %s", pattern)
VLogger.warning(fb_fd, "Reachable")
stdlib.free(pattern)
stdlib.free(find_data)
stdlib.free(sha1_arr)
stdlib.free(mod_arr)
stdlib.free(worklist)
return -1
while True:
fname: str = find_data.cFileName
if fname is not None:
fname_len: t.CSizeT = string.strlen(fname)
if fname_len > 3:
is_py: int = 0
if fname[fname_len - 3] == '.' and fname[fname_len - 2] == 'p' and fname[fname_len - 1] == 'y':
is_py = 1
if is_py != 0:
full_path: bytes = stdlib.malloc(dir_len + fname_len + 2)
if full_path is not None:
viperlib.snprintf(full_path, dir_len + fname_len + 2, "%s/%s", source_dir, fname)
sf: fileio.File | t.CPtr = fileio.File(full_path, fileio.MODE.R)
if not sf.closed:
sbuf: bytes = stdlib.malloc(SRC_BUF_SIZE_R)
if sbuf is not None:
br: LONG = sf.read_all(sbuf, SRC_BUF_SIZE_R)
sf.close()
if br > 0:
if br < SRC_BUF_SIZE_R:
sbuf[br] = 0
else:
sbuf[SRC_BUF_SIZE_R - 1] = 0
# 解析 AST 获取 _imported_modules
lx: ast.Lexer | t.CPtr = ast.new_lexer(mb)
if lx is not None:
ast._lexer_init(lx, sbuf, mb)
tokens: ast.Token | t.CPtr = ast.tokenize(lx)
tree: ast.AST | t.CPtr = ast.parse_tokens(mb, tokens)
if tree is not None:
tr: HandlesTranslator.Translator | t.CPtr = HandlesTranslator.Translator()
if tr is not None:
tr._declare_only = 2
# 源文件包名：fname 是顶级文件名（无目录分隔符），包为 None
tr.CurrentPackage = HandlesImports.compute_package_from_relpath(mb, fname)
HandlesStruct.reset_visible_structs(mb, 0)
tr.translate(tree)
# 获取 _imported_modules
if tr._imported_modules is not None:
im_len: t.CSizeT = string.strlen(tr._imported_modules)
if im_len > 0 and wl_len + im_len + 1 < 8192:
string.strcpy(worklist + wl_len, tr._imported_modules)
wl_len += im_len
worklist[wl_len] = ' '
wl_len += 1
worklist[wl_len] = '\0'
# 计算 App 源文件 SHA1，填充依赖到内存存储器
# worklist 遍历时，LookupIncludesDeps 能找到 App 源文件的依赖
app_sha1_r: str = IncludesScanner.compute_file_sha1(mb, full_path)
if app_sha1_r is not None:
PopulateIncludesDeps(app_sha1_r, tr._imported_modules)
stdlib.free(app_sha1_r)
# 释放 Translator 资源
if tr._global_names is not None:
stdlib.free(tr._global_names)
if tr._nonlocal_names is not None:
stdlib.free(tr._nonlocal_names)
stdlib.free(sbuf)
else:
stdlib.free(sbuf)
@@ -1395,16 +1535,16 @@ def _BuildReachableSha1Set(mb: memhub.MemBuddy | t.CPtr, source_dir: str,
sf.close()
stdlib.free(full_path)
if win32file.FindNextFileA(handle, find_data) == 0:
break
win32base.FindClose(handle)
stdlib.free(pattern)
stdlib.free(find_data)
if win32file.FindNextFileA(handle, find_data) == 0:
break
win32base.FindClose(handle)
stdlib.free(pattern)
stdlib.free(find_data)
# 去掉末尾多余空格
if wl_len > 0 and worklist[wl_len - 1] == ' ':
worklist[wl_len - 1] = '\0'
wl_len -= 1
# 去掉末尾多余空格
if wl_len > 0 and worklist[wl_len - 1] == ' ':
worklist[wl_len - 1] = '\0'
wl_len -= 1
fb: t.CChar | t.CPtr = VLogger.fmt_buf()
if fb is not None:
@@ -1512,36 +1652,55 @@ def _BuildReachableSha1Set(mb: memhub.MemBuddy | t.CPtr, source_dir: str,
# 加入 reachable_set
reachable_count = _AddSha1ToSet(reachable_set, reachable_count, found_sha1)
# 读取 .deps.txt 追加到 worklist
deps_path: str = _sliced_path(temp_dir, td_len_r, found_sha1, "deps.txt")
if deps_path is not None:
df: fileio.File | t.CPtr = fileio.File(deps_path, fileio.MODE.R)
if not df.closed:
deps_buf: bytes = stdlib.malloc(2048)
if deps_buf is not None:
dbr: t.CInt64T = df.read_all(deps_buf, 2048)
df.close()
if dbr > 0:
if dbr < 2048:
deps_buf[dbr] = '\0'
else:
deps_buf[2047] = '\0'
# 追加到 worklist（嵌套 if 拆分 + 用 = 代替 += 绕过 TPC AUGASGN bug）
dl: t.CSizeT = string.strlen(deps_buf)
if worklist is not None:
if dl > 0 and wl_len + dl + 2 < 8192:
if wl_len > 0 and worklist[wl_len - 1] != ' ':
worklist[wl_len] = ' '
wl_len = wl_len + 1
string.strcpy(worklist + wl_len, deps_buf)
wl_len = wl_len + dl
worklist[wl_len] = ' '
wl_len = wl_len + 1
worklist[wl_len] = '\0'
stdlib.free(deps_buf)
else:
df.close()
stdlib.free(deps_path)
# 从内存查找依赖（不读 deps.txt 过程文件）
# Phase B+ 前用 declare_only 翻译所有 includes 文件，提取 _imported_modules 存入内存
deps_str: str = LookupIncludesDeps(found_sha1)
if deps_str is not None:
dl: t.CSizeT = string.strlen(deps_str)
if dl > 0:
if wl_len + dl + 2 < 8192:
if wl_len > 0 and worklist[wl_len - 1] != ' ':
worklist[wl_len] = ' '
wl_len = wl_len + 1
string.strcpy(worklist + wl_len, deps_str)
wl_len = wl_len + dl
worklist[wl_len] = ' '
wl_len = wl_len + 1
worklist[wl_len] = '\0'
# deps_str 为空字符串视为无依赖（合法情况，如叶子模块）
else:
# 依赖未在内存中找到
# 按需过滤: 检查是否是 App 源文件（source_dir 下的 .py 文件）
# App 源文件不需要加入 reachable_set（只用于 includes 过滤），跳过
sd_len_ff: t.CSizeT = string.strlen(source_dir)
mb_len_ff: t.CSizeT = string.strlen(mod_buf)
app_mod_path: bytes = stdlib.malloc(sd_len_ff + mb_len_ff + 6)
is_app_src: int = 0
if app_mod_path is not None:
viperlib.snprintf(app_mod_path, sd_len_ff + mb_len_ff + 6, "%s/%s.py", source_dir, mod_buf)
af: fileio.File | t.CPtr = fileio.File(app_mod_path, fileio.MODE.R)
if not af.closed:
is_app_src = 1
af.close()
stdlib.free(app_mod_path)
if is_app_src != 0:
# App 源文件，跳过（不 fast-fail）
fb_skip: t.CChar | t.CPtr = VLogger.fmt_buf()
if fb_skip is not None:
viperlib.snprintf(fb_skip, 1024, "跳过 App 源文件: %s (sha1=%s)", mod_buf, found_sha1)
VLogger.debug(fb_skip, "Reachable")
else:
# fast-fail: includes 文件依赖未在内存中找到，Phase 1a-pre 未正确填充
fb_ff: t.CChar | t.CPtr = VLogger.fmt_buf()
if fb_ff is not None:
viperlib.snprintf(fb_ff, 1024, "依赖未找到 fast-fail: 模块 %s (sha1=%s) 的依赖未在内存中注册", mod_buf, found_sha1)
VLogger.error(fb_ff, "Reachable")
stdlib.free(sha1_arr)
stdlib.free(mod_arr)
stdlib.free(worklist)
stdlib.free(processed)
stdlib.free(mod_buf)
return -1
stdlib.free(mod_buf)
stdlib.free(sha1_arr)
stdlib.free(mod_arr)