GVSDS/Viper_VRTE
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
785e82551895dfe755dbecc9d579a4f9f59583f2
Viper_VRTE/gzip.vp
T
TermiNexus 728d090815 feat(vqt6/vipermath/vipersimd/w32/requests): 自绘 Canvas/Graph3D/ChartView 与 Qt6 Charts、新增 gzip 与 torch-lite 模块, 修包装类悬垂指针与中文路径/exp/大文件下载 
- vqt6: 包装类统一补 __new__ 堆分配, 修"创建函数栈帧 alloca"被存进字段/跨帧返回后 _ptr 悬垂(传给 QLayout::addWidget 等即崩溃); 菜单类按依赖顺序前移, menuBar()/statusBar() 改返回裸指针; 接入 Qt6 Charts 并新增 _bridge_canvas(Canvas 手绘板 / Graph3D 软件 3D 网络图 / ChartView 折线图); 新增自绘 QRadioButton(圆环+内点, 过渡动画)、剪贴板、应用级字体族 API
- 新增模块: gzip.vp(RFC1952/1951 解码器)、torch/(torch-lite 稠密张量 + conv/pool 前反向)
- vipersimd: 新增 scalar_fma/add/mul/sqrt_array 未向量化对照内核(与对应 SIMD 版逐位一致)
- w32: 新增 UTF-8/UTF-16 路径转换, CreateFile/CreateDirectory 走 W 系列, 修中文路径被按 GBK 误读; read_all 参数 max_count -> buf_size 并文档化"容量含终止符"契约
- requests: https 响应缓冲按需翻倍扩容(上限 256MB), 修 MB 级正文(如 MNIST .gz)被 4KB 静默截断
- vipermath: 重写 exp 为范围归约 + 短泰勒, 修负值发散(x=-30 得 3.7e10)与 NaN 触发 2^31 次缩放导致的卡死
- builtins: 新增 sizet/uzt 别名; stdarg: va_arg 参数 CType -> CPtr
- chore: 删除 vendored jpeg change.log
2026-09-10 23:09:05 +08:00

664 lines
20 KiB
Plaintext
Raw Blame History

# -*- coding: utf-8 -*-
# ============================================================
# gzip - 单文件 gzip 解码器(RFC 1952 + RFC 1951 deflate)
#
# OOP 范式对齐 zipfile: 数据由 GzipFile 对象持有并负责释放;
# 内部位流/Huffman 助手为模块级私有函数。
#
# 用法:
# gz: gzip.GzipFile | t.CPtr = gzip.GzipFile("train-images-idx3-ubyte.gz")
# if gz is None or gz.ok == 0: ...
# buf: bytes = gz.read_all() # 内部 malloc, read_all 返回即解压完成
# n: t.CSizeT = gz.dst_len # 原始长度(由对象持有, close 时释放)
# gz.close()
#
# 覆盖 stored/fixed/dynamic 三种 deflate 块; 不解 CRC。
# ============================================================
import t, c
import stdlib
import string
import w32.fileio as fileio
from stdint import *
# ============================================================
# GzipFile
# ============================================================
class GzipFile:
ok: t.CInt # 1 = 可正常使用
err: t.CInt # 错误码(0 无, <0 出错)
src: BYTEPTR # 整个 .gz 文件的内存副本(构造时读入, close 释放)
src_len: t.CSizeT
dst: BYTEPTR # 解压输出缓冲(read_all 生成, close 释放)
dst_len: t.CSizeT # 解压结果长度
_need_close_src: t.CInt # 是否持有 src(读失败时为 0)
def __init__(self, path: str):
self.ok = 0
self.err = 0
self.src = None
self.src_len = 0
self.dst = None
self.dst_len = 0
self._need_close_src = 0
if path is None:
self.err = -1
return
f: fileio.File | t.CPtr = fileio.File(path, fileio.MODE.R)
if f.closed:
self.err = -2 # 打不开文件
return
sz: LONGLONG = f.size()
if sz < 18:
f.close()
self.err = -3 # 文件过小
return
self.src_len = t.CSizeT(sz)
# read_all 契约: 最多读 max_count-1 字节并补 NUL; 故按 src_len+1 申请与请求
buf: BYTEPTR = stdlib.malloc(self.src_len + 1)
if buf is None:
f.close()
self.err = -4
return
got: LONG = f.read_all(buf, ULONG(self.src_len + 1))
f.close()
if got <= 0 or t.CSizeT(got) != self.src_len:
stdlib.free(buf)
self.err = -5 # 读取失败或被截断
return
# gzip 魔数校验
ub: t.CUnsignedChar | t.CPtr = buf
if ub[0] != 0x1F or ub[1] != 0x8B or ub[2] != 8:
stdlib.free(buf)
self.err = -6
return
self.src = buf
self._need_close_src = 1
self.ok = 1
def read_all(self) -> BYTEPTR:
"""解压整个成员。返回内部持有的输出缓冲(或 None); 长度见 self.dst_len。"""
if self.ok == 0 or self.src is None:
return None
if self.dst is not None:
return self.dst # 已解压过
want: t.CUInt32T = gzip_isize(self.src, self.src_len)
cap: t.CSizeT = t.CSizeT(want)
if cap == 0:
cap = self.src_len * 64 # 无 ISIZE 时保守初值
if cap < 1024:
cap = 1024
# 保险起见: ISIZE 与实际不符时报"目标缓冲不足", 就翻倍重试(正常情况下一次成功)
out: BYTEPTR = None
n: LONG = 0
while True:
out = stdlib.malloc(cap)
if out is None:
self.err = -7
return None
n = _gzip_decode(self.src, self.src_len, out, cap)
if n == -9 or n == -18 or n == -23:
stdlib.free(out)
out = None
if cap >= 1073741824:
self.err = t.CInt(n)
return None
cap = cap * 2
continue
break
if n <= 0:
stdlib.free(out)
self.err = t.CInt(n) # n 为负错误码, 原样保存
return None
self.dst = out
self.dst_len = t.CSizeT(n)
return out
def close(self):
if self._need_close_src != 0 and self.src is not None:
stdlib.free(self.src)
if self.dst is not None:
stdlib.free(self.dst)
self.src = None
self.dst = None
self.src_len = 0
self.dst_len = 0
self._need_close_src = 0
self.ok = 0
# ============================================================
# 公共访问器(跨模块读字段用函数, 避免 stub 字段访问陷阱)
# ============================================================
def gzip_open(path: str) -> GzipFile | t.CPtr:
return GzipFile(path)
def gzip_ok(gz: GzipFile | t.CPtr) -> t.CInt:
if gz is None:
return 0
return gz.ok
def gzip_err(gz: GzipFile | t.CPtr) -> t.CInt:
if gz is None:
return -1
return gz.err
def gzip_dst_len(gz: GzipFile | t.CPtr) -> t.CSizeT:
if gz is None:
return 0
return gz.dst_len
def gzip_read(gz: GzipFile | t.CPtr) -> BYTEPTR:
if gz is None:
return None
return gz.read_all()
def gzip_close(gz: GzipFile | t.CPtr):
if gz is not None:
gz.close()
def gzip_decode_bytes(src: t.CPtr, src_len: t.CSizeT,
dst: t.CPtr, dst_cap: t.CSizeT) -> LONG:
"""原始缓冲到缓冲的解压(不读文件), 供下载流程直接使用"""
return _gzip_decode(src, src_len, dst, dst_cap)
# ============================================================
# 位流状态(模块级, 单线程; 每次 _gzip_decode 重置)
# ============================================================
_g_bits: t.CUInt64T = 0
_g_bitcnt: t.CInt = 0
_g_pos: t.CSizeT = 0
_g_src: t.CPtr = None
_g_src_len: t.CSizeT = 0
def gzip_isize(src: t.CPtr, src_len: t.CSizeT) -> t.CUInt32T:
"""gzip 尾部 ISIZE(小端 u32), 即原始长度提示"""
if src is None or src_len < 4:
return 0
ub: t.CUnsignedChar | t.CPtr = src
p: t.CSizeT = src_len - 4
# 小端 u32 拼装(64 位中间量后收窄)
v64: t.CUInt64T = t.CUInt64T(ub[p]) \
+ t.CUInt64T(ub[p + 1]) * 256 \
+ t.CUInt64T(ub[p + 2]) * 65536 \
+ t.CUInt64T(ub[p + 3]) * 16777216
return t.CUInt32T(v64)
def _reset(src: t.CPtr, src_len: t.CSizeT):
global _g_bits, _g_bitcnt, _g_pos, _g_src, _g_src_len
_g_bits = 0
_g_bitcnt = 0
_g_pos = 0
_g_src = src
_g_src_len = src_len
def _fill(n: t.CInt) -> t.CInt:
global _g_bits, _g_bitcnt, _g_pos, _g_src, _g_src_len
sb: t.CUnsignedChar | t.CPtr = _g_src
while _g_bitcnt < n:
if _g_pos >= _g_src_len:
return -1
_g_bits = _g_bits | (t.CUInt64T(sb[_g_pos]) << _g_bitcnt)
_g_pos += 1
_g_bitcnt += 8
return 0
def _bits(n: t.CInt) -> t.CInt:
global _g_bits, _g_bitcnt
if _fill(n) != 0:
return -1
v: t.CUInt64T = _g_bits & ((t.CUInt64T(1) << n) - 1)
_g_bits = _g_bits >> n
_g_bitcnt -= n
return t.CInt(v)
def _byte() -> t.CInt:
return _bits(8)
def _align():
global _g_bits, _g_bitcnt
_g_bits = 0
_g_bitcnt = 0
def _decode(count: t.CInt | t.CPtr, first: t.CInt | t.CPtr,
off: t.CInt | t.CPtr, sorted: t.CInt | t.CPtr) -> t.CInt:
code: t.CInt = 0
ln: t.CInt = 1
while ln <= 15:
b: t.CInt = _bits(1)
if b < 0:
return -1
code = (code << 1) | b
if count[ln] != 0:
d: t.CInt = code - first[ln]
if d >= 0 and d < count[ln]:
return sorted[off[ln] + d]
ln += 1
return -1
def _build(first: t.CInt | t.CPtr, off: t.CInt | t.CPtr,
count: t.CInt | t.CPtr) -> t.CInt:
anylen: t.CInt = 0
l1: t.CInt = 1
while l1 <= 15:
if count[l1] != 0:
anylen = 1
l1 += 1
if anylen == 0:
return -1
left: t.CInt = 1
l2: t.CInt = 1
while l2 <= 15:
left = left << 1
if count[l2] > left:
return -1
left = left - count[l2]
l2 += 1
code: t.CInt = 0
l3: t.CInt = 1
while l3 <= 15:
code = (code + count[l3 - 1]) << 1
first[l3] = code
l3 += 1
offsum: t.CInt = 0
l4: t.CInt = 1
while l4 <= 15:
off[l4] = offsum
offsum += count[l4]
l4 += 1
return 0
def _collect(len_arr: t.CInt | t.CPtr, cnt: t.CInt | t.CPtr,
sorted_arr: t.CInt | t.CPtr, nsym: t.CInt,
pos: t.CInt | t.CPtr):
string.memset(cnt, 0, 17 * 4)
s1: t.CInt = 0
while s1 < nsym:
if len_arr[s1] != 0:
cnt[len_arr[s1]] += 1
s1 += 1
run: t.CInt = 0
l5: t.CInt = 1
while l5 <= 15:
pos[l5] = run
run += cnt[l5]
l5 += 1
s2: t.CInt = 0
while s2 < nsym:
if len_arr[s2] != 0:
sorted_arr[pos[len_arr[s2]]] = s2
pos[len_arr[s2]] += 1
s2 += 1
def _len_extra(code: t.CInt) -> t.CInt:
if code == 285:
return 0
if code >= 281:
return 5
if code >= 277:
return 4
if code >= 273:
return 3
if code >= 269:
return 2
if code >= 265:
return 1
return 0
def _len_base(code: t.CInt) -> t.CInt:
if code == 285:
return 258
if code >= 281:
return 131 + ((code - 281) << 5)
if code >= 277:
return 67 + ((code - 277) << 4)
if code >= 273:
return 35 + ((code - 273) << 3)
if code >= 269:
return 19 + ((code - 269) << 2)
if code >= 265:
return 11 + ((code - 265) << 1)
return 3 + (code - 257)
def _dist_extra(code: t.CInt) -> t.CInt:
if code < 4:
return 0
return (code - 4) // 2 + 1
def _dist_base(code: t.CInt) -> t.CInt:
if code < 4:
return code + 1
e: t.CInt = (code - 4) // 2 + 1
f: t.CInt = (1 << (e + 1)) + 1
if code % 2 == 0:
return f
return f + (1 << e)
def _cl_order_set(arr: t.CInt | t.CPtr, idx: t.CInt, val: t.CInt):
arr[idx] = val
# ============================================================
# 核心解压(模块级私有)
# ============================================================
def _gzip_decode(src: t.CPtr, src_len: t.CSizeT,
dst: t.CPtr, dst_cap: t.CSizeT) -> LONG:
if src is None or dst is None or src_len < 18 or dst_cap == 0:
return -1
ub: t.CUnsignedChar | t.CPtr = src
if ub[0] != 0x1F or ub[1] != 0x8B or ub[2] != 8:
return -2
_reset(src, src_len)
flg: t.CInt = t.CInt(ub[3])
hdr_len: t.CSizeT = 10
if flg & 0x04 != 0:
xlen: t.CSizeT = t.CSizeT(ub[hdr_len]) | (t.CSizeT(ub[hdr_len + 1]) << 8)
hdr_len += 2 + xlen
if flg & 0x08 != 0:
while hdr_len < src_len and ub[hdr_len] != 0:
hdr_len += 1
hdr_len += 1
if flg & 0x10 != 0:
while hdr_len < src_len and ub[hdr_len] != 0:
hdr_len += 1
hdr_len += 1
if flg & 0x02 != 0:
hdr_len += 2
if hdr_len >= src_len:
return -3
global _g_pos
_g_pos = hdr_len
dout: t.CUnsignedChar | t.CPtr = dst
wpos: t.CSizeT = 0
lit_len: t.CInt | t.CPtr = stdlib.malloc(288 * 4)
lit_cnt: t.CInt | t.CPtr = stdlib.malloc(17 * 4)
lit_first: t.CInt | t.CPtr = stdlib.malloc(17 * 4)
lit_off: t.CInt | t.CPtr = stdlib.malloc(17 * 4)
lit_sorted: t.CInt | t.CPtr = stdlib.malloc(288 * 4)
dst_len: t.CInt | t.CPtr = stdlib.malloc(32 * 4)
dst_cnt: t.CInt | t.CPtr = stdlib.malloc(17 * 4)
dst_first: t.CInt | t.CPtr = stdlib.malloc(17 * 4)
dst_off: t.CInt | t.CPtr = stdlib.malloc(17 * 4)
dst_sorted: t.CInt | t.CPtr = stdlib.malloc(32 * 4)
cl_len: t.CInt | t.CPtr = stdlib.malloc(19 * 4)
cl_order: t.CInt | t.CPtr = stdlib.malloc(19 * 4)
cl_cnt: t.CInt | t.CPtr = stdlib.malloc(17 * 4)
cl_first: t.CInt | t.CPtr = stdlib.malloc(17 * 4)
cl_off: t.CInt | t.CPtr = stdlib.malloc(17 * 4)
cl_sorted: t.CInt | t.CPtr = stdlib.malloc(19 * 4)
pos_tmp: t.CInt | t.CPtr = stdlib.malloc(17 * 4)
if lit_len is None or lit_cnt is None or lit_first is None \
or lit_off is None or lit_sorted is None or dst_len is None \
or dst_cnt is None or dst_first is None or dst_off is None \
or dst_sorted is None or cl_len is None or cl_order is None \
or cl_cnt is None or cl_first is None or cl_off is None \
or cl_sorted is None or pos_tmp is None:
return -4
# 全部表缓冲清零(初始 malloc 未清零; 防个别路径读到垃圾表导致提前/错乱收尾)
string.memset(lit_len, 0, 288 * 4)
string.memset(lit_cnt, 0, 17 * 4)
string.memset(lit_first, 0, 17 * 4)
string.memset(lit_off, 0, 17 * 4)
string.memset(lit_sorted, 0, 288 * 4)
string.memset(dst_len, 0, 32 * 4)
string.memset(dst_cnt, 0, 17 * 4)
string.memset(dst_first, 0, 17 * 4)
string.memset(dst_off, 0, 17 * 4)
string.memset(dst_sorted, 0, 32 * 4)
string.memset(cl_len, 0, 19 * 4)
string.memset(cl_order, 0, 19 * 4)
string.memset(cl_cnt, 0, 17 * 4)
string.memset(cl_first, 0, 17 * 4)
string.memset(cl_off, 0, 17 * 4)
string.memset(cl_sorted, 0, 19 * 4)
string.memset(pos_tmp, 0, 17 * 4)
_cl_order_set(cl_order, 0, 16)
_cl_order_set(cl_order, 1, 17)
_cl_order_set(cl_order, 2, 18)
_cl_order_set(cl_order, 3, 0)
_cl_order_set(cl_order, 4, 8)
_cl_order_set(cl_order, 5, 7)
_cl_order_set(cl_order, 6, 9)
_cl_order_set(cl_order, 7, 6)
_cl_order_set(cl_order, 8, 10)
_cl_order_set(cl_order, 9, 5)
_cl_order_set(cl_order, 10, 11)
_cl_order_set(cl_order, 11, 4)
_cl_order_set(cl_order, 12, 12)
_cl_order_set(cl_order, 13, 3)
_cl_order_set(cl_order, 14, 13)
_cl_order_set(cl_order, 15, 2)
_cl_order_set(cl_order, 16, 14)
_cl_order_set(cl_order, 17, 1)
_cl_order_set(cl_order, 18, 15)
err: t.CInt = 0
final: t.CInt = 0
while final == 0 and err == 0:
bfinal: t.CInt = _bits(1)
btype: t.CInt = _bits(2)
if bfinal < 0 or btype < 0:
err = -5
break
final = bfinal
if btype == 0:
_align()
slen: t.CInt = _bits(16)
snlen: t.CInt = _bits(16)
if slen < 0 or snlen < 0:
err = -6
break
if (slen ^ snlen) != 0xFFFF:
err = -7
break
i: t.CSizeT = 0
while i < t.CSizeT(slen):
ch: t.CInt = _byte()
if ch < 0:
err = -8
break
if wpos >= dst_cap:
err = -9
break
dout[wpos] = t.CUnsignedChar(ch)
wpos += 1
i += 1
if err != 0:
break
elif btype == 1:
string.memset(lit_cnt, 0, 17 * 4)
j: t.CInt = 0
while j < 144:
lit_len[j] = 8
j += 1
while j < 256:
lit_len[j] = 9
j += 1
while j < 280:
lit_len[j] = 7
j += 1
while j < 288:
lit_len[j] = 8
j += 1
_collect(lit_len, lit_cnt, lit_sorted, 288, pos_tmp)
if _build(lit_first, lit_off, lit_cnt) != 0:
err = -10
break
else:
hlit: t.CInt = _bits(5) + 257
hdist: t.CInt = _bits(5) + 1
hclen: t.CInt = _bits(4) + 4
if hlit < 0 or hdist < 0 or hclen < 0:
err = -11
break
string.memset(cl_len, 0, 19 * 4)
k: t.CInt = 0
while k < hclen:
vv: t.CInt = _bits(3)
if vv < 0:
err = -12
break
cl_len[cl_order[k]] = vv
k += 1
if err != 0:
break
_collect(cl_len, cl_cnt, cl_sorted, 19, pos_tmp)
if _build(cl_first, cl_off, cl_cnt) != 0:
err = -13
break
string.memset(lit_len, 0, 288 * 4)
string.memset(dst_len, 0, 32 * 4)
sym_i: t.CInt = 0
while sym_i < hlit + hdist:
sym: t.CInt = _decode(cl_cnt, cl_first, cl_off, cl_sorted)
if sym < 0:
err = -14
break
if sym < 16:
if sym_i < hlit:
lit_len[sym_i] = sym
else:
dst_len[sym_i - hlit] = sym
sym_i += 1
elif sym == 16:
prev: t.CInt = 0
if sym_i > 0:
if sym_i - 1 < hlit:
prev = lit_len[sym_i - 1]
else:
prev = dst_len[sym_i - hlit - 1]
rp: t.CInt = _bits(2) + 3
r2: t.CInt = 0
while r2 < rp and sym_i < hlit + hdist:
if sym_i < hlit:
lit_len[sym_i] = prev
else:
dst_len[sym_i - hlit] = prev
sym_i += 1
r2 += 1
elif sym == 17:
rp2: t.CInt = _bits(3) + 3
r3: t.CInt = 0
while r3 < rp2 and sym_i < hlit + hdist:
if sym_i < hlit:
lit_len[sym_i] = 0
else:
dst_len[sym_i - hlit] = 0
sym_i += 1
r3 += 1
else:
rp3: t.CInt = _bits(7) + 11
r4: t.CInt = 0
while r4 < rp3 and sym_i < hlit + hdist:
if sym_i < hlit:
lit_len[sym_i] = 0
else:
dst_len[sym_i - hlit] = 0
sym_i += 1
r4 += 1
if err != 0:
break
_collect(lit_len, lit_cnt, lit_sorted, hlit, pos_tmp)
if _build(lit_first, lit_off, lit_cnt) != 0:
err = -15
break
_collect(dst_len, dst_cnt, dst_sorted, hdist, pos_tmp)
if _build(dst_first, dst_off, dst_cnt) != 0:
err = -16
break
if err != 0:
break
run: t.CInt = 1
while run == 1 and err == 0:
sym: t.CInt = _decode(lit_cnt, lit_first, lit_off, lit_sorted)
if sym < 0:
err = -17
break
if sym < 256:
if wpos >= dst_cap:
err = -18
break
dout[wpos] = t.CUnsignedChar(sym)
wpos += 1
elif sym == 256:
run = 0
else:
le: t.CInt = _len_extra(sym)
ev: t.CInt = _bits(le)
if ev < 0:
err = -19
break
length: t.CSizeT = t.CSizeT(_len_base(sym) + ev)
ds: t.CInt = _decode(dst_cnt, dst_first, dst_off, dst_sorted)
if ds < 0:
err = -20
break
de: t.CInt = _dist_extra(ds)
dv: t.CInt = _bits(de)
if dv < 0:
err = -21
break
dist: t.CSizeT = t.CSizeT(_dist_base(ds) + dv)
if dist == 0 or dist > wpos:
err = -22
break
ci: t.CSizeT = 0
while ci < length:
if wpos >= dst_cap:
err = -23
break
dout[wpos] = dout[wpos - dist]
wpos += 1
ci += 1
if err != 0:
break
stdlib.free(lit_len)
stdlib.free(lit_cnt)
stdlib.free(lit_first)
stdlib.free(lit_off)
stdlib.free(lit_sorted)
stdlib.free(dst_len)
stdlib.free(dst_cnt)
stdlib.free(dst_first)
stdlib.free(dst_off)
stdlib.free(dst_sorted)
stdlib.free(cl_len)
stdlib.free(cl_order)
stdlib.free(cl_cnt)
stdlib.free(cl_first)
stdlib.free(cl_off)
stdlib.free(cl_sorted)
stdlib.free(pos_tmp)
if err != 0:
return LONG(err) # err 本身为负错误码, 不能再取负(否则被上层当成"长度>0 成功")
return LONG(wpos)
Reference in new issue View Git Blame Copy Permalink