GVSDS/TransPyC
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

snapshot before regression test

Browse Source
This commit is contained in:
t
2026-07-18 19:25:40 +08:00
commit 796222a300
2295 changed files with 206453 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
includes/zlib/__init__.py Normal file
View File

@@ -0,0 +1 @@
from .pyzlib import *

695
includes/zlib/pyzlib.py Normal file
View File

@@ -0,0 +1,695 @@
from stdint import *
import zlib.zdeflate as zdeflate
import zlib.zinflate as zinflate
import zlib.zchecksum as zchecksum
import zlib.zdef as zdef
import zlib.zhuff as zhuff
import stdlib
import string
import memhub
import t, c
# ============================================================
# Constants - matching Python zlib module names and values
# ============================================================
# Compression levels
Z_NO_COMPRESSION: t.CDefine = 0
Z_BEST_SPEED: t.CDefine = 1
Z_BEST_COMPRESSION: t.CDefine = 9
Z_DEFAULT_COMPRESSION: t.CDefine = (-1)
# Compression methods
DEFLATED: t.CDefine = 8
# Flush modes
Z_NO_FLUSH: t.CDefine = 0
Z_PARTIAL_FLUSH: t.CDefine = 1
Z_SYNC_FLUSH: t.CDefine = 2
Z_FULL_FLUSH: t.CDefine = 3
Z_FINISH: t.CDefine = 4
Z_BLOCK: t.CDefine = 5
Z_TREES: t.CDefine = 6
# Strategies
Z_DEFAULT_STRATEGY: t.CDefine = 0
Z_FILTERED: t.CDefine = 1
Z_HUFFMAN_ONLY: t.CDefine = 2
Z_RLE: t.CDefine = 3
Z_FIXED: t.CDefine = 4
# Return codes
Z_OK: t.CDefine = 0
Z_STREAM_END: t.CDefine = 1
Z_NEED_DICT: t.CDefine = 2
Z_ERRNO: t.CDefine = (-1)
Z_STREAM_ERROR: t.CDefine = (-2)
Z_DATA_ERROR: t.CDefine = (-3)
Z_MEM_ERROR: t.CDefine = (-4)
Z_BUF_ERROR: t.CDefine = (-5)
Z_VERSION_ERROR: t.CDefine = (-6)
# Window / Buffer constants
MAX_WBITS: t.CDefine = 15
DEF_BUF_SIZE: t.CDefine = 16384
DEF_MEM_LEVEL: t.CDefine = 8
# Version
ZLIB_VERSION: t.CDefine = "1.3.2"
pyzlib_error_msg: t.CArray[t.CChar, 512] = ""
pyzlib_error_code_val: t.CInt = 0
# ============================================================
# Structures
# ============================================================
@t.Object
class Compress:
pool: memhub.MemManager | t.CPtr
stream: VOIDPTR
is_initialized: t.CInt
is_finished: t.CInt
level: t.CInt
method: t.CInt
wbits: t.CInt
memLevel: t.CInt
strategy: t.CInt
zdict: BYTEPTR
zdict_len: t.CSizeT
last_pos: t.CSizeT
input_buf: BYTEPTR
input_buf_len: t.CSizeT
input_buf_cap: t.CSizeT
header_written: t.CInt
def compress(self,
data: BYTEPTR, data_len: t.CSizeT,
out_len: t.CSizeT | t.CPtr) -> BYTEPTR:
if not self.is_initialized:
set_error(Z_STREAM_ERROR, "compress object not initialized")
return None
if self.is_finished:
set_error(Z_STREAM_ERROR, "compress object already finished")
return None
if not out_len:
set_error(Z_STREAM_ERROR, "out_len is None")
return None
if data_len == 0:
c.Set(c.Deref(out_len), 0)
if self.pool == None:
return BYTEPTR(calloc(1, 1))
else:
p: BYTEPTR = BYTEPTR(self.pool.alloc(1))
memset(p, 0, 1)
return p
while self.input_buf_len + data_len > self.input_buf_cap:
self.input_buf_cap *= 2
if self.pool == None:
new_buf: BYTEPTR = BYTEPTR(realloc(self.input_buf, self.input_buf_cap))
else:
new_buf: BYTEPTR = BYTEPTR(self.pool.alloc(self.input_buf_cap))
if new_buf:
memcpy(new_buf, self.input_buf, self.input_buf_len)
self.pool.free(self.input_buf)
if not new_buf:
set_error(Z_MEM_ERROR, "out of memory")
c.Set(c.Deref(out_len), 0)
if self.pool == None:
return BYTEPTR(calloc(1, 1))
else:
p2: BYTEPTR = BYTEPTR(self.pool.alloc(1))
memset(p2, 0, 1)
return p2
self.input_buf = new_buf
memcpy(self.input_buf + self.input_buf_len, data, data_len)
self.input_buf_len += data_len
c.Set(c.Deref(out_len), 0)
if self.pool == None:
return BYTEPTR(calloc(1, 1))
else:
p3: BYTEPTR = BYTEPTR(self.pool.alloc(1))
memset(p3, 0, 1)
return p3
def flush(self, mode: t.CInt, out_len: t.CSizeT | t.CPtr) -> BYTEPTR:
if not self.is_initialized:
set_error(Z_STREAM_ERROR, "compress object not initialized")
return None
if not out_len:
set_error(Z_STREAM_ERROR, "out_len is None")
return None
s: zdeflate.zdeflate_stream | t.CPtr = self.stream # 隐式类型转换
if mode == Z_FINISH:
if not self.header_written:
self.header_written = 1
input_data: BYTEPTR = self.input_buf
input_len: t.CSizeT = self.input_buf_len
if input_len == 0:
s.writer.write_bits(1, 1)
s.writer.write_bits(1, 2)
lit_tree = zhuff.zhuff_tree()
lit_tree.build_fixed_lit_tree()
lit_tree.encode_symbol(256, c.Addr(s.writer))
else:
s.adler = zchecksum.zchecksum_adler32(input_data, input_len, 1)
s.compress_block(input_data, input_len, 1)
s.is_finished = 1
if s.wbits >= 0:
s.writer.align()
adler: t.CUInt32T = s.adler
s.writer.buf[s.writer.byte_pos + 0] = (adler >> 24) & 0xFF
s.writer.buf[s.writer.byte_pos + 1] = (adler >> 16) & 0xFF
s.writer.buf[s.writer.byte_pos + 2] = (adler >> 8) & 0xFF
s.writer.buf[s.writer.byte_pos + 3] = (adler >> 0) & 0xFF
s.writer.byte_pos += 4
self.is_finished = 1
total: t.CSizeT = s.writer.total()
if self.pool == None:
result: BYTEPTR = BYTEPTR(malloc(total))
else:
result: BYTEPTR = BYTEPTR(self.pool.alloc(total))
if result: memcpy(result, s.writer.buf, total)
c.Set(c.Deref(out_len), total)
if self.pool == None:
free(self.input_buf)
else:
self.pool.free(self.input_buf)
self.input_buf = None
self.input_buf_len = 0
self.input_buf_cap = 0
return result
elif mode == Z_SYNC_FLUSH or mode == Z_FULL_FLUSH:
input_data: BYTEPTR = self.input_buf
input_len: t.CSizeT = self.input_buf_len
if input_len > 0:
s.adler = zchecksum.zchecksum_adler32(input_data, input_len, s.adler)
s.compress_block(input_data, input_len, 0)
self.input_buf_len = 0
s.writer.write_bits(0, 1)
s.writer.write_bits(0, 2)
s.writer.align()
prev_pos: t.CSizeT = self.last_pos
total: t.CSizeT = s.writer.total()
delta: t.CSizeT = total - prev_pos
self.last_pos = s.writer.byte_pos
if delta == 0:
c.Set(c.Deref(out_len), 0)
if self.pool == None:
return BYTEPTR(calloc(1, 1))
else:
p4: BYTEPTR = BYTEPTR(self.pool.alloc(1))
memset(p4, 0, 1)
return p4
if self.pool == None:
result: BYTEPTR = BYTEPTR(malloc(delta))
else:
result: BYTEPTR = BYTEPTR(self.pool.alloc(delta))
if result: memcpy(result, s.writer.buf + prev_pos, delta)
c.Set(c.Deref(out_len), delta)
return result
c.Set(c.Deref(out_len), 0)
if self.pool == None:
return BYTEPTR(calloc(1, 1))
else:
p5: BYTEPTR = BYTEPTR(self.pool.alloc(1))
memset(p5, 0, 1)
return p5
def copy(self) -> Compress | t.CPtr:
if not self.is_initialized:
set_error(Z_STREAM_ERROR, "compress object not initialized")
return None
if self.pool == None:
copy_obj: Compress | t.CPtr = calloc(1, Compress.__sizeof__())
else:
copy_obj: Compress | t.CPtr = self.pool.alloc(Compress.__sizeof__())
memset(copy_obj, 0, Compress.__sizeof__())
if not copy_obj:
set_error(Z_MEM_ERROR, "out of memory")
return None
copy_obj.pool = self.pool
s: zdeflate.zdeflate_stream | t.CPtr = self.stream # 隐式类型转换
copy_obj.stream = s.copy()
if not copy_obj.stream:
if self.pool == None:
free(copy_obj)
else:
self.pool.free(copy_obj)
set_error(Z_MEM_ERROR, "out of memory")
return None
copy_obj.is_initialized = 1
copy_obj.is_finished = self.is_finished
copy_obj.level = self.level
copy_obj.method = self.method
copy_obj.wbits = self.wbits
copy_obj.memLevel = self.memLevel
copy_obj.strategy = self.strategy
copy_obj.last_pos = self.last_pos
copy_obj.header_written = self.header_written
if self.input_buf and self.input_buf_len > 0:
if self.pool == None:
copy_obj.input_buf = BYTEPTR(calloc(1, self.input_buf_cap))
else:
copy_obj.input_buf = BYTEPTR(self.pool.alloc(self.input_buf_cap))
memset(copy_obj.input_buf, 0, self.input_buf_cap)
if copy_obj.input_buf:
memcpy(copy_obj.input_buf, self.input_buf, self.input_buf_len)
copy_obj.input_buf_cap = self.input_buf_cap
copy_obj.input_buf_len = self.input_buf_len
else:
if self.pool == None:
copy_obj.input_buf = BYTEPTR(calloc(1, 256))
else:
copy_obj.input_buf = BYTEPTR(self.pool.alloc(256))
memset(copy_obj.input_buf, 0, 256)
copy_obj.input_buf_cap = 256
copy_obj.input_buf_len = 0
if self.zdict and self.zdict_len > 0:
copy_obj.zdict = clone_bytes(self.pool, self.zdict, self.zdict_len)
copy_obj.zdict_len = self.zdict_len
return copy_obj
def delete(self):
if self.is_initialized and self.stream:
s: zdeflate.zdeflate_stream | t.CPtr = self.stream # 隐式类型转换
s.destroy()
if self.pool == None:
free(self.zdict)
free(self.input_buf)
free(self)
else:
self.pool.free(self.zdict)
self.pool.free(self.input_buf)
self.pool.free(self)
def __del__(self):
self.delete()
@t.Object
class Decompress:
pool: memhub.MemManager | t.CPtr
stream: VOIDPTR
is_initialized: t.CInt
eof: t.CInt
wbits: t.CInt
zdict: BYTEPTR
zdict_len: t.CSizeT
_unused_data: BYTEPTR
_unused_data_len: t.CSizeT
_unused_data_cap: t.CSizeT
_unconsumed_tail: BYTEPTR
_unconsumed_tail_len: t.CSizeT
_unconsumed_tail_cap: t.CSizeT
input_buf: BYTEPTR
input_buf_len: t.CSizeT
input_buf_cap: t.CSizeT
def decompress(self, data: BYTEPTR, data_len: t.CSizeT,
max_length: t.CSizeT, out_len: t.CSizeT | t.CPtr) -> BYTEPTR:
if not self.is_initialized:
set_error(Z_STREAM_ERROR, "decompress object not initialized")
return None
if not out_len:
set_error(Z_STREAM_ERROR, "out_len is None")
return None
if self.eof:
if data and data_len > 0:
append_bytes(self.pool, c.Addr(self._unused_data), c.Addr(self._unused_data_len),
c.Addr(self._unused_data_cap), data, data_len)
c.Set(c.Deref(out_len), 0)
if self.pool == None:
return BYTEPTR(calloc(1, 1))
else:
p6: BYTEPTR = BYTEPTR(self.pool.alloc(1))
memset(p6, 0, 1)
return p6
self._unconsumed_tail_len = 0
if data and data_len > 0:
append_bytes(self.pool, c.Addr(self.input_buf), c.Addr(self.input_buf_len),
c.Addr(self.input_buf_cap), data, data_len)
if self.input_buf_len == 0:
c.Set(c.Deref(out_len), 0)
if self.pool == None:
return BYTEPTR(calloc(1, 1))
else:
p7: BYTEPTR = BYTEPTR(self.pool.alloc(1))
memset(p7, 0, 1)
return p7
s: zinflate.zinflate_stream | t.CPtr = zinflate.zinflate_create(self.pool, self.wbits)
if not s:
set_error(Z_MEM_ERROR, "out of memory")
return None
if self.zdict and self.zdict_len > 0:
s.set_dictionary(self.zdict, self.zdict_len)
out: t.CUInt8T | t.CPtr = None
err: t.CInt = s.decompress(self.input_buf, self.input_buf_len,
max_length, c.Addr(out), out_len)
if err != 0:
s.destroy()
set_error(Z_DATA_ERROR, "decompression failed")
return None
if s.is_finished:
self.eof = 1
if s.input_pos < s.input_len:
append_bytes(self.pool, c.Addr(self._unused_data), c.Addr(self._unused_data_len),
c.Addr(self._unused_data_cap),
s.input_data + s.input_pos,
s.input_len - s.input_pos)
self.input_buf_len = 0
else:
consumed: t.CSizeT = s.input_pos
if consumed < self.input_buf_len:
remaining: t.CSizeT = self.input_buf_len - consumed
memmove(self.input_buf, self.input_buf + consumed, remaining)
self.input_buf_len = remaining
else:
self.input_buf_len = 0
s.destroy()
return out
def flush(self, length: t.CSizeT, out_len: t.CSizeT | t.CPtr) -> BYTEPTR:
if not self.is_initialized:
set_error(Z_STREAM_ERROR, "decompress object not initialized")
return None
if not out_len:
set_error(Z_STREAM_ERROR, "out_len is None")
return None
c.Set(c.Deref(out_len), 0)
if self.pool == None:
return BYTEPTR(calloc(1, 1))
else:
p8: BYTEPTR = BYTEPTR(self.pool.alloc(1))
memset(p8, 0, 1)
return p8
def copy(self) -> Decompress | t.CPtr:
if not self.is_initialized:
set_error(Z_STREAM_ERROR, "decompress object not initialized")
return None
if self.pool == None:
copy_obj: Decompress | t.CPtr = calloc(1, Decompress.__sizeof__())
else:
copy_obj: Decompress | t.CPtr = self.pool.alloc(Decompress.__sizeof__())
memset(copy_obj, 0, Decompress.__sizeof__())
if not copy_obj:
set_error(Z_MEM_ERROR, "out of memory")
return None
copy_obj.pool = self.pool
copy_obj.is_initialized = 1
copy_obj.eof = self.eof
copy_obj.wbits = self.wbits
if self.zdict and self.zdict_len > 0:
copy_obj.zdict = clone_bytes(self.pool, self.zdict, self.zdict_len)
copy_obj.zdict_len = self.zdict_len
if self._unused_data and self._unused_data_len > 0:
copy_obj._unused_data = clone_bytes(self.pool, self._unused_data, self._unused_data_len)
copy_obj._unused_data_len = self._unused_data_len
copy_obj._unused_data_cap = self._unused_data_len
if self._unconsumed_tail and self._unconsumed_tail_len > 0:
copy_obj._unconsumed_tail = clone_bytes(self.pool, self._unconsumed_tail, self._unconsumed_tail_len)
copy_obj._unconsumed_tail_len = self._unconsumed_tail_len
copy_obj._unconsumed_tail_cap = self._unconsumed_tail_len
if self.input_buf and self.input_buf_len > 0:
copy_obj.input_buf = clone_bytes(self.pool, self.input_buf, self.input_buf_len)
copy_obj.input_buf_len = self.input_buf_len
copy_obj.input_buf_cap = self.input_buf_cap
else:
if self.pool == None:
copy_obj.input_buf = BYTEPTR(calloc(1, 256))
else:
copy_obj.input_buf = BYTEPTR(self.pool.alloc(256))
memset(copy_obj.input_buf, 0, 256)
copy_obj.input_buf_cap = 256
copy_obj.input_buf_len = 0
return copy_obj
def delete(self):
if self.pool == None:
free(self.zdict)
free(self._unused_data)
free(self._unconsumed_tail)
free(self.input_buf)
free(self)
else:
self.pool.free(self.zdict)
self.pool.free(self._unused_data)
self.pool.free(self._unconsumed_tail)
self.pool.free(self.input_buf)
self.pool.free(self)
def unused_data(self, length: t.CSizeT | t.CPtr) -> BYTEPTR:
#if not self:
# if length:
# c.Set(c.Deref(length), 0)
# return None
if length:
c.Set(c.Deref(length), self._unused_data_len)
return self._unused_data
def unconsumed_tail(self, length: t.CSizeT | t.CPtr) -> BYTEPTR:
#if not self:
# if length:
# c.Set(c.Deref(length), 0)
# return None
if length:
c.Set(c.Deref(length), self._unconsumed_tail_len)
return self._unconsumed_tail
# def eof(self) -> t.CInt:
# # if not obj: return 0
# return self.eof
def set_error(code: int, msg: str):
global pyzlib_error_code_val, pyzlib_error_msg
pyzlib_error_code_val = code
if msg:
strncpy(pyzlib_error_msg, msg, pyzlib_error_msg.__sizeof__() - 1)
pyzlib_error_msg[pyzlib_error_msg.__sizeof__() - 1] = '\0'
else:
pyzlib_error_msg[0] = '\0'
def clone_bytes(pool: memhub.MemManager | t.CPtr, src: BYTEPTR, length: t.CSizeT) -> BYTEPTR:
if not src or length == 0: return None
if pool == None:
dst: BYTEPTR = BYTEPTR(malloc(length))
else:
dst: BYTEPTR = BYTEPTR(pool.alloc(length))
if dst: memcpy(dst, src, length)
return dst
def append_bytes(pool: memhub.MemManager | t.CPtr, buf: BYTE | t.CPtr[t.CPtr], length: t.CSizeT | t.CPtr, cap: t.CSizeT | t.CPtr,
data: BYTEPTR, data_len: t.CSizeT) -> t.CInt:
if not data or data_len == 0: return 0
needed: t.CSizeT = c.Deref(length) + data_len
if needed > c.Deref(cap):
new_cap: t.CSizeT = c.Deref(cap) * 2
if new_cap < needed: new_cap = needed
if pool == None:
new_buf: BYTEPTR = BYTEPTR(realloc(c.Deref(buf), new_cap))
else:
new_buf: BYTEPTR = BYTEPTR(pool.alloc(new_cap))
if new_buf:
memcpy(new_buf, c.Deref(buf), c.Deref(length))
pool.free(c.Deref(buf))
if not new_buf: return -1
c.Set(c.Deref(buf), new_buf)
c.Set(c.Deref(cap), new_cap)
memcpy(c.Deref(buf) + c.Deref(length), data, data_len)
c.Set(c.Deref(length), c.Deref(length) + data_len)
return 0
def shrink_to_fit(pool: memhub.MemManager | t.CPtr, buf: BYTEPTR, length: t.CSizeT) -> BYTEPTR:
if length == 0:
if pool == None:
free(buf)
return BYTEPTR(calloc(1, 1))
else:
pool.free(buf)
p: BYTEPTR = BYTEPTR(pool.alloc(1))
memset(p, 0, 1)
return p
if pool == None:
result: BYTEPTR = BYTEPTR(realloc(buf, length))
return result if result else buf
else:
result: BYTEPTR = BYTEPTR(pool.alloc(length))
if result:
memcpy(result, buf, length)
pool.free(buf)
return result
return buf
def runtime_version() -> str:
return ZLIB_VERSION
def get_error() -> str:
return pyzlib_error_msg
def get_error_code() -> t.CInt:
return pyzlib_error_code_val
def clear_error():
global pyzlib_error_msg, pyzlib_error_code_val
pyzlib_error_msg[0] = '\0'
pyzlib_error_code_val = 0
def compress(pool: memhub.MemManager | t.CPtr, data: BYTEPTR, data_len: t.CSizeT,
level: t.CInt, wbits: t.CInt, out_len: t.CSizeT | t.CPtr) -> BYTEPTR:
if not data and data_len > 0:
set_error(Z_STREAM_ERROR, "data is None but data_len > 0")
return None
if not out_len:
set_error(Z_STREAM_ERROR, "out_len is None")
return None
actual_wbits: t.CInt = wbits
if wbits > MAX_WBITS:
actual_wbits = -MAX_WBITS
raw_result: UINT8PTR = zdeflate.zdeflate_one_shot(pool, data, data_len, level, actual_wbits, out_len)
if not raw_result:
set_error(Z_DATA_ERROR, "compression failed")
return None
if wbits > MAX_WBITS:
gzip_len: t.CSizeT = 10 + c.Deref(out_len) + 8
if pool == None:
gzip_buf: BYTEPTR = BYTEPTR(malloc(gzip_len))
else:
gzip_buf: BYTEPTR = BYTEPTR(pool.alloc(gzip_len))
if not gzip_buf:
if pool == None:
free(raw_result)
else:
pool.free(raw_result)
set_error(Z_MEM_ERROR, "out of memory")
return None
pos: t.CSizeT = 0
gzip_buf[pos + 0] = 0x1F
gzip_buf[pos + 1] = 0x8B
gzip_buf[pos + 2] = 0x08
gzip_buf[pos + 3] = 0x00
gzip_buf[pos + 4] = 0x00
gzip_buf[pos + 5] = 0x00
gzip_buf[pos + 6] = 0x00
gzip_buf[pos + 7] = 0x00
gzip_buf[pos + 8] = 0x00
gzip_buf[pos + 9] = 0xFF
pos += 10
memcpy(gzip_buf + pos, raw_result, c.Deref(out_len))
pos += c.Deref(out_len)
if pool == None:
free(raw_result)
else:
pool.free(raw_result)
crc: t.CUInt32T = zchecksum.zchecksum_crc32(data, data_len, 0)
gzip_buf[pos + 0] = (crc) & 0xFF
gzip_buf[pos + 1] = (crc >> 8) & 0xFF
gzip_buf[pos + 2] = (crc >> 16) & 0xFF
gzip_buf[pos + 3] = (crc >> 24) & 0xFF
gzip_buf[pos + 4] = (data_len) & 0xFF
gzip_buf[pos + 5] = (data_len >> 8) & 0xFF
gzip_buf[pos + 6] = (data_len >> 16) & 0xFF
gzip_buf[pos + 7] = (data_len >> 24) & 0xFF
pos += 8
c.Set(c.Deref(out_len), pos)
return shrink_to_fit(pool, gzip_buf, pos)
return shrink_to_fit(pool, raw_result, c.Deref(out_len))
def decompress(pool: memhub.MemManager | t.CPtr, data: BYTEPTR, data_len: t.CSizeT,
wbits: t.CInt, bufsize: t.CSizeT, out_len: t.CSizeT | t.CPtr) -> BYTEPTR:
if not data and data_len > 0:
set_error(Z_STREAM_ERROR, "data is None but data_len > 0")
return None
if not out_len:
set_error(Z_STREAM_ERROR, "out_len is None")
return None
result: UINT8PTR = zinflate.zinflate_one_shot(pool, data, data_len, wbits, bufsize, out_len)
if not result:
set_error(Z_DATA_ERROR, "decompression failed")
return None
return shrink_to_fit(pool, result, c.Deref(out_len))
def compressobj(pool: memhub.MemManager | t.CPtr, level: t.CInt, method: t.CInt, wbits: t.CInt,
memLevel: t.CInt, strategy: t.CInt,
zdict: BYTEPTR, zdict_len: t.CSizeT) -> Compress | t.CPtr:
if pool == None:
obj: Compress | t.CPtr = calloc(1, Compress.__sizeof__())
else:
obj: Compress | t.CPtr = pool.alloc(Compress.__sizeof__())
memset(obj, 0, Compress.__sizeof__())
if not obj:
set_error(Z_MEM_ERROR, "out of memory")
return None
obj.pool = pool
actual_wbits: t.CInt = wbits
if wbits > MAX_WBITS: actual_wbits = wbits - 16
s: zdeflate.zdeflate_stream | t.CPtr = zdeflate.zdeflate_create(pool, level, actual_wbits, memLevel, strategy)
if not s:
if pool == None:
free(obj)
else:
pool.free(obj)
set_error(Z_MEM_ERROR, "out of memory")
return None
obj.stream = s
obj.is_initialized = 1
obj.level = level
obj.method = method
obj.wbits = wbits
obj.memLevel = memLevel
obj.strategy = strategy
if pool == None:
obj.input_buf = BYTEPTR(calloc(1, 256))
else:
obj.input_buf = BYTEPTR(pool.alloc(256))
memset(obj.input_buf, 0, 256)
obj.input_buf_cap = 256
obj.input_buf_len = 0
obj.header_written = 0
if zdict and zdict_len > 0:
s.set_dictionary(zdict, zdict_len)
obj.zdict = clone_bytes(pool, zdict, zdict_len)
obj.zdict_len = zdict_len
return obj
def decompressobj(pool: memhub.MemManager | t.CPtr, wbits: t.CInt, zdict: BYTEPTR, zdict_len: t.CSizeT) -> Decompress | t.CPtr:
if pool == None:
obj: Decompress | t.CPtr = calloc(1, Decompress.__sizeof__())
else:
obj: Decompress | t.CPtr = pool.alloc(Decompress.__sizeof__())
memset(obj, 0, Decompress.__sizeof__())
if not obj:
set_error(Z_MEM_ERROR, "out of memory")
return None
obj.pool = pool
actual_wbits: t.CInt = wbits
if wbits > MAX_WBITS: actual_wbits = wbits - 16
obj.stream = None
obj.is_initialized = 1
obj.wbits = actual_wbits
if zdict and zdict_len > 0:
obj.zdict = clone_bytes(pool, zdict, zdict_len)
obj.zdict_len = zdict_len
if pool == None:
obj.input_buf = BYTEPTR(calloc(1, 256))
else:
obj.input_buf = BYTEPTR(pool.alloc(256))
memset(obj.input_buf, 0, 256)
obj.input_buf_cap = 256
obj.input_buf_len = 0
return obj
def zlib_adler32(data: BYTEPTR, length: t.CSizeT, value: t.CUnsignedLong) -> t.CUnsignedLong:
return zchecksum.zchecksum_adler32(data, length, UINT32(value))
def zlib_crc32(data: BYTEPTR, length: t.CSizeT, value: t.CUnsignedLong) -> t.CUnsignedLong:
return zchecksum.zchecksum_crc32(data, length, UINT32(value))

89
includes/zlib/zchecksum.py Normal file
View File

@@ -0,0 +1,89 @@
from stdint import *
import t, c
def zchecksum_adler32(data: UINT8PTR, length: t.CSizeT, init: UINT32) -> t.CUInt32T:
a: t.CUInt32T = (init >> 0) & 0xFFFF
b: t.CUInt32T = (init >> 16) & 0xFFFF
if not data or length == 0:
return init
i: t.CSizeT
for i in range(length):
a = (a + data[i]) % 65521
b = (b + a) % 65521
return (b << 16) | a
crc32_table: t.CArray[t.CUInt32T, 256] = [
0x00000000, 0x77073096, 0xEE0E612C, 0x990951BA,
0x076DC419, 0x706AF48F, 0xE963A535, 0x9E6495A3,
0x0EDB8832, 0x79DCB8A4, 0xE0D5E91E, 0x97D2D988,
0x09B64C2B, 0x7EB17CBD, 0xE7B82D07, 0x90BF1D91,
0x1DB71064, 0x6AB020F2, 0xF3B97148, 0x84BE41DE,
0x1ADAD47D, 0x6DDDE4EB, 0xF4D4B551, 0x83D385C7,
0x136C9856, 0x646BA8C0, 0xFD62F97A, 0x8A65C9EC,
0x14015C4F, 0x63066CD9, 0xFA0F3D63, 0x8D080DF5,
0x3B6E20C8, 0x4C69105E, 0xD56041E4, 0xA2677172,
0x3C03E4D1, 0x4B04D447, 0xD20D85FD, 0xA50AB56B,
0x35B5A8FA, 0x42B2986C, 0xDBBBBBD6, 0xACBCCB40,
0x32D86CE3, 0x45DF5C75, 0xDCD60DCF, 0xABD13D59,
0x26D930AC, 0x51DE003A, 0xC8D75180, 0xBFD06116,
0x21B4F4B5, 0x56B3C423, 0xCFBA9599, 0xB8BDA50F,
0x2802B89E, 0x5F058808, 0xC60CD9B2, 0xB10BE924,
0x2F6F7C87, 0x58684C11, 0xC1611DAB, 0xB6662D3D,
0x76DC4190, 0x01DB7106, 0x98D220BC, 0xEFD5102A,
0x71B18589, 0x06B6B51F, 0x9FBFE4A5, 0xE8B8D433,
0x7807C9A2, 0x0F00F934, 0x9609A88E, 0xE10E9818,
0x7F6A0DBB, 0x086D3D2D, 0x91646C97, 0xE6635C01,
0x6B6B51F4, 0x1C6C6162, 0x856530D8, 0xF262004E,
0x6C0695ED, 0x1B01A57B, 0x8208F4C1, 0xF50FC457,
0x65B0D9C6, 0x12B7E950, 0x8BBEB8EA, 0xFCB9887C,
0x62DD1DDF, 0x15DA2D49, 0x8CD37CF3, 0xFBD44C65,
0x4DB26158, 0x3AB551CE, 0xA3BC0074, 0xD4BB30E2,
0x4ADFA541, 0x3DD895D7, 0xA4D1C46D, 0xD3D6F4FB,
0x4369E96A, 0x346ED9FC, 0xAD678846, 0xDA60B8D0,
0x44042D73, 0x33031DE5, 0xAA0A4C5F, 0xDD0D7D49,
0x5005713C, 0x270241AA, 0xBE0B1010, 0xC90C2086,
0x5768B525, 0x206F85B3, 0xB966D409, 0xCE61E49F,
0x5EDEF90E, 0x29D9C998, 0xB0D09822, 0xC7D7A8B4,
0x59B33D17, 0x2EB40D81, 0xB7BD5C3B, 0xC0BA6CAD,
0xEDB88320, 0x9ABFB3B6, 0x03B6E20C, 0x74B1D29A,
0xEAD54739, 0x9DD277AF, 0x04DB2615, 0x73DC1683,
0xE3630B12, 0x94643B84, 0x0D6D6A3E, 0x7A6A5AA8,
0xE40ECF0B, 0x9309FF9D, 0x0A00AE27, 0x7D079EB1,
0xF00F9344, 0x8708A3D2, 0x1E01F268, 0x6906C2FE,
0xF762575D, 0x806567CB, 0x196C3671, 0x6E6B06E7,
0xFED41B76, 0x89D32BE0, 0x10DA7A5A, 0x67DD4ACC,
0xF9B9DF6F, 0x8EBEEFF9, 0x17B7BE43, 0x60B08ED5,
0xD6D6A3E8, 0xA1D1937E, 0x38D8C2C4, 0x4FDFF252,
0xD1BB67F1, 0xA6BC5767, 0x3FB506DD, 0x48B2364B,
0xD80D2BDA, 0xAF0A1B4C, 0x36034AF6, 0x41047A60,
0xDF60EFC3, 0xA867DF55, 0x316E8EEF, 0x4669BE79,
0xCB61B38C, 0xBC66831A, 0x256FD2A0, 0x5268E236,
0xCC0C7795, 0xBB0B4703, 0x220216B9, 0x5505262F,
0xC5BA3BBE, 0xB2BD0B28, 0x2BB45A92, 0x5CB36A04,
0xC2D7FFA7, 0xB5D0CF31, 0x2CD99E8B, 0x5BDEAE1D,
0x9B64C2B0, 0xEC63F226, 0x756AA39C, 0x026D930A,
0x9C0906A9, 0xEB0E363F, 0x72076785, 0x05005713,
0x95BF4A82, 0xE2B87A14, 0x7BB12BAE, 0x0CB61B38,
0x92D28E9B, 0xE5D5BE0D, 0x7CDCEFB7, 0x0BDBDF21,
0x86D3D2D4, 0xF1D4E242, 0x68DDB3F8, 0x1FDA836E,
0x81BE16CD, 0xF6B9265B, 0x6FB077E1, 0x18B74777,
0x88085AE6, 0xFF0F6A70, 0x66063BCA, 0x11010B5C,
0x8F659EFF, 0xF862AE69, 0x616BFFD3, 0x166CCF45,
0xA00AE278, 0xD70DD2EE, 0x4E048354, 0x3903B3C2,
0xA7672661, 0xD06016F7, 0x4969474D, 0x3E6E77DB,
0xAED16A4A, 0xD9D65ADC, 0x40DF0B66, 0x37D83BF0,
0xA9BCAE53, 0xDede9EC5, 0x47D7897F, 0x30D0B8E9,
0xBDDA8B1C, 0xCADD8B8A, 0x53D3903A, 0x24D4C2AC,
0xBAD03605, 0xCDD70693, 0x54DE5729, 0x23D967BF,
0xB3667A2E, 0xC4614AB8, 0x5D681B02, 0x2A6F2B94,
0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D
]
def zchecksum_crc32(data: UINT8PTR, length: t.CSizeT, init: t.CUInt32T) -> t.CUInt32T:
crc: t.CUInt32T = init ^ 0xFFFFFFFF
if not data or length == 0: return init
i: t.CSizeT
for i in range(length):
crc = crc32_table[(crc ^ data[i]) & 0xFF] ^ (crc >> 8)
return crc ^ 0xFFFFFFFF

288
includes/zlib/zdef.py Normal file
View File

@@ -0,0 +1,288 @@
from stdint import *
import stddef
import string
import stdlib
import memhub
import t, c
# ============================================================
# DEFLATE / zlib constants
# ============================================================
ZDEFLATE_WINDOW_BITS: t.CDefine = 15
ZDEFLATE_WINDOW_SIZE: t.CDefine = (1 << ZDEFLATE_WINDOW_BITS)
ZDEFLATE_WINDOW_MASK: t.CDefine = (ZDEFLATE_WINDOW_SIZE - 1)
ZDEFLATE_MIN_MATCH: t.CDefine = 3
ZDEFLATE_MAX_MATCH: t.CDefine = 258
ZDEFLATE_HASH_BITS: t.CDefine = 15
ZDEFLATE_HASH_SIZE: t.CDefine = (1 << ZDEFLATE_HASH_BITS)
ZDEFLATE_HASH_MASK: t.CDefine = (ZDEFLATE_HASH_SIZE - 1)
ZDEFLATE_MAX_CODES: t.CDefine = 288
ZDEFLATE_MAX_DIST_CODES: t.CDefine = 32
ZDEFLATE_MAX_BITS: t.CDefine = 15
ZDEFLATE_MAX_CODELEN_CODES: t.CDefine = 19
ZDEFLATE_LIT_COUNT: t.CDefine = 286
ZDEFLATE_DIST_COUNT: t.CDefine = 30
ZDEFLATE_LEN_SYMBOLS_BASE: t.CDefine = 257
ZDEFLATE_END_OF_BLOCK: t.CDefine = 256
# ============================================================
# Length / Distance extra bits tables (RFC 1951)
# ============================================================
zdeflate_len_extra_bits: t.CArray[t.CInt, 29] = [
0, 0, 0, 0, 0, 0, 0, 0,
1, 1, 1, 1,
2, 2, 2, 2,
3, 3, 3, 3,
4, 4, 4, 4,
5, 5, 5, 5,
0
]
zdeflate_len_base: t.CArray[t.CInt, 29] = [
3, 4, 5, 6, 7, 8, 9, 10,
11, 13, 15, 17,
19, 23, 27, 31,
35, 43, 51, 59,
67, 83, 99, 115,
131, 163, 195, 227,
258
]
zdeflate_dist_extra_bits: t.CArray[t.CInt, 30] = [
0, 0, 0, 0,
1, 1,
2, 2,
3, 3,
4, 4,
5, 5,
6, 6,
7, 7,
8, 8,
9, 9,
10, 10,
11, 11,
12, 12,
13, 13
]
zdeflate_dist_base: t.CArray[t.CInt, 30] = [
1, 2, 3, 4,
5, 7,
9, 13,
17, 25,
33, 49,
65, 97,
129, 193,
257, 385,
513, 769,
1025, 1537,
2049, 3073,
4097, 6145,
8193, 12289,
16385, 24577
]
zdeflate_codelen_order: t.CArray[int, 19] = [
16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15
]
# ============================================================
# Bit writer - writes bits LSB first into a byte buffer
# ============================================================
@t.Object
class zbit_writer:
pool: memhub.MemManager | t.CPtr
buf: BYTE | t.CPtr
cap: t.CSizeT
byte_pos: t.CSizeT
bit_pos: t.CInt
def __init__(self, pool: memhub.MemManager | t.CPtr):
self.pool = pool
self.cap = 4096
if pool == None:
self.buf = BYTEPTR(malloc(self.cap))
else:
self.buf = BYTEPTR(pool.alloc(self.cap))
memset(self.buf, 0, self.cap)
self.byte_pos = 0
self.bit_pos = 0
def ensure(self, need: t.CSizeT):
while self.byte_pos + need + 4 >= self.cap:
new_cap: t.CSizeT = self.cap * 2
if self.pool == None:
new_buf: BYTE | t.CPtr = BYTEPTR(realloc(self.buf, new_cap))
else:
new_buf: BYTE | t.CPtr = BYTEPTR(self.pool.alloc(new_cap))
if new_buf:
memcpy(new_buf, self.buf, self.byte_pos + (self.bit_pos + 7) / 8)
self.pool.free(self.buf)
if not new_buf: return
memset(new_buf + self.cap, 0, new_cap - self.cap)
self.buf = new_buf
self.cap = new_cap
def write_bits(self, value: UINT, nbits: t.CInt):
self.ensure((nbits + 7) / 8 + 1)
for i in range(nbits):
if value & (UINT(1) << i):
self.buf[self.byte_pos] |= (UINT(1) << self.bit_pos)
self.bit_pos += 1
if self.bit_pos == 8:
self.bit_pos = 0
self.byte_pos += 1
def write_bits_rev(self, code: UINT, nbits: t.CInt):
self.ensure((nbits + 7) / 8 + 1)
for i in range(nbits - 1, -1, -1):
if code & (UINT(1) << i):
self.buf[self.byte_pos] |= (UINT(1) << self.bit_pos)
self.bit_pos += 1
if self.bit_pos == 8:
self.bit_pos = 0
self.byte_pos += 1
def stored_block(self, data: UINT8PTR, length: t.CSizeT, final: t.CInt):
self.write_bits(1 if final else 0, 1)
self.write_bits(0, 2)
self.align()
pos: t.CSizeT = 0
while pos < length:
block_length: t.CSizeT = length - pos
if block_length > 65535:
block_length = 65535
n: t.CUInt16T = t.CUInt16T(block_length)
ncomp: t.CUInt16T = ~n
self.write_bits(n, 16)
self.write_bits(ncomp, 16)
self.ensure(block_length)
i: t.CSizeT = 0
for i in range(block_length):
self.buf[self.byte_pos] = data[pos + i]
self.byte_pos += 1
pos += block_length
if pos < length:
self.write_bits(0, 1)
self.write_bits(0, 2)
self.align()
def write_zlib_header(self, wbits: t.CInt, level: t.CInt):
cinfo: t.CInt = wbits - 8
if cinfo < 1: cinfo = 1
if cinfo > 7: cinfo = 7
cmf: t.CUnsignedChar = t.CUnsignedChar((cinfo << 4) | 8)
flevel: t.CInt = 0
if level >= 5: flevel = 3
elif level >= 3: flevel = 1
flg: t.CUnsignedChar = t.CUnsignedChar(flevel << 6)
flg |= t.CUnsignedChar(31 - (cmf * 256 + flg) % 31)
self.buf[self.byte_pos + 0] = cmf
self.buf[self.byte_pos + 1] = flg
self.byte_pos += 2
def write_gzip_header(self):
self.buf[self.byte_pos + 0] = 0x1F
self.buf[self.byte_pos + 1] = 0x8B
self.buf[self.byte_pos + 2] = 0x08
self.buf[self.byte_pos + 3] = 0x00
self.buf[self.byte_pos + 4] = 0x00
self.buf[self.byte_pos + 5] = 0x00
self.buf[self.byte_pos + 6] = 0x00
self.buf[self.byte_pos + 7] = 0x00
self.buf[self.byte_pos + 8] = 0x00
self.buf[self.byte_pos + 9] = 0xFF
self.byte_pos += 10
def align(self):
if self.bit_pos > 0:
self.bit_pos = 0
self.byte_pos += 1
def total(self) -> t.CSizeT:
return self.byte_pos + (1 if (self.bit_pos > 0) else 0)
def free(self):
if self.pool == None:
free(self.buf)
else:
self.pool.free(self.buf)
self.buf = None
self.cap = 0
self.byte_pos = 0
self.bit_pos = 0
def __del__(self):
self.free()
# ============================================================
# Bit reader - reads bits LSB first from a byte buffer
# ============================================================
@t.Object
class zbit_reader:
buf: BYTE | t.CPtr
length: t.CSizeT
byte_pos: t.CSizeT
bit_pos: t.CInt
def init(self, buf: BYTE | t.CPtr, length: t.CSizeT):
self.buf = buf
self.length = length
self.byte_pos = 0
self.bit_pos = 0
def read_bits(self, nbits: t.CInt, out: UINTPTR) -> t.CInt:
val: UINT = 0
for i in range(nbits):
if self.byte_pos >= self.length: return -1
if self.buf[self.byte_pos] & (UINT(1) << self.bit_pos):
val |= (UINT(1) << i)
self.bit_pos += 1
if self.bit_pos == 8:
self.bit_pos = 0
self.byte_pos += 1
c.Set(c.Deref(out), val)
return 0
def read_bits_rev(self, nbits: t.CInt, out: UINTPTR) -> t.CInt:
val: UINT = 0
for i in range(nbits - 1, 0, -1):
if self.byte_pos >= self.length: return -1
if self.buf[self.byte_pos] & (UINT(1) << self.bit_pos):
val |= (UINT(1) << i)
self.bit_pos += 1
if self.bit_pos == 8:
self.bit_pos = 0
self.byte_pos += 1
c.Set(c.Deref(out), val)
return 0
def align(self):
if self.bit_pos > 0:
self.bit_pos = 0
self.byte_pos += 1
# ============================================================
# Memory allocation helper for bare metal
# Can be replaced with custom allocator
# ============================================================
def zdef_alloc(pool: memhub.MemManager | t.CPtr, n: t.CSizeT) -> VOIDPTR:
if pool == None: return malloc(n)
return pool.alloc(n)
def zdef_free(pool: memhub.MemManager | t.CPtr, p: VOIDPTR):
if pool == None:
free(p)
return
pool.free(p)

434
includes/zlib/zdeflate.py Normal file
View File

@@ -0,0 +1,434 @@
from stdint import *
import zlib.zchecksum as zchecksum
import zlib.zhuff as zhuff
import zlib.zdef as zdef
import string
import memhub
import t, c
@t.Object
class zdeflate_stream:
pool: memhub.MemManager | t.CPtr
writer: zdef.zbit_writer
window: BYTE | t.CPtr
window_pos: t.CInt
window_size: t.CInt
hash_head: t.CInt | t.CPtr
hash_prev: t.CInt | t.CPtr
level: t.CInt
strategy: t.CInt
wbits: t.CInt
is_finished: t.CInt
adler: t.CUInt32T
def find_match(self, data: BYTE | t.CPtr, data_len: t.CSizeT,
pos: t.CSizeT, best_dist: t.CInt | t.CPtr) -> t.CInt:
if pos + 2 >= data_len: return 0
h: t.CUnsignedInt = zdeflate_hash(data + pos)
chain: t.CInt = self.hash_head[h]
best_len: t.CInt = zdef.ZDEFLATE_MIN_MATCH - 1
c.Set(c.Deref(best_dist), 0)
max_chain: t.CInt = 128 if (self.level >= 5) else (32 if (self.level >= 2) else 4)
limit: t.CInt = (1 << self.wbits) if (self.wbits > 0) else zdef.ZDEFLATE_WINDOW_SIZE
if limit > zdef.ZDEFLATE_WINDOW_SIZE:
limit = zdef.ZDEFLATE_WINDOW_SIZE
attempts: t.CInt = 0
while chain >= 0 and attempts < max_chain:
dist: t.CInt = t.CInt(pos) - chain
if dist <= 0 or dist > limit: break
match_len: t.CInt = 0
max_len: t.CInt = t.CInt(data_len - pos)
if max_len > zdef.ZDEFLATE_MAX_MATCH:
max_len = zdef.ZDEFLATE_MAX_MATCH
while match_len < max_len and data[pos + match_len] == data[chain + match_len]:
match_len += 1
if match_len > best_len:
best_len = match_len
c.Set(c.Deref(best_dist), dist)
if best_len >= zdef.ZDEFLATE_MAX_MATCH: break
chain = self.hash_prev[chain & zdef.ZDEFLATE_WINDOW_MASK]
if chain <= t.CInt(pos) - limit: break
attempts += 1
return best_len if (best_len >= zdef.ZDEFLATE_MIN_MATCH) else 0
def update_hash(self, data: BYTE | t.CPtr, pos: t.CSizeT):
if pos + 2 < pos: return
h: t.CUnsignedInt = zdeflate_hash(data + pos)
self.hash_prev[pos & zdef.ZDEFLATE_WINDOW_MASK] = self.hash_head[h]
self.hash_head[h] = t.CInt(pos)
def write_fixed_block(self, data: UINT8PTR, length: t.CSizeT, final: t.CInt):
lit_tree = zhuff.zhuff_tree()
dist_tree = zhuff.zhuff_tree()
lit_tree.build_fixed_lit_tree()
dist_tree.build_fixed_dist_tree()
self.writer.write_bits(1 if final else 0, 1)
self.writer.write_bits(1, 2)
pos: t.CSizeT = 0
while pos < length:
best_dist: t.CInt = 0
match_len: t.CInt = 0
if self.level > 0 and pos + 2 < length:
match_len = self.find_match(data, length, pos, c.Addr(best_dist))
if match_len >= zdef.ZDEFLATE_MIN_MATCH:
len_sym: t.CInt = zdeflate_len_to_symbol(match_len)
lit_tree.encode_symbol(len_sym, c.Addr(self.writer))
extra: t.CInt = zdef.zdeflate_len_extra_bits[len_sym - 257]
if extra > 0:
self.writer.write_bits(match_len - zdef.zdeflate_len_base[len_sym - 257], extra)
dist_sym: t.CInt = zdeflate_dist_to_symbol(best_dist)
dist_tree.encode_symbol(dist_sym, c.Addr(self.writer))
extra = zdef.zdeflate_dist_extra_bits[dist_sym]
if extra > 0:
self.writer.write_bits(best_dist - zdef.zdeflate_dist_base[dist_sym], extra)
for i in range(match_len):
self.update_hash(data, pos + i)
pos += match_len
else:
lit_tree.encode_symbol(data[pos], c.Addr(self.writer))
self.update_hash(data, pos)
pos += 1
lit_tree.encode_symbol(256, c.Addr(self.writer))
def encode_block_data(self, data: UINT8PTR, length: t.CSizeT,
lit_tree: zhuff.zhuff_tree | t.CPtr, dist_tree: zhuff.zhuff_tree | t.CPtr):
pos: t.CSizeT = 0
while pos < length:
best_dist: t.CInt = 0
match_len: t.CInt = 0
if self.level > 0 and pos + 2 < length:
match_len = self.find_match(data, length, pos, c.Addr(best_dist))
if match_len >= zdef.ZDEFLATE_MIN_MATCH:
len_sym: t.CInt = zdeflate_len_to_symbol(match_len)
lit_tree.encode_symbol(len_sym, c.Addr(self.writer))
extra: t.CInt = zdef.zdeflate_len_extra_bits[len_sym - 257]
if extra > 0:
self.writer.write_bits(match_len - zdef.zdeflate_len_base[len_sym - 257], extra)
dist_sym: t.CInt = zdeflate_dist_to_symbol(best_dist)
dist_tree.encode_symbol(dist_sym, c.Addr(self.writer))
extra = zdef.zdeflate_dist_extra_bits[dist_sym]
if extra > 0:
self.writer.write_bits(best_dist - zdef.zdeflate_dist_base[dist_sym], extra)
for j in range(match_len):
self.update_hash(data, pos + j)
pos += match_len
else:
lit_tree.encode_symbol(data[pos], c.Addr(self.writer))
self.update_hash(data, pos)
pos += 1
lit_tree.encode_symbol(256, c.Addr(self.writer))
def write_dynamic_block(self, data: UINT8PTR, length: t.CSizeT, final: t.CInt):
lit_freqs: t.CArray[t.CInt, 288]
dist_freqs: t.CArray[t.CInt, 32]
zdeflate_count_freqs(lit_freqs, dist_freqs, self, data, length)
hlit: t.CInt = 286
while hlit > 257 and lit_freqs[hlit - 1] == 0: hlit -= 1
hdist: t.CInt = 30
while hdist > 1 and dist_freqs[hdist - 1] == 0: hdist -= 1
lit_tree = zhuff.zhuff_tree()
dist_tree = zhuff.zhuff_tree()
lit_tree.build_codes(lit_freqs, hlit, zdef.ZDEFLATE_MAX_BITS)
dist_tree.build_codes(dist_freqs, hdist, zdef.ZDEFLATE_MAX_BITS)
all_lengths: t.CArray[t.CInt, 288 + 32]
for i in range(hlit): all_lengths[i] = lit_tree.codes[i].bits
for i in range(hdist): all_lengths[hlit + i] = dist_tree.codes[i].bits
total_lengths: t.CInt = hlit + hdist
cl_freqs: t.CArray[t.CInt, 19]
zdeflate_count_cl_freqs(all_lengths, total_lengths, cl_freqs)
cl_tree = zhuff.zhuff_tree()
cl_tree.build_codes(cl_freqs, 19, 7)
hclen: int = 19
while hclen > 4 and cl_tree.codes[zdef.zdeflate_codelen_order[hclen - 1]].bits == 0: hclen -= 1
self.writer.write_bits(1 if final else 0, 1)
self.writer.write_bits(2, 2)
self.writer.write_bits(hlit - 257, 5)
self.writer.write_bits(hdist - 1, 5)
self.writer.write_bits(hclen - 4, 4)
for j in range(hclen):
self.writer.write_bits(cl_tree.codes[zdef.zdeflate_codelen_order[j]].bits, 3)
zdeflate_write_cl_encoded(all_lengths, total_lengths, c.Addr(self.writer), c.Addr(cl_tree))
self.encode_block_data(data, length, c.Addr(lit_tree), c.Addr(dist_tree))
def compress_block(self, data: UINT8PTR, length: t.CSizeT, final: t.CInt):
if self.level == 0:
self.write_stored_block(data, length, final)
elif self.strategy == 4 or length < 128:
self.write_fixed_block(data, length, final)
else:
self.write_dynamic_block(data, length, final)
def write_stored_block(self, data: UINT8PTR, length: t.CSizeT, final: t.CInt):
self.writer.stored_block(data, length, final)
def compress(self, data: UINT8PTR, length: t.CSizeT,
out: t.CUInt8T | t.CPtr[t.CPtr], out_len: t.CSizeT | t.CPtr) -> t.CInt:
if self.is_finished: return -2
if not data or length == 0:
c.Set(c.Deref(out), None)
c.Set(c.Deref(out_len), 0)
return 0
self.adler = zchecksum.zchecksum_adler32(data, length, self.adler)
memset(self.hash_head, -1, zdef.ZDEFLATE_HASH_SIZE * int.__sizeof__())
memset(self.hash_prev, -1, zdef.ZDEFLATE_WINDOW_SIZE * int.__sizeof__())
self.compress_block(data, length, 0)
c.Set(c.Deref(out_len), self.writer.total())
c.Set(c.Deref(out), UINT8PTR(zdef.zdef_alloc(self.pool, c.Deref(out_len))))
if not c.Deref(out): return -4
memcpy(c.Deref(out), self.writer.buf, c.Deref(out_len))
return 0
def flush(self, mode: t.CInt, out: t.CUInt8T | t.CPtr[t.CPtr], out_len: t.CSizeT | t.CPtr) -> t.CInt:
if mode == 4:
if not self.is_finished:
if self.level == 0:
self.writer.write_bits(1, 1)
self.writer.write_bits(0, 2)
self.writer.align()
else:
lit_tree = zhuff.zhuff_tree()
lit_tree.build_fixed_lit_tree()
self.writer.write_bits(1, 1)
self.writer.write_bits(1, 2)
lit_tree.encode_symbol(256, c.Addr(self.writer))
self.is_finished = 1
if self.wbits >= 0:
self.writer.align()
adler: t.CUInt32T = self.adler
self.writer.buf[self.writer.byte_pos + 0] = (adler >> 24) & 0xFF
self.writer.buf[self.writer.byte_pos + 1] = (adler >> 16) & 0xFF
self.writer.buf[self.writer.byte_pos + 2] = (adler >> 8) & 0xFF
self.writer.buf[self.writer.byte_pos + 3] = (adler >> 0) & 0xFF
self.writer.byte_pos += 4
elif mode == 2 or mode == 3:
self.writer.write_bits(0, 1)
self.writer.write_bits(0, 2)
self.writer.align()
c.Set(c.Deref(out_len), self.writer.total())
c.Set(c.Deref(out), UINT8PTR(zdef.zdef_alloc(self.pool, c.Deref(out_len))))
if not c.Deref(out): return -4
memcpy(c.Deref(out), self.writer.buf, c.Deref(out_len))
return 0
def set_dictionary(self, dict: UINT8PTR, length: t.CSizeT) -> t.CInt:
if not dict: return -2
use: t.CSizeT = length
if use > zdef.ZDEFLATE_WINDOW_SIZE:
dict += length - zdef.ZDEFLATE_WINDOW_SIZE
use = zdef.ZDEFLATE_WINDOW_SIZE
memcpy(self.window, dict, use)
self.window_size = t.CInt(use)
self.window_pos = t.CInt(use)
self.adler = zchecksum.zchecksum_adler32(dict, length, self.adler)
return 0
def copy(self) -> zdeflate_stream | t.CPtr:
z: zdeflate_stream | t.CPtr = zdef.zdef_alloc(self.pool, zdeflate_stream.__sizeof__())
if not z: return None
memcpy(z, self, zdeflate_stream.__sizeof__())
z.writer.buf = BYTEPTR(zdef.zdef_alloc(self.pool, self.writer.cap))
if not z.writer.buf:
zdef.zdef_free(self.pool, z)
return None
memcpy(z.writer.buf, self.writer.buf, self.writer.cap)
z.window = BYTEPTR(zdef.zdef_alloc(self.pool, zdef.ZDEFLATE_WINDOW_SIZE))
z.hash_head = INTPTR(zdef.zdef_alloc(self.pool, zdef.ZDEFLATE_HASH_SIZE * int.__sizeof__()))
z.hash_prev = INTPTR(zdef.zdef_alloc(self.pool, zdef.ZDEFLATE_WINDOW_SIZE * int.__sizeof__()))
if not z.window or not z.hash_head or not z.hash_prev:
if z.writer.buf: zdef.zdef_free(self.pool, z.writer.buf)
zdef.zdef_free(self.pool, z)
return None
memcpy(z.window, self.window, zdef.ZDEFLATE_WINDOW_SIZE)
memcpy(z.hash_head, self.hash_head, zdef.ZDEFLATE_HASH_SIZE * int.__sizeof__())
memcpy(z.hash_prev, self.hash_prev, zdef.ZDEFLATE_WINDOW_SIZE * int.__sizeof__())
return z
def destroy(self):
if self.writer.buf: zdef.zdef_free(self.pool, self.writer.buf)
if self.window: zdef.zdef_free(self.pool, self.window)
if self.hash_head: zdef.zdef_free(self.pool, self.hash_head)
if self.hash_prev: zdef.zdef_free(self.pool, self.hash_prev)
zdef.zdef_free(self.pool, self)
def zdeflate_count_freqs(lit_freqs: INTPTR, dist_freqs: INTPTR,
s: zdeflate_stream | t.CPtr, data: UINT8PTR, length: t.CSizeT):
memset(lit_freqs, 0, 288 * int.__sizeof__())
memset(dist_freqs, 0, 32 * int.__sizeof__())
pos: t.CSizeT = 0
while pos < length:
best_dist: t.CInt = 0
match_len: t.CInt = 0
if s.level > 0 and pos + 2 < length:
match_len = zdeflate_stream.find_match(s, data, length, pos, c.Addr(best_dist))
if match_len >= zdef.ZDEFLATE_MIN_MATCH:
len_sym: t.CInt = zdeflate_len_to_symbol(match_len)
lit_freqs[len_sym] += 1
dist_freqs[zdeflate_dist_to_symbol(best_dist)] += 1
for i in range(match_len):
zdeflate_stream.update_hash(s, data, pos + i)
pos += match_len
else:
lit_freqs[data[pos]] += 1
zdeflate_stream.update_hash(s, data, pos)
pos += 1
lit_freqs[256] = 1
def zdeflate_hash(p: BYTE | t.CPtr) -> t.CUnsignedInt:
return (t.CUnsignedInt(p[0]) ^ (t.CUnsignedInt(p[1]) << 5) ^ (t.CUnsignedInt(p[2]) << 10)) & zdef.ZDEFLATE_HASH_MASK
def zdeflate_len_to_symbol(length: t.CInt) -> t.CInt:
for i in range(29):
if length <= zdef.zdeflate_len_base[i] + (1 << zdef.zdeflate_len_extra_bits[i]) - 1:
return 257 + i
return 285
def zdeflate_dist_to_symbol(dist: t.CInt) -> t.CInt:
for i in range(30):
if dist <= zdef.zdeflate_dist_base[i] + (1 << zdef.zdeflate_dist_extra_bits[i]) - 1:
return i
return 29
def zdeflate_count_cl_freqs(all_lengths: INTPTR, total: t.CInt, cl_freqs: INTPTR):
memset(cl_freqs, 0, 19 * int.__sizeof__())
i: t.CInt = 0
while i < total:
if all_lengths[i] == 0:
run: t.CInt = 0
while i + run < total and all_lengths[i + run] == 0: run += 1
while run > 0:
if run >= 11:
count: t.CInt = run
if count > 138: count = 138
cl_freqs[18] += 1
run -= count
elif run >= 3:
count: t.CInt = run
if count > 10: count = 10
cl_freqs[17] += 1
run -= count
else:
cl_freqs[0] += 1
run -= 1
while i < total and all_lengths[i] == 0: i += 1
else:
cl_freqs[all_lengths[i]] += 1
val: t.CInt = all_lengths[i]
i += 1
run: t.CInt = 0
while i + run < total and all_lengths[i + run] == val: run += 1
while run >= 3:
count: t.CInt = run
if count > 6: count = 6
cl_freqs[16] += 1
run -= count
i += run
def zdeflate_write_cl_encoded(all_lengths: INTPTR, total: t.CInt,
w: zdef.zbit_writer | t.CPtr, cl_tree: zhuff.zhuff_tree | t.CPtr):
i: t.CInt = 0
while i < total:
if all_lengths[i] == 0:
run: t.CInt = 0
while i + run < total and all_lengths[i + run] == 0: run += 1
while run > 0:
if run >= 11:
count: t.CInt = run
if count > 138: count = 138
cl_tree.encode_symbol(18, w)
w.write_bits(count - 11, 7)
run -= count
elif run >= 3:
count: t.CInt = run
if count > 10: count = 10
cl_tree.encode_symbol(17, w)
w.write_bits(count - 3, 3)
run -= count
else:
cl_tree.encode_symbol(0, w)
run -= 1
while i < total and all_lengths[i] == 0: i += 1
else:
cl_tree.encode_symbol(all_lengths[i], w)
val: t.CInt = all_lengths[i]
i += 1
run: t.CInt = 0
while i + run < total and all_lengths[i + run] == val: run += 1
while run >= 3:
count: t.CInt = run
if count > 6: count = 6
cl_tree.encode_symbol(16, w)
w.write_bits(count - 3, 2)
run -= count
i += run
def zdeflate_create(pool: memhub.MemManager | t.CPtr, level: t.CInt, wbits: t.CInt, mem_level: t.CInt, strategy: t.CInt) -> zdeflate_stream | t.CPtr:
n: t.CSizeT = zdeflate_stream.__sizeof__()
s: zdeflate_stream | t.CPtr = zdef.zdef_alloc(pool, n)
if not s: return None
memset(s, 0, zdeflate_stream.__sizeof__())
s.pool = pool
s.writer = zdef.zbit_writer(pool)
s.window = BYTEPTR(zdef.zdef_alloc(pool, zdef.ZDEFLATE_WINDOW_SIZE))
s.hash_head = INTPTR(zdef.zdef_alloc(pool, zdef.ZDEFLATE_HASH_SIZE * int.__sizeof__()))
s.hash_prev = INTPTR(zdef.zdef_alloc(pool, zdef.ZDEFLATE_WINDOW_SIZE * int.__sizeof__()))
if not s.window or not s.hash_head or not s.hash_prev:
s.destroy()
return None
memset(s.hash_head, -1, zdef.ZDEFLATE_HASH_SIZE * int.__sizeof__())
memset(s.hash_prev, -1, zdef.ZDEFLATE_WINDOW_SIZE * int.__sizeof__())
s.level = level
s.wbits = wbits
s.strategy = strategy
s.is_finished = 0
s.adler = 1
if wbits > 0:
s.writer.write_zlib_header(wbits, level)
elif wbits == 0:
s.writer.write_zlib_header(15, level)
return s
def zdeflate_one_shot(pool: memhub.MemManager | t.CPtr, data: UINT8PTR, length: t.CSizeT,
level: t.CInt, wbits: t.CInt, out_len: t.CSizeT | t.CPtr) -> t.CUInt8T | t.CPtr:
s: zdeflate_stream | t.CPtr = zdeflate_create(pool, level, wbits, 8, 0)
if not s: return None
memset(s.hash_head, -1, zdef.ZDEFLATE_HASH_SIZE * int.__sizeof__())
memset(s.hash_prev, -1, zdef.ZDEFLATE_WINDOW_SIZE * int.__sizeof__())
s.adler = zchecksum.zchecksum_adler32(data, length, 1)
if length == 0:
s.writer.write_bits(1, 1)
s.writer.write_bits(1, 2)
lit_tree = zhuff.zhuff_tree()
lit_tree.build_fixed_lit_tree()
lit_tree.encode_symbol(256, c.Addr(s.writer))
else:
s.compress_block(data, length, 1)
s.is_finished = 1
if wbits >= 0:
s.writer.align()
adler: t.CUInt32T = s.adler
s.writer.buf[s.writer.byte_pos + 0] = (adler >> 24) & 0xFF
s.writer.buf[s.writer.byte_pos + 1] = (adler >> 16) & 0xFF
s.writer.buf[s.writer.byte_pos + 2] = (adler >> 8) & 0xFF
s.writer.buf[s.writer.byte_pos + 3] = (adler >> 0) & 0xFF
s.writer.byte_pos += 4
c.Set(c.Deref(out_len), s.writer.total())
result: UINT8PTR = UINT8PTR(zdef.zdef_alloc(pool, c.Deref(out_len)))
if result: memcpy(result, s.writer.buf, c.Deref(out_len))
s.destroy()
return result

356
includes/zlib/zhuff.py Normal file
View File

@@ -0,0 +1,356 @@
#include <string.h>
from stdint import *
import zlib.zdef as zdef
import memhub
import t, c
ZHUFF_MAX_CODES: t.CDefine = 288
ZHUFF_MAX_BITS: t.CDefine = 15
class zhuff_code:
code: UINT
bits: t.CInt
@t.Object
class zhuff_tree:
pool: memhub.MemManager | t.CPtr
codes: t.CArray[zhuff_code, ZHUFF_MAX_CODES]
count: t.CInt
max_bits: t.CInt
def __init__(self):
self.pool = None
def build_codes(self, freqs: INTPTR, count: t.CInt, max_bits: t.CInt):
lengths: t.CArray[t.CInt, ZHUFF_MAX_CODES]
bl_count: t.CArray[t.CInt, ZHUFF_MAX_BITS + 1]
next_code: t.CArray[UINT, ZHUFF_MAX_BITS + 1]
self.count = count
self.max_bits = max_bits
self.build_code_lengths(lengths, freqs, count, max_bits)
memset(bl_count, 0, bl_count.__sizeof__())
for i in range(count):
if lengths[i] > 0:
bl_count[lengths[i]] += 1
code: t.CUnsignedInt = 0
next_code[0] = 0
for bits in range(1, max_bits + 1):
code = (code + bl_count[bits - 1]) << 1
next_code[bits] = code
for i in range(count):
self.codes[i].bits = lengths[i]
if lengths[i] > 0:
self.codes[i].code = next_code[lengths[i]]
next_code[lengths[i]] += 1
else:
self.codes[i].code = 0
def build_fixed_lit_tree(self):
lengths: t.CArray[t.CInt, 288]
bl_count: t.CArray[t.CInt, 16]
next_code: t.CArray[t.CUnsignedInt, 16]
self.get_fixed_lit_lengths(lengths)
memset(bl_count, 0, bl_count.__sizeof__())
for i in range(288):
if lengths[i] > 0:
bl_count[lengths[i]] += 1
code: t.CUnsignedInt = 0
next_code[0] = 0
for bits in range(1, 9 + 1):
code = (code + bl_count[bits - 1]) << 1
next_code[bits] = code
self.count = 288
self.max_bits = 9
for i in range(288):
self.codes[i].bits = lengths[i]
if lengths[i] > 0:
self.codes[i].code = next_code[lengths[i]]
next_code[lengths[i]] += 1
else:
self.codes[i].code = 0
def build_fixed_dist_tree(self):
lengths: t.CArray[t.CInt, 32]
bl_count: t.CArray[t.CInt, 16]
next_code: t.CArray[t.CUnsignedInt, 16]
self.get_fixed_dist_lengths(lengths)
memset(bl_count, 0, bl_count.__sizeof__())
for i in range(32):
if lengths[i] > 0:
bl_count[lengths[i]] += 1
code: t.CUnsignedInt = 0
next_code[0] = 0
for bits in range(1, 5 + 1):
code = (code + bl_count[bits - 1]) << 1
next_code[bits] = code
self.count = 32
self.max_bits = 5
for i in range(32):
self.codes[i].bits = lengths[i]
if lengths[i] > 0:
self.codes[i].code = next_code[lengths[i]]
next_code[lengths[i]] += 1
else:
self.codes[i].code = 0
def encode_symbol(self, symbol: int, writer: zdef.zbit_writer | t.CPtr):
if symbol < 0 or symbol >= self.count:
return
if self.codes[symbol].bits == 0:
return
writer.write_bits_rev(self.codes[symbol].code, self.codes[symbol].bits)
def build_code_lengths(self, lengths: t.CInt | t.CPtr, freqs: t.CInt | t.CPtr, count: t.CInt, max_bits: t.CInt):
bl_count: t.CArray[t.CInt, ZHUFF_MAX_BITS + 2]
sort_count: t.CInt = 0
memset(bl_count, 0, bl_count.__sizeof__())
memset(lengths, 0, int.__sizeof__() * count)
for i in range(count):
if freqs[i] > 0:
sort_count += 1
if sort_count == 0: return
if sort_count == 1:
for i in range(count):
if freqs[i] > 0:
lengths[i] = 1
return
items: t.CInt | t.CPtr = INTPTR(zdef.zdef_alloc(self.pool, sort_count * int.__sizeof__()))
item_freqs: t.CInt | t.CPtr = INTPTR(zdef.zdef_alloc(self.pool, sort_count * int.__sizeof__()))
idx: t.CInt = 0
for i in range(count):
if freqs[i] > 0:
items[idx] = i
item_freqs[idx] = freqs[i]
idx += 1
for i in range(sort_count):
key_item: t.CInt = items[i]
key_freq: t.CInt = item_freqs[i]
j: t.CInt = i - 1
while j >= 0 and item_freqs[j] > key_freq:
items[j + 1] = items[j]
item_freqs[j + 1] = item_freqs[j]
j -= 1
items[j + 1] = key_item
item_freqs[j + 1] = key_freq
parent: INTPTR = INTPTR(zdef.zdef_alloc(self.pool, (sort_count * 2) * int.__sizeof__()))
for i in range(sort_count * 2):
parent[i] = -1
heap: INTPTR = INTPTR(zdef.zdef_alloc(self.pool, (sort_count + 1) * int.__sizeof__()))
heap_size: t.CInt = 0
for i in range(sort_count):
heap[heap_size] = i
heap_size += 1
pos: t.CInt = heap_size - 1
while pos > 0:
par: t.CInt = (pos - 1) / 2
if item_freqs[heap[par]] > item_freqs[heap[pos]]:
tmp: t.CInt = heap[par]
heap[par] = heap[pos]
heap[pos] = tmp
pos = par
else: break
combined_freq: INTPTR = INTPTR(zdef.zdef_alloc(self.pool, (sort_count * 2) * int.__sizeof__()))
for i in range(sort_count):
combined_freq[i] = item_freqs[i]
internal: t.CInt = sort_count
while heap_size > 1:
a: t.CInt = heap[0]
heap[0] = heap[heap_size - 1]
heap_size -= 1
pos: t.CInt = 0
while True:
left: t.CInt = 2 * pos + 1
right: t.CInt = 2 * pos + 2
smallest: t.CInt = pos
if left < heap_size and combined_freq[heap[left]] < combined_freq[heap[smallest]]:
smallest = left
if right < heap_size and combined_freq[heap[right]] < combined_freq[heap[smallest]]:
smallest = right
if smallest != pos:
tmp: t.CInt = heap[pos]
heap[pos] = heap[smallest]
heap[smallest] = tmp
pos = smallest
else: break
b: t.CInt = heap[0]
heap[0] = heap[heap_size - 1]
heap_size -= 1
pos = 0
while True:
left: t.CInt = 2 * pos + 1
right: t.CInt = 2 * pos + 2
smallest: t.CInt = pos
if left < heap_size and combined_freq[heap[left]] < combined_freq[heap[smallest]]:
smallest = left
if right < heap_size and combined_freq[heap[right]] < combined_freq[heap[smallest]]:
smallest = right
if smallest != pos:
tmp: t.CInt = heap[pos]
heap[pos] = heap[smallest]
heap[smallest] = tmp
pos = smallest
else: break
if internal >= sort_count * 2 - 1: break
combined_freq[internal] = combined_freq[a] + combined_freq[b]
parent[a] = internal
parent[b] = internal
heap[heap_size] = internal
heap_size += 1
pos = heap_size - 1
while pos > 0:
par: t.CInt = (pos - 1) / 2
if combined_freq[heap[par]] > combined_freq[heap[pos]]:
tmp: t.CInt = heap[par]
heap[par] = heap[pos]
heap[pos] = tmp
pos = par
else: break
internal += 1
for i in range(sort_count):
node: t.CInt = i
length: t.CInt = 0
while parent[node] >= 0:
length += 1
node = parent[node]
lengths[items[i]] = length
memset(bl_count, 0, bl_count.__sizeof__())
for i in range(count):
if lengths[i] > 0:
bl_count[lengths[i]] += 1
overflow: t.CInt = 0
for i in range(count):
if lengths[i] > max_bits:
overflow += 1
bl_count[lengths[i]] -= 1
lengths[i] = max_bits
bl_count[max_bits] += 1
while overflow > 0:
bits: t.CInt = max_bits - 1
while bits > 0 and bl_count[bits] == 0:
bits -= 1
if bits == 0: break
bl_count[bits] -= 1
bl_count[bits + 1] += 2
bl_count[max_bits] -= 1
overflow -= 2
sorted_items: INTPTR = INTPTR(zdef.zdef_alloc(self.pool, sort_count * int.__sizeof__()))
si: t.CInt = 0
for i in range(count):
if freqs[i] > 0:
sorted_items[si] = i
si += 1
for i in range(si - 1):
for j in range(i + 1, si):
if lengths[sorted_items[i]] < lengths[sorted_items[j]]:
tmp: t.CInt = sorted_items[i]
sorted_items[i] = sorted_items[j]
sorted_items[j] = tmp
sidx: t.CInt = 0
for bits in range(max_bits, 1, -1):
n: t.CInt = bl_count[bits]
while n > 0 and sidx < si:
lengths[sorted_items[sidx]] = bits
sidx += 1
n -= 1
zdef.zdef_free(self.pool, sorted_items)
zdef.zdef_free(self.pool, heap)
zdef.zdef_free(self.pool, parent)
zdef.zdef_free(self.pool, combined_freq)
zdef.zdef_free(self.pool, item_freqs)
zdef.zdef_free(self.pool, items)
def get_fixed_lit_lengths(self, lengths: INTPTR):
for i in range( 0, 143 + 1): lengths[i] = 8
for i in range(143 + 1, 255 + 1): lengths[i] = 9
for i in range(255 + 1, 279 + 1): lengths[i] = 7
for i in range(279 + 1, 287 + 1): lengths[i] = 8
def get_fixed_dist_lengths(self, lengths: INTPTR):
for i in range(32):
lengths[i] = 5
def build_tree_from_lengths(self, lengths: INTPTR, count: t.CInt, max_bits: t.CInt):
bl_count: t.CArray[t.CInt, 16]
next_code: t.CArray[t.CUnsignedInt, 16]
memset(bl_count, 0, bl_count.__sizeof__())
for i in range(count):
if lengths[i] > 0:
bl_count[lengths[i]] += 1
code: UINT = 0
next_code[0] = 0
for bits in range(1, max_bits + 1):
code = (code + bl_count[bits - 1]) << 1
next_code[bits] = code
self.count = count
self.max_bits = max_bits
for i in range(count):
self.codes[i].bits = lengths[i]
if lengths[i] > 0:
self.codes[i].code = next_code[lengths[i]]
next_code[lengths[i]] += 1
else:
self.codes[i].code = 0
class zhuff_decode_node:
children: t.CArray[t.CInt, 2]
symbol: t.CInt
@t.Object
class zhuff_decode_tree:
pool: memhub.MemManager | t.CPtr
nodes: t.CArray[zhuff_decode_node, 2 * ZHUFF_MAX_CODES]
node_count: t.CInt
root: t.CInt
def __init__(self):
self.pool = None
def build_decode_tree(self, ht: zhuff_tree | t.CPtr):
self.node_count = 1
self.root = 0
self.nodes[0].children[0] = -1
self.nodes[0].children[1] = -1
self.nodes[0].symbol = -1
for i in range(ht.count):
if ht.codes[i].bits == 0: continue
node: int = 0
for bit in range(ht.codes[i].bits - 1, -1, -1):
dir: t.CInt = (ht.codes[i].code >> bit) & 1
if bit > 0:
# Internal node: traverse or create
if self.nodes[node].children[dir] == -1:
new_node: int = self.node_count
self.node_count += 1
self.nodes[new_node].children[0] = -1
self.nodes[new_node].children[1] = -1
self.nodes[new_node].symbol = -1
self.nodes[node].children[dir] = new_node
node = self.nodes[node].children[dir]
else:
# Leaf: bit 0 - set symbol on the child
if self.nodes[node].children[dir] == -1:
leaf: int = self.node_count
self.node_count += 1
self.nodes[leaf].children[0] = -1
self.nodes[leaf].children[1] = -1
self.nodes[leaf].symbol = i
self.nodes[node].children[dir] = leaf
else:
self.nodes[self.nodes[node].children[dir]].symbol = i
def decode_symbol(self, reader: zdef.zbit_reader | t.CPtr) -> t.CInt:
node: int = self.root
while self.nodes[node].symbol == -1:
bit: t.CUnsignedInt
if reader.read_bits(1, c.Addr(bit)) != 0: return -1
dir: t.CInt = t.CInt(bit)
if self.nodes[node].children[dir] == -1: return -1
node = self.nodes[node].children[dir]
return self.nodes[node].symbol

456
includes/zlib/zinflate.py Normal file
View File

@@ -0,0 +1,456 @@
from stdint import *
import zlib.zchecksum as zchecksum
import zlib.zdef as zdef
import zlib.zhuff as zhuff
import string
import memhub
import t, c
@t.Object
class zinflate_stream:
pool: memhub.MemManager | t.CPtr
reader: zdef.zbit_reader
window: BYTEPTR
window_pos: t.CInt
window_size: t.CInt
wbits: t.CInt
is_finished: t.CInt
is_initialized: t.CInt
header_parsed: t.CInt
is_gzip: t.CInt
adler: t.CUInt32T
crc: t.CUInt32T
output: BYTEPTR
output_len: t.CSizeT
output_cap: t.CSizeT
input_data: t.CConst | BYTEPTR
input_len: t.CSizeT
input_pos: t.CSizeT
max_length: t.CSizeT
def __init__(self):
pass
def output_byte(self, byte: BYTE):
if self.max_length > 0 and self.output_len >= self.max_length: return
if self.output_len >= self.output_cap:
new_cap: t.CSizeT = self.output_cap * 2
if new_cap < 256: new_cap = 256
new_buf: BYTEPTR = BYTEPTR(zdef.zdef_alloc(self.pool, new_cap))
if not new_buf: return
memcpy(new_buf, self.output, self.output_len)
zdef.zdef_free(self.pool, self.output)
self.output = new_buf
self.output_cap = new_cap
self.output[self.output_len] = byte
self.output_len += 1
self.window[self.window_pos & (zdef.ZDEFLATE_WINDOW_SIZE - 1)] = byte
self.window_pos += 1
def parse_header(self) -> t.CInt:
if self.header_parsed: return 0
if self.wbits < 0:
self.header_parsed = 1
self.is_gzip = 0
return 0
if self.input_pos + 2 > self.input_len: return -3
b0: BYTE = self.input_data[self.input_pos]
b1: BYTE = self.input_data[self.input_pos + 1]
if b0 == 0x1F and b1 == 0x8B:
self.is_gzip = 1
if self.input_pos + 10 > self.input_len: return -3
self.input_pos += 10
flg: BYTE = self.input_data[self.input_pos - 6]
if flg & 0x04:
if self.input_pos + 2 > self.input_len: return -3
xlen: UINT = self.input_data[self.input_pos] | (self.input_data[self.input_pos + 1] << 8)
self.input_pos += 2 + xlen
if flg & 0x08:
while self.input_pos < self.input_len and self.input_data[self.input_pos] != 0: self.input_pos += 1
self.input_pos += 1
if flg & 0x10:
while self.input_pos < self.input_len and self.input_data[self.input_pos] != 0: self.input_pos += 1
self.input_pos += 1
if flg & 0x02:
self.input_pos += 2
if self.input_pos > self.input_len: return -3
else:
self.is_gzip = 0
if (b0 * 256 + b1) % 31 != 0: return -3
cm: t.CInt = b0 & 0x0F
if cm != 8: return -3
self.input_pos += 2
self.header_parsed = 1
return 0
def read_bits_from_input(self, nbits: t.CInt, out: UINTPTR) -> t.CInt:
val: UINT = 0
for i in range(nbits):
if self.input_pos >= self.input_len: return -3
if self.input_data[self.input_pos] & (UINT(1) << (self.reader.bit_pos)):
val |= (UINT(1) << i)
self.reader.bit_pos += 1
if self.reader.bit_pos == 8:
self.reader.bit_pos = 0
self.input_pos += 1
c.Set(c.Deref(out), val)
return 0
def read_huffman(self, dt: zhuff.zhuff_decode_tree | t.CPtr) -> t.CInt:
node: int = dt.root
while dt.nodes[node].symbol == -1:
bit: UINT
if self.read_bits_from_input(1, c.Addr(bit)) != 0: return -1
dir: int = t.CInt(bit)
if dt.nodes[node].children[dir] == -1: return -3
node = dt.nodes[node].children[dir]
return dt.nodes[node].symbol
def inflate_block_stored(self) -> t.CInt:
if self.reader.bit_pos != 0:
self.reader.bit_pos = 0
self.input_pos += 1
if self.input_pos + 4 > self.input_len: return -3
length: UINT = self.input_data[self.input_pos] | (self.input_data[self.input_pos + 1] << 8)
nlen: UINT = self.input_data[self.input_pos + 2] | (self.input_data[self.input_pos + 3] << 8)
self.input_pos += 4
if (length ^ nlen) != 0xFFFF: return -3
if self.input_pos + length > self.input_len: return -3
i: t.CUnsignedInt
for i in range(length):
self.output_byte(self.input_data[self.input_pos])
self.input_pos += 1
return 0
def inflate_block_fixed(self) -> t.CInt:
lit_tree = zhuff.zhuff_tree()
dist_tree = zhuff.zhuff_tree()
lit_tree.build_fixed_lit_tree()
dist_tree.build_fixed_dist_tree()
lit_dt: zhuff.zhuff_decode_tree | t.CPtr = zdef.zdef_alloc(self.pool, zhuff.zhuff_decode_tree.__sizeof__())
dist_dt: zhuff.zhuff_decode_tree | t.CPtr = zdef.zdef_alloc(self.pool, zhuff.zhuff_decode_tree.__sizeof__())
if not lit_dt or not dist_dt:
if lit_dt: zdef.zdef_free(self.pool, lit_dt)
if dist_dt: zdef.zdef_free(self.pool, dist_dt)
return -4
memset(lit_dt, 0, zhuff.zhuff_decode_tree.__sizeof__())
memset(dist_dt, 0, zhuff.zhuff_decode_tree.__sizeof__())
lit_dt.build_decode_tree(c.Addr(lit_tree))
dist_dt.build_decode_tree(c.Addr(dist_tree))
result: t.CInt = 0
while True:
if self.max_length > 0 and self.output_len >= self.max_length: break
sym: t.CInt = self.read_huffman(lit_dt)
if sym < 0:
result = -3
break
if sym == 256: break
if sym < 256:
self.output_byte(BYTE(sym))
else:
len_idx: t.CInt = sym - 257
if len_idx < 0 or len_idx >= 29:
result = -3
break
length: t.CInt = zdef.zdeflate_len_base[len_idx]
extra: t.CInt = zdef.zdeflate_len_extra_bits[len_idx]
if extra > 0:
extra_val: UINT
if self.read_bits_from_input(extra, c.Addr(extra_val)) != 0:
result = -3
break
length += extra_val
dist_sym: t.CInt = self.read_huffman(dist_dt)
if dist_sym < 0 or dist_sym >= 30:
result = -3
break
dist: t.CInt = zdef.zdeflate_dist_base[dist_sym]
extra = zdef.zdeflate_dist_extra_bits[dist_sym]
if extra > 0:
extra_val: UINT
if self.read_bits_from_input(extra, c.Addr(extra_val)) != 0:
result = -3
break
dist += extra_val
for i in range(length):
if self.max_length > 0 and self.output_len >= self.max_length: break
src_pos: t.CInt = (self.window_pos - dist) & (zdef.ZDEFLATE_WINDOW_SIZE - 1)
byte: BYTE = self.window[src_pos]
self.output_byte(byte)
zdef.zdef_free(self.pool, lit_dt)
zdef.zdef_free(self.pool, dist_dt)
return result
def inflate_block_dynamic(self) -> t.CInt:
hlit_val: UINT
hdist_val: UINT
hclen_val: UINT
if self.read_bits_from_input(5, c.Addr(hlit_val)) != 0: return -3
if self.read_bits_from_input(5, c.Addr(hdist_val)) != 0: return -3
if self.read_bits_from_input(4, c.Addr(hclen_val)) != 0: return -3
hlit: t.CInt = t.CInt(hlit_val) + 257
hdist: t.CInt = t.CInt(hdist_val) + 1
hclen: t.CInt = t.CInt(hclen_val) + 4
cl_lengths: t.CArray[t.CInt, 19]
memset(cl_lengths, 0, cl_lengths.__sizeof__())
for i in range(hclen):
len_val: UINT
if self.read_bits_from_input(3, c.Addr(len_val)) != 0: return -3
cl_lengths[zdef.zdeflate_codelen_order[i]] = t.CInt(len_val)
cl_tree = zhuff.zhuff_tree()
cl_tree.build_tree_from_lengths(cl_lengths, 19, 7)
cl_dt = zhuff.zhuff_decode_tree()
cl_dt.build_decode_tree(c.Addr(cl_tree))
total: t.CInt = hlit + hdist
all_lengths: t.CInt | t.CPtr = zdef.zdef_alloc(self.pool, total * int.__sizeof__())
if not all_lengths: return -4
idx: t.CInt = 0
while idx < total:
sym: t.CInt = self.read_huffman(c.Addr(cl_dt))
if sym < 0 or sym > 18:
zdef.zdef_free(self.pool, all_lengths)
return -3
if sym < 16:
all_lengths[idx] = sym
idx += 1
elif sym == 16:
rep: UINT
if self.read_bits_from_input(2, c.Addr(rep)) != 0:
zdef.zdef_free(self.pool, all_lengths)
return -3
rep += 3
if idx == 0 or idx + rep > total:
zdef.zdef_free(self.pool, all_lengths)
return -3
prev: t.CInt = all_lengths[idx - 1]
i: t.CUnsignedInt
for i in range(rep):
all_lengths[idx] = prev
idx += 1
elif sym == 17:
rep: t.CUnsignedInt
if self.read_bits_from_input(3, c.Addr(rep)) != 0:
zdef.zdef_free(self.pool, all_lengths)
return -3
rep += 3
if idx + rep > total:
zdef.zdef_free(self.pool, all_lengths)
return -3
i: t.CUnsignedInt
for i in range(rep):
all_lengths[idx] = 0
idx += 1
else:
rep: t.CUnsignedInt
if self.read_bits_from_input(7, c.Addr(rep)) != 0:
zdef.zdef_free(self.pool, all_lengths)
return -3
rep += 11
if idx + rep > total:
zdef.zdef_free(self.pool, all_lengths)
return -3
i: t.CUnsignedInt
for i in range(rep):
all_lengths[idx] = 0
idx += 1
lit_tree = zhuff.zhuff_tree()
dist_tree = zhuff.zhuff_tree()
lit_lengths: t.CArray[t.CInt, 288]
memset(lit_lengths, 0, lit_lengths.__sizeof__())
for i in range(hlit):
lit_lengths[i] = all_lengths[i]
lit_tree.build_tree_from_lengths(lit_lengths, hlit, zdef.ZDEFLATE_MAX_BITS)
dist_lengths: t.CArray[t.CInt, 32]
memset(dist_lengths, 0, dist_lengths.__sizeof__())
for i in range(hdist):
dist_lengths[i] = all_lengths[hlit + i]
dist_tree.build_tree_from_lengths(dist_lengths, hdist, zdef.ZDEFLATE_MAX_BITS)
zdef.zdef_free(self.pool, all_lengths)
lit_dt = zhuff.zhuff_decode_tree()
dist_dt = zhuff.zhuff_decode_tree()
lit_dt.build_decode_tree(c.Addr(lit_tree))
dist_dt.build_decode_tree(c.Addr(dist_tree))
while True:
if self.max_length > 0 and self.output_len >= self.max_length: return 0
sym: t.CInt = self.read_huffman(c.Addr(lit_dt))
if sym < 0: return -3
if sym == 256: return 0
if sym < 256:
self.output_byte(BYTE(sym))
else:
len_idx: t.CInt = sym - 257
if len_idx < 0 or len_idx >= 29: return -3
length: t.CInt = zdef.zdeflate_len_base[len_idx]
extra: t.CInt = zdef.zdeflate_len_extra_bits[len_idx]
if extra > 0:
extra_val: UINT
if self.read_bits_from_input(extra, c.Addr(extra_val)) != 0: return -3
length += extra_val
dist_sym: t.CInt = self.read_huffman(c.Addr(dist_dt))
if dist_sym < 0 or dist_sym >= 30: return -3
dist: t.CInt = zdef.zdeflate_dist_base[dist_sym]
extra = zdef.zdeflate_dist_extra_bits[dist_sym]
if extra > 0:
extra_val: UINT
if self.read_bits_from_input(extra, c.Addr(extra_val)) != 0: return -3
dist += extra_val
for i in range(length):
if self.max_length > 0 and self.output_len >= self.max_length: return 0
src_pos: t.CInt = (self.window_pos - dist) & (zdef.ZDEFLATE_WINDOW_SIZE - 1)
byte: BYTE = self.window[src_pos]
self.output_byte(byte)
def inflate_stream(self) -> t.CInt:
err: t.CInt = self.parse_header()
if err != 0: return err
while not self.is_finished:
bfinal_val: UINT
btype_val: UINT
if self.read_bits_from_input(1, c.Addr(bfinal_val)) != 0: return -3
if self.read_bits_from_input(2, c.Addr(btype_val)) != 0: return -3
bfinal: t.CInt = t.CInt(bfinal_val)
btype: t.CInt = t.CInt(btype_val)
if btype == 0:
err = self.inflate_block_stored()
elif btype == 1:
err = self.inflate_block_fixed()
elif btype == 2:
err = self.inflate_block_dynamic()
else:
return -3
if err != 0: return err
if bfinal: self.is_finished = 1
if self.max_length > 0 and self.output_len >= self.max_length: return 0
if self.is_gzip:
if self.reader.bit_pos != 0:
self.reader.bit_pos = 0
self.input_pos += 1
if self.input_pos + 8 <= self.input_len:
self.crc = ((t.CUInt32T(self.input_data[self.input_pos + 0]) << 0) |
(t.CUInt32T(self.input_data[self.input_pos + 1]) << 8) |
(t.CUInt32T(self.input_data[self.input_pos + 2]) << 16) |
(t.CUInt32T(self.input_data[self.input_pos + 3]) << 24))
self.input_pos += 8
elif self.wbits >= 0:
if self.reader.bit_pos != 0:
self.reader.bit_pos = 0
self.input_pos += 1
if self.input_pos + 4 <= self.input_len:
self.adler = ((t.CUInt32T(self.input_data[self.input_pos + 0]) << 24) |
(t.CUInt32T(self.input_data[self.input_pos + 1]) << 16) |
(t.CUInt32T(self.input_data[self.input_pos + 2]) << 8) |
(t.CUInt32T(self.input_data[self.input_pos + 3]) << 0))
self.input_pos += 4
return 0
def decompress(self, data: UINT8PTR, length: t.CSizeT,
max_length: t.CSizeT, out: t.CUInt8T | t.CPtr[t.CPtr], out_len: t.CSizeT | t.CPtr) -> t.CInt:
if not self.is_initialized: return -2
self.input_data = data
self.input_len = length
self.input_pos = 0
self.reader.bit_pos = 0
self.output_len = 0
self.max_length = max_length
err: t.CInt = self.inflate_stream()
if err != 0: return err
c.Set(c.Deref(out_len), self.output_len)
if self.output_len == 0:
c.Set(c.Deref(out), UINT8PTR(zdef.zdef_alloc(self.pool, 1)))
else:
dst: UINT8PTR = UINT8PTR(zdef.zdef_alloc(self.pool, self.output_len))
c.Set(c.Deref(out), dst)
if dst:
memcpy(dst, self.output, self.output_len)
return 0
def set_dictionary(self, dict: UINT8PTR, length: t.CSizeT) -> t.CInt:
if not dict: return -2
use: t.CSizeT = length
if use > zdef.ZDEFLATE_WINDOW_SIZE:
dict += length - zdef.ZDEFLATE_WINDOW_SIZE
use = zdef.ZDEFLATE_WINDOW_SIZE
memcpy(self.window, dict, use)
self.window_pos = t.CInt(use)
return 0
def copy(self) -> zinflate_stream | t.CPtr:
z: zinflate_stream | t.CPtr = zdef.zdef_alloc(self.pool, zinflate_stream.__sizeof__())
if not z: return None
memcpy(z, self, zinflate_stream.__sizeof__())
z.window = BYTEPTR(zdef.zdef_alloc(self.pool, zdef.ZDEFLATE_WINDOW_SIZE))
z.output = BYTEPTR(zdef.zdef_alloc(self.pool, self.output_cap))
if not z.window or not z.output:
zinflate_stream.destroy(z)
return None
memcpy(z.window, self.window, zdef.ZDEFLATE_WINDOW_SIZE)
memcpy(z.output, self.output, self.output_cap)
return z
def destroy(self):
if not self: return
if self.window: zdef.zdef_free(self.pool, self.window)
if self.output: zdef.zdef_free(self.pool, self.output)
zdef.zdef_free(self.pool, self)
def zinflate_one_shot(pool: memhub.MemManager | t.CPtr, data: UINT8PTR, length: t.CSizeT,
wbits: t.CInt, bufsize: t.CSizeT, out_len: t.CSizeT | t.CPtr) -> t.CUInt8T | t.CPtr:
s: zinflate_stream | t.CPtr = zinflate_create(pool, wbits)
if not s: return None
out: t.CUInt8T | t.CPtr = None
err: t.CInt = s.decompress(data, length, 0, c.Addr(out), out_len)
s.destroy()
if err != 0: return None
return out
def zinflate_create(pool: memhub.MemManager | t.CPtr, wbits: t.CInt) -> zinflate_stream | t.CPtr:
s: zinflate_stream | t.CPtr = zdef.zdef_alloc(pool, zinflate_stream.__sizeof__())
if not s: return None
memset(s, 0, zinflate_stream.__sizeof__())
s.pool = pool
s.wbits = wbits
s.is_initialized = 1
s.adler = 1
s.window = BYTEPTR(zdef.zdef_alloc(pool, zdef.ZDEFLATE_WINDOW_SIZE))
s.output = BYTEPTR(zdef.zdef_alloc(pool, 256))
s.output_cap = 256
s.output_len = 0
s.max_length = 0
if not s.window or not s.output:
s.destroy()
return None
return s