Files
TransPyC/includes/memhub.py

558 lines
21 KiB
Python

import t, c
from stdint import *
import string
import atom
import viperio
import stdio
# ============================================================
# memhub - Unified memory management library
# MemManager (polymorphic base class, vtable)
# ├─ MemPool (arena bump allocator, no support for individual free)
# ├─ MemSlab (fixed-size block pool, alloc/free tracked via bitmap)
# └─ MemBuddy (buddy system, supports alloc/free/realloc with coalescing)
# All subclasses override alloc/free/reset via vtable, invokable polymorphically through MemManager pointer
# ============================================================
MEMHUB_ALIGN: t.CDefine = 8
# MemSlab bitmap constants
MEMSLAB_MIN_BLOCK: t.CDefine = 16
MEMSLAB_BITMAP_BYTES: t.CDefine = 256
# MemBuddy buddy constants
MEMBUDDY_MIN_BLOCK: t.CDefine = 32
MEMBUDDY_MAX_ORDERS: t.CDefine = 32
MEMBUDDY_HEADER_SIZE: t.CDefine = 8
def _align_up(val: t.CSizeT, align: t.CSizeT) -> t.CSizeT:
if align == 0: return val
return (val + align - 1) & ~(align - 1)
def _largest_pow2_le(val: t.CSizeT) -> t.CSizeT:
if val == 0: return 0
p: t.CSizeT = 1
while p * 2 <= val:
p = p * 2
return p
def _block_size_at_order(order: t.CInt) -> t.CSizeT:
bs: t.CSizeT = MEMBUDDY_MIN_BLOCK
i: t.CInt
for i in range(order):
bs = bs << 1
return bs
# ============================================================
# MemManager - Polymorphic base class
# @t.CVTable explicit annotation to ensure recognition as vtable class during cross-module imports
# (HandlesImports only detects IsCVTable via decorator; automatic inference is unavailable across modules)
# ============================================================
@t.CVTable
class MemManager:
__provides__: list[str] = ['__memmgr__']
base: t.CVoid | t.CPtr # Starting address of memory region
size: t.CSizeT # Total size of memory region
def __init__(self, base: t.CVoid | t.CPtr, size: t.CSizeT):
self.base = base
self.size = size
# === Virtual functions (overridden by subclasses) ===
def alloc(self, size: t.CSizeT) -> t.CVoid | t.CPtr:
# Default: allocation not supported
return None
def free(self, ptr: t.CVoid | t.CPtr) -> t.CInt:
# Default: no operation
return 0
def reset(self) -> t.CInt:
# Default: no operation
return 0
# === Concrete methods (invoke virtual functions for automatic polymorphic dispatch) ===
def calloc(self, count: t.CSizeT, size: t.CSizeT) -> t.CVoid | t.CPtr:
total: t.CSizeT = count * size
ptr: t.CVoid | t.CPtr = self.alloc(total)
if ptr is not None:
string.memset(ptr, 0, total)
return ptr
def realloc(self, ptr: t.CVoid | t.CPtr, new_size: t.CSizeT) -> t.CVoid | t.CPtr:
if ptr is None:
return self.alloc(new_size)
if new_size == 0:
self.free(ptr)
return None
new_ptr: t.CVoid | t.CPtr = self.alloc(new_size)
if new_ptr is None:
return ptr
# Base class cannot track original allocation size; data will not be copied. Subclasses may override to preserve data
self.free(ptr)
return new_ptr
def __enter__(self) -> 'MemManager' | t.CPtr:
return self
def __exit__(self):
self.reset()
def alloc_buf(self, capacity: t.CSizeT) -> viperio.Buf | t.CPtr:
buf: viperio.Buf | t.CPtr = self.alloc(viperio.Buf.__sizeof__())
if buf is None:
return None
data_ptr: t.CVoid | t.CPtr = self.alloc(capacity)
if data_ptr is None:
return None
buf.__before_init__()
buf.__init__(t.CChar(t.CUInt64T(data_ptr), t.CPtr), capacity)
return buf
# ============================================================
# MemPool - Arena bump allocator
# Individual free operations are unsupported; reset reclaims all memory at once
# ============================================================
class MemPool(MemManager):
offset: t.CSizeT # Bump pointer offset
high_water: t.CSizeT # High-water mark
def __init__(self, base: t.CVoid | t.CPtr, size: t.CSizeT):
self.base = base
self.size = size
self.offset = 0
self.high_water = 0
def alloc(self, size: t.CSizeT) -> t.CVoid | t.CPtr:
if size == 0: return None
aligned: t.CSizeT = _align_up(size, MEMHUB_ALIGN)
if self.offset + aligned > self.size: return None
ptr: t.CVoid | t.CPtr = t.CVoid(t.CUInt64T(self.base) + self.offset, t.CPtr)
self.offset += aligned
if self.offset > self.high_water:
self.high_water = self.offset
return ptr
def free(self, ptr: t.CVoid | t.CPtr) -> t.CInt:
# Bump allocator does not support individual free operations
return 1
def reset(self) -> t.CInt:
self.offset = 0
self.high_water = 0
return 1
# ============================================================
# MemSlab - Fixed-size block pool
# Bitmap stored at the start of arena; subsequent area split into block_size units managed by free list
# ============================================================
class MemSlab(MemManager):
block_size: t.CSizeT # Block size (aligned)
block_count: t.CSizeT # Total block count
used_count: t.CSizeT # Used block count
free_list: t.CVoid | t.CPtr # Free list head
alloc_map: t.CUInt8T | t.CPtr # Allocation bitmap (at start of arena)
alloc_map_size: t.CSizeT # Bitmap byte count
usable: t.CVoid | t.CPtr # Usable block area start (after bitmap)
usable_size: t.CSizeT # Usable block area size
def __init__(self, base: t.CVoid | t.CPtr, size: t.CSizeT, block_size: t.CSizeT):
self.base = base
self.size = size
self.block_size = 0
self.block_count = 0
self.used_count = 0
self.free_list = None
self.alloc_map = None
self.alloc_map_size = 0
self.usable = None
self.usable_size = 0
if base is None: return
bs: t.CSizeT = _align_up(block_size, MEMHUB_ALIGN)
if bs < MEMSLAB_MIN_BLOCK: bs = MEMSLAB_MIN_BLOCK
map_bytes: t.CSizeT = _align_up(MEMSLAB_BITMAP_BYTES, MEMHUB_ALIGN)
if size < map_bytes + bs: return
self.alloc_map = base
self.alloc_map_size = map_bytes
self.usable = t.CVoid(t.CUInt64T(base) + map_bytes, t.CPtr)
self.usable_size = size - map_bytes
self.block_size = bs
# Clear bitmap
idx: t.CSizeT = 0
while idx < map_bytes:
self.alloc_map[idx] = 0
idx += 1
self.block_count = self.usable_size / bs
if self.block_count == 0: return
# Construct free list: the first 8 bytes of each block store pointer to next block
self.free_list = None
i: t.CSizeT = 0
while i < self.block_count:
block: t.CVoid | t.CPtr = t.CVoid(t.CUInt64T(self.usable) + i * bs, t.CPtr)
c.DerefAs(block, self.free_list)
self.free_list = block
i += 1
def alloc(self, size: t.CSizeT) -> t.CVoid | t.CPtr:
# Slab mode: allocate one block only if requested size <= block_size
if self.free_list is None: return None
if size > self.block_size: return None
block: t.CVoid | t.CPtr = self.free_list
self.free_list = t.CVoid(c.Deref(t.CUInt64T(block, t.CPtr)), t.CPtr)
self.used_count += 1
idx: t.CSizeT = (t.CUInt64T(block) - t.CUInt64T(self.usable)) / self.block_size
self.alloc_map[idx / 8] = self.alloc_map[idx / 8] | t.CUInt8T(1 << (idx % 8))
return block
def free(self, ptr: t.CVoid | t.CPtr) -> t.CInt:
if ptr is None: return 0
p: t.CUInt64T = t.CUInt64T(ptr)
if p < t.CUInt64T(self.usable): return 0
if p >= t.CUInt64T(self.usable) + self.block_count * self.block_size: return 0
if (p - t.CUInt64T(self.usable)) % self.block_size != 0: return 0
idx: t.CSizeT = (p - t.CUInt64T(self.usable)) / self.block_size
byte_idx: t.CSizeT = idx / 8
bit_idx: t.CInt = t.CInt(idx % 8)
if (self.alloc_map[byte_idx] >> bit_idx) & 1 == 0: return 0
self.alloc_map[byte_idx] = self.alloc_map[byte_idx] & t.CUInt8T(~(1 << bit_idx))
c.DerefAs(ptr, self.free_list)
self.free_list = ptr
self.used_count -= 1
return 1
def reset(self) -> t.CInt:
self.used_count = 0
self.free_list = None
i: t.CSizeT = 0
while i < self.block_count:
block: t.CVoid | t.CPtr = t.CVoid(t.CUInt64T(self.usable) + i * self.block_size, t.CPtr)
c.DerefAs(block, self.free_list)
self.free_list = block
if i / 8 < self.alloc_map_size:
self.alloc_map[i / 8] = 0
i += 1
return 1
# ============================================================
# MemBuddy - Buddy system allocator
# Free list head array stored at arena start; subsequent memory managed in power-of-two sized blocks
# Thread-safe (spinlock enabled), supports block coalescing
# ============================================================
class MemBuddy(MemManager):
# Override parent __provides__: MemBuddy provides both __mbuddy__ and __memmgr__.
# Within `with MemBuddy(...)` context, classes (e.g. _str) with __requires__=['__mbuddy__']
# can be automatically injected via _find_provider
__provides__: list[str] = ['__mbuddy__', '__memmgr__']
max_order: t.CInt # Maximum order
free_lists: t.CUInt64T | t.CPtr # Free list head array (located at arena start)
lock_val: t.CVolatile | t.CInt # Spinlock
usable: t.CVoid | t.CPtr # Start of usable memory region (skips free_lists)
usable_size: t.CSizeT # Size of usable memory (power of two)
def __init__(self, base: t.CVoid | t.CPtr, size: t.CSizeT):
self.base = base
self.size = size
self.max_order = 0
self.free_lists = None
self.lock_val = 0
self.usable = None
self.usable_size = 0
self.lock_val = 0
fl_bytes: t.CSizeT = (MEMBUDDY_MAX_ORDERS + 1) * 8
self.free_lists = base
# Initialize all free list heads to be 0
i: t.CInt
for i in range(MEMBUDDY_MAX_ORDERS + 1):
self.free_lists[i] = 0
if size <= fl_bytes:
self.usable = None
self.usable_size = 0
self.max_order = 0
return
remaining: t.CSizeT = size - fl_bytes
usable: t.CSizeT = _largest_pow2_le(remaining)
if usable < MEMBUDDY_MIN_BLOCK:
self.usable = None
self.usable_size = 0
self.max_order = 0
return
self.usable = t.CVoid(t.CUInt64T(base) + fl_bytes, t.CPtr)
self.usable_size = usable
# 计算 max_order: log2(usable / MIN_BLOCK)
self.max_order = 0
bs: t.CSizeT = MEMBUDDY_MIN_BLOCK
while bs < usable:
bs = bs << 1
self.max_order += 1
# Entire usable memory region as a max_order free block
self._fl_push(self.max_order, self.usable)
# === Free list operations ===
def _fl_push(self, order: t.CInt, block: t.CVoid | t.CPtr):
old_head: t.CUInt64T = self.free_lists[order]
c.DerefAs(block, t.CVoid(old_head, t.CPtr))
self.free_lists[order] = t.CUInt64T(block)
def _fl_pop(self, order: t.CInt) -> t.CVoid | t.CPtr:
head_val: t.CUInt64T = self.free_lists[order]
if head_val == 0:
return None
block: t.CVoid | t.CPtr = t.CVoid(head_val, t.CPtr)
next_ptr: t.CVoid | t.CPtr = t.CVoid(c.Deref(t.CUInt64T(block, t.CPtr)), t.CPtr)
self.free_lists[order] = t.CUInt64T(next_ptr)
return block
def _fl_find_and_remove(self, order: t.CInt, target: t.CVoid | t.CPtr) -> t.CInt:
head_val: t.CUInt64T = self.free_lists[order]
if head_val == 0:
return 0
head: t.CVoid | t.CPtr = t.CVoid(head_val, t.CPtr)
if t.CUInt64T(head) == t.CUInt64T(target):
next_ptr: t.CVoid | t.CPtr = t.CVoid(c.Deref(t.CUInt64T(head, t.CPtr)), t.CPtr)
self.free_lists[order] = t.CUInt64T(next_ptr)
return 1
prev: t.CVoid | t.CPtr = head
cur: t.CVoid | t.CPtr = t.CVoid(c.Deref(t.CUInt64T(head, t.CPtr)), t.CPtr)
while t.CUInt64T(cur) != 0:
if t.CUInt64T(cur) == t.CUInt64T(target):
next_ptr: t.CVoid | t.CPtr = t.CVoid(c.Deref(t.CUInt64T(cur, t.CPtr)), t.CPtr)
c.DerefAs(prev, next_ptr)
return 1
prev = cur
cur = t.CVoid(c.Deref(t.CUInt64T(cur, t.CPtr)), t.CPtr)
return 0
# === Buddy system core ===
def _buddy_of(self, block: t.CVoid | t.CPtr, order: t.CInt) -> t.CVoid | t.CPtr:
offset: t.CSizeT = t.CUInt64T(block) - t.CUInt64T(self.usable)
bs: t.CSizeT = _block_size_at_order(order)
buddy_offset: t.CSizeT = offset ^ bs
return t.CVoid(t.CUInt64T(self.usable) + buddy_offset, t.CPtr)
def _order_for_size(self, size: t.CSizeT) -> t.CInt:
order: t.CInt = 0
bs: t.CSizeT = MEMBUDDY_MIN_BLOCK
while bs < size:
bs = bs << 1
order += 1
return order
def _split_to_order(self, to_order: t.CInt) -> t.CVoid | t.CPtr:
found_order: t.CInt = to_order
while found_order <= self.max_order:
if self.free_lists[found_order] != 0:
break
found_order += 1
if found_order > self.max_order:
return None
block: t.CVoid | t.CPtr = self._fl_pop(found_order)
while found_order > to_order:
found_order -= 1
bs: t.CSizeT = _block_size_at_order(found_order)
buddy: t.CVoid | t.CPtr = t.CVoid(t.CUInt64T(block) + bs, t.CPtr)
self._fl_push(found_order, buddy)
return block
def _coalesce(self, block: t.CVoid | t.CPtr, order: t.CInt):
while order < self.max_order:
buddy: t.CVoid | t.CPtr = self._buddy_of(block, order)
found: t.CInt = self._fl_find_and_remove(order, buddy)
if found == 0:
break
if t.CUInt64T(buddy) < t.CUInt64T(block):
block = buddy
order += 1
self._fl_push(order, block)
def _is_valid_ptr(self, ptr: t.CVoid | t.CPtr) -> t.CInt:
if ptr is None: return 0
if self.usable is None: return 0
block: t.CVoid | t.CPtr = t.CVoid(t.CUInt64T(ptr) - MEMBUDDY_HEADER_SIZE, t.CPtr)
if t.CUInt64T(block) < t.CUInt64T(self.usable): return 0
if t.CUInt64T(block) >= t.CUInt64T(self.usable) + self.usable_size: return 0
offset: t.CSizeT = t.CUInt64T(block) - t.CUInt64T(self.usable)
if offset % MEMBUDDY_MIN_BLOCK != 0: return 0
stored: t.CVoid | t.CPtr = t.CVoid(c.Deref(t.CUInt64T(block, t.CPtr)), t.CPtr)
stored_val: t.CUInt64T = t.CUInt64T(stored)
if (stored_val & 1) == 0: return 0
order: t.CInt = t.CInt(stored_val >> 1)
if order < 0: return 0
if order > self.max_order: return 0
return 1
# === Spinlock ===
def _lock(self): # can use the spinlock library for decoupling
while atom.__atomic_test_and_set(c.Addr(self.lock_val), atom.ATOMIC_ACQUIRE):
pass
def _unlock(self):
atom.__atomic_clear(c.Addr(self.lock_val), atom.ATOMIC_RELEASE)
# === Virtual function overrides ===
def alloc(self, size: t.CSizeT) -> t.CVoid | t.CPtr:
self._lock()
result: t.CVoid | t.CPtr = None
if self.usable is not None:
if size != 0:
needed: t.CSizeT = size + MEMBUDDY_HEADER_SIZE
order: t.CInt = self._order_for_size(needed)
if order <= self.max_order:
block: t.CVoid | t.CPtr = self._split_to_order(order)
if block is not None:
c.DerefAs(block, t.CVoid(t.CUInt64T((order << 1) | 1), t.CPtr))
result = t.CVoid(t.CUInt64T(block) + MEMBUDDY_HEADER_SIZE, t.CPtr)
if result is None:
# Allocation failure: print memory usage statistics
fb: t.CSizeT = self.free_bytes()
ub: t.CSizeT = self.usable_size - fb
stdio.printf("[MEM-FAIL] alloc(%zu) failed: total=%zu used=%zu free=%zu free_blocks=%zu\n",
size, self.usable_size, ub, fb, self.free_count())
self._unlock()
return result
def free(self, ptr: t.CVoid | t.CPtr) -> t.CInt:
self._lock()
if ptr is not None:
if self._is_valid_ptr(ptr) != 0:
block: t.CVoid | t.CPtr = t.CVoid(t.CUInt64T(ptr) - MEMBUDDY_HEADER_SIZE, t.CPtr)
stored: t.CVoid | t.CPtr = t.CVoid(c.Deref(t.CUInt64T(block, t.CPtr)), t.CPtr)
stored_val: t.CUInt64T = t.CUInt64T(stored)
order: t.CInt = t.CInt(stored_val >> 1)
c.DerefAs(block, t.CVoid(0, t.CPtr))
self._coalesce(block, order)
self._unlock()
return 1
def reset(self) -> t.CInt:
if self.usable is None:
return 0
i: t.CInt
for i in range(MEMBUDDY_MAX_ORDERS + 1):
self.free_lists[i] = 0
self._fl_push(self.max_order, self.usable)
return 1
# realloc override: buddy system reads original order from block header; old_size is not required
def realloc(self, ptr: t.CVoid | t.CPtr, new_size: t.CSizeT) -> t.CVoid | t.CPtr:
if ptr is None:
return self.alloc(new_size)
if new_size == 0:
self.free(ptr)
return None
if self._is_valid_ptr(ptr) == 0:
return None
block: t.CVoid | t.CPtr = t.CVoid(t.CUInt64T(ptr) - MEMBUDDY_HEADER_SIZE, t.CPtr)
stored: t.CVoid | t.CPtr = t.CVoid(c.Deref(t.CUInt64T(block, t.CPtr)), t.CPtr)
stored_val: t.CUInt64T = t.CUInt64T(stored)
old_order: t.CInt = t.CInt(stored_val >> 1)
needed: t.CSizeT = new_size + MEMBUDDY_HEADER_SIZE
new_order: t.CInt = self._order_for_size(needed)
if new_order <= old_order:
return ptr
new_ptr: t.CVoid | t.CPtr = self.alloc(new_size)
if new_ptr is None:
return ptr
old_block_size: t.CSizeT = _block_size_at_order(old_order)
old_data_size: t.CSizeT = old_block_size - MEMBUDDY_HEADER_SIZE
string.memcpy(new_ptr, ptr, old_data_size)
self.free(ptr)
return new_ptr
# === Status Queries ===
@property
def mem_size(self) -> t.CSizeT:
# Total usable region size (power of two). Maximum allocatable size = mem_size - MEMBUDDY_HEADER_SIZE
return self.usable_size
def stats(self) -> t.CSizeT:
# Return total usable region size, for status statistics
return self.usable_size
def free_count(self) -> t.CSizeT:
# Count total blocks in all order lists for status statistics
count: t.CSizeT = 0
i: t.CInt
for i in range(MEMBUDDY_MAX_ORDERS + 1):
head_val: t.CUInt64T = self.free_lists[i]
cur: t.CVoid | t.CPtr = t.CVoid(head_val, t.CPtr)
while t.CUInt64T(cur) != 0:
count += 1
cur = t.CVoid(c.Deref(t.CUInt64T(cur, t.CPtr)), t.CPtr)
return count
def free_bytes(self) -> t.CSizeT:
# Count total bytes in all free blocks for status statistics
total: t.CSizeT = 0
i: t.CInt
for i in range(MEMBUDDY_MAX_ORDERS + 1):
head_val: t.CUInt64T = self.free_lists[i]
cur: t.CVoid | t.CPtr = t.CVoid(head_val, t.CPtr)
while t.CUInt64T(cur) != 0:
total += _block_size_at_order(i)
cur = t.CVoid(c.Deref(t.CUInt64T(cur, t.CPtr)), t.CPtr)
return total
def used_bytes(self) -> t.CSizeT:
# Used bytes = total usable region size - free bytes in all free blocks
return self.usable_size - self.free_bytes()
def dump_stats(self, label: t.CChar | t.CPtr):
# Print memory usage statistics
fb: t.CSizeT = self.free_bytes()
ub: t.CSizeT = self.usable_size - fb
stdio.printf("[MEM] %s: total=%zu used=%zu (%zu%%) free=%zu free_blocks=%zu\n",
label, self.usable_size, ub,
(ub * 100) / self.usable_size if self.usable_size > 0 else 0,
fb, self.free_count())
def self_check(self) -> t.CInt:
# Validate buddy allocator internal consistency, return 0=OK, non-0=damage
if self.usable is None:
return 0
i: t.CInt
for i in range(MEMBUDDY_MAX_ORDERS + 1):
head_val: t.CUInt64T = self.free_lists[i]
cur: t.CVoid | t.CPtr = t.CVoid(head_val, t.CPtr)
while t.CUInt64T(cur) != 0:
# Check pointer is in usable region range
if t.CUInt64T(cur) < t.CUInt64T(self.usable):
return 1
if t.CUInt64T(cur) >= t.CUInt64T(self.usable) + self.usable_size:
return 2
# Check alignment: order i blocks must be aligned to size _block_size_at_order(i)
offset: t.CSizeT = t.CUInt64T(cur) - t.CUInt64T(self.usable)
bs: t.CSizeT = _block_size_at_order(i)
if offset % bs != 0:
return 3
cur = t.CVoid(c.Deref(t.CUInt64T(cur, t.CPtr)), t.CPtr)
return 0