383 lines
21 KiB
Plaintext
383 lines
21 KiB
Plaintext
from stdint import *
|
|
import ast
|
|
import mlirlite
|
|
import memhub
|
|
import string
|
|
import stdlib
|
|
import viperlib
|
|
import lib.core.CompilerFactory as CompilerFactory
|
|
import lib.core.Gen.Translator as HT
|
|
import lib.Stub.StubGen.VpiReader as VpiReader
|
|
|
|
import lib.core.Gen.GenTypes as GenTypes
|
|
import lib.core.Gen.Expr.ExprGenBase as ExprGenBase
|
|
import lib.Stub.StubMerger.StubMergerStore as StubMergerStore
|
|
|
|
|
|
# docstring registry capacity constants (kept in sync with ExprGenMixin)
|
|
define DOC_MAX = 512
|
|
define DOC_KEY_SZ = 192
|
|
define DOC_VAL_SZ = 4096
|
|
|
|
|
|
# ExprGenCls - split of ExprGenMixin (stage D): class-name inference / docstring registry
|
|
novirtual class ExprGenCls(ExprGenBase.ExprGenBase):
|
|
"""Expression class inference and docstring registry."""
|
|
|
|
# docstring extract/register/lookup (__doc__ support): pre-registration scans
|
|
# ASTs into a global "{module_sha1}.{symbol}" table; lookups inline the string.
|
|
|
|
def _extract_docstring_from_body(self, body: list[ast.AST*]*) -> str:
|
|
"""Extract a docstring from a body when body[0] is an Expr(Constant str)."""
|
|
if body is None or len(body) == 0: return None
|
|
first: ast.AST* = body.get(0)
|
|
if first is None or first.kind() != ast.ASTKind.Expr: return None
|
|
ex: ast.Expr* = ast.Expr*(first)
|
|
if ex is None or ex.value is None or ex.value.kind() != ast.ASTKind.Constant: return None
|
|
cn: ast.Constant* = ast.Constant*(ex.value)
|
|
if cn is None or cn.const_kind != ast.CONST_STR: return None
|
|
return cn.str_val
|
|
def register_docstring(self, module_sha1: str, symbol: str, doc: str) -> None:
|
|
"""Register a docstring under key "{module_sha1}.{symbol}", deduplicated."""
|
|
if module_sha1 is None or symbol is None or doc is None: return
|
|
if string.strlen(doc) == 0: return
|
|
if self.DocKeys is None:
|
|
self.DocKeys = CompilerFactory.factory.PermPool.alloc(DOC_MAX * DOC_KEY_SZ)
|
|
self.DocValues = CompilerFactory.factory.PermPool.alloc(DOC_MAX * DOC_VAL_SZ)
|
|
if self.DocKeys is None or self.DocValues is None: return
|
|
string.memset(self.DocKeys, 0, DOC_MAX * DOC_KEY_SZ)
|
|
string.memset(self.DocValues, 0, DOC_MAX * DOC_VAL_SZ)
|
|
if self.DocCount >= DOC_MAX: return
|
|
ml: sizet = string.strlen(module_sha1)
|
|
sl: sizet = string.strlen(symbol)
|
|
key_buf: bytes = stdlib.malloc(ml + sl + 2)
|
|
if key_buf is None: return
|
|
viperlib.snprintf(key_buf, ml + sl + 2, "%s.%s", module_sha1, symbol)
|
|
for i in range(int(self.DocCount)):
|
|
if string.strcmp(self.DocKeys + i * DOC_KEY_SZ, key_buf) == 0:
|
|
stdlib.free(key_buf)
|
|
return
|
|
string.strcpy(self.DocKeys + self.DocCount * DOC_KEY_SZ, key_buf)
|
|
string.strcpy(self.DocValues + self.DocCount * DOC_VAL_SZ, doc)
|
|
self.DocCount += 1
|
|
stdlib.free(key_buf)
|
|
def lookup_docstring(self, module_sha1: str, symbol: str) -> str:
|
|
"""Look up a docstring by "{module_sha1}.{symbol}", or None on miss."""
|
|
if self.DocKeys is None or module_sha1 is None or symbol is None: return None
|
|
ml: sizet = string.strlen(module_sha1)
|
|
sl: sizet = string.strlen(symbol)
|
|
key_buf: bytes = stdlib.malloc(ml + sl + 2)
|
|
if key_buf is None: return None
|
|
viperlib.snprintf(key_buf, ml + sl + 2, "%s.%s", module_sha1, symbol)
|
|
for i in range(int(self.DocCount)):
|
|
if string.strcmp(self.DocKeys + i * DOC_KEY_SZ, key_buf) == 0:
|
|
stdlib.free(key_buf)
|
|
return self.DocValues + i * DOC_VAL_SZ
|
|
stdlib.free(key_buf)
|
|
return None
|
|
def extract_module_docstrings(self, tree: ast.AST*, module_sha1: str) -> None:
|
|
"""Scan a module AST and register docstrings of its functions, classes and methods."""
|
|
if tree is None or module_sha1 is None: return
|
|
ch: list[ast.AST*]* = tree.children
|
|
if ch is None: return
|
|
cn: sizet = len(ch)
|
|
for ci in range(int(cn)):
|
|
child: ast.AST* = ch.get(ci)
|
|
if child is None: continue
|
|
kd: int = child.kind()
|
|
if kd == ast.ASTKind.FunctionDef:
|
|
fd: ast.FunctionDef* = ast.FunctionDef*(child)
|
|
if fd is not None and fd.name is not None:
|
|
fd_body: list[ast.AST*]* = fd.children
|
|
fdoc: str = self._extract_docstring_from_body(fd_body)
|
|
if fdoc is not None: self.register_docstring(module_sha1, fd.name, fdoc)
|
|
elif kd == ast.ASTKind.ClassDef:
|
|
cd: ast.ClassDef* = ast.ClassDef*(child)
|
|
if cd is None or cd.name is None: continue
|
|
cd_body0: list[ast.AST*]* = cd.children
|
|
cdoc: str = self._extract_docstring_from_body(cd_body0)
|
|
if cdoc is not None: self.register_docstring(module_sha1, cd.name, cdoc)
|
|
# class method docstring (key="{ClassName}.{method}")
|
|
cd_body: list[ast.AST*]* = cd.children
|
|
if cd_body is not None:
|
|
cbn: sizet = len(cd_body)
|
|
for mi in range(int(cbn)):
|
|
m: ast.AST* = cd_body.get(mi)
|
|
if m is None or m.kind() != ast.ASTKind.FunctionDef: continue
|
|
mfd: ast.FunctionDef* = ast.FunctionDef*(m)
|
|
if mfd is None or mfd.name is None: continue
|
|
mdoc: str = self._extract_docstring_from_body(mfd.children)
|
|
if mdoc is not None:
|
|
cl_nl: sizet = string.strlen(cd.name)
|
|
mt_nl: sizet = string.strlen(mfd.name)
|
|
sym_buf: bytes = stdlib.malloc(cl_nl + mt_nl + 2)
|
|
if sym_buf is not None:
|
|
viperlib.snprintf(sym_buf, cl_nl + mt_nl + 2, "%s.%s", cd.name, mfd.name)
|
|
self.register_docstring(module_sha1, sym_buf, mdoc)
|
|
stdlib.free(sym_buf)
|
|
def _resolve_doc_symbol_name(self, trans: HT.Translator*, name_id: str, out_mod: str, out_sym: str) -> int:
|
|
"""Resolve a Name symbol's docstring owner into out_mod/out_sym; return 1 on success."""
|
|
if trans is None or name_id is None or out_mod is None or out_sym is None: return 0
|
|
if trans.ModuleSha1 is None: return 0
|
|
# 1. local variable (function/global)
|
|
if self.Gen.VarGen.lookup_var(trans.SymTab, name_id) is not None:
|
|
string.strcpy(out_mod, trans.ModuleSha1)
|
|
string.strcpy(out_sym, name_id)
|
|
return 1
|
|
# 2. struct class (entry.ModuleSha1 distinguishes cross-module pre-registration)
|
|
se: GenTypes.StructEntry* = self.Gen.StructGen.find_struct(name_id)
|
|
if se is not None:
|
|
if se.ModuleSha1 is not None: string.strcpy(out_mod, se.ModuleSha1)
|
|
else: string.strcpy(out_mod, trans.ModuleSha1)
|
|
string.strcpy(out_sym, name_id)
|
|
return 1
|
|
# 3. module name (rare); use StubMerger module-level functions to avoid a
|
|
# cross-Mixin self call that would emit a nonexistent symbol.
|
|
mod_sha1: str = StubMergerStore.LookupSha1ByModName(name_id)
|
|
if mod_sha1 is None: mod_sha1 = StubMergerStore.LookupSha1ByModNameSuffix(name_id)
|
|
if mod_sha1 is not None:
|
|
string.strcpy(out_mod, mod_sha1)
|
|
string.strcpy(out_sym, name_id)
|
|
return 1
|
|
# 4. fall back to the current module
|
|
string.strcpy(out_mod, trans.ModuleSha1)
|
|
string.strcpy(out_sym, name_id)
|
|
return 1
|
|
def _resolve_doc_symbol_attr(self, trans: HT.Translator*, at: ast.Attribute*, out_mod: str, out_sym: str) -> int:
|
|
"""Resolve a nested Attribute chain's docstring owner into out_mod/out_sym."""
|
|
if trans is None or at is None or out_mod is None or out_sym is None: return 0
|
|
if trans.ModuleSha1 is None: return 0
|
|
# collect the Attribute chain (outermost attr first)
|
|
DOC_CHAIN_MAX: int = 8
|
|
items: bytes = stdlib.malloc(DOC_CHAIN_MAX * 128)
|
|
if items is None: return 0
|
|
n_items: int = 0
|
|
cur: ast.AST* = (ast.AST*)(at)
|
|
while cur is not None and cur.kind() == ast.ASTKind.Attribute and n_items < DOC_CHAIN_MAX:
|
|
cat: ast.Attribute* = ast.Attribute*(cur)
|
|
if cat.attr is None: break
|
|
string.strcpy(items + n_items * 128, cat.attr)
|
|
n_items += 1
|
|
cur = cat.value
|
|
# leftmost must be a Name
|
|
head_id: str = None
|
|
if cur is not None and cur.kind() == ast.ASTKind.Name:
|
|
hnm: ast.Name* = ast.Name*(cur)
|
|
head_id = hnm.id
|
|
if head_id is None:
|
|
stdlib.free(items)
|
|
return 0
|
|
# head decision: module name or variable/class
|
|
is_mod: int = 0
|
|
mod_sha1: str = None
|
|
if self.Gen.VarGen.lookup_var(trans.SymTab, head_id) is None:
|
|
mod_sha1 = StubMergerStore.LookupSha1ByModName(head_id)
|
|
if mod_sha1 is None: mod_sha1 = StubMergerStore.LookupSha1ByModNameSuffix(head_id)
|
|
if mod_sha1 is not None: is_mod = 1
|
|
if is_mod != 0:
|
|
# module-qualified: symbol = items reversed (excluding head_id)
|
|
if n_items == 0:
|
|
string.strcpy(out_mod, mod_sha1)
|
|
string.strcpy(out_sym, head_id)
|
|
stdlib.free(items)
|
|
return 1
|
|
total: sizet = 0
|
|
for j in range(int(n_items)): total += string.strlen(items + j * 128) + 1
|
|
sym_buf: bytes = stdlib.malloc(total + 1)
|
|
if sym_buf is None:
|
|
stdlib.free(items)
|
|
return 0
|
|
sym_buf[0] = '\0'
|
|
for j in range(int(n_items)):
|
|
if j > 0: string.strcat(sym_buf, ".")
|
|
string.strcat(sym_buf, items + (n_items - 1 - j) * 128)
|
|
string.strcpy(out_mod, mod_sha1)
|
|
string.strcpy(out_sym, sym_buf)
|
|
stdlib.free(sym_buf)
|
|
stdlib.free(items)
|
|
return 1
|
|
# variable/class: symbol = head_id + '.' + items reversed
|
|
total2: sizet = string.strlen(head_id)
|
|
for j in range(int(n_items)): total2 += string.strlen(items + j * 128) + 1
|
|
sym2: bytes = stdlib.malloc(total2 + 1)
|
|
if sym2 is None:
|
|
stdlib.free(items)
|
|
return 0
|
|
sym2[0] = '\0'
|
|
string.strcat(sym2, head_id)
|
|
for j in range(int(n_items)):
|
|
string.strcat(sym2, ".")
|
|
string.strcat(sym2, items + (n_items - 1 - j) * 128)
|
|
se2: GenTypes.StructEntry* = self.Gen.StructGen.find_struct(head_id)
|
|
if se2 is not None and se2.ModuleSha1 is not None: string.strcpy(out_mod, se2.ModuleSha1)
|
|
else: string.strcpy(out_mod, trans.ModuleSha1)
|
|
string.strcpy(out_sym, sym2)
|
|
stdlib.free(sym2)
|
|
stdlib.free(items)
|
|
return 1
|
|
# P3 migrated methods (from HandlesExprOps.vp: operator overload + binary ops)
|
|
|
|
# infer_expr_class - infer the struct class name from an expression (shared by
|
|
# unannotated assignment and method dispatch); returns None if undecidable.
|
|
def infer_expr_class(self, pool: memhub.MemBuddy*, mod: mlirlite.LLVMModule*, node: ast.AST*, trans: HT.Translator*) -> str:
|
|
"""Infer the struct class name from an expression, or None if undecidable."""
|
|
if node is None or trans is None: return None
|
|
k: int = node.kind()
|
|
# Call: class instantiation / method call
|
|
if k == ast.ASTKind.Call:
|
|
cl: ast.Call* = ast.Call*(node)
|
|
if cl is None or cl.func is None: return None
|
|
if cl.func.kind() == ast.ASTKind.Name:
|
|
cand: str = (ast.Name*(cl.func)).id
|
|
if cand is None: return None
|
|
if self.Gen.StructGen.find_struct(cand) is not None: return cand
|
|
return None
|
|
if cl.func.kind() == ast.ASTKind.Attribute:
|
|
# Method call a.cross(b): resolve the return class from the vpi return
|
|
# annotation cache (may differ from the receiver class).
|
|
ca: ast.Attribute* = ast.Attribute*(cl.func)
|
|
if ca is None or ca.attr is None or ca.value is None: return None
|
|
# module-level call (Module.func()): receiver is an imported module name
|
|
if ca.value.kind() == ast.ASTKind.Name:
|
|
mod_nm2: str = (ast.Name*(ca.value)).id
|
|
if mod_nm2 is not None and trans._from_imports is not None:
|
|
mod_sha1_2: str = self.Gen.CallGenBase._lookup_mod_sha1(pool, trans._from_imports, mod_nm2)
|
|
if mod_sha1_2 is not None:
|
|
temp_dir2: str = None
|
|
if self.Gen.CallGenBase.TempDir is not None: temp_dir2 = self.Gen.CallGenBase.TempDir
|
|
if temp_dir2 is not None: VpiReader._ensure_vpi_ret_annot_loaded(mod_sha1_2, temp_dir2)
|
|
annot_ret2: str = VpiReader._lookup_vpi_ret_annot(mod_sha1_2, None, ca.attr)
|
|
if annot_ret2 is not None:
|
|
ret_cls2: str = self._extract_annot_class_name(pool, annot_ret2)
|
|
if ret_cls2 is not None: return ret_cls2
|
|
obj_cls: str = self.infer_expr_class(pool, mod, ca.value, trans)
|
|
if obj_cls is None: return None
|
|
if self._cls_has_dunder(pool, mod, obj_cls, ca.attr) == 0: return None
|
|
cls_sha1: str = self.Gen.StructGen.get_struct_sha1(obj_cls)
|
|
if cls_sha1 is None and trans.ModuleSha1 is not None: cls_sha1 = trans.ModuleSha1
|
|
ret_cls: str = None
|
|
if cls_sha1 is not None:
|
|
temp_dir: str = None
|
|
# cross sub-object access via self.Gen for CallGenBase.TempDir
|
|
if self.Gen.CallGenBase.TempDir is not None: temp_dir = self.Gen.CallGenBase.TempDir
|
|
if temp_dir is not None: VpiReader._ensure_vpi_ret_annot_loaded(cls_sha1, temp_dir)
|
|
annot_ret: str = VpiReader._lookup_vpi_ret_annot(cls_sha1, obj_cls, ca.attr)
|
|
if annot_ret is not None: ret_cls = self._extract_annot_class_name(pool, annot_ret)
|
|
return ret_cls
|
|
return None
|
|
# Name: source variable annotation
|
|
if k == ast.ASTKind.Name:
|
|
nm: ast.Name* = ast.Name*(node)
|
|
if nm is None or nm.id is None: return None
|
|
ve: GenTypes.VarEntry* = self.Gen.VarGen.lookup_var_entry(trans.SymTab, nm.id)
|
|
if ve is None: return None
|
|
acn_n: str = self.Gen.VarGen.var_annot_class_name(ve)
|
|
if acn_n is None: return None
|
|
# Strip the module prefix (json.JsonValue -> JsonValue): the lookups use
|
|
# bare names; generic names containing '[' are not truncated.
|
|
acn_dot_n: str = string.strrchr(acn_n, 46)
|
|
if acn_dot_n is not None and string.strchr(acn_n, 91) is None: acn_n = acn_dot_n + 1
|
|
return acn_n
|
|
# BinOp/UnaryOp: operator overload returns the same class object
|
|
if k == ast.ASTKind.BinOp:
|
|
bop: ast.BinOp* = ast.BinOp*(node)
|
|
if bop is None or bop.left is None: return None
|
|
dunder: str = self.Gen.ExprGenOps._binop_to_dunder(bop.op)
|
|
if dunder is None: return None
|
|
lhs_cls: str = self.infer_expr_class(pool, mod, bop.left, trans)
|
|
if lhs_cls is None or self._cls_has_dunder(pool, mod, lhs_cls, dunder) == 0: return None
|
|
return lhs_cls
|
|
if k == ast.ASTKind.UnaryOp:
|
|
uop: ast.UnaryOp* = ast.UnaryOp*(node)
|
|
if uop is None or uop.operand is None: return None
|
|
dunder: str = self.Gen.ExprGenOps._unaryop_to_dunder(uop.op)
|
|
if dunder is None: return None
|
|
opr_cls: str = self.infer_expr_class(pool, mod, uop.operand, trans)
|
|
if opr_cls is None or self._cls_has_dunder(pool, mod, opr_cls, dunder) == 0: return None
|
|
return opr_cls
|
|
# Attribute: field annotation class name (recursive chain, e.g. p = r.origin)
|
|
if k == ast.ASTKind.Attribute: return self.Gen.ExprGenAttr._resolve_attr_field_class(pool, trans, node)
|
|
return None
|
|
# _cls_has_dunder - whether a class (or its MRO ancestors) has the dunder/method;
|
|
# defined in this module or declared in a cross-module stub SHA1.
|
|
def _cls_has_dunder(self, pool: memhub.MemBuddy*, mod: mlirlite.LLVMModule*, class_name: str, dunder: str) -> int:
|
|
"""Return 1 if the class or its MRO ancestors define the dunder method."""
|
|
if class_name is None or dunder is None or mod is None: return 0
|
|
lookup_name: str = pool.alloc(128)
|
|
if lookup_name is None: return 0
|
|
viperlib.snprintf(lookup_name, 128, "%s.%s", class_name, dunder)
|
|
if self.Gen.CallGenBase.find_func_in_module(mod, lookup_name) is not None: return 1
|
|
# walk MRO ancestors
|
|
mro_ptr: sizet* = self.Gen.StructGen.get_mro_chain(pool, class_name)
|
|
mi: int = 1
|
|
while mro_ptr is not None and mro_ptr[mi] != 0:
|
|
anc: str = rawptr(mro_ptr[mi])
|
|
if anc is not None:
|
|
anc_lookup: str = pool.alloc(128)
|
|
if anc_lookup is not None:
|
|
viperlib.snprintf(anc_lookup, 128, "%s.%s", anc, dunder)
|
|
if self.Gen.CallGenBase.find_func_in_module(mod, anc_lookup) is not None: return 1
|
|
mi += 1
|
|
# cross-module: a class with a SHA1 (method declared in a stub) counts as existing
|
|
if self.Gen.StructGen.get_struct_sha1(class_name) is not None: return 1
|
|
return 0
|
|
# _extract_annot_class_name - extract a registered struct class name from a vpi
|
|
# return annotation, splitting on '|' and taking the first hit.
|
|
def _extract_annot_class_name(self, pool: memhub.MemBuddy*, annot: str) -> str:
|
|
"""Extract a registered struct class name from return annotation text, or None."""
|
|
if pool is None or annot is None: return None
|
|
alen: sizet = string.strlen(annot)
|
|
if alen == 0: return None
|
|
seg: str = pool.alloc(256)
|
|
if seg is None: return None
|
|
start: sizet = 0
|
|
i: sizet
|
|
for i in range(int(alen)):
|
|
if annot[i] == '|':
|
|
s: sizet = start
|
|
e: sizet = i
|
|
while s < e and (annot[s] == ' ' or annot[s] == '\'' or annot[s] == '"'): s += 1
|
|
# Also strip a trailing '*' (e.g. `{sha}.AST*`), matching the tail
|
|
# stripping in ExprGenName._list_elem_obj_ty_from_struct_name.
|
|
while e > s and (annot[e - 1] == ' ' or annot[e - 1] == '\'' or annot[e - 1] == '"' or annot[e - 1] == '*'): e -= 1
|
|
seglen: sizet = e - s
|
|
if seglen > 0 and seglen < 256:
|
|
string.strncpy(seg, annot + s, seglen)
|
|
seg[seglen] = '\0'
|
|
# Unwrap the CPtr[T] pointer marker: the class name is inside.
|
|
while seglen > 6 and string.strncmp(seg, "CPtr[", 5) == 0 and seg[seglen - 1] == ']':
|
|
seg = seg + 5
|
|
seglen -= 6
|
|
seg[seglen] = '\0'
|
|
if self.Gen.StructGen.find_struct(seg) is not None: return seg
|
|
# On a find_struct miss, retry with the last '.'-prefix (module
|
|
# qualifier) stripped: memhub.MemBuddy -> MemBuddy.
|
|
seg_dot2: str = string.strrchr(seg, 46)
|
|
if seg_dot2 is not None and seg_dot2 != seg:
|
|
seg_bare2: str = seg_dot2 + 1
|
|
if self.Gen.StructGen.find_struct(seg_bare2) is not None: return seg_bare2
|
|
start = i + 1
|
|
# last segment (no '|' suffix)
|
|
s: sizet = start
|
|
e: sizet = alen
|
|
while s < e and (annot[s] == ' ' or annot[s] == '\'' or annot[s] == '"'): s += 1
|
|
# Also strip a trailing '*' (e.g. `{sha}.AST*`), matching the tail
|
|
# stripping in ExprGenName._list_elem_obj_ty_from_struct_name.
|
|
while e > s and (annot[e - 1] == ' ' or annot[e - 1] == '\'' or annot[e - 1] == '"' or annot[e - 1] == '*'): e -= 1
|
|
seglen: sizet = e - s
|
|
if seglen > 0 and seglen < 256:
|
|
string.strncpy(seg, annot + s, seglen)
|
|
seg[seglen] = '\0'
|
|
# Unwrap the CPtr[T] pointer marker: the class name is inside.
|
|
while seglen > 6 and string.strncmp(seg, "CPtr[", 5) == 0 and seg[seglen - 1] == ']':
|
|
seg = seg + 5
|
|
seglen -= 6
|
|
seg[seglen] = '\0'
|
|
if self.Gen.StructGen.find_struct(seg) is not None: return seg
|
|
# On a find_struct miss, retry with the last '.'-prefix (module
|
|
# qualifier) stripped: memhub.MemBuddy -> MemBuddy.
|
|
seg_dot3: str = string.strrchr(seg, 46)
|
|
if seg_dot3 is not None and seg_dot3 != seg:
|
|
seg_bare3: str = seg_dot3 + 1
|
|
if self.Gen.StructGen.find_struct(seg_bare3) is not None: return seg_bare3
|
|
return None |