chore: 同步 TransPyV→ViperC 改名、new 表达式支持与定宽字面量/窄化适配

This commit is contained in:
TermiNexus committed 2026-10-12 20:59:39 +08:00
1 parent 15f77da9d3
commit 23b47053d9
10 files changed
+10283 -10041

No files matched your search

File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
+5 -1
View File
@@ -38,7 +38,11 @@ class LlvmCodeGenerator(
"""
#builder: ir.IRBuilder = None
#_Trans: LlvmGeneratorMixin = None
pass
# `new memgr(x) T(args)` 的存储覆盖:_HandleNewExprLlvm 取好池存储(T*)后置入,
# _HandleClassNewLlvm / _HandleREnumConstructLlvm 有则复用之(否则各自 alloca)。
# 默认 None = 无覆盖;构造结束由 try/finally 复位。
NewStorageOverride: object = None
# 向后兼容:重新导出常量和辅助类
+1621 -1656
View File
File diff suppressed because it is too large. Load diff
+93 -2
View File
@@ -23,6 +23,30 @@ from lib.vast.Tokenizer import (
)
# ---------------------------------------------------------------------------
# `new [描述符...] T(args)` 生命周期分配表达式(与 ViperC includes/ast/exprs.vp
# 的 NEW_SRC_*/NEW_LIFE_* 同值)。TPC 侧同样按 VPC 口径解析成 New 节点,语义落地
# 由 Handles 负责(见 HandlesExprCall._HandleNewExprLlvm)。
# ---------------------------------------------------------------------------
NEW_SRC_DEFAULT = 0
NEW_SRC_STACK = 1
NEW_SRC_HEAP = 2
NEW_SRC_MEMGR = 3
NEW_LIFE_DEFAULT = 0
NEW_LIFE_SLIFE = 1
NEW_LIFE_HLIFE = 2
# `new` 之后的普通名不能是这些保留字(它们在 vast 里也是 NAME token)。
# 缺此判据时 `for new in x:` 会被误判为 new 表达式(`in` 紧随其后)。
_NEW_NEXT_RESERVED: frozenset[str] = frozenset({
'in', 'is', 'and', 'or', 'not', 'if', 'else', 'elif', 'for', 'while',
'lambda', 'await', 'yield', 'return', 'pass', 'break', 'continue',
'import', 'from', 'as', 'with', 'try', 'except', 'finally', 'raise',
'def', 'class', 'global', 'nonlocal', 'del', 'assert', 'match', 'case',
'True', 'False', 'None',
})
def _JoinConstParts(parts: list[object]) -> str | bytes:
"""拼接连续字符串字面量值(str 或 bytes,与 CPython 语义一致)。"""
if not parts:
@@ -705,6 +729,58 @@ class ExprParserMixin(TokenStream):
# ==================================================================
# 原子表达式
# ==================================================================
def _ParseNewExpr(self) -> N.expr:
"""`new [hlife|slife] [stack|heap|memgr(x)] T(args)`(软关键字,与 VPC 同口径)。
描述符顺序无关、各轴最多一次;T 走主表达式(Name/Attribute/Subscript + 调用
trailer),若解析结果本身是 Call(`T(args)`)则拆成 func/args,否则 args=None。
结果恒为 T*(语义在 Handles 侧落地:取存储 + 构造)。
"""
start: Token = self._Next() # 吃掉 `new`
src: int = NEW_SRC_DEFAULT
life: int = NEW_LIFE_DEFAULT
memgr: N.expr | None = None
while True:
tok: Token = self._Peek()
if tok.Type != NAME:
break
v: str = tok.Value
if v == 'hlife':
self._Next()
life = NEW_LIFE_HLIFE
continue
if v == 'slife':
self._Next()
life = NEW_LIFE_SLIFE
continue
if v == 'stack':
self._Next()
src = NEW_SRC_STACK
continue
if v == 'heap':
self._Next()
src = NEW_SRC_HEAP
continue
if v == 'memgr':
self._Next()
self._ExpectOp('(')
self._SkipNL()
memgr = self.ParseExpr()
self._SkipNL()
self._ExpectOp(')')
src = NEW_SRC_MEMGR
continue
break
callee: N.expr = self._ParsePrimary()
func: N.expr = callee
args: list[N.expr] | None = None
if isinstance(callee, N.Call):
func = callee.func
args = callee.args
node: N.New = N.New(Source=src, Life=life, memgr=memgr, func=func, args=args)
self._FixLoc(node, start)
return node
def _ParseAtom(self) -> N.expr:
self._SkipNL()
tok: Token = self._Peek()
@@ -713,6 +789,10 @@ class ExprParserMixin(TokenStream):
if tok.Type in (STRING, FSTRING):
return self._ParseStringConcat()
if tok.Type == NAME:
if tok.Value == 'new':
nxt: Token = self._Peek(1)
if nxt.Type == NAME and nxt.Value not in _NEW_NEXT_RESERVED:
return self._ParseNewExpr()
if tok.Value in ('True', 'False', 'None'):
self._Next()
node: N.Constant = N.Constant(value={'True': True, 'False': False, 'None': None}[tok.Value], kind=None)
@@ -744,6 +824,13 @@ class ExprParserMixin(TokenStream):
def _ParseNumber(self) -> N.Constant:
tok: Token = self._Next()
text: str = tok.Value
# 显式宽度后缀(_int<N>/_uint<N>/_i<N>/_u<N>):先从文本里剥掉再解析数值
width: int = getattr(tok, 'IntWidth', 0)
unsigned: int = getattr(tok, 'IntUnsigned', 0)
if width > 0:
cut: int = text.rfind('_')
if cut > 0:
text = text[:cut]
try:
if text.lower().endswith('j'):
num: object = complex(text[:-1].replace('_', ''))
@@ -757,6 +844,10 @@ class ExprParserMixin(TokenStream):
# 无法解析:保留原字符串(理论上不会发生)
num = text
node: N.Constant = N.Constant(value=num, kind=None)
if width > 0:
# 显式宽度随节点带走,由 codegen 做「不截断」严格校验
node.int_width = width
node.int_unsigned = unsigned
self._FixLoc(node, tok)
return node
@@ -1605,8 +1696,8 @@ class ExprParserMixin(TokenStream):
int** → (int | t.CPtr) | t.CPtr (i32**)
int****** → 6 层嵌套 (i32******)
展开为 BinOp(BitOr) 链后,TPC/TPV 现有类型解析(TypeAnnotationResolver
的 PtrCount 相加、TPV TypeGenResolve 的递归 BinOp 分支)零改动即可
展开为 BinOp(BitOr) 链后,TPC/VPC 现有类型解析(TypeAnnotationResolver
的 PtrCount 相加、VPC TypeGenResolve 的递归 BinOp 分支)零改动即可
生成多级指针 LLVM 类型。
无法解析的后缀形态**响亮失败**(SyntaxError,含 行:列 + 源码片段 + 形态 +
+9 -2
View File
@@ -109,9 +109,14 @@ _NODE_DEFS: dict[str, tuple[tuple[str, ...], tuple[str, ...]]] = {
'YieldFrom': (('value',), _POS_ATTRS),
'Compare': (('left', 'ops', 'comparators'), _POS_ATTRS),
'Call': (('func', 'args', 'keywords'), _POS_ATTRS),
# Viper `new` 生命周期分配表达式(软关键字):new [描述符...] T(args)。
# Source/Life 为描述符整数(见 NEW_SRC_*/NEW_LIFE_*);memgr 为 `memgr(x)` 的 x
# (无 memgr 描述符时为 None);func 是被构造的 callee(Name/Attribute/Subscript),
# args 为构造实参列表(无括号时为 None)。
'New': (('Source', 'Life', 'memgr', 'func', 'args'), _POS_ATTRS),
'FormattedValue': (('value', 'conversion', 'format_spec'), _POS_ATTRS),
'JoinedStr': (('values',), _POS_ATTRS),
'Constant': (('value', 'kind'), _POS_ATTRS),
'Constant': (('value', 'kind'), _POS_ATTRS + ('int_width', 'int_unsigned')),
'Attribute': (('value', 'attr', 'ctx'), _POS_ATTRS),
'Subscript': (('value', 'slice', 'ctx'), _POS_ATTRS),
'Starred': (('value', 'ctx'), _POS_ATTRS),
@@ -245,6 +250,7 @@ _CLASS_DEFAULTS: dict[str, dict[str, object]] = {
'MatchAs': {'pattern': None, 'name': None},
'MatchMapping': {'rest': None},
'MatchStar': {'name': None},
'New': {'memgr': None, 'args': None},
'Raise': {'exc': None, 'cause': None},
'Return': {'value': None},
'Slice': {'lower': None, 'upper': None, 'step': None},
@@ -276,7 +282,8 @@ _BASE_CLASS: dict[str, str] = {
'Call': 'expr', 'Compare': 'expr', 'Constant': 'expr', 'Dict': 'expr',
'DictComp': 'expr', 'FormattedValue': 'expr', 'GeneratorExp': 'expr',
'IfExp': 'expr', 'JoinedStr': 'expr', 'Lambda': 'expr', 'List': 'expr',
'ListComp': 'expr', 'Name': 'expr', 'NamedExpr': 'expr', 'Set': 'expr',
'ListComp': 'expr', 'Name': 'expr', 'NamedExpr': 'expr',
'New': 'expr', 'Set': 'expr',
'SetComp': 'expr', 'Slice': 'expr', 'Starred': 'expr', 'Subscript': 'expr',
'Tuple': 'expr', 'UnaryOp': 'expr', 'Yield': 'expr', 'YieldFrom': 'expr',
# mod
+57 -7
View File
@@ -76,10 +76,12 @@ class Token:
Start: 源码字符偏移(含 UTF-8 字节偏移换算依赖,供 f-string 内部定位)
End: 源码字符偏移(exclusive)
"""
__slots__ = ('Type', 'Value', 'Line', 'Col', 'EndLine', 'EndCol', 'Start', 'End')
__slots__ = ('Type', 'Value', 'Line', 'Col', 'EndLine', 'EndCol', 'Start', 'End',
'IntWidth', 'IntUnsigned')
def __init__(self, tok_type: str, value: str, line: int, col: int,
end_line: int, end_col: int) -> None:
end_line: int, end_col: int,
int_width: int = 0, int_unsigned: int = 0) -> None:
self.Type: str = tok_type
self.Value: str = value
self.Line: int = line
@@ -88,6 +90,9 @@ class Token:
self.EndCol: int = end_col
self.Start: int = -1
self.End: int = -1
# 整型字面量显式宽度后缀(_int<N>/_uint<N>/_i<N>/_u<N>):0 = 未指定
self.IntWidth: int = int_width
self.IntUnsigned: int = int_unsigned
def __repr__(self) -> str:
return (f'Token({self.Type}, {self.Value!r}, '
@@ -152,12 +157,13 @@ class Tokenizer:
col_bytes = len(seg.encode('utf-8'))
return line, col_bytes
def _Make(self, tok_type: str, start: int, end: int) -> Token:
def _Make(self, tok_type: str, start: int, end: int,
int_width: int = 0, int_unsigned: int = 0) -> Token:
"""以 [start, end) 源码区间构造 token。"""
start_line, start_col = self._PosInfo(start)
end_line, end_col = self._PosInfo(end)
tok: Token = Token(tok_type, self.Source[start:end], start_line, start_col,
end_line, end_col)
end_line, end_col, int_width, int_unsigned)
tok.Start = start
tok.End = end
return tok
@@ -335,8 +341,45 @@ class Tokenizer:
self.Tokens.append(self._Make(NAME, start, self.Pos))
self.AtLineStart = False
def _TryScanWidthSuffix(self) -> tuple[int, int]:
"""整型字面量显式宽度后缀 `_int<N>`/`_uint<N>`/`_i<N>`/`_u<N>`。
命中则推进 Pos 并返回 `(N, 无符?)`;否则返回 `(0, 0)` 且不动位置。
N 取 1..64(字面量值本身是 64 位);后缀之后不得直接跟标识符字符。
"""
src: str = self.Source
pos: int = self.Pos
if pos >= self.Length or src[pos] != '_':
return 0, 0
p: int = pos + 1
unsigned: int = 0
if src.startswith('uint', p):
p += 4
unsigned = 1
elif src.startswith('int', p):
p += 3
elif src.startswith('u', p):
p += 1
unsigned = 1
elif src.startswith('i', p):
p += 1
else:
return 0, 0
digits_start: int = p
while p < self.Length and src[p].isdigit():
p += 1
if p == digits_start:
return 0, 0
width: int = int(src[digits_start:p])
if width <= 0 or width > 64:
return 0, 0
if p < self.Length and (src[p].isalnum() or src[p] == '_'):
return 0, 0
self.Pos = p
return width, unsigned
def _ScanNumber(self) -> None:
"""扫描数字字面量:进制前缀/十进制/浮点/指数/复数 j/下划线。"""
"""扫描数字字面量:进制前缀/十进制/浮点/指数/复数 j/下划线/显式宽度后缀。"""
start: int = self.Pos
# 进制前缀 0x/0o/0b
if self.Source[self.Pos] == '0' and self.Pos + 1 < self.Length:
@@ -346,7 +389,8 @@ class Tokenizer:
while self.Pos < self.Length and (self.Source[self.Pos].isalnum()
or self.Source[self.Pos] == '_'):
self.Pos += 1
self.Tokens.append(self._Make(NUMBER, start, self.Pos))
hw, hu = self._TryScanWidthSuffix()
self.Tokens.append(self._Make(NUMBER, start, self.Pos, hw, hu))
self.AtLineStart = False
return
# 整数部分
@@ -375,7 +419,13 @@ class Tokenizer:
# 复数 j
if self.Pos < self.Length and self.Source[self.Pos] in 'jJ':
self.Pos += 1
self.Tokens.append(self._Make(NUMBER, start, self.Pos))
# 显式宽度后缀(仅整数字面量;浮点/复数不接受)
iw: int = 0
iu: int = 0
body: str = self.Source[start:self.Pos]
if not any(c in body for c in '.eEjJ'):
iw, iu = self._TryScanWidthSuffix()
self.Tokens.append(self._Make(NUMBER, start, self.Pos, iw, iu))
self.AtLineStart = False
# ------------------------------------------------------------------
+21
View File
@@ -319,12 +319,33 @@ class Unparser:
'Await': self._Await,
'Yield': self._Yield,
'YieldFrom': self._YieldFrom,
'New': self._New,
}
fn = m.get(name)
if fn is not None:
return fn(node) # type: ignore[operator]
raise NotImplementedError(f'unparse expr: {name}')
def _New(self, node: N.New) -> str:
"""`new [hlife|slife] [stack|heap|memgr(x)] T(args)`(与 ExprParser 的取值同口径:
NEW_SRC_STACK=1/HEAP=2/MEMGR=3,NEW_LIFE_SLIFE=1/HLIFE=2,DEFAULT=0)。"""
out: str = 'new'
if getattr(node, 'Life', 0) == 1:
out += ' slife'
elif getattr(node, 'Life', 0) == 2:
out += ' hlife'
src: int = getattr(node, 'Source', 0)
if src == 1:
out += ' stack'
elif src == 2:
out += ' heap'
elif src == 3 and node.memgr is not None:
out += f' memgr({self._expr(node.memgr)})'
callee: str = self._expr(node.func) if node.func is not None else ''
if node.args:
callee += '(' + ', '.join(self._expr(a) for a in node.args) + ')'
return f'{out} {callee}'
def _Name(self, node: N.Name) -> str:
return node.id
+12 -1
View File
@@ -138,6 +138,16 @@ def _HumanizeSyntaxError(exc: SyntaxError, source: str, filename: str) -> Syntax
return new
# Viper `new` 生命周期软关键字表达式:vast_c(stage 0 子集)不认它,会把 `new` 当普通名
# 产出 Name 节点。命中时强制走纯 Python 解析(语义基准),避免 C 路径产出错误 AST。
_NEW_EXPR_RE: _re.Pattern[str] = _re.compile(r'(?<![\w.])new[ \t]+[A-Za-z_]')
def _SourceUsesNewExpr(source: str) -> bool:
"""源码里是否出现 `new <普通名>` 形态(用于规避 C 解析器对 new 的误解)。"""
return _NEW_EXPR_RE.search(source) is not None
def parse(source: str, filename: str = '<unknown>', mode: str = 'exec', *,
type_comments: bool = False, feature_version: tuple[int, int] | int | None = None) -> AST:
"""解析源码字符串为 AST(与 ast.parse 同签名、同结构)。
@@ -147,7 +157,8 @@ def parse(source: str, filename: str = '<unknown>', mode: str = 'exec', *,
异常直接抛出。两条路径的 SyntaxError 都收口成带 文件/行/列/源码片段 的统一
报文(不再向用户漏出解析器内部 traceback)。
"""
if _VAST_C_PARSE is not None and _os.environ.get('VAST_USE_C', '1') != '0':
if (_VAST_C_PARSE is not None and _os.environ.get('VAST_USE_C', '1') != '0'
and not _SourceUsesNewExpr(source)):
try:
return _VAST_C_PARSE(source, mode)
except SyntaxError as _exc: