chore: 同步 TransPyV→ViperC 改名、new 表达式支持与定宽字面量/窄化适配
This commit is contained in:
1 parent
15f77da9d3
commit
23b47053d9
10 files changed
+10283
-10041
No files matched your search
+2278
-2276
File diff suppressed because it is too large.
Load diff
+618
-608
File diff suppressed because it is too large.
Load diff
+5569
-5488
File diff suppressed because it is too large.
Load diff
@@ -38,7 +38,11 @@ class LlvmCodeGenerator(
|
||||
"""
|
||||
#builder: ir.IRBuilder = None
|
||||
#_Trans: LlvmGeneratorMixin = None
|
||||
pass
|
||||
|
||||
# `new memgr(x) T(args)` 的存储覆盖:_HandleNewExprLlvm 取好池存储(T*)后置入,
|
||||
# _HandleClassNewLlvm / _HandleREnumConstructLlvm 有则复用之(否则各自 alloca)。
|
||||
# 默认 None = 无覆盖;构造结束由 try/finally 复位。
|
||||
NewStorageOverride: object = None
|
||||
|
||||
|
||||
# 向后兼容:重新导出常量和辅助类
|
||||
|
||||
+1621
-1656
File diff suppressed because it is too large.
Load diff
+93
-2
@@ -23,6 +23,30 @@ from lib.vast.Tokenizer import (
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# `new [描述符...] T(args)` 生命周期分配表达式(与 ViperC includes/ast/exprs.vp
|
||||
# 的 NEW_SRC_*/NEW_LIFE_* 同值)。TPC 侧同样按 VPC 口径解析成 New 节点,语义落地
|
||||
# 由 Handles 负责(见 HandlesExprCall._HandleNewExprLlvm)。
|
||||
# ---------------------------------------------------------------------------
|
||||
NEW_SRC_DEFAULT = 0
|
||||
NEW_SRC_STACK = 1
|
||||
NEW_SRC_HEAP = 2
|
||||
NEW_SRC_MEMGR = 3
|
||||
NEW_LIFE_DEFAULT = 0
|
||||
NEW_LIFE_SLIFE = 1
|
||||
NEW_LIFE_HLIFE = 2
|
||||
|
||||
# `new` 之后的普通名不能是这些保留字(它们在 vast 里也是 NAME token)。
|
||||
# 缺此判据时 `for new in x:` 会被误判为 new 表达式(`in` 紧随其后)。
|
||||
_NEW_NEXT_RESERVED: frozenset[str] = frozenset({
|
||||
'in', 'is', 'and', 'or', 'not', 'if', 'else', 'elif', 'for', 'while',
|
||||
'lambda', 'await', 'yield', 'return', 'pass', 'break', 'continue',
|
||||
'import', 'from', 'as', 'with', 'try', 'except', 'finally', 'raise',
|
||||
'def', 'class', 'global', 'nonlocal', 'del', 'assert', 'match', 'case',
|
||||
'True', 'False', 'None',
|
||||
})
|
||||
|
||||
|
||||
def _JoinConstParts(parts: list[object]) -> str | bytes:
|
||||
"""拼接连续字符串字面量值(str 或 bytes,与 CPython 语义一致)。"""
|
||||
if not parts:
|
||||
@@ -705,6 +729,58 @@ class ExprParserMixin(TokenStream):
|
||||
# ==================================================================
|
||||
# 原子表达式
|
||||
# ==================================================================
|
||||
def _ParseNewExpr(self) -> N.expr:
|
||||
"""`new [hlife|slife] [stack|heap|memgr(x)] T(args)`(软关键字,与 VPC 同口径)。
|
||||
|
||||
描述符顺序无关、各轴最多一次;T 走主表达式(Name/Attribute/Subscript + 调用
|
||||
trailer),若解析结果本身是 Call(`T(args)`)则拆成 func/args,否则 args=None。
|
||||
结果恒为 T*(语义在 Handles 侧落地:取存储 + 构造)。
|
||||
"""
|
||||
start: Token = self._Next() # 吃掉 `new`
|
||||
src: int = NEW_SRC_DEFAULT
|
||||
life: int = NEW_LIFE_DEFAULT
|
||||
memgr: N.expr | None = None
|
||||
while True:
|
||||
tok: Token = self._Peek()
|
||||
if tok.Type != NAME:
|
||||
break
|
||||
v: str = tok.Value
|
||||
if v == 'hlife':
|
||||
self._Next()
|
||||
life = NEW_LIFE_HLIFE
|
||||
continue
|
||||
if v == 'slife':
|
||||
self._Next()
|
||||
life = NEW_LIFE_SLIFE
|
||||
continue
|
||||
if v == 'stack':
|
||||
self._Next()
|
||||
src = NEW_SRC_STACK
|
||||
continue
|
||||
if v == 'heap':
|
||||
self._Next()
|
||||
src = NEW_SRC_HEAP
|
||||
continue
|
||||
if v == 'memgr':
|
||||
self._Next()
|
||||
self._ExpectOp('(')
|
||||
self._SkipNL()
|
||||
memgr = self.ParseExpr()
|
||||
self._SkipNL()
|
||||
self._ExpectOp(')')
|
||||
src = NEW_SRC_MEMGR
|
||||
continue
|
||||
break
|
||||
callee: N.expr = self._ParsePrimary()
|
||||
func: N.expr = callee
|
||||
args: list[N.expr] | None = None
|
||||
if isinstance(callee, N.Call):
|
||||
func = callee.func
|
||||
args = callee.args
|
||||
node: N.New = N.New(Source=src, Life=life, memgr=memgr, func=func, args=args)
|
||||
self._FixLoc(node, start)
|
||||
return node
|
||||
|
||||
def _ParseAtom(self) -> N.expr:
|
||||
self._SkipNL()
|
||||
tok: Token = self._Peek()
|
||||
@@ -713,6 +789,10 @@ class ExprParserMixin(TokenStream):
|
||||
if tok.Type in (STRING, FSTRING):
|
||||
return self._ParseStringConcat()
|
||||
if tok.Type == NAME:
|
||||
if tok.Value == 'new':
|
||||
nxt: Token = self._Peek(1)
|
||||
if nxt.Type == NAME and nxt.Value not in _NEW_NEXT_RESERVED:
|
||||
return self._ParseNewExpr()
|
||||
if tok.Value in ('True', 'False', 'None'):
|
||||
self._Next()
|
||||
node: N.Constant = N.Constant(value={'True': True, 'False': False, 'None': None}[tok.Value], kind=None)
|
||||
@@ -744,6 +824,13 @@ class ExprParserMixin(TokenStream):
|
||||
def _ParseNumber(self) -> N.Constant:
|
||||
tok: Token = self._Next()
|
||||
text: str = tok.Value
|
||||
# 显式宽度后缀(_int<N>/_uint<N>/_i<N>/_u<N>):先从文本里剥掉再解析数值
|
||||
width: int = getattr(tok, 'IntWidth', 0)
|
||||
unsigned: int = getattr(tok, 'IntUnsigned', 0)
|
||||
if width > 0:
|
||||
cut: int = text.rfind('_')
|
||||
if cut > 0:
|
||||
text = text[:cut]
|
||||
try:
|
||||
if text.lower().endswith('j'):
|
||||
num: object = complex(text[:-1].replace('_', ''))
|
||||
@@ -757,6 +844,10 @@ class ExprParserMixin(TokenStream):
|
||||
# 无法解析:保留原字符串(理论上不会发生)
|
||||
num = text
|
||||
node: N.Constant = N.Constant(value=num, kind=None)
|
||||
if width > 0:
|
||||
# 显式宽度随节点带走,由 codegen 做「不截断」严格校验
|
||||
node.int_width = width
|
||||
node.int_unsigned = unsigned
|
||||
self._FixLoc(node, tok)
|
||||
return node
|
||||
|
||||
@@ -1605,8 +1696,8 @@ class ExprParserMixin(TokenStream):
|
||||
int** → (int | t.CPtr) | t.CPtr (i32**)
|
||||
int****** → 6 层嵌套 (i32******)
|
||||
|
||||
展开为 BinOp(BitOr) 链后,TPC/TPV 现有类型解析(TypeAnnotationResolver
|
||||
的 PtrCount 相加、TPV TypeGenResolve 的递归 BinOp 分支)零改动即可
|
||||
展开为 BinOp(BitOr) 链后,TPC/VPC 现有类型解析(TypeAnnotationResolver
|
||||
的 PtrCount 相加、VPC TypeGenResolve 的递归 BinOp 分支)零改动即可
|
||||
生成多级指针 LLVM 类型。
|
||||
|
||||
无法解析的后缀形态**响亮失败**(SyntaxError,含 行:列 + 源码片段 + 形态 +
|
||||
|
||||
+9
-2
@@ -109,9 +109,14 @@ _NODE_DEFS: dict[str, tuple[tuple[str, ...], tuple[str, ...]]] = {
|
||||
'YieldFrom': (('value',), _POS_ATTRS),
|
||||
'Compare': (('left', 'ops', 'comparators'), _POS_ATTRS),
|
||||
'Call': (('func', 'args', 'keywords'), _POS_ATTRS),
|
||||
# Viper `new` 生命周期分配表达式(软关键字):new [描述符...] T(args)。
|
||||
# Source/Life 为描述符整数(见 NEW_SRC_*/NEW_LIFE_*);memgr 为 `memgr(x)` 的 x
|
||||
# (无 memgr 描述符时为 None);func 是被构造的 callee(Name/Attribute/Subscript),
|
||||
# args 为构造实参列表(无括号时为 None)。
|
||||
'New': (('Source', 'Life', 'memgr', 'func', 'args'), _POS_ATTRS),
|
||||
'FormattedValue': (('value', 'conversion', 'format_spec'), _POS_ATTRS),
|
||||
'JoinedStr': (('values',), _POS_ATTRS),
|
||||
'Constant': (('value', 'kind'), _POS_ATTRS),
|
||||
'Constant': (('value', 'kind'), _POS_ATTRS + ('int_width', 'int_unsigned')),
|
||||
'Attribute': (('value', 'attr', 'ctx'), _POS_ATTRS),
|
||||
'Subscript': (('value', 'slice', 'ctx'), _POS_ATTRS),
|
||||
'Starred': (('value', 'ctx'), _POS_ATTRS),
|
||||
@@ -245,6 +250,7 @@ _CLASS_DEFAULTS: dict[str, dict[str, object]] = {
|
||||
'MatchAs': {'pattern': None, 'name': None},
|
||||
'MatchMapping': {'rest': None},
|
||||
'MatchStar': {'name': None},
|
||||
'New': {'memgr': None, 'args': None},
|
||||
'Raise': {'exc': None, 'cause': None},
|
||||
'Return': {'value': None},
|
||||
'Slice': {'lower': None, 'upper': None, 'step': None},
|
||||
@@ -276,7 +282,8 @@ _BASE_CLASS: dict[str, str] = {
|
||||
'Call': 'expr', 'Compare': 'expr', 'Constant': 'expr', 'Dict': 'expr',
|
||||
'DictComp': 'expr', 'FormattedValue': 'expr', 'GeneratorExp': 'expr',
|
||||
'IfExp': 'expr', 'JoinedStr': 'expr', 'Lambda': 'expr', 'List': 'expr',
|
||||
'ListComp': 'expr', 'Name': 'expr', 'NamedExpr': 'expr', 'Set': 'expr',
|
||||
'ListComp': 'expr', 'Name': 'expr', 'NamedExpr': 'expr',
|
||||
'New': 'expr', 'Set': 'expr',
|
||||
'SetComp': 'expr', 'Slice': 'expr', 'Starred': 'expr', 'Subscript': 'expr',
|
||||
'Tuple': 'expr', 'UnaryOp': 'expr', 'Yield': 'expr', 'YieldFrom': 'expr',
|
||||
# mod
|
||||
|
||||
+57
-7
@@ -76,10 +76,12 @@ class Token:
|
||||
Start: 源码字符偏移(含 UTF-8 字节偏移换算依赖,供 f-string 内部定位)
|
||||
End: 源码字符偏移(exclusive)
|
||||
"""
|
||||
__slots__ = ('Type', 'Value', 'Line', 'Col', 'EndLine', 'EndCol', 'Start', 'End')
|
||||
__slots__ = ('Type', 'Value', 'Line', 'Col', 'EndLine', 'EndCol', 'Start', 'End',
|
||||
'IntWidth', 'IntUnsigned')
|
||||
|
||||
def __init__(self, tok_type: str, value: str, line: int, col: int,
|
||||
end_line: int, end_col: int) -> None:
|
||||
end_line: int, end_col: int,
|
||||
int_width: int = 0, int_unsigned: int = 0) -> None:
|
||||
self.Type: str = tok_type
|
||||
self.Value: str = value
|
||||
self.Line: int = line
|
||||
@@ -88,6 +90,9 @@ class Token:
|
||||
self.EndCol: int = end_col
|
||||
self.Start: int = -1
|
||||
self.End: int = -1
|
||||
# 整型字面量显式宽度后缀(_int<N>/_uint<N>/_i<N>/_u<N>):0 = 未指定
|
||||
self.IntWidth: int = int_width
|
||||
self.IntUnsigned: int = int_unsigned
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (f'Token({self.Type}, {self.Value!r}, '
|
||||
@@ -152,12 +157,13 @@ class Tokenizer:
|
||||
col_bytes = len(seg.encode('utf-8'))
|
||||
return line, col_bytes
|
||||
|
||||
def _Make(self, tok_type: str, start: int, end: int) -> Token:
|
||||
def _Make(self, tok_type: str, start: int, end: int,
|
||||
int_width: int = 0, int_unsigned: int = 0) -> Token:
|
||||
"""以 [start, end) 源码区间构造 token。"""
|
||||
start_line, start_col = self._PosInfo(start)
|
||||
end_line, end_col = self._PosInfo(end)
|
||||
tok: Token = Token(tok_type, self.Source[start:end], start_line, start_col,
|
||||
end_line, end_col)
|
||||
end_line, end_col, int_width, int_unsigned)
|
||||
tok.Start = start
|
||||
tok.End = end
|
||||
return tok
|
||||
@@ -335,8 +341,45 @@ class Tokenizer:
|
||||
self.Tokens.append(self._Make(NAME, start, self.Pos))
|
||||
self.AtLineStart = False
|
||||
|
||||
def _TryScanWidthSuffix(self) -> tuple[int, int]:
|
||||
"""整型字面量显式宽度后缀 `_int<N>`/`_uint<N>`/`_i<N>`/`_u<N>`。
|
||||
|
||||
命中则推进 Pos 并返回 `(N, 无符?)`;否则返回 `(0, 0)` 且不动位置。
|
||||
N 取 1..64(字面量值本身是 64 位);后缀之后不得直接跟标识符字符。
|
||||
"""
|
||||
src: str = self.Source
|
||||
pos: int = self.Pos
|
||||
if pos >= self.Length or src[pos] != '_':
|
||||
return 0, 0
|
||||
p: int = pos + 1
|
||||
unsigned: int = 0
|
||||
if src.startswith('uint', p):
|
||||
p += 4
|
||||
unsigned = 1
|
||||
elif src.startswith('int', p):
|
||||
p += 3
|
||||
elif src.startswith('u', p):
|
||||
p += 1
|
||||
unsigned = 1
|
||||
elif src.startswith('i', p):
|
||||
p += 1
|
||||
else:
|
||||
return 0, 0
|
||||
digits_start: int = p
|
||||
while p < self.Length and src[p].isdigit():
|
||||
p += 1
|
||||
if p == digits_start:
|
||||
return 0, 0
|
||||
width: int = int(src[digits_start:p])
|
||||
if width <= 0 or width > 64:
|
||||
return 0, 0
|
||||
if p < self.Length and (src[p].isalnum() or src[p] == '_'):
|
||||
return 0, 0
|
||||
self.Pos = p
|
||||
return width, unsigned
|
||||
|
||||
def _ScanNumber(self) -> None:
|
||||
"""扫描数字字面量:进制前缀/十进制/浮点/指数/复数 j/下划线。"""
|
||||
"""扫描数字字面量:进制前缀/十进制/浮点/指数/复数 j/下划线/显式宽度后缀。"""
|
||||
start: int = self.Pos
|
||||
# 进制前缀 0x/0o/0b
|
||||
if self.Source[self.Pos] == '0' and self.Pos + 1 < self.Length:
|
||||
@@ -346,7 +389,8 @@ class Tokenizer:
|
||||
while self.Pos < self.Length and (self.Source[self.Pos].isalnum()
|
||||
or self.Source[self.Pos] == '_'):
|
||||
self.Pos += 1
|
||||
self.Tokens.append(self._Make(NUMBER, start, self.Pos))
|
||||
hw, hu = self._TryScanWidthSuffix()
|
||||
self.Tokens.append(self._Make(NUMBER, start, self.Pos, hw, hu))
|
||||
self.AtLineStart = False
|
||||
return
|
||||
# 整数部分
|
||||
@@ -375,7 +419,13 @@ class Tokenizer:
|
||||
# 复数 j
|
||||
if self.Pos < self.Length and self.Source[self.Pos] in 'jJ':
|
||||
self.Pos += 1
|
||||
self.Tokens.append(self._Make(NUMBER, start, self.Pos))
|
||||
# 显式宽度后缀(仅整数字面量;浮点/复数不接受)
|
||||
iw: int = 0
|
||||
iu: int = 0
|
||||
body: str = self.Source[start:self.Pos]
|
||||
if not any(c in body for c in '.eEjJ'):
|
||||
iw, iu = self._TryScanWidthSuffix()
|
||||
self.Tokens.append(self._Make(NUMBER, start, self.Pos, iw, iu))
|
||||
self.AtLineStart = False
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@@ -319,12 +319,33 @@ class Unparser:
|
||||
'Await': self._Await,
|
||||
'Yield': self._Yield,
|
||||
'YieldFrom': self._YieldFrom,
|
||||
'New': self._New,
|
||||
}
|
||||
fn = m.get(name)
|
||||
if fn is not None:
|
||||
return fn(node) # type: ignore[operator]
|
||||
raise NotImplementedError(f'unparse expr: {name}')
|
||||
|
||||
def _New(self, node: N.New) -> str:
|
||||
"""`new [hlife|slife] [stack|heap|memgr(x)] T(args)`(与 ExprParser 的取值同口径:
|
||||
NEW_SRC_STACK=1/HEAP=2/MEMGR=3,NEW_LIFE_SLIFE=1/HLIFE=2,DEFAULT=0)。"""
|
||||
out: str = 'new'
|
||||
if getattr(node, 'Life', 0) == 1:
|
||||
out += ' slife'
|
||||
elif getattr(node, 'Life', 0) == 2:
|
||||
out += ' hlife'
|
||||
src: int = getattr(node, 'Source', 0)
|
||||
if src == 1:
|
||||
out += ' stack'
|
||||
elif src == 2:
|
||||
out += ' heap'
|
||||
elif src == 3 and node.memgr is not None:
|
||||
out += f' memgr({self._expr(node.memgr)})'
|
||||
callee: str = self._expr(node.func) if node.func is not None else ''
|
||||
if node.args:
|
||||
callee += '(' + ', '.join(self._expr(a) for a in node.args) + ')'
|
||||
return f'{out} {callee}'
|
||||
|
||||
def _Name(self, node: N.Name) -> str:
|
||||
return node.id
|
||||
|
||||
|
||||
+12
-1
@@ -138,6 +138,16 @@ def _HumanizeSyntaxError(exc: SyntaxError, source: str, filename: str) -> Syntax
|
||||
return new
|
||||
|
||||
|
||||
# Viper `new` 生命周期软关键字表达式:vast_c(stage 0 子集)不认它,会把 `new` 当普通名
|
||||
# 产出 Name 节点。命中时强制走纯 Python 解析(语义基准),避免 C 路径产出错误 AST。
|
||||
_NEW_EXPR_RE: _re.Pattern[str] = _re.compile(r'(?<![\w.])new[ \t]+[A-Za-z_]')
|
||||
|
||||
|
||||
def _SourceUsesNewExpr(source: str) -> bool:
|
||||
"""源码里是否出现 `new <普通名>` 形态(用于规避 C 解析器对 new 的误解)。"""
|
||||
return _NEW_EXPR_RE.search(source) is not None
|
||||
|
||||
|
||||
def parse(source: str, filename: str = '<unknown>', mode: str = 'exec', *,
|
||||
type_comments: bool = False, feature_version: tuple[int, int] | int | None = None) -> AST:
|
||||
"""解析源码字符串为 AST(与 ast.parse 同签名、同结构)。
|
||||
@@ -147,7 +157,8 @@ def parse(source: str, filename: str = '<unknown>', mode: str = 'exec', *,
|
||||
异常直接抛出。两条路径的 SyntaxError 都收口成带 文件/行/列/源码片段 的统一
|
||||
报文(不再向用户漏出解析器内部 traceback)。
|
||||
"""
|
||||
if _VAST_C_PARSE is not None and _os.environ.get('VAST_USE_C', '1') != '0':
|
||||
if (_VAST_C_PARSE is not None and _os.environ.get('VAST_USE_C', '1') != '0'
|
||||
and not _SourceUsesNewExpr(source)):
|
||||
try:
|
||||
return _VAST_C_PARSE(source, mode)
|
||||
except SyntaxError as _exc:
|
||||
|
||||
Reference in new issue
Block a user