TermiNexus
b49b5c3ba6
ExprParser 的 _REMOVED_CIF_NAMES 与 vast_c.c 的 Interesting/Removed 名字表原本只认 CIf/CIfdef/CIfndef，现补齐三件套六个拼写（含 c. 前缀形式），使 parser 阶段即可给出指向 cif/celif/celse 的 FailFast；_parser.pyd 同步重编。
1849 lines
76 KiB
Python
1849 lines
76 KiB
Python
"""vast.exprparser - Viper/Python 表达式语法分析器
|
||
|
||
实现与 CPython 3.12 语法规则一致的递归下降表达式解析：
|
||
|
||
- 完整运算符优先级（lambda/条件/or/and/not/比较/位运算/移位/算术/一元/幂/await）
|
||
- 原子表达式：标识符/数字/字符串/常量/f-string/元组/列表/字典/集合
|
||
- 推导式（list/set/dict/generator）、lambda、命名表达式（walrus）
|
||
- 调用（位置实参/关键字实参/*args/**kwargs）、属性、下标、切片
|
||
- f-string（JoinedStr/FormattedValue，含嵌套表达式与格式说明符）
|
||
|
||
本模块为 Mixin 设计：``Parser`` 继承 ``ExprParserMixin`` 获得表达式能力。
|
||
所有产出节点与 CPython ``ast`` 结构完全一致，并填充 lineno/col_offset/
|
||
end_lineno/end_col_offset 位置属性。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import bisect
|
||
|
||
from lib.vast import Nodes as N
|
||
from lib.vast.Tokenizer import (
|
||
Token, NAME, NUMBER, STRING, FSTRING, OP, NEWLINE, NL, ENDMARKER,
|
||
Tokenizer, _STRING_PREFIX_CHARS,
|
||
)
|
||
|
||
|
||
def _JoinConstParts(parts: list[object]) -> str | bytes:
|
||
"""拼接连续字符串字面量值（str 或 bytes，与 CPython 语义一致）。"""
|
||
if not parts:
|
||
return ''
|
||
if isinstance(parts[0], bytes):
|
||
buf: list[bytes] = []
|
||
for p in parts:
|
||
if isinstance(p, bytes):
|
||
buf.append(p)
|
||
else:
|
||
buf.append(str(p).encode('utf-8'))
|
||
return b''.join(buf)
|
||
return ''.join(str(p) for p in parts)
|
||
|
||
|
||
_HEX_ALPHABET: str = '0123456789abcdefABCDEF'
|
||
|
||
# 已删除的用户可见 CIf 族名（CIf/CIfdef/CIfndef/CElif/CElse/CEndif 及 c. 前缀形式）：
|
||
# 用户再写即 FailFast，提示改用原生 cif/celif/celse 与 defined() 谓词
|
||
#（内部标记 c.@CIf 族由解析期合成，不走此处）。
|
||
_REMOVED_CIF_NAMES: frozenset[str] = frozenset({
|
||
'CIf', 'CIfdef', 'CIfndef', 'CElif', 'CElse', 'CEndif',
|
||
'c.CIf', 'c.CIfdef', 'c.CIfndef', 'c.CElif', 'c.CElse', 'c.CEndif',
|
||
})
|
||
|
||
|
||
def _HexValOf(ch: str) -> int:
|
||
"""单个字符的十六进制位值；非十六进制字符返回 0（与 C 解析器 vast_c.HexVal 同口径）。"""
|
||
idx: int = _HEX_ALPHABET.find(ch)
|
||
if idx < 0:
|
||
return 0
|
||
return idx if idx < 16 else idx - 6
|
||
|
||
|
||
def _HexEscapeVal(body: str, start: int, count: int) -> int:
|
||
"""取转义序列后 count 位十六进制的值（越界位置与非十六进制字符均按 0 计）。
|
||
|
||
两侧解析器对非法 `\\x` / `\\u` / `\\U` 转义必须产出同一结果且都不许抛异常：
|
||
C 解析器走 HexVal（非 hex → 0），此处原来直接 `int(..., 16)`，遇 `\\xNN`
|
||
这类字面量会抛 ValueError（includes/ast/lexer.vp 的文档字符串即触发）。
|
||
"""
|
||
val: int = 0
|
||
n: int = len(body)
|
||
for k in range(count):
|
||
idx: int = start + k
|
||
val = val * 16 + _HexValOf(body[idx] if idx < n else '0')
|
||
return val
|
||
|
||
|
||
class TokenStream:
|
||
"""token 流管理基类：统一提供游标操作、位置标注工具。"""
|
||
|
||
def __init__(self, tokens: list[Token], source: str, line_starts: list[int]) -> None:
|
||
self.Tokens: list[Token] = tokens
|
||
self.Source: str = source
|
||
self.LineStarts: list[int] = line_starts
|
||
self.Idx: int = 0
|
||
# _Peek(0) 缓存：递归下降解析常在同一游标位置重复窥视
|
||
# （_At/_AtOp/_AtName 等），命中时跳过 len() 与列表索引。
|
||
# 游标移动后 Idx 变化自然失配，无需显式失效。
|
||
self._PeekCacheIdx: int = -1
|
||
self._PeekCacheTok: Token | None = None
|
||
# 类型注解解析模式：为 True 时 '*' / '**' 按 C 风格指针后缀处理
|
||
# （int* → int | t.CPtr、int** → 嵌套两层），不按乘法/幂运算消费。
|
||
self._InTypeAnnotation: bool = False
|
||
# 下标实参嵌套层数：'[' 实参内是表达式位置（t.CArray[T, A * B] 的尺寸），
|
||
# 此处的 '*' 在有右操作数时按乘法解析（见 _ParseTerm / _AtOperandAfterStarRun）。
|
||
self._SubscriptArgs: int = 0
|
||
# Tail token of the last consumed C-style pointer suffix ('*' / '**').
|
||
# _ParseParen compares it against _Prev() to recognize the paren-less cast
|
||
# `(T*)x`, which must not be mistaken for a plain group `(a | b)`.
|
||
self._PtrSuffixTail: Token | None = None
|
||
|
||
# ------------------------------------------------------------------
|
||
# 游标操作
|
||
# ------------------------------------------------------------------
|
||
def _Peek(self, offset: int = 0) -> Token:
|
||
i: int = self.Idx + offset
|
||
if i >= len(self.Tokens):
|
||
return self.Tokens[-1]
|
||
if offset == 0:
|
||
if self._PeekCacheIdx == i:
|
||
return self._PeekCacheTok
|
||
self._PeekCacheIdx = i
|
||
self._PeekCacheTok = self.Tokens[i]
|
||
return self._PeekCacheTok
|
||
return self.Tokens[i]
|
||
|
||
def _Next(self) -> Token:
|
||
tok: Token = self.Tokens[self.Idx]
|
||
if self.Idx < len(self.Tokens) - 1:
|
||
self.Idx += 1
|
||
return tok
|
||
|
||
def _Prev(self) -> Token:
|
||
return self.Tokens[max(0, self.Idx - 1)]
|
||
|
||
def _At(self, tok_type: str, value: str | None = None) -> bool:
|
||
# 内联 _Peek(0) 缓存逻辑（行为与 _Peek(offset=0) 完全一致）：
|
||
# 命中缓存直接取 _PeekCacheTok，miss 时读 Tokens[i] 并更新缓存；
|
||
# i >= len(Tokens) 时取末尾哨兵 token（不更新缓存）。
|
||
i: int = self.Idx
|
||
if i >= len(self.Tokens):
|
||
tok: Token = self.Tokens[-1]
|
||
elif self._PeekCacheIdx == i:
|
||
tok = self._PeekCacheTok
|
||
else:
|
||
self._PeekCacheIdx = i
|
||
self._PeekCacheTok = self.Tokens[i]
|
||
tok = self._PeekCacheTok
|
||
if tok.Type != tok_type:
|
||
return False
|
||
if value is not None and tok.Value != value:
|
||
return False
|
||
return True
|
||
|
||
def _AtOp(self, value: str) -> bool:
|
||
return self._At(OP, value)
|
||
|
||
def _AtName(self, value: str) -> bool:
|
||
return self._At(NAME, value)
|
||
|
||
def _SkipNL(self) -> None:
|
||
"""跳过括号内换行 token（NL）。"""
|
||
while self._Peek().Type == NL:
|
||
self._Next()
|
||
|
||
def _SkipNEWLINE(self) -> None:
|
||
"""跳过逻辑换行 token（NEWLINE）与空行/注释行（NL）。"""
|
||
while self._Peek().Type in (NEWLINE, NL):
|
||
self._Next()
|
||
|
||
def _ExpectOp(self, value: str) -> Token:
|
||
tok: Token = self._Next()
|
||
if tok.Type != OP or tok.Value != value:
|
||
raise SyntaxError(
|
||
f"expected '{value}', got {tok.Type}({tok.Value!r}) "
|
||
f"at line {tok.Line}"
|
||
)
|
||
return tok
|
||
|
||
def _ExpectName(self, value: str | None = None) -> Token:
|
||
tok: Token = self._Next()
|
||
if tok.Type != NAME or (value is not None and tok.Value != value):
|
||
want: str = f" {value!r}" if value else ''
|
||
raise SyntaxError(
|
||
f"expected name{want}, "
|
||
f"got {tok.Type}({tok.Value!r}) at line {tok.Line}"
|
||
)
|
||
return tok
|
||
|
||
def _ExpectNEWLINE(self) -> None:
|
||
tok: Token = self._Next()
|
||
if tok.Type != NEWLINE:
|
||
raise SyntaxError(
|
||
f"expected newline, got {tok.Type}({tok.Value!r}) at line {tok.Line}"
|
||
)
|
||
|
||
# ------------------------------------------------------------------
|
||
# 位置标注
|
||
# ------------------------------------------------------------------
|
||
def _FixLoc(self, node: N.AST, start_tok: Token, end_tok: Token | None = None) -> N.AST:
|
||
"""为节点设置 4 个位置属性（end 缺省取前一个 token）。"""
|
||
if end_tok is None:
|
||
end_tok = self._Prev()
|
||
if 'lineno' in getattr(node, '_attributes', ()):
|
||
node.lineno = start_tok.Line
|
||
node.col_offset = start_tok.Col
|
||
node.end_lineno = end_tok.EndLine
|
||
node.end_col_offset = end_tok.EndCol
|
||
return node
|
||
|
||
def _PosFromOffset(self, offset: int) -> tuple[int, int]:
|
||
"""字符偏移 → (行号, UTF-8 字节列偏移)。"""
|
||
idx: int = bisect.bisect_right(self.LineStarts, offset) - 1
|
||
line: int = idx + 1
|
||
col: int = len(self.Source[self.LineStarts[idx]:offset].encode('utf-8'))
|
||
return line, col
|
||
|
||
def _Relocate(self, tok: Token, base: int) -> None:
|
||
"""将子表达式 token 按绝对偏移重定位（f-string 内部表达式用）。"""
|
||
tok.Start = base + tok.Start
|
||
tok.End = base + tok.End
|
||
line, col = self._PosFromOffset(tok.Start)
|
||
e_line, e_col = self._PosFromOffset(tok.End)
|
||
tok.Line, tok.Col, tok.EndLine, tok.EndCol = line, col, e_line, e_col
|
||
|
||
|
||
class ExprParserMixin(TokenStream):
|
||
"""表达式解析器（Mixin，供 Parser 使用）。"""
|
||
|
||
# ==================================================================
|
||
# 表达式顶层：lambda | 条件表达式
|
||
# ==================================================================
|
||
def ParseExpr(self) -> N.expr:
|
||
"""完整表达式：lambda 或条件表达式（含 walrus 前置）。"""
|
||
if self._AtName('lambda'):
|
||
return self._ParseLambda()
|
||
return self._ParseConditional()
|
||
|
||
def ParseExprList(self) -> N.expr:
|
||
"""testlist_star_expr：表达式或逗号分隔元组（赋值右值/return 值）。"""
|
||
start: Token = self._Peek()
|
||
first: N.expr = self.ParseExpr()
|
||
if self._AtOp(','):
|
||
elts: list[N.expr] = [first]
|
||
while self._AtOp(','):
|
||
self._Next()
|
||
self._SkipNL()
|
||
elts.append(self.ParseExpr())
|
||
tup: N.Tuple = N.Tuple(elts=elts, ctx=N.Load())
|
||
self._FixLoc(tup, start)
|
||
return tup
|
||
return first
|
||
|
||
def _ParseConditional(self) -> N.expr:
|
||
"""条件表达式：disjunction ('if' disjunction 'else' expression)?"""
|
||
start: Token = self._Peek()
|
||
body: N.expr = self._ParseDisjunction()
|
||
if self._AtName('if'):
|
||
self._Next()
|
||
test: N.expr = self._ParseDisjunction()
|
||
self._ExpectName('else')
|
||
orelse: N.expr = self.ParseExpr()
|
||
node: N.IfExp = N.IfExp(test=test, body=body, orelse=orelse)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
return body
|
||
|
||
# ==================================================================
|
||
# 布尔/比较
|
||
# ==================================================================
|
||
def _ParseDisjunction(self) -> N.expr:
|
||
start: Token = self._Peek()
|
||
values: list[N.expr] = [self._ParseConjunction()]
|
||
while True:
|
||
self._SkipNL()
|
||
if self._AtName('or'):
|
||
self._Next()
|
||
values.append(self._ParseConjunction())
|
||
continue
|
||
break
|
||
if len(values) == 1:
|
||
return values[0]
|
||
node: N.BoolOp = N.BoolOp(op=N.Or(), values=values)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
|
||
def _ParseConjunction(self) -> N.expr:
|
||
start: Token = self._Peek()
|
||
values: list[N.expr] = [self._ParseInversion()]
|
||
while True:
|
||
self._SkipNL()
|
||
if self._AtName('and'):
|
||
self._Next()
|
||
values.append(self._ParseInversion())
|
||
continue
|
||
break
|
||
if len(values) == 1:
|
||
return values[0]
|
||
node: N.BoolOp = N.BoolOp(op=N.And(), values=values)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
|
||
def _ParseInversion(self) -> N.expr:
|
||
self._SkipNL()
|
||
if self._AtName('not'):
|
||
start: Token = self._Next()
|
||
operand: N.expr = self._ParseInversion()
|
||
node: N.UnaryOp = N.UnaryOp(op=N.Not(), operand=operand)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
return self._ParseComparison()
|
||
|
||
# 比较操作符映射
|
||
_COMPARE_OPS: dict[str, type] = {
|
||
'<': N.Lt, '>': N.Gt, '<=': N.LtE, '>=': N.GtE,
|
||
'==': N.Eq, '!=': N.NotEq,
|
||
'is': N.Is, 'is not': N.IsNot,
|
||
'in': N.In, 'not in': N.NotIn,
|
||
}
|
||
|
||
def _ParseComparison(self) -> N.expr:
|
||
start: Token = self._Peek()
|
||
left: N.expr = self._ParseBitOr()
|
||
ops: list[N.AST] = []
|
||
comparators: list[N.expr] = []
|
||
while True:
|
||
self._SkipNL()
|
||
tok: Token = self._Peek()
|
||
op_type: type | None = None
|
||
if tok.Type == OP and tok.Value in ('<', '>', '<=', '>=', '==', '!='):
|
||
op_type = self._COMPARE_OPS[tok.Value]
|
||
self._Next()
|
||
elif tok.Type == NAME and tok.Value == 'is':
|
||
self._Next()
|
||
if self._AtName('not'):
|
||
self._Next()
|
||
op_type = N.IsNot
|
||
else:
|
||
op_type = N.Is
|
||
elif tok.Type == NAME and tok.Value == 'in':
|
||
self._Next()
|
||
op_type = N.In
|
||
elif tok.Type == NAME and tok.Value == 'not' and self._Peek(1).Type == NAME and self._Peek(1).Value == 'in':
|
||
self._Next()
|
||
self._Next()
|
||
op_type = N.NotIn
|
||
else:
|
||
break
|
||
ops.append(op_type())
|
||
comparators.append(self._ParseBitOr())
|
||
if not ops:
|
||
return left
|
||
node: N.Compare = N.Compare(left=left, ops=ops, comparators=comparators)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
|
||
# ==================================================================
|
||
# 位运算
|
||
# ==================================================================
|
||
def _ParseBitOr(self) -> N.expr:
|
||
start: Token = self._Peek()
|
||
left: N.expr = self._ParseBitXor()
|
||
while True:
|
||
self._SkipNL()
|
||
if self._AtOp('|'):
|
||
self._Next()
|
||
right: N.expr = self._ParseBitXor()
|
||
node: N.BinOp = N.BinOp(left=left, op=N.BitOr(), right=right)
|
||
self._FixLoc(node, start)
|
||
left = node
|
||
continue
|
||
break
|
||
return left
|
||
|
||
def _ParseBitXor(self) -> N.expr:
|
||
start: Token = self._Peek()
|
||
left: N.expr = self._ParseBitAnd()
|
||
while True:
|
||
self._SkipNL()
|
||
if self._AtOp('^'):
|
||
self._Next()
|
||
right: N.expr = self._ParseBitAnd()
|
||
node: N.BinOp = N.BinOp(left=left, op=N.BitXor(), right=right)
|
||
self._FixLoc(node, start)
|
||
left = node
|
||
continue
|
||
break
|
||
return left
|
||
|
||
def _ParseBitAnd(self) -> N.expr:
|
||
start: Token = self._Peek()
|
||
left: N.expr = self._ParseShift()
|
||
while True:
|
||
self._SkipNL()
|
||
if self._AtOp('&'):
|
||
self._Next()
|
||
right: N.expr = self._ParseShift()
|
||
node: N.BinOp = N.BinOp(left=left, op=N.BitAnd(), right=right)
|
||
self._FixLoc(node, start)
|
||
left = node
|
||
continue
|
||
break
|
||
return left
|
||
|
||
def _ParseShift(self) -> N.expr:
|
||
start: Token = self._Peek()
|
||
left: N.expr = self._ParseArith()
|
||
while True:
|
||
self._SkipNL()
|
||
if self._AtOp('<<'):
|
||
self._Next()
|
||
right: N.expr = self._ParseArith()
|
||
node = N.BinOp(left=left, op=N.LShift(), right=right)
|
||
elif self._AtOp('>>'):
|
||
self._Next()
|
||
right = self._ParseArith()
|
||
node = N.BinOp(left=left, op=N.RShift(), right=right)
|
||
else:
|
||
break
|
||
self._FixLoc(node, start)
|
||
left = node
|
||
return left
|
||
|
||
# ==================================================================
|
||
# 算术
|
||
# ==================================================================
|
||
def _ParseArith(self) -> N.expr:
|
||
start: Token = self._Peek()
|
||
left: N.expr = self._ParseTerm()
|
||
while True:
|
||
self._SkipNL()
|
||
if self._AtOp('+'):
|
||
self._Next()
|
||
right: N.expr = self._ParseTerm()
|
||
node = N.BinOp(left=left, op=N.Add(), right=right)
|
||
elif self._AtOp('-'):
|
||
self._Next()
|
||
right = self._ParseTerm()
|
||
node = N.BinOp(left=left, op=N.Sub(), right=right)
|
||
else:
|
||
break
|
||
self._FixLoc(node, start)
|
||
left = node
|
||
return left
|
||
|
||
def _ParseTerm(self) -> N.expr:
|
||
start: Token = self._Peek()
|
||
left: N.expr = self._ParseFactor()
|
||
while True:
|
||
self._SkipNL()
|
||
tok: Token = self._Peek()
|
||
op_type: type | None = None
|
||
if tok.Type == OP:
|
||
if tok.Value == '*' and self._AtPtrSuffix(left):
|
||
# C 风格指针后缀（'*' 一层）；展开为 base | t.CPtr 后继续。
|
||
left = self._ConsumePtrSuffix(left)
|
||
continue
|
||
if self._InTypeAnnotation and tok.Value == '*':
|
||
# 注解模式：'*' 后已无操作数（']' / ',' / ')' / ':' / '=' / 行尾 / ';'）
|
||
# 时它是 C 风格指针后缀标记，交上方判据与 _ParseTypeAnnotation 统一消费；
|
||
# 但下标实参内的尺寸表达式（t.CArray[T, A * B]）'*' 后仍有操作数，
|
||
# 必须按乘法继续——否则下标收尾报 `expected ']'`。
|
||
if not (self._SubscriptArgs > 0 and self._AtOperandAfterStarRun()):
|
||
break
|
||
op_type = {'*': N.Mult, '/': N.Div, '//': N.FloorDiv,
|
||
'%': N.Mod, '@': N.MatMult}.get(tok.Value)
|
||
if op_type is None:
|
||
break
|
||
self._Next()
|
||
right: N.expr = self._ParseFactor()
|
||
node: N.BinOp = N.BinOp(left=left, op=op_type(), right=right)
|
||
self._FixLoc(node, start)
|
||
left = node
|
||
return left
|
||
|
||
# ==================================================================
|
||
# 一元 / 幂 / await
|
||
# ==================================================================
|
||
def _ParseFactor(self) -> N.expr:
|
||
tok: Token = self._Peek()
|
||
if tok.Type == OP and tok.Value in ('+', '-', '~', '*', '&'):
|
||
self._Next()
|
||
operand: N.expr = self._ParseFactor()
|
||
op_type: type = {'+': N.UAdd, '-': N.USub, '~': N.Invert,
|
||
'*': N.Deref, '&': N.AddrOf}[tok.Value]
|
||
node: N.UnaryOp = N.UnaryOp(op=op_type(), operand=operand)
|
||
self._FixLoc(node, tok)
|
||
return node
|
||
return self._ParsePower()
|
||
|
||
def _ParsePower(self) -> N.expr:
|
||
start: Token = self._Peek()
|
||
base: N.expr = self._ParseAwaitPrimary()
|
||
if self._AtOp('**'):
|
||
if self._AtPtrSuffix(base):
|
||
# C 风格指针后缀（'**' 计 2 层）
|
||
return self._ConsumePtrSuffix(base)
|
||
if self._InTypeAnnotation:
|
||
return base # 注解模式：'**' 是 C 风格指针后缀标记（2 层），不按幂处理
|
||
self._Next()
|
||
exponent: N.expr = self._ParseFactor() # ** 右结合
|
||
node: N.BinOp = N.BinOp(left=base, op=N.Pow(), right=exponent)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
return base
|
||
|
||
def _ParseAwaitPrimary(self) -> N.expr:
|
||
if self._AtName('await'):
|
||
start: Token = self._Next()
|
||
value: N.expr = self._ParsePrimary()
|
||
node: N.Await = N.Await(value=value)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
return self._ParsePrimary()
|
||
|
||
# ==================================================================
|
||
# 主表达式：原子 + trailer（调用/属性/下标）
|
||
# ==================================================================
|
||
def _ParsePrimary(self) -> N.expr:
|
||
start: Token = self._Peek()
|
||
node: N.expr = self._ParseAtom()
|
||
while True:
|
||
self._SkipNL()
|
||
tok: Token = self._Peek()
|
||
if tok.Type == OP and tok.Value in ('*', '**') and self._AtPtrSuffixCall(node):
|
||
# `T*(x)` / `T**(x)` 指针后缀强转：解析为 cast 调用（与
|
||
# `(T | t.CPtr)(x)` 同形）。在 trailer 循环里消费，故其后的
|
||
# `.attr` / `[i]` / `(…)` 仍由本循环继续挂接（如
|
||
# `ast.Attribute*(expr_root).value`）。
|
||
node = self._ConsumePtrSuffixCall(node, start)
|
||
elif tok.Type == OP and tok.Value == '.':
|
||
self._Next()
|
||
attr_tok: Token = self._ExpectName()
|
||
attr_node: N.Attribute = N.Attribute(value=node, attr=attr_tok.Value, ctx=N.Load())
|
||
self._FixLoc(attr_node, start)
|
||
node = attr_node
|
||
elif tok.Type == OP and tok.Value == '(':
|
||
call_node: N.Call = self._ParseCallTail(node, start)
|
||
node = call_node
|
||
elif tok.Type == OP and tok.Value == '[':
|
||
sub_node: N.Subscript = self._ParseSubscriptTail(node, start)
|
||
node = sub_node
|
||
else:
|
||
break
|
||
return node
|
||
|
||
def _ParseCallTail(self, func: N.expr, start: Token) -> N.Call:
|
||
self._ExpectOp('(')
|
||
args: list[N.expr] = []
|
||
keywords: list[N.keyword] = []
|
||
self._SkipNL()
|
||
if not self._AtOp(')'):
|
||
while True:
|
||
self._SkipNL()
|
||
# **kwargs
|
||
if self._AtOp('**'):
|
||
self._Next()
|
||
value: N.expr = self.ParseExpr()
|
||
kw: N.keyword = N.keyword(arg=None, value=value)
|
||
self._FixLoc(kw, self._Prev())
|
||
keywords.append(kw)
|
||
else:
|
||
# *args
|
||
if self._AtOp('*'):
|
||
self._Next()
|
||
value = self.ParseExpr()
|
||
starred: N.Starred = N.Starred(value=value, ctx=N.Load())
|
||
self._FixLoc(starred, self._Prev())
|
||
args.append(starred)
|
||
else:
|
||
# 关键字参数：NAME '='
|
||
if self._Peek().Type == NAME and self._Peek(1).Type == OP and self._Peek(1).Value == '=':
|
||
name_tok: Token = self._Next()
|
||
self._Next() # '='
|
||
value = self.ParseExpr()
|
||
kw = N.keyword(arg=name_tok.Value, value=value)
|
||
self._FixLoc(kw, name_tok)
|
||
keywords.append(kw)
|
||
else:
|
||
# 生成器表达式作为唯一实参：f(x for y in z)
|
||
# 先解析第一个表达式，若随后是 'for' 则回填为 GeneratorExp
|
||
if not args and not keywords:
|
||
gen_start: Token = self._Peek()
|
||
gen_elt: N.expr = self.ParseExpr()
|
||
if self._AtName('for'):
|
||
gen_gens: list[N.comprehension] = self._ParseComprehensionGenerators()
|
||
gen: N.GeneratorExp = N.GeneratorExp(elt=gen_elt, generators=gen_gens)
|
||
self._FixLoc(gen, gen_start)
|
||
args.append(gen)
|
||
else:
|
||
args.append(gen_elt)
|
||
else:
|
||
args.append(self.ParseExpr())
|
||
self._SkipNL()
|
||
if self._AtOp(','):
|
||
self._Next()
|
||
self._SkipNL()
|
||
if self._AtOp(')'):
|
||
break
|
||
continue
|
||
break
|
||
self._ExpectOp(')')
|
||
node: N.Call = N.Call(func=func, args=args, keywords=keywords)
|
||
self._FixLoc(node, start)
|
||
return self._DesugarSpecialCall(node, start)
|
||
|
||
def _CallCalleeName(self, func: N.expr) -> str:
|
||
"""被调用者的限定名：Name('X')→'X'；Attribute(c,'X')→'c.X'；否则空串。"""
|
||
if isinstance(func, N.Name):
|
||
return func.id
|
||
if isinstance(func, N.Attribute) and isinstance(func.value, N.Name):
|
||
return f'{func.value.id}.{func.attr}'
|
||
return ''
|
||
|
||
def _DesugarSpecialCall(self, node: N.Call, start: Token) -> N.expr:
|
||
"""调用点的特殊 desugar / fail-fast。
|
||
|
||
- `defined(NAME)`：仅 cif/celif 条件内合法（`self._InCIfCond`），
|
||
desugar 为内部标记调用 `c.@CIfdef(NAME)`；其余位置 FailFast。
|
||
- `c.CIf` / `c.CIfdef` / `c.CIfndef`（及裸名）：已删除，FailFast 指向新语法。
|
||
"""
|
||
callee: str = self._CallCalleeName(node.func)
|
||
if callee in _REMOVED_CIF_NAMES:
|
||
raise SyntaxError(
|
||
f"'{callee}(...)' has been removed; use the native 'cif' / 'celif' / "
|
||
f"'celse' statements together with the 'defined(NAME)' predicate "
|
||
f"instead, at line {start.Line}")
|
||
if callee == 'defined':
|
||
if not self._InCIfCond:
|
||
raise SyntaxError(
|
||
"'defined(NAME)' is only allowed inside a 'cif' / 'celif' "
|
||
f"condition, at line {start.Line}")
|
||
if len(node.args) != 1 or node.keywords:
|
||
raise SyntaxError(
|
||
f"'defined' expects exactly one macro name (e.g. defined(MAX_SIZE)), "
|
||
f"at line {start.Line}")
|
||
marker: N.Call = N.Call(
|
||
func=N.Attribute(value=N.Name(id='c', ctx=N.Load()),
|
||
attr='@CIfdef', ctx=N.Load()),
|
||
args=[node.args[0]], keywords=[])
|
||
self._FixLoc(marker, start)
|
||
return marker
|
||
return node
|
||
|
||
def _ParseSubscriptTail(self, value: N.expr, start: Token) -> N.Subscript:
|
||
self._ExpectOp('[')
|
||
self._SkipNL()
|
||
slice_node: N.AST = self._ParseSliceList()
|
||
self._SkipNL()
|
||
self._ExpectOp(']')
|
||
node: N.Subscript = N.Subscript(value=value, slice=slice_node, ctx=N.Load())
|
||
self._FixLoc(node, start)
|
||
return node
|
||
|
||
def _ParseSliceList(self) -> N.AST:
|
||
"""解析下标列表：a[1] / a[1:2] / a[1:2:3] / a[1,2] / a[...] / a[::]
|
||
|
||
下标实参内是**表达式**位置（如 `t.CArray[T, A * B]` 的尺寸），此处的 '*'
|
||
应按乘法解析；同时 `t.CArray[str*, 4]` 的指针后缀仍要成立——两者由
|
||
`_AtOperandAfterStarRun`（'*' 后是否还有操作数）区分，故此处登记已进入实参。
|
||
"""
|
||
self._SubscriptArgs += 1
|
||
try:
|
||
start: Token = self._Peek()
|
||
elts: list[N.AST] = [self._ParseSliceItem()]
|
||
while self._AtOp(','):
|
||
self._Next()
|
||
self._SkipNL()
|
||
elts.append(self._ParseSliceItem())
|
||
finally:
|
||
self._SubscriptArgs -= 1
|
||
if len(elts) == 1:
|
||
return elts[0]
|
||
tup: N.Tuple = N.Tuple(elts=elts, ctx=N.Load())
|
||
self._FixLoc(tup, start)
|
||
return tup
|
||
|
||
def _ParseSliceItem(self) -> N.AST:
|
||
start: Token = self._Peek()
|
||
# 含冒号 → Slice
|
||
if self._AtOp(':'):
|
||
lower: N.expr | None = None
|
||
upper: N.expr | None = None
|
||
step: N.expr | None = None
|
||
self._Next() # ':'
|
||
if not self._AtOp(':') and not self._AtOp(','):
|
||
if not self._AtOp(']'):
|
||
upper = self.ParseExpr()
|
||
if self._AtOp(':'):
|
||
self._Next()
|
||
if not self._AtOp(']') and not self._AtOp(',') and not self._AtOp(':'):
|
||
step = self.ParseExpr()
|
||
node: N.Slice = N.Slice(lower=lower, upper=upper, step=step)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
# 检查 a:b 形式：第一个元素后直接冒号
|
||
first: N.expr = self.ParseExpr()
|
||
if self._AtOp(':'):
|
||
lower = first
|
||
self._Next()
|
||
upper = None
|
||
step = None
|
||
if not self._AtOp(':') and not self._AtOp(']') and not self._AtOp(','):
|
||
upper = self.ParseExpr()
|
||
if self._AtOp(':'):
|
||
self._Next()
|
||
if not self._AtOp(']') and not self._AtOp(','):
|
||
step = self.ParseExpr()
|
||
node = N.Slice(lower=lower, upper=upper, step=step)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
# 简单下标（含 Ellipsis）
|
||
if self._AtOp('...'):
|
||
# 已在 _ParseExpr 中处理为 Constant(Ellipsis)
|
||
pass
|
||
return first
|
||
|
||
# ==================================================================
|
||
# 原子表达式
|
||
# ==================================================================
|
||
def _ParseAtom(self) -> N.expr:
|
||
self._SkipNL()
|
||
tok: Token = self._Peek()
|
||
if tok.Type == NUMBER:
|
||
return self._ParseNumber()
|
||
if tok.Type in (STRING, FSTRING):
|
||
return self._ParseStringConcat()
|
||
if tok.Type == NAME:
|
||
if tok.Value in ('True', 'False', 'None'):
|
||
self._Next()
|
||
node: N.Constant = N.Constant(value={'True': True, 'False': False, 'None': None}[tok.Value], kind=None)
|
||
self._FixLoc(node, tok)
|
||
return node
|
||
self._Next()
|
||
node = N.Name(id=tok.Value, ctx=N.Load())
|
||
self._FixLoc(node, tok)
|
||
return node
|
||
if tok.Type == OP:
|
||
if tok.Value == '(':
|
||
return self._ParseParen()
|
||
if tok.Value == '[':
|
||
return self._ParseList()
|
||
if tok.Value == '{':
|
||
return self._ParseDictSet()
|
||
if tok.Value == '...':
|
||
self._Next()
|
||
node = N.Constant(value=Ellipsis, kind=None)
|
||
self._FixLoc(node, tok)
|
||
return node
|
||
raise SyntaxError(
|
||
f"unexpected token {tok.Type}({tok.Value!r}) at line {tok.Line}"
|
||
)
|
||
|
||
# ------------------------------------------------------------------
|
||
# 数字
|
||
# ------------------------------------------------------------------
|
||
def _ParseNumber(self) -> N.Constant:
|
||
tok: Token = self._Next()
|
||
text: str = tok.Value
|
||
try:
|
||
if text.lower().endswith('j'):
|
||
num: object = complex(text[:-1].replace('_', ''))
|
||
elif text.lower().startswith(('0x', '0o', '0b')):
|
||
num = int(text.replace('_', ''), 0)
|
||
elif any(c in text for c in '.eE'):
|
||
num = float(text.replace('_', ''))
|
||
else:
|
||
num = int(text.replace('_', ''), 10)
|
||
except ValueError:
|
||
# 无法解析：保留原字符串（理论上不会发生）
|
||
num = text
|
||
node: N.Constant = N.Constant(value=num, kind=None)
|
||
self._FixLoc(node, tok)
|
||
return node
|
||
|
||
# ------------------------------------------------------------------
|
||
# 字符串（含相邻拼接与 f-string 混合）
|
||
# ------------------------------------------------------------------
|
||
def _ParseStringConcat(self) -> N.expr:
|
||
"""解析连续字符串字面量并拼接。
|
||
|
||
规则（与 CPython 一致）：
|
||
- 全部为普通字符串 → 单个 Constant（值合并）
|
||
- 含 f-string → JoinedStr（字符串为 Constant，f-string 展开为 values）
|
||
"""
|
||
start: Token = self._Peek()
|
||
parts: list[N.expr] = [] # JoinedStr 的 values 或常量
|
||
const_parts: list[object] = []
|
||
has_fstring: bool = False
|
||
while True:
|
||
# 相邻字符串可跨括号内换行（NL）合并
|
||
self._SkipNL()
|
||
tok: Token = self._Peek()
|
||
if tok.Type == STRING:
|
||
self._Next()
|
||
val: object = self._DecodeString(tok.Value)
|
||
const_parts.append(val)
|
||
elif tok.Type == FSTRING:
|
||
self._Next()
|
||
joined: N.JoinedStr = self._ParseFStringToken(tok)
|
||
has_fstring = True
|
||
if const_parts:
|
||
parts.append(self._MakeStrConstant(_JoinConstParts(const_parts), start))
|
||
const_parts = []
|
||
# 相邻 Constant 合并（f-string 字面量 + 后续常量段）
|
||
if parts and isinstance(parts[-1], N.Constant) and joined.values and isinstance(joined.values[0], N.Constant):
|
||
merged: str = str(parts[-1].value) + str(joined.values[0].value)
|
||
merged_const: N.Constant = N.Constant(value=merged, kind=None)
|
||
merged_const.lineno = parts[-1].lineno
|
||
merged_const.col_offset = parts[-1].col_offset
|
||
parts[-1] = merged_const
|
||
parts.extend(joined.values[1:])
|
||
else:
|
||
parts.extend(joined.values)
|
||
else:
|
||
break
|
||
if not has_fstring:
|
||
node: N.Constant = N.Constant(value=_JoinConstParts(const_parts), kind=None)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
if const_parts:
|
||
if parts and isinstance(parts[-1], N.Constant):
|
||
merged = str(parts[-1].value) + str(_JoinConstParts(const_parts))
|
||
merged_const = N.Constant(value=merged, kind=None)
|
||
merged_const.lineno = parts[-1].lineno
|
||
merged_const.col_offset = parts[-1].col_offset
|
||
parts[-1] = merged_const
|
||
else:
|
||
parts.append(self._MakeStrConstant(str(_JoinConstParts(const_parts)), start))
|
||
node = N.JoinedStr(values=parts)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
|
||
def _MakeStrConstant(self, value: str, start: Token) -> N.Constant:
|
||
node: N.Constant = N.Constant(value=value, kind=None)
|
||
node.lineno = start.Line
|
||
node.col_offset = start.Col
|
||
return node
|
||
|
||
# ------------------------------------------------------------------
|
||
# 字符串解码
|
||
# ------------------------------------------------------------------
|
||
def _DecodeString(self, raw: str) -> str | bytes:
|
||
"""解码字符串字面量原始文本（含前缀与引号）为实际值。
|
||
|
||
支持 r/b/u/f 前缀组合、单引号/双引号/三引号、常见转义序列。
|
||
"""
|
||
# 提取前缀与引号
|
||
i: int = 0
|
||
while i < len(raw) and raw[i] in _STRING_PREFIX_CHARS:
|
||
i += 1
|
||
prefix: str = raw[:i]
|
||
quote: str = raw[i]
|
||
triple: bool = raw.startswith(quote * 3, i)
|
||
body_start: int = i + (3 if triple else 1)
|
||
body_end: int = len(raw) - (3 if triple else 1)
|
||
body: str = raw[body_start:body_end]
|
||
is_raw: bool = 'r' in prefix.lower()
|
||
is_bytes: bool = 'b' in prefix.lower()
|
||
if is_bytes:
|
||
# bytes 字面量：逐字节解码（转义为字节值）
|
||
return self._DecodeBytes(body, is_raw)
|
||
# str 字面量
|
||
return self._DecodeEscapes(body, is_raw)
|
||
|
||
def _DecodeEscapes(self, body: str, is_raw: bool) -> str:
|
||
"""解码字符串转义序列。"""
|
||
if is_raw:
|
||
return body
|
||
out: list[str] = []
|
||
i: int = 0
|
||
n: int = len(body)
|
||
while i < n:
|
||
ch: str = body[i]
|
||
if ch != '\\':
|
||
out.append(ch)
|
||
i += 1
|
||
continue
|
||
i += 1
|
||
if i >= n:
|
||
out.append('\\')
|
||
break
|
||
esc: str = body[i]
|
||
simple: dict[str, str] = {
|
||
'\\': '\\', "'": "'", '"': '"', 'n': '\n', 't': '\t',
|
||
'r': '\r', 'b': '\b', 'f': '\f', 'v': '\v', 'a': '\a',
|
||
}
|
||
if esc in simple:
|
||
out.append(simple[esc])
|
||
i += 1
|
||
elif esc == 'x':
|
||
out.append(chr(_HexEscapeVal(body, i + 1, 2)))
|
||
i += 3
|
||
elif esc == 'u':
|
||
out.append(chr(_HexEscapeVal(body, i + 1, 4)))
|
||
i += 5
|
||
elif esc == 'U':
|
||
out.append(chr(_HexEscapeVal(body, i + 1, 8)))
|
||
i += 9
|
||
elif esc.isdigit() and '0' <= esc <= '7':
|
||
oct_digits: str = body[i:i + 3]
|
||
j: int = 0
|
||
while j < 3 and j < len(oct_digits) and '0' <= oct_digits[j] <= '7':
|
||
j += 1
|
||
out.append(chr(int(oct_digits[:j], 8)))
|
||
i += j
|
||
else:
|
||
# 未知转义：保留原样（CPython 保留反斜杠）
|
||
out.append('\\' + esc)
|
||
i += 1
|
||
return ''.join(out)
|
||
|
||
def _DecodeBytes(self, body: str, is_raw: bool) -> bytes:
|
||
"""解码 bytes 字面量（转义序列按字节处理，支持 r 前缀）。"""
|
||
if is_raw:
|
||
return body.encode('latin-1')
|
||
out: list[int] = []
|
||
i: int = 0
|
||
n: int = len(body)
|
||
while i < n:
|
||
ch: str = body[i]
|
||
if ch != '\\':
|
||
try:
|
||
out.append(ord(body[i]))
|
||
except Exception:
|
||
out.append(0)
|
||
i += 1
|
||
continue
|
||
i += 1
|
||
if i >= n:
|
||
out.append(ord('\\'))
|
||
break
|
||
esc: str = body[i]
|
||
simple: dict[str, int] = {
|
||
'\\': ord('\\'), "'": ord("'"), '"': ord('"'),
|
||
'n': 10, 't': 9, 'r': 13, 'b': 8, 'f': 12, 'v': 11, 'a': 7,
|
||
}
|
||
if esc in simple:
|
||
out.append(simple[esc])
|
||
i += 1
|
||
elif esc == 'x':
|
||
out.append(_HexEscapeVal(body, i + 1, 2))
|
||
i += 3
|
||
elif esc == 'u':
|
||
out.append(_HexEscapeVal(body, i + 1, 4))
|
||
i += 5
|
||
elif esc == 'U':
|
||
out.append(_HexEscapeVal(body, i + 1, 8))
|
||
i += 9
|
||
elif esc.isdigit() and '0' <= esc <= '7':
|
||
oct_digits: str = body[i:i + 3]
|
||
j: int = 0
|
||
while j < 3 and j < len(oct_digits) and '0' <= oct_digits[j] <= '7':
|
||
j += 1
|
||
out.append(int(oct_digits[:j], 8))
|
||
i += j
|
||
else:
|
||
out.append(ord('\\'))
|
||
out.append(ord(esc))
|
||
i += 1
|
||
return bytes(out)
|
||
|
||
# ------------------------------------------------------------------
|
||
# f-string 解析
|
||
# ------------------------------------------------------------------
|
||
def _ParseFStringToken(self, tok: Token) -> N.JoinedStr:
|
||
"""解析 FSTRING token 为 JoinedStr（values 为 Constant/FormattedValue）。"""
|
||
raw: str = tok.Value
|
||
values: list[N.expr] = self._ParseFStringBody(raw, tok, 0)
|
||
node: N.JoinedStr = N.JoinedStr(values=values)
|
||
self._FixLoc(node, tok)
|
||
return node
|
||
|
||
def _ParseFStringBody(self, raw: str, outer_tok: Token, base_offset: int) -> list[N.expr]:
|
||
"""解析 f-string 文本（含前缀与引号）为 values 列表。
|
||
|
||
Args:
|
||
raw: f-string 原始文本
|
||
outer_tok: FSTRING token（用于绝对偏移换算）
|
||
base_offset: 文本在 token Value 中的起始偏移（嵌套用）
|
||
"""
|
||
# 提取前缀与引号
|
||
i: int = 0
|
||
while i < len(raw) and raw[i] in _STRING_PREFIX_CHARS:
|
||
i += 1
|
||
prefix: str = raw[:i]
|
||
quote: str = raw[i]
|
||
triple: bool = raw.startswith(quote * 3, i)
|
||
is_raw: bool = 'r' in prefix.lower()
|
||
body_start: int = i + (3 if triple else 1)
|
||
body_end: int = len(raw) - (3 if triple else 1)
|
||
body: str = raw[body_start:body_end]
|
||
values: list[N.expr] = []
|
||
literal: list[str] = []
|
||
j: int = 0
|
||
n: int = len(body)
|
||
rel_base: int = body_start + base_offset # 相对外层的偏移
|
||
while j < n:
|
||
ch: str = body[j]
|
||
if ch == '{':
|
||
if j + 1 < n and body[j + 1] == '{':
|
||
literal.append('{')
|
||
j += 2
|
||
continue
|
||
# 表达式段开始
|
||
if literal:
|
||
cnode: N.Constant = N.Constant(value=self._DecodeEscapes(''.join(literal), is_raw), kind=None)
|
||
cnode.lineno, cnode.col_offset = outer_tok.Line, outer_tok.Col
|
||
values.append(cnode)
|
||
literal = []
|
||
fv: N.FormattedValue = self._ParseFStringValue(body, j, outer_tok, rel_base)
|
||
values.append(fv)
|
||
# 更新 j 到表达式段之后
|
||
j = self._FStringExprEnd
|
||
continue
|
||
elif ch == '}':
|
||
if j + 1 < n and body[j + 1] == '}':
|
||
literal.append('}')
|
||
j += 2
|
||
continue
|
||
# 单独的 } 是语法错误，宽松处理为字面量
|
||
literal.append('}')
|
||
j += 1
|
||
continue
|
||
else:
|
||
literal.append(ch)
|
||
j += 1
|
||
if literal:
|
||
cnode = N.Constant(value=self._DecodeEscapes(''.join(literal), is_raw), kind=None)
|
||
cnode.lineno, cnode.col_offset = outer_tok.Line, outer_tok.Col
|
||
values.append(cnode)
|
||
return values
|
||
|
||
_FStringExprEnd: int = 0 # 表达式段结束位置（body 内偏移），供调用方跳转
|
||
|
||
def _ParseFStringValue(self, body: str, start: int, outer_tok: Token, rel_base: int) -> N.FormattedValue:
|
||
"""解析 f-string 单个 {expr[!conv][:spec]} 段。
|
||
|
||
返回 FormattedValue；将表达式段结束位置存入 self._FStringExprEnd。
|
||
"""
|
||
j: int = start + 1 # 跳过 '{'
|
||
n: int = len(body)
|
||
depth: int = 1
|
||
expr_start: int = j
|
||
expr_end: int = j
|
||
# 扫描结束字符：'}' 正常闭合；'!'/':' 为转换/格式说明符起点
|
||
stop_char: str = '}'
|
||
# 扫描到匹配的 '}'（考虑括号嵌套与内部字符串）；
|
||
# 顶层 '!' 或 ':' 表示表达式结束（转换/格式说明符开始）
|
||
while j < n:
|
||
ch: str = body[j]
|
||
if ch == '\\':
|
||
j += 2
|
||
continue
|
||
if ch in '\'"':
|
||
# 跳过内部字符串字面量
|
||
j = self._SkipInnerLiteral(body, j)
|
||
continue
|
||
if ch in '([{':
|
||
depth += 1
|
||
j += 1
|
||
continue
|
||
if ch in ')]}':
|
||
if depth == 1 and ch == '}':
|
||
expr_end = j
|
||
j += 1
|
||
break
|
||
if depth == 1 and ch in ')}]':
|
||
# 不匹配：语法错误，宽松按结束处理
|
||
expr_end = j
|
||
j += 1
|
||
break
|
||
depth -= 1
|
||
j += 1
|
||
continue
|
||
if depth == 1 and (ch == '!' or ch == ':'):
|
||
# 转换/格式说明符起点：表达式结束（j 停在操作符处）
|
||
expr_end = j
|
||
stop_char = ch
|
||
break
|
||
j += 1
|
||
self._FStringExprEnd = j
|
||
expr_text: str = body[expr_start:expr_end]
|
||
# 转换与格式说明符（仅当扫描停在 ! 或 : 时才解析）
|
||
conversion: int = -1
|
||
format_spec: N.expr | None = None
|
||
k: int = j
|
||
if stop_char == '!':
|
||
k += 1
|
||
if k < n and body[k] in 'rsa':
|
||
conversion = ord(body[k])
|
||
k += 1
|
||
# 转换后可能还有 :format_spec
|
||
if k < n and body[k] == ':':
|
||
stop_char = ':'
|
||
else:
|
||
# 无格式说明符：跳过 '}' 结束表达式段
|
||
if k < n and body[k] == '}':
|
||
k += 1
|
||
self._FStringExprEnd = k
|
||
if stop_char == ':':
|
||
k += 1
|
||
# 格式说明符可含嵌套 {}（如 {x:{w}}）
|
||
spec_body: list[str] = []
|
||
depth2: int = 1
|
||
while k < n:
|
||
c2: str = body[k]
|
||
if c2 == '{':
|
||
depth2 += 1
|
||
spec_body.append(c2)
|
||
k += 1
|
||
elif c2 == '}':
|
||
depth2 -= 1
|
||
if depth2 == 0:
|
||
break
|
||
spec_body.append(c2)
|
||
k += 1
|
||
else:
|
||
spec_body.append(c2)
|
||
k += 1
|
||
spec_text: str = ''.join(spec_body)
|
||
if any(c == '{' for c in spec_text):
|
||
# 嵌套表达式：构造 JoinedStr
|
||
inner_fs: str = 'f"' + spec_text.replace('"', '\\"') + '"'
|
||
inner_tok: Token = Token(FSTRING, inner_fs, outer_tok.Line, outer_tok.Col,
|
||
outer_tok.EndLine, outer_tok.EndCol)
|
||
format_spec = self._ParseFStringToken(inner_tok)
|
||
else:
|
||
# 静态格式说明符：CPython 包为 JoinedStr(values=[Constant])
|
||
cnode: N.Constant = N.Constant(value=spec_text, kind=None)
|
||
cnode.lineno = outer_tok.Line
|
||
cnode.col_offset = outer_tok.Col
|
||
jstr: N.JoinedStr = N.JoinedStr(values=[cnode])
|
||
format_spec = jstr
|
||
# k 现指向 '}'：跳过它作为表达式段结束
|
||
if k < n and body[k] == '}':
|
||
k += 1
|
||
self._FStringExprEnd = k
|
||
# 解析表达式：子 token 流
|
||
# abs_base = FSTRING token 源码起始 + body 内偏移，映射回源码绝对偏移
|
||
expr_node: N.expr = self._ParseSubExpression(expr_text, outer_tok, outer_tok.Start + expr_start + rel_base)
|
||
fv: N.FormattedValue = N.FormattedValue(value=expr_node, conversion=conversion, format_spec=format_spec)
|
||
fv.lineno = outer_tok.Line
|
||
fv.col_offset = outer_tok.Col
|
||
fv.end_lineno = outer_tok.EndLine
|
||
fv.end_col_offset = outer_tok.EndCol
|
||
return fv
|
||
|
||
def _SkipInnerLiteral(self, body: str, start: int) -> int:
|
||
"""跳过 f-string 表达式内部的字符串字面量，返回结束位置。"""
|
||
j: int = start
|
||
# 前缀
|
||
while j < len(body) and body[j] in _STRING_PREFIX_CHARS:
|
||
j += 1
|
||
if j >= len(body) or body[j] not in '\'"':
|
||
return start + 1
|
||
quote: str = body[j]
|
||
triple: bool = body.startswith(quote * 3, j)
|
||
j += 3 if triple else 1
|
||
while j < len(body):
|
||
ch: str = body[j]
|
||
if ch == '\\':
|
||
j += 2
|
||
continue
|
||
if ch == quote:
|
||
if triple:
|
||
if body.startswith(quote * 3, j):
|
||
j += 3
|
||
return j
|
||
j += 1
|
||
else:
|
||
j += 1
|
||
return j
|
||
j += 1
|
||
return j
|
||
|
||
def _ParseSubExpression(self, text: str, outer_tok: Token, abs_base: int) -> N.expr:
|
||
"""解析子表达式文本（f-string 内部），并重定位到绝对偏移。"""
|
||
sub_toks: list[Token] = Tokenizer(text).Tokenize()
|
||
# 过滤 ENDMARKER 后的 NEWLINE 等
|
||
for t in sub_toks:
|
||
self._Relocate(t, abs_base)
|
||
# 保存主游标，切换到子 token 流
|
||
saved_idx: int = self.Idx
|
||
saved_tokens: list[Token] = self.Tokens
|
||
self.Tokens = sub_toks
|
||
self.Idx = 0
|
||
try:
|
||
expr: N.expr = self.ParseExpr()
|
||
finally:
|
||
self.Tokens = saved_tokens
|
||
self.Idx = saved_idx
|
||
return expr
|
||
|
||
# ------------------------------------------------------------------
|
||
# 括号 / 列表 / 字典 / 集合 / 推导式
|
||
# ------------------------------------------------------------------
|
||
def _ParseParen(self) -> N.expr:
|
||
start: Token = self._Next() # '('
|
||
self._SkipNL()
|
||
# 空元组 ()
|
||
if self._AtOp(')'):
|
||
self._Next()
|
||
tup: N.Tuple = N.Tuple(elts=[], ctx=N.Load())
|
||
self._FixLoc(tup, start)
|
||
return tup
|
||
# yield 表达式
|
||
if self._AtName('yield'):
|
||
yield_node: N.expr = self._ParseYield()
|
||
self._SkipNL()
|
||
self._ExpectOp(')')
|
||
return yield_node
|
||
# 命名表达式 (x := y)
|
||
if self._Peek().Type == NAME and self._Peek(1).Type == OP and self._Peek(1).Value == ':=':
|
||
name_tok: Token = self._Next()
|
||
self._Next() # ':='
|
||
value: N.expr = self.ParseExpr()
|
||
self._SkipNL()
|
||
self._ExpectOp(')')
|
||
walrus: N.NamedExpr = N.NamedExpr(target=N.Name(id=name_tok.Value, ctx=N.Store()), value=value)
|
||
self._FixLoc(walrus, start)
|
||
return walrus
|
||
first: N.expr = self.ParseExpr()
|
||
# Paren-less C cast `(T*)x`: the inner expression ends with a pointer suffix
|
||
# (_Prev() is that suffix's tail token) and the operand follows ')'. Same
|
||
# product as `(T*)(x)` / `T*(x)`: a Call whose func is `T | t.CPtr`.
|
||
is_ptr_cast: bool = self._PtrSuffixTail is not None and self._AtOp(')') and self._Prev() is self._PtrSuffixTail
|
||
self._SkipNL()
|
||
# 生成器表达式 (x for ...)
|
||
if self._AtName('for'):
|
||
generators: list[N.comprehension] = self._ParseComprehensionGenerators()
|
||
self._SkipNL()
|
||
self._ExpectOp(')')
|
||
gen: N.GeneratorExp = N.GeneratorExp(elt=first, generators=generators)
|
||
self._FixLoc(gen, start)
|
||
return gen
|
||
# 元组 (x,) / (x, y)
|
||
if self._AtOp(','):
|
||
elts: list[N.expr] = [first]
|
||
while self._AtOp(','):
|
||
self._Next()
|
||
self._SkipNL()
|
||
if self._AtOp(')'):
|
||
break
|
||
elts.append(self.ParseExpr())
|
||
self._SkipNL()
|
||
self._ExpectOp(')')
|
||
tup = N.Tuple(elts=elts, ctx=N.Load())
|
||
self._FixLoc(tup, start)
|
||
return tup
|
||
# 普通括号表达式 (x)
|
||
self._ExpectOp(')')
|
||
if is_ptr_cast and self._AtCastOperand():
|
||
operand: N.expr = self._ParseFactor()
|
||
cast_call: N.Call = N.Call(func=first, args=[operand], keywords=[])
|
||
self._FixLoc(cast_call, start)
|
||
return cast_call
|
||
return first
|
||
|
||
def _ParseList(self) -> N.expr:
|
||
start: Token = self._Next() # '['
|
||
self._SkipNL()
|
||
if self._AtOp(']'):
|
||
self._Next()
|
||
lst: N.List = N.List(elts=[], ctx=N.Load())
|
||
self._FixLoc(lst, start)
|
||
return lst
|
||
first: N.expr = self.ParseExpr()
|
||
self._SkipNL()
|
||
# 列表推导式
|
||
if self._AtName('for'):
|
||
generators = self._ParseComprehensionGenerators()
|
||
self._SkipNL()
|
||
self._ExpectOp(']')
|
||
lc: N.ListComp = N.ListComp(elt=first, generators=generators)
|
||
self._FixLoc(lc, start)
|
||
return lc
|
||
# 普通列表
|
||
elts = [first]
|
||
while self._AtOp(','):
|
||
self._Next()
|
||
self._SkipNL()
|
||
if self._AtOp(']'):
|
||
break
|
||
elts.append(self.ParseExpr())
|
||
self._SkipNL()
|
||
self._ExpectOp(']')
|
||
lst = N.List(elts=elts, ctx=N.Load())
|
||
self._FixLoc(lst, start)
|
||
return lst
|
||
|
||
def _ParseDictSet(self) -> N.expr:
|
||
start: Token = self._Next() # '{'
|
||
self._SkipNL()
|
||
if self._AtOp('}'):
|
||
self._Next()
|
||
dct: N.Dict = N.Dict(keys=[], values=[])
|
||
self._FixLoc(dct, start)
|
||
return dct
|
||
# ** 展开：{**d}
|
||
keys: list[N.expr | None] = []
|
||
values: list[N.expr] = []
|
||
while True:
|
||
self._SkipNL()
|
||
if self._AtOp('**'):
|
||
self._Next()
|
||
val: N.expr = self.ParseExpr()
|
||
keys.append(None)
|
||
values.append(val)
|
||
else:
|
||
key: N.expr = self.ParseExpr()
|
||
self._SkipNL()
|
||
if self._AtOp(':'):
|
||
self._Next()
|
||
self._SkipNL()
|
||
value: N.expr = self.ParseExpr()
|
||
keys.append(key)
|
||
values.append(value)
|
||
else:
|
||
# 集合元素
|
||
elts: list[N.expr] = [key]
|
||
while self._AtOp(','):
|
||
self._Next()
|
||
self._SkipNL()
|
||
if self._AtOp('}'):
|
||
break
|
||
elts.append(self.ParseExpr())
|
||
self._SkipNL()
|
||
# 集合推导式？
|
||
self._SkipNL()
|
||
if self._AtName('for'):
|
||
generators = self._ParseComprehensionGenerators()
|
||
self._SkipNL()
|
||
self._ExpectOp('}')
|
||
sc: N.SetComp = N.SetComp(elt=elts[0], generators=generators)
|
||
self._FixLoc(sc, start)
|
||
return sc
|
||
self._ExpectOp('}')
|
||
st: N.Set = N.Set(elts=elts)
|
||
self._FixLoc(st, start)
|
||
return st
|
||
self._SkipNL()
|
||
if self._AtOp(','):
|
||
self._Next()
|
||
self._SkipNL()
|
||
if self._AtOp('}'):
|
||
break
|
||
continue
|
||
break
|
||
# 字典推导式？
|
||
self._SkipNL()
|
||
if self._AtName('for'):
|
||
generators = self._ParseComprehensionGenerators()
|
||
self._SkipNL()
|
||
self._ExpectOp('}')
|
||
dc: N.DictComp = N.DictComp(key=keys[0] or N.Constant(value=None), value=values[0], generators=generators)
|
||
self._FixLoc(dc, start)
|
||
return dc
|
||
self._ExpectOp('}')
|
||
dct = N.Dict(keys=keys, values=values)
|
||
self._FixLoc(dct, start)
|
||
return dct
|
||
|
||
def _ParseComprehensionGenerators(self) -> list[N.comprehension]:
|
||
"""解析推导式生成器子句：for target in iter [if cond]* [async for ...]"""
|
||
generators: list[N.comprehension] = []
|
||
while True:
|
||
is_async: bool = False
|
||
if self._AtName('async') and self._Peek(1).Type == NAME and self._Peek(1).Value == 'for':
|
||
self._Next()
|
||
is_async = True
|
||
if not self._AtName('for'):
|
||
break
|
||
start: Token = self._Next() # 'for'
|
||
target: N.expr = self._ParseComprehensionTarget()
|
||
self._ExpectName('in')
|
||
# iter 用 disjunction 级别解析，避免吞掉 'if' 条件
|
||
iterable: N.expr = self._ParseDisjunction()
|
||
ifs: list[N.expr] = []
|
||
while self._AtName('if'):
|
||
self._Next()
|
||
ifs.append(self._ParseDisjunction())
|
||
comp: N.comprehension = N.comprehension(target=target, iter=iterable, ifs=ifs, is_async=int(is_async))
|
||
self._FixLoc(comp, start)
|
||
generators.append(comp)
|
||
if self._AtName('for') or (self._AtName('async') and self._Peek(1).Type == NAME and self._Peek(1).Value == 'for'):
|
||
continue
|
||
break
|
||
return generators
|
||
|
||
def _ParseComprehensionTarget(self) -> N.expr:
|
||
"""解析推导式 for 目标（支持裸元组 a, b 解包）。"""
|
||
start: Token = self._Peek()
|
||
first: N.expr = self._ParseTarget()
|
||
if self._AtOp(','):
|
||
elts: list[N.expr] = [first]
|
||
while self._AtOp(','):
|
||
self._Next()
|
||
self._SkipNL()
|
||
elts.append(self._ParseTarget())
|
||
tup: N.Tuple = N.Tuple(elts=elts, ctx=N.Store())
|
||
self._FixLoc(tup, start)
|
||
return tup
|
||
return first
|
||
|
||
def _ParseTarget(self) -> N.expr:
|
||
"""解析赋值目标（Store 上下文），支持元组/列表解包。"""
|
||
start: Token = self._Peek()
|
||
if self._AtOp('('):
|
||
self._Next()
|
||
self._SkipNL()
|
||
elts: list[N.expr] = []
|
||
if not self._AtOp(')'):
|
||
while True:
|
||
elts.append(self._ParseTarget())
|
||
self._SkipNL()
|
||
if self._AtOp(','):
|
||
self._Next()
|
||
self._SkipNL()
|
||
if self._AtOp(')'):
|
||
break
|
||
continue
|
||
break
|
||
self._ExpectOp(')')
|
||
tup: N.Tuple = N.Tuple(elts=elts, ctx=N.Store())
|
||
self._FixLoc(tup, start)
|
||
return tup
|
||
if self._AtOp('['):
|
||
self._Next()
|
||
self._SkipNL()
|
||
elts = []
|
||
if not self._AtOp(']'):
|
||
while True:
|
||
elts.append(self._ParseTarget())
|
||
self._SkipNL()
|
||
if self._AtOp(','):
|
||
self._Next()
|
||
self._SkipNL()
|
||
if self._AtOp(']'):
|
||
break
|
||
continue
|
||
break
|
||
self._ExpectOp(']')
|
||
lst: N.List = N.List(elts=elts, ctx=N.Store())
|
||
self._FixLoc(lst, start)
|
||
return lst
|
||
if self._AtOp('*'):
|
||
self._Next()
|
||
value: N.expr = self._ParseTarget()
|
||
starred: N.Starred = N.Starred(value=value, ctx=N.Store())
|
||
self._FixLoc(starred, start)
|
||
return starred
|
||
# 名称/属性/下标（Store）
|
||
tok: Token = self._Peek()
|
||
if tok.Type == NAME:
|
||
self._Next()
|
||
node: N.Name = N.Name(id=tok.Value, ctx=N.Store())
|
||
self._FixLoc(node, tok)
|
||
return node
|
||
# 属性/下标目标：解析 Load 后改 Store
|
||
node = self._ParsePrimary()
|
||
self._SetStoreContext(node)
|
||
return node
|
||
|
||
def _SetStoreContext(self, node: N.AST) -> None:
|
||
"""将 Load 上下文节点递归改为 Store。"""
|
||
if isinstance(node, N.Attribute):
|
||
node.ctx = N.Store()
|
||
elif isinstance(node, N.Subscript):
|
||
node.ctx = N.Store()
|
||
elif isinstance(node, N.Name):
|
||
node.ctx = N.Store()
|
||
elif isinstance(node, (N.Tuple, N.List)):
|
||
node.ctx = N.Store()
|
||
for elt in node.elts:
|
||
self._SetStoreContext(elt)
|
||
elif isinstance(node, N.Starred):
|
||
node.ctx = N.Store()
|
||
self._SetStoreContext(node.value)
|
||
|
||
# ------------------------------------------------------------------
|
||
# lambda / yield
|
||
# ------------------------------------------------------------------
|
||
def _ParseLambda(self) -> N.Lambda:
|
||
start: Token = self._Next() # 'lambda'
|
||
args: N.arguments
|
||
if self._AtOp(':'):
|
||
# 无参 lambda
|
||
args = N.arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[],
|
||
kw_defaults=[], kwarg=None, defaults=[])
|
||
else:
|
||
args = self._ParseArguments(skip_open=True, allow_lambda=True)
|
||
self._ExpectOp(':')
|
||
body: N.expr = self.ParseExpr()
|
||
node: N.Lambda = N.Lambda(args=args, body=body)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
|
||
def _ParseYield(self) -> N.expr:
|
||
start: Token = self._Next() # 'yield'
|
||
if self._AtName('from'):
|
||
self._Next()
|
||
value: N.expr = self.ParseExpr()
|
||
node: N.YieldFrom = N.YieldFrom(value=value)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
if not self._AtOp(')'):
|
||
value = self.ParseExprList()
|
||
node: N.Yield = N.Yield(value=value)
|
||
else:
|
||
node = N.Yield(value=None)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
|
||
# ==================================================================
|
||
# 参数列表（lambda 与函数定义共用）
|
||
# ==================================================================
|
||
def _ParseArguments(self, skip_open: bool = False, allow_lambda: bool = False) -> N.arguments:
|
||
"""解析参数列表。
|
||
|
||
支持 posonly (/)、*args、kwonly、**kwargs、默认值、注解。
|
||
defaults 与 posonlyargs+args 尾部对齐（CPython 语义）。
|
||
|
||
Args:
|
||
skip_open: True 时假定 '(' 已消费（lambda 场景无括号）
|
||
"""
|
||
if not skip_open:
|
||
self._ExpectOp('(')
|
||
self._SkipNL()
|
||
posonlyargs: list[N.arg] = []
|
||
args: list[N.arg] = []
|
||
kwonlyargs: list[N.arg] = []
|
||
kw_defaults: list[N.expr | None] = []
|
||
defaults: list[N.expr] = []
|
||
vararg: N.arg | None = None
|
||
kwarg: N.arg | None = None
|
||
seen_star: bool = False
|
||
# 按解析顺序收集 (arg, default) 用于尾部对齐
|
||
ordered: list[tuple[N.arg, N.expr | None]] = []
|
||
if not self._AtOp(')'):
|
||
while True:
|
||
self._SkipNL()
|
||
if self._AtOp(')'):
|
||
break
|
||
if self._AtOp('*'):
|
||
self._Next()
|
||
if self._AtOp(','):
|
||
# 裸 *：分隔 kwonly
|
||
seen_star = True
|
||
self._Next()
|
||
if self._AtOp(')') or self._AtOp(':'):
|
||
break
|
||
continue
|
||
vararg = self._ParseArg(allow_annotation=not allow_lambda)
|
||
seen_star = True
|
||
elif self._AtOp('**'):
|
||
self._Next()
|
||
kwarg = self._ParseArg(allow_annotation=not allow_lambda)
|
||
else:
|
||
a: N.arg = self._ParseArg(allow_annotation=not allow_lambda)
|
||
default: N.expr | None = None
|
||
if self._AtOp('='):
|
||
self._Next()
|
||
default = self.ParseExpr()
|
||
if seen_star:
|
||
kwonlyargs.append(a)
|
||
kw_defaults.append(default)
|
||
else:
|
||
ordered.append((a, default))
|
||
self._SkipNL()
|
||
if self._AtOp(','):
|
||
self._Next()
|
||
if self._AtOp(')'):
|
||
break
|
||
continue
|
||
break
|
||
# 切分 posonly 与普通 args：CPython 无显式 '/' 信息时全为普通 args
|
||
# 注意：本实现未显式解析 '/' 分隔，posonlyargs 保持空（TPC 源码不使用）
|
||
for a, d in ordered:
|
||
args.append(a)
|
||
defaults.append(d)
|
||
# defaults 尾部对齐：去掉开头的 None，且仅保留末尾连续默认值
|
||
# CPython 语义：defaults 与 args 尾部对齐，若中间有缺省则语法错误。
|
||
# 宽松处理：取最后一个 None 之后的全部默认值。
|
||
last_none: int = -1
|
||
for i, d in enumerate(defaults):
|
||
if d is None:
|
||
last_none = i
|
||
defaults = [d for d in defaults[last_none + 1:] if d is not None]
|
||
# 将带默认值的 arg 之前的参数清掉默认值列表长度 = 默认参数个数
|
||
self._SkipNL()
|
||
if not skip_open:
|
||
self._ExpectOp(')')
|
||
node: N.arguments = N.arguments(
|
||
posonlyargs=posonlyargs, args=args, vararg=vararg,
|
||
kwonlyargs=kwonlyargs, kw_defaults=kw_defaults, kwarg=kwarg,
|
||
defaults=defaults,
|
||
)
|
||
return node
|
||
|
||
def _ParseArg(self, allow_annotation: bool) -> N.arg:
|
||
start: Token = self._ExpectName()
|
||
annotation: N.expr | None = None
|
||
if allow_annotation and self._AtOp(':'):
|
||
self._Next()
|
||
annotation = self._ParseTypeAnnotation()
|
||
a: N.arg = N.arg(arg=start.Value, annotation=annotation, type_comment=None)
|
||
self._FixLoc(a, start)
|
||
return a
|
||
|
||
def _ParseTypeAnnotation(self) -> N.expr:
|
||
"""解析类型注解，支持 C 风格指针后缀（* / **）。
|
||
|
||
注解模式开启后，'_ParseTerm'/'_ParsePower' 不再把 '*' / '**' 当
|
||
乘法/幂运算消费，而由本方法统一循环消费并展开为嵌套联合类型：
|
||
|
||
int* → int | t.CPtr （i32*）
|
||
int** → (int | t.CPtr) | t.CPtr （i32**）
|
||
int****** → 6 层嵌套 （i32******）
|
||
|
||
展开为 BinOp(BitOr) 链后，TPC/TPV 现有类型解析（TypeAnnotationResolver
|
||
的 PtrCount 相加、TPV TypeGenResolve 的递归 BinOp 分支）零改动即可
|
||
生成多级指针 LLVM 类型。
|
||
|
||
无法解析的后缀形态**响亮失败**（SyntaxError，含 行:列 + 源码片段 + 形态 +
|
||
正确写法示例），绝不静默丢弃：注解末尾若仍遗留未被 `_AtPtrSuffix` 接受的
|
||
'*'/'**'（如 `a: int * 2` / `a: int* | t.CPtr`），判据拒绝消费即终止编译
|
||
（历史事故：静默退化会产出错误的指针层数）。
|
||
|
||
值引用参数注解（仅形参位有效）：前导 `&` 为 ref 模式、`&temp` 为 temp 模式，
|
||
合成联合形态 `T | t.CRef` / `T | t.CTempRef` 承载（CRef/CTempRef 非真实类，
|
||
仅按名字识别；语义侧见 HandlesFunctions / FuncGen）。表达式位的 `&x` 不受影响。
|
||
"""
|
||
ref_mode: str | None = None
|
||
ref_tok: Token = self._Peek()
|
||
if self._AtOp('&'):
|
||
self._Next()
|
||
if self._AtName('temp'):
|
||
self._Next()
|
||
ref_mode = 'temp'
|
||
else:
|
||
ref_mode = 'ref'
|
||
# `&` / `&temp` 之后必须紧跟类型；缺失即响亮失败（禁止静默丢弃）
|
||
end_tok: Token = self._Peek()
|
||
if end_tok.Type in (NEWLINE, NL, ENDMARKER) or (
|
||
end_tok.Type == OP and end_tok.Value in (')', ',', ':', '=', ']', '}', ';')):
|
||
snippet: str = ''
|
||
try:
|
||
snippet = (self.Source.splitlines()[end_tok.Line - 1] or '').strip()
|
||
except Exception:
|
||
snippet = ''
|
||
raise SyntaxError(
|
||
f"[vast] 值引用参数注解缺少类型：第 {end_tok.Line} 行第 {end_tok.Col} 列的 "
|
||
f"`&` 后必须紧跟类型 T（ref：`&T`；temp：`&temp T`）。"
|
||
f"源码片段 {snippet!r}。正确示例：`def f(p: &int)` / `def f(p: &temp void)`。")
|
||
saved_flag: bool = self._InTypeAnnotation
|
||
self._InTypeAnnotation = True
|
||
try:
|
||
expr: N.expr = self.ParseExpr()
|
||
finally:
|
||
self._InTypeAnnotation = saved_flag
|
||
if expr is None:
|
||
if ref_mode is not None:
|
||
raise SyntaxError(
|
||
"[vast] 值引用参数注解缺少类型：`&` 后必须紧跟类型 T"
|
||
"（ref：`&T`；temp：`&temp T`）。")
|
||
return None
|
||
if self._AtPtrSuffix(expr):
|
||
expr = self._ConsumePtrSuffix(expr)
|
||
tok: Token = self._Peek()
|
||
if tok.Type == OP and tok.Value in ('*', '**'):
|
||
snippet: str = ''
|
||
try:
|
||
snippet = (self.Source.splitlines()[tok.Line - 1] or '').strip()
|
||
except Exception:
|
||
snippet = ''
|
||
raise SyntaxError(
|
||
f"[vast] 注解位置的指针后缀无法解析：第 {tok.Line} 行第 {tok.Col} 列的 "
|
||
f"{tok.Value!r} 前面不是可加后缀的类型（Name / Attribute / Subscript），"
|
||
f"或其后不是后缀的合法结束位置（注解结尾，或 ']' / ',' / ')' / ':' / '='）。"
|
||
f"源码片段 {snippet!r}。"
|
||
f"期望形态：`T*`（1 层）/ `T**`（2 层），T 可为裸名 / t.X / mod.Type / "
|
||
f"rawptr[T] / t.CPtr[T]。"
|
||
f"正确示例：`a: t.CPtr*` / `a: rawptr*` / `a: t.X**` / "
|
||
f"`a: rawptr[ast.AST*]` / `def f(p: int*) -> int**`。"
|
||
)
|
||
if ref_mode is not None:
|
||
expr = self._MakeRefMarker(expr, ref_mode, ref_tok)
|
||
return expr
|
||
|
||
def _MakeRefMarker(self, base: N.expr, mode: str, tok: Token) -> N.expr:
|
||
"""构造值引用标记联合：`base | t.CRef`（ref）/ `base | t.CTempRef`（temp）。
|
||
|
||
CRef / CTempRef 非真实类，仅作为注解 AST 内的名字标记（语义侧按名字识别）。
|
||
"""
|
||
marker: N.expr = N.Attribute(
|
||
value=N.Name(id='t', ctx=N.Load()),
|
||
attr='CRef' if mode == 'ref' else 'CTempRef',
|
||
ctx=N.Load())
|
||
node: N.BinOp = N.BinOp(left=base, op=N.BitOr(), right=marker)
|
||
self._FixLoc(node, tok)
|
||
return node
|
||
|
||
# 类型结束后缀：'_AtPtrSuffix' 的判据表
|
||
# - 全局（任意表达式上下文）：'*'/'**' 串后紧跟这些 operator 时，乘法/幂本就缺右操作数
|
||
# （即原语法错误），故按指针后缀处理不会改变任何合法表达式的语义；
|
||
# - 注解模式额外允许语句结束位置（NEWLINE/NL/ENDMARKER/';'）。
|
||
_PTR_SUFFIX_END_OPS: tuple[str, ...] = (']', ',', ')', ':', '=')
|
||
_PTR_SUFFIX_STMT_END_OPS: tuple[str, ...] = (';',)
|
||
# 指针后缀「强转调用」后缀：'*'/'**' 串后**紧贴**（无空白）'(' 时，写成 C 风格
|
||
# 强转调用 `T*(x)`。'(' **不**放进 `_PTR_SUFFIX_END_OPS`：`a * (b)` 是合法乘法
|
||
# （'(' 正是乘法的右操作数起点），无条件把 '(' 计入会误伤乘法；故另立一条
|
||
# 「紧贴」判据（见 `_AtPtrSuffixCall`），只认 `T*(` 这种**无空白**的强转写法。
|
||
_PTR_SUFFIX_CALL_OP: str = '('
|
||
|
||
def _AtPtrSuffix(self, base: N.expr) -> bool:
|
||
"""判定当前 '*'/'**' 串是否为 C 风格指针后缀（作用于 base）。
|
||
|
||
判据（两条同时成立）:
|
||
1. base 是"类型原子"形态（Name / Attribute / Subscript）—— 即 ``ast.AST``、
|
||
``t.CArray[X, N]``、``rawptr[X]`` 之类；
|
||
2. 该 '*'/'**' 串（可跨括号内换行 NL）之后紧跟"无右操作数"位置：
|
||
``]`` / ``,`` / ``)`` / ``:`` / ``=``（全局），或注解模式下的
|
||
NEWLINE / NL / ENDMARKER / ``;``。
|
||
|
||
因此合法表达式 ``x[a * b]`` / ``f(x * y)`` / ``x[1 * 2]`` / ``a ** b``
|
||
（'*' 之后是操作数）永不被判为指针后缀。
|
||
"""
|
||
if not isinstance(base, (N.Name, N.Attribute, N.Subscript)):
|
||
return False
|
||
j: int = 0
|
||
stars: int = 0
|
||
while True:
|
||
tok: Token = self._Peek(j)
|
||
if tok.Type == OP and tok.Value in ('*', '**'):
|
||
stars += 1 if tok.Value == '*' else 2
|
||
j += 1
|
||
elif tok.Type == NL:
|
||
j += 1
|
||
else:
|
||
break
|
||
if stars == 0:
|
||
return False
|
||
nxt: Token = self._Peek(j)
|
||
if nxt.Type == OP and nxt.Value in self._PTR_SUFFIX_END_OPS:
|
||
return True
|
||
if self._InTypeAnnotation:
|
||
if nxt.Type in (NEWLINE, NL, ENDMARKER):
|
||
return True
|
||
if nxt.Type == OP and nxt.Value in self._PTR_SUFFIX_STMT_END_OPS:
|
||
return True
|
||
return False
|
||
|
||
def _ConsumePtrSuffix(self, expr: N.expr) -> N.expr:
|
||
"""消费连续的 C 风格指针后缀（'*' 一层 / '**' 两层），展开为 base | t.CPtr 链。"""
|
||
while True:
|
||
tok: Token = self._Peek()
|
||
if tok.Type == OP and tok.Value == '*':
|
||
self._Next()
|
||
self._PtrSuffixTail = tok
|
||
expr = self._MakePtrType(expr, tok)
|
||
elif tok.Type == OP and tok.Value == '**':
|
||
self._Next()
|
||
self._PtrSuffixTail = tok
|
||
expr = self._MakePtrType(expr, tok)
|
||
expr = self._MakePtrType(expr, tok)
|
||
else:
|
||
break
|
||
return expr
|
||
|
||
def _AtCastOperand(self) -> bool:
|
||
"""Whether the current token may start the operand of a paren-less cast `(T*)x`.
|
||
|
||
'(' is deliberately absent: `(T*)(x)` is already handled by the call trailer,
|
||
so that path's AST stays untouched. NL is not skipped either: a line break
|
||
after ')' starts a new statement, not a cast operand.
|
||
"""
|
||
tok: Token = self._Peek()
|
||
if tok.Type in (NAME, NUMBER, STRING):
|
||
return True
|
||
return tok.Type == OP and tok.Value in ('[', '{', '*', '&', '-', '+', '~')
|
||
|
||
def _AtOperandAfterStarRun(self) -> bool:
|
||
"""当前 '*'/'**' 串之后是否还有操作数起点（可跨括号内换行 NL）。
|
||
|
||
与 `_AtPtrSuffix` 互补：后者判「'*' 后是无右操作数位置」（']' / ',' / ')' /
|
||
':' / '=' / 行尾 / ';' ⇒ 指针后缀），本方法判其反面（'*' 后是 NAME / NUMBER /
|
||
STRING / FSTRING 或 '(' / '[' / '{' / 一元前缀 ⇒ 乘法）。用于下标实参内区分
|
||
`t.CArray[T, A * B]`（乘法）与 `t.CArray[T*, 4]`（指针后缀）。
|
||
"""
|
||
j: int = 0
|
||
while True:
|
||
tok: Token = self._Peek(j)
|
||
if tok.Type == OP and tok.Value in ('*', '**'):
|
||
j += 1
|
||
elif tok.Type == NL:
|
||
j += 1
|
||
else:
|
||
break
|
||
nxt: Token = self._Peek(j)
|
||
if nxt.Type in (NAME, NUMBER, STRING, FSTRING):
|
||
return True
|
||
return nxt.Type == OP and nxt.Value in ('(', '[', '{', '*', '&', '-', '+', '~', '...')
|
||
|
||
def _AtPtrSuffixCall(self, base: N.expr) -> bool:
|
||
"""判定当前 '*'/'**' 串是否为 `T*(x)` 形式的指针后缀**强转调用**。
|
||
|
||
判据（三条同时成立）:
|
||
1. base 是"类型原子"形态（Name / Attribute / Subscript）—— 与
|
||
`_AtPtrSuffix` 的判据 1 同源（`ast.Call` / `list[ast.AST*]` 之类）；
|
||
2. base 与 '*'/'**' 串**紧贴**（源码字符相邻，无空白/换行）；
|
||
3. 该 '*'/'**' 串（串内**逐个紧贴**）之后**紧贴**一个 '('。
|
||
|
||
即整段写作 **无空白**的 `T*(x)`（C 风格强转写法）。为什么 '(' 不在
|
||
`_PTR_SUFFIX_END_OPS` 里：`a * (b)` 是合法乘法（'(' 可作乘法右操作数的
|
||
起点），把 '(' 无条件计入表会误伤乘法。故此处叠加"前驱是类型原子 + 两侧
|
||
紧贴"两条判据：`a * (b + c)` / `f(x) * (y)` / `n * (m - 1)` /
|
||
`(i + 1) * (j)`（'*' 前后有空白，且前驱多为非类型原子）仍按乘法解析。
|
||
"""
|
||
if not isinstance(base, (N.Name, N.Attribute, N.Subscript)):
|
||
return False
|
||
first: Token = self._Peek(0)
|
||
prev: Token = self._Prev()
|
||
if prev.Type in (NEWLINE, NL) or prev.End != first.Start:
|
||
return False # base 与 '*'/'**' 之间有空白/换行：非 C 风格强转写法
|
||
j: int = 0
|
||
stars: int = 0
|
||
last_end: int = -1
|
||
while True:
|
||
tok: Token = self._Peek(j)
|
||
if tok.Type == OP and tok.Value in ('*', '**'):
|
||
if last_end != -1 and tok.Start != last_end:
|
||
return False # 星串内部出现空白：非紧邻，不判为强转
|
||
last_end = tok.End
|
||
stars += 1
|
||
j += 1
|
||
else:
|
||
break
|
||
if stars == 0:
|
||
return False
|
||
nxt: Token = self._Peek(j)
|
||
return nxt.Type == OP and nxt.Value == self._PTR_SUFFIX_CALL_OP \
|
||
and nxt.Start == last_end
|
||
|
||
def _ConsumePtrSuffixCall(self, base: N.expr, start: Token) -> N.expr:
|
||
"""把已认定为 `T*(...)` 的指针后缀强转解析为**调用**。
|
||
|
||
产物与既有强转形态 `(T | t.CPtr)(...)` 逐字同形：同为 `ast.Call`
|
||
（func = `T` 展开的 `base | t.CPtr` 链，args/keywords = 括号实参），
|
||
不新造第二套 cast 表示。
|
||
"""
|
||
func: N.expr = self._ConsumePtrSuffix(base)
|
||
return self._ParseCallTail(func, start)
|
||
|
||
def _MakePtrType(self, base: N.expr, tok: Token) -> N.expr:
|
||
"""构造一层指针包装：base | t.CPtr。"""
|
||
cptr: N.expr = N.Attribute(value=N.Name(id='t', ctx=N.Load()),
|
||
attr='CPtr', ctx=N.Load())
|
||
node: N.BinOp = N.BinOp(left=base, op=N.BitOr(), right=cptr)
|
||
self._FixLoc(node, tok)
|
||
return node
|