"""vast.exprparser - Viper/Python 表达式语法分析器 实现与 CPython 3.12 语法规则一致的递归下降表达式解析： - 完整运算符优先级（lambda/条件/or/and/not/比较/位运算/移位/算术/一元/幂/await） - 原子表达式：标识符/数字/字符串/常量/f-string/元组/列表/字典/集合 - 推导式（list/set/dict/generator）、lambda、命名表达式（walrus） - 调用（位置实参/关键字实参/*args/**kwargs）、属性、下标、切片 - f-string（JoinedStr/FormattedValue，含嵌套表达式与格式说明符） 本模块为 Mixin 设计：``Parser`` 继承 ``ExprParserMixin`` 获得表达式能力。 所有产出节点与 CPython ``ast`` 结构完全一致，并填充 lineno/col_offset/ end_lineno/end_col_offset 位置属性。 """ from __future__ import annotations import bisect from lib.vast import Nodes as N from lib.vast.Tokenizer import ( Token, NAME, NUMBER, STRING, FSTRING, OP, NEWLINE, NL, ENDMARKER, Tokenizer, _STRING_PREFIX_CHARS, ) # --------------------------------------------------------------------------- # `new [描述符...] T(args)` 生命周期分配表达式（与 ViperC includes/ast/exprs.vp # 的 NEW_SRC_*/NEW_LIFE_* 同值）。TPC 侧同样按 VPC 口径解析成 New 节点，语义落地 # 由 Handles 负责（见 HandlesExprCall._HandleNewExprLlvm）。 # --------------------------------------------------------------------------- NEW_SRC_DEFAULT = 0 NEW_SRC_STACK = 1 NEW_SRC_HEAP = 2 NEW_SRC_MEMGR = 3 NEW_LIFE_DEFAULT = 0 NEW_LIFE_SLIFE = 1 NEW_LIFE_HLIFE = 2 # `new` 之后的普通名不能是这些保留字（它们在 vast 里也是 NAME token）。 # 缺此判据时 `for new in x:` 会被误判为 new 表达式（`in` 紧随其后）。 _NEW_NEXT_RESERVED: frozenset[str] = frozenset({ 'in', 'is', 'and', 'or', 'not', 'if', 'else', 'elif', 'for', 'while', 'lambda', 'await', 'yield', 'return', 'pass', 'break', 'continue', 'import', 'from', 'as', 'with', 'try', 'except', 'finally', 'raise', 'def', 'class', 'global', 'nonlocal', 'del', 'assert', 'match', 'case', 'True', 'False', 'None', }) def _JoinConstParts(parts: list[object]) -> str | bytes: """拼接连续字符串字面量值（str 或 bytes，与 CPython 语义一致）。""" if not parts: return '' if isinstance(parts[0], bytes): buf: list[bytes] = [] for p in parts: if isinstance(p, bytes): buf.append(p) else: buf.append(str(p).encode('utf-8')) return b''.join(buf) return ''.join(str(p) for p in parts) _HEX_ALPHABET: str = '0123456789abcdefABCDEF' # 已删除的用户可见 CIf 族名（CIf/CIfdef/CIfndef/CElif/CElse/CEndif 及 c. 前缀形式）： # 用户再写即 FailFast，提示改用原生 cif/celif/celse 与 defined() 谓词 #（内部标记 c.@CIf 族由解析期合成，不走此处）。 _REMOVED_CIF_NAMES: frozenset[str] = frozenset({ 'CIf', 'CIfdef', 'CIfndef', 'CElif', 'CElse', 'CEndif', 'CIf', 'CIfdef', 'CIfndef', 'CElif', 'CElse', 'CEndif', }) def _HexValOf(ch: str) -> int: """单个字符的十六进制位值；非十六进制字符返回 0（与 C 解析器 vast_c.HexVal 同口径）。""" idx: int = _HEX_ALPHABET.find(ch) if idx < 0: return 0 return idx if idx < 16 else idx - 6 def _FindSourceCPtr(Node: N.AST) -> N.AST | None: """在注解 AST 中查找**源码书写**的 `CPtr`（裸名 / 限定名）。 内部表示仍是 `Subscript(Name('CPtr'), T)`（`T*` 脱糖产物），但该合成 Name 不带位置属性；源码中的 CPtr 一定带 lineno。据此只拒绝用户源码写法， 不影响内部 AST。 """ if isinstance(Node, (N.Name, N.Attribute)): Name: str = Node.id if isinstance(Node, N.Name) else Node.attr if Name == 'CPtr' and hasattr(Node, 'lineno'): return Node for Field in getattr(Node, '_fields', ()): Value = getattr(Node, Field, None) if isinstance(Value, N.AST): Found: N.AST | None = _FindSourceCPtr(Value) if Found is not None: return Found elif isinstance(Value, list): for Item in Value: if isinstance(Item, N.AST): Found = _FindSourceCPtr(Item) if Found is not None: return Found return None def _HexEscapeVal(body: str, start: int, count: int) -> int: """取转义序列后 count 位十六进制的值（越界位置与非十六进制字符均按 0 计）。 两侧解析器对非法 `\\x` / `\\u` / `\\U` 转义必须产出同一结果且都不许抛异常： C 解析器走 HexVal（非 hex → 0），此处原来直接 `int(..., 16)`，遇 `\\xNN` 这类字面量会抛 ValueError（includes/ast/lexer.vp 的文档字符串即触发）。 """ val: int = 0 n: int = len(body) for k in range(count): idx: int = start + k val = val * 16 + _HexValOf(body[idx] if idx < n else '0') return val class TokenStream: """token 流管理基类：统一提供游标操作、位置标注工具。""" def __init__(self, tokens: list[Token], source: str, line_starts: list[int]) -> None: self.Tokens: list[Token] = tokens self.Source: str = source self.LineStarts: list[int] = line_starts self.Idx: int = 0 # _Peek(0) 缓存：递归下降解析常在同一游标位置重复窥视 # （_At/_AtOp/_AtName 等），命中时跳过 len() 与列表索引。 # 游标移动后 Idx 变化自然失配，无需显式失效。 self._PeekCacheIdx: int = -1 self._PeekCacheTok: Token | None = None # 类型注解解析模式：为 True 时 '*' / '**' 按 C 风格指针后缀处理 # （int* → CPtr[int]、int** → 嵌套两层），不按乘法/幂运算消费。 self._InTypeAnnotation: bool = False # 下标实参嵌套层数：'[' 实参内是表达式位置（CArray[T, A * B] 的尺寸）， # 此处的 '*' 在有右操作数时按乘法解析（见 _ParseTerm / _AtOperandAfterStarRun）。 self._SubscriptArgs: int = 0 # Tail token of the last consumed C-style pointer suffix ('*' / '**'). # _ParseParen compares it against _Prev() to recognize the paren-less cast # `(T*)x`, which must not be mistaken for a plain group `(a | b)`. self._PtrSuffixTail: Token | None = None # ------------------------------------------------------------------ # 游标操作 # ------------------------------------------------------------------ def _Peek(self, offset: int = 0) -> Token: i: int = self.Idx + offset if i >= len(self.Tokens): return self.Tokens[-1] if offset == 0: if self._PeekCacheIdx == i: return self._PeekCacheTok self._PeekCacheIdx = i self._PeekCacheTok = self.Tokens[i] return self._PeekCacheTok return self.Tokens[i] def _Next(self) -> Token: tok: Token = self.Tokens[self.Idx] if self.Idx < len(self.Tokens) - 1: self.Idx += 1 return tok def _Prev(self) -> Token: return self.Tokens[max(0, self.Idx - 1)] def _At(self, tok_type: str, value: str | None = None) -> bool: # 内联 _Peek(0) 缓存逻辑（行为与 _Peek(offset=0) 完全一致）： # 命中缓存直接取 _PeekCacheTok，miss 时读 Tokens[i] 并更新缓存； # i >= len(Tokens) 时取末尾哨兵 token（不更新缓存）。 i: int = self.Idx if i >= len(self.Tokens): tok: Token = self.Tokens[-1] elif self._PeekCacheIdx == i: tok = self._PeekCacheTok else: self._PeekCacheIdx = i self._PeekCacheTok = self.Tokens[i] tok = self._PeekCacheTok if tok.Type != tok_type: return False if value is not None and tok.Value != value: return False return True def _AtOp(self, value: str) -> bool: return self._At(OP, value) def _AtName(self, value: str) -> bool: return self._At(NAME, value) def _SkipNL(self) -> None: """跳过括号内换行 token（NL）。""" while self._Peek().Type == NL: self._Next() def _SkipNEWLINE(self) -> None: """跳过逻辑换行 token（NEWLINE）与空行/注释行（NL）。""" while self._Peek().Type in (NEWLINE, NL): self._Next() def _ExpectOp(self, value: str) -> Token: tok: Token = self._Next() if tok.Type != OP or tok.Value != value: raise SyntaxError( f"expected '{value}', got {tok.Type}({tok.Value!r}) " f"at line {tok.Line}" ) return tok def _ExpectName(self, value: str | None = None) -> Token: tok: Token = self._Next() if tok.Type != NAME or (value is not None and tok.Value != value): want: str = f" {value!r}" if value else '' raise SyntaxError( f"expected name{want}, " f"got {tok.Type}({tok.Value!r}) at line {tok.Line}" ) return tok def _ExpectNEWLINE(self) -> None: tok: Token = self._Next() if tok.Type != NEWLINE: raise SyntaxError( f"expected newline, got {tok.Type}({tok.Value!r}) at line {tok.Line}" ) # ------------------------------------------------------------------ # 位置标注 # ------------------------------------------------------------------ def _FixLoc(self, node: N.AST, start_tok: Token, end_tok: Token | None = None) -> N.AST: """为节点设置 4 个位置属性（end 缺省取前一个 token）。""" if end_tok is None: end_tok = self._Prev() if 'lineno' in getattr(node, '_attributes', ()): node.lineno = start_tok.Line node.col_offset = start_tok.Col node.end_lineno = end_tok.EndLine node.end_col_offset = end_tok.EndCol return node def _PosFromOffset(self, offset: int) -> tuple[int, int]: """字符偏移 → (行号, UTF-8 字节列偏移)。""" idx: int = bisect.bisect_right(self.LineStarts, offset) - 1 line: int = idx + 1 col: int = len(self.Source[self.LineStarts[idx]:offset].encode('utf-8')) return line, col def _Relocate(self, tok: Token, base: int) -> None: """将子表达式 token 按绝对偏移重定位（f-string 内部表达式用）。""" tok.Start = base + tok.Start tok.End = base + tok.End line, col = self._PosFromOffset(tok.Start) e_line, e_col = self._PosFromOffset(tok.End) tok.Line, tok.Col, tok.EndLine, tok.EndCol = line, col, e_line, e_col class ExprParserMixin(TokenStream): """表达式解析器（Mixin，供 Parser 使用）。""" # ================================================================== # 表达式顶层：lambda | 条件表达式 # ================================================================== def ParseExpr(self) -> N.expr: """完整表达式：lambda 或条件表达式（含 walrus 前置）。""" if self._AtName('lambda'): return self._ParseLambda() return self._ParseConditional() def ParseExprList(self) -> N.expr: """testlist_star_expr：表达式或逗号分隔元组（赋值右值/return 值）。""" start: Token = self._Peek() first: N.expr = self.ParseExpr() if self._AtOp(','): elts: list[N.expr] = [first] while self._AtOp(','): self._Next() self._SkipNL() elts.append(self.ParseExpr()) tup: N.Tuple = N.Tuple(elts=elts, ctx=N.Load()) self._FixLoc(tup, start) return tup return first def _ParseConditional(self) -> N.expr: """条件表达式：disjunction ('if' disjunction 'else' expression)?""" start: Token = self._Peek() body: N.expr = self._ParseDisjunction() if self._AtName('if'): self._Next() test: N.expr = self._ParseDisjunction() self._ExpectName('else') orelse: N.expr = self.ParseExpr() node: N.IfExp = N.IfExp(test=test, body=body, orelse=orelse) self._FixLoc(node, start) return node return body # ================================================================== # 布尔/比较 # ================================================================== def _ParseDisjunction(self) -> N.expr: start: Token = self._Peek() values: list[N.expr] = [self._ParseConjunction()] while True: self._SkipNL() if self._AtName('or'): self._Next() values.append(self._ParseConjunction()) continue break if len(values) == 1: return values[0] node: N.BoolOp = N.BoolOp(op=N.Or(), values=values) self._FixLoc(node, start) return node def _ParseConjunction(self) -> N.expr: start: Token = self._Peek() values: list[N.expr] = [self._ParseInversion()] while True: self._SkipNL() if self._AtName('and'): self._Next() values.append(self._ParseInversion()) continue break if len(values) == 1: return values[0] node: N.BoolOp = N.BoolOp(op=N.And(), values=values) self._FixLoc(node, start) return node def _ParseInversion(self) -> N.expr: self._SkipNL() if self._AtName('not'): start: Token = self._Next() operand: N.expr = self._ParseInversion() node: N.UnaryOp = N.UnaryOp(op=N.Not(), operand=operand) self._FixLoc(node, start) return node return self._ParseComparison() # 比较操作符映射 _COMPARE_OPS: dict[str, type] = { '<': N.Lt, '>': N.Gt, '<=': N.LtE, '>=': N.GtE, '==': N.Eq, '!=': N.NotEq, 'is': N.Is, 'is not': N.IsNot, 'in': N.In, 'not in': N.NotIn, } def _ParseComparison(self) -> N.expr: start: Token = self._Peek() left: N.expr = self._ParseBitOr() ops: list[N.AST] = [] comparators: list[N.expr] = [] while True: self._SkipNL() tok: Token = self._Peek() op_type: type | None = None if tok.Type == OP and tok.Value in ('<', '>', '<=', '>=', '==', '!='): op_type = self._COMPARE_OPS[tok.Value] self._Next() elif tok.Type == NAME and tok.Value == 'is': self._Next() if self._AtName('not'): self._Next() op_type = N.IsNot else: op_type = N.Is elif tok.Type == NAME and tok.Value == 'in': self._Next() op_type = N.In elif tok.Type == NAME and tok.Value == 'not' and self._Peek(1).Type == NAME and self._Peek(1).Value == 'in': self._Next() self._Next() op_type = N.NotIn else: break ops.append(op_type()) comparators.append(self._ParseBitOr()) if not ops: return left node: N.Compare = N.Compare(left=left, ops=ops, comparators=comparators) self._FixLoc(node, start) return node # ================================================================== # 位运算 # ================================================================== def _ParseBitOr(self) -> N.expr: start: Token = self._Peek() left: N.expr = self._ParseBitXor() while True: self._SkipNL() if self._AtOp('|'): self._Next() right: N.expr = self._ParseBitXor() node: N.BinOp = N.BinOp(left=left, op=N.BitOr(), right=right) self._FixLoc(node, start) left = node continue break return left def _ParseBitXor(self) -> N.expr: start: Token = self._Peek() left: N.expr = self._ParseBitAnd() while True: self._SkipNL() if self._AtOp('^'): self._Next() right: N.expr = self._ParseBitAnd() node: N.BinOp = N.BinOp(left=left, op=N.BitXor(), right=right) self._FixLoc(node, start) left = node continue break return left def _ParseBitAnd(self) -> N.expr: start: Token = self._Peek() left: N.expr = self._ParseShift() while True: self._SkipNL() if self._AtOp('&'): self._Next() right: N.expr = self._ParseShift() node: N.BinOp = N.BinOp(left=left, op=N.BitAnd(), right=right) self._FixLoc(node, start) left = node continue break return left def _ParseShift(self) -> N.expr: start: Token = self._Peek() left: N.expr = self._ParseArith() while True: self._SkipNL() if self._AtOp('<<'): self._Next() right: N.expr = self._ParseArith() node = N.BinOp(left=left, op=N.LShift(), right=right) elif self._AtOp('>>'): self._Next() right = self._ParseArith() node = N.BinOp(left=left, op=N.RShift(), right=right) else: break self._FixLoc(node, start) left = node return left # ================================================================== # 算术 # ================================================================== def _ParseArith(self) -> N.expr: start: Token = self._Peek() left: N.expr = self._ParseTerm() while True: self._SkipNL() if self._AtOp('+'): self._Next() right: N.expr = self._ParseTerm() node = N.BinOp(left=left, op=N.Add(), right=right) elif self._AtOp('-'): self._Next() right = self._ParseTerm() node = N.BinOp(left=left, op=N.Sub(), right=right) else: break self._FixLoc(node, start) left = node return left def _ParseTerm(self) -> N.expr: start: Token = self._Peek() left: N.expr = self._ParseFactor() while True: self._SkipNL() tok: Token = self._Peek() op_type: type | None = None if tok.Type == OP: if tok.Value == '*' and self._AtPtrSuffix(left): # C 风格指针后缀（'*' 一层）；折叠为 CPtr[...] 后继续。 left = self._ConsumePtrSuffix(left) continue if self._InTypeAnnotation and tok.Value == '*': # 注解模式：'*' 后已无操作数（']' / ',' / ')' / ':' / '=' / 行尾 / ';'） # 时它是 C 风格指针后缀标记，交上方判据与 _ParseTypeAnnotation 统一消费； # 但下标实参内的尺寸表达式（CArray[T, A * B]）'*' 后仍有操作数， # 必须按乘法继续——否则下标收尾报 `expected ']'`。 if not (self._SubscriptArgs > 0 and self._AtOperandAfterStarRun()): break op_type = {'*': N.Mult, '/': N.Div, '//': N.FloorDiv, '%': N.Mod, '@': N.MatMult}.get(tok.Value) if op_type is None: break self._Next() right: N.expr = self._ParseFactor() node: N.BinOp = N.BinOp(left=left, op=op_type(), right=right) self._FixLoc(node, start) left = node return left # ================================================================== # 一元 / 幂 / await # ================================================================== def _ParseFactor(self) -> N.expr: tok: Token = self._Peek() if tok.Type == OP and tok.Value in ('+', '-', '~', '*', '&'): self._Next() operand: N.expr = self._ParseFactor() op_type: type = {'+': N.UAdd, '-': N.USub, '~': N.Invert, '*': N.Deref, '&': N.AddrOf}[tok.Value] node: N.UnaryOp = N.UnaryOp(op=op_type(), operand=operand) self._FixLoc(node, tok) return node return self._ParsePower() def _ParsePower(self) -> N.expr: start: Token = self._Peek() base: N.expr = self._ParseAwaitPrimary() if self._AtOp('**'): if self._AtPtrSuffix(base): # C 风格指针后缀（'**' 计 2 层） return self._ConsumePtrSuffix(base) if self._InTypeAnnotation: return base # 注解模式：'**' 是 C 风格指针后缀标记（2 层），不按幂处理 self._Next() exponent: N.expr = self._ParseFactor() # ** 右结合 node: N.BinOp = N.BinOp(left=base, op=N.Pow(), right=exponent) self._FixLoc(node, start) return node return base def _ParseAwaitPrimary(self) -> N.expr: if self._AtName('await'): start: Token = self._Next() value: N.expr = self._ParsePrimary() node: N.Await = N.Await(value=value) self._FixLoc(node, start) return node return self._ParsePrimary() # ================================================================== # 主表达式：原子 + trailer（调用/属性/下标） # ================================================================== def _ParsePrimary(self) -> N.expr: start: Token = self._Peek() node: N.expr = self._ParseAtom() while True: self._SkipNL() tok: Token = self._Peek() if tok.Type == OP and tok.Value in ('*', '**') and self._AtPtrSuffixCall(node): # `T*(x)` / `T**(x)` 指针后缀强转：解析为 cast 调用（与 # `(T | CPtr)(x)` 同形）。在 trailer 循环里消费，故其后的 # `.attr` / `[i]` / `(…)` 仍由本循环继续挂接（如 # `ast.Attribute*(expr_root).value`）。 node = self._ConsumePtrSuffixCall(node, start) elif tok.Type == OP and tok.Value == '.': self._Next() attr_tok: Token = self._ExpectName() attr_node: N.Attribute = N.Attribute(value=node, attr=attr_tok.Value, ctx=N.Load()) self._FixLoc(attr_node, start) node = attr_node elif tok.Type == OP and tok.Value == '(': call_node: N.Call = self._ParseCallTail(node, start) node = call_node elif tok.Type == OP and tok.Value == '[': sub_node: N.Subscript = self._ParseSubscriptTail(node, start) node = sub_node else: break return node def _ParseCallTail(self, func: N.expr, start: Token) -> N.Call: self._ExpectOp('(') args: list[N.expr] = [] keywords: list[N.keyword] = [] self._SkipNL() if not self._AtOp(')'): while True: self._SkipNL() # **kwargs if self._AtOp('**'): self._Next() value: N.expr = self.ParseExpr() kw: N.keyword = N.keyword(arg=None, value=value) self._FixLoc(kw, self._Prev()) keywords.append(kw) else: # *args if self._AtOp('*'): self._Next() value = self.ParseExpr() starred: N.Starred = N.Starred(value=value, ctx=N.Load()) self._FixLoc(starred, self._Prev()) args.append(starred) else: # 关键字参数：NAME '=' if self._Peek().Type == NAME and self._Peek(1).Type == OP and self._Peek(1).Value == '=': name_tok: Token = self._Next() self._Next() # '=' value = self.ParseExpr() kw = N.keyword(arg=name_tok.Value, value=value) self._FixLoc(kw, name_tok) keywords.append(kw) else: # 生成器表达式作为唯一实参：f(x for y in z) # 先解析第一个表达式，若随后是 'for' 则回填为 GeneratorExp if not args and not keywords: gen_start: Token = self._Peek() gen_elt: N.expr = self.ParseExpr() if self._AtName('for'): gen_gens: list[N.comprehension] = self._ParseComprehensionGenerators() gen: N.GeneratorExp = N.GeneratorExp(elt=gen_elt, generators=gen_gens) self._FixLoc(gen, gen_start) args.append(gen) else: args.append(gen_elt) else: args.append(self.ParseExpr()) self._SkipNL() if self._AtOp(','): self._Next() self._SkipNL() if self._AtOp(')'): break continue break self._ExpectOp(')') node: N.Call = N.Call(func=func, args=args, keywords=keywords) self._FixLoc(node, start) return self._DesugarSpecialCall(node, start) def _CallCalleeName(self, func: N.expr) -> str: """被调用者的限定名：Name('X')→'X'；Attribute(c,'X')→'c.X'；否则空串。""" if isinstance(func, N.Name): return func.id if isinstance(func, N.Attribute) and isinstance(func.value, N.Name): return f'{func.value.id}.{func.attr}' return '' def _DesugarSpecialCall(self, node: N.Call, start: Token) -> N.expr: """调用点的特殊 desugar / fail-fast。 - `defined(NAME)`：仅 cif/celif 条件内合法（`self._InCIfCond`）， desugar 为内部标记调用 `c.@CIfdef(NAME)`；其余位置 FailFast。 - `CIf` / `CIfdef` / `CIfndef`（及裸名）：已删除，FailFast 指向新语法。 """ callee: str = self._CallCalleeName(node.func) if callee in _REMOVED_CIF_NAMES: raise SyntaxError( f"'{callee}(...)' has been removed; use the native 'cif' / 'celif' / " f"'celse' statements together with the 'defined(NAME)' predicate " f"instead, at line {start.Line}") if callee == 'defined': if not self._InCIfCond: raise SyntaxError( "'defined(NAME)' is only allowed inside a 'cif' / 'celif' " f"condition, at line {start.Line}") if len(node.args) != 1 or node.keywords: raise SyntaxError( f"'defined' expects exactly one macro name (e.g. defined(MAX_SIZE)), " f"at line {start.Line}") marker: N.Call = N.Call( func=N.Name(id='CIfdef', ctx=N.Load()), args=[node.args[0]], keywords=[]) self._FixLoc(marker, start) return marker return node def _ParseSubscriptTail(self, value: N.expr, start: Token) -> N.Subscript: self._ExpectOp('[') self._SkipNL() slice_node: N.AST = self._ParseSliceList() self._SkipNL() self._ExpectOp(']') node: N.Subscript = N.Subscript(value=value, slice=slice_node, ctx=N.Load()) self._FixLoc(node, start) return node def _ParseSliceList(self) -> N.AST: """解析下标列表：a[1] / a[1:2] / a[1:2:3] / a[1,2] / a[...] / a[::] 下标实参内是**表达式**位置（如 `CArray[T, A * B]` 的尺寸），此处的 '*' 应按乘法解析；同时 `CArray[str*, 4]` 的指针后缀仍要成立——两者由 `_AtOperandAfterStarRun`（'*' 后是否还有操作数）区分，故此处登记已进入实参。 """ self._SubscriptArgs += 1 try: start: Token = self._Peek() elts: list[N.AST] = [self._ParseSliceItem()] while self._AtOp(','): self._Next() self._SkipNL() elts.append(self._ParseSliceItem()) finally: self._SubscriptArgs -= 1 if len(elts) == 1: return elts[0] tup: N.Tuple = N.Tuple(elts=elts, ctx=N.Load()) self._FixLoc(tup, start) return tup def _ParseSliceItem(self) -> N.AST: start: Token = self._Peek() # 含冒号 → Slice if self._AtOp(':'): lower: N.expr | None = None upper: N.expr | None = None step: N.expr | None = None self._Next() # ':' if not self._AtOp(':') and not self._AtOp(','): if not self._AtOp(']'): upper = self.ParseExpr() if self._AtOp(':'): self._Next() if not self._AtOp(']') and not self._AtOp(',') and not self._AtOp(':'): step = self.ParseExpr() node: N.Slice = N.Slice(lower=lower, upper=upper, step=step) self._FixLoc(node, start) return node # 检查 a:b 形式：第一个元素后直接冒号 first: N.expr = self.ParseExpr() if self._AtOp(':'): lower = first self._Next() upper = None step = None if not self._AtOp(':') and not self._AtOp(']') and not self._AtOp(','): upper = self.ParseExpr() if self._AtOp(':'): self._Next() if not self._AtOp(']') and not self._AtOp(','): step = self.ParseExpr() node = N.Slice(lower=lower, upper=upper, step=step) self._FixLoc(node, start) return node # 简单下标（含 Ellipsis） if self._AtOp('...'): # 已在 _ParseExpr 中处理为 Constant(Ellipsis) pass return first # ================================================================== # 原子表达式 # ================================================================== def _ParseNewExpr(self) -> N.expr: """`new [hlife|slife] [stack|heap|memgr(x)] T(args)`（软关键字，与 VPC 同口径）。 描述符顺序无关、各轴最多一次；T 走主表达式（Name/Attribute/Subscript + 调用 trailer），若解析结果本身是 Call（`T(args)`）则拆成 func/args，否则 args=None。 结果恒为 T*（语义在 Handles 侧落地：取存储 + 构造）。 """ start: Token = self._Next() # 吃掉 `new` src: int = NEW_SRC_DEFAULT life: int = NEW_LIFE_DEFAULT memgr: N.expr | None = None while True: tok: Token = self._Peek() if tok.Type != NAME: break v: str = tok.Value if v == 'hlife': self._Next() life = NEW_LIFE_HLIFE continue if v == 'slife': self._Next() life = NEW_LIFE_SLIFE continue if v == 'stack': self._Next() src = NEW_SRC_STACK continue if v == 'heap': self._Next() src = NEW_SRC_HEAP continue if v == 'memgr': self._Next() self._ExpectOp('(') self._SkipNL() memgr = self.ParseExpr() self._SkipNL() self._ExpectOp(')') src = NEW_SRC_MEMGR continue break callee: N.expr = self._ParsePrimary() func: N.expr = callee args: list[N.expr] | None = None if isinstance(callee, N.Call): func = callee.func args = callee.args node: N.New = N.New(Source=src, Life=life, memgr=memgr, func=func, args=args) self._FixLoc(node, start) return node def _ParseAtom(self) -> N.expr: self._SkipNL() tok: Token = self._Peek() if tok.Type == NUMBER: return self._ParseNumber() if tok.Type in (STRING, FSTRING): return self._ParseStringConcat() if tok.Type == NAME: if tok.Value == 'new': nxt: Token = self._Peek(1) if nxt.Type == NAME and nxt.Value not in _NEW_NEXT_RESERVED: return self._ParseNewExpr() if tok.Value in ('True', 'False', 'None'): self._Next() node: N.Constant = N.Constant(value={'True': True, 'False': False, 'None': None}[tok.Value], kind=None) self._FixLoc(node, tok) return node self._Next() node = N.Name(id=tok.Value, ctx=N.Load()) self._FixLoc(node, tok) return node if tok.Type == OP: if tok.Value == '(': return self._ParseParen() if tok.Value == '[': return self._ParseList() if tok.Value == '{': return self._ParseDictSet() if tok.Value == '...': self._Next() node = N.Constant(value=Ellipsis, kind=None) self._FixLoc(node, tok) return node raise SyntaxError( f"unexpected token {tok.Type}({tok.Value!r}) at line {tok.Line}" ) # ------------------------------------------------------------------ # 数字 # ------------------------------------------------------------------ def _ParseNumber(self) -> N.Constant: tok: Token = self._Next() text: str = tok.Value # 显式宽度后缀（_int/_uint/_i/_u）：先从文本里剥掉再解析数值 width: int = getattr(tok, 'IntWidth', 0) unsigned: int = getattr(tok, 'IntUnsigned', 0) if width > 0: cut: int = text.rfind('_') if cut > 0: text = text[:cut] try: if text.lower().endswith('j'): num: object = complex(text[:-1].replace('_', '')) elif text.lower().startswith(('0x', '0o', '0b')): num = int(text.replace('_', ''), 0) elif any(c in text for c in '.eE'): num = float(text.replace('_', '')) else: num = int(text.replace('_', ''), 10) except ValueError: # 无法解析：保留原字符串（理论上不会发生） num = text node: N.Constant = N.Constant(value=num, kind=None) if width > 0: # 显式宽度随节点带走，由 codegen 做「不截断」严格校验 node.int_width = width node.int_unsigned = unsigned self._FixLoc(node, tok) return node # ------------------------------------------------------------------ # 字符串（含相邻拼接与 f-string 混合） # ------------------------------------------------------------------ def _ParseStringConcat(self) -> N.expr: """解析连续字符串字面量并拼接。 规则（与 CPython 一致）： - 全部为普通字符串 → 单个 Constant（值合并） - 含 f-string → JoinedStr（字符串为 Constant，f-string 展开为 values） """ start: Token = self._Peek() parts: list[N.expr] = [] # JoinedStr 的 values 或常量 const_parts: list[object] = [] has_fstring: bool = False while True: # 相邻字符串可跨括号内换行（NL）合并 self._SkipNL() tok: Token = self._Peek() if tok.Type == STRING: self._Next() val: object = self._DecodeString(tok.Value) const_parts.append(val) elif tok.Type == FSTRING: self._Next() joined: N.JoinedStr = self._ParseFStringToken(tok) has_fstring = True if const_parts: parts.append(self._MakeStrConstant(_JoinConstParts(const_parts), start)) const_parts = [] # 相邻 Constant 合并（f-string 字面量 + 后续常量段） if parts and isinstance(parts[-1], N.Constant) and joined.values and isinstance(joined.values[0], N.Constant): merged: str = str(parts[-1].value) + str(joined.values[0].value) merged_const: N.Constant = N.Constant(value=merged, kind=None) merged_const.lineno = parts[-1].lineno merged_const.col_offset = parts[-1].col_offset parts[-1] = merged_const parts.extend(joined.values[1:]) else: parts.extend(joined.values) else: break if not has_fstring: node: N.Constant = N.Constant(value=_JoinConstParts(const_parts), kind=None) self._FixLoc(node, start) return node if const_parts: if parts and isinstance(parts[-1], N.Constant): merged = str(parts[-1].value) + str(_JoinConstParts(const_parts)) merged_const = N.Constant(value=merged, kind=None) merged_const.lineno = parts[-1].lineno merged_const.col_offset = parts[-1].col_offset parts[-1] = merged_const else: parts.append(self._MakeStrConstant(str(_JoinConstParts(const_parts)), start)) node = N.JoinedStr(values=parts) self._FixLoc(node, start) return node def _MakeStrConstant(self, value: str, start: Token) -> N.Constant: node: N.Constant = N.Constant(value=value, kind=None) node.lineno = start.Line node.col_offset = start.Col return node # ------------------------------------------------------------------ # 字符串解码 # ------------------------------------------------------------------ def _DecodeString(self, raw: str) -> str | bytes: """解码字符串字面量原始文本（含前缀与引号）为实际值。 支持 r/b/u/f 前缀组合、单引号/双引号/三引号、常见转义序列。 """ # 提取前缀与引号 i: int = 0 while i < len(raw) and raw[i] in _STRING_PREFIX_CHARS: i += 1 prefix: str = raw[:i] quote: str = raw[i] triple: bool = raw.startswith(quote * 3, i) body_start: int = i + (3 if triple else 1) body_end: int = len(raw) - (3 if triple else 1) body: str = raw[body_start:body_end] is_raw: bool = 'r' in prefix.lower() is_bytes: bool = 'b' in prefix.lower() if is_bytes: # bytes 字面量：逐字节解码（转义为字节值） return self._DecodeBytes(body, is_raw) # str 字面量 return self._DecodeEscapes(body, is_raw) def _DecodeEscapes(self, body: str, is_raw: bool) -> str: """解码字符串转义序列。""" if is_raw: return body out: list[str] = [] i: int = 0 n: int = len(body) while i < n: ch: str = body[i] if ch != '\\': out.append(ch) i += 1 continue i += 1 if i >= n: out.append('\\') break esc: str = body[i] simple: dict[str, str] = { '\\': '\\', "'": "'", '"': '"', 'n': '\n', 't': '\t', 'r': '\r', 'b': '\b', 'f': '\f', 'v': '\v', 'a': '\a', } if esc in simple: out.append(simple[esc]) i += 1 elif esc == 'x': out.append(chr(_HexEscapeVal(body, i + 1, 2))) i += 3 elif esc == 'u': out.append(chr(_HexEscapeVal(body, i + 1, 4))) i += 5 elif esc == 'U': out.append(chr(_HexEscapeVal(body, i + 1, 8))) i += 9 elif esc.isdigit() and '0' <= esc <= '7': oct_digits: str = body[i:i + 3] j: int = 0 while j < 3 and j < len(oct_digits) and '0' <= oct_digits[j] <= '7': j += 1 out.append(chr(int(oct_digits[:j], 8))) i += j else: # 未知转义：保留原样（CPython 保留反斜杠） out.append('\\' + esc) i += 1 return ''.join(out) def _DecodeBytes(self, body: str, is_raw: bool) -> bytes: """解码 bytes 字面量（转义序列按字节处理，支持 r 前缀）。""" if is_raw: return body.encode('latin-1') out: list[int] = [] i: int = 0 n: int = len(body) while i < n: ch: str = body[i] if ch != '\\': try: out.append(ord(body[i])) except Exception: out.append(0) i += 1 continue i += 1 if i >= n: out.append(ord('\\')) break esc: str = body[i] simple: dict[str, int] = { '\\': ord('\\'), "'": ord("'"), '"': ord('"'), 'n': 10, 't': 9, 'r': 13, 'b': 8, 'f': 12, 'v': 11, 'a': 7, } if esc in simple: out.append(simple[esc]) i += 1 elif esc == 'x': out.append(_HexEscapeVal(body, i + 1, 2)) i += 3 elif esc == 'u': out.append(_HexEscapeVal(body, i + 1, 4)) i += 5 elif esc == 'U': out.append(_HexEscapeVal(body, i + 1, 8)) i += 9 elif esc.isdigit() and '0' <= esc <= '7': oct_digits: str = body[i:i + 3] j: int = 0 while j < 3 and j < len(oct_digits) and '0' <= oct_digits[j] <= '7': j += 1 out.append(int(oct_digits[:j], 8)) i += j else: out.append(ord('\\')) out.append(ord(esc)) i += 1 return bytes(out) # ------------------------------------------------------------------ # f-string 解析 # ------------------------------------------------------------------ def _ParseFStringToken(self, tok: Token) -> N.JoinedStr: """解析 FSTRING token 为 JoinedStr（values 为 Constant/FormattedValue）。""" raw: str = tok.Value values: list[N.expr] = self._ParseFStringBody(raw, tok, 0) node: N.JoinedStr = N.JoinedStr(values=values) self._FixLoc(node, tok) return node def _ParseFStringBody(self, raw: str, outer_tok: Token, base_offset: int) -> list[N.expr]: """解析 f-string 文本（含前缀与引号）为 values 列表。 Args: raw: f-string 原始文本 outer_tok: FSTRING token（用于绝对偏移换算） base_offset: 文本在 token Value 中的起始偏移（嵌套用） """ # 提取前缀与引号 i: int = 0 while i < len(raw) and raw[i] in _STRING_PREFIX_CHARS: i += 1 prefix: str = raw[:i] quote: str = raw[i] triple: bool = raw.startswith(quote * 3, i) is_raw: bool = 'r' in prefix.lower() body_start: int = i + (3 if triple else 1) body_end: int = len(raw) - (3 if triple else 1) body: str = raw[body_start:body_end] values: list[N.expr] = [] literal: list[str] = [] j: int = 0 n: int = len(body) rel_base: int = body_start + base_offset # 相对外层的偏移 while j < n: ch: str = body[j] if ch == '{': if j + 1 < n and body[j + 1] == '{': literal.append('{') j += 2 continue # 表达式段开始 if literal: cnode: N.Constant = N.Constant(value=self._DecodeEscapes(''.join(literal), is_raw), kind=None) cnode.lineno, cnode.col_offset = outer_tok.Line, outer_tok.Col values.append(cnode) literal = [] fv: N.FormattedValue = self._ParseFStringValue(body, j, outer_tok, rel_base) values.append(fv) # 更新 j 到表达式段之后 j = self._FStringExprEnd continue elif ch == '}': if j + 1 < n and body[j + 1] == '}': literal.append('}') j += 2 continue # 单独的 } 是语法错误，宽松处理为字面量 literal.append('}') j += 1 continue else: literal.append(ch) j += 1 if literal: cnode = N.Constant(value=self._DecodeEscapes(''.join(literal), is_raw), kind=None) cnode.lineno, cnode.col_offset = outer_tok.Line, outer_tok.Col values.append(cnode) return values _FStringExprEnd: int = 0 # 表达式段结束位置（body 内偏移），供调用方跳转 def _ParseFStringValue(self, body: str, start: int, outer_tok: Token, rel_base: int) -> N.FormattedValue: """解析 f-string 单个 {expr[!conv][:spec]} 段。 返回 FormattedValue；将表达式段结束位置存入 self._FStringExprEnd。 """ j: int = start + 1 # 跳过 '{' n: int = len(body) depth: int = 1 expr_start: int = j expr_end: int = j # 扫描结束字符：'}' 正常闭合；'!'/':' 为转换/格式说明符起点 stop_char: str = '}' # 扫描到匹配的 '}'（考虑括号嵌套与内部字符串）； # 顶层 '!' 或 ':' 表示表达式结束（转换/格式说明符开始） while j < n: ch: str = body[j] if ch == '\\': j += 2 continue if ch in '\'"': # 跳过内部字符串字面量 j = self._SkipInnerLiteral(body, j) continue if ch in '([{': depth += 1 j += 1 continue if ch in ')]}': if depth == 1 and ch == '}': expr_end = j j += 1 break if depth == 1 and ch in ')}]': # 不匹配：语法错误，宽松按结束处理 expr_end = j j += 1 break depth -= 1 j += 1 continue if depth == 1 and (ch == '!' or ch == ':'): # 转换/格式说明符起点：表达式结束（j 停在操作符处） expr_end = j stop_char = ch break j += 1 self._FStringExprEnd = j expr_text: str = body[expr_start:expr_end] # 转换与格式说明符（仅当扫描停在 ! 或 : 时才解析） conversion: int = -1 format_spec: N.expr | None = None k: int = j if stop_char == '!': k += 1 if k < n and body[k] in 'rsa': conversion = ord(body[k]) k += 1 # 转换后可能还有 :format_spec if k < n and body[k] == ':': stop_char = ':' else: # 无格式说明符：跳过 '}' 结束表达式段 if k < n and body[k] == '}': k += 1 self._FStringExprEnd = k if stop_char == ':': k += 1 # 格式说明符可含嵌套 {}（如 {x:{w}}） spec_body: list[str] = [] depth2: int = 1 while k < n: c2: str = body[k] if c2 == '{': depth2 += 1 spec_body.append(c2) k += 1 elif c2 == '}': depth2 -= 1 if depth2 == 0: break spec_body.append(c2) k += 1 else: spec_body.append(c2) k += 1 spec_text: str = ''.join(spec_body) if any(c == '{' for c in spec_text): # 嵌套表达式：构造 JoinedStr inner_fs: str = 'f"' + spec_text.replace('"', '\\"') + '"' inner_tok: Token = Token(FSTRING, inner_fs, outer_tok.Line, outer_tok.Col, outer_tok.EndLine, outer_tok.EndCol) format_spec = self._ParseFStringToken(inner_tok) else: # 静态格式说明符：CPython 包为 JoinedStr(values=[Constant]) cnode: N.Constant = N.Constant(value=spec_text, kind=None) cnode.lineno = outer_tok.Line cnode.col_offset = outer_tok.Col jstr: N.JoinedStr = N.JoinedStr(values=[cnode]) format_spec = jstr # k 现指向 '}'：跳过它作为表达式段结束 if k < n and body[k] == '}': k += 1 self._FStringExprEnd = k # 解析表达式：子 token 流 # abs_base = FSTRING token 源码起始 + body 内偏移，映射回源码绝对偏移 expr_node: N.expr = self._ParseSubExpression(expr_text, outer_tok, outer_tok.Start + expr_start + rel_base) fv: N.FormattedValue = N.FormattedValue(value=expr_node, conversion=conversion, format_spec=format_spec) fv.lineno = outer_tok.Line fv.col_offset = outer_tok.Col fv.end_lineno = outer_tok.EndLine fv.end_col_offset = outer_tok.EndCol return fv def _SkipInnerLiteral(self, body: str, start: int) -> int: """跳过 f-string 表达式内部的字符串字面量，返回结束位置。""" j: int = start # 前缀 while j < len(body) and body[j] in _STRING_PREFIX_CHARS: j += 1 if j >= len(body) or body[j] not in '\'"': return start + 1 quote: str = body[j] triple: bool = body.startswith(quote * 3, j) j += 3 if triple else 1 while j < len(body): ch: str = body[j] if ch == '\\': j += 2 continue if ch == quote: if triple: if body.startswith(quote * 3, j): j += 3 return j j += 1 else: j += 1 return j j += 1 return j def _ParseSubExpression(self, text: str, outer_tok: Token, abs_base: int) -> N.expr: """解析子表达式文本（f-string 内部），并重定位到绝对偏移。""" sub_toks: list[Token] = Tokenizer(text).Tokenize() # 过滤 ENDMARKER 后的 NEWLINE 等 for t in sub_toks: self._Relocate(t, abs_base) # 保存主游标，切换到子 token 流 saved_idx: int = self.Idx saved_tokens: list[Token] = self.Tokens self.Tokens = sub_toks self.Idx = 0 try: expr: N.expr = self.ParseExpr() finally: self.Tokens = saved_tokens self.Idx = saved_idx return expr # ------------------------------------------------------------------ # 括号 / 列表 / 字典 / 集合 / 推导式 # ------------------------------------------------------------------ def _ParseParen(self) -> N.expr: start: Token = self._Next() # '(' self._SkipNL() # 空元组 () if self._AtOp(')'): self._Next() tup: N.Tuple = N.Tuple(elts=[], ctx=N.Load()) self._FixLoc(tup, start) return tup # yield 表达式 if self._AtName('yield'): yield_node: N.expr = self._ParseYield() self._SkipNL() self._ExpectOp(')') return yield_node # 命名表达式 (x := y) if self._Peek().Type == NAME and self._Peek(1).Type == OP and self._Peek(1).Value == ':=': name_tok: Token = self._Next() self._Next() # ':=' value: N.expr = self.ParseExpr() self._SkipNL() self._ExpectOp(')') walrus: N.NamedExpr = N.NamedExpr(target=N.Name(id=name_tok.Value, ctx=N.Store()), value=value) self._FixLoc(walrus, start) return walrus first: N.expr = self.ParseExpr() # Paren-less C cast `(T*)x`: the inner expression ends with a pointer suffix # (_Prev() is that suffix's tail token) and the operand follows ')'. Same # product as `(T*)(x)` / `T*(x)`: a Call whose func is `CPtr[T]`. is_ptr_cast: bool = self._PtrSuffixTail is not None and self._AtOp(')') and self._Prev() is self._PtrSuffixTail self._SkipNL() # 生成器表达式 (x for ...) if self._AtName('for'): generators: list[N.comprehension] = self._ParseComprehensionGenerators() self._SkipNL() self._ExpectOp(')') gen: N.GeneratorExp = N.GeneratorExp(elt=first, generators=generators) self._FixLoc(gen, start) return gen # 元组 (x,) / (x, y) if self._AtOp(','): elts: list[N.expr] = [first] while self._AtOp(','): self._Next() self._SkipNL() if self._AtOp(')'): break elts.append(self.ParseExpr()) self._SkipNL() self._ExpectOp(')') tup = N.Tuple(elts=elts, ctx=N.Load()) self._FixLoc(tup, start) return tup # 普通括号表达式 (x) self._ExpectOp(')') if is_ptr_cast and self._AtCastOperand(): operand: N.expr = self._ParseFactor() cast_call: N.Call = N.Call(func=first, args=[operand], keywords=[]) self._FixLoc(cast_call, start) return cast_call return first def _ParseList(self) -> N.expr: start: Token = self._Next() # '[' self._SkipNL() if self._AtOp(']'): self._Next() lst: N.List = N.List(elts=[], ctx=N.Load()) self._FixLoc(lst, start) return lst first: N.expr = self.ParseExpr() self._SkipNL() # 列表推导式 if self._AtName('for'): generators = self._ParseComprehensionGenerators() self._SkipNL() self._ExpectOp(']') lc: N.ListComp = N.ListComp(elt=first, generators=generators) self._FixLoc(lc, start) return lc # 普通列表 elts = [first] while self._AtOp(','): self._Next() self._SkipNL() if self._AtOp(']'): break elts.append(self.ParseExpr()) self._SkipNL() self._ExpectOp(']') lst = N.List(elts=elts, ctx=N.Load()) self._FixLoc(lst, start) return lst def _ParseDictSet(self) -> N.expr: start: Token = self._Next() # '{' self._SkipNL() if self._AtOp('}'): self._Next() dct: N.Dict = N.Dict(keys=[], values=[]) self._FixLoc(dct, start) return dct # ** 展开：{**d} keys: list[N.expr | None] = [] values: list[N.expr] = [] while True: self._SkipNL() if self._AtOp('**'): self._Next() val: N.expr = self.ParseExpr() keys.append(None) values.append(val) else: key: N.expr = self.ParseExpr() self._SkipNL() if self._AtOp(':'): self._Next() self._SkipNL() value: N.expr = self.ParseExpr() keys.append(key) values.append(value) else: # 集合元素 elts: list[N.expr] = [key] while self._AtOp(','): self._Next() self._SkipNL() if self._AtOp('}'): break elts.append(self.ParseExpr()) self._SkipNL() # 集合推导式？ self._SkipNL() if self._AtName('for'): generators = self._ParseComprehensionGenerators() self._SkipNL() self._ExpectOp('}') sc: N.SetComp = N.SetComp(elt=elts[0], generators=generators) self._FixLoc(sc, start) return sc self._ExpectOp('}') st: N.Set = N.Set(elts=elts) self._FixLoc(st, start) return st self._SkipNL() if self._AtOp(','): self._Next() self._SkipNL() if self._AtOp('}'): break continue break # 字典推导式？ self._SkipNL() if self._AtName('for'): generators = self._ParseComprehensionGenerators() self._SkipNL() self._ExpectOp('}') dc: N.DictComp = N.DictComp(key=keys[0] or N.Constant(value=None), value=values[0], generators=generators) self._FixLoc(dc, start) return dc self._ExpectOp('}') dct = N.Dict(keys=keys, values=values) self._FixLoc(dct, start) return dct def _ParseComprehensionGenerators(self) -> list[N.comprehension]: """解析推导式生成器子句：for target in iter [if cond]* [async for ...]""" generators: list[N.comprehension] = [] while True: is_async: bool = False if self._AtName('async') and self._Peek(1).Type == NAME and self._Peek(1).Value == 'for': self._Next() is_async = True if not self._AtName('for'): break start: Token = self._Next() # 'for' target: N.expr = self._ParseComprehensionTarget() self._ExpectName('in') # iter 用 disjunction 级别解析，避免吞掉 'if' 条件 iterable: N.expr = self._ParseDisjunction() ifs: list[N.expr] = [] while self._AtName('if'): self._Next() ifs.append(self._ParseDisjunction()) comp: N.comprehension = N.comprehension(target=target, iter=iterable, ifs=ifs, is_async=int(is_async)) self._FixLoc(comp, start) generators.append(comp) if self._AtName('for') or (self._AtName('async') and self._Peek(1).Type == NAME and self._Peek(1).Value == 'for'): continue break return generators def _ParseComprehensionTarget(self) -> N.expr: """解析推导式 for 目标（支持裸元组 a, b 解包）。""" start: Token = self._Peek() first: N.expr = self._ParseTarget() if self._AtOp(','): elts: list[N.expr] = [first] while self._AtOp(','): self._Next() self._SkipNL() elts.append(self._ParseTarget()) tup: N.Tuple = N.Tuple(elts=elts, ctx=N.Store()) self._FixLoc(tup, start) return tup return first def _ParseTarget(self) -> N.expr: """解析赋值目标（Store 上下文），支持元组/列表解包。""" start: Token = self._Peek() if self._AtOp('('): self._Next() self._SkipNL() elts: list[N.expr] = [] if not self._AtOp(')'): while True: elts.append(self._ParseTarget()) self._SkipNL() if self._AtOp(','): self._Next() self._SkipNL() if self._AtOp(')'): break continue break self._ExpectOp(')') tup: N.Tuple = N.Tuple(elts=elts, ctx=N.Store()) self._FixLoc(tup, start) return tup if self._AtOp('['): self._Next() self._SkipNL() elts = [] if not self._AtOp(']'): while True: elts.append(self._ParseTarget()) self._SkipNL() if self._AtOp(','): self._Next() self._SkipNL() if self._AtOp(']'): break continue break self._ExpectOp(']') lst: N.List = N.List(elts=elts, ctx=N.Store()) self._FixLoc(lst, start) return lst if self._AtOp('*'): self._Next() value: N.expr = self._ParseTarget() starred: N.Starred = N.Starred(value=value, ctx=N.Store()) self._FixLoc(starred, start) return starred # 名称/属性/下标（Store） tok: Token = self._Peek() if tok.Type == NAME: self._Next() node: N.Name = N.Name(id=tok.Value, ctx=N.Store()) self._FixLoc(node, tok) return node # 属性/下标目标：解析 Load 后改 Store node = self._ParsePrimary() self._SetStoreContext(node) return node def _SetStoreContext(self, node: N.AST) -> None: """将 Load 上下文节点递归改为 Store。""" if isinstance(node, N.Attribute): node.ctx = N.Store() elif isinstance(node, N.Subscript): node.ctx = N.Store() elif isinstance(node, N.Name): node.ctx = N.Store() elif isinstance(node, (N.Tuple, N.List)): node.ctx = N.Store() for elt in node.elts: self._SetStoreContext(elt) elif isinstance(node, N.Starred): node.ctx = N.Store() self._SetStoreContext(node.value) # ------------------------------------------------------------------ # lambda / yield # ------------------------------------------------------------------ def _ParseLambda(self) -> N.Lambda: start: Token = self._Next() # 'lambda' args: N.arguments if self._AtOp(':'): # 无参 lambda args = N.arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]) else: args = self._ParseArguments(skip_open=True, allow_lambda=True) self._ExpectOp(':') body: N.expr = self.ParseExpr() node: N.Lambda = N.Lambda(args=args, body=body) self._FixLoc(node, start) return node def _ParseYield(self) -> N.expr: start: Token = self._Next() # 'yield' if self._AtName('from'): self._Next() value: N.expr = self.ParseExpr() node: N.YieldFrom = N.YieldFrom(value=value) self._FixLoc(node, start) return node if not self._AtOp(')'): value = self.ParseExprList() node: N.Yield = N.Yield(value=value) else: node = N.Yield(value=None) self._FixLoc(node, start) return node # ================================================================== # 参数列表（lambda 与函数定义共用） # ================================================================== def _ParseArguments(self, skip_open: bool = False, allow_lambda: bool = False) -> N.arguments: """解析参数列表。 支持 posonly (/)、*args、kwonly、**kwargs、默认值、注解。 defaults 与 posonlyargs+args 尾部对齐（CPython 语义）。 Args: skip_open: True 时假定 '(' 已消费（lambda 场景无括号） """ if not skip_open: self._ExpectOp('(') self._SkipNL() posonlyargs: list[N.arg] = [] args: list[N.arg] = [] kwonlyargs: list[N.arg] = [] kw_defaults: list[N.expr | None] = [] defaults: list[N.expr] = [] vararg: N.arg | None = None kwarg: N.arg | None = None seen_star: bool = False # 按解析顺序收集 (arg, default) 用于尾部对齐 ordered: list[tuple[N.arg, N.expr | None]] = [] if not self._AtOp(')'): while True: self._SkipNL() if self._AtOp(')'): break if self._AtOp('*'): self._Next() if self._AtOp(','): # 裸 *：分隔 kwonly seen_star = True self._Next() if self._AtOp(')') or self._AtOp(':'): break continue vararg = self._ParseArg(allow_annotation=not allow_lambda) seen_star = True elif self._AtOp('**'): self._Next() kwarg = self._ParseArg(allow_annotation=not allow_lambda) else: a: N.arg = self._ParseArg(allow_annotation=not allow_lambda) default: N.expr | None = None if self._AtOp('='): self._Next() default = self.ParseExpr() if seen_star: kwonlyargs.append(a) kw_defaults.append(default) else: ordered.append((a, default)) self._SkipNL() if self._AtOp(','): self._Next() if self._AtOp(')'): break continue break # 切分 posonly 与普通 args：CPython 无显式 '/' 信息时全为普通 args # 注意：本实现未显式解析 '/' 分隔，posonlyargs 保持空（TPC 源码不使用） for a, d in ordered: args.append(a) defaults.append(d) # defaults 尾部对齐：去掉开头的 None，且仅保留末尾连续默认值 # CPython 语义：defaults 与 args 尾部对齐，若中间有缺省则语法错误。 # 宽松处理：取最后一个 None 之后的全部默认值。 last_none: int = -1 for i, d in enumerate(defaults): if d is None: last_none = i defaults = [d for d in defaults[last_none + 1:] if d is not None] # 将带默认值的 arg 之前的参数清掉默认值列表长度 = 默认参数个数 self._SkipNL() if not skip_open: self._ExpectOp(')') node: N.arguments = N.arguments( posonlyargs=posonlyargs, args=args, vararg=vararg, kwonlyargs=kwonlyargs, kw_defaults=kw_defaults, kwarg=kwarg, defaults=defaults, ) return node def _ParseArg(self, allow_annotation: bool) -> N.arg: start: Token = self._ExpectName() annotation: N.expr | None = None if allow_annotation and self._AtOp(':'): self._Next() annotation = self._ParseTypeAnnotation() a: N.arg = N.arg(arg=start.Value, annotation=annotation, type_comment=None) self._FixLoc(a, start) return a # 注解起始的修饰词：只在该位置生效，写入类型节点的 Markers 字段。 _TYPE_MARKER_WORDS: frozenset[str] = frozenset({ 'const', 'volatile', 'static', 'export', 'extern', 'inline', 'state', }) def _ParseTypeAnnotation(self) -> N.expr: """解析类型注解，支持修饰前缀与 C 风格指针后缀（* / **）。 修饰前缀（任意顺序、可组合）写入类型节点的 AST 字段，不再展开为 `T1 | T2` 联合： const volatile int → Name('int').Markers = ('const', 'volatile') x: __attribute__((...)) T → T.Attributes = (...) BigEndian uint16 → ByteOrder = 'big' Bit(3) int / Bit[3] int → BitWidth = 3 注解模式开启后，'_ParseTerm'/'_ParsePower' 不再把 '*' / '**' 当 乘法/幂运算消费，而由本方法统一循环消费并折叠为 Subscript： int* → CPtr[int] （i32*） int** → CPtr[CPtr[int]] （i32**） int****** → 6 层嵌套 （i32******） 无法解析的后缀形态**响亮失败**（SyntaxError，含 行:列 + 源码片段 + 形态 + 正确写法示例），绝不静默丢弃：注解末尾若仍遗留未被 `_AtPtrSuffix` 接受的 '*'/'**'（如 `a: int * 2`），判据拒绝消费即终止编译 （静默退化会产出错误的指针层数）。 用户源码不再接受 `CPtr[...]` / `CPtr` 指针写法（TypeError，提示改写成 `T*`）；内部 AST 仍以 `Subscript(Name('CPtr'), T)` 表示指针，`T*` 的 脱糖产物不受此限制。 值引用参数注解（仅形参位有效）：前导 `&` 为 ref 模式、`&temp` 为 temp 模式， 合成 Subscript 形态 `CRef[T]` / `CTempRef[T]`（CRef/CTempRef 非真实类， 仅按名字识别；语义侧见 HandlesFunctions / FuncGen）。表达式位的 `&x` 不受影响。 """ markers: list[str] = [] attributes: list[str] = [] byte_order: str = '' bit_width: int = 0 while True: tok: Token = self._Peek() if tok.Type != NAME: break word: str = tok.Value if word in self._TYPE_MARKER_WORDS: self._Next() markers.append(word) continue if word == '__attribute__' and self._Peek(1).Type == OP \ and self._Peek(1).Value == '(': self._Next() attributes.extend(self._ParseAttributeSpecifier()) continue if word in ('BigEndian', 'LittleEndian') and self._AtTypeStart(1): self._Next() byte_order = 'big' if word == 'BigEndian' else 'little' continue if word == 'Bit' and self._Peek(1).Type == OP \ and self._Peek(1).Value in ('(', '['): self._Next() bit_width = self._ParseBitWidth() continue break ref_mode: str | None = None ref_tok: Token = self._Peek() if self._AtOp('&'): self._Next() if self._AtName('temp'): self._Next() ref_mode = 'temp' else: ref_mode = 'ref' # `&` / `&temp` 之后必须紧跟类型；缺失即响亮失败（禁止静默丢弃） end_tok: Token = self._Peek() if end_tok.Type in (NEWLINE, NL, ENDMARKER) or ( end_tok.Type == OP and end_tok.Value in (')', ',', ':', '=', ']', '}', ';')): snippet: str = '' try: snippet = (self.Source.splitlines()[end_tok.Line - 1] or '').strip() except Exception: snippet = '' raise SyntaxError( f"[vast] 值引用参数注解缺少类型：第 {end_tok.Line} 行第 {end_tok.Col} 列的 " f"`&` 后必须紧跟类型 T（ref：`&T`；temp：`&temp T`）。" f"源码片段 {snippet!r}。正确示例：`def f(p: &int)` / `def f(p: &temp void)`。") saved_flag: bool = self._InTypeAnnotation self._InTypeAnnotation = True try: expr: N.expr = self.ParseExpr() finally: self._InTypeAnnotation = saved_flag if expr is None: if ref_mode is not None: raise SyntaxError( "[vast] 值引用参数注解缺少类型：`&` 后必须紧跟类型 T" "（ref：`&T`；temp：`&temp T`）。") if markers or attributes or byte_order or bit_width: raise SyntaxError( "[vast] 类型注解的修饰词后缺少类型（如 `x: const int`）。") return None SourceCPtr: N.AST | None = _FindSourceCPtr(expr) if SourceCPtr is not None: Line: int = getattr(SourceCPtr, 'lineno', 0) or 0 Col: int = (getattr(SourceCPtr, 'col_offset', 0) or 0) + 1 raise TypeError( f"[vast] 源码不支持 CPtr 指针写法（第 {Line} 行第 {Col} 列）：" f"指针请写 T*（如 int*）") if isinstance(expr, N.BinOp) and isinstance(expr.op, N.BitOr): def _is_none_literal(n: N.AST) -> bool: if isinstance(n, N.Constant): return getattr(n, 'value', ...) is None return isinstance(n, N.Name) and getattr(n, 'id', '') == 'None' _left_none: bool = _is_none_literal(expr.left) _right_none: bool = _is_none_literal(expr.right) if _left_none != _right_none: # `T | None` / `None | T` 等价于 T expr = expr.right if _left_none else expr.left else: Line: int = getattr(expr, 'lineno', 0) or 0 raise TypeError( f"[vast] 不支持类型联合 'T1 | T2'（第 {Line} 行）；" f"类型注解只能是单一类型：指针请写 T*（如 int*），修饰符请用关键字") if self._AtPtrSuffix(expr): expr = self._ConsumePtrSuffix(expr) tok: Token = self._Peek() if tok.Type == OP and tok.Value in ('*', '**'): snippet: str = '' try: snippet = (self.Source.splitlines()[tok.Line - 1] or '').strip() except Exception: snippet = '' raise SyntaxError( f"[vast] 注解位置的指针后缀无法解析：第 {tok.Line} 行第 {tok.Col} 列的 " f"{tok.Value!r} 前面不是可加后缀的类型（Name / Attribute / Subscript），" f"或其后不是后缀的合法结束位置（注解结尾，或 ']' / ',' / ')' / ':' / '='）。" f"源码片段 {snippet!r}。" f"期望形态：`T*`（1 层）/ `T**`（2 层），T 可为裸名 / X / mod.Type / " f"rawptr[T]。" f"正确示例：`a: int*` / `a: rawptr*` / `a: X**` / " f"`a: rawptr[ast.AST*]` / `def f(p: int*) -> int**`。") if ref_mode is not None: expr = self._MakeRefMarker(expr, ref_mode, ref_tok) self._ApplyTypePrefix(expr, markers, attributes, byte_order, bit_width) return expr def _AtTypeStart(self, offset: int) -> bool: """offset 处的 token 是否可作为类型起点（用于识别字节序前缀）。""" tok: Token = self._Peek(offset) if tok.Type in (NAME, NUMBER, STRING): return True return tok.Type == OP and tok.Value in ('[', '(', '&') @staticmethod def _SplitTopLevel(text: str, sep: str = ',') -> list[str]: """按顶层分隔符切分文本（忽略括号/方括号/花括号内与字符串内的分隔符）。""" parts: list[str] = [] buf: list[str] = [] depth: int = 0 quote: str = '' escaped: bool = False for ch in text: if quote: buf.append(ch) if escaped: escaped = False elif ch == '\\': escaped = True elif ch == quote: quote = '' continue if ch in ('"', "'"): quote = ch buf.append(ch) continue if ch in '([{': depth += 1 elif ch in ')]}': depth -= 1 if ch == sep and depth == 0: parts.append(''.join(buf)) buf = [] continue buf.append(ch) parts.append(''.join(buf)) return parts @staticmethod def _StripOuterParens(text: str) -> str: """去掉一对完整包裹文本的圆括号（`((a), (b))` 不误删）。""" s: str = text.strip() if not (s.startswith('(') and s.endswith(')')): return s depth: int = 0 quote: str = '' escaped: bool = False for i, ch in enumerate(s): if quote: if escaped: escaped = False elif ch == '\\': escaped = True elif ch == quote: quote = '' continue if ch in ('"', "'"): quote = ch elif ch == '(': depth += 1 elif ch == ')': depth -= 1 if depth == 0 and i != len(s) - 1: return s if depth != 0: return s return s[1:-1].strip() def _ParseAttributeSpecifier(self) -> tuple[str, ...]: """解析 `__attribute__(...)` 的括号内容（调用名已消费），返回属性文本元组。""" open_tok: Token = self._ExpectOp('(') depth: int = 1 inner: str = '' while True: tok: Token = self._Next() if tok.Type == ENDMARKER: raise SyntaxError( "[vast] __attribute__ 描述符缺少右括号：" "期望 `__attribute__((aligned(16)))` 形态。") if tok.Type == OP and tok.Value == '(': depth += 1 elif tok.Type == OP and tok.Value == ')': depth -= 1 if depth == 0: inner = self.Source[open_tok.End:tok.Start].strip() break inner = self._StripOuterParens(inner) return tuple(part.strip() for part in self._SplitTopLevel(inner) if part.strip()) def _ParseBitWidth(self) -> int: """解析 `Bit(N)` / `Bit[N]` 的位宽（调用名已消费）。""" open_tok: Token = self._Next() close: str = ')' if open_tok.Value == '(' else ']' self._SkipNL() width_tok: Token = self._Next() if width_tok.Type != NUMBER: raise SyntaxError( f"[vast] Bit 描述符缺少整数位宽：第 {width_tok.Line} 行" f"第 {width_tok.Col} 列，期望 `Bit(N)` / `Bit[N]`。") try: width: int = int(width_tok.Value.replace('_', ''), 0) except ValueError as exc: raise SyntaxError( f"[vast] Bit 描述符的位宽必须是整数字面量，得到 {width_tok.Value!r}。" ) from exc self._SkipNL() self._ExpectOp(close) return width @staticmethod def _ApplyTypePrefix(node: N.expr, markers: list[str], attributes: list[str], byte_order: str, bit_width: int) -> None: """把注解前缀写入类型节点的 Markers / Attributes / ByteOrder / BitWidth。""" if not hasattr(type(node), 'Markers'): return if markers: existing: tuple[str, ...] = tuple(getattr(node, 'Markers', ()) or ()) node.Markers = tuple(markers) + tuple(m for m in existing if m not in markers) if attributes: existing_attrs: tuple[str, ...] = tuple(getattr(node, 'Attributes', ()) or ()) node.Attributes = existing_attrs + tuple(attributes) if byte_order: node.ByteOrder = byte_order if bit_width: node.BitWidth = bit_width def _MakeRefMarker(self, base: N.expr, mode: str, tok: Token) -> N.expr: """构造值引用包装：`CRef[T]`（ref）/ `CTempRef[T]`（temp）。 CRef / CTempRef 非真实类，仅作为注解 AST 内的名字标记（语义侧按名字识别）。 """ marker: N.expr = N.Name( id='CRef' if mode == 'ref' else 'CTempRef', ctx=N.Load()) node: N.Subscript = N.Subscript(value=marker, slice=base, ctx=N.Load()) self._FixLoc(node, tok) return node # 类型结束后缀：'_AtPtrSuffix' 的判据表 # - 全局（任意表达式上下文）：'*'/'**' 串后紧跟这些 operator 时，乘法/幂本就缺右操作数 # （即原语法错误），故按指针后缀处理不会改变任何合法表达式的语义； # - 注解模式额外允许语句结束位置（NEWLINE/NL/ENDMARKER/';'）。 _PTR_SUFFIX_END_OPS: tuple[str, ...] = (']', ',', ')', ':', '=') _PTR_SUFFIX_STMT_END_OPS: tuple[str, ...] = (';',) # 指针后缀「强转调用」后缀：'*'/'**' 串后**紧贴**（无空白）'(' 时，写成 C 风格 # 强转调用 `T*(x)`。'(' **不**放进 `_PTR_SUFFIX_END_OPS`：`a * (b)` 是合法乘法 # （'(' 正是乘法的右操作数起点），无条件把 '(' 计入会误伤乘法；故另立一条 # 「紧贴」判据（见 `_AtPtrSuffixCall`），只认 `T*(` 这种**无空白**的强转写法。 _PTR_SUFFIX_CALL_OP: str = '(' def _AtPtrSuffix(self, base: N.expr) -> bool: """判定当前 '*'/'**' 串是否为 C 风格指针后缀（作用于 base）。 判据（两条同时成立）: 1. base 是"类型原子"形态（Name / Attribute / Subscript）—— 即 ``ast.AST``、 ``CArray[X, N]``、``rawptr[X]`` 之类； 2. 该 '*'/'**' 串（可跨括号内换行 NL）之后紧跟"无右操作数"位置： ``]`` / ``,`` / ``)`` / ``:`` / ``=``（全局），或注解模式下的 NEWLINE / NL / ENDMARKER / ``;``。 因此合法表达式 ``x[a * b]`` / ``f(x * y)`` / ``x[1 * 2]`` / ``a ** b`` （'*' 之后是操作数）永不被判为指针后缀。 """ if not isinstance(base, (N.Name, N.Attribute, N.Subscript)): return False j: int = 0 stars: int = 0 while True: tok: Token = self._Peek(j) if tok.Type == OP and tok.Value in ('*', '**'): stars += 1 if tok.Value == '*' else 2 j += 1 elif tok.Type == NL: j += 1 else: break if stars == 0: return False nxt: Token = self._Peek(j) if nxt.Type == OP and nxt.Value in self._PTR_SUFFIX_END_OPS: return True if self._InTypeAnnotation: if nxt.Type in (NEWLINE, NL, ENDMARKER): return True if nxt.Type == OP and nxt.Value in self._PTR_SUFFIX_STMT_END_OPS: return True return False def _ConsumePtrSuffix(self, expr: N.expr) -> N.expr: """消费连续的 C 风格指针后缀（'*' 一层 / '**' 两层），折叠为 CPtr[...] 嵌套。""" while True: tok: Token = self._Peek() if tok.Type == OP and tok.Value == '*': self._Next() self._PtrSuffixTail = tok expr = self._MakePtrType(expr, tok) elif tok.Type == OP and tok.Value == '**': self._Next() self._PtrSuffixTail = tok expr = self._MakePtrType(expr, tok) expr = self._MakePtrType(expr, tok) else: break return expr def _AtCastOperand(self) -> bool: """Whether the current token may start the operand of a paren-less cast `(T*)x`. '(' is deliberately absent: `(T*)(x)` is already handled by the call trailer, so that path's AST stays untouched. NL is not skipped either: a line break after ')' starts a new statement, not a cast operand. """ tok: Token = self._Peek() if tok.Type in (NAME, NUMBER, STRING): return True return tok.Type == OP and tok.Value in ('[', '{', '*', '&', '-', '+', '~') def _AtOperandAfterStarRun(self) -> bool: """当前 '*'/'**' 串之后是否还有操作数起点（可跨括号内换行 NL）。 与 `_AtPtrSuffix` 互补：后者判「'*' 后是无右操作数位置」（']' / ',' / ')' / ':' / '=' / 行尾 / ';' ⇒ 指针后缀），本方法判其反面（'*' 后是 NAME / NUMBER / STRING / FSTRING 或 '(' / '[' / '{' / 一元前缀 ⇒ 乘法）。用于下标实参内区分 `CArray[T, A * B]`（乘法）与 `CArray[T*, 4]`（指针后缀）。 """ j: int = 0 while True: tok: Token = self._Peek(j) if tok.Type == OP and tok.Value in ('*', '**'): j += 1 elif tok.Type == NL: j += 1 else: break nxt: Token = self._Peek(j) if nxt.Type in (NAME, NUMBER, STRING, FSTRING): return True return nxt.Type == OP and nxt.Value in ('(', '[', '{', '*', '&', '-', '+', '~', '...') def _AtPtrSuffixCall(self, base: N.expr) -> bool: """判定当前 '*'/'**' 串是否为 `T*(x)` 形式的指针后缀**强转调用**。 判据（三条同时成立）: 1. base 是"类型原子"形态（Name / Attribute / Subscript）—— 与 `_AtPtrSuffix` 的判据 1 同源（`ast.Call` / `list[ast.AST*]` 之类）； 2. base 与 '*'/'**' 串**紧贴**（源码字符相邻，无空白/换行）； 3. 该 '*'/'**' 串（串内**逐个紧贴**）之后**紧贴**一个 '('。 即整段写作 **无空白**的 `T*(x)`（C 风格强转写法）。为什么 '(' 不在 `_PTR_SUFFIX_END_OPS` 里：`a * (b)` 是合法乘法（'(' 可作乘法右操作数的 起点），把 '(' 无条件计入表会误伤乘法。故此处叠加"前驱是类型原子 + 两侧 紧贴"两条判据：`a * (b + c)` / `f(x) * (y)` / `n * (m - 1)` / `(i + 1) * (j)`（'*' 前后有空白，且前驱多为非类型原子）仍按乘法解析。 """ if not isinstance(base, (N.Name, N.Attribute, N.Subscript)): return False first: Token = self._Peek(0) prev: Token = self._Prev() if prev.Type in (NEWLINE, NL) or prev.End != first.Start: return False # base 与 '*'/'**' 之间有空白/换行：非 C 风格强转写法 j: int = 0 stars: int = 0 last_end: int = -1 while True: tok: Token = self._Peek(j) if tok.Type == OP and tok.Value in ('*', '**'): if last_end != -1 and tok.Start != last_end: return False # 星串内部出现空白：非紧邻，不判为强转 last_end = tok.End stars += 1 j += 1 else: break if stars == 0: return False nxt: Token = self._Peek(j) return nxt.Type == OP and nxt.Value == self._PTR_SUFFIX_CALL_OP \ and nxt.Start == last_end def _ConsumePtrSuffixCall(self, base: N.expr, start: Token) -> N.expr: """把已认定为 `T*(...)` 的指针后缀强转解析为**调用**。 产物为 `ast.Call`（func = `CPtr[T]` 指针包装，args/keywords = 括号实参）， 不新造第二套 cast 表示。 """ func: N.expr = self._ConsumePtrSuffix(base) return self._ParseCallTail(func, start) def _MakePtrType(self, base: N.expr, tok: Token) -> N.expr: """构造一层指针包装：`CPtr[base]`。""" cptr: N.expr = N.Name(id='CPtr', ctx=N.Load()) node: N.Subscript = N.Subscript(value=cptr, slice=base, ctx=N.Load()) self._FixLoc(node, tok) return node