"""vast.exprparser - Viper/Python 表达式语法分析器 实现与 CPython 3.12 语法规则一致的递归下降表达式解析: - 完整运算符优先级(lambda/条件/or/and/not/比较/位运算/移位/算术/一元/幂/await) - 原子表达式:标识符/数字/字符串/常量/f-string/元组/列表/字典/集合 - 推导式(list/set/dict/generator)、lambda、命名表达式(walrus) - 调用(位置实参/关键字实参/*args/**kwargs)、属性、下标、切片 - f-string(JoinedStr/FormattedValue,含嵌套表达式与格式说明符) 本模块为 Mixin 设计:``Parser`` 继承 ``ExprParserMixin`` 获得表达式能力。 所有产出节点与 CPython ``ast`` 结构完全一致,并填充 lineno/col_offset/ end_lineno/end_col_offset 位置属性。 """ from __future__ import annotations import bisect from lib.vast import Nodes as N from lib.vast.Tokenizer import ( Token, NAME, NUMBER, STRING, FSTRING, OP, NEWLINE, NL, ENDMARKER, Tokenizer, _STRING_PREFIX_CHARS, ) def _JoinConstParts(parts: list[object]) -> str | bytes: """拼接连续字符串字面量值(str 或 bytes,与 CPython 语义一致)。""" if not parts: return '' if isinstance(parts[0], bytes): buf: list[bytes] = [] for p in parts: if isinstance(p, bytes): buf.append(p) else: buf.append(str(p).encode('utf-8')) return b''.join(buf) return ''.join(str(p) for p in parts) class TokenStream: """token 流管理基类:统一提供游标操作、位置标注工具。""" def __init__(self, tokens: list[Token], source: str, line_starts: list[int]) -> None: self.Tokens: list[Token] = tokens self.Source: str = source self.LineStarts: list[int] = line_starts self.Idx: int = 0 # ------------------------------------------------------------------ # 游标操作 # ------------------------------------------------------------------ def _Peek(self, offset: int = 0) -> Token: i: int = self.Idx + offset if i >= len(self.Tokens): return self.Tokens[-1] return self.Tokens[i] def _Next(self) -> Token: tok: Token = self.Tokens[self.Idx] if self.Idx < len(self.Tokens) - 1: self.Idx += 1 return tok def _Prev(self) -> Token: return self.Tokens[max(0, self.Idx - 1)] def _At(self, tok_type: str, value: str | None = None) -> bool: tok: Token = self._Peek() if tok.Type != tok_type: return False if value is not None and tok.Value != value: return False return True def _AtOp(self, value: str) -> bool: return self._At(OP, value) def _AtName(self, value: str) -> bool: return self._At(NAME, value) def _SkipNL(self) -> None: """跳过括号内换行 token(NL)。""" while self._Peek().Type == NL: self._Next() def _SkipNEWLINE(self) -> None: """跳过逻辑换行 token(NEWLINE)与空行/注释行(NL)。""" while self._Peek().Type in (NEWLINE, NL): self._Next() def _ExpectOp(self, value: str) -> Token: tok: Token = self._Next() if tok.Type != OP or tok.Value != value: raise SyntaxError( f"expected '{value}', got {tok.Type}({tok.Value!r}) " f"at line {tok.Line}" ) return tok def _ExpectName(self, value: str | None = None) -> Token: tok: Token = self._Next() if tok.Type != NAME or (value is not None and tok.Value != value): want: str = f" {value!r}" if value else '' raise SyntaxError( f"expected name{want}, " f"got {tok.Type}({tok.Value!r}) at line {tok.Line}" ) return tok def _ExpectNEWLINE(self) -> None: tok: Token = self._Next() if tok.Type != NEWLINE: raise SyntaxError( f"expected newline, got {tok.Type}({tok.Value!r}) at line {tok.Line}" ) # ------------------------------------------------------------------ # 位置标注 # ------------------------------------------------------------------ def _FixLoc(self, node: N.AST, start_tok: Token, end_tok: Token | None = None) -> N.AST: """为节点设置 4 个位置属性(end 缺省取前一个 token)。""" if end_tok is None: end_tok = self._Prev() if 'lineno' in getattr(node, '_attributes', ()): node.lineno = start_tok.Line node.col_offset = start_tok.Col node.end_lineno = end_tok.EndLine node.end_col_offset = end_tok.EndCol return node def _PosFromOffset(self, offset: int) -> tuple[int, int]: """字符偏移 → (行号, UTF-8 字节列偏移)。""" idx: int = bisect.bisect_right(self.LineStarts, offset) - 1 line: int = idx + 1 col: int = len(self.Source[self.LineStarts[idx]:offset].encode('utf-8')) return line, col def _Relocate(self, tok: Token, base: int) -> None: """将子表达式 token 按绝对偏移重定位(f-string 内部表达式用)。""" tok.Start = base + tok.Start tok.End = base + tok.End line, col = self._PosFromOffset(tok.Start) e_line, e_col = self._PosFromOffset(tok.End) tok.Line, tok.Col, tok.EndLine, tok.EndCol = line, col, e_line, e_col class ExprParserMixin(TokenStream): """表达式解析器(Mixin,供 Parser 使用)。""" # ================================================================== # 表达式顶层:lambda | 条件表达式 # ================================================================== def ParseExpr(self) -> N.expr: """完整表达式:lambda 或条件表达式(含 walrus 前置)。""" if self._AtName('lambda'): return self._ParseLambda() return self._ParseConditional() def ParseExprList(self) -> N.expr: """testlist_star_expr:表达式或逗号分隔元组(赋值右值/return 值)。""" start: Token = self._Peek() first: N.expr = self.ParseExpr() if self._AtOp(','): elts: list[N.expr] = [first] while self._AtOp(','): self._Next() self._SkipNL() elts.append(self.ParseExpr()) tup: N.Tuple = N.Tuple(elts=elts, ctx=N.Load()) self._FixLoc(tup, start) return tup return first def _ParseConditional(self) -> N.expr: """条件表达式:disjunction ('if' disjunction 'else' expression)?""" start: Token = self._Peek() body: N.expr = self._ParseDisjunction() if self._AtName('if'): self._Next() test: N.expr = self._ParseDisjunction() self._ExpectName('else') orelse: N.expr = self.ParseExpr() node: N.IfExp = N.IfExp(test=test, body=body, orelse=orelse) self._FixLoc(node, start) return node return body # ================================================================== # 布尔/比较 # ================================================================== def _ParseDisjunction(self) -> N.expr: start: Token = self._Peek() values: list[N.expr] = [self._ParseConjunction()] while True: self._SkipNL() if self._AtName('or'): self._Next() values.append(self._ParseConjunction()) continue break if len(values) == 1: return values[0] node: N.BoolOp = N.BoolOp(op=N.Or(), values=values) self._FixLoc(node, start) return node def _ParseConjunction(self) -> N.expr: start: Token = self._Peek() values: list[N.expr] = [self._ParseInversion()] while True: self._SkipNL() if self._AtName('and'): self._Next() values.append(self._ParseInversion()) continue break if len(values) == 1: return values[0] node: N.BoolOp = N.BoolOp(op=N.And(), values=values) self._FixLoc(node, start) return node def _ParseInversion(self) -> N.expr: self._SkipNL() if self._AtName('not'): start: Token = self._Next() operand: N.expr = self._ParseInversion() node: N.UnaryOp = N.UnaryOp(op=N.Not(), operand=operand) self._FixLoc(node, start) return node return self._ParseComparison() # 比较操作符映射 _COMPARE_OPS: dict[str, type] = { '<': N.Lt, '>': N.Gt, '<=': N.LtE, '>=': N.GtE, '==': N.Eq, '!=': N.NotEq, 'is': N.Is, 'is not': N.IsNot, 'in': N.In, 'not in': N.NotIn, } def _ParseComparison(self) -> N.expr: start: Token = self._Peek() left: N.expr = self._ParseBitOr() ops: list[N.AST] = [] comparators: list[N.expr] = [] while True: self._SkipNL() tok: Token = self._Peek() op_type: type | None = None if tok.Type == OP and tok.Value in ('<', '>', '<=', '>=', '==', '!='): op_type = self._COMPARE_OPS[tok.Value] self._Next() elif tok.Type == NAME and tok.Value == 'is': self._Next() if self._AtName('not'): self._Next() op_type = N.IsNot else: op_type = N.Is elif tok.Type == NAME and tok.Value == 'in': self._Next() op_type = N.In elif tok.Type == NAME and tok.Value == 'not' and self._Peek(1).Type == NAME and self._Peek(1).Value == 'in': self._Next() self._Next() op_type = N.NotIn else: break ops.append(op_type()) comparators.append(self._ParseBitOr()) if not ops: return left node: N.Compare = N.Compare(left=left, ops=ops, comparators=comparators) self._FixLoc(node, start) return node # ================================================================== # 位运算 # ================================================================== def _ParseBitOr(self) -> N.expr: start: Token = self._Peek() left: N.expr = self._ParseBitXor() while True: self._SkipNL() if self._AtOp('|'): self._Next() right: N.expr = self._ParseBitXor() node: N.BinOp = N.BinOp(left=left, op=N.BitOr(), right=right) self._FixLoc(node, start) left = node continue break return left def _ParseBitXor(self) -> N.expr: start: Token = self._Peek() left: N.expr = self._ParseBitAnd() while True: self._SkipNL() if self._AtOp('^'): self._Next() right: N.expr = self._ParseBitAnd() node: N.BinOp = N.BinOp(left=left, op=N.BitXor(), right=right) self._FixLoc(node, start) left = node continue break return left def _ParseBitAnd(self) -> N.expr: start: Token = self._Peek() left: N.expr = self._ParseShift() while True: self._SkipNL() if self._AtOp('&'): self._Next() right: N.expr = self._ParseShift() node: N.BinOp = N.BinOp(left=left, op=N.BitAnd(), right=right) self._FixLoc(node, start) left = node continue break return left def _ParseShift(self) -> N.expr: start: Token = self._Peek() left: N.expr = self._ParseArith() while True: self._SkipNL() if self._AtOp('<<'): self._Next() right: N.expr = self._ParseArith() node = N.BinOp(left=left, op=N.LShift(), right=right) elif self._AtOp('>>'): self._Next() right = self._ParseArith() node = N.BinOp(left=left, op=N.RShift(), right=right) else: break self._FixLoc(node, start) left = node return left # ================================================================== # 算术 # ================================================================== def _ParseArith(self) -> N.expr: start: Token = self._Peek() left: N.expr = self._ParseTerm() while True: self._SkipNL() if self._AtOp('+'): self._Next() right: N.expr = self._ParseTerm() node = N.BinOp(left=left, op=N.Add(), right=right) elif self._AtOp('-'): self._Next() right = self._ParseTerm() node = N.BinOp(left=left, op=N.Sub(), right=right) else: break self._FixLoc(node, start) left = node return left def _ParseTerm(self) -> N.expr: start: Token = self._Peek() left: N.expr = self._ParseFactor() while True: self._SkipNL() tok: Token = self._Peek() op_type: type | None = None if tok.Type == OP: op_type = {'*': N.Mult, '/': N.Div, '//': N.FloorDiv, '%': N.Mod, '@': N.MatMult}.get(tok.Value) if op_type is None: break self._Next() right: N.expr = self._ParseFactor() node: N.BinOp = N.BinOp(left=left, op=op_type(), right=right) self._FixLoc(node, start) left = node return left # ================================================================== # 一元 / 幂 / await # ================================================================== def _ParseFactor(self) -> N.expr: tok: Token = self._Peek() if tok.Type == OP and tok.Value in ('+', '-', '~'): self._Next() operand: N.expr = self._ParseFactor() op_type: type = {'+': N.UAdd, '-': N.USub, '~': N.Invert}[tok.Value] node: N.UnaryOp = N.UnaryOp(op=op_type(), operand=operand) self._FixLoc(node, tok) return node return self._ParsePower() def _ParsePower(self) -> N.expr: start: Token = self._Peek() base: N.expr = self._ParseAwaitPrimary() if self._AtOp('**'): self._Next() exponent: N.expr = self._ParseFactor() # ** 右结合 node: N.BinOp = N.BinOp(left=base, op=N.Pow(), right=exponent) self._FixLoc(node, start) return node return base def _ParseAwaitPrimary(self) -> N.expr: if self._AtName('await'): start: Token = self._Next() value: N.expr = self._ParsePrimary() node: N.Await = N.Await(value=value) self._FixLoc(node, start) return node return self._ParsePrimary() # ================================================================== # 主表达式:原子 + trailer(调用/属性/下标) # ================================================================== def _ParsePrimary(self) -> N.expr: start: Token = self._Peek() node: N.expr = self._ParseAtom() while True: self._SkipNL() tok: Token = self._Peek() if tok.Type == OP and tok.Value == '.': self._Next() attr_tok: Token = self._ExpectName() attr_node: N.Attribute = N.Attribute(value=node, attr=attr_tok.Value, ctx=N.Load()) self._FixLoc(attr_node, start) node = attr_node elif tok.Type == OP and tok.Value == '(': call_node: N.Call = self._ParseCallTail(node, start) node = call_node elif tok.Type == OP and tok.Value == '[': sub_node: N.Subscript = self._ParseSubscriptTail(node, start) node = sub_node else: break return node def _ParseCallTail(self, func: N.expr, start: Token) -> N.Call: self._ExpectOp('(') args: list[N.expr] = [] keywords: list[N.keyword] = [] self._SkipNL() if not self._AtOp(')'): while True: self._SkipNL() # **kwargs if self._AtOp('**'): self._Next() value: N.expr = self.ParseExpr() kw: N.keyword = N.keyword(arg=None, value=value) self._FixLoc(kw, self._Prev()) keywords.append(kw) else: # *args if self._AtOp('*'): self._Next() value = self.ParseExpr() starred: N.Starred = N.Starred(value=value, ctx=N.Load()) self._FixLoc(starred, self._Prev()) args.append(starred) else: # 关键字参数:NAME '=' if self._Peek().Type == NAME and self._Peek(1).Type == OP and self._Peek(1).Value == '=': name_tok: Token = self._Next() self._Next() # '=' value = self.ParseExpr() kw = N.keyword(arg=name_tok.Value, value=value) self._FixLoc(kw, name_tok) keywords.append(kw) else: # 生成器表达式作为唯一实参:f(x for y in z) # 先解析第一个表达式,若随后是 'for' 则回填为 GeneratorExp if not args and not keywords: gen_start: Token = self._Peek() gen_elt: N.expr = self.ParseExpr() if self._AtName('for'): gen_gens: list[N.comprehension] = self._ParseComprehensionGenerators() gen: N.GeneratorExp = N.GeneratorExp(elt=gen_elt, generators=gen_gens) self._FixLoc(gen, gen_start) args.append(gen) else: args.append(gen_elt) else: args.append(self.ParseExpr()) self._SkipNL() if self._AtOp(','): self._Next() self._SkipNL() if self._AtOp(')'): break continue break self._ExpectOp(')') node: N.Call = N.Call(func=func, args=args, keywords=keywords) self._FixLoc(node, start) return node def _ParseSubscriptTail(self, value: N.expr, start: Token) -> N.Subscript: self._ExpectOp('[') self._SkipNL() slice_node: N.AST = self._ParseSliceList() self._SkipNL() self._ExpectOp(']') node: N.Subscript = N.Subscript(value=value, slice=slice_node, ctx=N.Load()) self._FixLoc(node, start) return node def _ParseSliceList(self) -> N.AST: """解析下标列表:a[1] / a[1:2] / a[1:2:3] / a[1,2] / a[...] / a[::]""" start: Token = self._Peek() elts: list[N.AST] = [self._ParseSliceItem()] while self._AtOp(','): self._Next() self._SkipNL() elts.append(self._ParseSliceItem()) if len(elts) == 1: return elts[0] tup: N.Tuple = N.Tuple(elts=elts, ctx=N.Load()) self._FixLoc(tup, start) return tup def _ParseSliceItem(self) -> N.AST: start: Token = self._Peek() # 含冒号 → Slice if self._AtOp(':'): lower: N.expr | None = None upper: N.expr | None = None step: N.expr | None = None self._Next() # ':' if not self._AtOp(':') and not self._AtOp(','): if not self._AtOp(']'): upper = self.ParseExpr() if self._AtOp(':'): self._Next() if not self._AtOp(']') and not self._AtOp(',') and not self._AtOp(':'): step = self.ParseExpr() node: N.Slice = N.Slice(lower=lower, upper=upper, step=step) self._FixLoc(node, start) return node # 检查 a:b 形式:第一个元素后直接冒号 first: N.expr = self.ParseExpr() if self._AtOp(':'): lower = first self._Next() upper = None step = None if not self._AtOp(':') and not self._AtOp(']') and not self._AtOp(','): upper = self.ParseExpr() if self._AtOp(':'): self._Next() if not self._AtOp(']') and not self._AtOp(','): step = self.ParseExpr() node = N.Slice(lower=lower, upper=upper, step=step) self._FixLoc(node, start) return node # 简单下标(含 Ellipsis) if self._AtOp('...'): # 已在 _ParseExpr 中处理为 Constant(Ellipsis) pass return first # ================================================================== # 原子表达式 # ================================================================== def _ParseAtom(self) -> N.expr: self._SkipNL() tok: Token = self._Peek() if tok.Type == NUMBER: return self._ParseNumber() if tok.Type in (STRING, FSTRING): return self._ParseStringConcat() if tok.Type == NAME: if tok.Value in ('True', 'False', 'None'): self._Next() node: N.Constant = N.Constant(value={'True': True, 'False': False, 'None': None}[tok.Value], kind=None) self._FixLoc(node, tok) return node self._Next() node = N.Name(id=tok.Value, ctx=N.Load()) self._FixLoc(node, tok) return node if tok.Type == OP: if tok.Value == '(': return self._ParseParen() if tok.Value == '[': return self._ParseList() if tok.Value == '{': return self._ParseDictSet() if tok.Value == '...': self._Next() node = N.Constant(value=Ellipsis, kind=None) self._FixLoc(node, tok) return node raise SyntaxError( f"unexpected token {tok.Type}({tok.Value!r}) at line {tok.Line}" ) # ------------------------------------------------------------------ # 数字 # ------------------------------------------------------------------ def _ParseNumber(self) -> N.Constant: tok: Token = self._Next() text: str = tok.Value try: if text.lower().endswith('j'): num: object = complex(text[:-1].replace('_', '')) elif text.lower().startswith(('0x', '0o', '0b')): num = int(text.replace('_', ''), 0) elif any(c in text for c in '.eE'): num = float(text.replace('_', '')) else: num = int(text.replace('_', ''), 10) except ValueError: # 无法解析:保留原字符串(理论上不会发生) num = text node: N.Constant = N.Constant(value=num, kind=None) self._FixLoc(node, tok) return node # ------------------------------------------------------------------ # 字符串(含相邻拼接与 f-string 混合) # ------------------------------------------------------------------ def _ParseStringConcat(self) -> N.expr: """解析连续字符串字面量并拼接。 规则(与 CPython 一致): - 全部为普通字符串 → 单个 Constant(值合并) - 含 f-string → JoinedStr(字符串为 Constant,f-string 展开为 values) """ start: Token = self._Peek() parts: list[N.expr] = [] # JoinedStr 的 values 或常量 const_parts: list[object] = [] has_fstring: bool = False while True: # 相邻字符串可跨括号内换行(NL)合并 self._SkipNL() tok: Token = self._Peek() if tok.Type == STRING: self._Next() val: object = self._DecodeString(tok.Value) const_parts.append(val) elif tok.Type == FSTRING: self._Next() joined: N.JoinedStr = self._ParseFStringToken(tok) has_fstring = True if const_parts: parts.append(self._MakeStrConstant(_JoinConstParts(const_parts), start)) const_parts = [] # 相邻 Constant 合并(f-string 字面量 + 后续常量段) if parts and isinstance(parts[-1], N.Constant) and joined.values and isinstance(joined.values[0], N.Constant): merged: str = str(parts[-1].value) + str(joined.values[0].value) merged_const: N.Constant = N.Constant(value=merged, kind=None) merged_const.lineno = parts[-1].lineno merged_const.col_offset = parts[-1].col_offset parts[-1] = merged_const parts.extend(joined.values[1:]) else: parts.extend(joined.values) else: break if not has_fstring: node: N.Constant = N.Constant(value=_JoinConstParts(const_parts), kind=None) self._FixLoc(node, start) return node if const_parts: if parts and isinstance(parts[-1], N.Constant): merged = str(parts[-1].value) + str(_JoinConstParts(const_parts)) merged_const = N.Constant(value=merged, kind=None) merged_const.lineno = parts[-1].lineno merged_const.col_offset = parts[-1].col_offset parts[-1] = merged_const else: parts.append(self._MakeStrConstant(str(_JoinConstParts(const_parts)), start)) node = N.JoinedStr(values=parts) self._FixLoc(node, start) return node def _MakeStrConstant(self, value: str, start: Token) -> N.Constant: node: N.Constant = N.Constant(value=value, kind=None) node.lineno = start.Line node.col_offset = start.Col return node # ------------------------------------------------------------------ # 字符串解码 # ------------------------------------------------------------------ def _DecodeString(self, raw: str) -> str | bytes: """解码字符串字面量原始文本(含前缀与引号)为实际值。 支持 r/b/u/f 前缀组合、单引号/双引号/三引号、常见转义序列。 """ # 提取前缀与引号 i: int = 0 while i < len(raw) and raw[i] in _STRING_PREFIX_CHARS: i += 1 prefix: str = raw[:i] quote: str = raw[i] triple: bool = raw.startswith(quote * 3, i) body_start: int = i + (3 if triple else 1) body_end: int = len(raw) - (3 if triple else 1) body: str = raw[body_start:body_end] is_raw: bool = 'r' in prefix.lower() is_bytes: bool = 'b' in prefix.lower() if is_bytes: # bytes 字面量:逐字节解码(转义为字节值) return self._DecodeBytes(body, is_raw) # str 字面量 return self._DecodeEscapes(body, is_raw) def _DecodeEscapes(self, body: str, is_raw: bool) -> str: """解码字符串转义序列。""" if is_raw: return body out: list[str] = [] i: int = 0 n: int = len(body) while i < n: ch: str = body[i] if ch != '\\': out.append(ch) i += 1 continue i += 1 if i >= n: out.append('\\') break esc: str = body[i] simple: dict[str, str] = { '\\': '\\', "'": "'", '"': '"', 'n': '\n', 't': '\t', 'r': '\r', 'b': '\b', 'f': '\f', 'v': '\v', 'a': '\a', } if esc in simple: out.append(simple[esc]) i += 1 elif esc == 'x': hex_digits: str = body[i + 1:i + 3] out.append(chr(int(hex_digits, 16))) i += 3 elif esc == 'u': hex_digits = body[i + 1:i + 5] out.append(chr(int(hex_digits, 16))) i += 5 elif esc == 'U': hex_digits = body[i + 1:i + 9] out.append(chr(int(hex_digits, 16))) i += 9 elif esc.isdigit() and '0' <= esc <= '7': oct_digits: str = body[i:i + 3] j: int = 0 while j < 3 and j < len(oct_digits) and '0' <= oct_digits[j] <= '7': j += 1 out.append(chr(int(oct_digits[:j], 8))) i += j else: # 未知转义:保留原样(CPython 保留反斜杠) out.append('\\' + esc) i += 1 return ''.join(out) def _DecodeBytes(self, body: str, is_raw: bool) -> bytes: """解码 bytes 字面量(转义序列按字节处理,支持 r 前缀)。""" if is_raw: return body.encode('latin-1') out: list[int] = [] i: int = 0 n: int = len(body) while i < n: ch: str = body[i] if ch != '\\': try: out.append(ord(body[i])) except Exception: out.append(0) i += 1 continue i += 1 if i >= n: out.append(ord('\\')) break esc: str = body[i] simple: dict[str, int] = { '\\': ord('\\'), "'": ord("'"), '"': ord('"'), 'n': 10, 't': 9, 'r': 13, 'b': 8, 'f': 12, 'v': 11, 'a': 7, } if esc in simple: out.append(simple[esc]) i += 1 elif esc == 'x': out.append(int(body[i + 1:i + 3], 16)) i += 3 elif esc == 'u': out.append(int(body[i + 1:i + 5], 16)) i += 5 elif esc == 'U': out.append(int(body[i + 1:i + 9], 16)) i += 9 elif esc.isdigit() and '0' <= esc <= '7': oct_digits: str = body[i:i + 3] j: int = 0 while j < 3 and j < len(oct_digits) and '0' <= oct_digits[j] <= '7': j += 1 out.append(int(oct_digits[:j], 8)) i += j else: out.append(ord('\\')) out.append(ord(esc)) i += 1 return bytes(out) # ------------------------------------------------------------------ # f-string 解析 # ------------------------------------------------------------------ def _ParseFStringToken(self, tok: Token) -> N.JoinedStr: """解析 FSTRING token 为 JoinedStr(values 为 Constant/FormattedValue)。""" raw: str = tok.Value values: list[N.expr] = self._ParseFStringBody(raw, tok, 0) node: N.JoinedStr = N.JoinedStr(values=values) self._FixLoc(node, tok) return node def _ParseFStringBody(self, raw: str, outer_tok: Token, base_offset: int) -> list[N.expr]: """解析 f-string 文本(含前缀与引号)为 values 列表。 Args: raw: f-string 原始文本 outer_tok: FSTRING token(用于绝对偏移换算) base_offset: 文本在 token Value 中的起始偏移(嵌套用) """ # 提取前缀与引号 i: int = 0 while i < len(raw) and raw[i] in _STRING_PREFIX_CHARS: i += 1 prefix: str = raw[:i] quote: str = raw[i] triple: bool = raw.startswith(quote * 3, i) is_raw: bool = 'r' in prefix.lower() body_start: int = i + (3 if triple else 1) body_end: int = len(raw) - (3 if triple else 1) body: str = raw[body_start:body_end] values: list[N.expr] = [] literal: list[str] = [] j: int = 0 n: int = len(body) rel_base: int = body_start + base_offset # 相对外层的偏移 while j < n: ch: str = body[j] if ch == '{': if j + 1 < n and body[j + 1] == '{': literal.append('{') j += 2 continue # 表达式段开始 if literal: cnode: N.Constant = N.Constant(value=self._DecodeEscapes(''.join(literal), is_raw), kind=None) cnode.lineno, cnode.col_offset = outer_tok.Line, outer_tok.Col values.append(cnode) literal = [] fv: N.FormattedValue = self._ParseFStringValue(body, j, outer_tok, rel_base) values.append(fv) # 更新 j 到表达式段之后 j = self._FStringExprEnd continue elif ch == '}': if j + 1 < n and body[j + 1] == '}': literal.append('}') j += 2 continue # 单独的 } 是语法错误,宽松处理为字面量 literal.append('}') j += 1 continue else: literal.append(ch) j += 1 if literal: cnode = N.Constant(value=self._DecodeEscapes(''.join(literal), is_raw), kind=None) cnode.lineno, cnode.col_offset = outer_tok.Line, outer_tok.Col values.append(cnode) return values _FStringExprEnd: int = 0 # 表达式段结束位置(body 内偏移),供调用方跳转 def _ParseFStringValue(self, body: str, start: int, outer_tok: Token, rel_base: int) -> N.FormattedValue: """解析 f-string 单个 {expr[!conv][:spec]} 段。 返回 FormattedValue;将表达式段结束位置存入 self._FStringExprEnd。 """ j: int = start + 1 # 跳过 '{' n: int = len(body) depth: int = 1 expr_start: int = j expr_end: int = j # 扫描结束字符:'}' 正常闭合;'!'/':' 为转换/格式说明符起点 stop_char: str = '}' # 扫描到匹配的 '}'(考虑括号嵌套与内部字符串); # 顶层 '!' 或 ':' 表示表达式结束(转换/格式说明符开始) while j < n: ch: str = body[j] if ch == '\\': j += 2 continue if ch in '\'"': # 跳过内部字符串字面量 j = self._SkipInnerLiteral(body, j) continue if ch in '([{': depth += 1 j += 1 continue if ch in ')]}': if depth == 1 and ch == '}': expr_end = j j += 1 break if depth == 1 and ch in ')}]': # 不匹配:语法错误,宽松按结束处理 expr_end = j j += 1 break depth -= 1 j += 1 continue if depth == 1 and (ch == '!' or ch == ':'): # 转换/格式说明符起点:表达式结束(j 停在操作符处) expr_end = j stop_char = ch break j += 1 self._FStringExprEnd = j expr_text: str = body[expr_start:expr_end] # 转换与格式说明符(仅当扫描停在 ! 或 : 时才解析) conversion: int = -1 format_spec: N.expr | None = None k: int = j if stop_char == '!': k += 1 if k < n and body[k] in 'rsa': conversion = ord(body[k]) k += 1 # 转换后可能还有 :format_spec if k < n and body[k] == ':': stop_char = ':' else: # 无格式说明符:跳过 '}' 结束表达式段 if k < n and body[k] == '}': k += 1 self._FStringExprEnd = k if stop_char == ':': k += 1 # 格式说明符可含嵌套 {}(如 {x:{w}}) spec_body: list[str] = [] depth2: int = 1 while k < n: c2: str = body[k] if c2 == '{': depth2 += 1 spec_body.append(c2) k += 1 elif c2 == '}': depth2 -= 1 if depth2 == 0: break spec_body.append(c2) k += 1 else: spec_body.append(c2) k += 1 spec_text: str = ''.join(spec_body) if any(c == '{' for c in spec_text): # 嵌套表达式:构造 JoinedStr inner_fs: str = 'f"' + spec_text.replace('"', '\\"') + '"' inner_tok: Token = Token(FSTRING, inner_fs, outer_tok.Line, outer_tok.Col, outer_tok.EndLine, outer_tok.EndCol) format_spec = self._ParseFStringToken(inner_tok) else: # 静态格式说明符:CPython 包为 JoinedStr(values=[Constant]) cnode: N.Constant = N.Constant(value=spec_text, kind=None) cnode.lineno = outer_tok.Line cnode.col_offset = outer_tok.Col jstr: N.JoinedStr = N.JoinedStr(values=[cnode]) format_spec = jstr # k 现指向 '}':跳过它作为表达式段结束 if k < n and body[k] == '}': k += 1 self._FStringExprEnd = k # 解析表达式:子 token 流 expr_node: N.expr = self._ParseSubExpression(expr_text, outer_tok, expr_start + rel_base) fv: N.FormattedValue = N.FormattedValue(value=expr_node, conversion=conversion, format_spec=format_spec) fv.lineno = outer_tok.Line fv.col_offset = outer_tok.Col fv.end_lineno = outer_tok.EndLine fv.end_col_offset = outer_tok.EndCol return fv def _SkipInnerLiteral(self, body: str, start: int) -> int: """跳过 f-string 表达式内部的字符串字面量,返回结束位置。""" j: int = start # 前缀 while j < len(body) and body[j] in _STRING_PREFIX_CHARS: j += 1 if j >= len(body) or body[j] not in '\'"': return start + 1 quote: str = body[j] triple: bool = body.startswith(quote * 3, j) j += 3 if triple else 1 while j < len(body): ch: str = body[j] if ch == '\\': j += 2 continue if ch == quote: if triple: if body.startswith(quote * 3, j): j += 3 return j j += 1 else: j += 1 return j j += 1 return j def _ParseSubExpression(self, text: str, outer_tok: Token, abs_base: int) -> N.expr: """解析子表达式文本(f-string 内部),并重定位到绝对偏移。""" sub_toks: list[Token] = Tokenizer(text).Tokenize() # 过滤 ENDMARKER 后的 NEWLINE 等 for t in sub_toks: self._Relocate(t, abs_base) # 保存主游标,切换到子 token 流 saved_idx: int = self.Idx saved_tokens: list[Token] = self.Tokens self.Tokens = sub_toks self.Idx = 0 try: expr: N.expr = self.ParseExpr() finally: self.Tokens = saved_tokens self.Idx = saved_idx return expr # ------------------------------------------------------------------ # 括号 / 列表 / 字典 / 集合 / 推导式 # ------------------------------------------------------------------ def _ParseParen(self) -> N.expr: start: Token = self._Next() # '(' self._SkipNL() # 空元组 () if self._AtOp(')'): self._Next() tup: N.Tuple = N.Tuple(elts=[], ctx=N.Load()) self._FixLoc(tup, start) return tup # yield 表达式 if self._AtName('yield'): yield_node: N.expr = self._ParseYield() self._SkipNL() self._ExpectOp(')') return yield_node # 命名表达式 (x := y) if self._Peek().Type == NAME and self._Peek(1).Type == OP and self._Peek(1).Value == ':=': name_tok: Token = self._Next() self._Next() # ':=' value: N.expr = self.ParseExpr() self._SkipNL() self._ExpectOp(')') walrus: N.NamedExpr = N.NamedExpr(target=N.Name(id=name_tok.Value, ctx=N.Store()), value=value) self._FixLoc(walrus, start) return walrus first: N.expr = self.ParseExpr() self._SkipNL() # 生成器表达式 (x for ...) if self._AtName('for'): generators: list[N.comprehension] = self._ParseComprehensionGenerators() self._SkipNL() self._ExpectOp(')') gen: N.GeneratorExp = N.GeneratorExp(elt=first, generators=generators) self._FixLoc(gen, start) return gen # 元组 (x,) / (x, y) if self._AtOp(','): elts: list[N.expr] = [first] while self._AtOp(','): self._Next() self._SkipNL() if self._AtOp(')'): break elts.append(self.ParseExpr()) self._SkipNL() self._ExpectOp(')') tup = N.Tuple(elts=elts, ctx=N.Load()) self._FixLoc(tup, start) return tup # 普通括号表达式 (x) self._ExpectOp(')') return first def _ParseList(self) -> N.expr: start: Token = self._Next() # '[' self._SkipNL() if self._AtOp(']'): self._Next() lst: N.List = N.List(elts=[], ctx=N.Load()) self._FixLoc(lst, start) return lst first: N.expr = self.ParseExpr() self._SkipNL() # 列表推导式 if self._AtName('for'): generators = self._ParseComprehensionGenerators() self._SkipNL() self._ExpectOp(']') lc: N.ListComp = N.ListComp(elt=first, generators=generators) self._FixLoc(lc, start) return lc # 普通列表 elts = [first] while self._AtOp(','): self._Next() self._SkipNL() if self._AtOp(']'): break elts.append(self.ParseExpr()) self._SkipNL() self._ExpectOp(']') lst = N.List(elts=elts, ctx=N.Load()) self._FixLoc(lst, start) return lst def _ParseDictSet(self) -> N.expr: start: Token = self._Next() # '{' self._SkipNL() if self._AtOp('}'): self._Next() dct: N.Dict = N.Dict(keys=[], values=[]) self._FixLoc(dct, start) return dct # ** 展开:{**d} keys: list[N.expr | None] = [] values: list[N.expr] = [] while True: self._SkipNL() if self._AtOp('**'): self._Next() val: N.expr = self.ParseExpr() keys.append(None) values.append(val) else: key: N.expr = self.ParseExpr() self._SkipNL() if self._AtOp(':'): self._Next() self._SkipNL() value: N.expr = self.ParseExpr() keys.append(key) values.append(value) else: # 集合元素 elts: list[N.expr] = [key] while self._AtOp(','): self._Next() self._SkipNL() if self._AtOp('}'): break elts.append(self.ParseExpr()) self._SkipNL() # 集合推导式? self._SkipNL() if self._AtName('for'): generators = self._ParseComprehensionGenerators() self._SkipNL() self._ExpectOp('}') sc: N.SetComp = N.SetComp(elt=elts[0], generators=generators) self._FixLoc(sc, start) return sc self._ExpectOp('}') st: N.Set = N.Set(elts=elts) self._FixLoc(st, start) return st self._SkipNL() if self._AtOp(','): self._Next() self._SkipNL() if self._AtOp('}'): break continue break # 字典推导式? self._SkipNL() if self._AtName('for'): generators = self._ParseComprehensionGenerators() self._SkipNL() self._ExpectOp('}') dc: N.DictComp = N.DictComp(key=keys[0] or N.Constant(value=None), value=values[0], generators=generators) self._FixLoc(dc, start) return dc self._ExpectOp('}') dct = N.Dict(keys=keys, values=values) self._FixLoc(dct, start) return dct def _ParseComprehensionGenerators(self) -> list[N.comprehension]: """解析推导式生成器子句:for target in iter [if cond]* [async for ...]""" generators: list[N.comprehension] = [] while True: is_async: bool = False if self._AtName('async') and self._Peek(1).Type == NAME and self._Peek(1).Value == 'for': self._Next() is_async = True if not self._AtName('for'): break start: Token = self._Next() # 'for' target: N.expr = self._ParseComprehensionTarget() self._ExpectName('in') # iter 用 disjunction 级别解析,避免吞掉 'if' 条件 iterable: N.expr = self._ParseDisjunction() ifs: list[N.expr] = [] while self._AtName('if'): self._Next() ifs.append(self._ParseDisjunction()) comp: N.comprehension = N.comprehension(target=target, iter=iterable, ifs=ifs, is_async=int(is_async)) self._FixLoc(comp, start) generators.append(comp) if self._AtName('for') or (self._AtName('async') and self._Peek(1).Type == NAME and self._Peek(1).Value == 'for'): continue break return generators def _ParseComprehensionTarget(self) -> N.expr: """解析推导式 for 目标(支持裸元组 a, b 解包)。""" start: Token = self._Peek() first: N.expr = self._ParseTarget() if self._AtOp(','): elts: list[N.expr] = [first] while self._AtOp(','): self._Next() self._SkipNL() elts.append(self._ParseTarget()) tup: N.Tuple = N.Tuple(elts=elts, ctx=N.Store()) self._FixLoc(tup, start) return tup return first def _ParseTarget(self) -> N.expr: """解析赋值目标(Store 上下文),支持元组/列表解包。""" start: Token = self._Peek() if self._AtOp('('): self._Next() self._SkipNL() elts: list[N.expr] = [] if not self._AtOp(')'): while True: elts.append(self._ParseTarget()) self._SkipNL() if self._AtOp(','): self._Next() self._SkipNL() if self._AtOp(')'): break continue break self._ExpectOp(')') tup: N.Tuple = N.Tuple(elts=elts, ctx=N.Store()) self._FixLoc(tup, start) return tup if self._AtOp('['): self._Next() self._SkipNL() elts = [] if not self._AtOp(']'): while True: elts.append(self._ParseTarget()) self._SkipNL() if self._AtOp(','): self._Next() self._SkipNL() if self._AtOp(']'): break continue break self._ExpectOp(']') lst: N.List = N.List(elts=elts, ctx=N.Store()) self._FixLoc(lst, start) return lst if self._AtOp('*'): self._Next() value: N.expr = self._ParseTarget() starred: N.Starred = N.Starred(value=value, ctx=N.Store()) self._FixLoc(starred, start) return starred # 名称/属性/下标(Store) tok: Token = self._Peek() if tok.Type == NAME: self._Next() node: N.Name = N.Name(id=tok.Value, ctx=N.Store()) self._FixLoc(node, tok) return node # 属性/下标目标:解析 Load 后改 Store node = self._ParsePrimary() self._SetStoreContext(node) return node def _SetStoreContext(self, node: N.AST) -> None: """将 Load 上下文节点递归改为 Store。""" if isinstance(node, N.Attribute): node.ctx = N.Store() elif isinstance(node, N.Subscript): node.ctx = N.Store() elif isinstance(node, N.Name): node.ctx = N.Store() elif isinstance(node, (N.Tuple, N.List)): node.ctx = N.Store() for elt in node.elts: self._SetStoreContext(elt) elif isinstance(node, N.Starred): node.ctx = N.Store() self._SetStoreContext(node.value) # ------------------------------------------------------------------ # lambda / yield # ------------------------------------------------------------------ def _ParseLambda(self) -> N.Lambda: start: Token = self._Next() # 'lambda' args: N.arguments if self._AtOp(':'): # 无参 lambda args = N.arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]) else: args = self._ParseArguments(skip_open=True, allow_lambda=True) self._ExpectOp(':') body: N.expr = self.ParseExpr() node: N.Lambda = N.Lambda(args=args, body=body) self._FixLoc(node, start) return node def _ParseYield(self) -> N.expr: start: Token = self._Next() # 'yield' if self._AtName('from'): self._Next() value: N.expr = self.ParseExpr() node: N.YieldFrom = N.YieldFrom(value=value) self._FixLoc(node, start) return node if not self._AtOp(')'): value = self.ParseExprList() node: N.Yield = N.Yield(value=value) else: node = N.Yield(value=None) self._FixLoc(node, start) return node # ================================================================== # 参数列表(lambda 与函数定义共用) # ================================================================== def _ParseArguments(self, skip_open: bool = False, allow_lambda: bool = False) -> N.arguments: """解析参数列表。 支持 posonly (/)、*args、kwonly、**kwargs、默认值、注解。 defaults 与 posonlyargs+args 尾部对齐(CPython 语义)。 Args: skip_open: True 时假定 '(' 已消费(lambda 场景无括号) """ if not skip_open: self._ExpectOp('(') self._SkipNL() posonlyargs: list[N.arg] = [] args: list[N.arg] = [] kwonlyargs: list[N.arg] = [] kw_defaults: list[N.expr | None] = [] defaults: list[N.expr] = [] vararg: N.arg | None = None kwarg: N.arg | None = None seen_star: bool = False # 按解析顺序收集 (arg, default) 用于尾部对齐 ordered: list[tuple[N.arg, N.expr | None]] = [] if not self._AtOp(')'): while True: self._SkipNL() if self._AtOp(')'): break if self._AtOp('*'): self._Next() if self._AtOp(','): # 裸 *:分隔 kwonly seen_star = True self._Next() if self._AtOp(')') or self._AtOp(':'): break continue vararg = self._ParseArg(allow_annotation=not allow_lambda) seen_star = True elif self._AtOp('**'): self._Next() kwarg = self._ParseArg(allow_annotation=not allow_lambda) else: a: N.arg = self._ParseArg(allow_annotation=not allow_lambda) default: N.expr | None = None if self._AtOp('='): self._Next() default = self.ParseExpr() if seen_star: kwonlyargs.append(a) kw_defaults.append(default) else: ordered.append((a, default)) self._SkipNL() if self._AtOp(','): self._Next() if self._AtOp(')'): break continue break # 切分 posonly 与普通 args:CPython 无显式 '/' 信息时全为普通 args # 注意:本实现未显式解析 '/' 分隔,posonlyargs 保持空(TPC 源码不使用) for a, d in ordered: args.append(a) defaults.append(d) # defaults 尾部对齐:去掉开头的 None,且仅保留末尾连续默认值 # CPython 语义:defaults 与 args 尾部对齐,若中间有缺省则语法错误。 # 宽松处理:取最后一个 None 之后的全部默认值。 last_none: int = -1 for i, d in enumerate(defaults): if d is None: last_none = i defaults = [d for d in defaults[last_none + 1:] if d is not None] # 将带默认值的 arg 之前的参数清掉默认值列表长度 = 默认参数个数 self._SkipNL() if not skip_open: self._ExpectOp(')') node: N.arguments = N.arguments( posonlyargs=posonlyargs, args=args, vararg=vararg, kwonlyargs=kwonlyargs, kw_defaults=kw_defaults, kwarg=kwarg, defaults=defaults, ) return node def _ParseArg(self, allow_annotation: bool) -> N.arg: start: Token = self._ExpectName() annotation: N.expr | None = None if allow_annotation and self._AtOp(':'): self._Next() annotation = self.ParseExpr() a: N.arg = N.arg(arg=start.Value, annotation=annotation, type_comment=None) self._FixLoc(a, start) return a