TermiNexus
6fc9e19e24
TPC 侧取缔对 Python 标准库 ast 模块的依赖，改用 lib/vast 包： - Nodes: 与 CPython 3.12 ast 完全同构的 131 个节点类(含抽象基类继承) - Tokenizer: 词法分析(缩进/字符串/f-string/数字/运算符/软关键字) - ExprParser: 表达式递归下降解析(优先级/lambda/推导式/f-string/切片) - Parser: 语句解析(def/class/if/for/try/with/match/import/注解) - Utils/Unparser: walk/dump/unparse/copy_location/literal_eval 等辅助 - 软关键字 fn/fun/func/function 并入解析器, 废除 ExpandSoftKeywords 预处理 - lib/_bootstrap 将 lib 目录加入 sys.path, import vast as ast 零修改替换 - 52 个文件 import ast 替换为 import vast as ast; 94 文件差分测试 0 失败
1454 lines
56 KiB
Python
1454 lines
56 KiB
Python
"""vast.exprparser - Viper/Python 表达式语法分析器
|
||
|
||
实现与 CPython 3.12 语法规则一致的递归下降表达式解析：
|
||
|
||
- 完整运算符优先级（lambda/条件/or/and/not/比较/位运算/移位/算术/一元/幂/await）
|
||
- 原子表达式：标识符/数字/字符串/常量/f-string/元组/列表/字典/集合
|
||
- 推导式（list/set/dict/generator）、lambda、命名表达式（walrus）
|
||
- 调用（位置实参/关键字实参/*args/**kwargs）、属性、下标、切片
|
||
- f-string（JoinedStr/FormattedValue，含嵌套表达式与格式说明符）
|
||
|
||
本模块为 Mixin 设计：``Parser`` 继承 ``ExprParserMixin`` 获得表达式能力。
|
||
所有产出节点与 CPython ``ast`` 结构完全一致，并填充 lineno/col_offset/
|
||
end_lineno/end_col_offset 位置属性。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import bisect
|
||
|
||
from lib.vast import Nodes as N
|
||
from lib.vast.Tokenizer import (
|
||
Token, NAME, NUMBER, STRING, FSTRING, OP, NEWLINE, NL, ENDMARKER,
|
||
Tokenizer, _STRING_PREFIX_CHARS,
|
||
)
|
||
|
||
|
||
def _JoinConstParts(parts: list[object]) -> str | bytes:
|
||
"""拼接连续字符串字面量值（str 或 bytes，与 CPython 语义一致）。"""
|
||
if not parts:
|
||
return ''
|
||
if isinstance(parts[0], bytes):
|
||
buf: list[bytes] = []
|
||
for p in parts:
|
||
if isinstance(p, bytes):
|
||
buf.append(p)
|
||
else:
|
||
buf.append(str(p).encode('utf-8'))
|
||
return b''.join(buf)
|
||
return ''.join(str(p) for p in parts)
|
||
|
||
|
||
class TokenStream:
|
||
"""token 流管理基类：统一提供游标操作、位置标注工具。"""
|
||
|
||
def __init__(self, tokens: list[Token], source: str, line_starts: list[int]) -> None:
|
||
self.Tokens: list[Token] = tokens
|
||
self.Source: str = source
|
||
self.LineStarts: list[int] = line_starts
|
||
self.Idx: int = 0
|
||
|
||
# ------------------------------------------------------------------
|
||
# 游标操作
|
||
# ------------------------------------------------------------------
|
||
def _Peek(self, offset: int = 0) -> Token:
|
||
i: int = self.Idx + offset
|
||
if i >= len(self.Tokens):
|
||
return self.Tokens[-1]
|
||
return self.Tokens[i]
|
||
|
||
def _Next(self) -> Token:
|
||
tok: Token = self.Tokens[self.Idx]
|
||
if self.Idx < len(self.Tokens) - 1:
|
||
self.Idx += 1
|
||
return tok
|
||
|
||
def _Prev(self) -> Token:
|
||
return self.Tokens[max(0, self.Idx - 1)]
|
||
|
||
def _At(self, tok_type: str, value: str | None = None) -> bool:
|
||
tok: Token = self._Peek()
|
||
if tok.Type != tok_type:
|
||
return False
|
||
if value is not None and tok.Value != value:
|
||
return False
|
||
return True
|
||
|
||
def _AtOp(self, value: str) -> bool:
|
||
return self._At(OP, value)
|
||
|
||
def _AtName(self, value: str) -> bool:
|
||
return self._At(NAME, value)
|
||
|
||
def _SkipNL(self) -> None:
|
||
"""跳过括号内换行 token（NL）。"""
|
||
while self._Peek().Type == NL:
|
||
self._Next()
|
||
|
||
def _SkipNEWLINE(self) -> None:
|
||
"""跳过逻辑换行 token（NEWLINE）与空行/注释行（NL）。"""
|
||
while self._Peek().Type in (NEWLINE, NL):
|
||
self._Next()
|
||
|
||
def _ExpectOp(self, value: str) -> Token:
|
||
tok: Token = self._Next()
|
||
if tok.Type != OP or tok.Value != value:
|
||
raise SyntaxError(
|
||
f"expected '{value}', got {tok.Type}({tok.Value!r}) "
|
||
f"at line {tok.Line}"
|
||
)
|
||
return tok
|
||
|
||
def _ExpectName(self, value: str | None = None) -> Token:
|
||
tok: Token = self._Next()
|
||
if tok.Type != NAME or (value is not None and tok.Value != value):
|
||
want: str = f" {value!r}" if value else ''
|
||
raise SyntaxError(
|
||
f"expected name{want}, "
|
||
f"got {tok.Type}({tok.Value!r}) at line {tok.Line}"
|
||
)
|
||
return tok
|
||
|
||
def _ExpectNEWLINE(self) -> None:
|
||
tok: Token = self._Next()
|
||
if tok.Type != NEWLINE:
|
||
raise SyntaxError(
|
||
f"expected newline, got {tok.Type}({tok.Value!r}) at line {tok.Line}"
|
||
)
|
||
|
||
# ------------------------------------------------------------------
|
||
# 位置标注
|
||
# ------------------------------------------------------------------
|
||
def _FixLoc(self, node: N.AST, start_tok: Token, end_tok: Token | None = None) -> N.AST:
|
||
"""为节点设置 4 个位置属性（end 缺省取前一个 token）。"""
|
||
if end_tok is None:
|
||
end_tok = self._Prev()
|
||
if 'lineno' in getattr(node, '_attributes', ()):
|
||
node.lineno = start_tok.Line
|
||
node.col_offset = start_tok.Col
|
||
node.end_lineno = end_tok.EndLine
|
||
node.end_col_offset = end_tok.EndCol
|
||
return node
|
||
|
||
def _PosFromOffset(self, offset: int) -> tuple[int, int]:
|
||
"""字符偏移 → (行号, UTF-8 字节列偏移)。"""
|
||
idx: int = bisect.bisect_right(self.LineStarts, offset) - 1
|
||
line: int = idx + 1
|
||
col: int = len(self.Source[self.LineStarts[idx]:offset].encode('utf-8'))
|
||
return line, col
|
||
|
||
def _Relocate(self, tok: Token, base: int) -> None:
|
||
"""将子表达式 token 按绝对偏移重定位（f-string 内部表达式用）。"""
|
||
tok.Start = base + tok.Start
|
||
tok.End = base + tok.End
|
||
line, col = self._PosFromOffset(tok.Start)
|
||
e_line, e_col = self._PosFromOffset(tok.End)
|
||
tok.Line, tok.Col, tok.EndLine, tok.EndCol = line, col, e_line, e_col
|
||
|
||
|
||
class ExprParserMixin(TokenStream):
|
||
"""表达式解析器（Mixin，供 Parser 使用）。"""
|
||
|
||
# ==================================================================
|
||
# 表达式顶层：lambda | 条件表达式
|
||
# ==================================================================
|
||
def ParseExpr(self) -> N.expr:
|
||
"""完整表达式：lambda 或条件表达式（含 walrus 前置）。"""
|
||
if self._AtName('lambda'):
|
||
return self._ParseLambda()
|
||
return self._ParseConditional()
|
||
|
||
def ParseExprList(self) -> N.expr:
|
||
"""testlist_star_expr：表达式或逗号分隔元组（赋值右值/return 值）。"""
|
||
start: Token = self._Peek()
|
||
first: N.expr = self.ParseExpr()
|
||
if self._AtOp(','):
|
||
elts: list[N.expr] = [first]
|
||
while self._AtOp(','):
|
||
self._Next()
|
||
self._SkipNL()
|
||
elts.append(self.ParseExpr())
|
||
tup: N.Tuple = N.Tuple(elts=elts, ctx=N.Load())
|
||
self._FixLoc(tup, start)
|
||
return tup
|
||
return first
|
||
|
||
def _ParseConditional(self) -> N.expr:
|
||
"""条件表达式：disjunction ('if' disjunction 'else' expression)?"""
|
||
start: Token = self._Peek()
|
||
body: N.expr = self._ParseDisjunction()
|
||
if self._AtName('if'):
|
||
self._Next()
|
||
test: N.expr = self._ParseDisjunction()
|
||
self._ExpectName('else')
|
||
orelse: N.expr = self.ParseExpr()
|
||
node: N.IfExp = N.IfExp(test=test, body=body, orelse=orelse)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
return body
|
||
|
||
# ==================================================================
|
||
# 布尔/比较
|
||
# ==================================================================
|
||
def _ParseDisjunction(self) -> N.expr:
|
||
start: Token = self._Peek()
|
||
values: list[N.expr] = [self._ParseConjunction()]
|
||
while True:
|
||
self._SkipNL()
|
||
if self._AtName('or'):
|
||
self._Next()
|
||
values.append(self._ParseConjunction())
|
||
continue
|
||
break
|
||
if len(values) == 1:
|
||
return values[0]
|
||
node: N.BoolOp = N.BoolOp(op=N.Or(), values=values)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
|
||
def _ParseConjunction(self) -> N.expr:
|
||
start: Token = self._Peek()
|
||
values: list[N.expr] = [self._ParseInversion()]
|
||
while True:
|
||
self._SkipNL()
|
||
if self._AtName('and'):
|
||
self._Next()
|
||
values.append(self._ParseInversion())
|
||
continue
|
||
break
|
||
if len(values) == 1:
|
||
return values[0]
|
||
node: N.BoolOp = N.BoolOp(op=N.And(), values=values)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
|
||
def _ParseInversion(self) -> N.expr:
|
||
self._SkipNL()
|
||
if self._AtName('not'):
|
||
start: Token = self._Next()
|
||
operand: N.expr = self._ParseInversion()
|
||
node: N.UnaryOp = N.UnaryOp(op=N.Not(), operand=operand)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
return self._ParseComparison()
|
||
|
||
# 比较操作符映射
|
||
_COMPARE_OPS: dict[str, type] = {
|
||
'<': N.Lt, '>': N.Gt, '<=': N.LtE, '>=': N.GtE,
|
||
'==': N.Eq, '!=': N.NotEq,
|
||
'is': N.Is, 'is not': N.IsNot,
|
||
'in': N.In, 'not in': N.NotIn,
|
||
}
|
||
|
||
def _ParseComparison(self) -> N.expr:
|
||
start: Token = self._Peek()
|
||
left: N.expr = self._ParseBitOr()
|
||
ops: list[N.AST] = []
|
||
comparators: list[N.expr] = []
|
||
while True:
|
||
self._SkipNL()
|
||
tok: Token = self._Peek()
|
||
op_type: type | None = None
|
||
if tok.Type == OP and tok.Value in ('<', '>', '<=', '>=', '==', '!='):
|
||
op_type = self._COMPARE_OPS[tok.Value]
|
||
self._Next()
|
||
elif tok.Type == NAME and tok.Value == 'is':
|
||
self._Next()
|
||
if self._AtName('not'):
|
||
self._Next()
|
||
op_type = N.IsNot
|
||
else:
|
||
op_type = N.Is
|
||
elif tok.Type == NAME and tok.Value == 'in':
|
||
self._Next()
|
||
op_type = N.In
|
||
elif tok.Type == NAME and tok.Value == 'not' and self._Peek(1).Type == NAME and self._Peek(1).Value == 'in':
|
||
self._Next()
|
||
self._Next()
|
||
op_type = N.NotIn
|
||
else:
|
||
break
|
||
ops.append(op_type())
|
||
comparators.append(self._ParseBitOr())
|
||
if not ops:
|
||
return left
|
||
node: N.Compare = N.Compare(left=left, ops=ops, comparators=comparators)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
|
||
# ==================================================================
|
||
# 位运算
|
||
# ==================================================================
|
||
def _ParseBitOr(self) -> N.expr:
|
||
start: Token = self._Peek()
|
||
left: N.expr = self._ParseBitXor()
|
||
while True:
|
||
self._SkipNL()
|
||
if self._AtOp('|'):
|
||
self._Next()
|
||
right: N.expr = self._ParseBitXor()
|
||
node: N.BinOp = N.BinOp(left=left, op=N.BitOr(), right=right)
|
||
self._FixLoc(node, start)
|
||
left = node
|
||
continue
|
||
break
|
||
return left
|
||
|
||
def _ParseBitXor(self) -> N.expr:
|
||
start: Token = self._Peek()
|
||
left: N.expr = self._ParseBitAnd()
|
||
while True:
|
||
self._SkipNL()
|
||
if self._AtOp('^'):
|
||
self._Next()
|
||
right: N.expr = self._ParseBitAnd()
|
||
node: N.BinOp = N.BinOp(left=left, op=N.BitXor(), right=right)
|
||
self._FixLoc(node, start)
|
||
left = node
|
||
continue
|
||
break
|
||
return left
|
||
|
||
def _ParseBitAnd(self) -> N.expr:
|
||
start: Token = self._Peek()
|
||
left: N.expr = self._ParseShift()
|
||
while True:
|
||
self._SkipNL()
|
||
if self._AtOp('&'):
|
||
self._Next()
|
||
right: N.expr = self._ParseShift()
|
||
node: N.BinOp = N.BinOp(left=left, op=N.BitAnd(), right=right)
|
||
self._FixLoc(node, start)
|
||
left = node
|
||
continue
|
||
break
|
||
return left
|
||
|
||
def _ParseShift(self) -> N.expr:
|
||
start: Token = self._Peek()
|
||
left: N.expr = self._ParseArith()
|
||
while True:
|
||
self._SkipNL()
|
||
if self._AtOp('<<'):
|
||
self._Next()
|
||
right: N.expr = self._ParseArith()
|
||
node = N.BinOp(left=left, op=N.LShift(), right=right)
|
||
elif self._AtOp('>>'):
|
||
self._Next()
|
||
right = self._ParseArith()
|
||
node = N.BinOp(left=left, op=N.RShift(), right=right)
|
||
else:
|
||
break
|
||
self._FixLoc(node, start)
|
||
left = node
|
||
return left
|
||
|
||
# ==================================================================
|
||
# 算术
|
||
# ==================================================================
|
||
def _ParseArith(self) -> N.expr:
|
||
start: Token = self._Peek()
|
||
left: N.expr = self._ParseTerm()
|
||
while True:
|
||
self._SkipNL()
|
||
if self._AtOp('+'):
|
||
self._Next()
|
||
right: N.expr = self._ParseTerm()
|
||
node = N.BinOp(left=left, op=N.Add(), right=right)
|
||
elif self._AtOp('-'):
|
||
self._Next()
|
||
right = self._ParseTerm()
|
||
node = N.BinOp(left=left, op=N.Sub(), right=right)
|
||
else:
|
||
break
|
||
self._FixLoc(node, start)
|
||
left = node
|
||
return left
|
||
|
||
def _ParseTerm(self) -> N.expr:
|
||
start: Token = self._Peek()
|
||
left: N.expr = self._ParseFactor()
|
||
while True:
|
||
self._SkipNL()
|
||
tok: Token = self._Peek()
|
||
op_type: type | None = None
|
||
if tok.Type == OP:
|
||
op_type = {'*': N.Mult, '/': N.Div, '//': N.FloorDiv,
|
||
'%': N.Mod, '@': N.MatMult}.get(tok.Value)
|
||
if op_type is None:
|
||
break
|
||
self._Next()
|
||
right: N.expr = self._ParseFactor()
|
||
node: N.BinOp = N.BinOp(left=left, op=op_type(), right=right)
|
||
self._FixLoc(node, start)
|
||
left = node
|
||
return left
|
||
|
||
# ==================================================================
|
||
# 一元 / 幂 / await
|
||
# ==================================================================
|
||
def _ParseFactor(self) -> N.expr:
|
||
tok: Token = self._Peek()
|
||
if tok.Type == OP and tok.Value in ('+', '-', '~'):
|
||
self._Next()
|
||
operand: N.expr = self._ParseFactor()
|
||
op_type: type = {'+': N.UAdd, '-': N.USub, '~': N.Invert}[tok.Value]
|
||
node: N.UnaryOp = N.UnaryOp(op=op_type(), operand=operand)
|
||
self._FixLoc(node, tok)
|
||
return node
|
||
return self._ParsePower()
|
||
|
||
def _ParsePower(self) -> N.expr:
|
||
start: Token = self._Peek()
|
||
base: N.expr = self._ParseAwaitPrimary()
|
||
if self._AtOp('**'):
|
||
self._Next()
|
||
exponent: N.expr = self._ParseFactor() # ** 右结合
|
||
node: N.BinOp = N.BinOp(left=base, op=N.Pow(), right=exponent)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
return base
|
||
|
||
def _ParseAwaitPrimary(self) -> N.expr:
|
||
if self._AtName('await'):
|
||
start: Token = self._Next()
|
||
value: N.expr = self._ParsePrimary()
|
||
node: N.Await = N.Await(value=value)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
return self._ParsePrimary()
|
||
|
||
# ==================================================================
|
||
# 主表达式：原子 + trailer（调用/属性/下标）
|
||
# ==================================================================
|
||
def _ParsePrimary(self) -> N.expr:
|
||
start: Token = self._Peek()
|
||
node: N.expr = self._ParseAtom()
|
||
while True:
|
||
self._SkipNL()
|
||
tok: Token = self._Peek()
|
||
if tok.Type == OP and tok.Value == '.':
|
||
self._Next()
|
||
attr_tok: Token = self._ExpectName()
|
||
attr_node: N.Attribute = N.Attribute(value=node, attr=attr_tok.Value, ctx=N.Load())
|
||
self._FixLoc(attr_node, start)
|
||
node = attr_node
|
||
elif tok.Type == OP and tok.Value == '(':
|
||
call_node: N.Call = self._ParseCallTail(node, start)
|
||
node = call_node
|
||
elif tok.Type == OP and tok.Value == '[':
|
||
sub_node: N.Subscript = self._ParseSubscriptTail(node, start)
|
||
node = sub_node
|
||
else:
|
||
break
|
||
return node
|
||
|
||
def _ParseCallTail(self, func: N.expr, start: Token) -> N.Call:
|
||
self._ExpectOp('(')
|
||
args: list[N.expr] = []
|
||
keywords: list[N.keyword] = []
|
||
self._SkipNL()
|
||
if not self._AtOp(')'):
|
||
while True:
|
||
self._SkipNL()
|
||
# **kwargs
|
||
if self._AtOp('**'):
|
||
self._Next()
|
||
value: N.expr = self.ParseExpr()
|
||
kw: N.keyword = N.keyword(arg=None, value=value)
|
||
self._FixLoc(kw, self._Prev())
|
||
keywords.append(kw)
|
||
else:
|
||
# *args
|
||
if self._AtOp('*'):
|
||
self._Next()
|
||
value = self.ParseExpr()
|
||
starred: N.Starred = N.Starred(value=value, ctx=N.Load())
|
||
self._FixLoc(starred, self._Prev())
|
||
args.append(starred)
|
||
else:
|
||
# 关键字参数：NAME '='
|
||
if self._Peek().Type == NAME and self._Peek(1).Type == OP and self._Peek(1).Value == '=':
|
||
name_tok: Token = self._Next()
|
||
self._Next() # '='
|
||
value = self.ParseExpr()
|
||
kw = N.keyword(arg=name_tok.Value, value=value)
|
||
self._FixLoc(kw, name_tok)
|
||
keywords.append(kw)
|
||
else:
|
||
# 生成器表达式作为唯一实参：f(x for y in z)
|
||
# 先解析第一个表达式，若随后是 'for' 则回填为 GeneratorExp
|
||
if not args and not keywords:
|
||
gen_start: Token = self._Peek()
|
||
gen_elt: N.expr = self.ParseExpr()
|
||
if self._AtName('for'):
|
||
gen_gens: list[N.comprehension] = self._ParseComprehensionGenerators()
|
||
gen: N.GeneratorExp = N.GeneratorExp(elt=gen_elt, generators=gen_gens)
|
||
self._FixLoc(gen, gen_start)
|
||
args.append(gen)
|
||
else:
|
||
args.append(gen_elt)
|
||
else:
|
||
args.append(self.ParseExpr())
|
||
self._SkipNL()
|
||
if self._AtOp(','):
|
||
self._Next()
|
||
self._SkipNL()
|
||
if self._AtOp(')'):
|
||
break
|
||
continue
|
||
break
|
||
self._ExpectOp(')')
|
||
node: N.Call = N.Call(func=func, args=args, keywords=keywords)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
|
||
def _ParseSubscriptTail(self, value: N.expr, start: Token) -> N.Subscript:
|
||
self._ExpectOp('[')
|
||
self._SkipNL()
|
||
slice_node: N.AST = self._ParseSliceList()
|
||
self._SkipNL()
|
||
self._ExpectOp(']')
|
||
node: N.Subscript = N.Subscript(value=value, slice=slice_node, ctx=N.Load())
|
||
self._FixLoc(node, start)
|
||
return node
|
||
|
||
def _ParseSliceList(self) -> N.AST:
|
||
"""解析下标列表：a[1] / a[1:2] / a[1:2:3] / a[1,2] / a[...] / a[::]"""
|
||
start: Token = self._Peek()
|
||
elts: list[N.AST] = [self._ParseSliceItem()]
|
||
while self._AtOp(','):
|
||
self._Next()
|
||
self._SkipNL()
|
||
elts.append(self._ParseSliceItem())
|
||
if len(elts) == 1:
|
||
return elts[0]
|
||
tup: N.Tuple = N.Tuple(elts=elts, ctx=N.Load())
|
||
self._FixLoc(tup, start)
|
||
return tup
|
||
|
||
def _ParseSliceItem(self) -> N.AST:
|
||
start: Token = self._Peek()
|
||
# 含冒号 → Slice
|
||
if self._AtOp(':'):
|
||
lower: N.expr | None = None
|
||
upper: N.expr | None = None
|
||
step: N.expr | None = None
|
||
self._Next() # ':'
|
||
if not self._AtOp(':') and not self._AtOp(','):
|
||
if not self._AtOp(']'):
|
||
upper = self.ParseExpr()
|
||
if self._AtOp(':'):
|
||
self._Next()
|
||
if not self._AtOp(']') and not self._AtOp(',') and not self._AtOp(':'):
|
||
step = self.ParseExpr()
|
||
node: N.Slice = N.Slice(lower=lower, upper=upper, step=step)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
# 检查 a:b 形式：第一个元素后直接冒号
|
||
first: N.expr = self.ParseExpr()
|
||
if self._AtOp(':'):
|
||
lower = first
|
||
self._Next()
|
||
upper = None
|
||
step = None
|
||
if not self._AtOp(':') and not self._AtOp(']') and not self._AtOp(','):
|
||
upper = self.ParseExpr()
|
||
if self._AtOp(':'):
|
||
self._Next()
|
||
if not self._AtOp(']') and not self._AtOp(','):
|
||
step = self.ParseExpr()
|
||
node = N.Slice(lower=lower, upper=upper, step=step)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
# 简单下标（含 Ellipsis）
|
||
if self._AtOp('...'):
|
||
# 已在 _ParseExpr 中处理为 Constant(Ellipsis)
|
||
pass
|
||
return first
|
||
|
||
# ==================================================================
|
||
# 原子表达式
|
||
# ==================================================================
|
||
def _ParseAtom(self) -> N.expr:
|
||
self._SkipNL()
|
||
tok: Token = self._Peek()
|
||
if tok.Type == NUMBER:
|
||
return self._ParseNumber()
|
||
if tok.Type in (STRING, FSTRING):
|
||
return self._ParseStringConcat()
|
||
if tok.Type == NAME:
|
||
if tok.Value in ('True', 'False', 'None'):
|
||
self._Next()
|
||
node: N.Constant = N.Constant(value={'True': True, 'False': False, 'None': None}[tok.Value], kind=None)
|
||
self._FixLoc(node, tok)
|
||
return node
|
||
self._Next()
|
||
node = N.Name(id=tok.Value, ctx=N.Load())
|
||
self._FixLoc(node, tok)
|
||
return node
|
||
if tok.Type == OP:
|
||
if tok.Value == '(':
|
||
return self._ParseParen()
|
||
if tok.Value == '[':
|
||
return self._ParseList()
|
||
if tok.Value == '{':
|
||
return self._ParseDictSet()
|
||
if tok.Value == '...':
|
||
self._Next()
|
||
node = N.Constant(value=Ellipsis, kind=None)
|
||
self._FixLoc(node, tok)
|
||
return node
|
||
raise SyntaxError(
|
||
f"unexpected token {tok.Type}({tok.Value!r}) at line {tok.Line}"
|
||
)
|
||
|
||
# ------------------------------------------------------------------
|
||
# 数字
|
||
# ------------------------------------------------------------------
|
||
def _ParseNumber(self) -> N.Constant:
|
||
tok: Token = self._Next()
|
||
text: str = tok.Value
|
||
try:
|
||
if text.lower().endswith('j'):
|
||
num: object = complex(text[:-1].replace('_', ''))
|
||
elif text.lower().startswith(('0x', '0o', '0b')):
|
||
num = int(text.replace('_', ''), 0)
|
||
elif any(c in text for c in '.eE'):
|
||
num = float(text.replace('_', ''))
|
||
else:
|
||
num = int(text.replace('_', ''), 10)
|
||
except ValueError:
|
||
# 无法解析：保留原字符串（理论上不会发生）
|
||
num = text
|
||
node: N.Constant = N.Constant(value=num, kind=None)
|
||
self._FixLoc(node, tok)
|
||
return node
|
||
|
||
# ------------------------------------------------------------------
|
||
# 字符串（含相邻拼接与 f-string 混合）
|
||
# ------------------------------------------------------------------
|
||
def _ParseStringConcat(self) -> N.expr:
|
||
"""解析连续字符串字面量并拼接。
|
||
|
||
规则（与 CPython 一致）：
|
||
- 全部为普通字符串 → 单个 Constant（值合并）
|
||
- 含 f-string → JoinedStr（字符串为 Constant，f-string 展开为 values）
|
||
"""
|
||
start: Token = self._Peek()
|
||
parts: list[N.expr] = [] # JoinedStr 的 values 或常量
|
||
const_parts: list[object] = []
|
||
has_fstring: bool = False
|
||
while True:
|
||
# 相邻字符串可跨括号内换行（NL）合并
|
||
self._SkipNL()
|
||
tok: Token = self._Peek()
|
||
if tok.Type == STRING:
|
||
self._Next()
|
||
val: object = self._DecodeString(tok.Value)
|
||
const_parts.append(val)
|
||
elif tok.Type == FSTRING:
|
||
self._Next()
|
||
joined: N.JoinedStr = self._ParseFStringToken(tok)
|
||
has_fstring = True
|
||
if const_parts:
|
||
parts.append(self._MakeStrConstant(_JoinConstParts(const_parts), start))
|
||
const_parts = []
|
||
# 相邻 Constant 合并（f-string 字面量 + 后续常量段）
|
||
if parts and isinstance(parts[-1], N.Constant) and joined.values and isinstance(joined.values[0], N.Constant):
|
||
merged: str = str(parts[-1].value) + str(joined.values[0].value)
|
||
merged_const: N.Constant = N.Constant(value=merged, kind=None)
|
||
merged_const.lineno = parts[-1].lineno
|
||
merged_const.col_offset = parts[-1].col_offset
|
||
parts[-1] = merged_const
|
||
parts.extend(joined.values[1:])
|
||
else:
|
||
parts.extend(joined.values)
|
||
else:
|
||
break
|
||
if not has_fstring:
|
||
node: N.Constant = N.Constant(value=_JoinConstParts(const_parts), kind=None)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
if const_parts:
|
||
if parts and isinstance(parts[-1], N.Constant):
|
||
merged = str(parts[-1].value) + str(_JoinConstParts(const_parts))
|
||
merged_const = N.Constant(value=merged, kind=None)
|
||
merged_const.lineno = parts[-1].lineno
|
||
merged_const.col_offset = parts[-1].col_offset
|
||
parts[-1] = merged_const
|
||
else:
|
||
parts.append(self._MakeStrConstant(str(_JoinConstParts(const_parts)), start))
|
||
node = N.JoinedStr(values=parts)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
|
||
def _MakeStrConstant(self, value: str, start: Token) -> N.Constant:
|
||
node: N.Constant = N.Constant(value=value, kind=None)
|
||
node.lineno = start.Line
|
||
node.col_offset = start.Col
|
||
return node
|
||
|
||
# ------------------------------------------------------------------
|
||
# 字符串解码
|
||
# ------------------------------------------------------------------
|
||
def _DecodeString(self, raw: str) -> str | bytes:
|
||
"""解码字符串字面量原始文本（含前缀与引号）为实际值。
|
||
|
||
支持 r/b/u/f 前缀组合、单引号/双引号/三引号、常见转义序列。
|
||
"""
|
||
# 提取前缀与引号
|
||
i: int = 0
|
||
while i < len(raw) and raw[i] in _STRING_PREFIX_CHARS:
|
||
i += 1
|
||
prefix: str = raw[:i]
|
||
quote: str = raw[i]
|
||
triple: bool = raw.startswith(quote * 3, i)
|
||
body_start: int = i + (3 if triple else 1)
|
||
body_end: int = len(raw) - (3 if triple else 1)
|
||
body: str = raw[body_start:body_end]
|
||
is_raw: bool = 'r' in prefix.lower()
|
||
is_bytes: bool = 'b' in prefix.lower()
|
||
if is_bytes:
|
||
# bytes 字面量：逐字节解码（转义为字节值）
|
||
return self._DecodeBytes(body, is_raw)
|
||
# str 字面量
|
||
return self._DecodeEscapes(body, is_raw)
|
||
|
||
def _DecodeEscapes(self, body: str, is_raw: bool) -> str:
|
||
"""解码字符串转义序列。"""
|
||
if is_raw:
|
||
return body
|
||
out: list[str] = []
|
||
i: int = 0
|
||
n: int = len(body)
|
||
while i < n:
|
||
ch: str = body[i]
|
||
if ch != '\\':
|
||
out.append(ch)
|
||
i += 1
|
||
continue
|
||
i += 1
|
||
if i >= n:
|
||
out.append('\\')
|
||
break
|
||
esc: str = body[i]
|
||
simple: dict[str, str] = {
|
||
'\\': '\\', "'": "'", '"': '"', 'n': '\n', 't': '\t',
|
||
'r': '\r', 'b': '\b', 'f': '\f', 'v': '\v', 'a': '\a',
|
||
}
|
||
if esc in simple:
|
||
out.append(simple[esc])
|
||
i += 1
|
||
elif esc == 'x':
|
||
hex_digits: str = body[i + 1:i + 3]
|
||
out.append(chr(int(hex_digits, 16)))
|
||
i += 3
|
||
elif esc == 'u':
|
||
hex_digits = body[i + 1:i + 5]
|
||
out.append(chr(int(hex_digits, 16)))
|
||
i += 5
|
||
elif esc == 'U':
|
||
hex_digits = body[i + 1:i + 9]
|
||
out.append(chr(int(hex_digits, 16)))
|
||
i += 9
|
||
elif esc.isdigit() and '0' <= esc <= '7':
|
||
oct_digits: str = body[i:i + 3]
|
||
j: int = 0
|
||
while j < 3 and j < len(oct_digits) and '0' <= oct_digits[j] <= '7':
|
||
j += 1
|
||
out.append(chr(int(oct_digits[:j], 8)))
|
||
i += j
|
||
else:
|
||
# 未知转义：保留原样（CPython 保留反斜杠）
|
||
out.append('\\' + esc)
|
||
i += 1
|
||
return ''.join(out)
|
||
|
||
def _DecodeBytes(self, body: str, is_raw: bool) -> bytes:
|
||
"""解码 bytes 字面量（转义序列按字节处理，支持 r 前缀）。"""
|
||
if is_raw:
|
||
return body.encode('latin-1')
|
||
out: list[int] = []
|
||
i: int = 0
|
||
n: int = len(body)
|
||
while i < n:
|
||
ch: str = body[i]
|
||
if ch != '\\':
|
||
try:
|
||
out.append(ord(body[i]))
|
||
except Exception:
|
||
out.append(0)
|
||
i += 1
|
||
continue
|
||
i += 1
|
||
if i >= n:
|
||
out.append(ord('\\'))
|
||
break
|
||
esc: str = body[i]
|
||
simple: dict[str, int] = {
|
||
'\\': ord('\\'), "'": ord("'"), '"': ord('"'),
|
||
'n': 10, 't': 9, 'r': 13, 'b': 8, 'f': 12, 'v': 11, 'a': 7,
|
||
}
|
||
if esc in simple:
|
||
out.append(simple[esc])
|
||
i += 1
|
||
elif esc == 'x':
|
||
out.append(int(body[i + 1:i + 3], 16))
|
||
i += 3
|
||
elif esc == 'u':
|
||
out.append(int(body[i + 1:i + 5], 16))
|
||
i += 5
|
||
elif esc == 'U':
|
||
out.append(int(body[i + 1:i + 9], 16))
|
||
i += 9
|
||
elif esc.isdigit() and '0' <= esc <= '7':
|
||
oct_digits: str = body[i:i + 3]
|
||
j: int = 0
|
||
while j < 3 and j < len(oct_digits) and '0' <= oct_digits[j] <= '7':
|
||
j += 1
|
||
out.append(int(oct_digits[:j], 8))
|
||
i += j
|
||
else:
|
||
out.append(ord('\\'))
|
||
out.append(ord(esc))
|
||
i += 1
|
||
return bytes(out)
|
||
|
||
# ------------------------------------------------------------------
|
||
# f-string 解析
|
||
# ------------------------------------------------------------------
|
||
def _ParseFStringToken(self, tok: Token) -> N.JoinedStr:
|
||
"""解析 FSTRING token 为 JoinedStr（values 为 Constant/FormattedValue）。"""
|
||
raw: str = tok.Value
|
||
values: list[N.expr] = self._ParseFStringBody(raw, tok, 0)
|
||
node: N.JoinedStr = N.JoinedStr(values=values)
|
||
self._FixLoc(node, tok)
|
||
return node
|
||
|
||
def _ParseFStringBody(self, raw: str, outer_tok: Token, base_offset: int) -> list[N.expr]:
|
||
"""解析 f-string 文本（含前缀与引号）为 values 列表。
|
||
|
||
Args:
|
||
raw: f-string 原始文本
|
||
outer_tok: FSTRING token（用于绝对偏移换算）
|
||
base_offset: 文本在 token Value 中的起始偏移（嵌套用）
|
||
"""
|
||
# 提取前缀与引号
|
||
i: int = 0
|
||
while i < len(raw) and raw[i] in _STRING_PREFIX_CHARS:
|
||
i += 1
|
||
prefix: str = raw[:i]
|
||
quote: str = raw[i]
|
||
triple: bool = raw.startswith(quote * 3, i)
|
||
is_raw: bool = 'r' in prefix.lower()
|
||
body_start: int = i + (3 if triple else 1)
|
||
body_end: int = len(raw) - (3 if triple else 1)
|
||
body: str = raw[body_start:body_end]
|
||
values: list[N.expr] = []
|
||
literal: list[str] = []
|
||
j: int = 0
|
||
n: int = len(body)
|
||
rel_base: int = body_start + base_offset # 相对外层的偏移
|
||
while j < n:
|
||
ch: str = body[j]
|
||
if ch == '{':
|
||
if j + 1 < n and body[j + 1] == '{':
|
||
literal.append('{')
|
||
j += 2
|
||
continue
|
||
# 表达式段开始
|
||
if literal:
|
||
cnode: N.Constant = N.Constant(value=self._DecodeEscapes(''.join(literal), is_raw), kind=None)
|
||
cnode.lineno, cnode.col_offset = outer_tok.Line, outer_tok.Col
|
||
values.append(cnode)
|
||
literal = []
|
||
fv: N.FormattedValue = self._ParseFStringValue(body, j, outer_tok, rel_base)
|
||
values.append(fv)
|
||
# 更新 j 到表达式段之后
|
||
j = self._FStringExprEnd
|
||
continue
|
||
elif ch == '}':
|
||
if j + 1 < n and body[j + 1] == '}':
|
||
literal.append('}')
|
||
j += 2
|
||
continue
|
||
# 单独的 } 是语法错误，宽松处理为字面量
|
||
literal.append('}')
|
||
j += 1
|
||
continue
|
||
else:
|
||
literal.append(ch)
|
||
j += 1
|
||
if literal:
|
||
cnode = N.Constant(value=self._DecodeEscapes(''.join(literal), is_raw), kind=None)
|
||
cnode.lineno, cnode.col_offset = outer_tok.Line, outer_tok.Col
|
||
values.append(cnode)
|
||
return values
|
||
|
||
_FStringExprEnd: int = 0 # 表达式段结束位置（body 内偏移），供调用方跳转
|
||
|
||
def _ParseFStringValue(self, body: str, start: int, outer_tok: Token, rel_base: int) -> N.FormattedValue:
|
||
"""解析 f-string 单个 {expr[!conv][:spec]} 段。
|
||
|
||
返回 FormattedValue；将表达式段结束位置存入 self._FStringExprEnd。
|
||
"""
|
||
j: int = start + 1 # 跳过 '{'
|
||
n: int = len(body)
|
||
depth: int = 1
|
||
expr_start: int = j
|
||
expr_end: int = j
|
||
# 扫描结束字符：'}' 正常闭合；'!'/':' 为转换/格式说明符起点
|
||
stop_char: str = '}'
|
||
# 扫描到匹配的 '}'（考虑括号嵌套与内部字符串）；
|
||
# 顶层 '!' 或 ':' 表示表达式结束（转换/格式说明符开始）
|
||
while j < n:
|
||
ch: str = body[j]
|
||
if ch == '\\':
|
||
j += 2
|
||
continue
|
||
if ch in '\'"':
|
||
# 跳过内部字符串字面量
|
||
j = self._SkipInnerLiteral(body, j)
|
||
continue
|
||
if ch in '([{':
|
||
depth += 1
|
||
j += 1
|
||
continue
|
||
if ch in ')]}':
|
||
if depth == 1 and ch == '}':
|
||
expr_end = j
|
||
j += 1
|
||
break
|
||
if depth == 1 and ch in ')}]':
|
||
# 不匹配：语法错误，宽松按结束处理
|
||
expr_end = j
|
||
j += 1
|
||
break
|
||
depth -= 1
|
||
j += 1
|
||
continue
|
||
if depth == 1 and (ch == '!' or ch == ':'):
|
||
# 转换/格式说明符起点：表达式结束（j 停在操作符处）
|
||
expr_end = j
|
||
stop_char = ch
|
||
break
|
||
j += 1
|
||
self._FStringExprEnd = j
|
||
expr_text: str = body[expr_start:expr_end]
|
||
# 转换与格式说明符（仅当扫描停在 ! 或 : 时才解析）
|
||
conversion: int = -1
|
||
format_spec: N.expr | None = None
|
||
k: int = j
|
||
if stop_char == '!':
|
||
k += 1
|
||
if k < n and body[k] in 'rsa':
|
||
conversion = ord(body[k])
|
||
k += 1
|
||
# 转换后可能还有 :format_spec
|
||
if k < n and body[k] == ':':
|
||
stop_char = ':'
|
||
else:
|
||
# 无格式说明符：跳过 '}' 结束表达式段
|
||
if k < n and body[k] == '}':
|
||
k += 1
|
||
self._FStringExprEnd = k
|
||
if stop_char == ':':
|
||
k += 1
|
||
# 格式说明符可含嵌套 {}（如 {x:{w}}）
|
||
spec_body: list[str] = []
|
||
depth2: int = 1
|
||
while k < n:
|
||
c2: str = body[k]
|
||
if c2 == '{':
|
||
depth2 += 1
|
||
spec_body.append(c2)
|
||
k += 1
|
||
elif c2 == '}':
|
||
depth2 -= 1
|
||
if depth2 == 0:
|
||
break
|
||
spec_body.append(c2)
|
||
k += 1
|
||
else:
|
||
spec_body.append(c2)
|
||
k += 1
|
||
spec_text: str = ''.join(spec_body)
|
||
if any(c == '{' for c in spec_text):
|
||
# 嵌套表达式：构造 JoinedStr
|
||
inner_fs: str = 'f"' + spec_text.replace('"', '\\"') + '"'
|
||
inner_tok: Token = Token(FSTRING, inner_fs, outer_tok.Line, outer_tok.Col,
|
||
outer_tok.EndLine, outer_tok.EndCol)
|
||
format_spec = self._ParseFStringToken(inner_tok)
|
||
else:
|
||
# 静态格式说明符：CPython 包为 JoinedStr(values=[Constant])
|
||
cnode: N.Constant = N.Constant(value=spec_text, kind=None)
|
||
cnode.lineno = outer_tok.Line
|
||
cnode.col_offset = outer_tok.Col
|
||
jstr: N.JoinedStr = N.JoinedStr(values=[cnode])
|
||
format_spec = jstr
|
||
# k 现指向 '}'：跳过它作为表达式段结束
|
||
if k < n and body[k] == '}':
|
||
k += 1
|
||
self._FStringExprEnd = k
|
||
# 解析表达式：子 token 流
|
||
expr_node: N.expr = self._ParseSubExpression(expr_text, outer_tok, expr_start + rel_base)
|
||
fv: N.FormattedValue = N.FormattedValue(value=expr_node, conversion=conversion, format_spec=format_spec)
|
||
fv.lineno = outer_tok.Line
|
||
fv.col_offset = outer_tok.Col
|
||
fv.end_lineno = outer_tok.EndLine
|
||
fv.end_col_offset = outer_tok.EndCol
|
||
return fv
|
||
|
||
def _SkipInnerLiteral(self, body: str, start: int) -> int:
|
||
"""跳过 f-string 表达式内部的字符串字面量，返回结束位置。"""
|
||
j: int = start
|
||
# 前缀
|
||
while j < len(body) and body[j] in _STRING_PREFIX_CHARS:
|
||
j += 1
|
||
if j >= len(body) or body[j] not in '\'"':
|
||
return start + 1
|
||
quote: str = body[j]
|
||
triple: bool = body.startswith(quote * 3, j)
|
||
j += 3 if triple else 1
|
||
while j < len(body):
|
||
ch: str = body[j]
|
||
if ch == '\\':
|
||
j += 2
|
||
continue
|
||
if ch == quote:
|
||
if triple:
|
||
if body.startswith(quote * 3, j):
|
||
j += 3
|
||
return j
|
||
j += 1
|
||
else:
|
||
j += 1
|
||
return j
|
||
j += 1
|
||
return j
|
||
|
||
def _ParseSubExpression(self, text: str, outer_tok: Token, abs_base: int) -> N.expr:
|
||
"""解析子表达式文本（f-string 内部），并重定位到绝对偏移。"""
|
||
sub_toks: list[Token] = Tokenizer(text).Tokenize()
|
||
# 过滤 ENDMARKER 后的 NEWLINE 等
|
||
for t in sub_toks:
|
||
self._Relocate(t, abs_base)
|
||
# 保存主游标，切换到子 token 流
|
||
saved_idx: int = self.Idx
|
||
saved_tokens: list[Token] = self.Tokens
|
||
self.Tokens = sub_toks
|
||
self.Idx = 0
|
||
try:
|
||
expr: N.expr = self.ParseExpr()
|
||
finally:
|
||
self.Tokens = saved_tokens
|
||
self.Idx = saved_idx
|
||
return expr
|
||
|
||
# ------------------------------------------------------------------
|
||
# 括号 / 列表 / 字典 / 集合 / 推导式
|
||
# ------------------------------------------------------------------
|
||
def _ParseParen(self) -> N.expr:
|
||
start: Token = self._Next() # '('
|
||
self._SkipNL()
|
||
# 空元组 ()
|
||
if self._AtOp(')'):
|
||
self._Next()
|
||
tup: N.Tuple = N.Tuple(elts=[], ctx=N.Load())
|
||
self._FixLoc(tup, start)
|
||
return tup
|
||
# yield 表达式
|
||
if self._AtName('yield'):
|
||
yield_node: N.expr = self._ParseYield()
|
||
self._SkipNL()
|
||
self._ExpectOp(')')
|
||
return yield_node
|
||
# 命名表达式 (x := y)
|
||
if self._Peek().Type == NAME and self._Peek(1).Type == OP and self._Peek(1).Value == ':=':
|
||
name_tok: Token = self._Next()
|
||
self._Next() # ':='
|
||
value: N.expr = self.ParseExpr()
|
||
self._SkipNL()
|
||
self._ExpectOp(')')
|
||
walrus: N.NamedExpr = N.NamedExpr(target=N.Name(id=name_tok.Value, ctx=N.Store()), value=value)
|
||
self._FixLoc(walrus, start)
|
||
return walrus
|
||
first: N.expr = self.ParseExpr()
|
||
self._SkipNL()
|
||
# 生成器表达式 (x for ...)
|
||
if self._AtName('for'):
|
||
generators: list[N.comprehension] = self._ParseComprehensionGenerators()
|
||
self._SkipNL()
|
||
self._ExpectOp(')')
|
||
gen: N.GeneratorExp = N.GeneratorExp(elt=first, generators=generators)
|
||
self._FixLoc(gen, start)
|
||
return gen
|
||
# 元组 (x,) / (x, y)
|
||
if self._AtOp(','):
|
||
elts: list[N.expr] = [first]
|
||
while self._AtOp(','):
|
||
self._Next()
|
||
self._SkipNL()
|
||
if self._AtOp(')'):
|
||
break
|
||
elts.append(self.ParseExpr())
|
||
self._SkipNL()
|
||
self._ExpectOp(')')
|
||
tup = N.Tuple(elts=elts, ctx=N.Load())
|
||
self._FixLoc(tup, start)
|
||
return tup
|
||
# 普通括号表达式 (x)
|
||
self._ExpectOp(')')
|
||
return first
|
||
|
||
def _ParseList(self) -> N.expr:
|
||
start: Token = self._Next() # '['
|
||
self._SkipNL()
|
||
if self._AtOp(']'):
|
||
self._Next()
|
||
lst: N.List = N.List(elts=[], ctx=N.Load())
|
||
self._FixLoc(lst, start)
|
||
return lst
|
||
first: N.expr = self.ParseExpr()
|
||
self._SkipNL()
|
||
# 列表推导式
|
||
if self._AtName('for'):
|
||
generators = self._ParseComprehensionGenerators()
|
||
self._SkipNL()
|
||
self._ExpectOp(']')
|
||
lc: N.ListComp = N.ListComp(elt=first, generators=generators)
|
||
self._FixLoc(lc, start)
|
||
return lc
|
||
# 普通列表
|
||
elts = [first]
|
||
while self._AtOp(','):
|
||
self._Next()
|
||
self._SkipNL()
|
||
if self._AtOp(']'):
|
||
break
|
||
elts.append(self.ParseExpr())
|
||
self._SkipNL()
|
||
self._ExpectOp(']')
|
||
lst = N.List(elts=elts, ctx=N.Load())
|
||
self._FixLoc(lst, start)
|
||
return lst
|
||
|
||
def _ParseDictSet(self) -> N.expr:
|
||
start: Token = self._Next() # '{'
|
||
self._SkipNL()
|
||
if self._AtOp('}'):
|
||
self._Next()
|
||
dct: N.Dict = N.Dict(keys=[], values=[])
|
||
self._FixLoc(dct, start)
|
||
return dct
|
||
# ** 展开：{**d}
|
||
keys: list[N.expr | None] = []
|
||
values: list[N.expr] = []
|
||
while True:
|
||
self._SkipNL()
|
||
if self._AtOp('**'):
|
||
self._Next()
|
||
val: N.expr = self.ParseExpr()
|
||
keys.append(None)
|
||
values.append(val)
|
||
else:
|
||
key: N.expr = self.ParseExpr()
|
||
self._SkipNL()
|
||
if self._AtOp(':'):
|
||
self._Next()
|
||
self._SkipNL()
|
||
value: N.expr = self.ParseExpr()
|
||
keys.append(key)
|
||
values.append(value)
|
||
else:
|
||
# 集合元素
|
||
elts: list[N.expr] = [key]
|
||
while self._AtOp(','):
|
||
self._Next()
|
||
self._SkipNL()
|
||
if self._AtOp('}'):
|
||
break
|
||
elts.append(self.ParseExpr())
|
||
self._SkipNL()
|
||
# 集合推导式？
|
||
self._SkipNL()
|
||
if self._AtName('for'):
|
||
generators = self._ParseComprehensionGenerators()
|
||
self._SkipNL()
|
||
self._ExpectOp('}')
|
||
sc: N.SetComp = N.SetComp(elt=elts[0], generators=generators)
|
||
self._FixLoc(sc, start)
|
||
return sc
|
||
self._ExpectOp('}')
|
||
st: N.Set = N.Set(elts=elts)
|
||
self._FixLoc(st, start)
|
||
return st
|
||
self._SkipNL()
|
||
if self._AtOp(','):
|
||
self._Next()
|
||
self._SkipNL()
|
||
if self._AtOp('}'):
|
||
break
|
||
continue
|
||
break
|
||
# 字典推导式？
|
||
self._SkipNL()
|
||
if self._AtName('for'):
|
||
generators = self._ParseComprehensionGenerators()
|
||
self._SkipNL()
|
||
self._ExpectOp('}')
|
||
dc: N.DictComp = N.DictComp(key=keys[0] or N.Constant(value=None), value=values[0], generators=generators)
|
||
self._FixLoc(dc, start)
|
||
return dc
|
||
self._ExpectOp('}')
|
||
dct = N.Dict(keys=keys, values=values)
|
||
self._FixLoc(dct, start)
|
||
return dct
|
||
|
||
def _ParseComprehensionGenerators(self) -> list[N.comprehension]:
|
||
"""解析推导式生成器子句：for target in iter [if cond]* [async for ...]"""
|
||
generators: list[N.comprehension] = []
|
||
while True:
|
||
is_async: bool = False
|
||
if self._AtName('async') and self._Peek(1).Type == NAME and self._Peek(1).Value == 'for':
|
||
self._Next()
|
||
is_async = True
|
||
if not self._AtName('for'):
|
||
break
|
||
start: Token = self._Next() # 'for'
|
||
target: N.expr = self._ParseComprehensionTarget()
|
||
self._ExpectName('in')
|
||
# iter 用 disjunction 级别解析，避免吞掉 'if' 条件
|
||
iterable: N.expr = self._ParseDisjunction()
|
||
ifs: list[N.expr] = []
|
||
while self._AtName('if'):
|
||
self._Next()
|
||
ifs.append(self._ParseDisjunction())
|
||
comp: N.comprehension = N.comprehension(target=target, iter=iterable, ifs=ifs, is_async=int(is_async))
|
||
self._FixLoc(comp, start)
|
||
generators.append(comp)
|
||
if self._AtName('for') or (self._AtName('async') and self._Peek(1).Type == NAME and self._Peek(1).Value == 'for'):
|
||
continue
|
||
break
|
||
return generators
|
||
|
||
def _ParseComprehensionTarget(self) -> N.expr:
|
||
"""解析推导式 for 目标（支持裸元组 a, b 解包）。"""
|
||
start: Token = self._Peek()
|
||
first: N.expr = self._ParseTarget()
|
||
if self._AtOp(','):
|
||
elts: list[N.expr] = [first]
|
||
while self._AtOp(','):
|
||
self._Next()
|
||
self._SkipNL()
|
||
elts.append(self._ParseTarget())
|
||
tup: N.Tuple = N.Tuple(elts=elts, ctx=N.Store())
|
||
self._FixLoc(tup, start)
|
||
return tup
|
||
return first
|
||
|
||
def _ParseTarget(self) -> N.expr:
|
||
"""解析赋值目标（Store 上下文），支持元组/列表解包。"""
|
||
start: Token = self._Peek()
|
||
if self._AtOp('('):
|
||
self._Next()
|
||
self._SkipNL()
|
||
elts: list[N.expr] = []
|
||
if not self._AtOp(')'):
|
||
while True:
|
||
elts.append(self._ParseTarget())
|
||
self._SkipNL()
|
||
if self._AtOp(','):
|
||
self._Next()
|
||
self._SkipNL()
|
||
if self._AtOp(')'):
|
||
break
|
||
continue
|
||
break
|
||
self._ExpectOp(')')
|
||
tup: N.Tuple = N.Tuple(elts=elts, ctx=N.Store())
|
||
self._FixLoc(tup, start)
|
||
return tup
|
||
if self._AtOp('['):
|
||
self._Next()
|
||
self._SkipNL()
|
||
elts = []
|
||
if not self._AtOp(']'):
|
||
while True:
|
||
elts.append(self._ParseTarget())
|
||
self._SkipNL()
|
||
if self._AtOp(','):
|
||
self._Next()
|
||
self._SkipNL()
|
||
if self._AtOp(']'):
|
||
break
|
||
continue
|
||
break
|
||
self._ExpectOp(']')
|
||
lst: N.List = N.List(elts=elts, ctx=N.Store())
|
||
self._FixLoc(lst, start)
|
||
return lst
|
||
if self._AtOp('*'):
|
||
self._Next()
|
||
value: N.expr = self._ParseTarget()
|
||
starred: N.Starred = N.Starred(value=value, ctx=N.Store())
|
||
self._FixLoc(starred, start)
|
||
return starred
|
||
# 名称/属性/下标（Store）
|
||
tok: Token = self._Peek()
|
||
if tok.Type == NAME:
|
||
self._Next()
|
||
node: N.Name = N.Name(id=tok.Value, ctx=N.Store())
|
||
self._FixLoc(node, tok)
|
||
return node
|
||
# 属性/下标目标：解析 Load 后改 Store
|
||
node = self._ParsePrimary()
|
||
self._SetStoreContext(node)
|
||
return node
|
||
|
||
def _SetStoreContext(self, node: N.AST) -> None:
|
||
"""将 Load 上下文节点递归改为 Store。"""
|
||
if isinstance(node, N.Attribute):
|
||
node.ctx = N.Store()
|
||
elif isinstance(node, N.Subscript):
|
||
node.ctx = N.Store()
|
||
elif isinstance(node, N.Name):
|
||
node.ctx = N.Store()
|
||
elif isinstance(node, (N.Tuple, N.List)):
|
||
node.ctx = N.Store()
|
||
for elt in node.elts:
|
||
self._SetStoreContext(elt)
|
||
elif isinstance(node, N.Starred):
|
||
node.ctx = N.Store()
|
||
self._SetStoreContext(node.value)
|
||
|
||
# ------------------------------------------------------------------
|
||
# lambda / yield
|
||
# ------------------------------------------------------------------
|
||
def _ParseLambda(self) -> N.Lambda:
|
||
start: Token = self._Next() # 'lambda'
|
||
args: N.arguments
|
||
if self._AtOp(':'):
|
||
# 无参 lambda
|
||
args = N.arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[],
|
||
kw_defaults=[], kwarg=None, defaults=[])
|
||
else:
|
||
args = self._ParseArguments(skip_open=True, allow_lambda=True)
|
||
self._ExpectOp(':')
|
||
body: N.expr = self.ParseExpr()
|
||
node: N.Lambda = N.Lambda(args=args, body=body)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
|
||
def _ParseYield(self) -> N.expr:
|
||
start: Token = self._Next() # 'yield'
|
||
if self._AtName('from'):
|
||
self._Next()
|
||
value: N.expr = self.ParseExpr()
|
||
node: N.YieldFrom = N.YieldFrom(value=value)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
if not self._AtOp(')'):
|
||
value = self.ParseExprList()
|
||
node: N.Yield = N.Yield(value=value)
|
||
else:
|
||
node = N.Yield(value=None)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
|
||
# ==================================================================
|
||
# 参数列表（lambda 与函数定义共用）
|
||
# ==================================================================
|
||
def _ParseArguments(self, skip_open: bool = False, allow_lambda: bool = False) -> N.arguments:
|
||
"""解析参数列表。
|
||
|
||
支持 posonly (/)、*args、kwonly、**kwargs、默认值、注解。
|
||
defaults 与 posonlyargs+args 尾部对齐（CPython 语义）。
|
||
|
||
Args:
|
||
skip_open: True 时假定 '(' 已消费（lambda 场景无括号）
|
||
"""
|
||
if not skip_open:
|
||
self._ExpectOp('(')
|
||
self._SkipNL()
|
||
posonlyargs: list[N.arg] = []
|
||
args: list[N.arg] = []
|
||
kwonlyargs: list[N.arg] = []
|
||
kw_defaults: list[N.expr | None] = []
|
||
defaults: list[N.expr] = []
|
||
vararg: N.arg | None = None
|
||
kwarg: N.arg | None = None
|
||
seen_star: bool = False
|
||
# 按解析顺序收集 (arg, default) 用于尾部对齐
|
||
ordered: list[tuple[N.arg, N.expr | None]] = []
|
||
if not self._AtOp(')'):
|
||
while True:
|
||
self._SkipNL()
|
||
if self._AtOp(')'):
|
||
break
|
||
if self._AtOp('*'):
|
||
self._Next()
|
||
if self._AtOp(','):
|
||
# 裸 *：分隔 kwonly
|
||
seen_star = True
|
||
self._Next()
|
||
if self._AtOp(')') or self._AtOp(':'):
|
||
break
|
||
continue
|
||
vararg = self._ParseArg(allow_annotation=not allow_lambda)
|
||
seen_star = True
|
||
elif self._AtOp('**'):
|
||
self._Next()
|
||
kwarg = self._ParseArg(allow_annotation=not allow_lambda)
|
||
else:
|
||
a: N.arg = self._ParseArg(allow_annotation=not allow_lambda)
|
||
default: N.expr | None = None
|
||
if self._AtOp('='):
|
||
self._Next()
|
||
default = self.ParseExpr()
|
||
if seen_star:
|
||
kwonlyargs.append(a)
|
||
kw_defaults.append(default)
|
||
else:
|
||
ordered.append((a, default))
|
||
self._SkipNL()
|
||
if self._AtOp(','):
|
||
self._Next()
|
||
if self._AtOp(')'):
|
||
break
|
||
continue
|
||
break
|
||
# 切分 posonly 与普通 args：CPython 无显式 '/' 信息时全为普通 args
|
||
# 注意：本实现未显式解析 '/' 分隔，posonlyargs 保持空（TPC 源码不使用）
|
||
for a, d in ordered:
|
||
args.append(a)
|
||
defaults.append(d)
|
||
# defaults 尾部对齐：去掉开头的 None，且仅保留末尾连续默认值
|
||
# CPython 语义：defaults 与 args 尾部对齐，若中间有缺省则语法错误。
|
||
# 宽松处理：取最后一个 None 之后的全部默认值。
|
||
last_none: int = -1
|
||
for i, d in enumerate(defaults):
|
||
if d is None:
|
||
last_none = i
|
||
defaults = [d for d in defaults[last_none + 1:] if d is not None]
|
||
# 将带默认值的 arg 之前的参数清掉默认值列表长度 = 默认参数个数
|
||
self._SkipNL()
|
||
if not skip_open:
|
||
self._ExpectOp(')')
|
||
node: N.arguments = N.arguments(
|
||
posonlyargs=posonlyargs, args=args, vararg=vararg,
|
||
kwonlyargs=kwonlyargs, kw_defaults=kw_defaults, kwarg=kwarg,
|
||
defaults=defaults,
|
||
)
|
||
return node
|
||
|
||
def _ParseArg(self, allow_annotation: bool) -> N.arg:
|
||
start: Token = self._ExpectName()
|
||
annotation: N.expr | None = None
|
||
if allow_annotation and self._AtOp(':'):
|
||
self._Next()
|
||
annotation = self.ParseExpr()
|
||
a: N.arg = N.arg(arg=start.Value, annotation=annotation, type_comment=None)
|
||
self._FixLoc(a, start)
|
||
return a
|