"""vast.parser - Viper/Python 语句级语法分析器 基于 vast.tokenizer + vast.exprparser,实现与 CPython 3.12 ``ast.parse`` 结构完全一致的语句解析: - 简单语句:赋值/增强赋值/带注解赋值/表达式/import/from-import/global/ nonlocal/del/assert/pass/break/continue/return/raise - 复合语句:if/while/for/try/with/match/def/class/async def-for-with - 装饰器、嵌套块(INDENT/DEDENT)、分号分隔、类型注解 - Viper 软关键字 fn/fun/func/function → FunctionDef(并入解析器,无需 ExpandSoftKeywords 预处理) 产出与 CPython ast 完全一致的节点,含 4 位置属性。 """ from __future__ import annotations from lib.vast import Nodes as N from lib.vast.ExprParser import ExprParserMixin from lib.vast.Tokenizer import ( Token, NAME, NUMBER, STRING, FSTRING, OP, NEWLINE, NL, INDENT, DEDENT, ENDMARKER, SOFT_KEYWORDS, ) class Parser(ExprParserMixin): """Viper/Python 完整语法分析器。""" # ================================================================== # 模块级入口 # ================================================================== def ParseModule(self) -> N.Module: """解析整个模块:返回 Module(body=..., type_ignores=[])。""" body: list[N.stmt] = [] self._SkipNEWLINE() while self._Peek().Type != ENDMARKER: stmt: N.stmt = self.ParseStatement() body.append(stmt) # 分号分隔后续语句(简单语句一行多条) while self._AtOp(';'): self._Next() if self._At(NEWLINE): break extra: N.stmt = self.ParseStatement() body.append(extra) self._SkipNEWLINE() module: N.Module = N.Module(body=body, type_ignores=[]) return module # ================================================================== # 语句分派 # ================================================================== def ParseStatement(self) -> N.stmt: """解析单条语句(含复合语句分派)。""" tok: Token = self._Peek() if tok.Type == NAME: v: str = tok.Value # 软关键字函数定义:fn/fun/func/function + 标识符 if v in SOFT_KEYWORDS and self._Peek(1).Type == NAME: return self._ParseFunctionDef() if v == 'async': nxt: Token = self._Peek(1) if nxt.Type == NAME and nxt.Value == 'def': return self._ParseFunctionDef(is_async=True) if nxt.Type == NAME and nxt.Value == 'for': return self._ParseFor(is_async=True) if nxt.Type == NAME and nxt.Value == 'with': return self._ParseWith(is_async=True) if v == 'def': return self._ParseFunctionDef() if v == 'class': return self._ParseClassDef() if v == 'if': return self._ParseIf() if v == 'while': return self._ParseWhile() if v == 'for': return self._ParseFor() if v == 'try': return self._ParseTry() if v == 'with': return self._ParseWith() if v == 'match': # 软关键字 match:尝试 match 语句,失败回退为表达式 save: int = self.Idx try: return self._ParseMatch() except SyntaxError: self.Idx = save if v == 'return': return self._ParseReturn() if v == 'raise': return self._ParseRaise() if v == 'import': return self._ParseImport() if v == 'from': return self._ParseImportFrom() if v == 'global': return self._ParseGlobal('global') if v == 'nonlocal': return self._ParseGlobal('nonlocal') if v == 'del': return self._ParseDelete() if v == 'assert': return self._ParseAssert() if v == 'pass': self._Next() node: N.Pass = N.Pass() self._FixLoc(node, tok) return node if v == 'break': self._Next() node = N.Break() self._FixLoc(node, tok) return node if v == 'continue': self._Next() node = N.Continue() self._FixLoc(node, tok) return node if v == 'yield': # yield 作为表达式语句 return self._ParseExprStmt() if tok.Type == OP and tok.Value == '@': return self._ParseDecorated() return self._ParseSimpleStmt() # ================================================================== # 简单语句 # ================================================================== def _ParseSimpleStmt(self) -> N.stmt: """解析简单语句(赋值/增强赋值/带注解赋值/表达式)。 不消费分号/NEWLINE,由调用方循环(ParseModule/_ParseSuite)统一处理。 """ start: Token = self._Peek() # 尝试带注解赋值 x: T [= v] ann: N.stmt | None = self._TryParseAnnAssign() if ann is not None: return ann # 表达式/赋值(testlist_star_expr) expr: N.expr = self.ParseExprList() # 赋值链 if self._AtOp('='): targets: list[N.expr] = [self._AsLoadTarget(expr)] while True: self._Next() # '=' value: N.expr = self.ParseExprList() if self._AtOp('='): targets.append(self._AsLoadTarget(value)) continue break node: N.Assign = N.Assign(targets=targets, value=value, type_comment=None) self._FixLoc(node, start) stmt = node elif self._AtOp('+='): self._Next() value = self.ParseExpr() node = N.AugAssign(target=self._AsLoadTarget(expr), op=N.Add(), value=value) self._FixLoc(node, start) stmt = node elif self._AtOp('-='): self._Next() value = self.ParseExpr() node = N.AugAssign(target=self._AsLoadTarget(expr), op=N.Sub(), value=value) self._FixLoc(node, start) stmt = node elif self._AtOp('*='): self._Next() value = self.ParseExpr() node = N.AugAssign(target=self._AsLoadTarget(expr), op=N.Mult(), value=value) self._FixLoc(node, start) stmt = node elif self._AtOp('/='): self._Next() value = self.ParseExpr() node = N.AugAssign(target=self._AsLoadTarget(expr), op=N.Div(), value=value) self._FixLoc(node, start) stmt = node elif self._AtOp('//='): self._Next() value = self.ParseExpr() node = N.AugAssign(target=self._AsLoadTarget(expr), op=N.FloorDiv(), value=value) self._FixLoc(node, start) stmt = node elif self._AtOp('%='): self._Next() value = self.ParseExpr() node = N.AugAssign(target=self._AsLoadTarget(expr), op=N.Mod(), value=value) self._FixLoc(node, start) stmt = node elif self._AtOp('**='): self._Next() value = self.ParseExpr() node = N.AugAssign(target=self._AsLoadTarget(expr), op=N.Pow(), value=value) self._FixLoc(node, start) stmt = node elif self._AtOp('<<='): self._Next() value = self.ParseExpr() node = N.AugAssign(target=self._AsLoadTarget(expr), op=N.LShift(), value=value) self._FixLoc(node, start) stmt = node elif self._AtOp('>>='): self._Next() value = self.ParseExpr() node = N.AugAssign(target=self._AsLoadTarget(expr), op=N.RShift(), value=value) self._FixLoc(node, start) stmt = node elif self._AtOp('&='): self._Next() value = self.ParseExpr() node = N.AugAssign(target=self._AsLoadTarget(expr), op=N.BitAnd(), value=value) self._FixLoc(node, start) stmt = node elif self._AtOp('|='): self._Next() value = self.ParseExpr() node = N.AugAssign(target=self._AsLoadTarget(expr), op=N.BitOr(), value=value) self._FixLoc(node, start) stmt = node elif self._AtOp('^='): self._Next() value = self.ParseExpr() node = N.AugAssign(target=self._AsLoadTarget(expr), op=N.BitXor(), value=value) self._FixLoc(node, start) stmt = node elif self._AtOp('@='): self._Next() value = self.ParseExpr() node = N.AugAssign(target=self._AsLoadTarget(expr), op=N.MatMult(), value=value) self._FixLoc(node, start) stmt = node else: node = N.Expr(value=expr) self._FixLoc(node, start) stmt = node return stmt def _AsLoadTarget(self, expr: N.expr) -> N.expr: """将已解析表达式(Load 上下文)转为赋值目标(Store 上下文)。""" self._SetStoreContext(expr) return expr def _TryParseAnnAssign(self) -> N.AnnAssign | None: """尝试解析带注解赋值:x: T [= v] 或 x: T(目标为 Name/Attribute/Subscript)。 注意:仅当目标为简单 Name/Attribute/Subscript 且后随 ':' 时匹配; 元组/列表解包不做注解(返回 None 走普通路径)。 """ start: Token = self._Peek() # 名称 if self._At(NAME) and self._Peek(1).Type == OP and self._Peek(1).Value == ':': name_tok: Token = self._Next() self._Next() # ':' annotation: N.expr = self.ParseExpr() value: N.expr | None = None if self._AtOp('='): self._Next() value = self.ParseExpr() target: N.Name = N.Name(id=name_tok.Value, ctx=N.Store()) self._FixLoc(target, name_tok) node: N.AnnAssign = N.AnnAssign(target=target, annotation=annotation, value=value, simple=1) self._FixLoc(node, start) return node # 属性/下标目标(无简单解包) if self._At(NAME) or (self._At(OP) and self._Peek().Value in ('(', '[')): save: int = self.Idx try: expr: N.expr = self.ParseExpr() except SyntaxError: self.Idx = save return None if self._AtOp(':') and isinstance(expr, (N.Name, N.Attribute, N.Subscript)): self._Next() annotation = self.ParseExpr() value = None if self._AtOp('='): self._Next() value = self.ParseExpr() self._SetStoreContext(expr) simple: int = 1 if isinstance(expr, N.Name) else 0 node = N.AnnAssign(target=expr, annotation=annotation, value=value, simple=simple) self._FixLoc(node, start) return node self.Idx = save return None # ------------------------------------------------------------------ # return / raise / import / global / del / assert # ------------------------------------------------------------------ def _ParseReturn(self) -> N.Return: start: Token = self._Next() # 'return' value: N.expr | None = None if not self._At(NEWLINE) and not self._AtOp(';'): value = self.ParseExprList() node: N.Return = N.Return(value=value) self._FixLoc(node, start) return node def _ParseRaise(self) -> N.Raise: start: Token = self._Next() # 'raise' exc: N.expr | None = None cause: N.expr | None = None if not self._At(NEWLINE) and not self._AtOp(';'): exc = self.ParseExpr() if self._AtName('from'): self._Next() cause = self.ParseExpr() node: N.Raise = N.Raise(exc=exc, cause=cause) self._FixLoc(node, start) return node def _ParseImport(self) -> N.Import: start: Token = self._Next() # 'import' names: list[N.alias] = [] while True: # 点分模块名:alias 位置从首个名称起、到末个名称(或 asname)止 name_tok: Token = self._ExpectName() end_tok: Token = name_tok name_parts: list[str] = [name_tok.Value] while self._AtOp('.'): self._Next() end_tok = self._ExpectName() name_parts.append(end_tok.Value) full: str = '.'.join(name_parts) asname: str | None = None if self._AtName('as'): self._Next() as_tok: Token = self._ExpectName() asname = as_tok.Value end_tok = as_tok alias: N.alias = N.alias(name=full, asname=asname) self._FixLoc(alias, name_tok, end_tok) names.append(alias) if self._AtOp(','): self._Next() continue break node: N.Import = N.Import(names=names) self._FixLoc(node, start) return node def _ParseImportFrom(self) -> N.ImportFrom: start: Token = self._Next() # 'from' level: int = 0 while self._AtOp('.'): self._Next() level += 1 module: str | None = None # from . import x → 点后直接是 import(无模块名) if not self._AtName('import'): parts: list[str] = [self._ExpectName().Value] while self._AtOp('.'): self._Next() parts.append(self._ExpectName().Value) module = '.'.join(parts) self._ExpectName('import') names: list[N.alias] = [] # from x import * | (a, b) | a, b if self._AtOp('('): self._Next() self._SkipNL() if not self._AtOp(')'): while True: names.append(self._ParseImportAlias()) self._SkipNL() if self._AtOp(','): self._Next() self._SkipNL() if self._AtOp(')'): break continue break self._ExpectOp(')') else: if self._AtOp('*'): star_tok: Token = self._Next() alias = N.alias(name='*', asname=None) self._FixLoc(alias, star_tok) names.append(alias) else: while True: names.append(self._ParseImportAlias()) if self._AtOp(','): self._Next() continue break node: N.ImportFrom = N.ImportFrom(module=module, names=names, level=level) self._FixLoc(node, start) return node def _ParseImportAlias(self) -> N.alias: """解析单个 import 别名:name [as alias]。 alias 节点位置与 CPython 一致:起点为首个名称 token, 终点为 asname(存在时)或名称结束 token。 """ name_tok: Token = self._ExpectName() asname: str | None = None end_tok: Token = name_tok if self._AtName('as'): self._Next() as_tok: Token = self._ExpectName() asname = as_tok.Value end_tok = as_tok alias: N.alias = N.alias(name=name_tok.Value, asname=asname) self._FixLoc(alias, name_tok, end_tok) return alias def _ParseGlobal(self, kw: str) -> N.AST: start: Token = self._Next() names: list[str] = [] while True: names.append(self._ExpectName().Value) if self._AtOp(','): self._Next() continue break node: N.AST = N.Global(names=names) if kw == 'global' else N.Nonlocal(names=names) self._FixLoc(node, start) return node def _ParseDelete(self) -> N.Delete: start: Token = self._Next() # 'del' targets: list[N.expr] = [] while True: expr: N.expr = self.ParseExpr() self._SetDelContext(expr) targets.append(expr) if self._AtOp(','): self._Next() continue break node: N.Delete = N.Delete(targets=targets) self._FixLoc(node, start) return node def _SetDelContext(self, node: N.AST) -> None: """将 Load 上下文节点递归改为 Del(del 语句目标)。""" if isinstance(node, N.Name): node.ctx = N.Del() elif isinstance(node, N.Attribute): node.ctx = N.Del() elif isinstance(node, N.Subscript): node.ctx = N.Del() elif isinstance(node, (N.Tuple, N.List)): node.ctx = N.Del() for elt in node.elts: self._SetDelContext(elt) def _ParseAssert(self) -> N.Assert: start: Token = self._Next() # 'assert' test: N.expr = self.ParseExpr() msg: N.expr | None = None if self._AtOp(','): self._Next() msg = self.ParseExpr() node: N.Assert = N.Assert(test=test, msg=msg) self._FixLoc(node, start) return node def _ConsumeSemicolons(self) -> None: """消费语句后的分号(允许空语句 ;;)。""" while self._AtOp(';'): self._Next() # ------------------------------------------------------------------ # 表达式语句(含 yield) # ------------------------------------------------------------------ def _ParseExprStmt(self) -> N.stmt: start: Token = self._Peek() # yield 语句:yield [expr_list] if self._AtName('yield'): expr: N.expr = self._ParseYield() else: expr = self.ParseExprList() node: N.Expr = N.Expr(value=expr) self._FixLoc(node, start) return node # ================================================================== # 装饰器 # ================================================================== def _ParseDecorated(self) -> N.stmt: """解析装饰器列表 + def/class。""" decorator_list: list[N.expr] = [] while self._AtOp('@'): deco_start: Token = self._Next() # '@' deco: N.expr = self.ParseExpr() self._FixLoc(deco, deco_start) decorator_list.append(deco) self._ExpectNEWLINE() self._SkipNEWLINE() tok: Token = self._Peek() if tok.Type == NAME and tok.Value == 'def': return self._ParseFunctionDef(decorators=decorator_list) if tok.Type == NAME and tok.Value == 'async': nxt: Token = self._Peek(1) if nxt.Type == NAME and nxt.Value == 'def': return self._ParseFunctionDef(is_async=True, decorators=decorator_list) if tok.Type == NAME and tok.Value == 'class': return self._ParseClassDef(decorators=decorator_list) raise SyntaxError(f"decorator must be followed by def/class, got {tok.Value!r}") # ================================================================== # 函数定义(含软关键字 fn/fun/func/function) # ================================================================== def _ParseFunctionDef(self, is_async: bool = False, decorators: list[N.expr] | None = None) -> N.FunctionDef: start: Token = self._Next() # def/fn/...(或 async) if start.Value == 'async': start = self._Next() # def name_tok: Token = self._ExpectName() # 类型参数(PEP 695):def f[T]( type_params: list[N.AST] = [] if self._AtOp('['): type_params = self._ParseTypeParams() args: N.arguments = self._ParseArguments() returns: N.expr | None = None if self._AtOp('->'): self._Next() returns = self.ParseExpr() self._ExpectOp(':') body: list[N.stmt] = self._ParseSuite() cls: type = N.AsyncFunctionDef if is_async else N.FunctionDef node: N.FunctionDef = cls( name=name_tok.Value, args=args, body=body, decorator_list=decorators or [], returns=returns, type_comment=None, type_params=type_params, ) self._FixLoc(node, start) return node def _ParseTypeParams(self) -> list[N.AST]: """解析 PEP 695 类型参数:[T, *Ts, **P, U: Bound]""" self._ExpectOp('[') self._SkipNL() params: list[N.AST] = [] if not self._AtOp(']'): while True: self._SkipNL() star: int = 0 if self._AtOp('*'): self._Next() star = 1 elif self._AtOp('**'): self._Next() star = 2 name_tok: Token = self._ExpectName() bound: N.expr | None = None if star == 0 and self._AtOp(':'): self._Next() bound = self.ParseExpr() if star == 1: tvt: N.TypeVarTuple = N.TypeVarTuple(name=name_tok.Value) self._FixLoc(tvt, name_tok) params.append(tvt) elif star == 2: ps: N.ParamSpec = N.ParamSpec(name=name_tok.Value) self._FixLoc(ps, name_tok) params.append(ps) else: tv: N.TypeVar = N.TypeVar(name=name_tok.Value, bound=bound) self._FixLoc(tv, name_tok) params.append(tv) self._SkipNL() if self._AtOp(','): self._Next() continue break self._ExpectOp(']') return params # ================================================================== # 类定义 # ================================================================== def _ParseClassDef(self, decorators: list[N.expr] | None = None) -> N.ClassDef: start: Token = self._Next() # class name_tok: Token = self._ExpectName() type_params: list[N.AST] = [] if self._AtOp('['): type_params = self._ParseTypeParams() bases: list[N.expr] = [] keywords: list[N.keyword] = [] if self._AtOp('('): self._Next() self._SkipNL() if not self._AtOp(')'): while True: self._SkipNL() if self._AtOp('**'): self._Next() value: N.expr = self.ParseExpr() kw: N.keyword = N.keyword(arg=None, value=value) self._FixLoc(kw, self._Prev()) keywords.append(kw) elif self._At(NAME) and self._Peek(1).Type == OP and self._Peek(1).Value == '=': arg_tok: Token = self._Next() self._Next() # '=' value = self.ParseExpr() kw = N.keyword(arg=arg_tok.Value, value=value) self._FixLoc(kw, arg_tok) keywords.append(kw) else: bases.append(self.ParseExpr()) self._SkipNL() if self._AtOp(','): self._Next() self._SkipNL() if self._AtOp(')'): break continue break self._ExpectOp(')') self._ExpectOp(':') body: list[N.stmt] = self._ParseSuite() node: N.ClassDef = N.ClassDef( name=name_tok.Value, bases=bases, keywords=keywords, body=body, decorator_list=decorators or [], type_params=type_params, ) self._FixLoc(node, start) return node # ================================================================== # 语句块(套件) # ================================================================== def _ParseSuite(self) -> list[N.stmt]: """解析语句块:':' NEWLINE INDENT ... DEDENT 或单行简单语句。 块首允许出现空行/注释行(NL),跳过后再检测 INDENT。 """ # 已消费 ':' 后的情况:检查是否换行 if self._At(NEWLINE): self._Next() # 跳过空行/注释行 while self._At(NL): self._Next() if self._At(INDENT): self._Next() body: list[N.stmt] = [] self._SkipNEWLINE() while self._Peek().Type != DEDENT and self._Peek().Type != ENDMARKER: stmt: N.stmt = self.ParseStatement() body.append(stmt) # 分号分隔后续语句 while self._AtOp(';'): self._Next() if self._At(NEWLINE): break extra: N.stmt = self.ParseStatement() body.append(extra) self._SkipNEWLINE() self._Expect(DEDENT) return body # 空套件(语法错误,宽松返回空) return [] # 单行套件:statement (; statement)* NEWLINE body = [self.ParseStatement()] while self._AtOp(';'): self._Next() if self._At(NEWLINE): break body.append(self.ParseStatement()) self._ExpectNEWLINE() return body def _Expect(self, tok_type: str, value: str | None = None) -> Token: """期望指定类型的 token。""" tok: Token = self._Next() if tok.Type != tok_type or (value is not None and tok.Value != value): raise SyntaxError( f"expected {tok_type}({value!r}), got {tok.Type}({tok.Value!r}) " f"at line {tok.Line}" ) return tok def _ConsumeNewlines(self) -> None: """消费连续 NEWLINE(空行)。""" while self._At(NEWLINE): self._Next() # ================================================================== # if / while / for / try / with / match # ================================================================== def _ParseIf(self) -> N.If: start: Token = self._Next() # if test: N.expr = self.ParseExpr() self._ExpectOp(':') body: list[N.stmt] = self._ParseSuite() orelse: list[N.stmt] = [] if self._AtName('elif'): # elif → 嵌套 If 于 orelse orelse.append(self._ParseIf()) elif self._AtName('else'): self._Next() self._ExpectOp(':') orelse = self._ParseSuite() node: N.If = N.If(test=test, body=body, orelse=orelse) self._FixLoc(node, start) return node def _ParseWhile(self) -> N.While: start: Token = self._Next() # while test: N.expr = self.ParseExpr() self._ExpectOp(':') body: list[N.stmt] = self._ParseSuite() orelse: list[N.stmt] = [] if self._AtName('else'): self._Next() self._ExpectOp(':') orelse = self._ParseSuite() node: N.While = N.While(test=test, body=body, orelse=orelse) self._FixLoc(node, start) return node def _ParseFor(self, is_async: bool = False) -> N.For: start: Token = self._Next() if start.Value == 'async': start = self._Next() # for target: N.expr = self._ParseTargetList() self._ExpectName('in') iterable: N.expr = self.ParseExpr() self._ExpectOp(':') body: list[N.stmt] = self._ParseSuite() orelse: list[N.stmt] = [] if self._AtName('else'): self._Next() self._ExpectOp(':') orelse = self._ParseSuite() cls: type = N.AsyncFor if is_async else N.For node: N.For = cls(target=target, iter=iterable, body=body, orelse=orelse, type_comment=None) self._FixLoc(node, start) return node def _ParseTargetList(self) -> N.expr: """解析 for 目标列表(支持裸元组 a, b = ...)。""" start: Token = self._Peek() first: N.expr = self._ParseTarget() if self._AtOp(','): elts: list[N.expr] = [first] while self._AtOp(','): self._Next() self._SkipNL() elts.append(self._ParseTarget()) tup: N.Tuple = N.Tuple(elts=elts, ctx=N.Store()) self._FixLoc(tup, start) return tup return first def _ParseTry(self) -> N.Try: start: Token = self._Next() # try self._ExpectOp(':') body: list[N.stmt] = self._ParseSuite() handlers: list[N.ExceptHandler] = [] orelse: list[N.stmt] = [] finalbody: list[N.stmt] = [] is_star: bool = False while self._AtName('except'): self._Next() if self._AtOp('*'): is_star = True self._Next() handlers.append(self._ParseExceptHandler()) if self._AtName('else'): self._Next() self._ExpectOp(':') orelse = self._ParseSuite() if self._AtName('finally'): self._Next() self._ExpectOp(':') finalbody = self._ParseSuite() cls: type = N.TryStar if is_star else N.Try node: N.Try = cls(body=body, handlers=handlers, orelse=orelse, finalbody=finalbody) self._FixLoc(node, start) return node def _ParseExceptHandler(self) -> N.ExceptHandler: start: Token = self._Peek() exc_type: N.expr | None = None name: str | None = None if not self._AtOp(':'): exc_type = self.ParseExpr() if self._AtName('as'): self._Next() name = self._ExpectName().Value self._ExpectOp(':') handler_body: list[N.stmt] = self._ParseSuite() node: N.ExceptHandler = N.ExceptHandler(type=exc_type, name=name, body=handler_body) self._FixLoc(node, start) return node def _ParseWith(self, is_async: bool = False) -> N.With: start: Token = self._Next() if start.Value == 'async': start = self._Next() # with items: list[N.withitem] = [] while True: ctx: N.expr = self.ParseExpr() optional_vars: N.expr | None = None if self._AtName('as'): self._Next() optional_vars = self._ParseTarget() item: N.withitem = N.withitem(context_expr=ctx, optional_vars=optional_vars) self._FixLoc(item, start) items.append(item) if self._AtOp(','): self._Next() continue break self._ExpectOp(':') body: list[N.stmt] = self._ParseSuite() cls: type = N.AsyncWith if is_async else N.With node: N.With = cls(items=items, body=body, type_comment=None) self._FixLoc(node, start) return node # ================================================================== # match 语句(PEP 634) # ================================================================== def _ParseMatch(self) -> N.Match: start: Token = self._Next() # match subject: N.expr = self.ParseExpr() self._ExpectOp(':') self._ExpectNEWLINE() self._Expect(INDENT) cases: list[N.match_case] = [] self._SkipNEWLINE() while self._AtName('case'): case_start: Token = self._Next() # case pattern: N.pattern = self._ParsePattern() guard: N.expr | None = None if self._AtName('if'): self._Next() guard = self.ParseExpr() self._ExpectOp(':') case_body: list[N.stmt] = self._ParseSuite() mc: N.match_case = N.match_case(pattern=pattern, guard=guard, body=case_body) self._FixLoc(mc, case_start) cases.append(mc) self._SkipNEWLINE() self._Expect(DEDENT) node: N.Match = N.Match(subject=subject, cases=cases) self._FixLoc(node, start) return node def _ParsePattern(self) -> N.pattern: """解析 match 模式(顶层,处理 | 或模式)。""" start: Token = self._Peek() patterns: list[N.pattern] = [self._ParsePatternOr()] while self._AtOp('|'): self._Next() patterns.append(self._ParsePatternOr()) if len(patterns) == 1: return patterns[0] node: N.MatchOr = N.MatchOr(patterns=patterns) self._FixLoc(node, start) return node def _ParsePatternOr(self) -> N.pattern: """解析单模式(含 as 绑定)。""" start: Token = self._Peek() pat: N.pattern = self._ParsePatternAtom() if self._AtName('as'): self._Next() name: str = self._ExpectName().Value node: N.MatchAs = N.MatchAs(pattern=pat, name=name) self._FixLoc(node, start) return node return pat def _ParsePatternAtom(self) -> N.pattern: """解析模式原子:字面量/捕获/序列/映射/类/通配符。""" start: Token = self._Peek() tok: Token = self._Peek() # 通配符 / 捕获 if tok.Type == NAME: # 字面量 True/False/None if tok.Value in ('True', 'False', 'None'): self._Next() node: N.MatchSingleton = N.MatchSingleton( value={'True': True, 'False': False, 'None': None}[tok.Value]) self._FixLoc(node, tok) return node # 类模式:Name(...) 或 a.b.c(...)——扫描属性链末尾判断 '(' peek_i: int = 1 while (self._Peek(peek_i).Type == OP and self._Peek(peek_i).Value == '.' and self._Peek(peek_i + 1).Type == NAME): peek_i += 2 if self._Peek(peek_i).Type == OP and self._Peek(peek_i).Value == '(': return self._ParseMatchClass() # 属性值模式 a.b(无括号) if self._Peek(1).Type == OP and self._Peek(1).Value == '.': return self._ParseMatchValueAttr() # 捕获模式(含 _ 通配符;_ 产生空 MatchAs 与 CPython 一致) self._Next() node = N.MatchAs(pattern=None, name=None if tok.Value == '_' else tok.Value) self._FixLoc(node, tok) return node # 序列模式 [a, b] / (a, b) if tok.Type == OP and tok.Value in ('[', '('): return self._ParseMatchSequence() # 映射模式 {k: v, **rest} if tok.Type == OP and tok.Value == '{': return self._ParseMatchMapping() # 值模式:数字/字符串/f-string/带符号字面量(不消费顶层 '|',留给 _ParsePattern 组 MatchOr) if tok.Type == NUMBER or tok.Type == STRING or tok.Type == FSTRING or ( tok.Type == OP and tok.Value in ('-', '+')): value: N.expr = self._ParsePatternValue() node = N.MatchValue(value=value) self._FixLoc(node, start) return node if tok.Type == OP and tok.Value == '*': self._Next() name = None if not self._At(NEWLINE) and not self._AtOp(','): name = self._ExpectName().Value node = N.MatchStar(name=name) self._FixLoc(node, tok) return node raise SyntaxError(f"unexpected pattern token {tok.Type}({tok.Value!r})") def _ParsePatternValue(self) -> N.expr: """解析值模式表达式:带可选 +/- 前缀的字面量。 与 CPython 一致:'|' 是模式或分隔符,这里只解析单个字面量 (数字/字符串/f-string),不消费顶层 '|'。 """ start: Token = self._Peek() tok: Token = self._Peek() if tok.Type == OP and tok.Value in ('-', '+'): op_tok: Token = self._Next() inner: N.expr if self._Peek().Type == NUMBER: inner = self._ParseNumber() elif self._Peek().Type in (STRING, FSTRING): inner = self._ParseStringConcat() else: raise SyntaxError( f"expected literal after '{op_tok.Value}' in pattern at line {op_tok.Line}" ) node: N.expr if op_tok.Value == '-': node = N.UnaryOp(op=N.USub(), operand=inner) else: node = N.UnaryOp(op=N.UAdd(), operand=inner) self._FixLoc(node, start) return node if tok.Type == NUMBER: return self._ParseNumber() return self._ParseStringConcat() def _ParseMatchValueAttr(self) -> N.pattern: """解析值模式属性访问:a.b.c(无括号)。""" start: Token = self._Peek() expr: N.expr = self.ParseExpr() node: N.MatchValue = N.MatchValue(value=expr) self._FixLoc(node, start) return node def _ParseMatchClass(self) -> N.pattern: """解析类模式:Name(pos, kw=v, ...) 或 a.b.c(...)。 类名仅解析 Name/Attribute 链(不消费调用括号),括号由本方法消费。 """ start: Token = self._Peek() cls_name: N.expr = self._ParsePatternClassName() self._ExpectOp('(') self._SkipNL() patterns: list[N.pattern] = [] kwd_attrs: list[str] = [] kwd_patterns: list[N.pattern] = [] if not self._AtOp(')'): while True: self._SkipNL() if self._AtOp('*'): self._Next() patterns.append(self._ParsePattern()) elif self._At(NAME) and self._Peek(1).Type == OP and self._Peek(1).Value == '=': attr_tok: Token = self._Next() self._Next() # '=' kwd_attrs.append(attr_tok.Value) kwd_patterns.append(self._ParsePattern()) else: patterns.append(self._ParsePattern()) self._SkipNL() if self._AtOp(','): self._Next() if self._AtOp(')'): break continue break self._ExpectOp(')') node: N.MatchClass = N.MatchClass(cls=cls_name, patterns=patterns, kwd_attrs=kwd_attrs, kwd_patterns=kwd_patterns) self._FixLoc(node, start) return node def _ParsePatternClassName(self) -> N.expr: """解析类模式的类名(Name / a.b.c 属性链,不含调用括号)。""" start: Token = self._Peek() name_tok: Token = self._ExpectName() node: N.expr = N.Name(id=name_tok.Value, ctx=N.Load()) self._FixLoc(node, name_tok) while self._AtOp('.'): self._Next() attr_tok: Token = self._ExpectName() attr_node: N.Attribute = N.Attribute(value=node, attr=attr_tok.Value, ctx=N.Load()) self._FixLoc(attr_node, start) node = attr_node return node def _ParseMatchSequence(self) -> N.pattern: start: Token = self._Next() # '[' 或 '(' close: str = ']' if start.Value == '[' else ')' patterns: list[N.pattern] = [] self._SkipNL() if not self._AtOp(close): while True: self._SkipNL() patterns.append(self._ParsePattern()) self._SkipNL() if self._AtOp(','): self._Next() if self._AtOp(close): break continue break self._ExpectOp(close) node: N.MatchSequence = N.MatchSequence(patterns=patterns) self._FixLoc(node, start) return node def _ParseMatchMapping(self) -> N.pattern: start: Token = self._Next() # '{' keys: list[N.expr] = [] patterns: list[N.pattern] = [] rest: str | None = None self._SkipNL() if not self._AtOp('}'): while True: self._SkipNL() if self._AtOp('**'): self._Next() rest = self._ExpectName().Value else: key: N.expr = self.ParseExpr() self._ExpectOp(':') keys.append(key) patterns.append(self._ParsePattern()) self._SkipNL() if self._AtOp(','): self._Next() if self._AtOp('}'): break continue break self._ExpectOp('}') node: N.MatchMapping = N.MatchMapping(keys=keys, patterns=patterns, rest=rest) self._FixLoc(node, start) return node def Parse(source: str) -> N.Module: """便捷入口:源码字符串 → Module AST(供 vast.parse 调用)。""" from lib.vast.Tokenizer import Tokenizer tokens = Tokenizer(source).Tokenize() line_starts: list[int] = [0] # 从 token 流重建行起始表(由 Tokenizer 维护,此处直接计算) line_starts = _ComputeLineStarts(source) parser: Parser = Parser(tokens, source, line_starts) return parser.ParseModule() def _ComputeLineStarts(source: str) -> list[int]: """计算源码所有行起始字符索引(含首行 0)。""" starts: list[int] = [0] for i, ch in enumerate(source): if ch == '\n': starts.append(i + 1) return starts