TermiNexus
aa9042173e
- case ClassName.attr(...) 类模式：扫描属性链末尾判断 '('，
此前只查首 token 后 '('，LLVMType.Ptr(...) 被误解析为
MatchValue(Call) → TPC 对 renum match 走值模式分支，对枚举
构造表达式运行时求值生成 ptrtoint SSA，LLVM switch 报
'case value is not a constant integer'
- case 0 | 1 值模式：字面量/符号不再消费顶层 '|'，留给
_ParsePattern 组 MatchOr（此前 ParseExpr 吞掉 '|' 成 BinOp）
1084 lines
42 KiB
Python
1084 lines
42 KiB
Python
"""vast.parser - Viper/Python 语句级语法分析器
|
||
|
||
基于 vast.tokenizer + vast.exprparser，实现与 CPython 3.12 ``ast.parse``
|
||
结构完全一致的语句解析：
|
||
|
||
- 简单语句：赋值/增强赋值/带注解赋值/表达式/import/from-import/global/
|
||
nonlocal/del/assert/pass/break/continue/return/raise
|
||
- 复合语句：if/while/for/try/with/match/def/class/async def-for-with
|
||
- 装饰器、嵌套块（INDENT/DEDENT）、分号分隔、类型注解
|
||
- Viper 软关键字 fn/fun/func/function → FunctionDef（并入解析器，无需
|
||
ExpandSoftKeywords 预处理）
|
||
|
||
产出与 CPython ast 完全一致的节点，含 4 位置属性。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
from lib.vast import Nodes as N
|
||
from lib.vast.ExprParser import ExprParserMixin
|
||
from lib.vast.Tokenizer import (
|
||
Token, NAME, NUMBER, STRING, FSTRING, OP, NEWLINE, NL, INDENT, DEDENT,
|
||
ENDMARKER, SOFT_KEYWORDS,
|
||
)
|
||
|
||
|
||
class Parser(ExprParserMixin):
|
||
"""Viper/Python 完整语法分析器。"""
|
||
|
||
# ==================================================================
|
||
# 模块级入口
|
||
# ==================================================================
|
||
def ParseModule(self) -> N.Module:
|
||
"""解析整个模块：返回 Module(body=..., type_ignores=[])。"""
|
||
body: list[N.stmt] = []
|
||
self._SkipNEWLINE()
|
||
while self._Peek().Type != ENDMARKER:
|
||
stmt: N.stmt = self.ParseStatement()
|
||
body.append(stmt)
|
||
# 分号分隔后续语句（简单语句一行多条）
|
||
while self._AtOp(';'):
|
||
self._Next()
|
||
if self._At(NEWLINE):
|
||
break
|
||
extra: N.stmt = self.ParseStatement()
|
||
body.append(extra)
|
||
self._SkipNEWLINE()
|
||
module: N.Module = N.Module(body=body, type_ignores=[])
|
||
return module
|
||
|
||
# ==================================================================
|
||
# 语句分派
|
||
# ==================================================================
|
||
def ParseStatement(self) -> N.stmt:
|
||
"""解析单条语句（含复合语句分派）。"""
|
||
tok: Token = self._Peek()
|
||
if tok.Type == NAME:
|
||
v: str = tok.Value
|
||
# 软关键字函数定义：fn/fun/func/function + 标识符
|
||
if v in SOFT_KEYWORDS and self._Peek(1).Type == NAME:
|
||
return self._ParseFunctionDef()
|
||
if v == 'async':
|
||
nxt: Token = self._Peek(1)
|
||
if nxt.Type == NAME and nxt.Value == 'def':
|
||
return self._ParseFunctionDef(is_async=True)
|
||
if nxt.Type == NAME and nxt.Value == 'for':
|
||
return self._ParseFor(is_async=True)
|
||
if nxt.Type == NAME and nxt.Value == 'with':
|
||
return self._ParseWith(is_async=True)
|
||
if v == 'def':
|
||
return self._ParseFunctionDef()
|
||
if v == 'class':
|
||
return self._ParseClassDef()
|
||
if v == 'if':
|
||
return self._ParseIf()
|
||
if v == 'while':
|
||
return self._ParseWhile()
|
||
if v == 'for':
|
||
return self._ParseFor()
|
||
if v == 'try':
|
||
return self._ParseTry()
|
||
if v == 'with':
|
||
return self._ParseWith()
|
||
if v == 'match':
|
||
# 软关键字 match：尝试 match 语句，失败回退为表达式
|
||
save: int = self.Idx
|
||
try:
|
||
return self._ParseMatch()
|
||
except SyntaxError:
|
||
self.Idx = save
|
||
if v == 'return':
|
||
return self._ParseReturn()
|
||
if v == 'raise':
|
||
return self._ParseRaise()
|
||
if v == 'import':
|
||
return self._ParseImport()
|
||
if v == 'from':
|
||
return self._ParseImportFrom()
|
||
if v == 'global':
|
||
return self._ParseGlobal('global')
|
||
if v == 'nonlocal':
|
||
return self._ParseGlobal('nonlocal')
|
||
if v == 'del':
|
||
return self._ParseDelete()
|
||
if v == 'assert':
|
||
return self._ParseAssert()
|
||
if v == 'pass':
|
||
self._Next()
|
||
node: N.Pass = N.Pass()
|
||
self._FixLoc(node, tok)
|
||
return node
|
||
if v == 'break':
|
||
self._Next()
|
||
node = N.Break()
|
||
self._FixLoc(node, tok)
|
||
return node
|
||
if v == 'continue':
|
||
self._Next()
|
||
node = N.Continue()
|
||
self._FixLoc(node, tok)
|
||
return node
|
||
if v == 'yield':
|
||
# yield 作为表达式语句
|
||
return self._ParseExprStmt()
|
||
if tok.Type == OP and tok.Value == '@':
|
||
return self._ParseDecorated()
|
||
return self._ParseSimpleStmt()
|
||
|
||
# ==================================================================
|
||
# 简单语句
|
||
# ==================================================================
|
||
def _ParseSimpleStmt(self) -> N.stmt:
|
||
"""解析简单语句（赋值/增强赋值/带注解赋值/表达式）。
|
||
|
||
不消费分号/NEWLINE，由调用方循环（ParseModule/_ParseSuite）统一处理。
|
||
"""
|
||
start: Token = self._Peek()
|
||
# 尝试带注解赋值 x: T [= v]
|
||
ann: N.stmt | None = self._TryParseAnnAssign()
|
||
if ann is not None:
|
||
return ann
|
||
# 表达式/赋值（testlist_star_expr）
|
||
expr: N.expr = self.ParseExprList()
|
||
# 赋值链
|
||
if self._AtOp('='):
|
||
targets: list[N.expr] = [self._AsLoadTarget(expr)]
|
||
while True:
|
||
self._Next() # '='
|
||
value: N.expr = self.ParseExprList()
|
||
if self._AtOp('='):
|
||
targets.append(self._AsLoadTarget(value))
|
||
continue
|
||
break
|
||
node: N.Assign = N.Assign(targets=targets, value=value, type_comment=None)
|
||
self._FixLoc(node, start)
|
||
stmt = node
|
||
elif self._AtOp('+='):
|
||
self._Next()
|
||
value = self.ParseExpr()
|
||
node = N.AugAssign(target=self._AsLoadTarget(expr), op=N.Add(), value=value)
|
||
self._FixLoc(node, start)
|
||
stmt = node
|
||
elif self._AtOp('-='):
|
||
self._Next()
|
||
value = self.ParseExpr()
|
||
node = N.AugAssign(target=self._AsLoadTarget(expr), op=N.Sub(), value=value)
|
||
self._FixLoc(node, start)
|
||
stmt = node
|
||
elif self._AtOp('*='):
|
||
self._Next()
|
||
value = self.ParseExpr()
|
||
node = N.AugAssign(target=self._AsLoadTarget(expr), op=N.Mult(), value=value)
|
||
self._FixLoc(node, start)
|
||
stmt = node
|
||
elif self._AtOp('/='):
|
||
self._Next()
|
||
value = self.ParseExpr()
|
||
node = N.AugAssign(target=self._AsLoadTarget(expr), op=N.Div(), value=value)
|
||
self._FixLoc(node, start)
|
||
stmt = node
|
||
elif self._AtOp('//='):
|
||
self._Next()
|
||
value = self.ParseExpr()
|
||
node = N.AugAssign(target=self._AsLoadTarget(expr), op=N.FloorDiv(), value=value)
|
||
self._FixLoc(node, start)
|
||
stmt = node
|
||
elif self._AtOp('%='):
|
||
self._Next()
|
||
value = self.ParseExpr()
|
||
node = N.AugAssign(target=self._AsLoadTarget(expr), op=N.Mod(), value=value)
|
||
self._FixLoc(node, start)
|
||
stmt = node
|
||
elif self._AtOp('**='):
|
||
self._Next()
|
||
value = self.ParseExpr()
|
||
node = N.AugAssign(target=self._AsLoadTarget(expr), op=N.Pow(), value=value)
|
||
self._FixLoc(node, start)
|
||
stmt = node
|
||
elif self._AtOp('<<='):
|
||
self._Next()
|
||
value = self.ParseExpr()
|
||
node = N.AugAssign(target=self._AsLoadTarget(expr), op=N.LShift(), value=value)
|
||
self._FixLoc(node, start)
|
||
stmt = node
|
||
elif self._AtOp('>>='):
|
||
self._Next()
|
||
value = self.ParseExpr()
|
||
node = N.AugAssign(target=self._AsLoadTarget(expr), op=N.RShift(), value=value)
|
||
self._FixLoc(node, start)
|
||
stmt = node
|
||
elif self._AtOp('&='):
|
||
self._Next()
|
||
value = self.ParseExpr()
|
||
node = N.AugAssign(target=self._AsLoadTarget(expr), op=N.BitAnd(), value=value)
|
||
self._FixLoc(node, start)
|
||
stmt = node
|
||
elif self._AtOp('|='):
|
||
self._Next()
|
||
value = self.ParseExpr()
|
||
node = N.AugAssign(target=self._AsLoadTarget(expr), op=N.BitOr(), value=value)
|
||
self._FixLoc(node, start)
|
||
stmt = node
|
||
elif self._AtOp('^='):
|
||
self._Next()
|
||
value = self.ParseExpr()
|
||
node = N.AugAssign(target=self._AsLoadTarget(expr), op=N.BitXor(), value=value)
|
||
self._FixLoc(node, start)
|
||
stmt = node
|
||
elif self._AtOp('@='):
|
||
self._Next()
|
||
value = self.ParseExpr()
|
||
node = N.AugAssign(target=self._AsLoadTarget(expr), op=N.MatMult(), value=value)
|
||
self._FixLoc(node, start)
|
||
stmt = node
|
||
else:
|
||
node = N.Expr(value=expr)
|
||
self._FixLoc(node, start)
|
||
stmt = node
|
||
return stmt
|
||
|
||
def _AsLoadTarget(self, expr: N.expr) -> N.expr:
|
||
"""将已解析表达式（Load 上下文）转为赋值目标（Store 上下文）。"""
|
||
self._SetStoreContext(expr)
|
||
return expr
|
||
|
||
def _TryParseAnnAssign(self) -> N.AnnAssign | None:
|
||
"""尝试解析带注解赋值：x: T [= v] 或 x: T（目标为 Name/Attribute/Subscript）。
|
||
|
||
注意：仅当目标为简单 Name/Attribute/Subscript 且后随 ':' 时匹配；
|
||
元组/列表解包不做注解（返回 None 走普通路径）。
|
||
"""
|
||
start: Token = self._Peek()
|
||
# 名称
|
||
if self._At(NAME) and self._Peek(1).Type == OP and self._Peek(1).Value == ':':
|
||
name_tok: Token = self._Next()
|
||
self._Next() # ':'
|
||
annotation: N.expr = self.ParseExpr()
|
||
value: N.expr | None = None
|
||
if self._AtOp('='):
|
||
self._Next()
|
||
value = self.ParseExpr()
|
||
target: N.Name = N.Name(id=name_tok.Value, ctx=N.Store())
|
||
self._FixLoc(target, name_tok)
|
||
node: N.AnnAssign = N.AnnAssign(target=target, annotation=annotation,
|
||
value=value, simple=1)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
# 属性/下标目标（无简单解包）
|
||
if self._At(NAME) or (self._At(OP) and self._Peek().Value in ('(', '[')):
|
||
save: int = self.Idx
|
||
try:
|
||
expr: N.expr = self.ParseExpr()
|
||
except SyntaxError:
|
||
self.Idx = save
|
||
return None
|
||
if self._AtOp(':') and isinstance(expr, (N.Name, N.Attribute, N.Subscript)):
|
||
self._Next()
|
||
annotation = self.ParseExpr()
|
||
value = None
|
||
if self._AtOp('='):
|
||
self._Next()
|
||
value = self.ParseExpr()
|
||
self._SetStoreContext(expr)
|
||
simple: int = 1 if isinstance(expr, N.Name) else 0
|
||
node = N.AnnAssign(target=expr, annotation=annotation, value=value, simple=simple)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
self.Idx = save
|
||
return None
|
||
|
||
# ------------------------------------------------------------------
|
||
# return / raise / import / global / del / assert
|
||
# ------------------------------------------------------------------
|
||
def _ParseReturn(self) -> N.Return:
|
||
start: Token = self._Next() # 'return'
|
||
value: N.expr | None = None
|
||
if not self._At(NEWLINE) and not self._AtOp(';'):
|
||
value = self.ParseExprList()
|
||
node: N.Return = N.Return(value=value)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
|
||
def _ParseRaise(self) -> N.Raise:
|
||
start: Token = self._Next() # 'raise'
|
||
exc: N.expr | None = None
|
||
cause: N.expr | None = None
|
||
if not self._At(NEWLINE) and not self._AtOp(';'):
|
||
exc = self.ParseExpr()
|
||
if self._AtName('from'):
|
||
self._Next()
|
||
cause = self.ParseExpr()
|
||
node: N.Raise = N.Raise(exc=exc, cause=cause)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
|
||
def _ParseImport(self) -> N.Import:
|
||
start: Token = self._Next() # 'import'
|
||
names: list[N.alias] = []
|
||
while True:
|
||
# 点分模块名：alias 位置从首个名称起、到末个名称（或 asname）止
|
||
name_tok: Token = self._ExpectName()
|
||
end_tok: Token = name_tok
|
||
name_parts: list[str] = [name_tok.Value]
|
||
while self._AtOp('.'):
|
||
self._Next()
|
||
end_tok = self._ExpectName()
|
||
name_parts.append(end_tok.Value)
|
||
full: str = '.'.join(name_parts)
|
||
asname: str | None = None
|
||
if self._AtName('as'):
|
||
self._Next()
|
||
as_tok: Token = self._ExpectName()
|
||
asname = as_tok.Value
|
||
end_tok = as_tok
|
||
alias: N.alias = N.alias(name=full, asname=asname)
|
||
self._FixLoc(alias, name_tok, end_tok)
|
||
names.append(alias)
|
||
if self._AtOp(','):
|
||
self._Next()
|
||
continue
|
||
break
|
||
node: N.Import = N.Import(names=names)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
|
||
def _ParseImportFrom(self) -> N.ImportFrom:
|
||
start: Token = self._Next() # 'from'
|
||
level: int = 0
|
||
while self._AtOp('.'):
|
||
self._Next()
|
||
level += 1
|
||
module: str | None = None
|
||
# from . import x → 点后直接是 import（无模块名）
|
||
if not self._AtName('import'):
|
||
parts: list[str] = [self._ExpectName().Value]
|
||
while self._AtOp('.'):
|
||
self._Next()
|
||
parts.append(self._ExpectName().Value)
|
||
module = '.'.join(parts)
|
||
self._ExpectName('import')
|
||
names: list[N.alias] = []
|
||
# from x import * | (a, b) | a, b
|
||
if self._AtOp('('):
|
||
self._Next()
|
||
self._SkipNL()
|
||
if not self._AtOp(')'):
|
||
while True:
|
||
names.append(self._ParseImportAlias())
|
||
self._SkipNL()
|
||
if self._AtOp(','):
|
||
self._Next()
|
||
self._SkipNL()
|
||
if self._AtOp(')'):
|
||
break
|
||
continue
|
||
break
|
||
self._ExpectOp(')')
|
||
else:
|
||
if self._AtOp('*'):
|
||
star_tok: Token = self._Next()
|
||
alias = N.alias(name='*', asname=None)
|
||
self._FixLoc(alias, star_tok)
|
||
names.append(alias)
|
||
else:
|
||
while True:
|
||
names.append(self._ParseImportAlias())
|
||
if self._AtOp(','):
|
||
self._Next()
|
||
continue
|
||
break
|
||
node: N.ImportFrom = N.ImportFrom(module=module, names=names, level=level)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
|
||
def _ParseImportAlias(self) -> N.alias:
|
||
"""解析单个 import 别名：name [as alias]。
|
||
|
||
alias 节点位置与 CPython 一致：起点为首个名称 token，
|
||
终点为 asname（存在时）或名称结束 token。
|
||
"""
|
||
name_tok: Token = self._ExpectName()
|
||
asname: str | None = None
|
||
end_tok: Token = name_tok
|
||
if self._AtName('as'):
|
||
self._Next()
|
||
as_tok: Token = self._ExpectName()
|
||
asname = as_tok.Value
|
||
end_tok = as_tok
|
||
alias: N.alias = N.alias(name=name_tok.Value, asname=asname)
|
||
self._FixLoc(alias, name_tok, end_tok)
|
||
return alias
|
||
|
||
def _ParseGlobal(self, kw: str) -> N.AST:
|
||
start: Token = self._Next()
|
||
names: list[str] = []
|
||
while True:
|
||
names.append(self._ExpectName().Value)
|
||
if self._AtOp(','):
|
||
self._Next()
|
||
continue
|
||
break
|
||
node: N.AST = N.Global(names=names) if kw == 'global' else N.Nonlocal(names=names)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
|
||
def _ParseDelete(self) -> N.Delete:
|
||
start: Token = self._Next() # 'del'
|
||
targets: list[N.expr] = []
|
||
while True:
|
||
expr: N.expr = self.ParseExpr()
|
||
self._SetDelContext(expr)
|
||
targets.append(expr)
|
||
if self._AtOp(','):
|
||
self._Next()
|
||
continue
|
||
break
|
||
node: N.Delete = N.Delete(targets=targets)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
|
||
def _SetDelContext(self, node: N.AST) -> None:
|
||
"""将 Load 上下文节点递归改为 Del（del 语句目标）。"""
|
||
if isinstance(node, N.Name):
|
||
node.ctx = N.Del()
|
||
elif isinstance(node, N.Attribute):
|
||
node.ctx = N.Del()
|
||
elif isinstance(node, N.Subscript):
|
||
node.ctx = N.Del()
|
||
elif isinstance(node, (N.Tuple, N.List)):
|
||
node.ctx = N.Del()
|
||
for elt in node.elts:
|
||
self._SetDelContext(elt)
|
||
|
||
def _ParseAssert(self) -> N.Assert:
|
||
start: Token = self._Next() # 'assert'
|
||
test: N.expr = self.ParseExpr()
|
||
msg: N.expr | None = None
|
||
if self._AtOp(','):
|
||
self._Next()
|
||
msg = self.ParseExpr()
|
||
node: N.Assert = N.Assert(test=test, msg=msg)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
|
||
def _ConsumeSemicolons(self) -> None:
|
||
"""消费语句后的分号（允许空语句 ;;）。"""
|
||
while self._AtOp(';'):
|
||
self._Next()
|
||
|
||
# ------------------------------------------------------------------
|
||
# 表达式语句（含 yield）
|
||
# ------------------------------------------------------------------
|
||
def _ParseExprStmt(self) -> N.stmt:
|
||
start: Token = self._Peek()
|
||
# yield 语句：yield [expr_list]
|
||
if self._AtName('yield'):
|
||
expr: N.expr = self._ParseYield()
|
||
else:
|
||
expr = self.ParseExprList()
|
||
node: N.Expr = N.Expr(value=expr)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
|
||
# ==================================================================
|
||
# 装饰器
|
||
# ==================================================================
|
||
def _ParseDecorated(self) -> N.stmt:
|
||
"""解析装饰器列表 + def/class。"""
|
||
decorator_list: list[N.expr] = []
|
||
while self._AtOp('@'):
|
||
deco_start: Token = self._Next() # '@'
|
||
deco: N.expr = self.ParseExpr()
|
||
self._FixLoc(deco, deco_start)
|
||
decorator_list.append(deco)
|
||
self._ExpectNEWLINE()
|
||
self._SkipNEWLINE()
|
||
tok: Token = self._Peek()
|
||
if tok.Type == NAME and tok.Value == 'def':
|
||
return self._ParseFunctionDef(decorators=decorator_list)
|
||
if tok.Type == NAME and tok.Value == 'async':
|
||
nxt: Token = self._Peek(1)
|
||
if nxt.Type == NAME and nxt.Value == 'def':
|
||
return self._ParseFunctionDef(is_async=True, decorators=decorator_list)
|
||
if tok.Type == NAME and tok.Value == 'class':
|
||
return self._ParseClassDef(decorators=decorator_list)
|
||
raise SyntaxError(f"decorator must be followed by def/class, got {tok.Value!r}")
|
||
|
||
# ==================================================================
|
||
# 函数定义（含软关键字 fn/fun/func/function）
|
||
# ==================================================================
|
||
def _ParseFunctionDef(self, is_async: bool = False,
|
||
decorators: list[N.expr] | None = None) -> N.FunctionDef:
|
||
start: Token = self._Next() # def/fn/...（或 async）
|
||
if start.Value == 'async':
|
||
start = self._Next() # def
|
||
name_tok: Token = self._ExpectName()
|
||
# 类型参数（PEP 695）：def f[T](
|
||
type_params: list[N.AST] = []
|
||
if self._AtOp('['):
|
||
type_params = self._ParseTypeParams()
|
||
args: N.arguments = self._ParseArguments()
|
||
returns: N.expr | None = None
|
||
if self._AtOp('->'):
|
||
self._Next()
|
||
returns = self.ParseExpr()
|
||
self._ExpectOp(':')
|
||
body: list[N.stmt] = self._ParseSuite()
|
||
cls: type = N.AsyncFunctionDef if is_async else N.FunctionDef
|
||
node: N.FunctionDef = cls(
|
||
name=name_tok.Value, args=args, body=body,
|
||
decorator_list=decorators or [], returns=returns,
|
||
type_comment=None, type_params=type_params,
|
||
)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
|
||
def _ParseTypeParams(self) -> list[N.AST]:
|
||
"""解析 PEP 695 类型参数：[T, *Ts, **P, U: Bound]"""
|
||
self._ExpectOp('[')
|
||
self._SkipNL()
|
||
params: list[N.AST] = []
|
||
if not self._AtOp(']'):
|
||
while True:
|
||
self._SkipNL()
|
||
star: int = 0
|
||
if self._AtOp('*'):
|
||
self._Next()
|
||
star = 1
|
||
elif self._AtOp('**'):
|
||
self._Next()
|
||
star = 2
|
||
name_tok: Token = self._ExpectName()
|
||
bound: N.expr | None = None
|
||
if star == 0 and self._AtOp(':'):
|
||
self._Next()
|
||
bound = self.ParseExpr()
|
||
if star == 1:
|
||
tvt: N.TypeVarTuple = N.TypeVarTuple(name=name_tok.Value)
|
||
self._FixLoc(tvt, name_tok)
|
||
params.append(tvt)
|
||
elif star == 2:
|
||
ps: N.ParamSpec = N.ParamSpec(name=name_tok.Value)
|
||
self._FixLoc(ps, name_tok)
|
||
params.append(ps)
|
||
else:
|
||
tv: N.TypeVar = N.TypeVar(name=name_tok.Value, bound=bound)
|
||
self._FixLoc(tv, name_tok)
|
||
params.append(tv)
|
||
self._SkipNL()
|
||
if self._AtOp(','):
|
||
self._Next()
|
||
continue
|
||
break
|
||
self._ExpectOp(']')
|
||
return params
|
||
|
||
# ==================================================================
|
||
# 类定义
|
||
# ==================================================================
|
||
def _ParseClassDef(self, decorators: list[N.expr] | None = None) -> N.ClassDef:
|
||
start: Token = self._Next() # class
|
||
name_tok: Token = self._ExpectName()
|
||
type_params: list[N.AST] = []
|
||
if self._AtOp('['):
|
||
type_params = self._ParseTypeParams()
|
||
bases: list[N.expr] = []
|
||
keywords: list[N.keyword] = []
|
||
if self._AtOp('('):
|
||
self._Next()
|
||
self._SkipNL()
|
||
if not self._AtOp(')'):
|
||
while True:
|
||
self._SkipNL()
|
||
if self._AtOp('**'):
|
||
self._Next()
|
||
value: N.expr = self.ParseExpr()
|
||
kw: N.keyword = N.keyword(arg=None, value=value)
|
||
self._FixLoc(kw, self._Prev())
|
||
keywords.append(kw)
|
||
elif self._At(NAME) and self._Peek(1).Type == OP and self._Peek(1).Value == '=':
|
||
arg_tok: Token = self._Next()
|
||
self._Next() # '='
|
||
value = self.ParseExpr()
|
||
kw = N.keyword(arg=arg_tok.Value, value=value)
|
||
self._FixLoc(kw, arg_tok)
|
||
keywords.append(kw)
|
||
else:
|
||
bases.append(self.ParseExpr())
|
||
self._SkipNL()
|
||
if self._AtOp(','):
|
||
self._Next()
|
||
self._SkipNL()
|
||
if self._AtOp(')'):
|
||
break
|
||
continue
|
||
break
|
||
self._ExpectOp(')')
|
||
self._ExpectOp(':')
|
||
body: list[N.stmt] = self._ParseSuite()
|
||
node: N.ClassDef = N.ClassDef(
|
||
name=name_tok.Value, bases=bases, keywords=keywords, body=body,
|
||
decorator_list=decorators or [], type_params=type_params,
|
||
)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
|
||
# ==================================================================
|
||
# 语句块（套件）
|
||
# ==================================================================
|
||
def _ParseSuite(self) -> list[N.stmt]:
|
||
"""解析语句块：':' NEWLINE INDENT ... DEDENT 或单行简单语句。
|
||
|
||
块首允许出现空行/注释行（NL），跳过后再检测 INDENT。
|
||
"""
|
||
# 已消费 ':' 后的情况：检查是否换行
|
||
if self._At(NEWLINE):
|
||
self._Next()
|
||
# 跳过空行/注释行
|
||
while self._At(NL):
|
||
self._Next()
|
||
if self._At(INDENT):
|
||
self._Next()
|
||
body: list[N.stmt] = []
|
||
self._SkipNEWLINE()
|
||
while self._Peek().Type != DEDENT and self._Peek().Type != ENDMARKER:
|
||
stmt: N.stmt = self.ParseStatement()
|
||
body.append(stmt)
|
||
# 分号分隔后续语句
|
||
while self._AtOp(';'):
|
||
self._Next()
|
||
if self._At(NEWLINE):
|
||
break
|
||
extra: N.stmt = self.ParseStatement()
|
||
body.append(extra)
|
||
self._SkipNEWLINE()
|
||
self._Expect(DEDENT)
|
||
return body
|
||
# 空套件（语法错误，宽松返回空）
|
||
return []
|
||
# 单行套件：statement (; statement)* NEWLINE
|
||
body = [self.ParseStatement()]
|
||
while self._AtOp(';'):
|
||
self._Next()
|
||
if self._At(NEWLINE):
|
||
break
|
||
body.append(self.ParseStatement())
|
||
self._ExpectNEWLINE()
|
||
return body
|
||
|
||
def _Expect(self, tok_type: str, value: str | None = None) -> Token:
|
||
"""期望指定类型的 token。"""
|
||
tok: Token = self._Next()
|
||
if tok.Type != tok_type or (value is not None and tok.Value != value):
|
||
raise SyntaxError(
|
||
f"expected {tok_type}({value!r}), got {tok.Type}({tok.Value!r}) "
|
||
f"at line {tok.Line}"
|
||
)
|
||
return tok
|
||
|
||
def _ConsumeNewlines(self) -> None:
|
||
"""消费连续 NEWLINE（空行）。"""
|
||
while self._At(NEWLINE):
|
||
self._Next()
|
||
|
||
# ==================================================================
|
||
# if / while / for / try / with / match
|
||
# ==================================================================
|
||
def _ParseIf(self) -> N.If:
|
||
start: Token = self._Next() # if
|
||
test: N.expr = self.ParseExpr()
|
||
self._ExpectOp(':')
|
||
body: list[N.stmt] = self._ParseSuite()
|
||
orelse: list[N.stmt] = []
|
||
if self._AtName('elif'):
|
||
# elif → 嵌套 If 于 orelse
|
||
orelse.append(self._ParseIf())
|
||
elif self._AtName('else'):
|
||
self._Next()
|
||
self._ExpectOp(':')
|
||
orelse = self._ParseSuite()
|
||
node: N.If = N.If(test=test, body=body, orelse=orelse)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
|
||
def _ParseWhile(self) -> N.While:
|
||
start: Token = self._Next() # while
|
||
test: N.expr = self.ParseExpr()
|
||
self._ExpectOp(':')
|
||
body: list[N.stmt] = self._ParseSuite()
|
||
orelse: list[N.stmt] = []
|
||
if self._AtName('else'):
|
||
self._Next()
|
||
self._ExpectOp(':')
|
||
orelse = self._ParseSuite()
|
||
node: N.While = N.While(test=test, body=body, orelse=orelse)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
|
||
def _ParseFor(self, is_async: bool = False) -> N.For:
|
||
start: Token = self._Next()
|
||
if start.Value == 'async':
|
||
start = self._Next() # for
|
||
target: N.expr = self._ParseTargetList()
|
||
self._ExpectName('in')
|
||
iterable: N.expr = self.ParseExpr()
|
||
self._ExpectOp(':')
|
||
body: list[N.stmt] = self._ParseSuite()
|
||
orelse: list[N.stmt] = []
|
||
if self._AtName('else'):
|
||
self._Next()
|
||
self._ExpectOp(':')
|
||
orelse = self._ParseSuite()
|
||
cls: type = N.AsyncFor if is_async else N.For
|
||
node: N.For = cls(target=target, iter=iterable, body=body, orelse=orelse,
|
||
type_comment=None)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
|
||
def _ParseTargetList(self) -> N.expr:
|
||
"""解析 for 目标列表（支持裸元组 a, b = ...）。"""
|
||
start: Token = self._Peek()
|
||
first: N.expr = self._ParseTarget()
|
||
if self._AtOp(','):
|
||
elts: list[N.expr] = [first]
|
||
while self._AtOp(','):
|
||
self._Next()
|
||
self._SkipNL()
|
||
elts.append(self._ParseTarget())
|
||
tup: N.Tuple = N.Tuple(elts=elts, ctx=N.Store())
|
||
self._FixLoc(tup, start)
|
||
return tup
|
||
return first
|
||
|
||
def _ParseTry(self) -> N.Try:
|
||
start: Token = self._Next() # try
|
||
self._ExpectOp(':')
|
||
body: list[N.stmt] = self._ParseSuite()
|
||
handlers: list[N.ExceptHandler] = []
|
||
orelse: list[N.stmt] = []
|
||
finalbody: list[N.stmt] = []
|
||
is_star: bool = False
|
||
while self._AtName('except'):
|
||
self._Next()
|
||
if self._AtOp('*'):
|
||
is_star = True
|
||
self._Next()
|
||
handlers.append(self._ParseExceptHandler())
|
||
if self._AtName('else'):
|
||
self._Next()
|
||
self._ExpectOp(':')
|
||
orelse = self._ParseSuite()
|
||
if self._AtName('finally'):
|
||
self._Next()
|
||
self._ExpectOp(':')
|
||
finalbody = self._ParseSuite()
|
||
cls: type = N.TryStar if is_star else N.Try
|
||
node: N.Try = cls(body=body, handlers=handlers, orelse=orelse, finalbody=finalbody)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
|
||
def _ParseExceptHandler(self) -> N.ExceptHandler:
|
||
start: Token = self._Peek()
|
||
exc_type: N.expr | None = None
|
||
name: str | None = None
|
||
if not self._AtOp(':'):
|
||
exc_type = self.ParseExpr()
|
||
if self._AtName('as'):
|
||
self._Next()
|
||
name = self._ExpectName().Value
|
||
self._ExpectOp(':')
|
||
handler_body: list[N.stmt] = self._ParseSuite()
|
||
node: N.ExceptHandler = N.ExceptHandler(type=exc_type, name=name, body=handler_body)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
|
||
def _ParseWith(self, is_async: bool = False) -> N.With:
|
||
start: Token = self._Next()
|
||
if start.Value == 'async':
|
||
start = self._Next() # with
|
||
items: list[N.withitem] = []
|
||
while True:
|
||
ctx: N.expr = self.ParseExpr()
|
||
optional_vars: N.expr | None = None
|
||
if self._AtName('as'):
|
||
self._Next()
|
||
optional_vars = self._ParseTarget()
|
||
item: N.withitem = N.withitem(context_expr=ctx, optional_vars=optional_vars)
|
||
self._FixLoc(item, start)
|
||
items.append(item)
|
||
if self._AtOp(','):
|
||
self._Next()
|
||
continue
|
||
break
|
||
self._ExpectOp(':')
|
||
body: list[N.stmt] = self._ParseSuite()
|
||
cls: type = N.AsyncWith if is_async else N.With
|
||
node: N.With = cls(items=items, body=body, type_comment=None)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
|
||
# ==================================================================
|
||
# match 语句（PEP 634）
|
||
# ==================================================================
|
||
def _ParseMatch(self) -> N.Match:
|
||
start: Token = self._Next() # match
|
||
subject: N.expr = self.ParseExpr()
|
||
self._ExpectOp(':')
|
||
self._ExpectNEWLINE()
|
||
self._Expect(INDENT)
|
||
cases: list[N.match_case] = []
|
||
self._SkipNEWLINE()
|
||
while self._AtName('case'):
|
||
case_start: Token = self._Next() # case
|
||
pattern: N.pattern = self._ParsePattern()
|
||
guard: N.expr | None = None
|
||
if self._AtName('if'):
|
||
self._Next()
|
||
guard = self.ParseExpr()
|
||
self._ExpectOp(':')
|
||
case_body: list[N.stmt] = self._ParseSuite()
|
||
mc: N.match_case = N.match_case(pattern=pattern, guard=guard, body=case_body)
|
||
self._FixLoc(mc, case_start)
|
||
cases.append(mc)
|
||
self._SkipNEWLINE()
|
||
self._Expect(DEDENT)
|
||
node: N.Match = N.Match(subject=subject, cases=cases)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
|
||
def _ParsePattern(self) -> N.pattern:
|
||
"""解析 match 模式（顶层，处理 | 或模式）。"""
|
||
start: Token = self._Peek()
|
||
patterns: list[N.pattern] = [self._ParsePatternOr()]
|
||
while self._AtOp('|'):
|
||
self._Next()
|
||
patterns.append(self._ParsePatternOr())
|
||
if len(patterns) == 1:
|
||
return patterns[0]
|
||
node: N.MatchOr = N.MatchOr(patterns=patterns)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
|
||
def _ParsePatternOr(self) -> N.pattern:
|
||
"""解析单模式（含 as 绑定）。"""
|
||
start: Token = self._Peek()
|
||
pat: N.pattern = self._ParsePatternAtom()
|
||
if self._AtName('as'):
|
||
self._Next()
|
||
name: str = self._ExpectName().Value
|
||
node: N.MatchAs = N.MatchAs(pattern=pat, name=name)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
return pat
|
||
|
||
def _ParsePatternAtom(self) -> N.pattern:
|
||
"""解析模式原子：字面量/捕获/序列/映射/类/通配符。"""
|
||
start: Token = self._Peek()
|
||
tok: Token = self._Peek()
|
||
# 通配符 / 捕获
|
||
if tok.Type == NAME:
|
||
# 字面量 True/False/None
|
||
if tok.Value in ('True', 'False', 'None'):
|
||
self._Next()
|
||
node: N.MatchSingleton = N.MatchSingleton(
|
||
value={'True': True, 'False': False, 'None': None}[tok.Value])
|
||
self._FixLoc(node, tok)
|
||
return node
|
||
# 类模式：Name(...) 或 a.b.c(...)——扫描属性链末尾判断 '('
|
||
peek_i: int = 1
|
||
while (self._Peek(peek_i).Type == OP and self._Peek(peek_i).Value == '.'
|
||
and self._Peek(peek_i + 1).Type == NAME):
|
||
peek_i += 2
|
||
if self._Peek(peek_i).Type == OP and self._Peek(peek_i).Value == '(':
|
||
return self._ParseMatchClass()
|
||
# 属性值模式 a.b（无括号）
|
||
if self._Peek(1).Type == OP and self._Peek(1).Value == '.':
|
||
return self._ParseMatchValueAttr()
|
||
# 捕获模式（含 _ 通配符；_ 产生空 MatchAs 与 CPython 一致）
|
||
self._Next()
|
||
node = N.MatchAs(pattern=None, name=None if tok.Value == '_' else tok.Value)
|
||
self._FixLoc(node, tok)
|
||
return node
|
||
# 序列模式 [a, b] / (a, b)
|
||
if tok.Type == OP and tok.Value in ('[', '('):
|
||
return self._ParseMatchSequence()
|
||
# 映射模式 {k: v, **rest}
|
||
if tok.Type == OP and tok.Value == '{':
|
||
return self._ParseMatchMapping()
|
||
# 值模式：数字/字符串/f-string/带符号字面量（不消费顶层 '|'，留给 _ParsePattern 组 MatchOr）
|
||
if tok.Type == NUMBER or tok.Type == STRING or tok.Type == FSTRING or (
|
||
tok.Type == OP and tok.Value in ('-', '+')):
|
||
value: N.expr = self._ParsePatternValue()
|
||
node = N.MatchValue(value=value)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
if tok.Type == OP and tok.Value == '*':
|
||
self._Next()
|
||
name = None
|
||
if not self._At(NEWLINE) and not self._AtOp(','):
|
||
name = self._ExpectName().Value
|
||
node = N.MatchStar(name=name)
|
||
self._FixLoc(node, tok)
|
||
return node
|
||
raise SyntaxError(f"unexpected pattern token {tok.Type}({tok.Value!r})")
|
||
|
||
def _ParsePatternValue(self) -> N.expr:
|
||
"""解析值模式表达式：带可选 +/- 前缀的字面量。
|
||
|
||
与 CPython 一致：'|' 是模式或分隔符，这里只解析单个字面量
|
||
（数字/字符串/f-string），不消费顶层 '|'。
|
||
"""
|
||
start: Token = self._Peek()
|
||
tok: Token = self._Peek()
|
||
if tok.Type == OP and tok.Value in ('-', '+'):
|
||
op_tok: Token = self._Next()
|
||
inner: N.expr
|
||
if self._Peek().Type == NUMBER:
|
||
inner = self._ParseNumber()
|
||
elif self._Peek().Type in (STRING, FSTRING):
|
||
inner = self._ParseStringConcat()
|
||
else:
|
||
raise SyntaxError(
|
||
f"expected literal after '{op_tok.Value}' in pattern at line {op_tok.Line}"
|
||
)
|
||
node: N.expr
|
||
if op_tok.Value == '-':
|
||
node = N.UnaryOp(op=N.USub(), operand=inner)
|
||
else:
|
||
node = N.UnaryOp(op=N.UAdd(), operand=inner)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
if tok.Type == NUMBER:
|
||
return self._ParseNumber()
|
||
return self._ParseStringConcat()
|
||
|
||
def _ParseMatchValueAttr(self) -> N.pattern:
|
||
"""解析值模式属性访问：a.b.c（无括号）。"""
|
||
start: Token = self._Peek()
|
||
expr: N.expr = self.ParseExpr()
|
||
node: N.MatchValue = N.MatchValue(value=expr)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
|
||
def _ParseMatchClass(self) -> N.pattern:
|
||
"""解析类模式：Name(pos, kw=v, ...) 或 a.b.c(...)。
|
||
|
||
类名仅解析 Name/Attribute 链（不消费调用括号），括号由本方法消费。
|
||
"""
|
||
start: Token = self._Peek()
|
||
cls_name: N.expr = self._ParsePatternClassName()
|
||
self._ExpectOp('(')
|
||
self._SkipNL()
|
||
patterns: list[N.pattern] = []
|
||
kwd_attrs: list[str] = []
|
||
kwd_patterns: list[N.pattern] = []
|
||
if not self._AtOp(')'):
|
||
while True:
|
||
self._SkipNL()
|
||
if self._AtOp('*'):
|
||
self._Next()
|
||
patterns.append(self._ParsePattern())
|
||
elif self._At(NAME) and self._Peek(1).Type == OP and self._Peek(1).Value == '=':
|
||
attr_tok: Token = self._Next()
|
||
self._Next() # '='
|
||
kwd_attrs.append(attr_tok.Value)
|
||
kwd_patterns.append(self._ParsePattern())
|
||
else:
|
||
patterns.append(self._ParsePattern())
|
||
self._SkipNL()
|
||
if self._AtOp(','):
|
||
self._Next()
|
||
if self._AtOp(')'):
|
||
break
|
||
continue
|
||
break
|
||
self._ExpectOp(')')
|
||
node: N.MatchClass = N.MatchClass(cls=cls_name, patterns=patterns,
|
||
kwd_attrs=kwd_attrs, kwd_patterns=kwd_patterns)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
|
||
def _ParsePatternClassName(self) -> N.expr:
|
||
"""解析类模式的类名（Name / a.b.c 属性链，不含调用括号）。"""
|
||
start: Token = self._Peek()
|
||
name_tok: Token = self._ExpectName()
|
||
node: N.expr = N.Name(id=name_tok.Value, ctx=N.Load())
|
||
self._FixLoc(node, name_tok)
|
||
while self._AtOp('.'):
|
||
self._Next()
|
||
attr_tok: Token = self._ExpectName()
|
||
attr_node: N.Attribute = N.Attribute(value=node, attr=attr_tok.Value, ctx=N.Load())
|
||
self._FixLoc(attr_node, start)
|
||
node = attr_node
|
||
return node
|
||
|
||
def _ParseMatchSequence(self) -> N.pattern:
|
||
start: Token = self._Next() # '[' 或 '('
|
||
close: str = ']' if start.Value == '[' else ')'
|
||
patterns: list[N.pattern] = []
|
||
self._SkipNL()
|
||
if not self._AtOp(close):
|
||
while True:
|
||
self._SkipNL()
|
||
patterns.append(self._ParsePattern())
|
||
self._SkipNL()
|
||
if self._AtOp(','):
|
||
self._Next()
|
||
if self._AtOp(close):
|
||
break
|
||
continue
|
||
break
|
||
self._ExpectOp(close)
|
||
node: N.MatchSequence = N.MatchSequence(patterns=patterns)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
|
||
def _ParseMatchMapping(self) -> N.pattern:
|
||
start: Token = self._Next() # '{'
|
||
keys: list[N.expr] = []
|
||
patterns: list[N.pattern] = []
|
||
rest: str | None = None
|
||
self._SkipNL()
|
||
if not self._AtOp('}'):
|
||
while True:
|
||
self._SkipNL()
|
||
if self._AtOp('**'):
|
||
self._Next()
|
||
rest = self._ExpectName().Value
|
||
else:
|
||
key: N.expr = self.ParseExpr()
|
||
self._ExpectOp(':')
|
||
keys.append(key)
|
||
patterns.append(self._ParsePattern())
|
||
self._SkipNL()
|
||
if self._AtOp(','):
|
||
self._Next()
|
||
if self._AtOp('}'):
|
||
break
|
||
continue
|
||
break
|
||
self._ExpectOp('}')
|
||
node: N.MatchMapping = N.MatchMapping(keys=keys, patterns=patterns, rest=rest)
|
||
self._FixLoc(node, start)
|
||
return node
|
||
|
||
|
||
def Parse(source: str) -> N.Module:
|
||
"""便捷入口：源码字符串 → Module AST（供 vast.parse 调用）。"""
|
||
from lib.vast.Tokenizer import Tokenizer
|
||
|
||
tokens = Tokenizer(source).Tokenize()
|
||
line_starts: list[int] = [0]
|
||
# 从 token 流重建行起始表（由 Tokenizer 维护，此处直接计算）
|
||
line_starts = _ComputeLineStarts(source)
|
||
parser: Parser = Parser(tokens, source, line_starts)
|
||
return parser.ParseModule()
|
||
|
||
|
||
def _ComputeLineStarts(source: str) -> list[int]:
|
||
"""计算源码所有行起始字符索引（含首行 0）。"""
|
||
starts: list[int] = [0]
|
||
for i, ch in enumerate(source):
|
||
if ch == '\n':
|
||
starts.append(i + 1)
|
||
return starts
|