GVSDS/TransPyC
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
efb4aada15040928da5ed7e81ebae2a13911d579
TransPyC/includes/ast/parser.vp
T

2444 lines
89 KiB
Plaintext
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import t, c
from stdint import *
import memhub
import string
import stdio
import stdlib
from .tokens import (Token, TokenType, Keyword, TokOp)
from .base import (
AST, ASTCtx, ASTKind, OpKind, ASTFlag,
_set_pos, _binop_from_op, _augop_from_op, _cmpop_from_op, _copy_str, _inherit_pos,
_init_ast,
CONST_INT, CONST_FLOAT, CONST_STR, CONST_BOOL, CONST_NONE, CONST_CHAR,
FLAG_IS_ASYNC,
)
from .exprs import (
BoolOp, BinOp, UnaryOp, Lambda, IfExp, Dict, Set,
ListComp, SetComp, DictComp, GeneratorExp,
Await, Yield, YieldFrom, FormattedValue, JoinedStr,
Constant, NamedExpr, Attribute, Subscript, Starred, Name,
List, Tuple, Slice, Call, Compare, OpNode,
)
from .stmts import (
Module, Expression, Interactive, FunctionType,
FunctionDef, ClassDef, Return, Delete, Assign, AugAssign, AnnAssign,
For, While, If, With, Raise, Try, Assert, Global, Nonlocal,
Pass, Break, Continue, Expr, Import, ImportFrom, Match,
)
from .astaux import (
ExceptHandler, Arguments, Arg, Keyword as KwNode, Alias, WithItem, Comprehension,
)
from .match import (
MatchCase, MatchValue, MatchSingleton, MatchSequence,
MatchMapping, MatchClass, MatchStar, MatchAs, MatchOr,
)
from .lexer import Lexer, _lexer_init, tokenize, new_lexer
# ============================================================
# Parser State
# ============================================================
class Parser:
tokens: Token | t.CPtr # token linked list head
cur: Token | t.CPtr # current token
pool: memhub.MemManager | t.CPtr # memory pool
error_count: t.CInt # error count
pending_decorators: list[AST | t.CPtr] | t.CPtr # pending decorators list
def __new__(self, pool: memhub.MemManager | t.CPtr):
ptr: Parser | t.CPtr = pool.alloc(Parser.__sizeof__())
if ptr:
string.memset(ptr, 0, Parser.__sizeof__())
return ptr
def new_parser(pool: memhub.MemManager | t.CPtr) -> Parser | t.CPtr:
"""Allocate from the pool and return the Parser (cross-module safe wrapper)"""
return Parser(pool)
def _parser_init(ps: Parser | t.CPtr, tokens: Token | t.CPtr,
pool: memhub.MemManager | t.CPtr):
if ps is None: return
ps.tokens = tokens
ps.cur = tokens
ps.pool = pool
ps.error_count = 0
ps.pending_decorators = None
# ============================================================
# Token Navigation Helpers
# ============================================================
def _advance(ps: Parser | t.CPtr):
"""Advance to the next token"""
if ps.cur is None: return
if ps.cur.next is not None:
ps.cur = ps.cur.next
def _cur_type(ps: Parser | t.CPtr) -> t.CInt:
if ps.cur is None: return TokenType.EndMarker
return ps.cur.type
def _cur_op(ps: Parser | t.CPtr) -> t.CInt:
if ps.cur is None: return 0
return ps.cur.op_subtype
def _cur_kw(ps: Parser | t.CPtr) -> t.CInt:
if ps.cur is None: return 0
return ps.cur.kw_subtype
def _cur_str(ps: Parser | t.CPtr) -> str:
if ps.cur is None: return None
return ps.cur.str_val
def _cur_lineno(ps: Parser | t.CPtr) -> t.CInt:
if ps.cur is None: return 0
return ps.cur.lineno
def _cur_col(ps: Parser | t.CPtr) -> t.CInt:
if ps.cur is None: return 0
return ps.cur.col_offset
def _skip_nl(ps: Parser | t.CPtr):
"""Skip TokenType.Nl token"""
while ps.cur is not None and ps.cur.type == TokenType.Nl:
ps.cur = ps.cur.next
def _skip_nls_and_newlines(ps: Parser | t.CPtr):
"""Skip TokenType.Nl and TokenType.NewLine (used for line continuation)"""
while ps.cur is not None:
if ps.cur.type == TokenType.Nl or ps.cur.type == TokenType.NewLine:
ps.cur = ps.cur.next
else:
break
def _expect_type(ps: Parser | t.CPtr, ttype: t.CInt) -> t.CInt:
"""Expect the current token to be of the specified type, advance and return 1, otherwise return 0"""
if _cur_type(ps) == ttype:
_advance(ps)
return 1
ps.error_count += 1
return 0
def _expect_op(ps: Parser | t.CPtr, op_id: t.CInt) -> t.CInt:
"""Expecting the current one to be some operator"""
if _cur_type(ps) == TokenType.Op and _cur_op(ps) == op_id:
_advance(ps)
return 1
ps.error_count += 1
return 0
def _match_op(ps: Parser | t.CPtr, op_id: t.CInt) -> t.CInt:
"""Check but not consume the current token"""
if _cur_type(ps) == TokenType.Op and _cur_op(ps) == op_id:
return 1
return 0
def _accept_op(ps: Parser | t.CPtr, op_id: t.CInt) -> t.CInt:
"""If match, consume and return 1"""
if _match_op(ps, op_id):
_advance(ps)
return 1
return 0
# ============================================================
# Constant Construction Helper (New API: Constant directly accepts lineno/col)
# ============================================================
def _new_const_int(pool: memhub.MemManager | t.CPtr, val: t.CInt64T,
lineno: t.CInt, col: t.CInt) -> AST | t.CPtr:
return Constant(pool, CONST_INT, val, 0.0, None, lineno, col)
def _new_const_float(pool: memhub.MemManager | t.CPtr, val: t.CDouble,
lineno: t.CInt, col: t.CInt) -> AST | t.CPtr:
global _g_dbg_pts, _g_dbg_vals, _g_dbg_cnt
new_c: Constant | t.CPtr = Constant(pool, CONST_FLOAT, 0, val, None, lineno, col)
# OVW-DBG: 检查之前构造的浮点节点是否已被覆写（定位覆写者）
if _g_dbg_cnt > 0 and _g_dbg_pts is not None and _g_dbg_vals is not None:
chk: t.CSizeT = 0
while chk < _g_dbg_cnt:
old_ptr: t.CUInt64T = _g_dbg_pts[chk]
if old_ptr != 0:
old_val: t.CDouble = _g_dbg_vals[chk]
old_c: Constant | t.CPtr = (Constant | t.CPtr)(t.CVoid(old_ptr, t.CPtr))
if old_c.float_val != old_val:
stdio.printf("[OVW-DBG] 构造新字面量 val=%.17g 后，之前节点 ptr=%p 的 float_val 从 %.17g 变 %.17g\n",
val, t.CUInt64T(old_ptr), old_val, old_c.float_val)
_g_dbg_pts[chk] = 0
chk += 1
# 记录当前节点
if _g_dbg_pts is None:
_g_dbg_pts = stdlib.malloc(512 * 8)
_g_dbg_vals = stdlib.malloc(512 * 8)
if _g_dbg_cnt < 512 and _g_dbg_pts is not None and _g_dbg_vals is not None:
_g_dbg_pts[_g_dbg_cnt] = t.CUInt64T(new_c)
_g_dbg_vals[_g_dbg_cnt] = val
_g_dbg_cnt += 1
# NEWF-DBG: 打印浮点常量构造值+节点指针+存储后读回（定位浮点 CDefine 值错误）
stdio.printf("[NEWF-DBG] val=%.17g ptr=%p readback=%.17g int_val=%lld\n",
val, t.CSizeT(new_c), new_c.float_val, new_c.int_val)
return new_c
# OVW-DBG: 浮点节点监控表（stdlib 分配，避免污染 pool）
_g_dbg_pts: t.CUInt64T | t.CPtr = None
_g_dbg_vals: t.CDouble | t.CPtr = None
_g_dbg_cnt: int = 0
def _new_const_str(pool: memhub.MemManager | t.CPtr, val: str,
lineno: t.CInt, col: t.CInt,
is_dquote: t.CInt = 1,
is_triple: t.CInt = 0,
is_char: t.CInt = 0) -> AST | t.CPtr:
"""Create a string/character constant node
is_dquote: 0 = single quote, non-0 = double quote (default double quote)
is_triple: non-0 = triple-quoted string
is_char: non-0 = single-character candidate (lexer has confirmed that after escaping it's exactly 1 byte, including '0')
Semantic distinctions:
- 'x' (single quote, single character, not triple-quoted) → CONST_CHAR (i8 value)
- "x" (double quote, includes 0 terminator) → CONST_STR (i8* pointer, i8 x 2)
- 'ab' (single quotes, multiple characters) → CONST_STR (i8* pointer)
- '''x''' / \"\"\"x\"\"\" (triple quotes) → CONST_STR (i8* pointer)
- '' / "" (empty string) → CONST_STR (i8* pointer)
"""
# Single quotes, non-triple quotes, single-character candidates marked by lexer → CONST_CHAR
# Mark with is_char instead of strlen(val), because '0' becomes a NUL byte after escaping, strlen would return 0
if is_dquote == 0 and is_triple == 0 and is_char != 0:
# Single quote # Single character → CONST_CHAR, int_val stores the character's ASCII value
# val[0] is a legal value of 0 even if it's NUL
ch_val: t.CInt64T = val[0]
return Constant(pool, CONST_CHAR, ch_val, 0.0, val, lineno, col)
# Other situations → CONST_STR
return Constant(pool, CONST_STR, 0, 0.0, val, lineno, col)
def _new_const_bool(pool: memhub.MemManager | t.CPtr, val: t.CInt,
lineno: t.CInt, col: t.CInt) -> AST | t.CPtr:
return Constant(pool, CONST_BOOL, val, 0.0, None, lineno, col)
def _new_const_none(pool: memhub.MemManager | t.CPtr,
lineno: t.CInt, col: t.CInt) -> AST | t.CPtr:
return Constant(pool, CONST_NONE, 0, 0.0, None, lineno, col)
# ============================================================
# ctx Helper (Recursive Fixing Store Context)
# ============================================================
def _set_ctx(node: AST | t.CPtr, ctx: t.CInt):
"""Set ctx field of node (based on kind conversion)"""
if node is None: return
k: t.CInt = node.kind()
if k == ASTKind.Name:
nm: Name | t.CPtr = (Name | t.CPtr)(node)
nm.ctx = ctx
elif k == ASTKind.Attribute:
at: Attribute | t.CPtr = (Attribute | t.CPtr)(node)
at.ctx = ctx
elif k == ASTKind.Subscript:
sb: Subscript | t.CPtr = (Subscript | t.CPtr)(node)
sb.ctx = ctx
elif k == ASTKind.Tuple:
tp: Tuple | t.CPtr = (Tuple | t.CPtr)(node)
tp.ctx = ctx
elif k == ASTKind.List:
ls: List | t.CPtr = (List | t.CPtr)(node)
ls.ctx = ctx
elif k == ASTKind.Starred:
st: Starred | t.CPtr = (Starred | t.CPtr)(node)
st.ctx = ctx
def _fix_store_ctx(node: AST | t.CPtr):
"""Fix ctx of store targets to be Store"""
if node is None: return
_set_ctx(node, ASTCtx.Store)
k: t.CInt = node.kind()
if k == ASTKind.Tuple or k == ASTKind.List:
tp_elts: list[AST | t.CPtr] | t.CPtr = None
if k == ASTKind.Tuple:
tp: Tuple | t.CPtr = (Tuple | t.CPtr)(node)
tp_elts = tp.elts
else:
ls: List | t.CPtr = (List | t.CPtr)(node)
tp_elts = ls.elts
if tp_elts is not None:
n: t.CSizeT = tp_elts.__len__()
i: t.CSizeT = 0
while i < n:
cur: AST | t.CPtr = tp_elts.get(i)
_fix_store_ctx(cur)
i += 1
elif k == ASTKind.Starred:
st2: Starred | t.CPtr = (Starred | t.CPtr)(node)
_fix_store_ctx(st2.value)
# ============================================================
# Expression Parser
# ============================================================
def _parse_expr(ps: Parser | t.CPtr) -> AST | t.CPtr:
"""expr: yield_expr | namedexpr_test"""
if _cur_kw(ps) == Keyword.Yield:
return _parse_yield(ps)
return _parse_namedexpr(ps)
def _parse_namedexpr(ps: Parser | t.CPtr) -> AST | t.CPtr:
"""namedexpr_test: test [':=' test]"""
left: AST | t.CPtr = _parse_test(ps)
if left is None: return None
if _match_op(ps, TokOp.ColonEq):
_advance(ps)
right: AST | t.CPtr = _parse_test(ps)
node: AST | t.CPtr = NamedExpr(ps.pool, left, right)
_set_pos(node, left.lineno, left.col_offset, 0, 0)
return node
return left
def _parse_test(ps: Parser | t.CPtr) -> AST | t.CPtr:
"""test: or_test ['if' or_test 'else' test] | lambdef"""
if _cur_kw(ps) == Keyword.Lambda:
return _parse_lambda(ps)
left: AST | t.CPtr = _parse_or_test(ps)
if left is None: return None
if _cur_kw(ps) == Keyword.If:
_advance(ps)
test: AST | t.CPtr = _parse_or_test(ps)
if _cur_kw(ps) == Keyword.Else:
_advance(ps)
orelse: AST | t.CPtr = _parse_test(ps)
node: AST | t.CPtr = IfExp(ps.pool, test, left, orelse)
_set_pos(node, left.lineno, left.col_offset, 0, 0)
return node
ps.error_count += 1
return left
return left
def _parse_lambda(ps: Parser | t.CPtr) -> AST | t.CPtr:
"""lambdef: 'lambda' [varargslist] ':' test"""
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps) # lambda
args: AST | t.CPtr = None
if not _match_op(ps, TokOp.Colon):
args = _parse_varargslist(ps, 1)
if not _expect_op(ps, TokOp.Colon):
return None
body: AST | t.CPtr = _parse_test(ps)
node: AST | t.CPtr = Lambda(ps.pool, args, body)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_or_test(ps: Parser | t.CPtr) -> AST | t.CPtr:
"""or_test: and_test ('or' and_test)*"""
left: AST | t.CPtr = _parse_and_test(ps)
if left is None: return None
if _cur_kw(ps) == Keyword.Or:
values: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
values.append(left)
while _cur_kw(ps) == Keyword.Or:
_advance(ps)
right: AST | t.CPtr = _parse_and_test(ps)
if right is None: break
values.append(right)
node: AST | t.CPtr = BoolOp(ps.pool, OpKind.Or, values)
_set_pos(node, left.lineno, left.col_offset, 0, 0)
return node
return left
def _parse_and_test(ps: Parser | t.CPtr) -> AST | t.CPtr:
"""and_test: not_test ('and' not_test)*"""
left: AST | t.CPtr = _parse_not_test(ps)
if left is None: return None
if _cur_kw(ps) == Keyword.And:
values: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
values.append(left)
while _cur_kw(ps) == Keyword.And:
_advance(ps)
right: AST | t.CPtr = _parse_not_test(ps)
if right is None: break
values.append(right)
node: AST | t.CPtr = BoolOp(ps.pool, OpKind.And, values)
_set_pos(node, left.lineno, left.col_offset, 0, 0)
return node
return left
def _parse_not_test(ps: Parser | t.CPtr) -> AST | t.CPtr:
"""not_test: 'not' not_test | comparison"""
if _cur_kw(ps) == Keyword.Not:
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps)
operand: AST | t.CPtr = _parse_not_test(ps)
node: AST | t.CPtr = UnaryOp(ps.pool, OpKind.Not, operand)
_set_pos(node, lineno, col, 0, 0)
return node
return _parse_comparison(ps)
def _parse_comparison(ps: Parser | t.CPtr) -> AST | t.CPtr:
"""comparison: bitor_expr (comp_op bitor_expr)*
Single comparison design: a < b → Compare(left=a, op=Lt, right=b)
Chained comparison a < b < c → BoolOp(And, [Compare(a,<,b), Compare(b,<,c)])
"""
left: AST | t.CPtr = _parse_bitor(ps)
if left is None: return None
op_id: t.CInt = _match_comp_op(ps)
if op_id == OpKind.NoneOp:
return left
_advance_comp_op(ps)
right: AST | t.CPtr = _parse_bitor(ps)
# Single comparison: construct ops and comparators list
ops: list[t.CInt] | t.CPtr = list[t.CInt](ps.pool, 8)
ops.append(op_id)
comparators: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
comparators.append(right)
first_cmp: AST | t.CPtr = Compare(ps.pool, left, ops, comparators)
_set_pos(first_cmp, left.lineno, left.col_offset, 0, 0)
# Check chained comparison
next_op: t.CInt = _match_comp_op(ps)
if next_op == OpKind.NoneOp:
return first_cmp
# Single comparison with chained conditions: a < b < c → BoolOp(And, [Compare(a,<,b), Compare(b,<,c), ...])
values: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
values.append(first_cmp)
cur_left: AST | t.CPtr = right
while next_op != OpKind.NoneOp:
_advance_comp_op(ps)
cur_right: AST | t.CPtr = _parse_bitor(ps)
ops2: list[t.CInt] | t.CPtr = list[t.CInt](ps.pool, 8)
ops2.append(next_op)
comparators2: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
comparators2.append(cur_right)
cmp: AST | t.CPtr = Compare(ps.pool, cur_left, ops2, comparators2)
_set_pos(cmp, cur_left.lineno, cur_left.col_offset, 0, 0)
values.append(cmp)
cur_left = cur_right
next_op = _match_comp_op(ps)
node: AST | t.CPtr = BoolOp(ps.pool, OpKind.And, values)
_set_pos(node, left.lineno, left.col_offset, 0, 0)
return node
def _match_comp_op(ps: Parser | t.CPtr) -> t.CInt:
"""Check if the current one is a comparison operator, return OP_* or OpKind.NoneOp"""
if _cur_type(ps) == TokenType.Op:
op_id: t.CInt = _cur_op(ps)
if op_id == TokOp.EqEq: return OpKind.Eq
if op_id == TokOp.ExclaimEq: return OpKind.Ne
if op_id == TokOp.Less: return OpKind.Lt
if op_id == TokOp.LessEq: return OpKind.Le
if op_id == TokOp.Greater: return OpKind.Gt
if op_id == TokOp.GreaterEq: return OpKind.Ge
if _cur_kw(ps) == Keyword.Is:
if ps.cur.next is not None and ps.cur.next.type == TokenType.Name and ps.cur.next.kw_subtype == Keyword.Not:
return OpKind.IsNot
return OpKind.Is
if _cur_kw(ps) == Keyword.Not:
if ps.cur.next is not None and ps.cur.next.kw_subtype == Keyword.In:
return OpKind.NotIn
if _cur_kw(ps) == Keyword.In:
return OpKind.In
return OpKind.NoneOp
def _advance_comp_op(ps: Parser | t.CPtr):
"""Consume comparison operator token"""
if _cur_type(ps) == TokenType.Op:
_advance(ps)
return
if _cur_kw(ps) == Keyword.Is:
_advance(ps)
if _cur_kw(ps) == Keyword.Not:
_advance(ps)
return
if _cur_kw(ps) == Keyword.Not:
_advance(ps)
if _cur_kw(ps) == Keyword.In:
_advance(ps)
return
if _cur_kw(ps) == Keyword.In:
_advance(ps)
def _parse_bitor(ps: Parser | t.CPtr) -> AST | t.CPtr:
left: AST | t.CPtr = _parse_bitxor(ps)
if left is None: return None
while _match_op(ps, TokOp.VBar):
_advance(ps)
_skip_nl(ps)
right: AST | t.CPtr = _parse_bitxor(ps)
node: AST | t.CPtr = BinOp(ps.pool, left, OpKind.BitOr, right)
_set_pos(node, left.lineno, left.col_offset, 0, 0)
left = node
return left
def _parse_bitxor(ps: Parser | t.CPtr) -> AST | t.CPtr:
left: AST | t.CPtr = _parse_bitand(ps)
if left is None: return None
while _match_op(ps, TokOp.Caret):
_advance(ps)
_skip_nl(ps)
right: AST | t.CPtr = _parse_bitand(ps)
node: AST | t.CPtr = BinOp(ps.pool, left, OpKind.BitXor, right)
_set_pos(node, left.lineno, left.col_offset, 0, 0)
left = node
return left
def _parse_bitand(ps: Parser | t.CPtr) -> AST | t.CPtr:
left: AST | t.CPtr = _parse_shift(ps)
if left is None: return None
while _match_op(ps, TokOp.Amp):
_advance(ps)
_skip_nl(ps)
right: AST | t.CPtr = _parse_shift(ps)
node: AST | t.CPtr = BinOp(ps.pool, left, OpKind.BitAnd, right)
_set_pos(node, left.lineno, left.col_offset, 0, 0)
left = node
return left
def _parse_shift(ps: Parser | t.CPtr) -> AST | t.CPtr:
left: AST | t.CPtr = _parse_arith(ps)
if left is None: return None
while _match_op(ps, TokOp.LtLt) or _match_op(ps, TokOp.GtGt):
op_token: t.CInt = _cur_op(ps)
_advance(ps)
_skip_nl(ps)
right: AST | t.CPtr = _parse_arith(ps)
ast_op: t.CInt = OpKind.LShift
if op_token == TokOp.GtGt:
ast_op = OpKind.RShift
node: AST | t.CPtr = BinOp(ps.pool, left, ast_op, right)
_set_pos(node, left.lineno, left.col_offset, 0, 0)
left = node
return left
def _parse_arith(ps: Parser | t.CPtr) -> AST | t.CPtr:
left: AST | t.CPtr = _parse_term(ps)
if left is None: return None
while _match_op(ps, TokOp.Plus) or _match_op(ps, TokOp.Minus):
op_token: t.CInt = _cur_op(ps)
_advance(ps)
_skip_nl(ps)
right: AST | t.CPtr = _parse_term(ps)
ast_op: t.CInt = OpKind.Add
if op_token == TokOp.Minus:
ast_op = OpKind.Sub
node: AST | t.CPtr = BinOp(ps.pool, left, ast_op, right)
_set_pos(node, left.lineno, left.col_offset, 0, 0)
left = node
return left
def _parse_term(ps: Parser | t.CPtr) -> AST | t.CPtr:
left: AST | t.CPtr = _parse_factor(ps)
if left is None: return None
while True:
op_token: t.CInt = 0
if _match_op(ps, TokOp.Star): op_token = TokOp.Star
elif _match_op(ps, TokOp.Slash): op_token = TokOp.Slash
elif _match_op(ps, TokOp.DSlash): op_token = TokOp.DSlash
elif _match_op(ps, TokOp.Percent): op_token = TokOp.Percent
elif _match_op(ps, TokOp.At): op_token = TokOp.At
if op_token == 0: break
_advance(ps)
_skip_nl(ps)
right: AST | t.CPtr = _parse_factor(ps)
ast_op: t.CInt = _binop_from_op(op_token)
node: AST | t.CPtr = BinOp(ps.pool, left, ast_op, right)
_set_pos(node, left.lineno, left.col_offset, 0, 0)
left = node
return left
def _parse_factor(ps: Parser | t.CPtr) -> AST | t.CPtr:
"""factor: '+' factor | '-' factor | '~' factor | power"""
if _match_op(ps, TokOp.Plus) or _match_op(ps, TokOp.Minus) or _match_op(ps, TokOp.Tilde):
op_token: t.CInt = _cur_op(ps)
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps)
operand: AST | t.CPtr = _parse_factor(ps)
ast_op: t.CInt = OpKind.UAdd
if op_token == TokOp.Minus:
ast_op = OpKind.USub
elif op_token == TokOp.Tilde:
ast_op = OpKind.Invert
node: AST | t.CPtr = UnaryOp(ps.pool, ast_op, operand)
_set_pos(node, lineno, col, 0, 0)
return node
return _parse_power(ps)
def _parse_power(ps: Parser | t.CPtr) -> AST | t.CPtr:
"""power: await_expr ['**' factor] | atom_expr ['**' factor]"""
base: AST | t.CPtr = _parse_atom_expr(ps)
if base is None: return None
if _match_op(ps, TokOp.StarStar):
_advance(ps)
exp: AST | t.CPtr = _parse_factor(ps)
node: AST | t.CPtr = BinOp(ps.pool, base, OpKind.Pow, exp)
_set_pos(node, base.lineno, base.col_offset, 0, 0)
return node
return base
def _parse_atom_expr(ps: Parser | t.CPtr) -> AST | t.CPtr:
"""atom_expr: ['await'] atom trailer*"""
if _cur_kw(ps) == Keyword.Await:
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps)
inner: AST | t.CPtr = _parse_atom_expr(ps)
node: AST | t.CPtr = Await(ps.pool, inner)
_set_pos(node, lineno, col, 0, 0)
return node
base: AST | t.CPtr = _parse_atom(ps)
if base is None: return None
# trailer loop
while True:
if _match_op(ps, TokOp.Dot):
_advance(ps)
if _cur_type(ps) != TokenType.Name:
ps.error_count += 1
break
attr_name: str = _cur_str(ps)
attr_lineno: t.CInt = _cur_lineno(ps)
attr_col: t.CInt = _cur_col(ps)
_advance(ps)
ctx: t.CInt = ASTCtx.Load
if _match_op(ps, TokOp.Equal):
ctx = ASTCtx.Store
elif _match_op(ps, TokOp.PlusEq) or _match_op(ps, TokOp.MinusEq):
ctx = ASTCtx.Store
base = Attribute(ps.pool, base, attr_name, ctx)
_set_pos(base, attr_lineno, attr_col, 0, 0)
elif _match_op(ps, TokOp.LPar):
base = _parse_call(ps, base)
elif _match_op(ps, TokOp.Lsqb):
base = _parse_subscript(ps, base)
else:
break
return base
def _parse_call(ps: Parser | t.CPtr, func: AST | t.CPtr) -> AST | t.CPtr:
"""Parse function call trailer. Both args and keywords are lists."""
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps) # (
_skip_nls_and_newlines(ps) # Skip the line break after '('
args: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
keywords: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
if not _match_op(ps, TokOp.RPar):
while True:
# **expr
if _match_op(ps, TokOp.StarStar):
_advance(ps)
_skip_nls_and_newlines(ps)
val: AST | t.CPtr = _parse_expr(ps)
kw: AST | t.CPtr = KwNode(ps.pool, None, val)
_set_pos(kw, _cur_lineno(ps), _cur_col(ps), 0, 0)
keywords.append(kw)
# *expr
elif _match_op(ps, TokOp.Star):
_advance(ps)
_skip_nls_and_newlines(ps)
val = _parse_expr(ps)
star: AST | t.CPtr = Starred(ps.pool, val, ASTCtx.Load)
_set_pos(star, val.lineno, val.col_offset, 0, 0)
args.append(star)
# keyword = expr
elif _cur_type(ps) == TokenType.Name:
if ps.cur.next is not None and ps.cur.next.type == TokenType.Op and ps.cur.next.op_subtype == TokOp.Equal:
kw_name: str = _cur_str(ps)
kw_lineno: t.CInt = _cur_lineno(ps)
kw_col: t.CInt = _cur_col(ps)
_advance(ps) # name
_advance(ps) # =
_skip_nls_and_newlines(ps)
val = _parse_expr(ps)
kw = KwNode(ps.pool, kw_name, val)
_set_pos(kw, kw_lineno, kw_col, 0, 0)
keywords.append(kw)
else:
val = _parse_expr(ps)
args.append(val)
else:
val = _parse_expr(ps)
args.append(val)
_skip_nls_and_newlines(ps) # Skip the line break after the parameter
if not _accept_op(ps, TokOp.Comma):
break
_skip_nls_and_newlines(ps) # Skip the line break after the comma
_skip_nls_and_newlines(ps) # Skip the line break before ')'
_expect_op(ps, TokOp.RPar)
node: AST | t.CPtr = Call(ps.pool, func, args, keywords)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_subscript(ps: Parser | t.CPtr, value: AST | t.CPtr) -> AST | t.CPtr:
"""Parse subscript trailer."""
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps) # [
_skip_nl(ps)
slice_node: AST | t.CPtr = _parse_slice(ps)
_expect_op(ps, TokOp.Rsqb)
ctx: t.CInt = ASTCtx.Load
if _match_op(ps, TokOp.Equal):
ctx = ASTCtx.Store
node: AST | t.CPtr = Subscript(ps.pool, value, slice_node, ctx)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_slice(ps: Parser | t.CPtr) -> AST | t.CPtr:
"""Parse slice: [expr] ':' [expr] [':' [expr]] | expr | expr, expr, ..."""
lower: AST | t.CPtr = None
if not _match_op(ps, TokOp.Colon):
lower = _parse_expr(ps)
# Check if there’s a comma (tuple syntax: a[x, y, ...])
if _match_op(ps, TokOp.Comma):
elts: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
if lower is not None:
elts.append(lower)
while _accept_op(ps, TokOp.Comma):
if _match_op(ps, TokOp.Rsqb):
break # Trailing comma
elem: AST | t.CPtr = _parse_expr(ps)
if elem is not None:
elts.append(elem)
lineno: t.CInt = 0
if lower is not None:
lineno = lower.lineno
node: AST | t.CPtr = Tuple(ps.pool, elts, ASTCtx.Load)
_set_pos(node, lineno, 0, 0, 0)
return node
if _match_op(ps, TokOp.Colon):
_advance(ps)
upper: AST | t.CPtr = None
if not _match_op(ps, TokOp.Colon) and not _match_op(ps, TokOp.Rsqb):
upper = _parse_expr(ps)
step: AST | t.CPtr = None
if _accept_op(ps, TokOp.Colon):
if not _match_op(ps, TokOp.Rsqb):
step = _parse_expr(ps)
lineno: t.CInt = 0
if lower is not None:
lineno = lower.lineno
elif upper is not None:
lineno = upper.lineno
else:
lineno = _cur_lineno(ps)
node: AST | t.CPtr = Slice(ps.pool, lower, upper, step)
_set_pos(node, lineno, 0, 0, 0)
return node
return lower
def _parse_atom(ps: Parser | t.CPtr) -> AST | t.CPtr:
"""atom: the smallest unit of expression"""
ttype: t.CInt = _cur_type(ps)
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
if ttype == TokenType.Name:
kw: t.CInt = _cur_kw(ps)
if kw == Keyword.True_:
_advance(ps)
return _new_const_bool(ps.pool, 1, lineno, col)
if kw == Keyword.False_:
_advance(ps)
return _new_const_bool(ps.pool, 0, lineno, col)
if kw == Keyword.None_:
_advance(ps)
return _new_const_none(ps.pool, lineno, col)
name_str: str = _cur_str(ps)
_advance(ps)
node: AST | t.CPtr = Name(ps.pool, name_str, ASTCtx.Load)
_set_pos(node, lineno, col, 0, 0)
return node
if ttype == TokenType.Number:
tok: Token | t.CPtr = ps.cur
_advance(ps)
if tok.is_float:
return _new_const_float(ps.pool, tok.float_val, lineno, col)
return _new_const_int(ps.pool, tok.int_val, lineno, col)
if ttype == TokenType.String:
sval: str = _cur_str(ps)
is_fstr: t.CInt = ps.cur.kw_subtype & 4
is_triple: t.CInt = ps.cur.kw_subtype & 8
is_dquote: t.CInt = ps.cur.kw_subtype & 16
is_char: t.CInt = ps.cur.kw_subtype & 32
_advance(ps)
if is_fstr:
return _parse_fstring(ps, sval, lineno, col)
return _new_const_str(ps.pool, sval, lineno, col, is_dquote, is_triple, is_char)
if _match_op(ps, TokOp.Ellipsis):
_advance(ps)
return _new_const_none(ps.pool, lineno, col)
if _match_op(ps, TokOp.LPar):
return _parse_paren(ps)
if _match_op(ps, TokOp.Lsqb):
return _parse_list(ps)
if _match_op(ps, TokOp.LBrace):
return _parse_brace(ps)
ps.error_count += 1
return None
def _parse_paren(ps: Parser | t.CPtr) -> AST | t.CPtr:
"""Parenthesis expressions: () | (expr) | (expr,) | (expr, ...) | (yield)"""
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps) # (
_skip_nl(ps) # Skip the line break after '(' (multi-line tuple)
if _match_op(ps, TokOp.RPar):
_advance(ps)
# Empty tuple
empty: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
node: AST | t.CPtr = Tuple(ps.pool, empty, ASTCtx.Load)
_set_pos(node, lineno, col, 0, 0)
return node
# yield
if _cur_kw(ps) == Keyword.Yield:
y: AST | t.CPtr = _parse_yield(ps)
_expect_op(ps, TokOp.RPar)
return y
first: AST | t.CPtr = _parse_expr(ps)
if _match_op(ps, TokOp.Comma):
# Tuple
elts: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
elts.append(first)
while _accept_op(ps, TokOp.Comma):
_skip_nl(ps)
if _match_op(ps, TokOp.RPar):
break
e: AST | t.CPtr = _parse_expr(ps)
elts.append(e)
_expect_op(ps, TokOp.RPar)
node = Tuple(ps.pool, elts, ASTCtx.Load)
_set_pos(node, lineno, col, 0, 0)
return node
_expect_op(ps, TokOp.RPar)
# Check for generator expression
if _cur_kw(ps) == Keyword.For:
return _parse_comprehension(ps, first, ASTKind.GeneratorExp, lineno, col)
return first
def _parse_list(ps: Parser | t.CPtr) -> AST | t.CPtr:
"""List expressions: [] | [expr_list] | [expr_list comp_for]"""
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps) # [
_skip_nl(ps) # Skip the line break after '[' (multi-line list)
if _match_op(ps, TokOp.Rsqb):
_advance(ps)
empty: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
node: AST | t.CPtr = List(ps.pool, empty, ASTCtx.Load)
_set_pos(node, lineno, col, 0, 0)
return node
first: AST | t.CPtr = _parse_expr(ps)
if _cur_kw(ps) == Keyword.For:
node = _parse_comprehension(ps, first, ASTKind.ListComp, lineno, col)
_expect_op(ps, TokOp.Rsqb)
return node
elts: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
elts.append(first)
while _accept_op(ps, TokOp.Comma):
_skip_nl(ps)
if _match_op(ps, TokOp.Rsqb):
break
e: AST | t.CPtr = _parse_expr(ps)
elts.append(e)
_expect_op(ps, TokOp.Rsqb)
node = List(ps.pool, elts, ASTCtx.Load)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_brace(ps: Parser | t.CPtr) -> AST | t.CPtr:
"""Brace expressions: {} | {set_items} | {dict_items} | {comp_for}"""
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps) # {
_skip_nl(ps) # Skip the line break after '{' (multi-line dict)
if _match_op(ps, TokOp.RBrace):
_advance(ps)
empty_k: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
empty_v: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
node: AST | t.CPtr = Dict(ps.pool, empty_k, empty_v)
_set_pos(node, lineno, col, 0, 0)
return node
first: AST | t.CPtr = _parse_expr(ps)
# dict: key: value
if _match_op(ps, TokOp.Colon):
_advance(ps)
val: AST | t.CPtr = _parse_expr(ps)
if _cur_kw(ps) == Keyword.For:
# dict comprehension
gens: list[AST | t.CPtr] | t.CPtr = _parse_comp_for(ps)
node = DictComp(ps.pool, first, val, gens)
_set_pos(node, lineno, col, 0, 0)
_expect_op(ps, TokOp.RBrace)
return node
keys: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
values: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
keys.append(first)
values.append(val)
while _accept_op(ps, TokOp.Comma):
_skip_nl(ps)
if _match_op(ps, TokOp.RBrace):
break
k: AST | t.CPtr = _parse_expr(ps)
_expect_op(ps, TokOp.Colon)
v: AST | t.CPtr = _parse_expr(ps)
keys.append(k)
values.append(v)
_expect_op(ps, TokOp.RBrace)
node = Dict(ps.pool, keys, values)
_set_pos(node, lineno, col, 0, 0)
return node
# set
if _cur_kw(ps) == Keyword.For:
node = _parse_comprehension(ps, first, ASTKind.SetComp, lineno, col)
_expect_op(ps, TokOp.RBrace)
return node
elts: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
elts.append(first)
while _accept_op(ps, TokOp.Comma):
if _match_op(ps, TokOp.RBrace):
break
e: AST | t.CPtr = _parse_expr(ps)
elts.append(e)
_expect_op(ps, TokOp.RBrace)
node = Set(ps.pool, elts)
_set_pos(node, lineno, col, 0, 0)
return node
# ============================================================
# f-string parsing
# ============================================================
def _parse_sub_expr(ps: Parser | t.CPtr, src: str) -> AST | t.CPtr:
"""Create a sub-lexer and parser to parse expressions (used for {expr} inside f-strings)"""
pool: memhub.MemManager | t.CPtr = ps.pool
lx: Lexer | t.CPtr = new_lexer(pool)
if lx is None: return None
_lexer_init(lx, src, pool)
tokens: Token | t.CPtr = tokenize(lx)
if tokens is None: return None
sub_ps: Parser | t.CPtr = new_parser(pool)
if sub_ps is None: return None
_parser_init(sub_ps, tokens, pool)
return _parse_test(sub_ps)
def _parse_fstring(ps: Parser | t.CPtr, content: str,
lineno: t.CInt, col: t.CInt) -> AST | t.CPtr:
"""Parse f-string content, building JoinedStr or FormattedValue nodes"""
pool: memhub.MemManager | t.CPtr = ps.pool
values: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](pool, 8)
i: t.CSizeT = 0
content_len: t.CSizeT = string.strlen(content)
# Literal text buffer for f-string content
lit_buf: str = pool.alloc(content_len + 1)
if lit_buf is None: return None
lit_pos: t.CSizeT = 0
while i < content_len:
ch: t.CChar = content[i]
if ch == '{':
# {{ → Literal {
if i + 1 < content_len and content[i + 1] == '{':
lit_buf[lit_pos] = '{'
lit_pos += 1
i += 2
continue
# Refresh literal buffer
if lit_pos > 0:
lit_buf[lit_pos] = '\0'
lit_node: AST | t.CPtr = _new_const_str(pool, lit_buf, lineno, col)
values.append(lit_node)
lit_buf = pool.alloc(content_len + 1)
if lit_buf is None: break
lit_pos = 0
# Find matching closing '}' while tracking nested '{}' and string literals inside the f-string
j: t.CSizeT = i + 1
depth: t.CInt = 1
conv_pos: t.CSizeT = 0
fmt_pos: t.CSizeT = 0
while j < content_len and depth > 0:
c: t.CChar = content[j]
if c == '{':
depth += 1
elif c == '}':
depth -= 1
if depth == 0:
break
elif depth == 1:
if c == '!' and conv_pos == 0 and fmt_pos == 0:
if j + 1 < content_len:
nc: t.CChar = content[j + 1]
if nc == 's' or nc == 'r' or nc == 'a':
if j + 2 < content_len:
after: t.CChar = content[j + 2]
if after == ':' or after == '}':
conv_pos = j
elif j + 2 == content_len:
conv_pos = j
elif c == ':' and fmt_pos == 0:
fmt_pos = j
j += 1
if depth != 0:
ps.error_count += 1
break
# Extract expression text
expr_end: t.CSizeT = j
if conv_pos > 0:
expr_end = conv_pos
elif fmt_pos > 0:
expr_end = fmt_pos
expr_len: t.CSizeT = expr_end - i - 1
if expr_len == 0:
ps.error_count += 1
i = j + 1
continue
expr_buf: str = pool.alloc(expr_len + 1)
if expr_buf is None: break
string.memcpy(expr_buf, content + i + 1, expr_len)
expr_buf[expr_len] = '\0'
# Parse expression text
expr_node: AST | t.CPtr = _parse_sub_expr(ps, expr_buf)
# Conversion specifier
conversion: t.CInt = -1
if conv_pos > 0:
conv_char: t.CChar = content[conv_pos + 1]
if conv_char == 's':
conversion = 115
elif conv_char == 'r':
conversion = 114
elif conv_char == 'a':
conversion = 97
# Format specifier
format_spec: AST | t.CPtr = None
if fmt_pos > 0:
spec_len: t.CSizeT = j - fmt_pos - 1
if spec_len > 0:
spec_buf: str = pool.alloc(spec_len + 1)
if spec_buf is not None:
string.memcpy(spec_buf, content + fmt_pos + 1, spec_len)
spec_buf[spec_len] = '\0'
format_spec = _new_const_str(pool, spec_buf, lineno, col)
fv: AST | t.CPtr = FormattedValue(pool, expr_node, conversion, format_spec)
_set_pos(fv, lineno, col, 0, 0)
values.append(fv)
i = j + 1
elif ch == '}':
if i + 1 < content_len and content[i + 1] == '}':
lit_buf[lit_pos] = '}'
lit_pos += 1
i += 2
else:
ps.error_count += 1
i += 1
else:
lit_buf[lit_pos] = ch
lit_pos += 1
i += 1
# Refresh remaining literal buffer
if lit_pos > 0:
lit_buf[lit_pos] = '\0'
lit_node2: AST | t.CPtr = _new_const_str(pool, lit_buf, lineno, col)
values.append(lit_node2)
# If no content, return empty string
if values.__len__() == 0:
return _new_const_str(pool, "", lineno, col)
# If only one FormattedValue and no literal, return FormattedValue
if values.__len__() == 1:
first: AST | t.CPtr = values.get(0)
if first.kind() == ASTKind.FormattedValue:
return first
node: AST | t.CPtr = JoinedStr(pool, values)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_comprehension(ps: Parser | t.CPtr, elt: AST | t.CPtr,
vtype: t.CInt, lineno: t.CInt, col: t.CInt) -> AST | t.CPtr:
"""General comprehension: elt comp_for"""
gens: list[AST | t.CPtr] | t.CPtr = _parse_comp_for(ps)
node: AST | t.CPtr = None
if vtype == ASTKind.ListComp:
node = ListComp(ps.pool, elt, gens)
elif vtype == ASTKind.SetComp:
node = SetComp(ps.pool, elt, gens)
else:
node = GeneratorExp(ps.pool, elt, gens)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_comp_for(ps: Parser | t.CPtr) -> list[AST | t.CPtr] | t.CPtr:
"""comp_for: ['async'] 'for' exprlist 'in' or_test [comp_if] (comp_for)*
Returns generators list。"""
gens: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
while True:
is_async: t.CInt = 0
if _cur_kw(ps) == Keyword.Async:
is_async = 1
_advance(ps)
if _cur_kw(ps) != Keyword.For:
break
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps)
target: AST | t.CPtr = _parse_target_list(ps)
if _cur_kw(ps) != Keyword.In:
ps.error_count += 1
break
_advance(ps)
iter_: AST | t.CPtr = _parse_or_test(ps)
# Collect ifs to list
ifs: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
while _cur_kw(ps) == Keyword.If:
_advance(ps)
cond: AST | t.CPtr = _parse_or_test(ps)
ifs.append(cond)
comp: AST | t.CPtr = Comprehension(ps.pool, target, iter_, ifs, is_async)
_set_pos(comp, lineno, col, 0, 0)
gens.append(comp)
return gens
def _parse_target_list(ps: Parser | t.CPtr) -> AST | t.CPtr:
"""Parse assignment target list (comma-separated)"""
first: AST | t.CPtr = _parse_target(ps)
if _match_op(ps, TokOp.Comma):
elts: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
elts.append(first)
while _accept_op(ps, TokOp.Comma):
if _cur_kw(ps) == Keyword.In:
break
e: AST | t.CPtr = _parse_target(ps)
elts.append(e)
node: AST | t.CPtr = Tuple(ps.pool, elts, ASTCtx.Store)
_set_pos(node, first.lineno, first.col_offset, 0, 0)
return node
return first
def _parse_target(ps: Parser | t.CPtr) -> AST | t.CPtr:
"""Parse assignment target"""
if _match_op(ps, TokOp.Star):
_advance(ps)
inner: AST | t.CPtr = _parse_atom_expr(ps)
node: AST | t.CPtr = Starred(ps.pool, inner, ASTCtx.Store)
_set_pos(node, inner.lineno, inner.col_offset, 0, 0)
return node
if _match_op(ps, TokOp.LPar):
_advance(ps)
_skip_nl(ps)
t_: AST | t.CPtr = _parse_target_list(ps)
_skip_nl(ps)
_expect_op(ps, TokOp.RPar)
return t_
if _match_op(ps, TokOp.Lsqb):
_advance(ps)
_skip_nl(ps)
first: AST | t.CPtr = _parse_target(ps)
elts: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
elts.append(first)
while _accept_op(ps, TokOp.Comma):
_skip_nl(ps)
if _match_op(ps, TokOp.Rsqb):
break
e: AST | t.CPtr = _parse_target(ps)
elts.append(e)
_expect_op(ps, TokOp.Rsqb)
node = List(ps.pool, elts, ASTCtx.Store)
_set_pos(node, first.lineno, first.col_offset, 0, 0)
return node
# Normal atom_expr, ctx=Store
node = _parse_atom_expr(ps)
if node is not None:
_set_ctx(node, ASTCtx.Store)
return node
def _parse_yield(ps: Parser | t.CPtr) -> AST | t.CPtr:
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps) # yield
if _cur_kw(ps) == Keyword.From:
_advance(ps)
val: AST | t.CPtr = _parse_expr(ps)
node: AST | t.CPtr = YieldFrom(ps.pool, val)
_set_pos(node, lineno, col, 0, 0)
return node
val2: AST | t.CPtr = None
if not _match_op(ps, TokOp.RPar) and _cur_type(ps) != TokenType.NewLine and _cur_type(ps) != TokenType.EndMarker:
if _cur_type(ps) == TokenType.Op and (_cur_op(ps) == TokOp.Comma or _cur_op(ps) == TokOp.Rsqb):
pass
else:
val2 = _parse_test(ps)
node = Yield(ps.pool, val2)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_varargslist(ps: Parser | t.CPtr, is_lambda: t.CInt = 0) -> AST | t.CPtr:
"""Parse function parameter list
New API: args/defaults/kw_defaults are lists, vararg/kwarg are fields."""
args: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
vararg: AST | t.CPtr = None
kwarg: AST | t.CPtr = None
defaults: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
kw_defaults: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
while True:
_skip_nl(ps)
if _match_op(ps, TokOp.RPar):
break
# *args 或 * (kwonly separated)
if _match_op(ps, TokOp.Star):
_advance(ps)
_skip_nl(ps)
if _cur_type(ps) == TokenType.Name:
vararg = _parse_arg_def(ps, is_lambda)
if not _accept_op(ps, TokOp.Comma):
break
_skip_nl(ps)
continue
# **kwargs
if _match_op(ps, TokOp.StarStar):
_advance(ps)
_skip_nl(ps)
kwarg = _parse_arg_def(ps, is_lambda)
if not _accept_op(ps, TokOp.Comma):
break
_skip_nl(ps)
continue
# / Delimiter (posonly end)— New API: posonly/regular are not distinguished
if _match_op(ps, TokOp.Slash):
_advance(ps)
if not _accept_op(ps, TokOp.Comma):
break
_skip_nl(ps)
continue
# Normal parameter
a: AST | t.CPtr = _parse_arg_def(ps, is_lambda)
if a is None: break
args.append(a)
if _match_op(ps, TokOp.Equal):
_advance(ps)
d: AST | t.CPtr = _parse_test(ps)
defaults.append(d)
if not _accept_op(ps, TokOp.Comma):
break
_skip_nl(ps)
return Arguments(ps.pool, args, vararg, kwarg, defaults, kw_defaults)
def _parse_arg_def(ps: Parser | t.CPtr, is_lambda: t.CInt = 0) -> AST | t.CPtr:
"""arg: TokenType.Name [':' annotation]"""
if _cur_type(ps) != TokenType.Name:
ps.error_count += 1
return None
name_str: str = _cur_str(ps)
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps)
annotation: AST | t.CPtr = None
if is_lambda == 0:
if _accept_op(ps, TokOp.Colon):
annotation = _parse_test(ps)
result: AST | t.CPtr = Arg(ps.pool, name_str, annotation)
_set_pos(result, lineno, col, 0, 0)
return result
# ============================================================
# Statement parsing (Design: directly append to parent, do not return a list)
# ============================================================
def _parse_suite_into_children(ps: Parser | t.CPtr, parent: AST | t.CPtr):
"""Parse statement suite, directly append to parent's children.
suite: TokenType.NewLine TokenType.Indent stmt+ TokenType.Dedent | simple_stmt"""
_skip_nl(ps)
if _cur_type(ps) == TokenType.NewLine:
_advance(ps)
_skip_nl(ps)
if _cur_type(ps) != TokenType.Indent:
ps.error_count += 1
return
_advance(ps)
sidx: t.CInt = 0
while _cur_type(ps) != TokenType.Dedent and _cur_type(ps) != TokenType.EndMarker:
sidx += 1
# Decorator (class body methods can have decorators)
if _match_op(ps, TokOp.At):
ps.pending_decorators = _parse_decorators(ps)
before_cur: Token | t.CPtr = ps.cur
_parse_statement_into(ps, parent)
# Defensive: if parse did not advance token, force advance to avoid infinite loop
if ps.cur is before_cur:
ps.error_count += 1
_advance(ps)
_skip_nl(ps)
_expect_type(ps, TokenType.Dedent)
return
# Single-line suite
_parse_simple_stmt_into(ps, parent)
def _parse_statement_into(ps: Parser | t.CPtr, parent: AST | t.CPtr):
"""Parse statement (compound or simple), append to the parent."""
ttype: t.CInt = _cur_type(ps)
kw: t.CInt = _cur_kw(ps)
if ttype == TokenType.Name:
if kw == Keyword.If or kw == Keyword.While or kw == Keyword.For or kw == Keyword.Try:
stmt: AST | t.CPtr = _parse_compound(ps)
if stmt is not None:
parent.append(stmt)
return
if kw == Keyword.With or kw == Keyword.Def or kw == Keyword.Class:
stmt = _parse_compound(ps)
if stmt is not None:
parent.append(stmt)
return
if kw == Keyword.Match or kw == Keyword.Async:
stmt = _parse_compound(ps)
if stmt is not None:
parent.append(stmt)
return
_parse_simple_stmt_into(ps, parent)
def _parse_simple_stmt_into(ps: Parser | t.CPtr, parent: AST | t.CPtr):
"""simple_stmt: small_stmt (';' small_stmt)* [';'] TokenType.NewLine"""
first: AST | t.CPtr = _parse_small_stmt(ps)
if first is not None:
parent.append(first)
while _accept_op(ps, TokOp.Semi):
if _cur_type(ps) == TokenType.NewLine or _cur_type(ps) == TokenType.EndMarker:
break
s: AST | t.CPtr = _parse_small_stmt(ps)
if s is None: break
parent.append(s)
_skip_nl(ps)
if _cur_type(ps) == TokenType.NewLine:
_advance(ps)
def _parse_small_stmt(ps: Parser | t.CPtr) -> AST | t.CPtr:
"""small_stmt: various simple statements"""
kw: t.CInt = _cur_kw(ps)
if kw == Keyword.Pass:
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps)
node: AST | t.CPtr = Pass(ps.pool)
_set_pos(node, lineno, col, 0, 0)
return node
if kw == Keyword.Break:
lineno = _cur_lineno(ps)
col = _cur_col(ps)
_advance(ps)
node = Break(ps.pool)
_set_pos(node, lineno, col, 0, 0)
return node
if kw == Keyword.Continue:
lineno = _cur_lineno(ps)
col = _cur_col(ps)
_advance(ps)
node = Continue(ps.pool)
_set_pos(node, lineno, col, 0, 0)
return node
if kw == Keyword.Return:
return _parse_return(ps)
if kw == Keyword.Raise:
return _parse_raise(ps)
if kw == Keyword.Global:
return _parse_global(ps)
if kw == Keyword.Nonlocal:
return _parse_nonlocal(ps)
if kw == Keyword.Import:
return _parse_import(ps)
if kw == Keyword.From:
return _parse_from_import(ps)
if kw == Keyword.Del:
return _parse_del(ps)
if kw == Keyword.Assert:
return _parse_assert(ps)
if kw == Keyword.Yield:
return _parse_expr_stmt(ps, _parse_yield(ps))
# Expression statement / assignment / augassign / annassign
return _parse_expr_stmt(ps, None)
def _maybe_tuple(ps: Parser | t.CPtr, first: AST | t.CPtr) -> AST | t.CPtr:
"""If there is a comma after first, collect the comma-separated expressions into a Tuple."""
if first is None: return None
if not _match_op(ps, TokOp.Comma):
return first
lineno: t.CInt = first.lineno
col: t.CInt = first.col_offset
elts: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
elts.append(first)
_accept_op(ps, TokOp.Comma)
while True:
tt: t.CInt = _cur_type(ps)
if tt == TokenType.NewLine or tt == TokenType.EndMarker:
break
if tt == TokenType.Op:
op: t.CInt = _cur_op(ps)
if op == TokOp.Equal or op == TokOp.Semi:
break
e: AST | t.CPtr = _parse_expr(ps)
if e is None: break
elts.append(e)
if not _accept_op(ps, TokOp.Comma):
break
node: AST | t.CPtr = Tuple(ps.pool, elts, ASTCtx.Load)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_testlist(ps: Parser | t.CPtr) -> AST | t.CPtr:
"""Parse a comma-separated list of expressions, if there’s more than one, form a tuple."""
first: AST | t.CPtr = _parse_expr(ps)
return _maybe_tuple(ps, first)
def _parse_expr_stmt(ps: Parser | t.CPtr, first: AST | t.CPtr) -> AST | t.CPtr:
"""Expression statement, handle assignment/augassign/annassign"""
if first is None:
first = _parse_testlist(ps)
else:
first = _maybe_tuple(ps, first)
if first is None:
return None
lineno: t.CInt = first.lineno
col: t.CInt = first.col_offset
# Assignment
if _match_op(ps, TokOp.Equal):
targets: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
targets.append(first)
_fix_store_ctx(first)
value: AST | t.CPtr = None
accept_result: t.CInt = _accept_op(ps, TokOp.Equal)
while accept_result:
v: AST | t.CPtr = _parse_testlist(ps)
if v is None: break
if value is not None:
targets.append(value)
_fix_store_ctx(value)
value = v
accept_result = _accept_op(ps, TokOp.Equal)
node: AST | t.CPtr = Assign(ps.pool, targets, value)
_set_pos(node, lineno, col, 0, 0)
return node
# augassign
if _cur_type(ps) == TokenType.Op:
op_id: t.CInt = _cur_op(ps)
aug_op: t.CInt = _augop_from_op(op_id)
if aug_op != OpKind.NoneOp:
_advance(ps)
value2: AST | t.CPtr = _parse_testlist(ps)
_fix_store_ctx(first)
node2: AST | t.CPtr = AugAssign(ps.pool, first, aug_op, value2)
_set_pos(node2, lineno, col, 0, 0)
return node2
# annassign: target ':' annotation [= value]
if _match_op(ps, TokOp.Colon):
_advance(ps)
annotation: AST | t.CPtr = _parse_test(ps)
value3: AST | t.CPtr = None
if _accept_op(ps, TokOp.Equal):
value3 = _parse_testlist(ps)
if ps.pool is None:
return None
_fix_store_ctx(first)
if ps.pool is None:
return None
node3: AST | t.CPtr = AnnAssign(ps.pool, first, annotation, value3, 1)
_set_pos(node3, lineno, col, 0, 0)
return node3
# Expression statement
node4: AST | t.CPtr = Expr(ps.pool, first)
_set_pos(node4, lineno, col, 0, 0)
return node4
def _parse_return(ps: Parser | t.CPtr) -> AST | t.CPtr:
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps)
val: AST | t.CPtr = None
if _cur_type(ps) != TokenType.NewLine and _cur_type(ps) != TokenType.EndMarker and _cur_type(ps) != TokenType.Op:
val = _parse_testlist(ps)
elif _cur_type(ps) == TokenType.Op and _cur_op(ps) != TokOp.Semi:
val = _parse_testlist(ps)
node: AST | t.CPtr = Return(ps.pool, val)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_raise(ps: Parser | t.CPtr) -> AST | t.CPtr:
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps)
exc: AST | t.CPtr = None
cause: AST | t.CPtr = None
if _cur_type(ps) != TokenType.NewLine and _cur_type(ps) != TokenType.EndMarker:
exc = _parse_test(ps)
if _cur_kw(ps) == Keyword.From:
_advance(ps)
cause = _parse_test(ps)
node: AST | t.CPtr = Raise(ps.pool, exc, cause)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_global(ps: Parser | t.CPtr) -> AST | t.CPtr:
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps)
names: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
if _cur_type(ps) == TokenType.Name:
n: AST | t.CPtr = Name(ps.pool, _cur_str(ps), ASTCtx.Store)
_set_pos(n, _cur_lineno(ps), _cur_col(ps), 0, 0)
names.append(n)
_advance(ps)
while _accept_op(ps, TokOp.Comma):
if _cur_type(ps) != TokenType.Name: break
n = Name(ps.pool, _cur_str(ps), ASTCtx.Store)
_set_pos(n, _cur_lineno(ps), _cur_col(ps), 0, 0)
names.append(n)
_advance(ps)
node: AST | t.CPtr = Global(ps.pool, names)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_nonlocal(ps: Parser | t.CPtr) -> AST | t.CPtr:
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps)
names: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
if _cur_type(ps) == TokenType.Name:
n: AST | t.CPtr = Name(ps.pool, _cur_str(ps), ASTCtx.Store)
_set_pos(n, _cur_lineno(ps), _cur_col(ps), 0, 0)
names.append(n)
_advance(ps)
while _accept_op(ps, TokOp.Comma):
if _cur_type(ps) != TokenType.Name: break
n = Name(ps.pool, _cur_str(ps), ASTCtx.Store)
_set_pos(n, _cur_lineno(ps), _cur_col(ps), 0, 0)
names.append(n)
_advance(ps)
node: AST | t.CPtr = Nonlocal(ps.pool, names)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_import(ps: Parser | t.CPtr) -> AST | t.CPtr:
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps)
names: list[AST | t.CPtr] | t.CPtr = _parse_alias_list(ps)
node: AST | t.CPtr = Import(ps.pool, names)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_from_import(ps: Parser | t.CPtr) -> AST | t.CPtr:
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps)
# Parse module (possibly with dots)
level: t.CInt = 0
while _match_op(ps, TokOp.Dot):
level += 1
_advance(ps)
while _match_op(ps, TokOp.Ellipsis):
level += 3
_advance(ps)
module_name: str = None
# The token type for keywords (like import) is also Name, so they must be excluded using kw_subtype,
# otherwise the `import` in `from . import` would be mistakenly treated as a module_name.
# Note: only `import` itself should be excluded. Module names can be keywords (like in `from .match import ...`,
# where match is a Viper match statement keyword), requiring NotKeyword would cause
# `from .match import` to fail parsing and the dependency scan would miss that module.
if _cur_type(ps) == TokenType.Name and _cur_kw(ps) != Keyword.Import:
module_name = _cur_str(ps)
_advance(ps)
while _match_op(ps, TokOp.Dot):
_advance(ps)
if _cur_type(ps) == TokenType.Name and _cur_kw(ps) == Keyword.NotKeyword:
next_name: str = _cur_str(ps)
mlen: t.CInt = string.strlen(module_name)
nlen: t.CInt = string.strlen(next_name)
new_mod: str = ps.pool.alloc(mlen + 1 + nlen + 1)
if new_mod is not None:
string.strcpy(new_mod, module_name)
new_mod[mlen] = '.'
string.strcpy(new_mod + mlen + 1, next_name)
module_name = new_mod
_advance(ps)
if _cur_kw(ps) != Keyword.Import:
ps.error_count += 1
return None
_advance(ps)
_skip_nl(ps)
# import *
if _match_op(ps, TokOp.Star):
_advance(ps)
star_alias: AST | t.CPtr = Alias(ps.pool, "*", None)
_set_pos(star_alias, lineno, col, 0, 0)
star_names: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
star_names.append(star_alias)
node: AST | t.CPtr = ImportFrom(ps.pool, module_name, star_names, level)
_set_pos(node, lineno, col, 0, 0)
return node
# ( names ) or names
if _match_op(ps, TokOp.LPar):
_advance(ps)
_skip_nl(ps)
names: list[AST | t.CPtr] | t.CPtr = _parse_alias_list(ps)
_skip_nl(ps)
_expect_op(ps, TokOp.RPar)
else:
names = _parse_alias_list(ps)
node: AST | t.CPtr = ImportFrom(ps.pool, module_name, names, level)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_alias_list(ps: Parser | t.CPtr) -> list[AST | t.CPtr] | t.CPtr:
"""Parse alias list."""
names: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
while True:
if _cur_type(ps) != TokenType.Name:
break
name_str: str = _cur_str(ps)
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps)
# dotted name
while _match_op(ps, TokOp.Dot):
_advance(ps)
if _cur_type(ps) == TokenType.Name:
next_n: str = _cur_str(ps)
nmlen: t.CInt = string.strlen(name_str)
nxtlen: t.CInt = string.strlen(next_n)
new_name: str = ps.pool.alloc(nmlen + 1 + nxtlen + 1)
if new_name is not None:
string.strcpy(new_name, name_str)
new_name[nmlen] = '.'
string.strcpy(new_name + nmlen + 1, next_n)
name_str = new_name
_advance(ps)
asname: str = None
if _cur_kw(ps) == Keyword.As:
_advance(ps)
if _cur_type(ps) == TokenType.Name:
asname = _cur_str(ps)
_advance(ps)
a: AST | t.CPtr = Alias(ps.pool, name_str, asname)
_set_pos(a, lineno, col, 0, 0)
names.append(a)
if not _accept_op(ps, TokOp.Comma):
break
_skip_nl(ps)
return names
def _parse_del(ps: Parser | t.CPtr) -> AST | t.CPtr:
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps)
targets: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
while True:
t_: AST | t.CPtr = _parse_atom_expr(ps)
if t_ is None: break
_set_ctx(t_, ASTCtx.Del)
targets.append(t_)
if not _accept_op(ps, TokOp.Comma):
break
node: AST | t.CPtr = Delete(ps.pool, targets)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_assert(ps: Parser | t.CPtr) -> AST | t.CPtr:
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps)
test: AST | t.CPtr = _parse_test(ps)
msg: AST | t.CPtr = None
if _accept_op(ps, TokOp.Comma):
msg = _parse_test(ps)
node: AST | t.CPtr = Assert(ps.pool, test, msg)
_set_pos(node, lineno, col, 0, 0)
return node
# ============================================================
# Compound statements
# ============================================================
def _parse_compound(ps: Parser | t.CPtr) -> AST | t.CPtr:
kw: t.CInt = _cur_kw(ps)
if kw == Keyword.If:
return _parse_if(ps)
if kw == Keyword.While:
return _parse_while(ps)
if kw == Keyword.For:
return _parse_for(ps, 0)
if kw == Keyword.Try:
return _parse_try(ps)
if kw == Keyword.With:
return _parse_with(ps, 0)
if kw == Keyword.Def:
return _parse_funcdef(ps, 0)
if kw == Keyword.Class:
return _parse_classdef(ps)
if kw == Keyword.Async:
_advance(ps)
if _cur_kw(ps) == Keyword.Def:
return _parse_funcdef(ps, 1)
if _cur_kw(ps) == Keyword.For:
return _parse_for(ps, 1)
if _cur_kw(ps) == Keyword.With:
return _parse_with(ps, 1)
ps.error_count += 1
return None
if kw == Keyword.Match:
return _parse_match(ps)
ps.error_count += 1
return None
def _parse_if(ps: Parser | t.CPtr) -> AST | t.CPtr:
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps) # if
test: AST | t.CPtr = _parse_test(ps)
_expect_op(ps, TokOp.Colon)
# First construct with empty orelse, then fill it in later
empty: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
node: If | t.CPtr = If(ps.pool, test, empty)
_parse_suite_into_children(ps, node)
orelse: list[AST | t.CPtr] | t.CPtr = _parse_if_tail(ps)
node.orelse = orelse
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_if_tail(ps: Parser | t.CPtr) -> list[AST | t.CPtr] | t.CPtr:
"""Parse elif/else chain and return orelse list.
elif: orelse = list containing single If node (nested)
else: orelse = statement list
"""
if _cur_kw(ps) == Keyword.Elif:
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps)
elif_test: AST | t.CPtr = _parse_test(ps)
_expect_op(ps, TokOp.Colon)
empty: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
elif_node: If | t.CPtr = If(ps.pool, elif_test, empty)
_parse_suite_into_children(ps, elif_node)
elif_orelse: list[AST | t.CPtr] | t.CPtr = _parse_if_tail(ps)
elif_node.orelse = elif_orelse
_set_pos(elif_node, lineno, col, 0, 0)
orelse: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
orelse.append(elif_node)
return orelse
if _cur_kw(ps) == Keyword.Else:
_advance(ps)
_expect_op(ps, TokOp.Colon)
scratch: Module | t.CPtr = Module(ps.pool)
_parse_suite_into_children(ps, scratch)
return scratch.children
return list[AST | t.CPtr](ps.pool, 8) # 空 list
# ============================================================
# Parsing Compound Statements (Continued)
# ============================================================
def _parse_while(ps: Parser | t.CPtr) -> AST | t.CPtr:
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps) # while
test: AST | t.CPtr = _parse_test(ps)
_expect_op(ps, TokOp.Colon)
empty: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
node: While | t.CPtr = While(ps.pool, test, empty)
_parse_suite_into_children(ps, node)
if _cur_kw(ps) == Keyword.Else:
_advance(ps)
_expect_op(ps, TokOp.Colon)
scratch: Module | t.CPtr = Module(ps.pool)
_parse_suite_into_children(ps, scratch)
node.orelse = scratch.children
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_for(ps: Parser | t.CPtr, is_async: t.CInt) -> AST | t.CPtr:
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps) # for
target: AST | t.CPtr = _parse_target_list(ps)
if _cur_kw(ps) != Keyword.In:
ps.error_count += 1
return None
_advance(ps)
iter_: AST | t.CPtr = _parse_or_test(ps)
_expect_op(ps, TokOp.Colon)
empty: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
node: For | t.CPtr = For(ps.pool, target, iter_, empty, is_async)
_fix_store_ctx(target)
_parse_suite_into_children(ps, node)
if _cur_kw(ps) == Keyword.Else:
_advance(ps)
_expect_op(ps, TokOp.Colon)
scratch: Module | t.CPtr = Module(ps.pool)
_parse_suite_into_children(ps, scratch)
node.orelse = scratch.children
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_try(ps: Parser | t.CPtr) -> AST | t.CPtr:
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps) # try
_expect_op(ps, TokOp.Colon)
empty: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
node: Try | t.CPtr = Try(ps.pool, empty, empty, empty)
_parse_suite_into_children(ps, node)
# handlers (except)
handlers: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
while _cur_kw(ps) == Keyword.Except:
h: AST | t.CPtr = _parse_except_handler(ps)
if h is not None:
handlers.append(h)
node.handlers = handlers
# else
if _cur_kw(ps) == Keyword.Else:
_advance(ps)
_expect_op(ps, TokOp.Colon)
scratch: Module | t.CPtr = Module(ps.pool)
_parse_suite_into_children(ps, scratch)
node.orelse = scratch.children
# finally
if _cur_kw(ps) == Keyword.Finally:
_advance(ps)
_expect_op(ps, TokOp.Colon)
scratch2: Module | t.CPtr = Module(ps.pool)
_parse_suite_into_children(ps, scratch2)
node.finalbody = scratch2.children
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_except_handler(ps: Parser | t.CPtr) -> AST | t.CPtr:
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps) # except
type_: AST | t.CPtr = None
name: str = None
if _cur_op(ps) != TokOp.Colon:
type_ = _parse_test(ps)
if _cur_kw(ps) == Keyword.As:
_advance(ps)
if _cur_type(ps) == TokenType.Name:
name = _cur_str(ps)
_advance(ps)
_expect_op(ps, TokOp.Colon)
node: AST | t.CPtr = ExceptHandler(ps.pool, type_, name)
_parse_suite_into_children(ps, node)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_with(ps: Parser | t.CPtr, is_async: t.CInt) -> AST | t.CPtr:
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps) # with
items: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
# items
while True:
item: AST | t.CPtr = _parse_with_item(ps)
if item is None: break
items.append(item)
if not _accept_op(ps, TokOp.Comma):
break
# Check if we can continue with a newline (with (...): form)
if _cur_type(ps) == TokenType.NewLine or _cur_type(ps) == TokenType.Nl:
break
_expect_op(ps, TokOp.Colon)
node: With | t.CPtr = With(ps.pool, items, is_async)
_parse_suite_into_children(ps, node)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_with_item(ps: Parser | t.CPtr) -> AST | t.CPtr:
"""with_item: test ['as' expr]"""
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
context_expr: AST | t.CPtr = _parse_or_test(ps)
if context_expr is None: return None
optional_vars: AST | t.CPtr = None
if _cur_kw(ps) == Keyword.As:
_advance(ps)
optional_vars = _parse_target(ps)
_fix_store_ctx(optional_vars)
node: AST | t.CPtr = WithItem(ps.pool, context_expr, optional_vars)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_funcdef(ps: Parser | t.CPtr, is_async: t.CInt) -> AST | t.CPtr:
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps) # def
if _cur_type(ps) != TokenType.Name:
ps.error_count += 1
return None
name: str = _cur_str(ps)
_advance(ps)
# PEP 695 泛型函数类型参数: def add[T](a: T, b: T) — 解析并存入 type_params。
# （与 class 解析器一致；之前缺失此处理导致 _expect_op(Colon) 失败、函数体被
# 误提升到模块级——b = Box(42) 被当作模块级全局注册，函数内赋值误报
# "需用 global 声明"。翻译时 type_params 映射到 int 兜底。）
type_params_fd: list[str] | t.CPtr = None
if _accept_op(ps, TokOp.Lsqb):
type_params_fd = list[str](ps.pool, 8)
while not _match_op(ps, TokOp.Rsqb):
if _cur_type(ps) == TokenType.EndMarker:
break
if _cur_type(ps) == TokenType.Name:
tp_name_fd: str = _cur_str(ps)
_advance(ps)
# 跳过可选 bound/约束: [T: bound] 或 [T: (A, B)]
if _match_op(ps, TokOp.Colon):
_advance(ps)
_parse_test(ps)
type_params_fd.append(tp_name_fd)
if not _accept_op(ps, TokOp.Comma):
break
_expect_op(ps, TokOp.Rsqb)
args: AST | t.CPtr = None
if _accept_op(ps, TokOp.LPar):
args = _parse_varargslist(ps, 0)
_expect_op(ps, TokOp.RPar)
returns: AST | t.CPtr = None
if _accept_op(ps, TokOp.RArrow):
returns = _parse_test(ps)
_expect_op(ps, TokOp.Colon)
dec_list: list[AST | t.CPtr] | t.CPtr = ps.pending_decorators
ps.pending_decorators = None
node: FunctionDef | t.CPtr = FunctionDef(ps.pool, name, args, dec_list, returns, is_async, type_params_fd)
_parse_suite_into_children(ps, node)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_classdef(ps: Parser | t.CPtr) -> AST | t.CPtr:
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps) # class
if _cur_type(ps) != TokenType.Name:
ps.error_count += 1
return None
name: str = _cur_str(ps)
_advance(ps)
# Parse type parameters [T] or [T, U] (PEP 695 generic class syntax)
type_params: list[str] | t.CPtr = None
if _accept_op(ps, TokOp.Lsqb):
type_params = list[str](ps.pool, 8)
while not _match_op(ps, TokOp.Rsqb):
if _cur_type(ps) == TokenType.EndMarker: break
if _cur_type(ps) == TokenType.Name:
tp_name: str = _cur_str(ps)
_advance(ps)
# Skip optional bound/constraint: [T: bound] or [T: (A, B)]
if _match_op(ps, TokOp.Colon):
_advance(ps)
_parse_test(ps)
type_params.append(tp_name)
if not _accept_op(ps, TokOp.Comma):
break
_expect_op(ps, TokOp.Rsqb)
bases: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
keywords: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
# bases / keywords
if _accept_op(ps, TokOp.LPar):
while not _match_op(ps, TokOp.RPar):
if _cur_type(ps) == TokenType.EndMarker: break
# base 或 keyword=value
if _cur_type(ps) == TokenType.Name:
# Parse as expression, if followed by '=', then as keyword=value
e: AST | t.CPtr = _parse_or_test(ps)
if _match_op(ps, TokOp.Equal):
_advance(ps)
v: AST | t.CPtr = _parse_test(ps)
kw_node: AST | t.CPtr = KwNode(ps.pool, e, v)
keywords.append(kw_node)
else:
bases.append(e)
if not _accept_op(ps, TokOp.Comma):
break
_expect_op(ps, TokOp.RPar)
_expect_op(ps, TokOp.Colon)
dec_list: list[AST | t.CPtr] | t.CPtr = ps.pending_decorators
ps.pending_decorators = None
node: ClassDef | t.CPtr = ClassDef(ps.pool, name, bases, keywords, dec_list, type_params)
_parse_suite_into_children(ps, node)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_match(ps: Parser | t.CPtr) -> AST | t.CPtr:
"""match statement: match subject ':' NewLine Indent case+ Dedent"""
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps) # match
subject: AST | t.CPtr = _parse_test(ps)
_expect_op(ps, TokOp.Colon)
cases: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
node: AST | t.CPtr = Match(ps.pool, subject, cases)
# Parsing body: Same NewLine/Indent/Dedent pattern as _parse_suite_into_children,
# but parses case clauses into a cases list instead of normal statements into children
_skip_nl(ps)
if _cur_type(ps) == TokenType.NewLine:
_advance(ps)
_skip_nl(ps)
if _cur_type(ps) == TokenType.Indent:
_advance(ps)
while _cur_type(ps) != TokenType.Dedent and _cur_type(ps) != TokenType.EndMarker:
if _cur_kw(ps) == Keyword.Case:
case_node: AST | t.CPtr = _parse_match_case(ps)
if case_node is not None:
cases.append(case_node)
else:
_advance(ps) # Skip non-case token to avoid infinite loop
_expect_type(ps, TokenType.Dedent)
_set_pos(node, lineno, col, 0, 0)
return node
# ============================================================
# Match pattern parsing
# ============================================================
def _make_name_ast(pool: memhub.MemManager | t.CPtr, node: Name | t.CPtr) -> AST | t.CPtr:
# Analysis: "Convert Name* to AST* (Solving the TransPyC Type Inference Problem)"
return node
def _parse_pattern(ps: Parser | t.CPtr) -> AST | t.CPtr:
"""pattern: or_pattern ['as' NAME]
or_pattern: closed_pattern ('|' closed_pattern)*"""
pat: AST | t.CPtr = _parse_closed_pattern(ps)
if pat is None: return None
# OR pattern
if _match_op(ps, TokOp.VBar):
patterns: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
patterns.append(pat)
while _accept_op(ps, TokOp.VBar):
p: AST | t.CPtr = _parse_closed_pattern(ps)
if p is not None:
patterns.append(p)
or_node: AST | t.CPtr = MatchOr(ps.pool, patterns)
_set_pos(or_node, pat.lineno, pat.col_offset, 0, 0)
pat = or_node
# AS pattern
if _cur_kw(ps) == Keyword.As:
_advance(ps)
if _cur_type(ps) != TokenType.Name:
ps.error_count += 1
return pat
name_str: str = _cur_str(ps)
_advance(ps)
as_node: AST | t.CPtr = MatchAs(ps.pool, pat, name_str)
_set_pos(as_node, pat.lineno, pat.col_offset, 0, 0)
return as_node
return pat
def _parse_closed_pattern(ps: Parser | t.CPtr) -> AST | t.CPtr:
"""closed_pattern: literal | capture | wildcard | value | group | sequence | mapping | class"""
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
ttype: t.CInt = _cur_type(ps)
val: AST | t.CPtr
node: AST | t.CPtr
tok: Token | t.CPtr
sval: str
# Number literal pattern
if ttype == TokenType.Number:
tok = ps.cur
_advance(ps)
if tok.is_float:
val = _new_const_float(ps.pool, tok.float_val, lineno, col)
else:
val = _new_const_int(ps.pool, tok.int_val, lineno, col)
node = MatchValue(ps.pool, val)
_set_pos(node, lineno, col, 0, 0)
return node
# String literal pattern
if ttype == TokenType.String:
sval = _cur_str(ps)
is_dquote_m: t.CInt = ps.cur.kw_subtype & 16
_advance(ps)
val = _new_const_str(ps.pool, sval, lineno, col, is_dquote_m)
node = MatchValue(ps.pool, val)
_set_pos(node, lineno, col, 0, 0)
return node
# Negative number literal pattern: -1, -1.5
if _match_op(ps, TokOp.Minus):
_advance(ps)
if _cur_type(ps) == TokenType.Number:
tok = ps.cur
_advance(ps)
if tok.is_float:
val = _new_const_float(ps.pool, -tok.float_val, lineno, col)
else:
val = _new_const_int(ps.pool, -tok.int_val, lineno, col)
node = MatchValue(ps.pool, val)
_set_pos(node, lineno, col, 0, 0)
return node
ps.error_count += 1
return None
# Name: None/True/False/_ /capture/value/class
if ttype == TokenType.Name:
kw: t.CInt = _cur_kw(ps)
# None/True/False → MatchSingleton
if kw == Keyword.None_ or kw == Keyword.True_ or kw == Keyword.False_:
_advance(ps)
if kw == Keyword.None_:
val = _new_const_none(ps.pool, lineno, col)
elif kw == Keyword.True_:
val = _new_const_bool(ps.pool, 1, lineno, col)
else:
val = _new_const_bool(ps.pool, 0, lineno, col)
node = MatchSingleton(ps.pool, val)
_set_pos(node, lineno, col, 0, 0)
return node
name_str: str = _cur_str(ps)
# Wildcard pattern: _
if string.strcmp(name_str, "_") == 0:
_advance(ps)
node = MatchAs(ps.pool, None, None)
_set_pos(node, lineno, col, 0, 0)
return node
_advance(ps)
# Check dotted name（value pattern 或 class pattern）
if _match_op(ps, TokOp.Dot):
name_node: Name | t.CPtr = Name(ps.pool, name_str, ASTCtx.Load)
_set_pos(name_node, lineno, col, 0, 0)
base: AST | t.CPtr = _make_name_ast(ps.pool, name_node)
while _match_op(ps, TokOp.Dot):
_advance(ps)
if _cur_type(ps) != TokenType.Name:
ps.error_count += 1
break
attr_name: str = _cur_str(ps)
attr_lineno: t.CInt = _cur_lineno(ps)
attr_col: t.CInt = _cur_col(ps)
_advance(ps)
base = Attribute(ps.pool, base, attr_name, ASTCtx.Load)
_set_pos(base, attr_lineno, attr_col, 0, 0)
# class pattern: dotted.name(...)
if _match_op(ps, TokOp.LPar):
return _parse_class_pattern(ps, base, lineno, col)
# value pattern: dotted.name
node = MatchValue(ps.pool, base)
_set_pos(node, lineno, col, 0, 0)
return node
# class pattern: Name(...)
if _match_op(ps, TokOp.LPar):
cls_node: Name | t.CPtr = Name(ps.pool, name_str, ASTCtx.Load)
_set_pos(cls_node, lineno, col, 0, 0)
cls: AST | t.CPtr = _make_name_ast(ps.pool, cls_node)
return _parse_class_pattern(ps, cls, lineno, col)
# capture pattern: NAME
node = MatchAs(ps.pool, None, name_str)
_set_pos(node, lineno, col, 0, 0)
return node
# group pattern / sequence pattern (tuple): (...)
if _match_op(ps, TokOp.LPar):
_advance(ps)
inner: AST | t.CPtr = _parse_pattern(ps)
if _accept_op(ps, TokOp.Comma):
# sequence pattern (tuple)
patterns: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
if inner is not None:
patterns.append(inner)
while not _match_op(ps, TokOp.RPar) and _cur_type(ps) != TokenType.EndMarker:
p: AST | t.CPtr = _parse_pattern(ps)
if p is not None:
patterns.append(p)
if not _accept_op(ps, TokOp.Comma):
break
_expect_op(ps, TokOp.RPar)
node = MatchSequence(ps.pool, patterns)
_set_pos(node, lineno, col, 0, 0)
return node
_expect_op(ps, TokOp.RPar)
return inner # group pattern
# sequence pattern (list): [...]
if _match_op(ps, TokOp.Lsqb):
return _parse_sequence_pattern(ps, TokOp.Rsqb, lineno, col)
# mapping pattern: {...}
if _match_op(ps, TokOp.LBrace):
return _parse_mapping_pattern(ps, lineno, col)
ps.error_count += 1
return None
def _parse_sequence_pattern(ps: Parser | t.CPtr, close_op: t.CInt,
lineno: t.CInt, col: t.CInt) -> AST | t.CPtr:
"""sequence_pattern: '[' [pattern_list] ']'"""
patterns: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
_advance(ps) # [
while not _match_op(ps, close_op) and _cur_type(ps) != TokenType.EndMarker:
# *name (star pattern)
if _match_op(ps, TokOp.Star):
_advance(ps)
if _cur_type(ps) == TokenType.Name:
star_name: str = _cur_str(ps)
_advance(ps)
star_node: AST | t.CPtr
if string.strcmp(star_name, "_") == 0:
star_node = MatchStar(ps.pool, None)
else:
star_node = MatchStar(ps.pool, star_name)
_set_pos(star_node, lineno, col, 0, 0)
patterns.append(star_node)
else:
p: AST | t.CPtr = _parse_pattern(ps)
if p is not None:
patterns.append(p)
if not _accept_op(ps, TokOp.Comma):
break
_expect_op(ps, close_op)
node: AST | t.CPtr = MatchSequence(ps.pool, patterns)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_mapping_pattern(ps: Parser | t.CPtr,
lineno: t.CInt, col: t.CInt) -> AST | t.CPtr:
"""mapping_pattern: '{' [items] '}'
items: key ':' pattern (',' key ':' pattern)* [',' '**' NAME] | '**' NAME"""
keys: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
patterns: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
rest: str = None
node: AST | t.CPtr
key: AST | t.CPtr
val_pat: AST | t.CPtr
_advance(ps) # {
if _match_op(ps, TokOp.RBrace):
_advance(ps)
node = MatchMapping(ps.pool, keys, patterns, rest)
_set_pos(node, lineno, col, 0, 0)
return node
while _cur_type(ps) != TokenType.EndMarker:
if _match_op(ps, TokOp.StarStar):
_advance(ps)
if _cur_type(ps) == TokenType.Name:
rest = _cur_str(ps)
_advance(ps)
break
key = _parse_expr(ps)
_expect_op(ps, TokOp.Colon)
val_pat = _parse_pattern(ps)
if key is not None:
keys.append(key)
if val_pat is not None:
patterns.append(val_pat)
if not _accept_op(ps, TokOp.Comma):
break
_expect_op(ps, TokOp.RBrace)
node = MatchMapping(ps.pool, keys, patterns, rest)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_class_pattern(ps: Parser | t.CPtr, cls: AST | t.CPtr,
lineno: t.CInt, col: t.CInt) -> AST | t.CPtr:
"""class_pattern: cls '(' [pattern_list] ')'
pattern_list: positional_pattern (',' positional_pattern)*
[',' keyword_pattern]* | keyword_pattern (',' keyword_pattern)*
keyword_pattern: NAME '=' pattern"""
patterns: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
kwd_attrs: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
kwd_patterns: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
p: AST | t.CPtr
_advance(ps) # (
if not _match_op(ps, TokOp.RPar):
while _cur_type(ps) != TokenType.EndMarker:
# keyword pattern: NAME '=' pattern
if _cur_type(ps) == TokenType.Name and ps.cur is not None:
nxt: Token | t.CPtr = ps.cur.next
if nxt is not None and nxt.type == TokenType.Op and nxt.op_subtype == TokOp.Equal:
attr_name: str = _cur_str(ps)
_advance(ps) # NAME
_advance(ps) # =
kwd_pat: AST | t.CPtr = _parse_pattern(ps)
attr_n: Name | t.CPtr = Name(ps.pool, attr_name, ASTCtx.Load)
_set_pos(attr_n, lineno, col, 0, 0)
attr_node: AST | t.CPtr = _make_name_ast(ps.pool, attr_n)
kwd_attrs.append(attr_node)
if kwd_pat is not None:
kwd_patterns.append(kwd_pat)
else:
p = _parse_pattern(ps)
if p is not None:
patterns.append(p)
else:
p = _parse_pattern(ps)
if p is not None:
patterns.append(p)
if not _accept_op(ps, TokOp.Comma):
break
_expect_op(ps, TokOp.RPar)
node: AST | t.CPtr = MatchClass(ps.pool, cls, patterns, kwd_attrs, kwd_patterns)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_match_case(ps: Parser | t.CPtr) -> AST | t.CPtr:
"""case pattern [if guard] ':' suite"""
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps) # case
guard: AST | t.CPtr = None
pattern: AST | t.CPtr = _parse_pattern(ps)
if _cur_kw(ps) == Keyword.If:
_advance(ps)
guard = _parse_or_test(ps)
_expect_op(ps, TokOp.Colon)
node: AST | t.CPtr = MatchCase(ps.pool, pattern, guard)
_parse_suite_into_children(ps, node)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_decorators(ps: Parser | t.CPtr) -> list[AST | t.CPtr] | t.CPtr:
"""解析装饰器链，返回 list。"""
decs: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
while _match_op(ps, TokOp.At):
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps) # @
expr: AST | t.CPtr = _parse_namedexpr(ps)
_skip_nl(ps)
if _cur_type(ps) == TokenType.NewLine:
_advance(ps)
decs.append(expr)
return decs
# ============================================================
# Module entry
# ============================================================
def _parse_module_body(ps: Parser | t.CPtr, module_node: AST | t.CPtr):
"""解析模块体，append 到 module_node。"""
_skip_nl(ps)
stmt_idx: t.CInt = 0
while _cur_type(ps) != TokenType.EndMarker:
stmt_idx += 1
# Decorator
if _match_op(ps, TokOp.At):
ps.pending_decorators = _parse_decorators(ps)
# Record current token pointer to check for infinite loop
before_cur: Token | t.CPtr = ps.cur
_parse_statement_into(ps, module_node)
# Defense: If parsing doesn't advance the token (like when _parse_atom encounters an unrecognized token and returns None),
# force it to move forward to avoid an infinite loop in the main loop; accumulate error_count for the caller to diagnose
if ps.cur is before_cur:
ps.error_count += 1
_advance(ps)
_skip_nl(ps)
def parse_tokens(pool: memhub.MemManager | t.CPtr,
tokens: Token | t.CPtr) -> AST | t.CPtr:
"""Parse an AST Module from a token list."""
if pool is None or tokens is None:
return None
ps: Parser | t.CPtr = new_parser(pool)
if ps is None: return None
_parser_init(ps, tokens, pool)
module_node: Module | t.CPtr = Module(pool)
_parse_module_body(ps, module_node)
return module_node
Reference in new issue View Git Blame Copy Permalink