GVSDS/Viper_VRTE
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
14ff9d109f5322d25e350f7cc14a49eaad22bb11
Viper_VRTE/ast/parser.vp
T

2750 lines
102 KiB
Plaintext
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import t, c
from stdint import *
import memhub
import string
import stdio
import stdlib
from .tokens import (Token, TokenType, Keyword, TokOp)
from .base import (
AST, ASTCtx, ASTKind, OpKind, ASTFlag,
_set_pos, _binop_from_op, _augop_from_op, _cmpop_from_op, _copy_str, _inherit_pos,
_init_ast,
CONST_INT, CONST_FLOAT, CONST_STR, CONST_BOOL, CONST_NONE, CONST_CHAR,
FLAG_IS_ASYNC, FLAG_IS_EXTERN,
)
from .exprs import (
BoolOp, BinOp, UnaryOp, Lambda, IfExp, Dict, Set,
ListComp, SetComp, DictComp, GeneratorExp,
Await, Yield, YieldFrom, FormattedValue, JoinedStr,
Constant, NamedExpr, Attribute, Subscript, Starred, Name,
List, Tuple, Slice, Call, Compare, OpNode,
)
from .stmts import (
Module, Expression, Interactive, FunctionType,
FunctionDef, ClassDef, Return, Delete, Assign, AugAssign, AnnAssign,
For, While, If, With, Raise, Try, Assert, Global, Nonlocal,
Pass, Break, Continue, Expr, Import, ImportFrom, Match,
)
from .astaux import (
ExceptHandler, Arguments, Arg, Keyword as KwNode, Alias, WithItem, Comprehension,
)
from .match import (
MatchCase, MatchValue, MatchSingleton, MatchSequence,
MatchMapping, MatchClass, MatchStar, MatchAs, MatchOr,
)
from .lexer import Lexer, _lexer_init, tokenize, new_lexer
# ============================================================
# Parser State
# ============================================================
class Parser:
tokens: Token | t.CPtr # token linked list head
cur: Token | t.CPtr # current token
pool: memhub.MemManager | t.CPtr # memory pool
error_count: t.CInt # error count
pending_decorators: list[AST | t.CPtr] | t.CPtr # pending decorators list
def __new__(self, pool: memhub.MemManager | t.CPtr):
ptr: Parser | t.CPtr = pool.alloc(Parser.__sizeof__())
if ptr:
string.memset(ptr, 0, Parser.__sizeof__())
return ptr
def new_parser(pool: memhub.MemManager | t.CPtr) -> Parser | t.CPtr:
"""Allocate from the pool and return the Parser (cross-module safe wrapper)"""
return Parser(pool)
def _parser_init(ps: Parser | t.CPtr, tokens: Token | t.CPtr,
pool: memhub.MemManager | t.CPtr):
if ps is None: return
ps.tokens = tokens
ps.cur = tokens
ps.pool = pool
ps.error_count = 0
ps.pending_decorators = None
# ============================================================
# Token Navigation Helpers
# ============================================================
def _advance(ps: Parser | t.CPtr):
"""Advance to the next token"""
if ps.cur is None: return
if ps.cur.next is not None:
ps.cur = ps.cur.next
def _cur_type(ps: Parser | t.CPtr) -> t.CInt:
if ps.cur is None: return TokenType.EndMarker
return ps.cur.type
def _cur_op(ps: Parser | t.CPtr) -> t.CInt:
if ps.cur is None: return 0
return ps.cur.op_subtype
def _cur_kw(ps: Parser | t.CPtr) -> t.CInt:
if ps.cur is None: return 0
return ps.cur.kw_subtype
def _is_funcdef_kw(ps: Parser | t.CPtr, kw: t.CInt) -> int:
"""判断当前 token 是否为函数定义关键字（def 及其别名 fn/fun/func/function，
以及 extern 声明关键字）
def 是硬关键字直接判定；fn/fun/func/function/extern 是软关键字，
仅当下一个 token 是 Name 且再下一个是 '('（或泛型 '[T]' 前的 '['）时
判定为函数定义。extern 还允许 'extern def A(...)' 形式（下一个是 def）。
否则按普通标识符处理，兼容既有代码把 func 用作
字段/参数名（如 AST 节点 func 字段、Call.func 注解 `func: AST | t.CPtr`）。
"""
if kw == Keyword.Def:
return 1
if kw == Keyword.Extern:
# extern def A(...) 或 extern A(...)（extern 后是 def 或 Name+'(/'['）
nxt: Token | t.CPtr = ps.cur.next
if nxt is None:
return 0
if nxt.kw_subtype == Keyword.Def:
return 1
if nxt.type != TokenType.Name:
return 0
nxt2: Token | t.CPtr = nxt.next
if nxt2 is None:
return 0
if nxt2.type == TokenType.Op \
and (nxt2.op_subtype == TokOp.LPar or nxt2.op_subtype == TokOp.Lsqb):
return 1
return 0
if kw == Keyword.Fn or kw == Keyword.Fun \
or kw == Keyword.Func or kw == Keyword.Function:
# 软关键字 lookahead: 下个是 Name，再下个是 '(' 或 '[T...]' 泛型参数（PEP 695）
nxt: Token | t.CPtr = ps.cur.next
if nxt is None:
return 0
if nxt.type != TokenType.Name:
return 0
nxt2: Token | t.CPtr = nxt.next
if nxt2 is None:
return 0
if nxt2.type == TokenType.Op \
and (nxt2.op_subtype == TokOp.LPar or nxt2.op_subtype == TokOp.Lsqb):
return 1
return 0
return 0
def _is_match_stmt(ps: Parser | t.CPtr) -> int:
"""match 是软关键字（CPython 3.10+）：lookahead 判断是否为 match 语句
match 后直接跟 ':'（变量注解，如 lexer.vp 的 `match: t.CInt = 1`）、
'='（赋值，如 `match = 0`）、换行/文件尾 → 按普通标识符；
否则扫描至行尾，在括号深度 0 处出现 ':' → match 语句（subject 后必须冒号）。
case/type 同理为软关键字（case 仅在 match 套件内消费，type 无语句分发）。
"""
nxt: Token | t.CPtr = ps.cur.next
if nxt is None:
return 0
if nxt.type == TokenType.NewLine or nxt.type == TokenType.EndMarker:
return 0
if nxt.type == TokenType.Op:
if nxt.op_subtype == TokOp.Colon:
return 0
if nxt.op_subtype == TokOp.Equal:
return 0
walk: Token | t.CPtr = nxt
depth: t.CInt = 0
while walk is not None:
if walk.type == TokenType.NewLine or walk.type == TokenType.EndMarker:
return 0
if walk.type == TokenType.Op:
wop: t.CInt = walk.op_subtype
if wop == TokOp.LPar or wop == TokOp.Lsqb or wop == TokOp.LBrace:
depth += 1
elif wop == TokOp.RPar or wop == TokOp.Rsqb or wop == TokOp.RBrace:
if depth > 0:
depth -= 1
elif wop == TokOp.Colon and depth == 0:
return 1
walk = walk.next
return 0
# 解析错误位置与消息（fail-fast 仿 Python SyntaxError 报告，首次错误记录）
_g_parse_err_line: t.CInt = 0
_g_parse_err_col: t.CInt = 0
_g_parse_err_msg: str = None
def _record_parse_err(ps: Parser | t.CPtr):
"""记录首个解析错误的位置与消息（只记录第一次，后续错误不覆盖）"""
global _g_parse_err_line, _g_parse_err_col, _g_parse_err_msg
if _g_parse_err_line != 0:
return
_g_parse_err_line = _cur_lineno(ps)
_g_parse_err_col = _cur_col(ps)
err_s: str = _cur_str(ps)
if err_s is None:
err_s = "(none)"
# 消息文本用 stdlib 拷贝（token 的 str_val 在 pool 中，生命周期覆盖本次编译但保险起见独立存储）
buf_m: str = stdlib.malloc(128)
if buf_m is not None:
stdio.snprintf(buf_m, 128, "意外的 token '%s'", err_s)
_g_parse_err_msg = buf_m
else:
_g_parse_err_msg = err_s
def get_parse_err_line() -> t.CInt:
"""获取最近一次解析错误的行号（0=无错误）"""
return _g_parse_err_line
def get_parse_err_col() -> t.CInt:
"""获取最近一次解析错误的列号（0=无错误）"""
return _g_parse_err_col
def get_parse_err_msg() -> str:
"""获取最近一次解析错误的错误消息（仿 Python SyntaxError 消息文本）"""
return _g_parse_err_msg
def _cur_str(ps: Parser | t.CPtr) -> str:
if ps.cur is None: return None
return ps.cur.str_val
def _cur_lineno(ps: Parser | t.CPtr) -> t.CInt:
if ps.cur is None: return 0
return ps.cur.lineno
def _cur_col(ps: Parser | t.CPtr) -> t.CInt:
if ps.cur is None: return 0
return ps.cur.col_offset
def _skip_nl(ps: Parser | t.CPtr):
"""Skip TokenType.Nl token"""
while ps.cur is not None and ps.cur.type == TokenType.Nl:
ps.cur = ps.cur.next
def _skip_nls_and_newlines(ps: Parser | t.CPtr):
"""Skip TokenType.Nl and TokenType.NewLine (used for line continuation)"""
while ps.cur is not None:
if ps.cur.type == TokenType.Nl or ps.cur.type == TokenType.NewLine:
ps.cur = ps.cur.next
else:
break
def _expect_type(ps: Parser | t.CPtr, ttype: t.CInt) -> t.CInt:
"""Expect the current token to be of the specified type, advance and return 1, otherwise return 0"""
if _cur_type(ps) == ttype:
_advance(ps)
return 1
_record_parse_err(ps)
ps.error_count += 1
return 0
def _expect_op(ps: Parser | t.CPtr, op_id: t.CInt) -> t.CInt:
"""Expecting the current one to be some operator"""
if _cur_type(ps) == TokenType.Op and _cur_op(ps) == op_id:
_advance(ps)
return 1
_record_parse_err(ps)
ps.error_count += 1
return 0
def _match_op(ps: Parser | t.CPtr, op_id: t.CInt) -> t.CInt:
"""Check but not consume the current token"""
if _cur_type(ps) == TokenType.Op and _cur_op(ps) == op_id:
return 1
return 0
def _accept_op(ps: Parser | t.CPtr, op_id: t.CInt) -> t.CInt:
"""If match, consume and return 1"""
if _match_op(ps, op_id):
_advance(ps)
return 1
return 0
# ============================================================
# Constant Construction Helper (New API: Constant directly accepts lineno/col)
# ============================================================
def _new_const_int(pool: memhub.MemManager | t.CPtr, val: t.CInt64T,
lineno: t.CInt, col: t.CInt) -> AST | t.CPtr:
return Constant(pool, CONST_INT, val, 0.0, None, lineno, col)
def _new_const_float(pool: memhub.MemManager | t.CPtr, val: t.CDouble,
lineno: t.CInt, col: t.CInt) -> AST | t.CPtr:
global _g_dbg_pts, _g_dbg_vals, _g_dbg_cnt
new_c: Constant | t.CPtr = Constant(pool, CONST_FLOAT, 0, val, None, lineno, col)
# 把数字文本存到 str_val（stdlib 分配，绕开 pool 对 float_val 字段的覆写）
# REG-SRC 用 string.atof(str_val) 恢复原始值
lit_txt: str = stdlib.malloc(64)
if lit_txt is not None:
stdio.snprintf(lit_txt, 64, "%.17g", val)
new_c.str_val = lit_txt
# 记录浮点节点 ptr→val 到 stdlib 表（构造时值正确，供 REG-SRC 查表绕开 pool 覆写）
# 表满时重置：旧记录对应已翻译完的文件，从当前节点重新记录
if _g_dbg_pts is None:
_g_dbg_pts = stdlib.malloc(4096 * 8)
_g_dbg_vals = stdlib.malloc(4096 * 8)
_g_dbg_cnt = 0
elif _g_dbg_cnt >= 4096:
_g_dbg_cnt = 0
if _g_dbg_pts is not None and _g_dbg_vals is not None:
_g_dbg_pts[_g_dbg_cnt] = t.CUInt64T(new_c)
_g_dbg_vals[_g_dbg_cnt] = val
_g_dbg_cnt += 1
return new_c
# 浮点节点构造值表（stdlib 分配，避免污染 pool）
_g_dbg_pts: t.CUInt64T | t.CPtr = None
_g_dbg_vals: t.CDouble | t.CPtr = None
_g_dbg_cnt: int = 0
# 最近一次 _dbg_lookup_lit_val 是否命中
_g_dbg_lit_found: int = 0
def _dbg_lookup_lit_val(ptr: t.CUInt64T) -> double:
"""查浮点节点构造值表：返回 ptr 节点的原始构造值（未命中返回 0 并置 found=0）
用于绕开 pool 分配对 AST float_val 字段的覆写（构造时值正确，读取时可能被覆写）。
"""
global _g_dbg_lit_found
_g_dbg_lit_found = 0
if _g_dbg_pts is None or _g_dbg_vals is None:
return 0.0
i: t.CSizeT = 0
while i < _g_dbg_cnt:
if _g_dbg_pts[i] == ptr:
_g_dbg_lit_found = 1
return _g_dbg_vals[i]
i += 1
return 0.0
def _dbg_lit_found() -> int:
"""检查最近一次 _dbg_lookup_lit_val 是否命中"""
return _g_dbg_lit_found
def _new_const_str(pool: memhub.MemManager | t.CPtr, val: str,
lineno: t.CInt, col: t.CInt,
is_dquote: t.CInt = 1,
is_triple: t.CInt = 0,
is_char: t.CInt = 0) -> AST | t.CPtr:
"""Create a string/character constant node
is_dquote: 0 = single quote, non-0 = double quote (default double quote)
is_triple: non-0 = triple-quoted string
is_char: non-0 = single-character candidate (lexer has confirmed that after escaping it's exactly 1 byte, including '0')
Semantic distinctions:
- 'x' (single quote, single character, not triple-quoted) → CONST_CHAR (i8 value)
- "x" (double quote, includes 0 terminator) → CONST_STR (i8* pointer, i8 x 2)
- 'ab' (single quotes, multiple characters) → CONST_STR (i8* pointer)
- '''x''' / \"\"\"x\"\"\" (triple quotes) → CONST_STR (i8* pointer)
- '' / "" (empty string) → CONST_STR (i8* pointer)
"""
# Single quotes, non-triple quotes, single-character candidates marked by lexer → CONST_CHAR
# Mark with is_char instead of strlen(val), because '0' becomes a NUL byte after escaping, strlen would return 0
if is_dquote == 0 and is_triple == 0 and is_char != 0:
# Single quote # Single character → CONST_CHAR, int_val stores the character's ASCII value
# val[0] is a legal value of 0 even if it's NUL
ch_val: t.CInt64T = val[0]
return Constant(pool, CONST_CHAR, ch_val, 0.0, val, lineno, col)
# Other situations → CONST_STR
return Constant(pool, CONST_STR, 0, 0.0, val, lineno, col)
def _new_const_bool(pool: memhub.MemManager | t.CPtr, val: t.CInt,
lineno: t.CInt, col: t.CInt) -> AST | t.CPtr:
return Constant(pool, CONST_BOOL, val, 0.0, None, lineno, col)
def _new_const_none(pool: memhub.MemManager | t.CPtr,
lineno: t.CInt, col: t.CInt) -> AST | t.CPtr:
return Constant(pool, CONST_NONE, 0, 0.0, None, lineno, col)
# ============================================================
# ctx Helper (Recursive Fixing Store Context)
# ============================================================
def _set_ctx(node: AST | t.CPtr, ctx: t.CInt):
"""Set ctx field of node (based on kind conversion)"""
if node is None: return
k: t.CInt = node.kind()
if k == ASTKind.Name:
nm: Name | t.CPtr = (Name | t.CPtr)(node)
nm.ctx = ctx
elif k == ASTKind.Attribute:
at: Attribute | t.CPtr = (Attribute | t.CPtr)(node)
at.ctx = ctx
elif k == ASTKind.Subscript:
sb: Subscript | t.CPtr = (Subscript | t.CPtr)(node)
sb.ctx = ctx
elif k == ASTKind.Tuple:
tp: Tuple | t.CPtr = (Tuple | t.CPtr)(node)
tp.ctx = ctx
elif k == ASTKind.List:
ls: List | t.CPtr = (List | t.CPtr)(node)
ls.ctx = ctx
elif k == ASTKind.Starred:
st: Starred | t.CPtr = (Starred | t.CPtr)(node)
st.ctx = ctx
def _fix_store_ctx(node: AST | t.CPtr):
"""Fix ctx of store targets to be Store"""
if node is None: return
_set_ctx(node, ASTCtx.Store)
k: t.CInt = node.kind()
if k == ASTKind.Tuple or k == ASTKind.List:
tp_elts: list[AST | t.CPtr] | t.CPtr = None
if k == ASTKind.Tuple:
tp: Tuple | t.CPtr = (Tuple | t.CPtr)(node)
tp_elts = tp.elts
else:
ls: List | t.CPtr = (List | t.CPtr)(node)
tp_elts = ls.elts
if tp_elts is not None:
n: t.CSizeT = tp_elts.__len__()
i: t.CSizeT = 0
while i < n:
cur: AST | t.CPtr = tp_elts.get(i)
_fix_store_ctx(cur)
i += 1
elif k == ASTKind.Starred:
st2: Starred | t.CPtr = (Starred | t.CPtr)(node)
_fix_store_ctx(st2.value)
# ============================================================
# Expression Parser
# ============================================================
def _parse_expr(ps: Parser | t.CPtr) -> AST | t.CPtr:
"""expr: yield_expr | namedexpr_test"""
if _cur_kw(ps) == Keyword.Yield:
return _parse_yield(ps)
return _parse_namedexpr(ps)
def _parse_namedexpr(ps: Parser | t.CPtr) -> AST | t.CPtr:
"""namedexpr_test: test [':=' test]"""
left: AST | t.CPtr = _parse_test(ps)
if left is None: return None
if _match_op(ps, TokOp.ColonEq):
_advance(ps)
right: AST | t.CPtr = _parse_test(ps)
node: AST | t.CPtr = NamedExpr(ps.pool, left, right)
_set_pos(node, left.lineno, left.col_offset, 0, 0)
return node
return left
def _parse_test(ps: Parser | t.CPtr) -> AST | t.CPtr:
"""test: or_test ['if' or_test 'else' test] | lambdef"""
if _cur_kw(ps) == Keyword.Lambda:
return _parse_lambda(ps)
left: AST | t.CPtr = _parse_or_test(ps)
if left is None: return None
if _cur_kw(ps) == Keyword.If:
_advance(ps)
test: AST | t.CPtr = _parse_or_test(ps)
if _cur_kw(ps) == Keyword.Else:
_advance(ps)
orelse: AST | t.CPtr = _parse_test(ps)
node: AST | t.CPtr = IfExp(ps.pool, test, left, orelse)
_set_pos(node, left.lineno, left.col_offset, 0, 0)
return node
ps.error_count += 1
return left
return left
def _parse_lambda(ps: Parser | t.CPtr) -> AST | t.CPtr:
"""lambdef: 'lambda' [varargslist] ':' test"""
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps) # lambda
args: AST | t.CPtr = None
if not _match_op(ps, TokOp.Colon):
args = _parse_varargslist(ps, 1)
if not _expect_op(ps, TokOp.Colon):
return None
body: AST | t.CPtr = _parse_test(ps)
node: AST | t.CPtr = Lambda(ps.pool, args, body)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_or_test(ps: Parser | t.CPtr) -> AST | t.CPtr:
"""or_test: and_test ('or' and_test)*"""
left: AST | t.CPtr = _parse_and_test(ps)
if left is None: return None
# 括号内行首 or（(A\n or B)）：or 前的 NL 在此跳过，否则 or_test 返回后
# 上层 _expect_op(RPar) 遇到 or 报"意外的 token 'or'"
# （CallGenMixin.vp L7918 跨模块调用检查根因；ClassGenMixin L1988 等
# 括号内换行行首 or 同理，按解析顺序未暴露）。
_skip_nl(ps)
if _cur_kw(ps) == Keyword.Or:
values: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
values.append(left)
while _cur_kw(ps) == Keyword.Or:
_advance(ps)
_skip_nl(ps) # 括号内跨行: a or\n b
right: AST | t.CPtr = _parse_and_test(ps)
if right is None: break
values.append(right)
node: AST | t.CPtr = BoolOp(ps.pool, OpKind.Or, values)
_set_pos(node, left.lineno, left.col_offset, 0, 0)
return node
return left
def _parse_and_test(ps: Parser | t.CPtr) -> AST | t.CPtr:
"""and_test: not_test ('and' not_test)*"""
left: AST | t.CPtr = _parse_not_test(ps)
if left is None: return None
# 括号内行首 and（(A\n and B)）：同 or_test 修复（or 前跳 NL）
_skip_nl(ps)
if _cur_kw(ps) == Keyword.And:
values: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
values.append(left)
while _cur_kw(ps) == Keyword.And:
_advance(ps)
_skip_nl(ps) # 括号内跨行: a and\n b
right: AST | t.CPtr = _parse_not_test(ps)
if right is None: break
values.append(right)
node: AST | t.CPtr = BoolOp(ps.pool, OpKind.And, values)
_set_pos(node, left.lineno, left.col_offset, 0, 0)
return node
return left
def _parse_not_test(ps: Parser | t.CPtr) -> AST | t.CPtr:
"""not_test: 'not' not_test | comparison"""
if _cur_kw(ps) == Keyword.Not:
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps)
operand: AST | t.CPtr = _parse_not_test(ps)
node: AST | t.CPtr = UnaryOp(ps.pool, OpKind.Not, operand)
_set_pos(node, lineno, col, 0, 0)
return node
return _parse_comparison(ps)
def _parse_comparison(ps: Parser | t.CPtr) -> AST | t.CPtr:
"""comparison: bitor_expr (comp_op bitor_expr)*
Single comparison design: a < b → Compare(left=a, op=Lt, right=b)
Chained comparison a < b < c → BoolOp(And, [Compare(a,<,b), Compare(b,<,c)])
"""
left: AST | t.CPtr = _parse_bitor(ps)
if left is None: return None
op_id: t.CInt = _match_comp_op(ps)
if op_id == OpKind.NoneOp:
return left
_advance_comp_op(ps)
_skip_nl(ps) # 括号内跨行: a ==\n b
right: AST | t.CPtr = _parse_bitor(ps)
# Single comparison: construct ops and comparators list
ops: list[t.CInt] | t.CPtr = list[t.CInt](ps.pool, 8)
ops.append(op_id)
comparators: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
comparators.append(right)
first_cmp: AST | t.CPtr = Compare(ps.pool, left, ops, comparators)
_set_pos(first_cmp, left.lineno, left.col_offset, 0, 0)
# Check chained comparison
next_op: t.CInt = _match_comp_op(ps)
if next_op == OpKind.NoneOp:
return first_cmp
# Single comparison with chained conditions: a < b < c → BoolOp(And, [Compare(a,<,b), Compare(b,<,c), ...])
values: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
values.append(first_cmp)
cur_left: AST | t.CPtr = right
while next_op != OpKind.NoneOp:
_advance_comp_op(ps)
_skip_nl(ps) # 括号内跨行链式比较
cur_right: AST | t.CPtr = _parse_bitor(ps)
ops2: list[t.CInt] | t.CPtr = list[t.CInt](ps.pool, 8)
ops2.append(next_op)
comparators2: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
comparators2.append(cur_right)
cmp: AST | t.CPtr = Compare(ps.pool, cur_left, ops2, comparators2)
_set_pos(cmp, cur_left.lineno, cur_left.col_offset, 0, 0)
values.append(cmp)
cur_left = cur_right
next_op = _match_comp_op(ps)
node: AST | t.CPtr = BoolOp(ps.pool, OpKind.And, values)
_set_pos(node, left.lineno, left.col_offset, 0, 0)
return node
def _match_comp_op(ps: Parser | t.CPtr) -> t.CInt:
"""Check if the current one is a comparison operator, return OP_* or OpKind.NoneOp"""
if _cur_type(ps) == TokenType.Op:
op_id: t.CInt = _cur_op(ps)
if op_id == TokOp.EqEq: return OpKind.Eq
if op_id == TokOp.ExclaimEq: return OpKind.Ne
if op_id == TokOp.Less: return OpKind.Lt
if op_id == TokOp.LessEq: return OpKind.Le
if op_id == TokOp.Greater: return OpKind.Gt
if op_id == TokOp.GreaterEq: return OpKind.Ge
if _cur_kw(ps) == Keyword.Is:
if ps.cur.next is not None and ps.cur.next.type == TokenType.Name and ps.cur.next.kw_subtype == Keyword.Not:
return OpKind.IsNot
return OpKind.Is
if _cur_kw(ps) == Keyword.Not:
if ps.cur.next is not None and ps.cur.next.kw_subtype == Keyword.In:
return OpKind.NotIn
if _cur_kw(ps) == Keyword.In:
return OpKind.In
return OpKind.NoneOp
def _advance_comp_op(ps: Parser | t.CPtr):
"""Consume comparison operator token"""
if _cur_type(ps) == TokenType.Op:
_advance(ps)
return
if _cur_kw(ps) == Keyword.Is:
_advance(ps)
if _cur_kw(ps) == Keyword.Not:
_advance(ps)
return
if _cur_kw(ps) == Keyword.Not:
_advance(ps)
if _cur_kw(ps) == Keyword.In:
_advance(ps)
return
if _cur_kw(ps) == Keyword.In:
_advance(ps)
def _parse_bitor(ps: Parser | t.CPtr) -> AST | t.CPtr:
left: AST | t.CPtr = _parse_bitxor(ps)
if left is None: return None
while _match_op(ps, TokOp.VBar):
_advance(ps)
_skip_nl(ps)
right: AST | t.CPtr = _parse_bitxor(ps)
node: AST | t.CPtr = BinOp(ps.pool, left, OpKind.BitOr, right)
_set_pos(node, left.lineno, left.col_offset, 0, 0)
left = node
return left
def _parse_bitxor(ps: Parser | t.CPtr) -> AST | t.CPtr:
left: AST | t.CPtr = _parse_bitand(ps)
if left is None: return None
while _match_op(ps, TokOp.Caret):
_advance(ps)
_skip_nl(ps)
right: AST | t.CPtr = _parse_bitand(ps)
node: AST | t.CPtr = BinOp(ps.pool, left, OpKind.BitXor, right)
_set_pos(node, left.lineno, left.col_offset, 0, 0)
left = node
return left
def _parse_bitand(ps: Parser | t.CPtr) -> AST | t.CPtr:
left: AST | t.CPtr = _parse_shift(ps)
if left is None: return None
while _match_op(ps, TokOp.Amp):
_advance(ps)
_skip_nl(ps)
right: AST | t.CPtr = _parse_shift(ps)
node: AST | t.CPtr = BinOp(ps.pool, left, OpKind.BitAnd, right)
_set_pos(node, left.lineno, left.col_offset, 0, 0)
left = node
return left
def _parse_shift(ps: Parser | t.CPtr) -> AST | t.CPtr:
left: AST | t.CPtr = _parse_arith(ps)
if left is None: return None
while _match_op(ps, TokOp.LtLt) or _match_op(ps, TokOp.GtGt):
op_token: t.CInt = _cur_op(ps)
_advance(ps)
_skip_nl(ps)
right: AST | t.CPtr = _parse_arith(ps)
ast_op: t.CInt = OpKind.LShift
if op_token == TokOp.GtGt:
ast_op = OpKind.RShift
node: AST | t.CPtr = BinOp(ps.pool, left, ast_op, right)
_set_pos(node, left.lineno, left.col_offset, 0, 0)
left = node
return left
def _parse_arith(ps: Parser | t.CPtr) -> AST | t.CPtr:
left: AST | t.CPtr = _parse_term(ps)
if left is None: return None
while _match_op(ps, TokOp.Plus) or _match_op(ps, TokOp.Minus):
op_token: t.CInt = _cur_op(ps)
_advance(ps)
_skip_nl(ps)
right: AST | t.CPtr = _parse_term(ps)
ast_op: t.CInt = OpKind.Add
if op_token == TokOp.Minus:
ast_op = OpKind.Sub
node: AST | t.CPtr = BinOp(ps.pool, left, ast_op, right)
_set_pos(node, left.lineno, left.col_offset, 0, 0)
left = node
return left
def _parse_term(ps: Parser | t.CPtr) -> AST | t.CPtr:
left: AST | t.CPtr = _parse_factor(ps)
if left is None: return None
while True:
op_token: t.CInt = 0
if _match_op(ps, TokOp.Star): op_token = TokOp.Star
elif _match_op(ps, TokOp.Slash): op_token = TokOp.Slash
elif _match_op(ps, TokOp.DSlash): op_token = TokOp.DSlash
elif _match_op(ps, TokOp.Percent): op_token = TokOp.Percent
elif _match_op(ps, TokOp.At): op_token = TokOp.At
if op_token == 0: break
_advance(ps)
_skip_nl(ps)
right: AST | t.CPtr = _parse_factor(ps)
ast_op: t.CInt = _binop_from_op(op_token)
node: AST | t.CPtr = BinOp(ps.pool, left, ast_op, right)
_set_pos(node, left.lineno, left.col_offset, 0, 0)
left = node
return left
def _parse_factor(ps: Parser | t.CPtr) -> AST | t.CPtr:
"""factor: '+' factor | '-' factor | '~' factor | power"""
if _match_op(ps, TokOp.Plus) or _match_op(ps, TokOp.Minus) or _match_op(ps, TokOp.Tilde):
op_token: t.CInt = _cur_op(ps)
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps)
operand: AST | t.CPtr = _parse_factor(ps)
ast_op: t.CInt = OpKind.UAdd
if op_token == TokOp.Minus:
ast_op = OpKind.USub
elif op_token == TokOp.Tilde:
ast_op = OpKind.Invert
node: AST | t.CPtr = UnaryOp(ps.pool, ast_op, operand)
_set_pos(node, lineno, col, 0, 0)
return node
return _parse_power(ps)
def _parse_power(ps: Parser | t.CPtr) -> AST | t.CPtr:
"""power: await_expr ['**' factor] | atom_expr ['**' factor]"""
base: AST | t.CPtr = _parse_atom_expr(ps)
if base is None: return None
if _match_op(ps, TokOp.StarStar):
_advance(ps)
exp: AST | t.CPtr = _parse_factor(ps)
node: AST | t.CPtr = BinOp(ps.pool, base, OpKind.Pow, exp)
_set_pos(node, base.lineno, base.col_offset, 0, 0)
return node
return base
def _parse_atom_expr(ps: Parser | t.CPtr) -> AST | t.CPtr:
"""atom_expr: ['await'] atom trailer*"""
if _cur_kw(ps) == Keyword.Await:
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps)
inner: AST | t.CPtr = _parse_atom_expr(ps)
node: AST | t.CPtr = Await(ps.pool, inner)
_set_pos(node, lineno, col, 0, 0)
return node
base: AST | t.CPtr = _parse_atom(ps)
if base is None: return None
# trailer loop
while True:
if _match_op(ps, TokOp.Dot):
_advance(ps)
if _cur_type(ps) != TokenType.Name:
ps.error_count += 1
break
attr_name: str = _cur_str(ps)
attr_lineno: t.CInt = _cur_lineno(ps)
attr_col: t.CInt = _cur_col(ps)
_advance(ps)
ctx: t.CInt = ASTCtx.Load
if _match_op(ps, TokOp.Equal):
ctx = ASTCtx.Store
elif _match_op(ps, TokOp.PlusEq) or _match_op(ps, TokOp.MinusEq):
ctx = ASTCtx.Store
base = Attribute(ps.pool, base, attr_name, ctx)
_set_pos(base, attr_lineno, attr_col, 0, 0)
elif _match_op(ps, TokOp.LPar):
base = _parse_call(ps, base)
elif _match_op(ps, TokOp.Lsqb):
base = _parse_subscript(ps, base)
else:
break
return base
def _parse_call(ps: Parser | t.CPtr, func: AST | t.CPtr) -> AST | t.CPtr:
"""Parse function call trailer. Both args and keywords are lists."""
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps) # (
_skip_nls_and_newlines(ps) # Skip the line break after '('
args: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
keywords: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
if not _match_op(ps, TokOp.RPar):
while True:
# **expr
if _match_op(ps, TokOp.StarStar):
_advance(ps)
_skip_nls_and_newlines(ps)
val: AST | t.CPtr = _parse_expr(ps)
kw: AST | t.CPtr = KwNode(ps.pool, None, val)
_set_pos(kw, _cur_lineno(ps), _cur_col(ps), 0, 0)
keywords.append(kw)
# *expr
elif _match_op(ps, TokOp.Star):
_advance(ps)
_skip_nls_and_newlines(ps)
val = _parse_expr(ps)
star: AST | t.CPtr = Starred(ps.pool, val, ASTCtx.Load)
_set_pos(star, val.lineno, val.col_offset, 0, 0)
args.append(star)
# keyword = expr
elif _cur_type(ps) == TokenType.Name:
if ps.cur.next is not None and ps.cur.next.type == TokenType.Op and ps.cur.next.op_subtype == TokOp.Equal:
kw_name: str = _cur_str(ps)
kw_lineno: t.CInt = _cur_lineno(ps)
kw_col: t.CInt = _cur_col(ps)
_advance(ps) # name
_advance(ps) # =
_skip_nls_and_newlines(ps)
val = _parse_expr(ps)
kw = KwNode(ps.pool, kw_name, val)
_set_pos(kw, kw_lineno, kw_col, 0, 0)
keywords.append(kw)
else:
val = _parse_expr(ps)
args.append(val)
else:
val = _parse_expr(ps)
args.append(val)
_skip_nls_and_newlines(ps) # Skip the line break after the parameter
if not _accept_op(ps, TokOp.Comma):
break
_skip_nls_and_newlines(ps) # Skip the line break after the comma
_skip_nls_and_newlines(ps) # Skip the line break before ')'
_expect_op(ps, TokOp.RPar)
node: AST | t.CPtr = Call(ps.pool, func, args, keywords)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_subscript(ps: Parser | t.CPtr, value: AST | t.CPtr) -> AST | t.CPtr:
"""Parse subscript trailer."""
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps) # [
_skip_nl(ps)
slice_node: AST | t.CPtr = _parse_slice(ps)
_expect_op(ps, TokOp.Rsqb)
ctx: t.CInt = ASTCtx.Load
if _match_op(ps, TokOp.Equal):
ctx = ASTCtx.Store
node: AST | t.CPtr = Subscript(ps.pool, value, slice_node, ctx)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_slice(ps: Parser | t.CPtr) -> AST | t.CPtr:
"""Parse slice: [expr] ':' [expr] [':' [expr]] | expr | expr, expr, ..."""
lower: AST | t.CPtr = None
if not _match_op(ps, TokOp.Colon):
lower = _parse_expr(ps)
# Check if there’s a comma (tuple syntax: a[x, y, ...])
if _match_op(ps, TokOp.Comma):
elts: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
if lower is not None:
elts.append(lower)
while _accept_op(ps, TokOp.Comma):
if _match_op(ps, TokOp.Rsqb):
break # Trailing comma
elem: AST | t.CPtr = _parse_expr(ps)
if elem is not None:
elts.append(elem)
lineno: t.CInt = 0
if lower is not None:
lineno = lower.lineno
node: AST | t.CPtr = Tuple(ps.pool, elts, ASTCtx.Load)
_set_pos(node, lineno, 0, 0, 0)
return node
if _match_op(ps, TokOp.Colon):
_advance(ps)
upper: AST | t.CPtr = None
if not _match_op(ps, TokOp.Colon) and not _match_op(ps, TokOp.Rsqb):
upper = _parse_expr(ps)
step: AST | t.CPtr = None
if _accept_op(ps, TokOp.Colon):
if not _match_op(ps, TokOp.Rsqb):
step = _parse_expr(ps)
lineno: t.CInt = 0
if lower is not None:
lineno = lower.lineno
elif upper is not None:
lineno = upper.lineno
else:
lineno = _cur_lineno(ps)
node: AST | t.CPtr = Slice(ps.pool, lower, upper, step)
_set_pos(node, lineno, 0, 0, 0)
return node
return lower
def _parse_atom(ps: Parser | t.CPtr) -> AST | t.CPtr:
"""atom: the smallest unit of expression"""
ttype: t.CInt = _cur_type(ps)
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
if ttype == TokenType.Name:
kw: t.CInt = _cur_kw(ps)
if kw == Keyword.True_:
_advance(ps)
return _new_const_bool(ps.pool, 1, lineno, col)
if kw == Keyword.False_:
_advance(ps)
return _new_const_bool(ps.pool, 0, lineno, col)
if kw == Keyword.None_:
_advance(ps)
return _new_const_none(ps.pool, lineno, col)
name_str: str = _cur_str(ps)
_advance(ps)
node: AST | t.CPtr = Name(ps.pool, name_str, ASTCtx.Load)
_set_pos(node, lineno, col, 0, 0)
return node
if ttype == TokenType.Number:
tok: Token | t.CPtr = ps.cur
_advance(ps)
if tok.is_float:
return _new_const_float(ps.pool, tok.float_val, lineno, col)
return _new_const_int(ps.pool, tok.int_val, lineno, col)
if ttype == TokenType.String:
sval: str = _cur_str(ps)
is_fstr: t.CInt = ps.cur.kw_subtype & 4
is_triple: t.CInt = ps.cur.kw_subtype & 8
is_dquote: t.CInt = ps.cur.kw_subtype & 16
is_char: t.CInt = ps.cur.kw_subtype & 32
_advance(ps)
node: AST | t.CPtr
if is_fstr:
node = _parse_fstring(ps, sval, lineno, col)
else:
node = _new_const_str(ps.pool, sval, lineno, col, is_dquote, is_triple, is_char)
# Python 相邻字符串字面量隐式拼接："a" "b" → "ab"
# Nl 是括号内换行 token（允许括号内跨行拼接）；语句级换行是 NewLine，不跳过
while True:
_skip_nl(ps)
if _cur_type(ps) != TokenType.String:
break
n_sval: str = _cur_str(ps)
n_fstr: t.CInt = ps.cur.kw_subtype & 4
n_triple: t.CInt = ps.cur.kw_subtype & 8
n_dquote: t.CInt = ps.cur.kw_subtype & 16
n_char: t.CInt = ps.cur.kw_subtype & 32
_advance(ps)
n_node: AST | t.CPtr
if n_fstr:
n_node = _parse_fstring(ps, n_sval, lineno, col)
else:
n_node = _new_const_str(ps.pool, n_sval, lineno, col, n_dquote, n_triple, n_char)
node = _concat_str_nodes(ps.pool, node, n_node)
return node
if _match_op(ps, TokOp.Ellipsis):
_advance(ps)
return _new_const_none(ps.pool, lineno, col)
if _match_op(ps, TokOp.LPar):
return _parse_paren(ps)
if _match_op(ps, TokOp.Lsqb):
return _parse_list(ps)
if _match_op(ps, TokOp.LBrace):
return _parse_brace(ps)
ps.error_count += 1
return None
def _parse_paren(ps: Parser | t.CPtr) -> AST | t.CPtr:
"""Parenthesis expressions: () | (expr) | (expr,) | (expr, ...) | (yield)"""
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps) # (
_skip_nl(ps) # Skip the line break after '(' (multi-line tuple)
if _match_op(ps, TokOp.RPar):
_advance(ps)
# Empty tuple
empty: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
node: AST | t.CPtr = Tuple(ps.pool, empty, ASTCtx.Load)
_set_pos(node, lineno, col, 0, 0)
return node
# yield
if _cur_kw(ps) == Keyword.Yield:
y: AST | t.CPtr = _parse_yield(ps)
_expect_op(ps, TokOp.RPar)
return y
first: AST | t.CPtr = _parse_expr(ps)
if _match_op(ps, TokOp.Comma):
# Tuple
elts: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
elts.append(first)
while _accept_op(ps, TokOp.Comma):
_skip_nl(ps)
if _match_op(ps, TokOp.RPar):
break
e: AST | t.CPtr = _parse_expr(ps)
elts.append(e)
_skip_nl(ps) # 末元素无尾逗号时收尾（多行元组）
_expect_op(ps, TokOp.RPar)
node = Tuple(ps.pool, elts, ASTCtx.Load)
_set_pos(node, lineno, col, 0, 0)
return node
_skip_nl(ps) # (expr\n) 单表达式括号跨行收尾
_expect_op(ps, TokOp.RPar)
# Check for generator expression
if _cur_kw(ps) == Keyword.For:
return _parse_comprehension(ps, first, ASTKind.GeneratorExp, lineno, col)
return first
def _parse_list(ps: Parser | t.CPtr) -> AST | t.CPtr:
"""List expressions: [] | [expr_list] | [expr_list comp_for]"""
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps) # [
_skip_nl(ps) # Skip the line break after '[' (multi-line list)
if _match_op(ps, TokOp.Rsqb):
_advance(ps)
empty: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
node: AST | t.CPtr = List(ps.pool, empty, ASTCtx.Load)
_set_pos(node, lineno, col, 0, 0)
return node
first: AST | t.CPtr = _parse_expr(ps)
if _cur_kw(ps) == Keyword.For:
node = _parse_comprehension(ps, first, ASTKind.ListComp, lineno, col)
_expect_op(ps, TokOp.Rsqb)
return node
elts: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
elts.append(first)
while _accept_op(ps, TokOp.Comma):
_skip_nl(ps)
if _match_op(ps, TokOp.Rsqb):
break
e: AST | t.CPtr = _parse_expr(ps)
elts.append(e)
_skip_nl(ps) # 末元素无尾逗号时收尾（base64.vp 多行列表）
_expect_op(ps, TokOp.Rsqb)
node = List(ps.pool, elts, ASTCtx.Load)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_brace(ps: Parser | t.CPtr) -> AST | t.CPtr:
"""Brace expressions: {} | {set_items} | {dict_items} | {comp_for}"""
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps) # {
_skip_nl(ps) # Skip the line break after '{' (multi-line dict)
if _match_op(ps, TokOp.RBrace):
_advance(ps)
empty_k: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
empty_v: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
node: AST | t.CPtr = Dict(ps.pool, empty_k, empty_v)
_set_pos(node, lineno, col, 0, 0)
return node
first: AST | t.CPtr = _parse_expr(ps)
# dict: key: value
if _match_op(ps, TokOp.Colon):
_advance(ps)
val: AST | t.CPtr = _parse_expr(ps)
if _cur_kw(ps) == Keyword.For:
# dict comprehension
gens: list[AST | t.CPtr] | t.CPtr = _parse_comp_for(ps)
node = DictComp(ps.pool, first, val, gens)
_set_pos(node, lineno, col, 0, 0)
_expect_op(ps, TokOp.RBrace)
return node
keys: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
values: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
keys.append(first)
values.append(val)
while _accept_op(ps, TokOp.Comma):
_skip_nl(ps)
if _match_op(ps, TokOp.RBrace):
break
k: AST | t.CPtr = _parse_expr(ps)
_expect_op(ps, TokOp.Colon)
v: AST | t.CPtr = _parse_expr(ps)
keys.append(k)
values.append(v)
_skip_nl(ps) # 末元素无尾逗号时收尾
_expect_op(ps, TokOp.RBrace)
node = Dict(ps.pool, keys, values)
_set_pos(node, lineno, col, 0, 0)
return node
# set
if _cur_kw(ps) == Keyword.For:
node = _parse_comprehension(ps, first, ASTKind.SetComp, lineno, col)
_expect_op(ps, TokOp.RBrace)
return node
elts: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
elts.append(first)
while _accept_op(ps, TokOp.Comma):
_skip_nl(ps) # 括号内跨行集合字面量
if _match_op(ps, TokOp.RBrace):
break
e: AST | t.CPtr = _parse_expr(ps)
elts.append(e)
_skip_nl(ps) # 末元素无尾逗号时收尾
_expect_op(ps, TokOp.RBrace)
node = Set(ps.pool, elts)
_set_pos(node, lineno, col, 0, 0)
return node
# ============================================================
# f-string parsing
# ============================================================
def _parse_sub_expr(ps: Parser | t.CPtr, src: str) -> AST | t.CPtr:
"""Create a sub-lexer and parser to parse expressions (used for {expr} inside f-strings)"""
pool: memhub.MemManager | t.CPtr = ps.pool
lx: Lexer | t.CPtr = new_lexer(pool)
if lx is None: return None
_lexer_init(lx, src, pool)
tokens: Token | t.CPtr = tokenize(lx)
if tokens is None: return None
sub_ps: Parser | t.CPtr = new_parser(pool)
if sub_ps is None: return None
_parser_init(sub_ps, tokens, pool)
return _parse_test(sub_ps)
def _append_str_value(pool: memhub.MemManager | t.CPtr,
values: list[AST | t.CPtr] | t.CPtr,
node: AST | t.CPtr):
"""将字符串节点展开加入 values（JoinedStr 展开其 values，其余直接追加）"""
if node is None:
return
if node.kind() == ASTKind.JoinedStr:
js_val: JoinedStr | t.CPtr = (JoinedStr | t.CPtr)(node)
if js_val.values is not None:
n_v: t.CSizeT = js_val.values.__len__()
i_v: t.CSizeT
for i_v in range(n_v):
values.append(js_val.values.get(i_v))
else:
values.append(node)
def _concat_str_nodes(pool: memhub.MemManager | t.CPtr,
a: AST | t.CPtr, b: AST | t.CPtr) -> AST | t.CPtr:
"""合并相邻字符串字面量节点（Python 隐式拼接 "a" "b" → "ab"）
纯常量字符串直接拼接文本；涉及 f-string（JoinedStr/FormattedValue）时
合并到 JoinedStr 的 values 列表。
"""
if a is None:
return b
if b is None:
return a
# 两个都是常量字符串 → 直接拼接文本
if a.kind() == ASTKind.Constant and b.kind() == ASTKind.Constant:
ca_c: Constant | t.CPtr = (Constant | t.CPtr)(a)
cb_c: Constant | t.CPtr = (Constant | t.CPtr)(b)
if ca_c.const_kind == CONST_STR and cb_c.const_kind == CONST_STR \
and ca_c.str_val is not None and cb_c.str_val is not None:
la_c: t.CSizeT = string.strlen(ca_c.str_val)
lb_c: t.CSizeT = string.strlen(cb_c.str_val)
buf_c: str = pool.alloc(la_c + lb_c + 1)
if buf_c is not None:
string.memcpy(buf_c, ca_c.str_val, la_c)
string.memcpy(buf_c + la_c, cb_c.str_val, lb_c + 1)
return _new_const_str(pool, buf_c, a.lineno, a.col_offset, 0, 0, 0)
# 涉及 f-string → 合并到 JoinedStr
values_c: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](pool, 8)
_append_str_value(pool, values_c, a)
_append_str_value(pool, values_c, b)
if values_c.__len__() == 1:
return values_c.get(0)
node_c: AST | t.CPtr = JoinedStr(pool, values_c)
_set_pos(node_c, a.lineno, a.col_offset, 0, 0)
return node_c
def _parse_fstring(ps: Parser | t.CPtr, content: str,
lineno: t.CInt, col: t.CInt) -> AST | t.CPtr:
"""Parse f-string content, building JoinedStr or FormattedValue nodes"""
pool: memhub.MemManager | t.CPtr = ps.pool
values: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](pool, 8)
i: t.CSizeT = 0
content_len: t.CSizeT = string.strlen(content)
# Literal text buffer for f-string content
lit_buf: str = pool.alloc(content_len + 1)
if lit_buf is None: return None
lit_pos: t.CSizeT = 0
while i < content_len:
ch: t.CChar = content[i]
if ch == '{':
# {{ → Literal {
if i + 1 < content_len and content[i + 1] == '{':
lit_buf[lit_pos] = '{'
lit_pos += 1
i += 2
continue
# Refresh literal buffer
if lit_pos > 0:
lit_buf[lit_pos] = '\0'
lit_node: AST | t.CPtr = _new_const_str(pool, lit_buf, lineno, col)
values.append(lit_node)
lit_buf = pool.alloc(content_len + 1)
if lit_buf is None: break
lit_pos = 0
# Find matching closing '}' while tracking nested '{}' and string literals inside the f-string
j: t.CSizeT = i + 1
depth: t.CInt = 1
conv_pos: t.CSizeT = 0
fmt_pos: t.CSizeT = 0
while j < content_len and depth > 0:
c: t.CChar = content[j]
if c == '{':
depth += 1
elif c == '}':
depth -= 1
if depth == 0:
break
elif depth == 1:
if c == '!' and conv_pos == 0 and fmt_pos == 0:
if j + 1 < content_len:
nc: t.CChar = content[j + 1]
if nc == 's' or nc == 'r' or nc == 'a':
if j + 2 < content_len:
after: t.CChar = content[j + 2]
if after == ':' or after == '}':
conv_pos = j
elif j + 2 == content_len:
conv_pos = j
elif c == ':' and fmt_pos == 0:
fmt_pos = j
j += 1
if depth != 0:
ps.error_count += 1
break
# Extract expression text
expr_end: t.CSizeT = j
if conv_pos > 0:
expr_end = conv_pos
elif fmt_pos > 0:
expr_end = fmt_pos
expr_len: t.CSizeT = expr_end - i - 1
if expr_len == 0:
ps.error_count += 1
i = j + 1
continue
expr_buf: str = pool.alloc(expr_len + 1)
if expr_buf is None: break
string.memcpy(expr_buf, content + i + 1, expr_len)
expr_buf[expr_len] = '\0'
# Parse expression text
expr_node: AST | t.CPtr = _parse_sub_expr(ps, expr_buf)
# Conversion specifier
conversion: t.CInt = -1
if conv_pos > 0:
conv_char: t.CChar = content[conv_pos + 1]
if conv_char == 's':
conversion = 115
elif conv_char == 'r':
conversion = 114
elif conv_char == 'a':
conversion = 97
# Format specifier
format_spec: AST | t.CPtr = None
if fmt_pos > 0:
spec_len: t.CSizeT = j - fmt_pos - 1
if spec_len > 0:
spec_buf: str = pool.alloc(spec_len + 1)
if spec_buf is not None:
string.memcpy(spec_buf, content + fmt_pos + 1, spec_len)
spec_buf[spec_len] = '\0'
format_spec = _new_const_str(pool, spec_buf, lineno, col)
fv: AST | t.CPtr = FormattedValue(pool, expr_node, conversion, format_spec)
_set_pos(fv, lineno, col, 0, 0)
values.append(fv)
i = j + 1
elif ch == '}':
if i + 1 < content_len and content[i + 1] == '}':
lit_buf[lit_pos] = '}'
lit_pos += 1
i += 2
else:
ps.error_count += 1
i += 1
else:
lit_buf[lit_pos] = ch
lit_pos += 1
i += 1
# Refresh remaining literal buffer
if lit_pos > 0:
lit_buf[lit_pos] = '\0'
lit_node2: AST | t.CPtr = _new_const_str(pool, lit_buf, lineno, col)
values.append(lit_node2)
# If no content, return empty string
if values.__len__() == 0:
return _new_const_str(pool, "", lineno, col)
# If only one FormattedValue and no literal, return FormattedValue
if values.__len__() == 1:
first: AST | t.CPtr = values.get(0)
if first.kind() == ASTKind.FormattedValue:
return first
node: AST | t.CPtr = JoinedStr(pool, values)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_comprehension(ps: Parser | t.CPtr, elt: AST | t.CPtr,
vtype: t.CInt, lineno: t.CInt, col: t.CInt) -> AST | t.CPtr:
"""General comprehension: elt comp_for"""
gens: list[AST | t.CPtr] | t.CPtr = _parse_comp_for(ps)
node: AST | t.CPtr = None
if vtype == ASTKind.ListComp:
node = ListComp(ps.pool, elt, gens)
elif vtype == ASTKind.SetComp:
node = SetComp(ps.pool, elt, gens)
else:
node = GeneratorExp(ps.pool, elt, gens)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_comp_for(ps: Parser | t.CPtr) -> list[AST | t.CPtr] | t.CPtr:
"""comp_for: ['async'] 'for' exprlist 'in' or_test [comp_if] (comp_for)*
Returns generators list。"""
gens: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
while True:
is_async: t.CInt = 0
if _cur_kw(ps) == Keyword.Async:
is_async = 1
_advance(ps)
if _cur_kw(ps) != Keyword.For:
break
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps)
target: AST | t.CPtr = _parse_target_list(ps)
if _cur_kw(ps) != Keyword.In:
ps.error_count += 1
break
_advance(ps)
iter_: AST | t.CPtr = _parse_or_test(ps)
# Collect ifs to list
ifs: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
while _cur_kw(ps) == Keyword.If:
_advance(ps)
cond: AST | t.CPtr = _parse_or_test(ps)
ifs.append(cond)
comp: AST | t.CPtr = Comprehension(ps.pool, target, iter_, ifs, is_async)
_set_pos(comp, lineno, col, 0, 0)
gens.append(comp)
return gens
def _parse_target_list(ps: Parser | t.CPtr) -> AST | t.CPtr:
"""Parse assignment target list (comma-separated)"""
first: AST | t.CPtr = _parse_target(ps)
if _match_op(ps, TokOp.Comma):
elts: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
elts.append(first)
while _accept_op(ps, TokOp.Comma):
if _cur_kw(ps) == Keyword.In:
break
e: AST | t.CPtr = _parse_target(ps)
elts.append(e)
node: AST | t.CPtr = Tuple(ps.pool, elts, ASTCtx.Store)
_set_pos(node, first.lineno, first.col_offset, 0, 0)
return node
return first
def _parse_target(ps: Parser | t.CPtr) -> AST | t.CPtr:
"""Parse assignment target"""
if _match_op(ps, TokOp.Star):
_advance(ps)
inner: AST | t.CPtr = _parse_atom_expr(ps)
node: AST | t.CPtr = Starred(ps.pool, inner, ASTCtx.Store)
_set_pos(node, inner.lineno, inner.col_offset, 0, 0)
return node
if _match_op(ps, TokOp.LPar):
_advance(ps)
_skip_nl(ps)
t_: AST | t.CPtr = _parse_target_list(ps)
_skip_nl(ps)
_expect_op(ps, TokOp.RPar)
return t_
if _match_op(ps, TokOp.Lsqb):
_advance(ps)
_skip_nl(ps)
first: AST | t.CPtr = _parse_target(ps)
elts: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
elts.append(first)
while _accept_op(ps, TokOp.Comma):
_skip_nl(ps)
if _match_op(ps, TokOp.Rsqb):
break
e: AST | t.CPtr = _parse_target(ps)
elts.append(e)
_expect_op(ps, TokOp.Rsqb)
node = List(ps.pool, elts, ASTCtx.Store)
_set_pos(node, first.lineno, first.col_offset, 0, 0)
return node
# Normal atom_expr, ctx=Store
node = _parse_atom_expr(ps)
if node is not None:
_set_ctx(node, ASTCtx.Store)
return node
def _parse_yield(ps: Parser | t.CPtr) -> AST | t.CPtr:
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps) # yield
if _cur_kw(ps) == Keyword.From:
_advance(ps)
val: AST | t.CPtr = _parse_expr(ps)
node: AST | t.CPtr = YieldFrom(ps.pool, val)
_set_pos(node, lineno, col, 0, 0)
return node
val2: AST | t.CPtr = None
if not _match_op(ps, TokOp.RPar) and _cur_type(ps) != TokenType.NewLine and _cur_type(ps) != TokenType.EndMarker:
if _cur_type(ps) == TokenType.Op and (_cur_op(ps) == TokOp.Comma or _cur_op(ps) == TokOp.Rsqb):
pass
else:
val2 = _parse_test(ps)
node = Yield(ps.pool, val2)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_varargslist(ps: Parser | t.CPtr, is_lambda: t.CInt = 0) -> AST | t.CPtr:
"""Parse function parameter list
New API: args/defaults/kw_defaults are lists, vararg/kwarg are fields."""
args: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
vararg: AST | t.CPtr = None
kwarg: AST | t.CPtr = None
defaults: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
kw_defaults: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
while True:
_skip_nl(ps)
if _match_op(ps, TokOp.RPar):
break
# *args 或 * (kwonly separated)
if _match_op(ps, TokOp.Star):
_advance(ps)
_skip_nl(ps)
if _cur_type(ps) == TokenType.Name:
vararg = _parse_arg_def(ps, is_lambda)
if not _accept_op(ps, TokOp.Comma):
break
_skip_nl(ps)
continue
# **kwargs
if _match_op(ps, TokOp.StarStar):
_advance(ps)
_skip_nl(ps)
kwarg = _parse_arg_def(ps, is_lambda)
if not _accept_op(ps, TokOp.Comma):
break
_skip_nl(ps)
continue
# / Delimiter (posonly end)— New API: posonly/regular are not distinguished
if _match_op(ps, TokOp.Slash):
_advance(ps)
if not _accept_op(ps, TokOp.Comma):
break
_skip_nl(ps)
continue
# Normal parameter
a: AST | t.CPtr = _parse_arg_def(ps, is_lambda)
if a is None: break
args.append(a)
if _match_op(ps, TokOp.Equal):
_advance(ps)
d: AST | t.CPtr = _parse_test(ps)
defaults.append(d)
if not _accept_op(ps, TokOp.Comma):
break
_skip_nl(ps)
return Arguments(ps.pool, args, vararg, kwarg, defaults, kw_defaults)
def _parse_arg_def(ps: Parser | t.CPtr, is_lambda: t.CInt = 0) -> AST | t.CPtr:
"""arg: TokenType.Name [':' annotation]"""
if _cur_type(ps) != TokenType.Name:
ps.error_count += 1
return None
name_str: str = _cur_str(ps)
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps)
annotation: AST | t.CPtr = None
if is_lambda == 0:
if _accept_op(ps, TokOp.Colon):
annotation = _parse_test(ps)
result: AST | t.CPtr = Arg(ps.pool, name_str, annotation)
_set_pos(result, lineno, col, 0, 0)
return result
# ============================================================
# Statement parsing (Design: directly append to parent, do not return a list)
# ============================================================
def _parse_suite_into_children(ps: Parser | t.CPtr, parent: AST | t.CPtr):
"""Parse statement suite, directly append to parent's children.
suite: TokenType.NewLine TokenType.Indent stmt+ TokenType.Dedent | simple_stmt"""
_skip_nl(ps)
if _cur_type(ps) == TokenType.NewLine:
_advance(ps)
_skip_nl(ps)
if _cur_type(ps) != TokenType.Indent:
ps.error_count += 1
return
_advance(ps)
sidx: t.CInt = 0
while _cur_type(ps) != TokenType.Dedent and _cur_type(ps) != TokenType.EndMarker:
sidx += 1
# Decorator (class body methods can have decorators)
if _match_op(ps, TokOp.At):
ps.pending_decorators = _parse_decorators(ps)
before_cur: Token | t.CPtr = ps.cur
_parse_statement_into(ps, parent)
# Defensive: if parse did not advance token, force advance to avoid infinite loop
if ps.cur is before_cur:
ps.error_count += 1
_advance(ps)
_skip_nl(ps)
_expect_type(ps, TokenType.Dedent)
return
# Single-line suite
_parse_simple_stmt_into(ps, parent)
def _parse_statement_into(ps: Parser | t.CPtr, parent: AST | t.CPtr):
"""Parse statement (compound or simple), append to the parent."""
ttype: t.CInt = _cur_type(ps)
kw: t.CInt = _cur_kw(ps)
if ttype == TokenType.Name:
if kw == Keyword.If or kw == Keyword.While or kw == Keyword.For or kw == Keyword.Try:
stmt: AST | t.CPtr = _parse_compound(ps)
if stmt is not None:
parent.append(stmt)
return
if kw == Keyword.With or _is_funcdef_kw(ps, kw) != 0 or kw == Keyword.Class:
stmt = _parse_compound(ps)
if stmt is not None:
parent.append(stmt)
return
if (kw == Keyword.Match and _is_match_stmt(ps) != 0) or kw == Keyword.Async:
stmt = _parse_compound(ps)
if stmt is not None:
parent.append(stmt)
return
_parse_simple_stmt_into(ps, parent)
def _parse_simple_stmt_into(ps: Parser | t.CPtr, parent: AST | t.CPtr):
"""simple_stmt: small_stmt (';' small_stmt)* [';'] TokenType.NewLine"""
first: AST | t.CPtr = _parse_small_stmt(ps)
if first is not None:
parent.append(first)
while _accept_op(ps, TokOp.Semi):
if _cur_type(ps) == TokenType.NewLine or _cur_type(ps) == TokenType.EndMarker:
break
s: AST | t.CPtr = _parse_small_stmt(ps)
if s is None: break
parent.append(s)
# 先跳过 Nl（括号内换行，如多行函数调用参数间的换行，属合法 token）
_skip_nl(ps)
# 语句结束检查：simple_stmt 必须以换行/文件尾结束。
# 残留 token（如相邻表达式 `func12344444 avg3(...)` 被静默拆成多条语句、
# `def` 缺失的关键字函数定义残留）是语法错误（fail-fast），禁止静默，
# 并仿 Python 报告错误位置（SyntaxError at line/col）。
if _cur_type(ps) != TokenType.NewLine and _cur_type(ps) != TokenType.EndMarker:
_record_parse_err(ps)
err_str: str = _cur_str(ps)
if err_str is None:
err_str = "(none)"
# 定位：从链表头找 cur，记录前一个 token 与下一个 token 的上下文
prev_tok: Token | t.CPtr = None
walk_tok: Token | t.CPtr = ps.tokens
while walk_tok is not None and walk_tok is not ps.cur:
prev_tok = walk_tok
walk_tok = walk_tok.next
pv_str: str = "(head)"
pv_line: t.CInt = 0
pv_col: t.CInt = 0
if prev_tok is not None:
pv_str = prev_tok.str_val
pv_line = prev_tok.lineno
pv_col = prev_tok.col_offset
nxt_tok: Token | t.CPtr = ps.cur.next
nxt_str: str = "(none)"
nxt_line: t.CInt = 0
nxt_col: t.CInt = 0
if nxt_tok is not None:
nxt_str = nxt_tok.str_val
nxt_line = nxt_tok.lineno
nxt_col = nxt_tok.col_offset
stdio.printf("[AST] SyntaxError: line %d col %d (end %d %d), 意外的 token '%s' (type=%d) | prev='%s' (line %d col %d) | next='%s' (line %d col %d)\n",
_g_parse_err_line, _g_parse_err_col, ps.cur.end_lineno, ps.cur.end_col_offset,
err_str, _cur_type(ps),
pv_str, pv_line, pv_col,
nxt_str, nxt_line, nxt_col)
ps.error_count += 1
return
if _cur_type(ps) == TokenType.NewLine:
_advance(ps)
def _parse_small_stmt(ps: Parser | t.CPtr) -> AST | t.CPtr:
"""small_stmt: various simple statements"""
kw: t.CInt = _cur_kw(ps)
if kw == Keyword.Pass:
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps)
node: AST | t.CPtr = Pass(ps.pool)
_set_pos(node, lineno, col, 0, 0)
return node
if kw == Keyword.Break:
lineno = _cur_lineno(ps)
col = _cur_col(ps)
_advance(ps)
node = Break(ps.pool)
_set_pos(node, lineno, col, 0, 0)
return node
if kw == Keyword.Continue:
lineno = _cur_lineno(ps)
col = _cur_col(ps)
_advance(ps)
node = Continue(ps.pool)
_set_pos(node, lineno, col, 0, 0)
return node
if kw == Keyword.Return:
return _parse_return(ps)
if kw == Keyword.Raise:
return _parse_raise(ps)
if kw == Keyword.Global:
return _parse_global(ps)
if kw == Keyword.Nonlocal:
return _parse_nonlocal(ps)
if kw == Keyword.Import:
return _parse_import(ps)
if kw == Keyword.From:
return _parse_from_import(ps)
if kw == Keyword.Del:
return _parse_del(ps)
if kw == Keyword.Assert:
return _parse_assert(ps)
if kw == Keyword.Yield:
return _parse_expr_stmt(ps, _parse_yield(ps))
# Expression statement / assignment / augassign / annassign
return _parse_expr_stmt(ps, None)
def _maybe_tuple(ps: Parser | t.CPtr, first: AST | t.CPtr) -> AST | t.CPtr:
"""If there is a comma after first, collect the comma-separated expressions into a Tuple."""
if first is None: return None
if not _match_op(ps, TokOp.Comma):
return first
lineno: t.CInt = first.lineno
col: t.CInt = first.col_offset
elts: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
elts.append(first)
_accept_op(ps, TokOp.Comma)
while True:
tt: t.CInt = _cur_type(ps)
if tt == TokenType.NewLine or tt == TokenType.EndMarker:
break
if tt == TokenType.Op:
op: t.CInt = _cur_op(ps)
if op == TokOp.Equal or op == TokOp.Semi:
break
e: AST | t.CPtr = _parse_expr(ps)
if e is None: break
elts.append(e)
if not _accept_op(ps, TokOp.Comma):
break
node: AST | t.CPtr = Tuple(ps.pool, elts, ASTCtx.Load)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_testlist(ps: Parser | t.CPtr) -> AST | t.CPtr:
"""Parse a comma-separated list of expressions, if there’s more than one, form a tuple."""
first: AST | t.CPtr = _parse_expr(ps)
return _maybe_tuple(ps, first)
def _parse_expr_stmt(ps: Parser | t.CPtr, first: AST | t.CPtr) -> AST | t.CPtr:
"""Expression statement, handle assignment/augassign/annassign"""
if first is None:
first = _parse_testlist(ps)
else:
first = _maybe_tuple(ps, first)
if first is None:
return None
lineno: t.CInt = first.lineno
col: t.CInt = first.col_offset
# Assignment
if _match_op(ps, TokOp.Equal):
targets: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
targets.append(first)
_fix_store_ctx(first)
value: AST | t.CPtr = None
accept_result: t.CInt = _accept_op(ps, TokOp.Equal)
while accept_result:
v: AST | t.CPtr = _parse_testlist(ps)
if v is None: break
if value is not None:
targets.append(value)
_fix_store_ctx(value)
value = v
accept_result = _accept_op(ps, TokOp.Equal)
node: AST | t.CPtr = Assign(ps.pool, targets, value)
_set_pos(node, lineno, col, 0, 0)
return node
# augassign
if _cur_type(ps) == TokenType.Op:
op_id: t.CInt = _cur_op(ps)
aug_op: t.CInt = _augop_from_op(op_id)
if aug_op != OpKind.NoneOp:
_advance(ps)
value2: AST | t.CPtr = _parse_testlist(ps)
_fix_store_ctx(first)
node2: AST | t.CPtr = AugAssign(ps.pool, first, aug_op, value2)
_set_pos(node2, lineno, col, 0, 0)
return node2
# annassign: target ':' annotation [= value]
if _match_op(ps, TokOp.Colon):
_advance(ps)
annotation: AST | t.CPtr = _parse_test(ps)
value3: AST | t.CPtr = None
if _accept_op(ps, TokOp.Equal):
value3 = _parse_testlist(ps)
if ps.pool is None:
return None
_fix_store_ctx(first)
if ps.pool is None:
return None
node3: AST | t.CPtr = AnnAssign(ps.pool, first, annotation, value3, 1)
_set_pos(node3, lineno, col, 0, 0)
return node3
# Expression statement
node4: AST | t.CPtr = Expr(ps.pool, first)
_set_pos(node4, lineno, col, 0, 0)
return node4
def _parse_return(ps: Parser | t.CPtr) -> AST | t.CPtr:
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps)
val: AST | t.CPtr = None
if _cur_type(ps) != TokenType.NewLine and _cur_type(ps) != TokenType.EndMarker and _cur_type(ps) != TokenType.Op:
val = _parse_testlist(ps)
elif _cur_type(ps) == TokenType.Op and _cur_op(ps) != TokOp.Semi:
val = _parse_testlist(ps)
node: AST | t.CPtr = Return(ps.pool, val)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_raise(ps: Parser | t.CPtr) -> AST | t.CPtr:
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps)
exc: AST | t.CPtr = None
cause: AST | t.CPtr = None
if _cur_type(ps) != TokenType.NewLine and _cur_type(ps) != TokenType.EndMarker:
exc = _parse_test(ps)
if _cur_kw(ps) == Keyword.From:
_advance(ps)
cause = _parse_test(ps)
node: AST | t.CPtr = Raise(ps.pool, exc, cause)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_global(ps: Parser | t.CPtr) -> AST | t.CPtr:
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps)
names: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
if _cur_type(ps) == TokenType.Name:
n: AST | t.CPtr = Name(ps.pool, _cur_str(ps), ASTCtx.Store)
_set_pos(n, _cur_lineno(ps), _cur_col(ps), 0, 0)
names.append(n)
_advance(ps)
while _accept_op(ps, TokOp.Comma):
if _cur_type(ps) != TokenType.Name: break
n = Name(ps.pool, _cur_str(ps), ASTCtx.Store)
_set_pos(n, _cur_lineno(ps), _cur_col(ps), 0, 0)
names.append(n)
_advance(ps)
node: AST | t.CPtr = Global(ps.pool, names)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_nonlocal(ps: Parser | t.CPtr) -> AST | t.CPtr:
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps)
names: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
if _cur_type(ps) == TokenType.Name:
n: AST | t.CPtr = Name(ps.pool, _cur_str(ps), ASTCtx.Store)
_set_pos(n, _cur_lineno(ps), _cur_col(ps), 0, 0)
names.append(n)
_advance(ps)
while _accept_op(ps, TokOp.Comma):
if _cur_type(ps) != TokenType.Name: break
n = Name(ps.pool, _cur_str(ps), ASTCtx.Store)
_set_pos(n, _cur_lineno(ps), _cur_col(ps), 0, 0)
names.append(n)
_advance(ps)
node: AST | t.CPtr = Nonlocal(ps.pool, names)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_import(ps: Parser | t.CPtr) -> AST | t.CPtr:
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps)
names: list[AST | t.CPtr] | t.CPtr = _parse_alias_list(ps)
node: AST | t.CPtr = Import(ps.pool, names)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_from_import(ps: Parser | t.CPtr) -> AST | t.CPtr:
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps)
# Parse module (possibly with dots)
level: t.CInt = 0
while _match_op(ps, TokOp.Dot):
level += 1
_advance(ps)
while _match_op(ps, TokOp.Ellipsis):
level += 3
_advance(ps)
module_name: str = None
# The token type for keywords (like import) is also Name, so they must be excluded using kw_subtype,
# otherwise the `import` in `from . import` would be mistakenly treated as a module_name.
# Note: only `import` itself should be excluded. Module names can be keywords (like in `from .match import ...`,
# where match is a Viper match statement keyword), requiring NotKeyword would cause
# `from .match import` to fail parsing and the dependency scan would miss that module.
if _cur_type(ps) == TokenType.Name and _cur_kw(ps) != Keyword.Import:
module_name = _cur_str(ps)
_advance(ps)
while _match_op(ps, TokOp.Dot):
_advance(ps)
if _cur_type(ps) == TokenType.Name and _cur_kw(ps) == Keyword.NotKeyword:
next_name: str = _cur_str(ps)
mlen: t.CInt = string.strlen(module_name)
nlen: t.CInt = string.strlen(next_name)
new_mod: str = ps.pool.alloc(mlen + 1 + nlen + 1)
if new_mod is not None:
string.strcpy(new_mod, module_name)
new_mod[mlen] = '.'
string.strcpy(new_mod + mlen + 1, next_name)
module_name = new_mod
_advance(ps)
if _cur_kw(ps) != Keyword.Import:
ps.error_count += 1
return None
_advance(ps)
_skip_nl(ps)
# import *
if _match_op(ps, TokOp.Star):
_advance(ps)
star_alias: AST | t.CPtr = Alias(ps.pool, "*", None)
_set_pos(star_alias, lineno, col, 0, 0)
star_names: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
star_names.append(star_alias)
node: AST | t.CPtr = ImportFrom(ps.pool, module_name, star_names, level)
_set_pos(node, lineno, col, 0, 0)
return node
# ( names ) or names
if _match_op(ps, TokOp.LPar):
_advance(ps)
_skip_nl(ps)
names: list[AST | t.CPtr] | t.CPtr = _parse_alias_list(ps)
_skip_nl(ps)
_expect_op(ps, TokOp.RPar)
else:
names = _parse_alias_list(ps)
node: AST | t.CPtr = ImportFrom(ps.pool, module_name, names, level)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_alias_list(ps: Parser | t.CPtr) -> list[AST | t.CPtr] | t.CPtr:
"""Parse alias list."""
names: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
while True:
if _cur_type(ps) != TokenType.Name:
break
name_str: str = _cur_str(ps)
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps)
# dotted name
while _match_op(ps, TokOp.Dot):
_advance(ps)
if _cur_type(ps) == TokenType.Name:
next_n: str = _cur_str(ps)
nmlen: t.CInt = string.strlen(name_str)
nxtlen: t.CInt = string.strlen(next_n)
new_name: str = ps.pool.alloc(nmlen + 1 + nxtlen + 1)
if new_name is not None:
string.strcpy(new_name, name_str)
new_name[nmlen] = '.'
string.strcpy(new_name + nmlen + 1, next_n)
name_str = new_name
_advance(ps)
asname: str = None
if _cur_kw(ps) == Keyword.As:
_advance(ps)
if _cur_type(ps) == TokenType.Name:
asname = _cur_str(ps)
_advance(ps)
a: AST | t.CPtr = Alias(ps.pool, name_str, asname)
_set_pos(a, lineno, col, 0, 0)
names.append(a)
if not _accept_op(ps, TokOp.Comma):
break
_skip_nl(ps)
return names
def _parse_del(ps: Parser | t.CPtr) -> AST | t.CPtr:
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps)
targets: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
while True:
t_: AST | t.CPtr = _parse_atom_expr(ps)
if t_ is None: break
_set_ctx(t_, ASTCtx.Del)
targets.append(t_)
if not _accept_op(ps, TokOp.Comma):
break
node: AST | t.CPtr = Delete(ps.pool, targets)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_assert(ps: Parser | t.CPtr) -> AST | t.CPtr:
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps)
test: AST | t.CPtr = _parse_test(ps)
msg: AST | t.CPtr = None
if _accept_op(ps, TokOp.Comma):
msg = _parse_test(ps)
node: AST | t.CPtr = Assert(ps.pool, test, msg)
_set_pos(node, lineno, col, 0, 0)
return node
# ============================================================
# Compound statements
# ============================================================
def _parse_compound(ps: Parser | t.CPtr) -> AST | t.CPtr:
kw: t.CInt = _cur_kw(ps)
if kw == Keyword.If:
return _parse_if(ps)
if kw == Keyword.While:
return _parse_while(ps)
if kw == Keyword.For:
return _parse_for(ps, 0)
if kw == Keyword.Try:
return _parse_try(ps)
if kw == Keyword.With:
return _parse_with(ps, 0)
if _is_funcdef_kw(ps, kw) != 0:
return _parse_funcdef(ps, 0)
if kw == Keyword.Class:
return _parse_classdef(ps)
if kw == Keyword.Async:
_advance(ps)
if _is_funcdef_kw(ps, _cur_kw(ps)) != 0:
return _parse_funcdef(ps, 1)
if _cur_kw(ps) == Keyword.For:
return _parse_for(ps, 1)
if _cur_kw(ps) == Keyword.With:
return _parse_with(ps, 1)
ps.error_count += 1
return None
if kw == Keyword.Match and _is_match_stmt(ps) != 0:
return _parse_match(ps)
ps.error_count += 1
return None
def _parse_if(ps: Parser | t.CPtr) -> AST | t.CPtr:
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps) # if
test: AST | t.CPtr = _parse_test(ps)
_expect_op(ps, TokOp.Colon)
# First construct with empty orelse, then fill it in later
empty: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
node: If | t.CPtr = If(ps.pool, test, empty)
_parse_suite_into_children(ps, node)
orelse: list[AST | t.CPtr] | t.CPtr = _parse_if_tail(ps)
node.orelse = orelse
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_if_tail(ps: Parser | t.CPtr) -> list[AST | t.CPtr] | t.CPtr:
"""Parse elif/else chain and return orelse list.
elif: orelse = list containing single If node (nested)
else: orelse = statement list
"""
if _cur_kw(ps) == Keyword.Elif:
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps)
elif_test: AST | t.CPtr = _parse_test(ps)
_expect_op(ps, TokOp.Colon)
empty: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
elif_node: If | t.CPtr = If(ps.pool, elif_test, empty)
_parse_suite_into_children(ps, elif_node)
elif_orelse: list[AST | t.CPtr] | t.CPtr = _parse_if_tail(ps)
elif_node.orelse = elif_orelse
_set_pos(elif_node, lineno, col, 0, 0)
orelse: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
orelse.append(elif_node)
return orelse
if _cur_kw(ps) == Keyword.Else:
_advance(ps)
_expect_op(ps, TokOp.Colon)
scratch: Module | t.CPtr = Module(ps.pool)
_parse_suite_into_children(ps, scratch)
return scratch.children
return list[AST | t.CPtr](ps.pool, 8) # 空 list
# ============================================================
# Parsing Compound Statements (Continued)
# ============================================================
def _parse_while(ps: Parser | t.CPtr) -> AST | t.CPtr:
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps) # while
test: AST | t.CPtr = _parse_test(ps)
_expect_op(ps, TokOp.Colon)
empty: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
node: While | t.CPtr = While(ps.pool, test, empty)
_parse_suite_into_children(ps, node)
if _cur_kw(ps) == Keyword.Else:
_advance(ps)
_expect_op(ps, TokOp.Colon)
scratch: Module | t.CPtr = Module(ps.pool)
_parse_suite_into_children(ps, scratch)
node.orelse = scratch.children
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_for(ps: Parser | t.CPtr, is_async: t.CInt) -> AST | t.CPtr:
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps) # for
target: AST | t.CPtr = _parse_target_list(ps)
if _cur_kw(ps) != Keyword.In:
ps.error_count += 1
return None
_advance(ps)
iter_: AST | t.CPtr = _parse_or_test(ps)
_expect_op(ps, TokOp.Colon)
empty: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
node: For | t.CPtr = For(ps.pool, target, iter_, empty, is_async)
_fix_store_ctx(target)
_parse_suite_into_children(ps, node)
if _cur_kw(ps) == Keyword.Else:
_advance(ps)
_expect_op(ps, TokOp.Colon)
scratch: Module | t.CPtr = Module(ps.pool)
_parse_suite_into_children(ps, scratch)
node.orelse = scratch.children
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_try(ps: Parser | t.CPtr) -> AST | t.CPtr:
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps) # try
_expect_op(ps, TokOp.Colon)
empty: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
node: Try | t.CPtr = Try(ps.pool, empty, empty, empty)
_parse_suite_into_children(ps, node)
# handlers (except)
handlers: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
while _cur_kw(ps) == Keyword.Except:
h: AST | t.CPtr = _parse_except_handler(ps)
if h is not None:
handlers.append(h)
node.handlers = handlers
# else
if _cur_kw(ps) == Keyword.Else:
_advance(ps)
_expect_op(ps, TokOp.Colon)
scratch: Module | t.CPtr = Module(ps.pool)
_parse_suite_into_children(ps, scratch)
node.orelse = scratch.children
# finally
if _cur_kw(ps) == Keyword.Finally:
_advance(ps)
_expect_op(ps, TokOp.Colon)
scratch2: Module | t.CPtr = Module(ps.pool)
_parse_suite_into_children(ps, scratch2)
node.finalbody = scratch2.children
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_except_handler(ps: Parser | t.CPtr) -> AST | t.CPtr:
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps) # except
type_: AST | t.CPtr = None
name: str = None
if _cur_op(ps) != TokOp.Colon:
type_ = _parse_test(ps)
if _cur_kw(ps) == Keyword.As:
_advance(ps)
if _cur_type(ps) == TokenType.Name:
name = _cur_str(ps)
_advance(ps)
_expect_op(ps, TokOp.Colon)
node: AST | t.CPtr = ExceptHandler(ps.pool, type_, name)
_parse_suite_into_children(ps, node)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_with(ps: Parser | t.CPtr, is_async: t.CInt) -> AST | t.CPtr:
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps) # with
items: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
# items
while True:
item: AST | t.CPtr = _parse_with_item(ps)
if item is None: break
items.append(item)
if not _accept_op(ps, TokOp.Comma):
break
# Check if we can continue with a newline (with (...): form)
if _cur_type(ps) == TokenType.NewLine or _cur_type(ps) == TokenType.Nl:
break
_expect_op(ps, TokOp.Colon)
node: With | t.CPtr = With(ps.pool, items, is_async)
_parse_suite_into_children(ps, node)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_with_item(ps: Parser | t.CPtr) -> AST | t.CPtr:
"""with_item: test ['as' expr]"""
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
context_expr: AST | t.CPtr = _parse_or_test(ps)
if context_expr is None: return None
optional_vars: AST | t.CPtr = None
if _cur_kw(ps) == Keyword.As:
_advance(ps)
optional_vars = _parse_target(ps)
_fix_store_ctx(optional_vars)
node: AST | t.CPtr = WithItem(ps.pool, context_expr, optional_vars)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_funcdef(ps: Parser | t.CPtr, is_async: t.CInt) -> AST | t.CPtr:
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
is_extern_fd: t.CInt = 0
if _cur_kw(ps) == Keyword.Extern:
# extern 声明：extern [def] A(...) [-> T] [": pass" 可选]
is_extern_fd = 1
_advance(ps) # extern
if _cur_kw(ps) == Keyword.Def:
_advance(ps)
else:
_advance(ps) # def / fn / ...
if _cur_type(ps) != TokenType.Name:
ps.error_count += 1
return None
name: str = _cur_str(ps)
_advance(ps)
# PEP 695 泛型函数类型参数: def add[T](a: T, b: T) — 解析并存入 type_params。
# （与 class 解析器一致；之前缺失此处理导致 _expect_op(Colon) 失败、函数体被
# 误提升到模块级——b = Box(42) 被当作模块级全局注册，函数内赋值误报
# "需用 global 声明"。翻译时 type_params 映射到 int 兜底。）
type_params_fd: list[str] | t.CPtr = None
if _accept_op(ps, TokOp.Lsqb):
type_params_fd = list[str](ps.pool, 8)
while not _match_op(ps, TokOp.Rsqb):
if _cur_type(ps) == TokenType.EndMarker:
break
if _cur_type(ps) == TokenType.Name:
tp_name_fd: str = _cur_str(ps)
_advance(ps)
# 跳过可选 bound/约束: [T: bound] 或 [T: (A, B)]
if _match_op(ps, TokOp.Colon):
_advance(ps)
_parse_test(ps)
type_params_fd.append(tp_name_fd)
if not _accept_op(ps, TokOp.Comma):
break
_expect_op(ps, TokOp.Rsqb)
args: AST | t.CPtr = None
if _accept_op(ps, TokOp.LPar):
args = _parse_varargslist(ps, 0)
_expect_op(ps, TokOp.RPar)
returns: AST | t.CPtr = None
if _accept_op(ps, TokOp.RArrow):
returns = _parse_test(ps)
if is_extern_fd:
# extern 声明：返回注解改写为 <原类型> | t.State，复用既有
# State/外部 C 函数识别路径（vpi 注解含 "State" → 定义端裸名 + 不生成函数体）
state_attr_fd: AST | t.CPtr = Attribute(ps.pool,
Name(ps.pool, "t", ASTCtx.Load),
"State", ASTCtx.Load)
if returns != None:
returns = BinOp(ps.pool, returns, OpKind.BitOr, state_attr_fd)
else:
returns = state_attr_fd
dec_list: list[AST | t.CPtr] | t.CPtr = ps.pending_decorators
ps.pending_decorators = None
node: FunctionDef | t.CPtr = FunctionDef(ps.pool, name, args, dec_list, returns, is_async, type_params_fd)
if is_extern_fd:
node.flags = node.flags | FLAG_IS_EXTERN
if is_extern_fd == 0 or _match_op(ps, TokOp.Colon):
_expect_op(ps, TokOp.Colon)
_parse_suite_into_children(ps, node)
# extern 无 ': pass' 形式（extern A(...) -> int）：body 为空，仅声明
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_classdef(ps: Parser | t.CPtr) -> AST | t.CPtr:
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps) # class
if _cur_type(ps) != TokenType.Name:
ps.error_count += 1
return None
name: str = _cur_str(ps)
_advance(ps)
# Parse type parameters [T] or [T, U] (PEP 695 generic class syntax)
type_params: list[str] | t.CPtr = None
if _accept_op(ps, TokOp.Lsqb):
type_params = list[str](ps.pool, 8)
while not _match_op(ps, TokOp.Rsqb):
if _cur_type(ps) == TokenType.EndMarker: break
if _cur_type(ps) == TokenType.Name:
tp_name: str = _cur_str(ps)
_advance(ps)
# Skip optional bound/constraint: [T: bound] or [T: (A, B)]
if _match_op(ps, TokOp.Colon):
_advance(ps)
_parse_test(ps)
type_params.append(tp_name)
if not _accept_op(ps, TokOp.Comma):
break
_expect_op(ps, TokOp.Rsqb)
bases: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
keywords: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
# bases / keywords
if _accept_op(ps, TokOp.LPar):
while not _match_op(ps, TokOp.RPar):
# 基类括号内跨行（class X(A,\n B):）：逗号后换行在此跳过，
# 否则 cur=NL 非 Name/Comma → break → _expect_op(RPar) 报
# "意外的 token"（Generator.vp L34 多继承跨行根因）
_skip_nl(ps)
if _cur_type(ps) == TokenType.EndMarker: break
# base 或 keyword=value
if _cur_type(ps) == TokenType.Name:
# Parse as expression, if followed by '=', then as keyword=value
e: AST | t.CPtr = _parse_or_test(ps)
if _match_op(ps, TokOp.Equal):
_advance(ps)
v: AST | t.CPtr = _parse_test(ps)
kw_node: AST | t.CPtr = KwNode(ps.pool, e, v)
keywords.append(kw_node)
else:
bases.append(e)
if not _accept_op(ps, TokOp.Comma):
break
_skip_nl(ps) # 末基类后换行收尾（class X(A,\n B):）
_expect_op(ps, TokOp.RPar)
_expect_op(ps, TokOp.Colon)
dec_list: list[AST | t.CPtr] | t.CPtr = ps.pending_decorators
ps.pending_decorators = None
node: ClassDef | t.CPtr = ClassDef(ps.pool, name, bases, keywords, dec_list, type_params)
_parse_suite_into_children(ps, node)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_match(ps: Parser | t.CPtr) -> AST | t.CPtr:
"""match statement: match subject ':' NewLine Indent case+ Dedent"""
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps) # match
subject: AST | t.CPtr = _parse_test(ps)
_expect_op(ps, TokOp.Colon)
cases: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
node: AST | t.CPtr = Match(ps.pool, subject, cases)
# Parsing body: Same NewLine/Indent/Dedent pattern as _parse_suite_into_children,
# but parses case clauses into a cases list instead of normal statements into children
_skip_nl(ps)
if _cur_type(ps) == TokenType.NewLine:
_advance(ps)
_skip_nl(ps)
if _cur_type(ps) == TokenType.Indent:
_advance(ps)
while _cur_type(ps) != TokenType.Dedent and _cur_type(ps) != TokenType.EndMarker:
if _cur_kw(ps) == Keyword.Case:
case_node: AST | t.CPtr = _parse_match_case(ps)
if case_node is not None:
cases.append(case_node)
else:
_advance(ps) # Skip non-case token to avoid infinite loop
_expect_type(ps, TokenType.Dedent)
_set_pos(node, lineno, col, 0, 0)
return node
# ============================================================
# Match pattern parsing
# ============================================================
def _make_name_ast(pool: memhub.MemManager | t.CPtr, node: Name | t.CPtr) -> AST | t.CPtr:
# Analysis: "Convert Name* to AST* (Solving the TransPyC Type Inference Problem)"
return node
def _parse_pattern(ps: Parser | t.CPtr) -> AST | t.CPtr:
"""pattern: or_pattern ['as' NAME]
or_pattern: closed_pattern ('|' closed_pattern)*"""
pat: AST | t.CPtr = _parse_closed_pattern(ps)
if pat is None: return None
# OR pattern
if _match_op(ps, TokOp.VBar):
patterns: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
patterns.append(pat)
while _accept_op(ps, TokOp.VBar):
p: AST | t.CPtr = _parse_closed_pattern(ps)
if p is not None:
patterns.append(p)
or_node: AST | t.CPtr = MatchOr(ps.pool, patterns)
_set_pos(or_node, pat.lineno, pat.col_offset, 0, 0)
pat = or_node
# AS pattern
if _cur_kw(ps) == Keyword.As:
_advance(ps)
if _cur_type(ps) != TokenType.Name:
ps.error_count += 1
return pat
name_str: str = _cur_str(ps)
_advance(ps)
as_node: AST | t.CPtr = MatchAs(ps.pool, pat, name_str)
_set_pos(as_node, pat.lineno, pat.col_offset, 0, 0)
return as_node
return pat
def _parse_closed_pattern(ps: Parser | t.CPtr) -> AST | t.CPtr:
"""closed_pattern: literal | capture | wildcard | value | group | sequence | mapping | class"""
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
ttype: t.CInt = _cur_type(ps)
val: AST | t.CPtr
node: AST | t.CPtr
tok: Token | t.CPtr
sval: str
# Number literal pattern
if ttype == TokenType.Number:
tok = ps.cur
_advance(ps)
if tok.is_float:
val = _new_const_float(ps.pool, tok.float_val, lineno, col)
else:
val = _new_const_int(ps.pool, tok.int_val, lineno, col)
node = MatchValue(ps.pool, val)
_set_pos(node, lineno, col, 0, 0)
return node
# String literal pattern
if ttype == TokenType.String:
sval = _cur_str(ps)
is_dquote_m: t.CInt = ps.cur.kw_subtype & 16
_advance(ps)
val = _new_const_str(ps.pool, sval, lineno, col, is_dquote_m)
node = MatchValue(ps.pool, val)
_set_pos(node, lineno, col, 0, 0)
return node
# Negative number literal pattern: -1, -1.5
if _match_op(ps, TokOp.Minus):
_advance(ps)
if _cur_type(ps) == TokenType.Number:
tok = ps.cur
_advance(ps)
if tok.is_float:
val = _new_const_float(ps.pool, -tok.float_val, lineno, col)
else:
val = _new_const_int(ps.pool, -tok.int_val, lineno, col)
node = MatchValue(ps.pool, val)
_set_pos(node, lineno, col, 0, 0)
return node
ps.error_count += 1
return None
# Name: None/True/False/_ /capture/value/class
if ttype == TokenType.Name:
kw: t.CInt = _cur_kw(ps)
# None/True/False → MatchSingleton
if kw == Keyword.None_ or kw == Keyword.True_ or kw == Keyword.False_:
_advance(ps)
if kw == Keyword.None_:
val = _new_const_none(ps.pool, lineno, col)
elif kw == Keyword.True_:
val = _new_const_bool(ps.pool, 1, lineno, col)
else:
val = _new_const_bool(ps.pool, 0, lineno, col)
node = MatchSingleton(ps.pool, val)
_set_pos(node, lineno, col, 0, 0)
return node
name_str: str = _cur_str(ps)
# Wildcard pattern: _
if string.strcmp(name_str, "_") == 0:
_advance(ps)
node = MatchAs(ps.pool, None, None)
_set_pos(node, lineno, col, 0, 0)
return node
_advance(ps)
# Check dotted name（value pattern 或 class pattern）
if _match_op(ps, TokOp.Dot):
name_node: Name | t.CPtr = Name(ps.pool, name_str, ASTCtx.Load)
_set_pos(name_node, lineno, col, 0, 0)
base: AST | t.CPtr = _make_name_ast(ps.pool, name_node)
while _match_op(ps, TokOp.Dot):
_advance(ps)
if _cur_type(ps) != TokenType.Name:
ps.error_count += 1
break
attr_name: str = _cur_str(ps)
attr_lineno: t.CInt = _cur_lineno(ps)
attr_col: t.CInt = _cur_col(ps)
_advance(ps)
base = Attribute(ps.pool, base, attr_name, ASTCtx.Load)
_set_pos(base, attr_lineno, attr_col, 0, 0)
# class pattern: dotted.name(...)
if _match_op(ps, TokOp.LPar):
return _parse_class_pattern(ps, base, lineno, col)
# value pattern: dotted.name
node = MatchValue(ps.pool, base)
_set_pos(node, lineno, col, 0, 0)
return node
# class pattern: Name(...)
if _match_op(ps, TokOp.LPar):
cls_node: Name | t.CPtr = Name(ps.pool, name_str, ASTCtx.Load)
_set_pos(cls_node, lineno, col, 0, 0)
cls: AST | t.CPtr = _make_name_ast(ps.pool, cls_node)
return _parse_class_pattern(ps, cls, lineno, col)
# capture pattern: NAME
node = MatchAs(ps.pool, None, name_str)
_set_pos(node, lineno, col, 0, 0)
return node
# group pattern / sequence pattern (tuple): (...)
if _match_op(ps, TokOp.LPar):
_advance(ps)
inner: AST | t.CPtr = _parse_pattern(ps)
if _accept_op(ps, TokOp.Comma):
# sequence pattern (tuple)
patterns: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
if inner is not None:
patterns.append(inner)
while not _match_op(ps, TokOp.RPar) and _cur_type(ps) != TokenType.EndMarker:
p: AST | t.CPtr = _parse_pattern(ps)
if p is not None:
patterns.append(p)
if not _accept_op(ps, TokOp.Comma):
break
_expect_op(ps, TokOp.RPar)
node = MatchSequence(ps.pool, patterns)
_set_pos(node, lineno, col, 0, 0)
return node
_expect_op(ps, TokOp.RPar)
return inner # group pattern
# sequence pattern (list): [...]
if _match_op(ps, TokOp.Lsqb):
return _parse_sequence_pattern(ps, TokOp.Rsqb, lineno, col)
# mapping pattern: {...}
if _match_op(ps, TokOp.LBrace):
return _parse_mapping_pattern(ps, lineno, col)
ps.error_count += 1
return None
def _parse_sequence_pattern(ps: Parser | t.CPtr, close_op: t.CInt,
lineno: t.CInt, col: t.CInt) -> AST | t.CPtr:
"""sequence_pattern: '[' [pattern_list] ']'"""
patterns: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
_advance(ps) # [
while not _match_op(ps, close_op) and _cur_type(ps) != TokenType.EndMarker:
# *name (star pattern)
if _match_op(ps, TokOp.Star):
_advance(ps)
if _cur_type(ps) == TokenType.Name:
star_name: str = _cur_str(ps)
_advance(ps)
star_node: AST | t.CPtr
if string.strcmp(star_name, "_") == 0:
star_node = MatchStar(ps.pool, None)
else:
star_node = MatchStar(ps.pool, star_name)
_set_pos(star_node, lineno, col, 0, 0)
patterns.append(star_node)
else:
p: AST | t.CPtr = _parse_pattern(ps)
if p is not None:
patterns.append(p)
if not _accept_op(ps, TokOp.Comma):
break
_expect_op(ps, close_op)
node: AST | t.CPtr = MatchSequence(ps.pool, patterns)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_mapping_pattern(ps: Parser | t.CPtr,
lineno: t.CInt, col: t.CInt) -> AST | t.CPtr:
"""mapping_pattern: '{' [items] '}'
items: key ':' pattern (',' key ':' pattern)* [',' '**' NAME] | '**' NAME"""
keys: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
patterns: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
rest: str = None
node: AST | t.CPtr
key: AST | t.CPtr
val_pat: AST | t.CPtr
_advance(ps) # {
if _match_op(ps, TokOp.RBrace):
_advance(ps)
node = MatchMapping(ps.pool, keys, patterns, rest)
_set_pos(node, lineno, col, 0, 0)
return node
while _cur_type(ps) != TokenType.EndMarker:
if _match_op(ps, TokOp.StarStar):
_advance(ps)
if _cur_type(ps) == TokenType.Name:
rest = _cur_str(ps)
_advance(ps)
break
key = _parse_expr(ps)
_expect_op(ps, TokOp.Colon)
val_pat = _parse_pattern(ps)
if key is not None:
keys.append(key)
if val_pat is not None:
patterns.append(val_pat)
if not _accept_op(ps, TokOp.Comma):
break
_expect_op(ps, TokOp.RBrace)
node = MatchMapping(ps.pool, keys, patterns, rest)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_class_pattern(ps: Parser | t.CPtr, cls: AST | t.CPtr,
lineno: t.CInt, col: t.CInt) -> AST | t.CPtr:
"""class_pattern: cls '(' [pattern_list] ')'
pattern_list: positional_pattern (',' positional_pattern)*
[',' keyword_pattern]* | keyword_pattern (',' keyword_pattern)*
keyword_pattern: NAME '=' pattern"""
patterns: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
kwd_attrs: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
kwd_patterns: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
p: AST | t.CPtr
_advance(ps) # (
if not _match_op(ps, TokOp.RPar):
while _cur_type(ps) != TokenType.EndMarker:
# keyword pattern: NAME '=' pattern
if _cur_type(ps) == TokenType.Name and ps.cur is not None:
nxt: Token | t.CPtr = ps.cur.next
if nxt is not None and nxt.type == TokenType.Op and nxt.op_subtype == TokOp.Equal:
attr_name: str = _cur_str(ps)
_advance(ps) # NAME
_advance(ps) # =
kwd_pat: AST | t.CPtr = _parse_pattern(ps)
attr_n: Name | t.CPtr = Name(ps.pool, attr_name, ASTCtx.Load)
_set_pos(attr_n, lineno, col, 0, 0)
attr_node: AST | t.CPtr = _make_name_ast(ps.pool, attr_n)
kwd_attrs.append(attr_node)
if kwd_pat is not None:
kwd_patterns.append(kwd_pat)
else:
p = _parse_pattern(ps)
if p is not None:
patterns.append(p)
else:
p = _parse_pattern(ps)
if p is not None:
patterns.append(p)
if not _accept_op(ps, TokOp.Comma):
break
_expect_op(ps, TokOp.RPar)
node: AST | t.CPtr = MatchClass(ps.pool, cls, patterns, kwd_attrs, kwd_patterns)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_match_case(ps: Parser | t.CPtr) -> AST | t.CPtr:
"""case pattern [if guard] ':' suite"""
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps) # case
guard: AST | t.CPtr = None
pattern: AST | t.CPtr = _parse_pattern(ps)
if _cur_kw(ps) == Keyword.If:
_advance(ps)
guard = _parse_or_test(ps)
_expect_op(ps, TokOp.Colon)
node: AST | t.CPtr = MatchCase(ps.pool, pattern, guard)
_parse_suite_into_children(ps, node)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_decorators(ps: Parser | t.CPtr) -> list[AST | t.CPtr] | t.CPtr:
"""解析装饰器链，返回 list。"""
decs: list[AST | t.CPtr] | t.CPtr = list[AST | t.CPtr](ps.pool, 8)
while _match_op(ps, TokOp.At):
lineno: t.CInt = _cur_lineno(ps)
col: t.CInt = _cur_col(ps)
_advance(ps) # @
expr: AST | t.CPtr = _parse_namedexpr(ps)
_skip_nl(ps)
if _cur_type(ps) == TokenType.NewLine:
_advance(ps)
decs.append(expr)
return decs
# ============================================================
# Module entry
# ============================================================
def _parse_module_body(ps: Parser | t.CPtr, module_node: AST | t.CPtr):
"""解析模块体，append 到 module_node。"""
_skip_nl(ps)
stmt_idx: t.CInt = 0
while _cur_type(ps) != TokenType.EndMarker:
stmt_idx += 1
# Decorator
if _match_op(ps, TokOp.At):
ps.pending_decorators = _parse_decorators(ps)
# Record current token pointer to check for infinite loop
before_cur: Token | t.CPtr = ps.cur
_parse_statement_into(ps, module_node)
# Defense: If parsing doesn't advance the token (like when _parse_atom encounters an unrecognized token and returns None),
# force it to move forward to avoid an infinite loop in the main loop; accumulate error_count for the caller to diagnose
if ps.cur is before_cur:
ps.error_count += 1
_advance(ps)
_skip_nl(ps)
def parse_tokens(pool: memhub.MemManager | t.CPtr,
tokens: Token | t.CPtr) -> AST | t.CPtr:
"""Parse an AST Module from a token list."""
if pool is None or tokens is None:
return None
global _g_parse_err_line, _g_parse_err_col, _g_parse_err_msg
_g_parse_err_line = 0
_g_parse_err_col = 0
_g_parse_err_msg = None
ps: Parser | t.CPtr = new_parser(pool)
if ps is None: return None
_parser_init(ps, tokens, pool)
module_node: Module | t.CPtr = Module(pool)
_parse_module_body(ps, module_node)
# fail-fast: 解析出错（error_count > 0）时返回 None，由调用方（TPV 等）报错终止，
# 避免错误代码静默编译（如 `func12344444 avg3(...)` 缺 def 关键字的函数定义残留）。
if ps.error_count > 0:
return None
return module_node
Reference in new issue View Git Blame Copy Permalink