2903 lines
111 KiB
Plaintext
2903 lines
111 KiB
Plaintext
import t, c
|
||
from stdint import *
|
||
import memhub
|
||
import string
|
||
import stdio
|
||
import stdlib
|
||
from .tokens import (Token, TokenType, Keyword, TokOp)
|
||
from .base import (
|
||
AST, ASTCtx, ASTKind, OpKind, ASTFlag,
|
||
_set_pos, _binop_from_op, _augop_from_op, _cmpop_from_op, _copy_str, _inherit_pos,
|
||
_init_ast,
|
||
CONST_INT, CONST_FLOAT, CONST_STR, CONST_BOOL, CONST_NONE, CONST_CHAR,
|
||
FLAG_IS_ASYNC, FLAG_IS_EXTERN, FLAG_IS_EXPORT, FLAG_IS_DEFINE,
|
||
)
|
||
from .exprs import (
|
||
BoolOp, BinOp, UnaryOp, Lambda, IfExp, Dict, Set,
|
||
ListComp, SetComp, DictComp, GeneratorExp,
|
||
Await, Yield, YieldFrom, FormattedValue, JoinedStr,
|
||
Constant, NamedExpr, Attribute, Subscript, Starred, Name,
|
||
List, Tuple, Slice, Call, Compare, OpNode,
|
||
)
|
||
from .stmts import (
|
||
Module, Expression, Interactive, FunctionType,
|
||
FunctionDef, ClassDef, Return, Delete, Assign, AugAssign, AnnAssign,
|
||
For, While, If, With, Raise, Try, Assert, Global, Nonlocal,
|
||
Pass, Break, Continue, Expr, Import, ImportFrom, Match,
|
||
)
|
||
from .astaux import (
|
||
ExceptHandler, Arguments, Arg, Keyword as KwNode, Alias, WithItem, Comprehension,
|
||
)
|
||
from .match import (
|
||
MatchCase, MatchValue, MatchSingleton, MatchSequence,
|
||
MatchMapping, MatchClass, MatchStar, MatchAs, MatchOr,
|
||
)
|
||
from .lexer import Lexer, _lexer_init, tokenize, new_lexer
|
||
|
||
|
||
# ============================================================
|
||
# Parser State
|
||
# ============================================================
|
||
class Parser:
|
||
tokens: rawptr[Token] # token linked list head
|
||
cur: rawptr[Token] # current token
|
||
pool: rawptr[memhub.MemManager] # memory pool
|
||
error_count: int # error count
|
||
pending_decorators: rawptr[list[rawptr[AST]]] # pending decorators list
|
||
|
||
def __new__(self, pool: rawptr[memhub.MemManager]):
|
||
ptr: rawptr[Parser] = pool.alloc(Parser.__sizeof__())
|
||
if ptr:
|
||
string.memset(ptr, 0, Parser.__sizeof__())
|
||
return ptr
|
||
|
||
|
||
def new_parser(pool: rawptr[memhub.MemManager]) -> rawptr[Parser]:
|
||
"""Allocate from the pool and return the Parser (cross-module safe wrapper)"""
|
||
return Parser(pool)
|
||
|
||
|
||
def _parser_init(ps: rawptr[Parser], tokens: rawptr[Token],
|
||
pool: rawptr[memhub.MemManager]):
|
||
if ps is None: return
|
||
ps.tokens = tokens
|
||
ps.cur = tokens
|
||
ps.pool = pool
|
||
ps.error_count = 0
|
||
ps.pending_decorators = None
|
||
|
||
|
||
# ============================================================
|
||
# Token Navigation Helpers
|
||
# ============================================================
|
||
|
||
def _advance(ps: rawptr[Parser]):
|
||
"""Advance to the next token"""
|
||
if ps.cur is None: return
|
||
if ps.cur.next is not None:
|
||
ps.cur = ps.cur.next
|
||
|
||
|
||
def _cur_type(ps: rawptr[Parser]) -> int:
|
||
if ps.cur is None: return TokenType.EndMarker
|
||
return ps.cur.type
|
||
|
||
|
||
def _cur_op(ps: rawptr[Parser]) -> int:
|
||
if ps.cur is None: return 0
|
||
return ps.cur.op_subtype
|
||
|
||
|
||
def _cur_kw(ps: rawptr[Parser]) -> int:
|
||
if ps.cur is None: return 0
|
||
return ps.cur.kw_subtype
|
||
|
||
|
||
def _is_funcdef_kw(ps: rawptr[Parser], kw: int) -> int:
|
||
"""判断当前 token 是否为函数定义关键字（def 及其别名 fn/fun/func/function，
|
||
以及 extern/export 声明关键字）
|
||
|
||
def 是硬关键字直接判定；fn/fun/func/function/extern/export 是软关键字，
|
||
仅当下一个 token 是 Name 且再下一个是 '('（或泛型 '[T]' 前的 '['）时
|
||
判定为函数定义。extern 还允许 'extern def A(...)' 形式（下一个是 def），
|
||
export 同样允许 'export def A(...)' 形式。
|
||
否则按普通标识符处理，兼容既有代码把 func 用作
|
||
字段/参数名（如 AST 节点 func 字段、Call.func 注解 `func: rawptr[AST]`）。
|
||
"""
|
||
if kw == Keyword.Def:
|
||
return 1
|
||
if kw == Keyword.Extern or kw == Keyword.Export:
|
||
# extern/export def A(...) 或 extern/export A(...)
|
||
# （extern/export 后是 def 或 Name+'(/'['）
|
||
nxt: rawptr[Token] = ps.cur.next
|
||
if nxt is None:
|
||
return 0
|
||
if nxt.kw_subtype == Keyword.Def:
|
||
return 1
|
||
if nxt.type != TokenType.Name:
|
||
return 0
|
||
nxt2: rawptr[Token] = nxt.next
|
||
if nxt2 is None:
|
||
return 0
|
||
if nxt2.type == TokenType.Op \
|
||
and (nxt2.op_subtype == TokOp.LPar or nxt2.op_subtype == TokOp.Lsqb):
|
||
return 1
|
||
return 0
|
||
if kw == Keyword.Fn or kw == Keyword.Fun \
|
||
or kw == Keyword.Func or kw == Keyword.Function \
|
||
or kw == Keyword.Define:
|
||
# 软关键字 lookahead: 下个是 Name，再下个是 '(' 或 '[T...]' 泛型参数（PEP 695）
|
||
# （define 宏函数与 def 语义一致，返回注解自动追加 t.CDefine）
|
||
nxt: rawptr[Token] = ps.cur.next
|
||
if nxt is None:
|
||
return 0
|
||
if nxt.type != TokenType.Name:
|
||
return 0
|
||
nxt2: rawptr[Token] = nxt.next
|
||
if nxt2 is None:
|
||
return 0
|
||
if nxt2.type == TokenType.Op \
|
||
and (nxt2.op_subtype == TokOp.LPar or nxt2.op_subtype == TokOp.Lsqb):
|
||
return 1
|
||
return 0
|
||
return 0
|
||
|
||
|
||
def _is_match_stmt(ps: rawptr[Parser]) -> int:
|
||
"""match 是软关键字（CPython 3.10+）：lookahead 判断是否为 match 语句
|
||
|
||
match 后直接跟 ':'（变量注解，如 lexer.vp 的 `match: int = 1`）、
|
||
'='（赋值，如 `match = 0`）、换行/文件尾 → 按普通标识符；
|
||
否则扫描至行尾，在括号深度 0 处出现 ':' → match 语句（subject 后必须冒号）。
|
||
case/type 同理为软关键字（case 仅在 match 套件内消费，type 无语句分发）。
|
||
"""
|
||
nxt: rawptr[Token] = ps.cur.next
|
||
if nxt is None:
|
||
return 0
|
||
if nxt.type == TokenType.NewLine or nxt.type == TokenType.EndMarker:
|
||
return 0
|
||
if nxt.type == TokenType.Op:
|
||
if nxt.op_subtype == TokOp.Colon:
|
||
return 0
|
||
if nxt.op_subtype == TokOp.Equal:
|
||
return 0
|
||
walk: rawptr[Token] = nxt
|
||
depth: int = 0
|
||
while walk is not None:
|
||
if walk.type == TokenType.NewLine or walk.type == TokenType.EndMarker:
|
||
return 0
|
||
if walk.type == TokenType.Op:
|
||
wop: int = walk.op_subtype
|
||
if wop == TokOp.LPar or wop == TokOp.Lsqb or wop == TokOp.LBrace:
|
||
depth += 1
|
||
elif wop == TokOp.RPar or wop == TokOp.Rsqb or wop == TokOp.RBrace:
|
||
if depth > 0:
|
||
depth -= 1
|
||
elif wop == TokOp.Colon and depth == 0:
|
||
return 1
|
||
walk = walk.next
|
||
return 0
|
||
|
||
|
||
# 解析错误位置与消息（fail-fast 仿 Python SyntaxError 报告，首次错误记录）
|
||
_g_parse_err_line: int = 0
|
||
_g_parse_err_col: int = 0
|
||
_g_parse_err_msg: str = None
|
||
|
||
|
||
def _record_parse_err(ps: rawptr[Parser]):
|
||
"""记录首个解析错误的位置与消息（只记录第一次，后续错误不覆盖）"""
|
||
global _g_parse_err_line, _g_parse_err_col, _g_parse_err_msg
|
||
if _g_parse_err_line != 0:
|
||
return
|
||
_g_parse_err_line = _cur_lineno(ps)
|
||
_g_parse_err_col = _cur_col(ps)
|
||
err_s: str = _cur_str(ps)
|
||
if err_s is None:
|
||
err_s = "(none)"
|
||
# 消息文本用 stdlib 拷贝（token 的 str_val 在 pool 中，生命周期覆盖本次编译但保险起见独立存储）
|
||
buf_m: str = stdlib.malloc(128)
|
||
if buf_m is not None:
|
||
stdio.snprintf(buf_m, 128, "意外的 token '%s'", err_s)
|
||
_g_parse_err_msg = buf_m
|
||
else:
|
||
_g_parse_err_msg = err_s
|
||
|
||
|
||
def get_parse_err_line() -> int:
|
||
"""获取最近一次解析错误的行号（0=无错误）"""
|
||
return _g_parse_err_line
|
||
|
||
|
||
def get_parse_err_col() -> int:
|
||
"""获取最近一次解析错误的列号（0=无错误）"""
|
||
return _g_parse_err_col
|
||
|
||
|
||
def get_parse_err_msg() -> str:
|
||
"""获取最近一次解析错误的错误消息（仿 Python SyntaxError 消息文本）"""
|
||
return _g_parse_err_msg
|
||
|
||
|
||
def _cur_str(ps: rawptr[Parser]) -> str:
|
||
if ps.cur is None: return None
|
||
return ps.cur.str_val
|
||
|
||
|
||
def _cur_lineno(ps: rawptr[Parser]) -> int:
|
||
if ps.cur is None: return 0
|
||
return ps.cur.lineno
|
||
|
||
|
||
def _cur_col(ps: rawptr[Parser]) -> int:
|
||
if ps.cur is None: return 0
|
||
return ps.cur.col_offset
|
||
|
||
|
||
def _skip_nl(ps: rawptr[Parser]):
|
||
"""Skip TokenType.Nl token"""
|
||
while ps.cur is not None and ps.cur.type == TokenType.Nl:
|
||
ps.cur = ps.cur.next
|
||
|
||
|
||
def _skip_nls_and_newlines(ps: rawptr[Parser]):
|
||
"""Skip TokenType.Nl and TokenType.NewLine (used for line continuation)"""
|
||
while ps.cur is not None:
|
||
if ps.cur.type == TokenType.Nl or ps.cur.type == TokenType.NewLine:
|
||
ps.cur = ps.cur.next
|
||
else:
|
||
break
|
||
|
||
|
||
def _expect_type(ps: rawptr[Parser], ttype: int) -> int:
|
||
"""Expect the current token to be of the specified type, advance and return 1, otherwise return 0"""
|
||
if _cur_type(ps) == ttype:
|
||
_advance(ps)
|
||
return 1
|
||
_record_parse_err(ps)
|
||
ps.error_count += 1
|
||
return 0
|
||
|
||
|
||
def _expect_op(ps: rawptr[Parser], op_id: int) -> int:
|
||
"""Expecting the current one to be some operator"""
|
||
if _cur_type(ps) == TokenType.Op and _cur_op(ps) == op_id:
|
||
_advance(ps)
|
||
return 1
|
||
_record_parse_err(ps)
|
||
ps.error_count += 1
|
||
return 0
|
||
|
||
|
||
def _match_op(ps: rawptr[Parser], op_id: int) -> int:
|
||
"""Check but not consume the current token"""
|
||
if _cur_type(ps) == TokenType.Op and _cur_op(ps) == op_id:
|
||
return 1
|
||
return 0
|
||
|
||
|
||
def _accept_op(ps: rawptr[Parser], op_id: int) -> int:
|
||
"""If match, consume and return 1"""
|
||
if _match_op(ps, op_id):
|
||
_advance(ps)
|
||
return 1
|
||
return 0
|
||
|
||
|
||
# ============================================================
|
||
# Constant Construction Helper (New API: Constant directly accepts lineno/col)
|
||
# ============================================================
|
||
|
||
def _new_const_int(pool: rawptr[memhub.MemManager], val: int64,
|
||
lineno: int, col: int) -> rawptr[AST]:
|
||
return Constant(pool, CONST_INT, val, 0.0, None, lineno, col)
|
||
|
||
|
||
def _new_const_float(pool: rawptr[memhub.MemManager], val: double,
|
||
lineno: int, col: int) -> rawptr[AST]:
|
||
global _g_dbg_pts, _g_dbg_vals, _g_dbg_cnt
|
||
new_c: rawptr[Constant] = Constant(pool, CONST_FLOAT, 0, val, None, lineno, col)
|
||
# 把数字文本存到 str_val（stdlib 分配，绕开 pool 对 float_val 字段的覆写）
|
||
# REG-SRC 用 string.atof(str_val) 恢复原始值
|
||
lit_txt: str = stdlib.malloc(64)
|
||
if lit_txt is not None:
|
||
stdio.snprintf(lit_txt, 64, "%.17g", val)
|
||
new_c.str_val = lit_txt
|
||
# 记录浮点节点 ptr→val 到 stdlib 表（构造时值正确，供 REG-SRC 查表绕开 pool 覆写）
|
||
# 表满时重置：旧记录对应已翻译完的文件，从当前节点重新记录
|
||
if _g_dbg_pts is None:
|
||
_g_dbg_pts = stdlib.malloc(4096 * 8)
|
||
_g_dbg_vals = stdlib.malloc(4096 * 8)
|
||
_g_dbg_cnt = 0
|
||
elif _g_dbg_cnt >= 4096:
|
||
_g_dbg_cnt = 0
|
||
if _g_dbg_pts is not None and _g_dbg_vals is not None:
|
||
_g_dbg_pts[_g_dbg_cnt] = uint64(new_c)
|
||
_g_dbg_vals[_g_dbg_cnt] = val
|
||
_g_dbg_cnt += 1
|
||
return new_c
|
||
|
||
|
||
# 浮点节点构造值表（stdlib 分配，避免污染 pool）
|
||
_g_dbg_pts: rawptr[uint64] = None
|
||
_g_dbg_vals: rawptr[double] = None
|
||
_g_dbg_cnt: int = 0
|
||
# 最近一次 _dbg_lookup_lit_val 是否命中
|
||
_g_dbg_lit_found: int = 0
|
||
|
||
|
||
def _dbg_lookup_lit_val(ptr: uint64) -> double:
|
||
"""查浮点节点构造值表：返回 ptr 节点的原始构造值（未命中返回 0 并置 found=0）
|
||
|
||
用于绕开 pool 分配对 AST float_val 字段的覆写（构造时值正确，读取时可能被覆写）。
|
||
"""
|
||
global _g_dbg_lit_found
|
||
_g_dbg_lit_found = 0
|
||
if _g_dbg_pts is None or _g_dbg_vals is None:
|
||
return 0.0
|
||
i: t.CSizeT = 0
|
||
while i < _g_dbg_cnt:
|
||
if _g_dbg_pts[i] == ptr:
|
||
_g_dbg_lit_found = 1
|
||
return _g_dbg_vals[i]
|
||
i += 1
|
||
return 0.0
|
||
|
||
|
||
def _dbg_lit_found() -> int:
|
||
"""检查最近一次 _dbg_lookup_lit_val 是否命中"""
|
||
return _g_dbg_lit_found
|
||
|
||
|
||
def _new_const_str(pool: rawptr[memhub.MemManager], val: str,
|
||
lineno: int, col: int,
|
||
is_dquote: int = 1,
|
||
is_triple: int = 0,
|
||
is_char: int = 0) -> rawptr[AST]:
|
||
"""Create a string/character constant node
|
||
|
||
is_dquote: 0 = single quote, non-0 = double quote (default double quote)
|
||
is_triple: non-0 = triple-quoted string
|
||
is_char: non-0 = single-character candidate (lexer has confirmed that after escaping it's exactly 1 byte, including '0')
|
||
Semantic distinctions:
|
||
- 'x' (single quote, single character, not triple-quoted) → CONST_CHAR (i8 value)
|
||
- "x" (double quote, includes 0 terminator) → CONST_STR (i8* pointer, i8 x 2)
|
||
- 'ab' (single quotes, multiple characters) → CONST_STR (i8* pointer)
|
||
- '''x''' / \"\"\"x\"\"\" (triple quotes) → CONST_STR (i8* pointer)
|
||
- '' / "" (empty string) → CONST_STR (i8* pointer)
|
||
"""
|
||
# Single quotes, non-triple quotes, single-character candidates marked by lexer → CONST_CHAR
|
||
# Mark with is_char instead of strlen(val), because '0' becomes a NUL byte after escaping, strlen would return 0
|
||
if is_dquote == 0 and is_triple == 0 and is_char != 0:
|
||
# Single quote # Single character → CONST_CHAR, int_val stores the character's ASCII value
|
||
# val[0] is a legal value of 0 even if it's NUL
|
||
ch_val: int64 = val[0]
|
||
return Constant(pool, CONST_CHAR, ch_val, 0.0, val, lineno, col)
|
||
# Other situations → CONST_STR
|
||
return Constant(pool, CONST_STR, 0, 0.0, val, lineno, col)
|
||
|
||
|
||
def _new_const_bool(pool: rawptr[memhub.MemManager], val: int,
|
||
lineno: int, col: int) -> rawptr[AST]:
|
||
return Constant(pool, CONST_BOOL, val, 0.0, None, lineno, col)
|
||
|
||
|
||
def _new_const_none(pool: rawptr[memhub.MemManager],
|
||
lineno: int, col: int) -> rawptr[AST]:
|
||
return Constant(pool, CONST_NONE, 0, 0.0, None, lineno, col)
|
||
|
||
|
||
# ============================================================
|
||
# ctx Helper (Recursive Fixing Store Context)
|
||
# ============================================================
|
||
|
||
def _set_ctx(node: rawptr[AST], ctx: int):
|
||
"""Set ctx field of node (based on kind conversion)"""
|
||
if node is None: return
|
||
k: int = node.kind()
|
||
if k == ASTKind.Name:
|
||
nm: rawptr[Name] = (rawptr[Name])(node)
|
||
nm.ctx = ctx
|
||
elif k == ASTKind.Attribute:
|
||
at: rawptr[Attribute] = (rawptr[Attribute])(node)
|
||
at.ctx = ctx
|
||
elif k == ASTKind.Subscript:
|
||
sb: rawptr[Subscript] = (rawptr[Subscript])(node)
|
||
sb.ctx = ctx
|
||
elif k == ASTKind.Tuple:
|
||
tp: rawptr[Tuple] = (rawptr[Tuple])(node)
|
||
tp.ctx = ctx
|
||
elif k == ASTKind.List:
|
||
ls: rawptr[List] = (rawptr[List])(node)
|
||
ls.ctx = ctx
|
||
elif k == ASTKind.Starred:
|
||
st: rawptr[Starred] = (rawptr[Starred])(node)
|
||
st.ctx = ctx
|
||
|
||
|
||
def _fix_store_ctx(node: rawptr[AST]):
|
||
"""Fix ctx of store targets to be Store"""
|
||
if node is None: return
|
||
_set_ctx(node, ASTCtx.Store)
|
||
k: int = node.kind()
|
||
if k == ASTKind.Tuple or k == ASTKind.List:
|
||
tp_elts: rawptr[list[rawptr[AST]]] = None
|
||
if k == ASTKind.Tuple:
|
||
tp: rawptr[Tuple] = (rawptr[Tuple])(node)
|
||
tp_elts = tp.elts
|
||
else:
|
||
ls: rawptr[List] = (rawptr[List])(node)
|
||
tp_elts = ls.elts
|
||
if tp_elts is not None:
|
||
n: t.CSizeT = tp_elts.__len__()
|
||
i: t.CSizeT = 0
|
||
while i < n:
|
||
cur: rawptr[AST] = tp_elts.get(i)
|
||
_fix_store_ctx(cur)
|
||
i += 1
|
||
elif k == ASTKind.Starred:
|
||
st2: rawptr[Starred] = (rawptr[Starred])(node)
|
||
_fix_store_ctx(st2.value)
|
||
|
||
|
||
# ============================================================
|
||
# Expression Parser
|
||
# ============================================================
|
||
|
||
def _parse_expr(ps: rawptr[Parser]) -> rawptr[AST]:
|
||
"""expr: yield_expr | namedexpr_test"""
|
||
if _cur_kw(ps) == Keyword.Yield:
|
||
return _parse_yield(ps)
|
||
return _parse_namedexpr(ps)
|
||
|
||
|
||
def _parse_namedexpr(ps: rawptr[Parser]) -> rawptr[AST]:
|
||
"""namedexpr_test: test [':=' test]"""
|
||
left: rawptr[AST] = _parse_test(ps)
|
||
if left is None: return None
|
||
if _match_op(ps, TokOp.ColonEq):
|
||
_advance(ps)
|
||
right: rawptr[AST] = _parse_test(ps)
|
||
node: rawptr[AST] = NamedExpr(ps.pool, left, right)
|
||
_set_pos(node, left.lineno, left.col_offset, 0, 0)
|
||
return node
|
||
return left
|
||
|
||
|
||
def _parse_test(ps: rawptr[Parser]) -> rawptr[AST]:
|
||
"""test: or_test ['if' or_test 'else' test] | lambdef"""
|
||
if _cur_kw(ps) == Keyword.Lambda:
|
||
return _parse_lambda(ps)
|
||
left: rawptr[AST] = _parse_or_test(ps)
|
||
if left is None: return None
|
||
if _cur_kw(ps) == Keyword.If:
|
||
_advance(ps)
|
||
test: rawptr[AST] = _parse_or_test(ps)
|
||
if _cur_kw(ps) == Keyword.Else:
|
||
_advance(ps)
|
||
orelse: rawptr[AST] = _parse_test(ps)
|
||
node: rawptr[AST] = IfExp(ps.pool, test, left, orelse)
|
||
_set_pos(node, left.lineno, left.col_offset, 0, 0)
|
||
return node
|
||
ps.error_count += 1
|
||
return left
|
||
return left
|
||
|
||
|
||
def _parse_lambda(ps: rawptr[Parser]) -> rawptr[AST]:
|
||
"""lambdef: 'lambda' [varargslist] ':' test"""
|
||
lineno: int = _cur_lineno(ps)
|
||
col: int = _cur_col(ps)
|
||
_advance(ps) # lambda
|
||
args: rawptr[AST] = None
|
||
if not _match_op(ps, TokOp.Colon):
|
||
args = _parse_varargslist(ps, 1)
|
||
if not _expect_op(ps, TokOp.Colon):
|
||
return None
|
||
body: rawptr[AST] = _parse_test(ps)
|
||
node: rawptr[AST] = Lambda(ps.pool, args, body)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
|
||
|
||
def _parse_or_test(ps: rawptr[Parser]) -> rawptr[AST]:
|
||
"""or_test: and_test ('or' and_test)*"""
|
||
left: rawptr[AST] = _parse_and_test(ps)
|
||
if left is None: return None
|
||
values: rawptr[list[rawptr[AST]]] = None
|
||
while True:
|
||
# 括号内行首 or（(A\n or B)）与行尾 or（A or\n B）都支持：
|
||
# 每轮先 _skip_nl 再检查 or，否则 A or B\n or C 在 B 后停在 Nl 上
|
||
# 循环退出，上层把续行 or 当非法 token（对齐 and_test / vast 实现）。
|
||
_skip_nl(ps)
|
||
if _cur_kw(ps) != Keyword.Or:
|
||
break
|
||
if values is None:
|
||
values = list[rawptr[AST]](ps.pool, 8)
|
||
values.append(left)
|
||
_advance(ps)
|
||
_skip_nl(ps) # 括号内跨行: a or\n b
|
||
right: rawptr[AST] = _parse_and_test(ps)
|
||
if right is None: break
|
||
values.append(right)
|
||
if values is not None:
|
||
node: rawptr[AST] = BoolOp(ps.pool, OpKind.Or, values)
|
||
_set_pos(node, left.lineno, left.col_offset, 0, 0)
|
||
return node
|
||
return left
|
||
|
||
|
||
def _parse_and_test(ps: rawptr[Parser]) -> rawptr[AST]:
|
||
"""and_test: not_test ('and' not_test)*"""
|
||
left: rawptr[AST] = _parse_not_test(ps)
|
||
if left is None: return None
|
||
values: rawptr[list[rawptr[AST]]] = None
|
||
while True:
|
||
# 括号内行首 and（(A\n and B)）与行尾 and（A and\n B）都支持：
|
||
# 每轮先 _skip_nl 再检查 and，否则 A and B\n and C 在 B 后停在
|
||
# Nl 上循环退出，上层把续行 and 当非法 token
|
||
# （ThreadingTest check 多行 and 链解析失败根因）。
|
||
_skip_nl(ps)
|
||
if _cur_kw(ps) != Keyword.And:
|
||
break
|
||
if values is None:
|
||
values = list[rawptr[AST]](ps.pool, 8)
|
||
values.append(left)
|
||
_advance(ps)
|
||
_skip_nl(ps) # 括号内跨行: a and\n b
|
||
right: rawptr[AST] = _parse_not_test(ps)
|
||
if right is None: break
|
||
values.append(right)
|
||
if values is not None:
|
||
node: rawptr[AST] = BoolOp(ps.pool, OpKind.And, values)
|
||
_set_pos(node, left.lineno, left.col_offset, 0, 0)
|
||
return node
|
||
return left
|
||
|
||
|
||
def _parse_not_test(ps: rawptr[Parser]) -> rawptr[AST]:
|
||
"""not_test: 'not' not_test | comparison"""
|
||
if _cur_kw(ps) == Keyword.Not:
|
||
lineno: int = _cur_lineno(ps)
|
||
col: int = _cur_col(ps)
|
||
_advance(ps)
|
||
operand: rawptr[AST] = _parse_not_test(ps)
|
||
node: rawptr[AST] = UnaryOp(ps.pool, OpKind.Not, operand)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
return _parse_comparison(ps)
|
||
|
||
|
||
def _parse_comparison(ps: rawptr[Parser]) -> rawptr[AST]:
|
||
"""comparison: bitor_expr (comp_op bitor_expr)*
|
||
|
||
Single comparison design: a < b → Compare(left=a, op=Lt, right=b)
|
||
Chained comparison a < b < c → BoolOp(And, [Compare(a,<,b), Compare(b,<,c)])
|
||
"""
|
||
left: rawptr[AST] = _parse_bitor(ps)
|
||
if left is None: return None
|
||
|
||
_skip_nl(ps) # 括号内行首比较运算符: (a\n < b)
|
||
op_id: int = _match_comp_op(ps)
|
||
if op_id == OpKind.NoneOp:
|
||
return left
|
||
_advance_comp_op(ps)
|
||
_skip_nl(ps) # 括号内跨行: a ==\n b
|
||
right: rawptr[AST] = _parse_bitor(ps)
|
||
# Single comparison: construct ops and comparators list
|
||
ops: rawptr[list[int]] = list[int](ps.pool, 8)
|
||
ops.append(op_id)
|
||
comparators: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
|
||
comparators.append(right)
|
||
first_cmp: rawptr[AST] = Compare(ps.pool, left, ops, comparators)
|
||
_set_pos(first_cmp, left.lineno, left.col_offset, 0, 0)
|
||
|
||
# Check chained comparison
|
||
_skip_nl(ps) # 括号内行首链式比较: (a < b\n < c)
|
||
next_op: int = _match_comp_op(ps)
|
||
if next_op == OpKind.NoneOp:
|
||
return first_cmp
|
||
|
||
# Single comparison with chained conditions: a < b < c → BoolOp(And, [Compare(a,<,b), Compare(b,<,c), ...])
|
||
values: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
|
||
values.append(first_cmp)
|
||
cur_left: rawptr[AST] = right
|
||
while next_op != OpKind.NoneOp:
|
||
_advance_comp_op(ps)
|
||
_skip_nl(ps) # 括号内跨行链式比较
|
||
cur_right: rawptr[AST] = _parse_bitor(ps)
|
||
ops2: rawptr[list[int]] = list[int](ps.pool, 8)
|
||
ops2.append(next_op)
|
||
comparators2: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
|
||
comparators2.append(cur_right)
|
||
cmp: rawptr[AST] = Compare(ps.pool, cur_left, ops2, comparators2)
|
||
_set_pos(cmp, cur_left.lineno, cur_left.col_offset, 0, 0)
|
||
values.append(cmp)
|
||
cur_left = cur_right
|
||
_skip_nl(ps) # 括号内行首链式比较: (a < b\n < c)
|
||
next_op = _match_comp_op(ps)
|
||
node: rawptr[AST] = BoolOp(ps.pool, OpKind.And, values)
|
||
_set_pos(node, left.lineno, left.col_offset, 0, 0)
|
||
return node
|
||
|
||
|
||
def _match_comp_op(ps: rawptr[Parser]) -> int:
|
||
"""Check if the current one is a comparison operator, return OP_* or OpKind.NoneOp"""
|
||
if _cur_type(ps) == TokenType.Op:
|
||
op_id: int = _cur_op(ps)
|
||
if op_id == TokOp.EqEq: return OpKind.Eq
|
||
if op_id == TokOp.ExclaimEq: return OpKind.Ne
|
||
if op_id == TokOp.Less: return OpKind.Lt
|
||
if op_id == TokOp.LessEq: return OpKind.Le
|
||
if op_id == TokOp.Greater: return OpKind.Gt
|
||
if op_id == TokOp.GreaterEq: return OpKind.Ge
|
||
if _cur_kw(ps) == Keyword.Is:
|
||
if ps.cur.next is not None and ps.cur.next.type == TokenType.Name and ps.cur.next.kw_subtype == Keyword.Not:
|
||
return OpKind.IsNot
|
||
return OpKind.Is
|
||
if _cur_kw(ps) == Keyword.Not:
|
||
if ps.cur.next is not None and ps.cur.next.kw_subtype == Keyword.In:
|
||
return OpKind.NotIn
|
||
if _cur_kw(ps) == Keyword.In:
|
||
return OpKind.In
|
||
return OpKind.NoneOp
|
||
|
||
|
||
def _advance_comp_op(ps: rawptr[Parser]):
|
||
"""Consume comparison operator token"""
|
||
if _cur_type(ps) == TokenType.Op:
|
||
_advance(ps)
|
||
return
|
||
if _cur_kw(ps) == Keyword.Is:
|
||
_advance(ps)
|
||
if _cur_kw(ps) == Keyword.Not:
|
||
_advance(ps)
|
||
return
|
||
if _cur_kw(ps) == Keyword.Not:
|
||
_advance(ps)
|
||
if _cur_kw(ps) == Keyword.In:
|
||
_advance(ps)
|
||
return
|
||
if _cur_kw(ps) == Keyword.In:
|
||
_advance(ps)
|
||
|
||
|
||
def _parse_bitor(ps: rawptr[Parser]) -> rawptr[AST]:
|
||
left: rawptr[AST] = _parse_bitxor(ps)
|
||
if left is None: return None
|
||
while True:
|
||
_skip_nl(ps) # 括号内行首 '|': (a\n | b)
|
||
if _match_op(ps, TokOp.VBar) == 0:
|
||
break
|
||
_advance(ps)
|
||
_skip_nl(ps) # 括号内跨行: a |\n b
|
||
right: rawptr[AST] = _parse_bitxor(ps)
|
||
node: rawptr[AST] = BinOp(ps.pool, left, OpKind.BitOr, right)
|
||
_set_pos(node, left.lineno, left.col_offset, 0, 0)
|
||
left = node
|
||
return left
|
||
|
||
|
||
def _parse_bitxor(ps: rawptr[Parser]) -> rawptr[AST]:
|
||
left: rawptr[AST] = _parse_bitand(ps)
|
||
if left is None: return None
|
||
while True:
|
||
_skip_nl(ps) # 括号内行首 '^': (a\n ^ b)
|
||
if _match_op(ps, TokOp.Caret) == 0:
|
||
break
|
||
_advance(ps)
|
||
_skip_nl(ps) # 括号内跨行: a ^\n b
|
||
right: rawptr[AST] = _parse_bitand(ps)
|
||
node: rawptr[AST] = BinOp(ps.pool, left, OpKind.BitXor, right)
|
||
_set_pos(node, left.lineno, left.col_offset, 0, 0)
|
||
left = node
|
||
return left
|
||
|
||
|
||
def _parse_bitand(ps: rawptr[Parser]) -> rawptr[AST]:
|
||
left: rawptr[AST] = _parse_shift(ps)
|
||
if left is None: return None
|
||
while True:
|
||
_skip_nl(ps) # 括号内行首 '&': (a\n & b)
|
||
if _match_op(ps, TokOp.Amp) == 0:
|
||
break
|
||
_advance(ps)
|
||
_skip_nl(ps) # 括号内跨行: a &\n b
|
||
right: rawptr[AST] = _parse_shift(ps)
|
||
node: rawptr[AST] = BinOp(ps.pool, left, OpKind.BitAnd, right)
|
||
_set_pos(node, left.lineno, left.col_offset, 0, 0)
|
||
left = node
|
||
return left
|
||
|
||
|
||
def _parse_shift(ps: rawptr[Parser]) -> rawptr[AST]:
|
||
left: rawptr[AST] = _parse_arith(ps)
|
||
if left is None: return None
|
||
while True:
|
||
_skip_nl(ps) # 括号内行首 '<<'/'>>': (a\n << b)
|
||
if _match_op(ps, TokOp.LtLt) == 0 and _match_op(ps, TokOp.GtGt) == 0:
|
||
break
|
||
op_token: int = _cur_op(ps)
|
||
_advance(ps)
|
||
_skip_nl(ps) # 括号内跨行: a <<\n b
|
||
right: rawptr[AST] = _parse_arith(ps)
|
||
ast_op: int = OpKind.LShift
|
||
if op_token == TokOp.GtGt:
|
||
ast_op = OpKind.RShift
|
||
node: rawptr[AST] = BinOp(ps.pool, left, ast_op, right)
|
||
_set_pos(node, left.lineno, left.col_offset, 0, 0)
|
||
left = node
|
||
return left
|
||
|
||
|
||
def _parse_arith(ps: rawptr[Parser]) -> rawptr[AST]:
|
||
left: rawptr[AST] = _parse_term(ps)
|
||
if left is None: return None
|
||
while True:
|
||
_skip_nl(ps) # 括号内行首 '+'/'-': (a\n + b)
|
||
if _match_op(ps, TokOp.Plus) == 0 and _match_op(ps, TokOp.Minus) == 0:
|
||
break
|
||
op_token: int = _cur_op(ps)
|
||
_advance(ps)
|
||
_skip_nl(ps) # 括号内跨行: a +\n b
|
||
right: rawptr[AST] = _parse_term(ps)
|
||
ast_op: int = OpKind.Add
|
||
if op_token == TokOp.Minus:
|
||
ast_op = OpKind.Sub
|
||
node: rawptr[AST] = BinOp(ps.pool, left, ast_op, right)
|
||
_set_pos(node, left.lineno, left.col_offset, 0, 0)
|
||
left = node
|
||
return left
|
||
|
||
|
||
def _parse_term(ps: rawptr[Parser]) -> rawptr[AST]:
|
||
left: rawptr[AST] = _parse_factor(ps)
|
||
if left is None: return None
|
||
while True:
|
||
_skip_nl(ps) # 括号内行首 '*'/'/'/'//'/'%'/'@': (a\n * b)
|
||
op_token: int = 0
|
||
if _match_op(ps, TokOp.Star): op_token = TokOp.Star
|
||
elif _match_op(ps, TokOp.Slash): op_token = TokOp.Slash
|
||
elif _match_op(ps, TokOp.DSlash): op_token = TokOp.DSlash
|
||
elif _match_op(ps, TokOp.Percent): op_token = TokOp.Percent
|
||
elif _match_op(ps, TokOp.At): op_token = TokOp.At
|
||
if op_token == 0: break
|
||
_advance(ps)
|
||
_skip_nl(ps) # 括号内跨行: a *\n b
|
||
right: rawptr[AST] = _parse_factor(ps)
|
||
ast_op: int = _binop_from_op(op_token)
|
||
node: rawptr[AST] = BinOp(ps.pool, left, ast_op, right)
|
||
_set_pos(node, left.lineno, left.col_offset, 0, 0)
|
||
left = node
|
||
return left
|
||
|
||
|
||
def _parse_factor(ps: rawptr[Parser]) -> rawptr[AST]:
|
||
"""factor: '+' factor | '-' factor | '~' factor | '*' factor | '&' factor | power"""
|
||
if (_match_op(ps, TokOp.Plus) or _match_op(ps, TokOp.Minus) or _match_op(ps, TokOp.Tilde)
|
||
or _match_op(ps, TokOp.Star) or _match_op(ps, TokOp.Amp)):
|
||
op_token: int = _cur_op(ps)
|
||
lineno: int = _cur_lineno(ps)
|
||
col: int = _cur_col(ps)
|
||
_advance(ps)
|
||
operand: rawptr[AST] = _parse_factor(ps)
|
||
ast_op: int = OpKind.UAdd
|
||
if op_token == TokOp.Minus:
|
||
ast_op = OpKind.USub
|
||
elif op_token == TokOp.Tilde:
|
||
ast_op = OpKind.Invert
|
||
elif op_token == TokOp.Star:
|
||
ast_op = OpKind.Deref
|
||
elif op_token == TokOp.Amp:
|
||
ast_op = OpKind.AddrOf
|
||
node: rawptr[AST] = UnaryOp(ps.pool, ast_op, operand)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
return _parse_power(ps)
|
||
|
||
|
||
def _parse_power(ps: rawptr[Parser]) -> rawptr[AST]:
|
||
"""power: await_expr ['**' factor] | atom_expr ['**' factor]"""
|
||
base: rawptr[AST] = _parse_atom_expr(ps)
|
||
if base is None: return None
|
||
if _match_op(ps, TokOp.StarStar):
|
||
_advance(ps)
|
||
exp: rawptr[AST] = _parse_factor(ps)
|
||
node: rawptr[AST] = BinOp(ps.pool, base, OpKind.Pow, exp)
|
||
_set_pos(node, base.lineno, base.col_offset, 0, 0)
|
||
return node
|
||
return base
|
||
|
||
|
||
def _parse_atom_expr(ps: rawptr[Parser]) -> rawptr[AST]:
|
||
"""atom_expr: ['await'] atom trailer*"""
|
||
if _cur_kw(ps) == Keyword.Await:
|
||
lineno: int = _cur_lineno(ps)
|
||
col: int = _cur_col(ps)
|
||
_advance(ps)
|
||
inner: rawptr[AST] = _parse_atom_expr(ps)
|
||
node: rawptr[AST] = Await(ps.pool, inner)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
base: rawptr[AST] = _parse_atom(ps)
|
||
if base is None: return None
|
||
# trailer loop
|
||
while True:
|
||
if _match_op(ps, TokOp.Dot):
|
||
_advance(ps)
|
||
if _cur_type(ps) != TokenType.Name:
|
||
ps.error_count += 1
|
||
break
|
||
attr_name: str = _cur_str(ps)
|
||
attr_lineno: int = _cur_lineno(ps)
|
||
attr_col: int = _cur_col(ps)
|
||
_advance(ps)
|
||
ctx: int = ASTCtx.Load
|
||
if _match_op(ps, TokOp.Equal):
|
||
ctx = ASTCtx.Store
|
||
elif _match_op(ps, TokOp.PlusEq) or _match_op(ps, TokOp.MinusEq):
|
||
ctx = ASTCtx.Store
|
||
base = Attribute(ps.pool, base, attr_name, ctx)
|
||
_set_pos(base, attr_lineno, attr_col, 0, 0)
|
||
elif _match_op(ps, TokOp.LPar):
|
||
base = _parse_call(ps, base)
|
||
elif _match_op(ps, TokOp.Lsqb):
|
||
base = _parse_subscript(ps, base)
|
||
else:
|
||
break
|
||
return base
|
||
|
||
|
||
def _parse_call(ps: rawptr[Parser], func: rawptr[AST]) -> rawptr[AST]:
|
||
"""Parse function call trailer. Both args and keywords are lists."""
|
||
lineno: int = _cur_lineno(ps)
|
||
col: int = _cur_col(ps)
|
||
_advance(ps) # (
|
||
_skip_nls_and_newlines(ps) # Skip the line break after '('
|
||
args: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
|
||
keywords: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
|
||
|
||
if not _match_op(ps, TokOp.RPar):
|
||
while True:
|
||
# **expr
|
||
if _match_op(ps, TokOp.StarStar):
|
||
_advance(ps)
|
||
_skip_nls_and_newlines(ps)
|
||
val: rawptr[AST] = _parse_expr(ps)
|
||
kw: rawptr[AST] = KwNode(ps.pool, None, val)
|
||
_set_pos(kw, _cur_lineno(ps), _cur_col(ps), 0, 0)
|
||
keywords.append(kw)
|
||
# *expr
|
||
elif _match_op(ps, TokOp.Star):
|
||
_advance(ps)
|
||
_skip_nls_and_newlines(ps)
|
||
val = _parse_expr(ps)
|
||
star: rawptr[AST] = Starred(ps.pool, val, ASTCtx.Load)
|
||
_set_pos(star, val.lineno, val.col_offset, 0, 0)
|
||
args.append(star)
|
||
# keyword = expr
|
||
elif _cur_type(ps) == TokenType.Name:
|
||
if ps.cur.next is not None and ps.cur.next.type == TokenType.Op and ps.cur.next.op_subtype == TokOp.Equal:
|
||
kw_name: str = _cur_str(ps)
|
||
kw_lineno: int = _cur_lineno(ps)
|
||
kw_col: int = _cur_col(ps)
|
||
_advance(ps) # name
|
||
_advance(ps) # =
|
||
_skip_nls_and_newlines(ps)
|
||
val = _parse_expr(ps)
|
||
kw = KwNode(ps.pool, kw_name, val)
|
||
_set_pos(kw, kw_lineno, kw_col, 0, 0)
|
||
keywords.append(kw)
|
||
else:
|
||
val = _parse_expr(ps)
|
||
args.append(val)
|
||
else:
|
||
val = _parse_expr(ps)
|
||
args.append(val)
|
||
_skip_nls_and_newlines(ps) # Skip the line break after the parameter
|
||
if not _accept_op(ps, TokOp.Comma):
|
||
break
|
||
_skip_nls_and_newlines(ps) # Skip the line break after the comma
|
||
_skip_nls_and_newlines(ps) # Skip the line break before ')'
|
||
_expect_op(ps, TokOp.RPar)
|
||
node: rawptr[AST] = Call(ps.pool, func, args, keywords)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
|
||
|
||
def _parse_subscript(ps: rawptr[Parser], value: rawptr[AST]) -> rawptr[AST]:
|
||
"""Parse subscript trailer."""
|
||
lineno: int = _cur_lineno(ps)
|
||
col: int = _cur_col(ps)
|
||
_advance(ps) # [
|
||
_skip_nl(ps)
|
||
slice_node: rawptr[AST] = _parse_slice(ps)
|
||
_expect_op(ps, TokOp.Rsqb)
|
||
ctx: int = ASTCtx.Load
|
||
if _match_op(ps, TokOp.Equal):
|
||
ctx = ASTCtx.Store
|
||
node: rawptr[AST] = Subscript(ps.pool, value, slice_node, ctx)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
|
||
|
||
def _parse_slice(ps: rawptr[Parser]) -> rawptr[AST]:
|
||
"""Parse slice: [expr] ':' [expr] [':' [expr]] | expr | expr, expr, ..."""
|
||
lower: rawptr[AST] = None
|
||
if not _match_op(ps, TokOp.Colon):
|
||
lower = _parse_expr(ps)
|
||
|
||
# Check if there’s a comma (tuple syntax: a[x, y, ...])
|
||
if _match_op(ps, TokOp.Comma):
|
||
elts: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
|
||
if lower is not None:
|
||
elts.append(lower)
|
||
while _accept_op(ps, TokOp.Comma):
|
||
if _match_op(ps, TokOp.Rsqb):
|
||
break # Trailing comma
|
||
elem: rawptr[AST] = _parse_expr(ps)
|
||
if elem is not None:
|
||
elts.append(elem)
|
||
lineno: int = 0
|
||
if lower is not None:
|
||
lineno = lower.lineno
|
||
node: rawptr[AST] = Tuple(ps.pool, elts, ASTCtx.Load)
|
||
_set_pos(node, lineno, 0, 0, 0)
|
||
return node
|
||
|
||
if _match_op(ps, TokOp.Colon):
|
||
_advance(ps)
|
||
upper: rawptr[AST] = None
|
||
if not _match_op(ps, TokOp.Colon) and not _match_op(ps, TokOp.Rsqb):
|
||
upper = _parse_expr(ps)
|
||
step: rawptr[AST] = None
|
||
if _accept_op(ps, TokOp.Colon):
|
||
if not _match_op(ps, TokOp.Rsqb):
|
||
step = _parse_expr(ps)
|
||
lineno: int = 0
|
||
if lower is not None:
|
||
lineno = lower.lineno
|
||
elif upper is not None:
|
||
lineno = upper.lineno
|
||
else:
|
||
lineno = _cur_lineno(ps)
|
||
node: rawptr[AST] = Slice(ps.pool, lower, upper, step)
|
||
_set_pos(node, lineno, 0, 0, 0)
|
||
return node
|
||
return lower
|
||
|
||
|
||
def _parse_atom(ps: rawptr[Parser]) -> rawptr[AST]:
|
||
"""atom: the smallest unit of expression"""
|
||
ttype: int = _cur_type(ps)
|
||
lineno: int = _cur_lineno(ps)
|
||
col: int = _cur_col(ps)
|
||
|
||
if ttype == TokenType.Name:
|
||
kw: int = _cur_kw(ps)
|
||
if kw == Keyword.True_:
|
||
_advance(ps)
|
||
return _new_const_bool(ps.pool, 1, lineno, col)
|
||
if kw == Keyword.False_:
|
||
_advance(ps)
|
||
return _new_const_bool(ps.pool, 0, lineno, col)
|
||
if kw == Keyword.None_:
|
||
_advance(ps)
|
||
return _new_const_none(ps.pool, lineno, col)
|
||
name_str: str = _cur_str(ps)
|
||
_advance(ps)
|
||
node: rawptr[AST] = Name(ps.pool, name_str, ASTCtx.Load)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
|
||
if ttype == TokenType.Number:
|
||
tok: rawptr[Token] = ps.cur
|
||
_advance(ps)
|
||
if tok.is_float:
|
||
return _new_const_float(ps.pool, tok.float_val, lineno, col)
|
||
return _new_const_int(ps.pool, tok.int_val, lineno, col)
|
||
|
||
if ttype == TokenType.String:
|
||
sval: str = _cur_str(ps)
|
||
is_fstr: int = ps.cur.kw_subtype & 4
|
||
is_triple: int = ps.cur.kw_subtype & 8
|
||
is_dquote: int = ps.cur.kw_subtype & 16
|
||
is_char: int = ps.cur.kw_subtype & 32
|
||
_advance(ps)
|
||
node: rawptr[AST]
|
||
if is_fstr:
|
||
node = _parse_fstring(ps, sval, lineno, col)
|
||
else:
|
||
node = _new_const_str(ps.pool, sval, lineno, col, is_dquote, is_triple, is_char)
|
||
# Python 相邻字符串字面量隐式拼接："a" "b" → "ab"
|
||
# Nl 是括号内换行 token（允许括号内跨行拼接）；语句级换行是 NewLine，不跳过
|
||
while True:
|
||
_skip_nl(ps)
|
||
if _cur_type(ps) != TokenType.String:
|
||
break
|
||
n_sval: str = _cur_str(ps)
|
||
n_fstr: int = ps.cur.kw_subtype & 4
|
||
n_triple: int = ps.cur.kw_subtype & 8
|
||
n_dquote: int = ps.cur.kw_subtype & 16
|
||
n_char: int = ps.cur.kw_subtype & 32
|
||
_advance(ps)
|
||
n_node: rawptr[AST]
|
||
if n_fstr:
|
||
n_node = _parse_fstring(ps, n_sval, lineno, col)
|
||
else:
|
||
n_node = _new_const_str(ps.pool, n_sval, lineno, col, n_dquote, n_triple, n_char)
|
||
node = _concat_str_nodes(ps.pool, node, n_node)
|
||
return node
|
||
|
||
if _match_op(ps, TokOp.Ellipsis):
|
||
_advance(ps)
|
||
return _new_const_none(ps.pool, lineno, col)
|
||
|
||
if _match_op(ps, TokOp.LPar):
|
||
return _parse_paren(ps)
|
||
if _match_op(ps, TokOp.Lsqb):
|
||
return _parse_list(ps)
|
||
if _match_op(ps, TokOp.LBrace):
|
||
return _parse_brace(ps)
|
||
|
||
_record_parse_err(ps)
|
||
ps.error_count += 1
|
||
return None
|
||
|
||
|
||
def _parse_paren(ps: rawptr[Parser]) -> rawptr[AST]:
|
||
"""Parenthesis expressions: () | (expr) | (expr,) | (expr, ...) | (yield)"""
|
||
lineno: int = _cur_lineno(ps)
|
||
col: int = _cur_col(ps)
|
||
_advance(ps) # (
|
||
_skip_nl(ps) # Skip the line break after '(' (multi-line tuple)
|
||
if _match_op(ps, TokOp.RPar):
|
||
_advance(ps)
|
||
# Empty tuple
|
||
empty: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
|
||
node: rawptr[AST] = Tuple(ps.pool, empty, ASTCtx.Load)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
# yield
|
||
if _cur_kw(ps) == Keyword.Yield:
|
||
y: rawptr[AST] = _parse_yield(ps)
|
||
_expect_op(ps, TokOp.RPar)
|
||
return y
|
||
first: rawptr[AST] = _parse_expr(ps)
|
||
if _match_op(ps, TokOp.Comma):
|
||
# Tuple
|
||
elts: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
|
||
elts.append(first)
|
||
while _accept_op(ps, TokOp.Comma):
|
||
_skip_nl(ps)
|
||
if _match_op(ps, TokOp.RPar):
|
||
break
|
||
e: rawptr[AST] = _parse_expr(ps)
|
||
elts.append(e)
|
||
_skip_nl(ps) # 末元素无尾逗号时收尾（多行元组）
|
||
_expect_op(ps, TokOp.RPar)
|
||
node = Tuple(ps.pool, elts, ASTCtx.Load)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
_skip_nl(ps) # (expr\n) 单表达式括号跨行收尾
|
||
_expect_op(ps, TokOp.RPar)
|
||
# Check for generator expression
|
||
if _cur_kw(ps) == Keyword.For:
|
||
return _parse_comprehension(ps, first, ASTKind.GeneratorExp, lineno, col)
|
||
return first
|
||
|
||
|
||
def _parse_list(ps: rawptr[Parser]) -> rawptr[AST]:
|
||
"""List expressions: [] | [expr_list] | [expr_list comp_for]"""
|
||
lineno: int = _cur_lineno(ps)
|
||
col: int = _cur_col(ps)
|
||
_advance(ps) # [
|
||
_skip_nl(ps) # Skip the line break after '[' (multi-line list)
|
||
if _match_op(ps, TokOp.Rsqb):
|
||
_advance(ps)
|
||
empty: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
|
||
node: rawptr[AST] = List(ps.pool, empty, ASTCtx.Load)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
first: rawptr[AST] = _parse_expr(ps)
|
||
if _cur_kw(ps) == Keyword.For:
|
||
node = _parse_comprehension(ps, first, ASTKind.ListComp, lineno, col)
|
||
_expect_op(ps, TokOp.Rsqb)
|
||
return node
|
||
elts: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
|
||
elts.append(first)
|
||
while _accept_op(ps, TokOp.Comma):
|
||
_skip_nl(ps)
|
||
if _match_op(ps, TokOp.Rsqb):
|
||
break
|
||
e: rawptr[AST] = _parse_expr(ps)
|
||
elts.append(e)
|
||
_skip_nl(ps) # 末元素无尾逗号时收尾（base64.vp 多行列表）
|
||
_expect_op(ps, TokOp.Rsqb)
|
||
node = List(ps.pool, elts, ASTCtx.Load)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
|
||
|
||
def _parse_brace(ps: rawptr[Parser]) -> rawptr[AST]:
|
||
"""Brace expressions: {} | {set_items} | {dict_items} | {comp_for}"""
|
||
lineno: int = _cur_lineno(ps)
|
||
col: int = _cur_col(ps)
|
||
_advance(ps) # {
|
||
_skip_nl(ps) # Skip the line break after '{' (multi-line dict)
|
||
if _match_op(ps, TokOp.RBrace):
|
||
_advance(ps)
|
||
empty_k: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
|
||
empty_v: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
|
||
node: rawptr[AST] = Dict(ps.pool, empty_k, empty_v)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
first: rawptr[AST] = _parse_expr(ps)
|
||
# dict: key: value
|
||
if _match_op(ps, TokOp.Colon):
|
||
_advance(ps)
|
||
val: rawptr[AST] = _parse_expr(ps)
|
||
if _cur_kw(ps) == Keyword.For:
|
||
# dict comprehension
|
||
gens: rawptr[list[rawptr[AST]]] = _parse_comp_for(ps)
|
||
node = DictComp(ps.pool, first, val, gens)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
_expect_op(ps, TokOp.RBrace)
|
||
return node
|
||
keys: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
|
||
values: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
|
||
keys.append(first)
|
||
values.append(val)
|
||
while _accept_op(ps, TokOp.Comma):
|
||
_skip_nl(ps)
|
||
if _match_op(ps, TokOp.RBrace):
|
||
break
|
||
k: rawptr[AST] = _parse_expr(ps)
|
||
_expect_op(ps, TokOp.Colon)
|
||
v: rawptr[AST] = _parse_expr(ps)
|
||
keys.append(k)
|
||
values.append(v)
|
||
_skip_nl(ps) # 末元素无尾逗号时收尾
|
||
_expect_op(ps, TokOp.RBrace)
|
||
node = Dict(ps.pool, keys, values)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
# set
|
||
if _cur_kw(ps) == Keyword.For:
|
||
node = _parse_comprehension(ps, first, ASTKind.SetComp, lineno, col)
|
||
_expect_op(ps, TokOp.RBrace)
|
||
return node
|
||
elts: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
|
||
elts.append(first)
|
||
while _accept_op(ps, TokOp.Comma):
|
||
_skip_nl(ps) # 括号内跨行集合字面量
|
||
if _match_op(ps, TokOp.RBrace):
|
||
break
|
||
e: rawptr[AST] = _parse_expr(ps)
|
||
elts.append(e)
|
||
_skip_nl(ps) # 末元素无尾逗号时收尾
|
||
_expect_op(ps, TokOp.RBrace)
|
||
node = Set(ps.pool, elts)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
|
||
|
||
# ============================================================
|
||
# f-string parsing
|
||
# ============================================================
|
||
|
||
def _parse_sub_expr(ps: rawptr[Parser], src: str) -> rawptr[AST]:
|
||
"""Create a sub-lexer and parser to parse expressions (used for {expr} inside f-strings)"""
|
||
pool: rawptr[memhub.MemManager] = ps.pool
|
||
lx: rawptr[Lexer] = new_lexer(pool)
|
||
if lx is None: return None
|
||
_lexer_init(lx, src, pool)
|
||
tokens: rawptr[Token] = tokenize(lx)
|
||
if tokens is None: return None
|
||
sub_ps: rawptr[Parser] = new_parser(pool)
|
||
if sub_ps is None: return None
|
||
_parser_init(sub_ps, tokens, pool)
|
||
return _parse_test(sub_ps)
|
||
|
||
|
||
def _append_str_value(pool: rawptr[memhub.MemManager],
|
||
values: rawptr[list[rawptr[AST]]],
|
||
node: rawptr[AST]):
|
||
"""将字符串节点展开加入 values（JoinedStr 展开其 values，其余直接追加）"""
|
||
if node is None:
|
||
return
|
||
if node.kind() == ASTKind.JoinedStr:
|
||
js_val: rawptr[JoinedStr] = (rawptr[JoinedStr])(node)
|
||
if js_val.values is not None:
|
||
n_v: t.CSizeT = js_val.values.__len__()
|
||
i_v: t.CSizeT
|
||
for i_v in range(n_v):
|
||
values.append(js_val.values.get(i_v))
|
||
else:
|
||
values.append(node)
|
||
|
||
|
||
def _concat_str_nodes(pool: rawptr[memhub.MemManager],
|
||
a: rawptr[AST], b: rawptr[AST]) -> rawptr[AST]:
|
||
"""合并相邻字符串字面量节点（Python 隐式拼接 "a" "b" → "ab"）
|
||
|
||
纯常量字符串直接拼接文本；涉及 f-string（JoinedStr/FormattedValue）时
|
||
合并到 JoinedStr 的 values 列表。
|
||
"""
|
||
if a is None:
|
||
return b
|
||
if b is None:
|
||
return a
|
||
# 两个都是常量字符串 → 直接拼接文本
|
||
if a.kind() == ASTKind.Constant and b.kind() == ASTKind.Constant:
|
||
ca_c: rawptr[Constant] = (rawptr[Constant])(a)
|
||
cb_c: rawptr[Constant] = (rawptr[Constant])(b)
|
||
if ca_c.const_kind == CONST_STR and cb_c.const_kind == CONST_STR \
|
||
and ca_c.str_val is not None and cb_c.str_val is not None:
|
||
la_c: t.CSizeT = string.strlen(ca_c.str_val)
|
||
lb_c: t.CSizeT = string.strlen(cb_c.str_val)
|
||
buf_c: str = pool.alloc(la_c + lb_c + 1)
|
||
if buf_c is not None:
|
||
string.memcpy(buf_c, ca_c.str_val, la_c)
|
||
string.memcpy(buf_c + la_c, cb_c.str_val, lb_c + 1)
|
||
return _new_const_str(pool, buf_c, a.lineno, a.col_offset, 0, 0, 0)
|
||
# 涉及 f-string → 合并到 JoinedStr
|
||
values_c: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](pool, 8)
|
||
_append_str_value(pool, values_c, a)
|
||
_append_str_value(pool, values_c, b)
|
||
if values_c.__len__() == 1:
|
||
return values_c.get(0)
|
||
node_c: rawptr[AST] = JoinedStr(pool, values_c)
|
||
_set_pos(node_c, a.lineno, a.col_offset, 0, 0)
|
||
return node_c
|
||
|
||
|
||
def _parse_fstring(ps: rawptr[Parser], content: str,
|
||
lineno: int, col: int) -> rawptr[AST]:
|
||
"""Parse f-string content, building JoinedStr or FormattedValue nodes"""
|
||
pool: rawptr[memhub.MemManager] = ps.pool
|
||
values: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](pool, 8)
|
||
i: t.CSizeT = 0
|
||
content_len: t.CSizeT = string.strlen(content)
|
||
# Literal text buffer for f-string content
|
||
lit_buf: str = pool.alloc(content_len + 1)
|
||
if lit_buf is None: return None
|
||
lit_pos: t.CSizeT = 0
|
||
|
||
while i < content_len:
|
||
ch: t.CChar = content[i]
|
||
if ch == '{':
|
||
# {{ → Literal {
|
||
if i + 1 < content_len and content[i + 1] == '{':
|
||
lit_buf[lit_pos] = '{'
|
||
lit_pos += 1
|
||
i += 2
|
||
continue
|
||
# Refresh literal buffer
|
||
if lit_pos > 0:
|
||
lit_buf[lit_pos] = '\0'
|
||
lit_node: rawptr[AST] = _new_const_str(pool, lit_buf, lineno, col)
|
||
values.append(lit_node)
|
||
lit_buf = pool.alloc(content_len + 1)
|
||
if lit_buf is None: break
|
||
lit_pos = 0
|
||
# Find matching closing '}' while tracking nested '{}' and string literals inside the f-string
|
||
j: t.CSizeT = i + 1
|
||
depth: int = 1
|
||
conv_pos: t.CSizeT = 0
|
||
fmt_pos: t.CSizeT = 0
|
||
while j < content_len and depth > 0:
|
||
c: t.CChar = content[j]
|
||
if c == '\'' or c == '"':
|
||
# 跳过表达式块内的嵌套字符串字面量（PEP 701 同引号嵌套），
|
||
# 避免其内部字符干扰 {} 深度/转换/格式分隔符定位
|
||
nq: t.CChar = c
|
||
j += 1
|
||
while j < content_len:
|
||
c2: t.CChar = content[j]
|
||
if c2 == '\\':
|
||
j += 2
|
||
continue
|
||
if c2 == nq:
|
||
j += 1
|
||
break
|
||
j += 1
|
||
continue
|
||
if c == '{':
|
||
depth += 1
|
||
elif c == '}':
|
||
depth -= 1
|
||
if depth == 0:
|
||
break
|
||
elif depth == 1:
|
||
if c == '!' and conv_pos == 0 and fmt_pos == 0:
|
||
if j + 1 < content_len:
|
||
nc: t.CChar = content[j + 1]
|
||
if nc == 's' or nc == 'r' or nc == 'a':
|
||
if j + 2 < content_len:
|
||
after: t.CChar = content[j + 2]
|
||
if after == ':' or after == '}':
|
||
conv_pos = j
|
||
elif j + 2 == content_len:
|
||
conv_pos = j
|
||
elif c == ':' and fmt_pos == 0:
|
||
fmt_pos = j
|
||
j += 1
|
||
if depth != 0:
|
||
_record_parse_err(ps)
|
||
ps.error_count += 1
|
||
break
|
||
# Extract expression text
|
||
expr_end: t.CSizeT = j
|
||
if conv_pos > 0:
|
||
expr_end = conv_pos
|
||
elif fmt_pos > 0:
|
||
expr_end = fmt_pos
|
||
expr_len: t.CSizeT = expr_end - i - 1
|
||
if expr_len == 0:
|
||
_record_parse_err(ps)
|
||
ps.error_count += 1
|
||
i = j + 1
|
||
continue
|
||
expr_buf: str = pool.alloc(expr_len + 1)
|
||
if expr_buf is None: break
|
||
string.memcpy(expr_buf, content + i + 1, expr_len)
|
||
expr_buf[expr_len] = '\0'
|
||
# Parse expression text
|
||
expr_node: rawptr[AST] = _parse_sub_expr(ps, expr_buf)
|
||
# Conversion specifier
|
||
conversion: int = -1
|
||
if conv_pos > 0:
|
||
conv_char: t.CChar = content[conv_pos + 1]
|
||
if conv_char == 's':
|
||
conversion = 115
|
||
elif conv_char == 'r':
|
||
conversion = 114
|
||
elif conv_char == 'a':
|
||
conversion = 97
|
||
# Format specifier
|
||
format_spec: rawptr[AST] = None
|
||
if fmt_pos > 0:
|
||
spec_len: t.CSizeT = j - fmt_pos - 1
|
||
if spec_len > 0:
|
||
spec_buf: str = pool.alloc(spec_len + 1)
|
||
if spec_buf is not None:
|
||
string.memcpy(spec_buf, content + fmt_pos + 1, spec_len)
|
||
spec_buf[spec_len] = '\0'
|
||
format_spec = _new_const_str(pool, spec_buf, lineno, col)
|
||
fv: rawptr[AST] = FormattedValue(pool, expr_node, conversion, format_spec)
|
||
_set_pos(fv, lineno, col, 0, 0)
|
||
values.append(fv)
|
||
i = j + 1
|
||
elif ch == '}':
|
||
if i + 1 < content_len and content[i + 1] == '}':
|
||
lit_buf[lit_pos] = '}'
|
||
lit_pos += 1
|
||
i += 2
|
||
else:
|
||
_record_parse_err(ps)
|
||
ps.error_count += 1
|
||
i += 1
|
||
else:
|
||
lit_buf[lit_pos] = ch
|
||
lit_pos += 1
|
||
i += 1
|
||
|
||
# Refresh remaining literal buffer
|
||
if lit_pos > 0:
|
||
lit_buf[lit_pos] = '\0'
|
||
lit_node2: rawptr[AST] = _new_const_str(pool, lit_buf, lineno, col)
|
||
values.append(lit_node2)
|
||
|
||
# If no content, return empty string
|
||
if values.__len__() == 0:
|
||
return _new_const_str(pool, "", lineno, col)
|
||
# If only one FormattedValue and no literal, return FormattedValue
|
||
if values.__len__() == 1:
|
||
first: rawptr[AST] = values.get(0)
|
||
if first.kind() == ASTKind.FormattedValue:
|
||
return first
|
||
node: rawptr[AST] = JoinedStr(pool, values)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
|
||
|
||
def _parse_comprehension(ps: rawptr[Parser], elt: rawptr[AST],
|
||
vtype: int, lineno: int, col: int) -> rawptr[AST]:
|
||
"""General comprehension: elt comp_for"""
|
||
gens: rawptr[list[rawptr[AST]]] = _parse_comp_for(ps)
|
||
node: rawptr[AST] = None
|
||
if vtype == ASTKind.ListComp:
|
||
node = ListComp(ps.pool, elt, gens)
|
||
elif vtype == ASTKind.SetComp:
|
||
node = SetComp(ps.pool, elt, gens)
|
||
else:
|
||
node = GeneratorExp(ps.pool, elt, gens)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
|
||
|
||
def _parse_comp_for(ps: rawptr[Parser]) -> rawptr[list[rawptr[AST]]]:
|
||
"""comp_for: ['async'] 'for' exprlist 'in' or_test [comp_if] (comp_for)*
|
||
Returns generators list。"""
|
||
gens: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
|
||
while True:
|
||
is_async: int = 0
|
||
if _cur_kw(ps) == Keyword.Async:
|
||
is_async = 1
|
||
_advance(ps)
|
||
if _cur_kw(ps) != Keyword.For:
|
||
break
|
||
lineno: int = _cur_lineno(ps)
|
||
col: int = _cur_col(ps)
|
||
_advance(ps)
|
||
target: rawptr[AST] = _parse_target_list(ps)
|
||
if _cur_kw(ps) != Keyword.In:
|
||
ps.error_count += 1
|
||
break
|
||
_advance(ps)
|
||
iter_: rawptr[AST] = _parse_or_test(ps)
|
||
# Collect ifs to list
|
||
ifs: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
|
||
while _cur_kw(ps) == Keyword.If:
|
||
_advance(ps)
|
||
cond: rawptr[AST] = _parse_or_test(ps)
|
||
ifs.append(cond)
|
||
comp: rawptr[AST] = Comprehension(ps.pool, target, iter_, ifs, is_async)
|
||
_set_pos(comp, lineno, col, 0, 0)
|
||
gens.append(comp)
|
||
return gens
|
||
|
||
|
||
def _parse_target_list(ps: rawptr[Parser]) -> rawptr[AST]:
|
||
"""Parse assignment target list (comma-separated)"""
|
||
first: rawptr[AST] = _parse_target(ps)
|
||
if _match_op(ps, TokOp.Comma):
|
||
elts: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
|
||
elts.append(first)
|
||
while _accept_op(ps, TokOp.Comma):
|
||
if _cur_kw(ps) == Keyword.In:
|
||
break
|
||
e: rawptr[AST] = _parse_target(ps)
|
||
elts.append(e)
|
||
node: rawptr[AST] = Tuple(ps.pool, elts, ASTCtx.Store)
|
||
_set_pos(node, first.lineno, first.col_offset, 0, 0)
|
||
return node
|
||
return first
|
||
|
||
|
||
def _parse_target(ps: rawptr[Parser]) -> rawptr[AST]:
|
||
"""Parse assignment target"""
|
||
if _match_op(ps, TokOp.Star):
|
||
_advance(ps)
|
||
inner: rawptr[AST] = _parse_atom_expr(ps)
|
||
node: rawptr[AST] = Starred(ps.pool, inner, ASTCtx.Store)
|
||
_set_pos(node, inner.lineno, inner.col_offset, 0, 0)
|
||
return node
|
||
if _match_op(ps, TokOp.LPar):
|
||
_advance(ps)
|
||
_skip_nl(ps)
|
||
t_: rawptr[AST] = _parse_target_list(ps)
|
||
_skip_nl(ps)
|
||
_expect_op(ps, TokOp.RPar)
|
||
return t_
|
||
if _match_op(ps, TokOp.Lsqb):
|
||
_advance(ps)
|
||
_skip_nl(ps)
|
||
first: rawptr[AST] = _parse_target(ps)
|
||
elts: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
|
||
elts.append(first)
|
||
while _accept_op(ps, TokOp.Comma):
|
||
_skip_nl(ps)
|
||
if _match_op(ps, TokOp.Rsqb):
|
||
break
|
||
e: rawptr[AST] = _parse_target(ps)
|
||
elts.append(e)
|
||
_expect_op(ps, TokOp.Rsqb)
|
||
node = List(ps.pool, elts, ASTCtx.Store)
|
||
_set_pos(node, first.lineno, first.col_offset, 0, 0)
|
||
return node
|
||
# Normal atom_expr, ctx=Store
|
||
node = _parse_atom_expr(ps)
|
||
if node is not None:
|
||
_set_ctx(node, ASTCtx.Store)
|
||
return node
|
||
|
||
|
||
def _parse_yield(ps: rawptr[Parser]) -> rawptr[AST]:
|
||
lineno: int = _cur_lineno(ps)
|
||
col: int = _cur_col(ps)
|
||
_advance(ps) # yield
|
||
if _cur_kw(ps) == Keyword.From:
|
||
_advance(ps)
|
||
val: rawptr[AST] = _parse_expr(ps)
|
||
node: rawptr[AST] = YieldFrom(ps.pool, val)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
val2: rawptr[AST] = None
|
||
if not _match_op(ps, TokOp.RPar) and _cur_type(ps) != TokenType.NewLine and _cur_type(ps) != TokenType.EndMarker:
|
||
if _cur_type(ps) == TokenType.Op and (_cur_op(ps) == TokOp.Comma or _cur_op(ps) == TokOp.Rsqb):
|
||
pass
|
||
else:
|
||
val2 = _parse_test(ps)
|
||
node = Yield(ps.pool, val2)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
|
||
|
||
def _parse_varargslist(ps: rawptr[Parser], is_lambda: int = 0) -> rawptr[AST]:
|
||
"""Parse function parameter list
|
||
New API: args/defaults/kw_defaults are lists, vararg/kwarg are fields."""
|
||
args: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
|
||
vararg: rawptr[AST] = None
|
||
kwarg: rawptr[AST] = None
|
||
defaults: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
|
||
kw_defaults: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
|
||
|
||
while True:
|
||
_skip_nl(ps)
|
||
if _match_op(ps, TokOp.RPar):
|
||
break
|
||
# *args 或 * (kwonly separated)
|
||
if _match_op(ps, TokOp.Star):
|
||
_advance(ps)
|
||
_skip_nl(ps)
|
||
if _cur_type(ps) == TokenType.Name:
|
||
vararg = _parse_arg_def(ps, is_lambda)
|
||
if not _accept_op(ps, TokOp.Comma):
|
||
break
|
||
_skip_nl(ps)
|
||
continue
|
||
# **kwargs
|
||
if _match_op(ps, TokOp.StarStar):
|
||
_advance(ps)
|
||
_skip_nl(ps)
|
||
kwarg = _parse_arg_def(ps, is_lambda)
|
||
if not _accept_op(ps, TokOp.Comma):
|
||
break
|
||
_skip_nl(ps)
|
||
continue
|
||
# / Delimiter (posonly end)— New API: posonly/regular are not distinguished
|
||
if _match_op(ps, TokOp.Slash):
|
||
_advance(ps)
|
||
if not _accept_op(ps, TokOp.Comma):
|
||
break
|
||
_skip_nl(ps)
|
||
continue
|
||
# Normal parameter
|
||
a: rawptr[AST] = _parse_arg_def(ps, is_lambda)
|
||
if a is None: break
|
||
args.append(a)
|
||
if _match_op(ps, TokOp.Equal):
|
||
_advance(ps)
|
||
d: rawptr[AST] = _parse_test(ps)
|
||
defaults.append(d)
|
||
if not _accept_op(ps, TokOp.Comma):
|
||
break
|
||
_skip_nl(ps)
|
||
return Arguments(ps.pool, args, vararg, kwarg, defaults, kw_defaults)
|
||
|
||
|
||
def _parse_type_ann(ps: rawptr[Parser]) -> rawptr[AST]:
|
||
"""解析类型注解，支持 C 风格指针后缀（* / **）。
|
||
|
||
int* → rawptr[int] （i32*）
|
||
int** → (rawptr[int]) | t.CPtr （i32**）
|
||
int****** → 6 层嵌套 （i32******）
|
||
|
||
_parse_test 会把后缀 '*' 消费为乘法（Mult）、'**' 消费为幂（Pow），
|
||
右操作数解析失败时产生 BinOp(op, right=None) 垃圾节点并 error_count+1
|
||
（'int***' 形态为 BinOp(Pow, UnaryOp(Deref, None))，StarStar+Star）。
|
||
本函数回收该形态并恢复 error_count，再按层数嵌套构造 BinOp(BitOr, t.CPtr)。
|
||
"""
|
||
before_ec: int = ps.error_count
|
||
base: rawptr[AST] = _parse_test(ps)
|
||
if base is None:
|
||
_record_parse_err(ps) # 注解解析失败：记录具体位置（error_count 已在 _parse_test 内增加）
|
||
return None
|
||
ptr_level: int = 0
|
||
# 回收垃圾节点（_parse_test 对后缀 * / ** 的消费产物）
|
||
if base.kind() == ASTKind.BinOp:
|
||
gb: rawptr[BinOp] = (rawptr[BinOp])(base)
|
||
if gb.op == OpKind.Mult and gb.right is None:
|
||
ptr_level += 1
|
||
base = gb.left
|
||
elif gb.op == OpKind.Pow:
|
||
if gb.right is None:
|
||
ptr_level += 2
|
||
base = gb.left
|
||
elif gb.right.kind() == ASTKind.UnaryOp:
|
||
gu: rawptr[UnaryOp] = (rawptr[UnaryOp])(gb.right)
|
||
if gu.op == OpKind.Deref and gu.operand is None:
|
||
ptr_level += 3
|
||
base = gb.left
|
||
if ptr_level > 0:
|
||
ps.error_count = before_ec # 回收成功，恢复错误计数避免整文件解析失败
|
||
# 循环消费残留后缀（'**' 是词法单 token 计 2 层，'*' 计 1 层）
|
||
while _match_op(ps, TokOp.StarStar):
|
||
_advance(ps)
|
||
ptr_level += 2
|
||
while _match_op(ps, TokOp.Star):
|
||
_advance(ps)
|
||
ptr_level += 1
|
||
# 嵌套构造 rawptr[base]（n 层）
|
||
li_an: int = 0
|
||
while li_an < ptr_level:
|
||
cptr_an: rawptr[AST] = Attribute(ps.pool,
|
||
Name(ps.pool, "t", ASTCtx.Load),
|
||
"CPtr", ASTCtx.Load)
|
||
base = BinOp(ps.pool, base, OpKind.BitOr, cptr_an)
|
||
li_an += 1
|
||
return base
|
||
|
||
|
||
def _parse_arg_def(ps: rawptr[Parser], is_lambda: int = 0) -> rawptr[AST]:
|
||
"""arg: TokenType.Name [':' annotation]"""
|
||
if _cur_type(ps) != TokenType.Name:
|
||
ps.error_count += 1
|
||
return None
|
||
name_str: str = _cur_str(ps)
|
||
lineno: int = _cur_lineno(ps)
|
||
col: int = _cur_col(ps)
|
||
_advance(ps)
|
||
annotation: rawptr[AST] = None
|
||
if is_lambda == 0:
|
||
if _accept_op(ps, TokOp.Colon):
|
||
annotation = _parse_type_ann(ps)
|
||
result: rawptr[AST] = Arg(ps.pool, name_str, annotation)
|
||
_set_pos(result, lineno, col, 0, 0)
|
||
return result
|
||
|
||
|
||
# ============================================================
|
||
# Statement parsing (Design: directly append to parent, do not return a list)
|
||
# ============================================================
|
||
|
||
def _parse_suite_into_children(ps: rawptr[Parser], parent: rawptr[AST]):
|
||
"""Parse statement suite, directly append to parent's children.
|
||
suite: TokenType.NewLine TokenType.Indent stmt+ TokenType.Dedent | simple_stmt"""
|
||
_skip_nl(ps)
|
||
if _cur_type(ps) == TokenType.NewLine:
|
||
_advance(ps)
|
||
_skip_nl(ps)
|
||
if _cur_type(ps) != TokenType.Indent:
|
||
ps.error_count += 1
|
||
return
|
||
_advance(ps)
|
||
sidx: int = 0
|
||
while _cur_type(ps) != TokenType.Dedent and _cur_type(ps) != TokenType.EndMarker:
|
||
sidx += 1
|
||
# Decorator (class body methods can have decorators)
|
||
if _match_op(ps, TokOp.At):
|
||
ps.pending_decorators = _parse_decorators(ps)
|
||
before_cur: rawptr[Token] = ps.cur
|
||
_parse_statement_into(ps, parent)
|
||
# Defensive: if parse did not advance token, force advance to avoid infinite loop
|
||
if ps.cur is before_cur:
|
||
ps.error_count += 1
|
||
_advance(ps)
|
||
_skip_nl(ps)
|
||
_expect_type(ps, TokenType.Dedent)
|
||
return
|
||
# Single-line suite
|
||
_parse_simple_stmt_into(ps, parent)
|
||
|
||
|
||
def _parse_statement_into(ps: rawptr[Parser], parent: rawptr[AST]):
|
||
"""Parse statement (compound or simple), append to the parent."""
|
||
ttype: int = _cur_type(ps)
|
||
kw: int = _cur_kw(ps)
|
||
if ttype == TokenType.Name:
|
||
if kw == Keyword.If or kw == Keyword.While or kw == Keyword.For or kw == Keyword.Try:
|
||
stmt: rawptr[AST] = _parse_compound(ps)
|
||
if stmt is not None:
|
||
parent.append(stmt)
|
||
return
|
||
if kw == Keyword.With or _is_funcdef_kw(ps, kw) != 0 or kw == Keyword.Class:
|
||
stmt = _parse_compound(ps)
|
||
if stmt is not None:
|
||
parent.append(stmt)
|
||
return
|
||
if (kw == Keyword.Match and _is_match_stmt(ps) != 0) or kw == Keyword.Async:
|
||
stmt = _parse_compound(ps)
|
||
if stmt is not None:
|
||
parent.append(stmt)
|
||
return
|
||
_parse_simple_stmt_into(ps, parent)
|
||
|
||
|
||
def _parse_simple_stmt_into(ps: rawptr[Parser], parent: rawptr[AST]):
|
||
"""simple_stmt: small_stmt (';' small_stmt)* [';'] TokenType.NewLine"""
|
||
first: rawptr[AST] = _parse_small_stmt(ps)
|
||
if first is not None:
|
||
parent.append(first)
|
||
while _accept_op(ps, TokOp.Semi):
|
||
if _cur_type(ps) == TokenType.NewLine or _cur_type(ps) == TokenType.EndMarker:
|
||
break
|
||
s: rawptr[AST] = _parse_small_stmt(ps)
|
||
if s is None: break
|
||
parent.append(s)
|
||
# 先跳过 Nl（括号内换行，如多行函数调用参数间的换行，属合法 token）
|
||
_skip_nl(ps)
|
||
# 语句结束检查：simple_stmt 必须以换行/文件尾结束。
|
||
# 残留 token（如相邻表达式 `func12344444 avg3(...)` 被静默拆成多条语句、
|
||
# `def` 缺失的关键字函数定义残留）是语法错误（fail-fast），禁止静默，
|
||
# 并仿 Python 报告错误位置（SyntaxError at line/col）。
|
||
if _cur_type(ps) != TokenType.NewLine and _cur_type(ps) != TokenType.EndMarker:
|
||
_record_parse_err(ps)
|
||
err_str: str = _cur_str(ps)
|
||
if err_str is None:
|
||
err_str = "(none)"
|
||
# 定位：从链表头找 cur，记录前一个 token 与下一个 token 的上下文
|
||
prev_tok: rawptr[Token] = None
|
||
walk_tok: rawptr[Token] = ps.tokens
|
||
while walk_tok is not None and walk_tok is not ps.cur:
|
||
prev_tok = walk_tok
|
||
walk_tok = walk_tok.next
|
||
pv_str: str = "(head)"
|
||
pv_line: int = 0
|
||
pv_col: int = 0
|
||
if prev_tok is not None:
|
||
pv_str = prev_tok.str_val
|
||
pv_line = prev_tok.lineno
|
||
pv_col = prev_tok.col_offset
|
||
nxt_tok: rawptr[Token] = ps.cur.next
|
||
nxt_str: str = "(none)"
|
||
nxt_line: int = 0
|
||
nxt_col: int = 0
|
||
if nxt_tok is not None:
|
||
nxt_str = nxt_tok.str_val
|
||
nxt_line = nxt_tok.lineno
|
||
nxt_col = nxt_tok.col_offset
|
||
stdio.printf("[AST] SyntaxError: line %d col %d (end %d %d), 意外的 token '%s' (type=%d) | prev='%s' (line %d col %d) | next='%s' (line %d col %d)\n",
|
||
_g_parse_err_line, _g_parse_err_col, ps.cur.end_lineno, ps.cur.end_col_offset,
|
||
err_str, _cur_type(ps),
|
||
pv_str, pv_line, pv_col,
|
||
nxt_str, nxt_line, nxt_col)
|
||
ps.error_count += 1
|
||
return
|
||
if _cur_type(ps) == TokenType.NewLine:
|
||
_advance(ps)
|
||
|
||
|
||
def _parse_small_stmt(ps: rawptr[Parser]) -> rawptr[AST]:
|
||
"""small_stmt: various simple statements"""
|
||
kw: int = _cur_kw(ps)
|
||
if kw == Keyword.Pass:
|
||
lineno: int = _cur_lineno(ps)
|
||
col: int = _cur_col(ps)
|
||
_advance(ps)
|
||
node: rawptr[AST] = Pass(ps.pool)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
if kw == Keyword.Break:
|
||
lineno = _cur_lineno(ps)
|
||
col = _cur_col(ps)
|
||
_advance(ps)
|
||
node = Break(ps.pool)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
if kw == Keyword.Continue:
|
||
lineno = _cur_lineno(ps)
|
||
col = _cur_col(ps)
|
||
_advance(ps)
|
||
node = Continue(ps.pool)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
if kw == Keyword.Return:
|
||
return _parse_return(ps)
|
||
if kw == Keyword.Raise:
|
||
return _parse_raise(ps)
|
||
if kw == Keyword.Global:
|
||
return _parse_global(ps)
|
||
if kw == Keyword.Nonlocal:
|
||
return _parse_nonlocal(ps)
|
||
if kw == Keyword.Import:
|
||
return _parse_import(ps)
|
||
if kw == Keyword.From:
|
||
return _parse_from_import(ps)
|
||
if kw == Keyword.Del:
|
||
return _parse_del(ps)
|
||
if kw == Keyword.Assert:
|
||
return _parse_assert(ps)
|
||
if kw == Keyword.Yield:
|
||
return _parse_expr_stmt(ps, _parse_yield(ps))
|
||
# Expression statement / assignment / augassign / annassign
|
||
return _parse_expr_stmt(ps, None)
|
||
|
||
|
||
def _maybe_tuple(ps: rawptr[Parser], first: rawptr[AST]) -> rawptr[AST]:
|
||
"""If there is a comma after first, collect the comma-separated expressions into a Tuple."""
|
||
if first is None: return None
|
||
if not _match_op(ps, TokOp.Comma):
|
||
return first
|
||
lineno: int = first.lineno
|
||
col: int = first.col_offset
|
||
elts: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
|
||
elts.append(first)
|
||
_accept_op(ps, TokOp.Comma)
|
||
while True:
|
||
tt: int = _cur_type(ps)
|
||
if tt == TokenType.NewLine or tt == TokenType.EndMarker:
|
||
break
|
||
if tt == TokenType.Op:
|
||
op: int = _cur_op(ps)
|
||
if op == TokOp.Equal or op == TokOp.Semi:
|
||
break
|
||
e: rawptr[AST] = _parse_expr(ps)
|
||
if e is None: break
|
||
elts.append(e)
|
||
if not _accept_op(ps, TokOp.Comma):
|
||
break
|
||
node: rawptr[AST] = Tuple(ps.pool, elts, ASTCtx.Load)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
|
||
|
||
def _parse_testlist(ps: rawptr[Parser]) -> rawptr[AST]:
|
||
"""Parse a comma-separated list of expressions, if there’s more than one, form a tuple."""
|
||
first: rawptr[AST] = _parse_expr(ps)
|
||
return _maybe_tuple(ps, first)
|
||
|
||
|
||
def _parse_expr_stmt(ps: rawptr[Parser], first: rawptr[AST]) -> rawptr[AST]:
|
||
"""Expression statement, handle assignment/augassign/annassign"""
|
||
if first is None:
|
||
first = _parse_testlist(ps)
|
||
else:
|
||
first = _maybe_tuple(ps, first)
|
||
if first is None:
|
||
return None
|
||
lineno: int = first.lineno
|
||
col: int = first.col_offset
|
||
# Assignment
|
||
if _match_op(ps, TokOp.Equal):
|
||
targets: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
|
||
targets.append(first)
|
||
_fix_store_ctx(first)
|
||
value: rawptr[AST] = None
|
||
accept_result: int = _accept_op(ps, TokOp.Equal)
|
||
while accept_result:
|
||
v: rawptr[AST] = _parse_testlist(ps)
|
||
if v is None: break
|
||
if value is not None:
|
||
targets.append(value)
|
||
_fix_store_ctx(value)
|
||
value = v
|
||
accept_result = _accept_op(ps, TokOp.Equal)
|
||
node: rawptr[AST] = Assign(ps.pool, targets, value)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
# augassign
|
||
if _cur_type(ps) == TokenType.Op:
|
||
op_id: int = _cur_op(ps)
|
||
aug_op: int = _augop_from_op(op_id)
|
||
if aug_op != OpKind.NoneOp:
|
||
_advance(ps)
|
||
value2: rawptr[AST] = _parse_testlist(ps)
|
||
_fix_store_ctx(first)
|
||
node2: rawptr[AST] = AugAssign(ps.pool, first, aug_op, value2)
|
||
_set_pos(node2, lineno, col, 0, 0)
|
||
return node2
|
||
# annassign: target ':' annotation [= value]
|
||
if _match_op(ps, TokOp.Colon):
|
||
_advance(ps)
|
||
annotation: rawptr[AST] = _parse_type_ann(ps)
|
||
value3: rawptr[AST] = None
|
||
if _accept_op(ps, TokOp.Equal):
|
||
value3 = _parse_testlist(ps)
|
||
if ps.pool is None:
|
||
return None
|
||
_fix_store_ctx(first)
|
||
if ps.pool is None:
|
||
return None
|
||
node3: rawptr[AST] = AnnAssign(ps.pool, first, annotation, value3, 1)
|
||
_set_pos(node3, lineno, col, 0, 0)
|
||
return node3
|
||
# Expression statement
|
||
node4: rawptr[AST] = Expr(ps.pool, first)
|
||
_set_pos(node4, lineno, col, 0, 0)
|
||
return node4
|
||
|
||
|
||
def _parse_return(ps: rawptr[Parser]) -> rawptr[AST]:
|
||
lineno: int = _cur_lineno(ps)
|
||
col: int = _cur_col(ps)
|
||
_advance(ps)
|
||
val: rawptr[AST] = None
|
||
if _cur_type(ps) != TokenType.NewLine and _cur_type(ps) != TokenType.EndMarker and _cur_type(ps) != TokenType.Op:
|
||
val = _parse_testlist(ps)
|
||
elif _cur_type(ps) == TokenType.Op and _cur_op(ps) != TokOp.Semi:
|
||
val = _parse_testlist(ps)
|
||
node: rawptr[AST] = Return(ps.pool, val)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
|
||
|
||
def _parse_raise(ps: rawptr[Parser]) -> rawptr[AST]:
|
||
lineno: int = _cur_lineno(ps)
|
||
col: int = _cur_col(ps)
|
||
_advance(ps)
|
||
exc: rawptr[AST] = None
|
||
cause: rawptr[AST] = None
|
||
if _cur_type(ps) != TokenType.NewLine and _cur_type(ps) != TokenType.EndMarker:
|
||
exc = _parse_test(ps)
|
||
if _cur_kw(ps) == Keyword.From:
|
||
_advance(ps)
|
||
cause = _parse_test(ps)
|
||
node: rawptr[AST] = Raise(ps.pool, exc, cause)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
|
||
|
||
def _parse_global(ps: rawptr[Parser]) -> rawptr[AST]:
|
||
lineno: int = _cur_lineno(ps)
|
||
col: int = _cur_col(ps)
|
||
_advance(ps)
|
||
names: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
|
||
if _cur_type(ps) == TokenType.Name:
|
||
n: rawptr[AST] = Name(ps.pool, _cur_str(ps), ASTCtx.Store)
|
||
_set_pos(n, _cur_lineno(ps), _cur_col(ps), 0, 0)
|
||
names.append(n)
|
||
_advance(ps)
|
||
while _accept_op(ps, TokOp.Comma):
|
||
if _cur_type(ps) != TokenType.Name: break
|
||
n = Name(ps.pool, _cur_str(ps), ASTCtx.Store)
|
||
_set_pos(n, _cur_lineno(ps), _cur_col(ps), 0, 0)
|
||
names.append(n)
|
||
_advance(ps)
|
||
node: rawptr[AST] = Global(ps.pool, names)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
|
||
|
||
def _parse_nonlocal(ps: rawptr[Parser]) -> rawptr[AST]:
|
||
lineno: int = _cur_lineno(ps)
|
||
col: int = _cur_col(ps)
|
||
_advance(ps)
|
||
names: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
|
||
if _cur_type(ps) == TokenType.Name:
|
||
n: rawptr[AST] = Name(ps.pool, _cur_str(ps), ASTCtx.Store)
|
||
_set_pos(n, _cur_lineno(ps), _cur_col(ps), 0, 0)
|
||
names.append(n)
|
||
_advance(ps)
|
||
while _accept_op(ps, TokOp.Comma):
|
||
if _cur_type(ps) != TokenType.Name: break
|
||
n = Name(ps.pool, _cur_str(ps), ASTCtx.Store)
|
||
_set_pos(n, _cur_lineno(ps), _cur_col(ps), 0, 0)
|
||
names.append(n)
|
||
_advance(ps)
|
||
node: rawptr[AST] = Nonlocal(ps.pool, names)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
|
||
|
||
def _parse_import(ps: rawptr[Parser]) -> rawptr[AST]:
|
||
lineno: int = _cur_lineno(ps)
|
||
col: int = _cur_col(ps)
|
||
_advance(ps)
|
||
names: rawptr[list[rawptr[AST]]] = _parse_alias_list(ps)
|
||
node: rawptr[AST] = Import(ps.pool, names)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
|
||
|
||
def _parse_from_import(ps: rawptr[Parser]) -> rawptr[AST]:
|
||
lineno: int = _cur_lineno(ps)
|
||
col: int = _cur_col(ps)
|
||
_advance(ps)
|
||
# Parse module (possibly with dots)
|
||
level: int = 0
|
||
while _match_op(ps, TokOp.Dot):
|
||
level += 1
|
||
_advance(ps)
|
||
while _match_op(ps, TokOp.Ellipsis):
|
||
level += 3
|
||
_advance(ps)
|
||
module_name: str = None
|
||
# The token type for keywords (like import) is also Name, so they must be excluded using kw_subtype,
|
||
# otherwise the `import` in `from . import` would be mistakenly treated as a module_name.
|
||
# Note: only `import` itself should be excluded. Module names can be keywords (like in `from .match import ...`,
|
||
# where match is a Viper match statement keyword), requiring NotKeyword would cause
|
||
# `from .match import` to fail parsing and the dependency scan would miss that module.
|
||
if _cur_type(ps) == TokenType.Name and _cur_kw(ps) != Keyword.Import:
|
||
module_name = _cur_str(ps)
|
||
_advance(ps)
|
||
while _match_op(ps, TokOp.Dot):
|
||
_advance(ps)
|
||
if _cur_type(ps) == TokenType.Name and _cur_kw(ps) == Keyword.NotKeyword:
|
||
next_name: str = _cur_str(ps)
|
||
mlen: int = string.strlen(module_name)
|
||
nlen: int = string.strlen(next_name)
|
||
new_mod: str = ps.pool.alloc(mlen + 1 + nlen + 1)
|
||
if new_mod is not None:
|
||
string.strcpy(new_mod, module_name)
|
||
new_mod[mlen] = '.'
|
||
string.strcpy(new_mod + mlen + 1, next_name)
|
||
module_name = new_mod
|
||
_advance(ps)
|
||
if _cur_kw(ps) != Keyword.Import:
|
||
ps.error_count += 1
|
||
return None
|
||
_advance(ps)
|
||
_skip_nl(ps)
|
||
# import *
|
||
if _match_op(ps, TokOp.Star):
|
||
_advance(ps)
|
||
star_alias: rawptr[AST] = Alias(ps.pool, "*", None)
|
||
_set_pos(star_alias, lineno, col, 0, 0)
|
||
star_names: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
|
||
star_names.append(star_alias)
|
||
node: rawptr[AST] = ImportFrom(ps.pool, module_name, star_names, level)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
# ( names ) or names
|
||
if _match_op(ps, TokOp.LPar):
|
||
_advance(ps)
|
||
_skip_nl(ps)
|
||
names: rawptr[list[rawptr[AST]]] = _parse_alias_list(ps)
|
||
_skip_nl(ps)
|
||
_expect_op(ps, TokOp.RPar)
|
||
else:
|
||
names = _parse_alias_list(ps)
|
||
node: rawptr[AST] = ImportFrom(ps.pool, module_name, names, level)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
|
||
|
||
def _parse_alias_list(ps: rawptr[Parser]) -> rawptr[list[rawptr[AST]]]:
|
||
"""Parse alias list."""
|
||
names: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
|
||
while True:
|
||
if _cur_type(ps) != TokenType.Name:
|
||
break
|
||
name_str: str = _cur_str(ps)
|
||
lineno: int = _cur_lineno(ps)
|
||
col: int = _cur_col(ps)
|
||
_advance(ps)
|
||
# dotted name
|
||
while _match_op(ps, TokOp.Dot):
|
||
_advance(ps)
|
||
if _cur_type(ps) == TokenType.Name:
|
||
next_n: str = _cur_str(ps)
|
||
nmlen: int = string.strlen(name_str)
|
||
nxtlen: int = string.strlen(next_n)
|
||
new_name: str = ps.pool.alloc(nmlen + 1 + nxtlen + 1)
|
||
if new_name is not None:
|
||
string.strcpy(new_name, name_str)
|
||
new_name[nmlen] = '.'
|
||
string.strcpy(new_name + nmlen + 1, next_n)
|
||
name_str = new_name
|
||
_advance(ps)
|
||
asname: str = None
|
||
if _cur_kw(ps) == Keyword.As:
|
||
_advance(ps)
|
||
if _cur_type(ps) == TokenType.Name:
|
||
asname = _cur_str(ps)
|
||
_advance(ps)
|
||
a: rawptr[AST] = Alias(ps.pool, name_str, asname)
|
||
_set_pos(a, lineno, col, 0, 0)
|
||
names.append(a)
|
||
if not _accept_op(ps, TokOp.Comma):
|
||
break
|
||
_skip_nl(ps)
|
||
return names
|
||
|
||
|
||
def _parse_del(ps: rawptr[Parser]) -> rawptr[AST]:
|
||
lineno: int = _cur_lineno(ps)
|
||
col: int = _cur_col(ps)
|
||
_advance(ps)
|
||
targets: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
|
||
while True:
|
||
t_: rawptr[AST] = _parse_atom_expr(ps)
|
||
if t_ is None: break
|
||
_set_ctx(t_, ASTCtx.Del)
|
||
targets.append(t_)
|
||
if not _accept_op(ps, TokOp.Comma):
|
||
break
|
||
node: rawptr[AST] = Delete(ps.pool, targets)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
|
||
|
||
def _parse_assert(ps: rawptr[Parser]) -> rawptr[AST]:
|
||
lineno: int = _cur_lineno(ps)
|
||
col: int = _cur_col(ps)
|
||
_advance(ps)
|
||
test: rawptr[AST] = _parse_test(ps)
|
||
msg: rawptr[AST] = None
|
||
if _accept_op(ps, TokOp.Comma):
|
||
msg = _parse_test(ps)
|
||
node: rawptr[AST] = Assert(ps.pool, test, msg)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
|
||
|
||
# ============================================================
|
||
# Compound statements
|
||
# ============================================================
|
||
|
||
def _parse_compound(ps: rawptr[Parser]) -> rawptr[AST]:
|
||
kw: int = _cur_kw(ps)
|
||
if kw == Keyword.If:
|
||
return _parse_if(ps)
|
||
if kw == Keyword.While:
|
||
return _parse_while(ps)
|
||
if kw == Keyword.For:
|
||
return _parse_for(ps, 0)
|
||
if kw == Keyword.Try:
|
||
return _parse_try(ps)
|
||
if kw == Keyword.With:
|
||
return _parse_with(ps, 0)
|
||
if _is_funcdef_kw(ps, kw) != 0:
|
||
return _parse_funcdef(ps, 0)
|
||
if kw == Keyword.Class:
|
||
return _parse_classdef(ps)
|
||
if kw == Keyword.Async:
|
||
_advance(ps)
|
||
if _is_funcdef_kw(ps, _cur_kw(ps)) != 0:
|
||
return _parse_funcdef(ps, 1)
|
||
if _cur_kw(ps) == Keyword.For:
|
||
return _parse_for(ps, 1)
|
||
if _cur_kw(ps) == Keyword.With:
|
||
return _parse_with(ps, 1)
|
||
ps.error_count += 1
|
||
return None
|
||
if kw == Keyword.Match and _is_match_stmt(ps) != 0:
|
||
return _parse_match(ps)
|
||
ps.error_count += 1
|
||
return None
|
||
|
||
|
||
def _parse_if(ps: rawptr[Parser]) -> rawptr[AST]:
|
||
lineno: int = _cur_lineno(ps)
|
||
col: int = _cur_col(ps)
|
||
_advance(ps) # if
|
||
test: rawptr[AST] = _parse_test(ps)
|
||
_expect_op(ps, TokOp.Colon)
|
||
# First construct with empty orelse, then fill it in later
|
||
empty: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
|
||
node: rawptr[If] = If(ps.pool, test, empty)
|
||
_parse_suite_into_children(ps, node)
|
||
orelse: rawptr[list[rawptr[AST]]] = _parse_if_tail(ps)
|
||
node.orelse = orelse
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
|
||
|
||
def _parse_if_tail(ps: rawptr[Parser]) -> rawptr[list[rawptr[AST]]]:
|
||
"""Parse elif/else chain and return orelse list.
|
||
|
||
elif: orelse = list containing single If node (nested)
|
||
else: orelse = statement list
|
||
"""
|
||
if _cur_kw(ps) == Keyword.Elif:
|
||
lineno: int = _cur_lineno(ps)
|
||
col: int = _cur_col(ps)
|
||
_advance(ps)
|
||
elif_test: rawptr[AST] = _parse_test(ps)
|
||
_expect_op(ps, TokOp.Colon)
|
||
empty: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
|
||
elif_node: rawptr[If] = If(ps.pool, elif_test, empty)
|
||
_parse_suite_into_children(ps, elif_node)
|
||
elif_orelse: rawptr[list[rawptr[AST]]] = _parse_if_tail(ps)
|
||
elif_node.orelse = elif_orelse
|
||
_set_pos(elif_node, lineno, col, 0, 0)
|
||
orelse: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
|
||
orelse.append(elif_node)
|
||
return orelse
|
||
if _cur_kw(ps) == Keyword.Else:
|
||
_advance(ps)
|
||
_expect_op(ps, TokOp.Colon)
|
||
scratch: rawptr[Module] = Module(ps.pool)
|
||
_parse_suite_into_children(ps, scratch)
|
||
return scratch.children
|
||
return list[rawptr[AST]](ps.pool, 8) # 空 list
|
||
|
||
|
||
# ============================================================
|
||
# Parsing Compound Statements (Continued)
|
||
# ============================================================
|
||
|
||
def _parse_while(ps: rawptr[Parser]) -> rawptr[AST]:
|
||
lineno: int = _cur_lineno(ps)
|
||
col: int = _cur_col(ps)
|
||
_advance(ps) # while
|
||
test: rawptr[AST] = _parse_test(ps)
|
||
_expect_op(ps, TokOp.Colon)
|
||
empty: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
|
||
node: rawptr[While] = While(ps.pool, test, empty)
|
||
_parse_suite_into_children(ps, node)
|
||
if _cur_kw(ps) == Keyword.Else:
|
||
_advance(ps)
|
||
_expect_op(ps, TokOp.Colon)
|
||
scratch: rawptr[Module] = Module(ps.pool)
|
||
_parse_suite_into_children(ps, scratch)
|
||
node.orelse = scratch.children
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
|
||
|
||
def _parse_for(ps: rawptr[Parser], is_async: int) -> rawptr[AST]:
|
||
lineno: int = _cur_lineno(ps)
|
||
col: int = _cur_col(ps)
|
||
_advance(ps) # for
|
||
target: rawptr[AST] = _parse_target_list(ps)
|
||
if _cur_kw(ps) != Keyword.In:
|
||
ps.error_count += 1
|
||
return None
|
||
_advance(ps)
|
||
iter_: rawptr[AST] = _parse_or_test(ps)
|
||
_expect_op(ps, TokOp.Colon)
|
||
empty: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
|
||
node: rawptr[For] = For(ps.pool, target, iter_, empty, is_async)
|
||
_fix_store_ctx(target)
|
||
_parse_suite_into_children(ps, node)
|
||
if _cur_kw(ps) == Keyword.Else:
|
||
_advance(ps)
|
||
_expect_op(ps, TokOp.Colon)
|
||
scratch: rawptr[Module] = Module(ps.pool)
|
||
_parse_suite_into_children(ps, scratch)
|
||
node.orelse = scratch.children
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
|
||
|
||
def _parse_try(ps: rawptr[Parser]) -> rawptr[AST]:
|
||
lineno: int = _cur_lineno(ps)
|
||
col: int = _cur_col(ps)
|
||
_advance(ps) # try
|
||
_expect_op(ps, TokOp.Colon)
|
||
empty: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
|
||
node: rawptr[Try] = Try(ps.pool, empty, empty, empty)
|
||
_parse_suite_into_children(ps, node)
|
||
# handlers (except)
|
||
handlers: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
|
||
while _cur_kw(ps) == Keyword.Except:
|
||
h: rawptr[AST] = _parse_except_handler(ps)
|
||
if h is not None:
|
||
handlers.append(h)
|
||
node.handlers = handlers
|
||
# else
|
||
if _cur_kw(ps) == Keyword.Else:
|
||
_advance(ps)
|
||
_expect_op(ps, TokOp.Colon)
|
||
scratch: rawptr[Module] = Module(ps.pool)
|
||
_parse_suite_into_children(ps, scratch)
|
||
node.orelse = scratch.children
|
||
# finally
|
||
if _cur_kw(ps) == Keyword.Finally:
|
||
_advance(ps)
|
||
_expect_op(ps, TokOp.Colon)
|
||
scratch2: rawptr[Module] = Module(ps.pool)
|
||
_parse_suite_into_children(ps, scratch2)
|
||
node.finalbody = scratch2.children
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
|
||
|
||
def _parse_except_handler(ps: rawptr[Parser]) -> rawptr[AST]:
|
||
lineno: int = _cur_lineno(ps)
|
||
col: int = _cur_col(ps)
|
||
_advance(ps) # except
|
||
type_: rawptr[AST] = None
|
||
name: str = None
|
||
if _cur_op(ps) != TokOp.Colon:
|
||
type_ = _parse_test(ps)
|
||
if _cur_kw(ps) == Keyword.As:
|
||
_advance(ps)
|
||
if _cur_type(ps) == TokenType.Name:
|
||
name = _cur_str(ps)
|
||
_advance(ps)
|
||
_expect_op(ps, TokOp.Colon)
|
||
node: rawptr[AST] = ExceptHandler(ps.pool, type_, name)
|
||
_parse_suite_into_children(ps, node)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
|
||
|
||
def _parse_with(ps: rawptr[Parser], is_async: int) -> rawptr[AST]:
|
||
lineno: int = _cur_lineno(ps)
|
||
col: int = _cur_col(ps)
|
||
_advance(ps) # with
|
||
items: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
|
||
# items
|
||
while True:
|
||
item: rawptr[AST] = _parse_with_item(ps)
|
||
if item is None: break
|
||
items.append(item)
|
||
if not _accept_op(ps, TokOp.Comma):
|
||
break
|
||
# Check if we can continue with a newline (with (...): form)
|
||
if _cur_type(ps) == TokenType.NewLine or _cur_type(ps) == TokenType.Nl:
|
||
break
|
||
_expect_op(ps, TokOp.Colon)
|
||
node: rawptr[With] = With(ps.pool, items, is_async)
|
||
_parse_suite_into_children(ps, node)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
|
||
|
||
def _parse_with_item(ps: rawptr[Parser]) -> rawptr[AST]:
|
||
"""with_item: test ['as' expr]"""
|
||
lineno: int = _cur_lineno(ps)
|
||
col: int = _cur_col(ps)
|
||
context_expr: rawptr[AST] = _parse_or_test(ps)
|
||
if context_expr is None: return None
|
||
optional_vars: rawptr[AST] = None
|
||
if _cur_kw(ps) == Keyword.As:
|
||
_advance(ps)
|
||
optional_vars = _parse_target(ps)
|
||
_fix_store_ctx(optional_vars)
|
||
node: rawptr[AST] = WithItem(ps.pool, context_expr, optional_vars)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
|
||
|
||
def _parse_funcdef(ps: rawptr[Parser], is_async: int) -> rawptr[AST]:
|
||
lineno: int = _cur_lineno(ps)
|
||
col: int = _cur_col(ps)
|
||
is_extern_fd: int = 0
|
||
is_export_fd: int = 0
|
||
is_define_fd: int = 0
|
||
if _cur_kw(ps) == Keyword.Extern:
|
||
# extern 声明：extern [def] A(...) [-> T] [": pass" 可选]
|
||
is_extern_fd = 1
|
||
_advance(ps) # extern
|
||
if _cur_kw(ps) == Keyword.Def:
|
||
_advance(ps)
|
||
elif _cur_kw(ps) == Keyword.Export:
|
||
# export 声明：export [def] A(...) [-> T] [": pass" 可选]
|
||
is_export_fd = 1
|
||
_advance(ps) # export
|
||
if _cur_kw(ps) == Keyword.Def:
|
||
_advance(ps)
|
||
elif _cur_kw(ps) == Keyword.Define:
|
||
# define 宏函数：语法与 def 完全一致，返回注解自动追加 t.CDefine
|
||
# 标记为宏函数（StubGen 保留原样 → C #define / alwaysinline 内联）
|
||
is_define_fd = 1
|
||
_advance(ps) # define
|
||
else:
|
||
_advance(ps) # def / fn / ...
|
||
if _cur_type(ps) != TokenType.Name:
|
||
ps.error_count += 1
|
||
return None
|
||
name: str = _cur_str(ps)
|
||
_advance(ps)
|
||
# PEP 695 泛型函数类型参数: def add[T](a: T, b: T) — 解析并存入 type_params。
|
||
# （与 class 解析器一致；之前缺失此处理导致 _expect_op(Colon) 失败、函数体被
|
||
# 误提升到模块级——b = Box(42) 被当作模块级全局注册，函数内赋值误报
|
||
# "需用 global 声明"。翻译时 type_params 映射到 int 兜底。）
|
||
type_params_fd: rawptr[list[str]] = None
|
||
if _accept_op(ps, TokOp.Lsqb):
|
||
type_params_fd = list[str](ps.pool, 8)
|
||
while not _match_op(ps, TokOp.Rsqb):
|
||
if _cur_type(ps) == TokenType.EndMarker:
|
||
break
|
||
if _cur_type(ps) == TokenType.Name:
|
||
tp_name_fd: str = _cur_str(ps)
|
||
_advance(ps)
|
||
# 跳过可选 bound/约束: [T: bound] 或 [T: (A, B)]
|
||
if _match_op(ps, TokOp.Colon):
|
||
_advance(ps)
|
||
_parse_test(ps)
|
||
type_params_fd.append(tp_name_fd)
|
||
if not _accept_op(ps, TokOp.Comma):
|
||
break
|
||
_expect_op(ps, TokOp.Rsqb)
|
||
args: rawptr[AST] = None
|
||
if _accept_op(ps, TokOp.LPar):
|
||
args = _parse_varargslist(ps, 0)
|
||
_expect_op(ps, TokOp.RPar)
|
||
returns: rawptr[AST] = None
|
||
if _accept_op(ps, TokOp.RArrow):
|
||
returns = _parse_type_ann(ps)
|
||
if is_extern_fd:
|
||
# extern 声明：返回注解改写为 <原类型> | t.State，复用既有
|
||
# State/外部 C 函数识别路径（vpi 注解含 "State" → 定义端裸名 + 不生成函数体）
|
||
state_attr_fd: rawptr[AST] = Attribute(ps.pool,
|
||
Name(ps.pool, "t", ASTCtx.Load),
|
||
"State", ASTCtx.Load)
|
||
if returns != None:
|
||
returns = BinOp(ps.pool, returns, OpKind.BitOr, state_attr_fd)
|
||
else:
|
||
returns = state_attr_fd
|
||
elif is_export_fd:
|
||
# export 声明：返回注解改写为 <原类型> | t.CExport，复用既有
|
||
# CExport 识别路径（vpi 注解含 "CExport" → 定义端裸名 + 不加模块 SHA1 头）
|
||
export_attr_fd: rawptr[AST] = Attribute(ps.pool,
|
||
Name(ps.pool, "t", ASTCtx.Load),
|
||
"CExport", ASTCtx.Load)
|
||
if returns != None:
|
||
returns = BinOp(ps.pool, returns, OpKind.BitOr, export_attr_fd)
|
||
else:
|
||
returns = export_attr_fd
|
||
elif is_define_fd:
|
||
# define 宏函数：返回注解改写为 <原类型> | t.CDefine，复用既有
|
||
# CDefine 宏函数识别路径（StubGen 保留原样 → C #define / alwaysinline）
|
||
define_attr_fd: rawptr[AST] = Attribute(ps.pool,
|
||
Name(ps.pool, "t", ASTCtx.Load),
|
||
"CDefine", ASTCtx.Load)
|
||
if returns != None:
|
||
returns = BinOp(ps.pool, returns, OpKind.BitOr, define_attr_fd)
|
||
else:
|
||
returns = define_attr_fd
|
||
dec_list: rawptr[list[rawptr[AST]]] = ps.pending_decorators
|
||
ps.pending_decorators = None
|
||
node: rawptr[FunctionDef] = FunctionDef(ps.pool, name, args, dec_list, returns, is_async, type_params_fd)
|
||
if is_extern_fd:
|
||
node.flags = node.flags | FLAG_IS_EXTERN
|
||
if is_export_fd:
|
||
node.flags = node.flags | FLAG_IS_EXPORT
|
||
if is_define_fd:
|
||
node.flags = node.flags | FLAG_IS_DEFINE
|
||
if (is_extern_fd == 0 and is_export_fd == 0 and is_define_fd == 0) or _match_op(ps, TokOp.Colon):
|
||
_expect_op(ps, TokOp.Colon)
|
||
_parse_suite_into_children(ps, node)
|
||
else:
|
||
# extern/export 无 ': pass' 形式（extern A(...) -> int）：body 为空，仅声明。
|
||
# 必须消费本行 NewLine：模块级/类体语句循环只 _skip_nl（Nl），
|
||
# 残留 NewLine 会被下一轮当表达式解析（_parse_atom → error_count++）
|
||
# → parse_tokens 返回 None（TPV 编译 extern 文件报 7 个扫描失败的根因）
|
||
if _cur_type(ps) == TokenType.NewLine:
|
||
_advance(ps)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
|
||
|
||
def _parse_classdef(ps: rawptr[Parser]) -> rawptr[AST]:
|
||
lineno: int = _cur_lineno(ps)
|
||
col: int = _cur_col(ps)
|
||
_advance(ps) # class
|
||
if _cur_type(ps) != TokenType.Name:
|
||
ps.error_count += 1
|
||
return None
|
||
name: str = _cur_str(ps)
|
||
_advance(ps)
|
||
# Parse type parameters [T] or [T, U] (PEP 695 generic class syntax)
|
||
type_params: rawptr[list[str]] = None
|
||
if _accept_op(ps, TokOp.Lsqb):
|
||
type_params = list[str](ps.pool, 8)
|
||
while not _match_op(ps, TokOp.Rsqb):
|
||
if _cur_type(ps) == TokenType.EndMarker: break
|
||
if _cur_type(ps) == TokenType.Name:
|
||
tp_name: str = _cur_str(ps)
|
||
_advance(ps)
|
||
# Skip optional bound/constraint: [T: bound] or [T: (A, B)]
|
||
if _match_op(ps, TokOp.Colon):
|
||
_advance(ps)
|
||
_parse_test(ps)
|
||
type_params.append(tp_name)
|
||
if not _accept_op(ps, TokOp.Comma):
|
||
break
|
||
_expect_op(ps, TokOp.Rsqb)
|
||
bases: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
|
||
keywords: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
|
||
# bases / keywords
|
||
if _accept_op(ps, TokOp.LPar):
|
||
while not _match_op(ps, TokOp.RPar):
|
||
# 基类括号内跨行（class X(A,\n B):）：逗号后换行在此跳过，
|
||
# 否则 cur=NL 非 Name/Comma → break → _expect_op(RPar) 报
|
||
# "意外的 token"（Generator.vp L34 多继承跨行根因）
|
||
_skip_nl(ps)
|
||
if _cur_type(ps) == TokenType.EndMarker: break
|
||
# base 或 keyword=value
|
||
if _cur_type(ps) == TokenType.Name:
|
||
# Parse as expression, if followed by '=', then as keyword=value
|
||
e: rawptr[AST] = _parse_or_test(ps)
|
||
if _match_op(ps, TokOp.Equal):
|
||
_advance(ps)
|
||
v: rawptr[AST] = _parse_test(ps)
|
||
kw_node: rawptr[AST] = KwNode(ps.pool, e, v)
|
||
keywords.append(kw_node)
|
||
else:
|
||
bases.append(e)
|
||
if not _accept_op(ps, TokOp.Comma):
|
||
break
|
||
_skip_nl(ps) # 末基类后换行收尾（class X(A,\n B):）
|
||
_expect_op(ps, TokOp.RPar)
|
||
_expect_op(ps, TokOp.Colon)
|
||
dec_list: rawptr[list[rawptr[AST]]] = ps.pending_decorators
|
||
ps.pending_decorators = None
|
||
node: rawptr[ClassDef] = ClassDef(ps.pool, name, bases, keywords, dec_list, type_params)
|
||
_parse_suite_into_children(ps, node)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
|
||
|
||
def _parse_match(ps: rawptr[Parser]) -> rawptr[AST]:
|
||
"""match statement: match subject ':' NewLine Indent case+ Dedent"""
|
||
lineno: int = _cur_lineno(ps)
|
||
col: int = _cur_col(ps)
|
||
_advance(ps) # match
|
||
subject: rawptr[AST] = _parse_test(ps)
|
||
_expect_op(ps, TokOp.Colon)
|
||
cases: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
|
||
node: rawptr[AST] = Match(ps.pool, subject, cases)
|
||
# Parsing body: Same NewLine/Indent/Dedent pattern as _parse_suite_into_children,
|
||
# but parses case clauses into a cases list instead of normal statements into children
|
||
_skip_nl(ps)
|
||
if _cur_type(ps) == TokenType.NewLine:
|
||
_advance(ps)
|
||
_skip_nl(ps)
|
||
if _cur_type(ps) == TokenType.Indent:
|
||
_advance(ps)
|
||
while _cur_type(ps) != TokenType.Dedent and _cur_type(ps) != TokenType.EndMarker:
|
||
if _cur_kw(ps) == Keyword.Case:
|
||
case_node: rawptr[AST] = _parse_match_case(ps)
|
||
if case_node is not None:
|
||
cases.append(case_node)
|
||
else:
|
||
_advance(ps) # Skip non-case token to avoid infinite loop
|
||
_expect_type(ps, TokenType.Dedent)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
|
||
|
||
# ============================================================
|
||
# Match pattern parsing
|
||
# ============================================================
|
||
|
||
def _make_name_ast(pool: rawptr[memhub.MemManager], node: rawptr[Name]) -> rawptr[AST]:
|
||
# Analysis: "Convert Name* to AST* (Solving the TransPyC Type Inference Problem)"
|
||
return node
|
||
|
||
|
||
def _parse_pattern(ps: rawptr[Parser]) -> rawptr[AST]:
|
||
"""pattern: or_pattern ['as' NAME]
|
||
or_pattern: closed_pattern ('|' closed_pattern)*"""
|
||
pat: rawptr[AST] = _parse_closed_pattern(ps)
|
||
if pat is None: return None
|
||
# OR pattern
|
||
if _match_op(ps, TokOp.VBar):
|
||
patterns: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
|
||
patterns.append(pat)
|
||
while _accept_op(ps, TokOp.VBar):
|
||
p: rawptr[AST] = _parse_closed_pattern(ps)
|
||
if p is not None:
|
||
patterns.append(p)
|
||
or_node: rawptr[AST] = MatchOr(ps.pool, patterns)
|
||
_set_pos(or_node, pat.lineno, pat.col_offset, 0, 0)
|
||
pat = or_node
|
||
# AS pattern
|
||
if _cur_kw(ps) == Keyword.As:
|
||
_advance(ps)
|
||
if _cur_type(ps) != TokenType.Name:
|
||
ps.error_count += 1
|
||
return pat
|
||
name_str: str = _cur_str(ps)
|
||
_advance(ps)
|
||
as_node: rawptr[AST] = MatchAs(ps.pool, pat, name_str)
|
||
_set_pos(as_node, pat.lineno, pat.col_offset, 0, 0)
|
||
return as_node
|
||
return pat
|
||
|
||
|
||
def _parse_closed_pattern(ps: rawptr[Parser]) -> rawptr[AST]:
|
||
"""closed_pattern: literal | capture | wildcard | value | group | sequence | mapping | class"""
|
||
lineno: int = _cur_lineno(ps)
|
||
col: int = _cur_col(ps)
|
||
ttype: int = _cur_type(ps)
|
||
val: rawptr[AST]
|
||
node: rawptr[AST]
|
||
tok: rawptr[Token]
|
||
sval: str
|
||
|
||
# Number literal pattern
|
||
if ttype == TokenType.Number:
|
||
tok = ps.cur
|
||
_advance(ps)
|
||
if tok.is_float:
|
||
val = _new_const_float(ps.pool, tok.float_val, lineno, col)
|
||
else:
|
||
val = _new_const_int(ps.pool, tok.int_val, lineno, col)
|
||
node = MatchValue(ps.pool, val)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
|
||
# String literal pattern
|
||
if ttype == TokenType.String:
|
||
sval = _cur_str(ps)
|
||
is_dquote_m: int = ps.cur.kw_subtype & 16
|
||
_advance(ps)
|
||
val = _new_const_str(ps.pool, sval, lineno, col, is_dquote_m)
|
||
node = MatchValue(ps.pool, val)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
|
||
# Negative number literal pattern: -1, -1.5
|
||
if _match_op(ps, TokOp.Minus):
|
||
_advance(ps)
|
||
if _cur_type(ps) == TokenType.Number:
|
||
tok = ps.cur
|
||
_advance(ps)
|
||
if tok.is_float:
|
||
val = _new_const_float(ps.pool, -tok.float_val, lineno, col)
|
||
else:
|
||
val = _new_const_int(ps.pool, -tok.int_val, lineno, col)
|
||
node = MatchValue(ps.pool, val)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
ps.error_count += 1
|
||
return None
|
||
|
||
# Name: None/True/False/_ /capture/value/class
|
||
if ttype == TokenType.Name:
|
||
kw: int = _cur_kw(ps)
|
||
# None/True/False → MatchSingleton
|
||
if kw == Keyword.None_ or kw == Keyword.True_ or kw == Keyword.False_:
|
||
_advance(ps)
|
||
if kw == Keyword.None_:
|
||
val = _new_const_none(ps.pool, lineno, col)
|
||
elif kw == Keyword.True_:
|
||
val = _new_const_bool(ps.pool, 1, lineno, col)
|
||
else:
|
||
val = _new_const_bool(ps.pool, 0, lineno, col)
|
||
node = MatchSingleton(ps.pool, val)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
name_str: str = _cur_str(ps)
|
||
# Wildcard pattern: _
|
||
if string.strcmp(name_str, "_") == 0:
|
||
_advance(ps)
|
||
node = MatchAs(ps.pool, None, None)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
_advance(ps)
|
||
# Check dotted name（value pattern 或 class pattern）
|
||
if _match_op(ps, TokOp.Dot):
|
||
name_node: rawptr[Name] = Name(ps.pool, name_str, ASTCtx.Load)
|
||
_set_pos(name_node, lineno, col, 0, 0)
|
||
base: rawptr[AST] = _make_name_ast(ps.pool, name_node)
|
||
while _match_op(ps, TokOp.Dot):
|
||
_advance(ps)
|
||
if _cur_type(ps) != TokenType.Name:
|
||
ps.error_count += 1
|
||
break
|
||
attr_name: str = _cur_str(ps)
|
||
attr_lineno: int = _cur_lineno(ps)
|
||
attr_col: int = _cur_col(ps)
|
||
_advance(ps)
|
||
base = Attribute(ps.pool, base, attr_name, ASTCtx.Load)
|
||
_set_pos(base, attr_lineno, attr_col, 0, 0)
|
||
# class pattern: dotted.name(...)
|
||
if _match_op(ps, TokOp.LPar):
|
||
return _parse_class_pattern(ps, base, lineno, col)
|
||
# value pattern: dotted.name
|
||
node = MatchValue(ps.pool, base)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
# class pattern: Name(...)
|
||
if _match_op(ps, TokOp.LPar):
|
||
cls_node: rawptr[Name] = Name(ps.pool, name_str, ASTCtx.Load)
|
||
_set_pos(cls_node, lineno, col, 0, 0)
|
||
cls: rawptr[AST] = _make_name_ast(ps.pool, cls_node)
|
||
return _parse_class_pattern(ps, cls, lineno, col)
|
||
# capture pattern: NAME
|
||
node = MatchAs(ps.pool, None, name_str)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
|
||
# group pattern / sequence pattern (tuple): (...)
|
||
if _match_op(ps, TokOp.LPar):
|
||
_advance(ps)
|
||
inner: rawptr[AST] = _parse_pattern(ps)
|
||
if _accept_op(ps, TokOp.Comma):
|
||
# sequence pattern (tuple)
|
||
patterns: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
|
||
if inner is not None:
|
||
patterns.append(inner)
|
||
while not _match_op(ps, TokOp.RPar) and _cur_type(ps) != TokenType.EndMarker:
|
||
p: rawptr[AST] = _parse_pattern(ps)
|
||
if p is not None:
|
||
patterns.append(p)
|
||
if not _accept_op(ps, TokOp.Comma):
|
||
break
|
||
_expect_op(ps, TokOp.RPar)
|
||
node = MatchSequence(ps.pool, patterns)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
_expect_op(ps, TokOp.RPar)
|
||
return inner # group pattern
|
||
|
||
# sequence pattern (list): [...]
|
||
if _match_op(ps, TokOp.Lsqb):
|
||
return _parse_sequence_pattern(ps, TokOp.Rsqb, lineno, col)
|
||
|
||
# mapping pattern: {...}
|
||
if _match_op(ps, TokOp.LBrace):
|
||
return _parse_mapping_pattern(ps, lineno, col)
|
||
|
||
ps.error_count += 1
|
||
return None
|
||
|
||
|
||
def _parse_sequence_pattern(ps: rawptr[Parser], close_op: int,
|
||
lineno: int, col: int) -> rawptr[AST]:
|
||
"""sequence_pattern: '[' [pattern_list] ']'"""
|
||
patterns: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
|
||
_advance(ps) # [
|
||
while not _match_op(ps, close_op) and _cur_type(ps) != TokenType.EndMarker:
|
||
# *name (star pattern)
|
||
if _match_op(ps, TokOp.Star):
|
||
_advance(ps)
|
||
if _cur_type(ps) == TokenType.Name:
|
||
star_name: str = _cur_str(ps)
|
||
_advance(ps)
|
||
star_node: rawptr[AST]
|
||
if string.strcmp(star_name, "_") == 0:
|
||
star_node = MatchStar(ps.pool, None)
|
||
else:
|
||
star_node = MatchStar(ps.pool, star_name)
|
||
_set_pos(star_node, lineno, col, 0, 0)
|
||
patterns.append(star_node)
|
||
else:
|
||
p: rawptr[AST] = _parse_pattern(ps)
|
||
if p is not None:
|
||
patterns.append(p)
|
||
if not _accept_op(ps, TokOp.Comma):
|
||
break
|
||
_expect_op(ps, close_op)
|
||
node: rawptr[AST] = MatchSequence(ps.pool, patterns)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
|
||
|
||
def _parse_mapping_pattern(ps: rawptr[Parser],
|
||
lineno: int, col: int) -> rawptr[AST]:
|
||
"""mapping_pattern: '{' [items] '}'
|
||
items: key ':' pattern (',' key ':' pattern)* [',' '**' NAME] | '**' NAME"""
|
||
keys: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
|
||
patterns: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
|
||
rest: str = None
|
||
node: rawptr[AST]
|
||
key: rawptr[AST]
|
||
val_pat: rawptr[AST]
|
||
_advance(ps) # {
|
||
if _match_op(ps, TokOp.RBrace):
|
||
_advance(ps)
|
||
node = MatchMapping(ps.pool, keys, patterns, rest)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
while _cur_type(ps) != TokenType.EndMarker:
|
||
if _match_op(ps, TokOp.StarStar):
|
||
_advance(ps)
|
||
if _cur_type(ps) == TokenType.Name:
|
||
rest = _cur_str(ps)
|
||
_advance(ps)
|
||
break
|
||
key = _parse_expr(ps)
|
||
_expect_op(ps, TokOp.Colon)
|
||
val_pat = _parse_pattern(ps)
|
||
if key is not None:
|
||
keys.append(key)
|
||
if val_pat is not None:
|
||
patterns.append(val_pat)
|
||
if not _accept_op(ps, TokOp.Comma):
|
||
break
|
||
_expect_op(ps, TokOp.RBrace)
|
||
node = MatchMapping(ps.pool, keys, patterns, rest)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
|
||
|
||
def _parse_class_pattern(ps: rawptr[Parser], cls: rawptr[AST],
|
||
lineno: int, col: int) -> rawptr[AST]:
|
||
"""class_pattern: cls '(' [pattern_list] ')'
|
||
pattern_list: positional_pattern (',' positional_pattern)*
|
||
[',' keyword_pattern]* | keyword_pattern (',' keyword_pattern)*
|
||
keyword_pattern: NAME '=' pattern"""
|
||
patterns: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
|
||
kwd_attrs: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
|
||
kwd_patterns: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
|
||
p: rawptr[AST]
|
||
_advance(ps) # (
|
||
if not _match_op(ps, TokOp.RPar):
|
||
while _cur_type(ps) != TokenType.EndMarker:
|
||
# keyword pattern: NAME '=' pattern
|
||
if _cur_type(ps) == TokenType.Name and ps.cur is not None:
|
||
nxt: rawptr[Token] = ps.cur.next
|
||
if nxt is not None and nxt.type == TokenType.Op and nxt.op_subtype == TokOp.Equal:
|
||
attr_name: str = _cur_str(ps)
|
||
_advance(ps) # NAME
|
||
_advance(ps) # =
|
||
kwd_pat: rawptr[AST] = _parse_pattern(ps)
|
||
attr_n: rawptr[Name] = Name(ps.pool, attr_name, ASTCtx.Load)
|
||
_set_pos(attr_n, lineno, col, 0, 0)
|
||
attr_node: rawptr[AST] = _make_name_ast(ps.pool, attr_n)
|
||
kwd_attrs.append(attr_node)
|
||
if kwd_pat is not None:
|
||
kwd_patterns.append(kwd_pat)
|
||
else:
|
||
p = _parse_pattern(ps)
|
||
if p is not None:
|
||
patterns.append(p)
|
||
else:
|
||
p = _parse_pattern(ps)
|
||
if p is not None:
|
||
patterns.append(p)
|
||
if not _accept_op(ps, TokOp.Comma):
|
||
break
|
||
_expect_op(ps, TokOp.RPar)
|
||
node: rawptr[AST] = MatchClass(ps.pool, cls, patterns, kwd_attrs, kwd_patterns)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
|
||
|
||
def _parse_match_case(ps: rawptr[Parser]) -> rawptr[AST]:
|
||
"""case pattern [if guard] ':' suite"""
|
||
lineno: int = _cur_lineno(ps)
|
||
col: int = _cur_col(ps)
|
||
_advance(ps) # case
|
||
guard: rawptr[AST] = None
|
||
pattern: rawptr[AST] = _parse_pattern(ps)
|
||
if _cur_kw(ps) == Keyword.If:
|
||
_advance(ps)
|
||
guard = _parse_or_test(ps)
|
||
_expect_op(ps, TokOp.Colon)
|
||
node: rawptr[AST] = MatchCase(ps.pool, pattern, guard)
|
||
_parse_suite_into_children(ps, node)
|
||
_set_pos(node, lineno, col, 0, 0)
|
||
return node
|
||
|
||
|
||
def _parse_decorators(ps: rawptr[Parser]) -> rawptr[list[rawptr[AST]]]:
|
||
"""解析装饰器链，返回 list。"""
|
||
decs: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
|
||
while _match_op(ps, TokOp.At):
|
||
lineno: int = _cur_lineno(ps)
|
||
col: int = _cur_col(ps)
|
||
_advance(ps) # @
|
||
expr: rawptr[AST] = _parse_namedexpr(ps)
|
||
_skip_nl(ps)
|
||
if _cur_type(ps) == TokenType.NewLine:
|
||
_advance(ps)
|
||
decs.append(expr)
|
||
return decs
|
||
|
||
|
||
# ============================================================
|
||
# Module entry
|
||
# ============================================================
|
||
|
||
def _parse_module_body(ps: rawptr[Parser], module_node: rawptr[AST]):
|
||
"""解析模块体，append 到 module_node。"""
|
||
_skip_nl(ps)
|
||
stmt_idx: int = 0
|
||
while _cur_type(ps) != TokenType.EndMarker:
|
||
stmt_idx += 1
|
||
# Decorator
|
||
if _match_op(ps, TokOp.At):
|
||
ps.pending_decorators = _parse_decorators(ps)
|
||
# Record current token pointer to check for infinite loop
|
||
before_cur: rawptr[Token] = ps.cur
|
||
_parse_statement_into(ps, module_node)
|
||
# Defense: If parsing doesn't advance the token (like when _parse_atom encounters an unrecognized token and returns None),
|
||
# force it to move forward to avoid an infinite loop in the main loop; accumulate error_count for the caller to diagnose
|
||
if ps.cur is before_cur:
|
||
ps.error_count += 1
|
||
_advance(ps)
|
||
_skip_nl(ps)
|
||
|
||
|
||
def parse_tokens(pool: rawptr[memhub.MemManager],
|
||
tokens: rawptr[Token]) -> rawptr[AST]:
|
||
"""Parse an AST Module from a token list."""
|
||
if pool is None or tokens is None:
|
||
return None
|
||
global _g_parse_err_line, _g_parse_err_col, _g_parse_err_msg
|
||
_g_parse_err_line = 0
|
||
_g_parse_err_col = 0
|
||
_g_parse_err_msg = None
|
||
ps: rawptr[Parser] = new_parser(pool)
|
||
if ps is None: return None
|
||
_parser_init(ps, tokens, pool)
|
||
module_node: rawptr[Module] = Module(pool)
|
||
_parse_module_body(ps, module_node)
|
||
# fail-fast: 解析出错（error_count > 0）时返回 None，由调用方（TPV 等）报错终止，
|
||
# 避免错误代码静默编译（如 `func12344444 avg3(...)` 缺 def 关键字的函数定义残留）。
|
||
if ps.error_count > 0:
|
||
return None
|
||
return module_node
|