GVSDS/Viper_VRTE
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
e71daeea0e6499c66f890b6de950d0c70f386793
Viper_VRTE/ast/parser.vp
T
TermiNexus e71daeea0e docs(includes): R11 注释英文化——75 个文件的中文注释改写为英文短注释 
按规范「注释只讲解作用，不掺杂历史原因，尽量英文」：把 includes 下 75 个 .vp 的中文 # 注释与三引号 docstring 改写为英文，每个注释块压到 2 行以内。

保留要点：作用、约束、危险警告（不得/必须/注意）、ABI 与类型约定（如 BYTEPTR | t.CPtr = i8** 的跨模块坍缩坑）、位宽/长度约定、fail-fast 要求、内存与句柄所有权。

未动：文件头注释（规范豁免，仍是中文）、运行时中文字符串常量（printf / fatal_error / MlirFail 等用户可见消息，属代码字面量）。

验证：独立脚本比对 HEAD 与工作区的代码骨架（剥离 # 注释与三引号 docstring 后的非空行）——75 个文件逐字节一致；行尾 CRLF/LF 未被改动；全文剩余中文仅 547 行头注释、0 条行尾注释。bootstrap 三重重编 SUCCESS；全量 77/77 顺序回归通过。
2026-10-05 16:18:44 +08:00

4620 lines
192 KiB
Plaintext
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import t, c
from stdint import *
import memhub
import string
import stdio
import stdlib
from .tokens import (Token, TokenType, Keyword, TokOp)
from .base import (
AST, ASTCtx, ASTKind, OpKind, ASTFlag,
_set_pos, _binop_from_op, _augop_from_op, _cmpop_from_op, _copy_str, _inherit_pos,
_init_ast,
CONST_INT, CONST_FLOAT, CONST_STR, CONST_BOOL, CONST_NONE, CONST_CHAR,
FLAG_IS_ASYNC, FLAG_IS_EXTERN, FLAG_IS_EXPORT, FLAG_IS_DEFINE,
)
from .exprs import (
BoolOp, BinOp, UnaryOp, Lambda, IfExp, Dict, Set,
ListComp, SetComp, DictComp, GeneratorExp,
Await, Yield, YieldFrom, FormattedValue, JoinedStr,
Constant, NamedExpr, Attribute, Subscript, Starred, Name,
List, Tuple, Slice, Call, Compare, OpNode,
)
from .stmts import (
Module, Expression, Interactive, FunctionType,
FunctionDef, ClassDef, Return, Delete, Assign, AugAssign, AnnAssign,
For, While, If, With, Raise, Try, Assert, Global, Nonlocal,
Pass, Break, Continue, FallThrough, Expr, Import, ImportFrom, Match,
)
from .astaux import (
ExceptHandler, Arguments, Arg, Keyword as KwNode, Alias, WithItem, Comprehension,
)
from .match import (
MatchCase, MatchValue, MatchSingleton, MatchSequence,
MatchMapping, MatchClass, MatchStar, MatchAs, MatchOr,
)
from .lexer import (
Lexer, _lexer_init, tokenize, new_lexer,
get_lex_err_line, get_lex_err_col, get_lex_err_msg,
_lex_int_neg_exceeds_i64,
)
# ============================================================
# Parser State
# ============================================================
# Hard cap on the token linked-list length (cycle detection): normal sources have far fewer
# tokens, so reaching this count without the tail means the list is cyclic (see parse_tokens).
define PARSER_MAX_TOKENS = 8000000
class Parser:
tokens: rawptr[Token] # token linked list head
cur: rawptr[Token] # current token
prev: rawptr[Token] # previous consumed token (for the `T*(x)` adjacency test)
pool: rawptr[memhub.MemManager] # memory pool
error_count: int # error count
pending_decorators: rawptr[list[rawptr[AST]]] # pending decorators list
# Pending parsed left operand for a leading `*` in argument position (see _parse_call / _parse_power).
starred_seed: rawptr[AST]
# `static` modifier (soft keyword) pending flag: set after consuming `static` before a `Name :`
# declaration; the next _parse_type_ann prepends t.CStatic and clears it (see _ParseStaticModified).
pending_static: int
# Whether the suite currently being parsed is directly owned by a class (1 = class body, 0 = other),
# maintained by _parse_suite_into_children for the virtual/novirtual class-body check.
in_class_body: int
# Step budget (_advance count / parse_tokens budget): the statement loop fails fast once exceeded,
# never looping forever (see the cycle detection in parse_tokens).
step_count: sizet
step_limit: sizet
# Watchdog: counter and convergence flag for "query current token" (see _tick). When tick_count
# exceeds step_limit, aborted=1, entries return converged values and every loop ends in bounded rounds.
tick_count: sizet
aborted: int
def __new__(self, pool: rawptr[memhub.MemManager]):
ptr: rawptr[Parser] = pool.alloc(Parser.__sizeof__())
if ptr: string.memset(ptr, 0, Parser.__sizeof__())
return ptr
def new_parser(pool: rawptr[memhub.MemManager]) -> rawptr[Parser]:
"""Allocate from the pool and return the Parser (cross-module safe wrapper)"""
return Parser(pool)
def _parser_init(ps: rawptr[Parser], tokens: rawptr[Token],
pool: rawptr[memhub.MemManager]):
if ps is None: return
ps.tokens = tokens
ps.cur = tokens
ps.prev = None
ps.pool = pool
ps.error_count = 0
ps.pending_decorators = None
ps.starred_seed = None
ps.pending_static = 0
ps.in_class_body = 0
ps.step_count = 0
# Default budget is the cap (parse_tokens later tightens it to the real token count) so entries that
# bypass parse_tokens (parse_expression etc.) do not misreport at 0, while a hard bound still prevents loops.
ps.step_limit = PARSER_MAX_TOKENS
ps.tick_count = 0
ps.aborted = 0
# ============================================================
# Parser watchdog (static convergence: no silent hang)
#
# Any loop that spins without consuming a token must repeatedly query the current token, so every query
# entry (_cur_type / _cur_op / _cur_kw / _match_op / _accept_op / _advance) ticks the watchdog; once the
# budget is exceeded it sets aborted, records a parse error and returns a converged value:
# _cur_type -> EndMarker, _cur_op / _cur_kw -> 0, _match_op / _accept_op -> 1 (jump to end),
# _advance -> jump to end. parse_tokens then returns None with error_count > 0 and the caller reports
# file:line:col SyntaxError; there is no silent-hang path left.
# ============================================================
def _tick(ps: rawptr[Parser]):
"""Tick the watchdog; over budget sets aborted and records the first parse error (with position)"""
if ps is None: return
if ps.aborted != 0: return
ps.tick_count += 1
if ps.tick_count > ps.step_limit:
ps.aborted = 1
_record_parse_err(ps)
ps.error_count += 1
# ============================================================
# Token Navigation Helpers
# ============================================================
def _advance(ps: rawptr[Parser]):
"""Advance to the next token"""
_tick(ps)
if ps.aborted != 0:
ps.cur = None
return
if ps.cur is None: return
if ps.cur.next is not None:
ps.prev = ps.cur
ps.cur = ps.cur.next
ps.step_count += 1
def _cur_type(ps: rawptr[Parser]) -> int:
_tick(ps)
if ps.aborted != 0: return TokenType.EndMarker
if ps.cur is None: return TokenType.EndMarker
return ps.cur.type
def _cur_op(ps: rawptr[Parser]) -> int:
_tick(ps)
if ps.aborted != 0: return 0
if ps.cur is None: return 0
return ps.cur.op_subtype
def _cur_kw(ps: rawptr[Parser]) -> int:
_tick(ps)
if ps.aborted != 0: return 0
if ps.cur is None: return 0
return ps.cur.kw_subtype
def _is_funcdef_kw(ps: rawptr[Parser], kw: int) -> int:
"""Whether the current token is a function-definition keyword (def and its aliases
fn/fun/func/function, plus extern/export).
def is a hard keyword. fn/fun/func/function/extern/export are soft: they count only when the next
token is a Name followed by '(' or a generic '['; extern/export also accept `<kw> def A(...)`.
Otherwise they are ordinary identifiers (e.g. the AST `func` field or `func: rawptr[AST]`).
"""
if kw == Keyword.Def:
return 1
if kw == Keyword.Extern or kw == Keyword.Export:
# extern/export def A(...) or extern/export A(...)
# (after extern/export comes def or Name + '(' / '[')
nxt: rawptr[Token] = ps.cur.next
if nxt is None: return 0
if nxt.kw_subtype == Keyword.Def: return 1
if nxt.type != TokenType.Name: return 0
nxt2: rawptr[Token] = nxt.next
if nxt2 is None: return 0
if nxt2.type == TokenType.Op and (nxt2.op_subtype == TokOp.LPar or nxt2.op_subtype == TokOp.Lsqb):
return 1
return 0
if kw == Keyword.Fn or kw == Keyword.Fun \
or kw == Keyword.Func or kw == Keyword.Function \
or kw == Keyword.Define:
# Soft-keyword lookahead: next is a Name, the one after is '(' or a generic '[T...]' (PEP 695)
# (the define macro function matches def, with t.CDefine appended to the return annotation)
nxt: rawptr[Token] = ps.cur.next
if nxt is None:
return 0
if nxt.type != TokenType.Name:
return 0
nxt2: rawptr[Token] = nxt.next
if nxt2 is None:
return 0
if nxt2.type == TokenType.Op \
and (nxt2.op_subtype == TokOp.LPar or nxt2.op_subtype == TokOp.Lsqb):
return 1
return 0
return 0
def _is_define_const(ps: rawptr[Parser]) -> int:
"""Soft-keyword lookahead for a define constant declaration (1 = yes, 0 = no).
`define NAME[: T] = <literal>`: next is a Name, then '=' (no annotation) or ':' (annotated).
`define NAME(...)` and `define` used as an identifier do not match. `define NAME: T` (missing '=')
does match and then fails loudly in _parse_define_const's _expect_op(Equal).
"""
if _cur_kw(ps) != Keyword.Define: return 0
nxt: rawptr[Token] = ps.cur.next
if nxt is None or nxt.type != TokenType.Name: return 0
nxt2: rawptr[Token] = nxt.next
if nxt2 is None or nxt2.type != TokenType.Op: return 0
if nxt2.op_subtype == TokOp.Equal or nxt2.op_subtype == TokOp.Colon: return 1
return 0
def _is_typedef(ps: rawptr[Parser]) -> int:
"""Soft-keyword test for a typedef alias declaration (1 = yes, 0 = no).
`typedef T1 = T2` is fully equivalent to `T1: t.CTypedef = T2`. `typedef` is a statement-level
reserved word (never an identifier in the repo); invalid forms fail loudly in _parse_typedef.
"""
if _cur_kw(ps) != Keyword.Typedef:
return 0
return 1
def _is_export_extern_var(ps: rawptr[Parser]) -> int:
"""Soft-keyword test for `export NAME :` / `extern NAME :` variable declarations (1 = yes, 0 = no).
export/extern also serve function declarations (see _is_funcdef_kw); the third token distinguishes
them: '(' or '[' is a function, ':' is a variable. Other forms do not match; invalid forms fail loudly.
"""
kw: int = _cur_kw(ps)
if kw != Keyword.Export and kw != Keyword.Extern:
return 0
nxt: rawptr[Token] = ps.cur.next
if nxt is None:
return 0
if nxt.type != TokenType.Name:
return 0
# The name must be a non-keyword identifier (excludes forms like `export def f()`)
if nxt.kw_subtype != Keyword.NotKeyword:
return 0
nxt2: rawptr[Token] = nxt.next
if nxt2 is None:
return 0
if nxt2.type != TokenType.Op:
return 0
if nxt2.op_subtype != TokOp.Colon:
return 0
return 1
# Parse error position and message (fail-fast, Python SyntaxError style, first error only)
_g_parse_err_line: int = 0
_g_parse_err_col: int = 0
_g_parse_err_msg: str = None
def _record_parse_err(ps: rawptr[Parser]):
"""Record the position and message of the first parse error (later errors do not overwrite it)"""
global _g_parse_err_line, _g_parse_err_col, _g_parse_err_msg
if _g_parse_err_line != 0:
return
_g_parse_err_line = _cur_lineno(ps)
_g_parse_err_col = _cur_col(ps)
err_s: str = _cur_str(ps)
if err_s is None:
err_s = "(none)"
# Copy the message text via stdlib (token.str_val lives in the pool; stored independently to be safe)
buf_m: str = stdlib.malloc(128)
if buf_m is not None:
stdio.snprintf(buf_m, 128, "意外的 token '%s'", err_s)
_g_parse_err_msg = buf_m
else:
_g_parse_err_msg = err_s
def _record_parse_err_ctx(ps: rawptr[Parser], tok_type: int,
pv_str: str, pv_line: int, pv_col: int,
nx_str: str, nx_line: int, nx_col: int):
"""Record the first parse error with token type and prev/next tokens appended to the message.
Same source as _record_parse_err (first only); the message must stay a single line since logs
inline it. file:line:col and the caret are filled later by the caller's dump_parse_traceback.
"""
global _g_parse_err_line, _g_parse_err_col, _g_parse_err_msg
if _g_parse_err_line != 0:
return
_g_parse_err_line = _cur_lineno(ps)
_g_parse_err_col = _cur_col(ps)
err_s: str = _cur_str(ps)
if err_s is None:
err_s = "(none)"
buf_m: str = stdlib.malloc(512)
if buf_m is None:
_g_parse_err_msg = err_s
return
stdio.snprintf(buf_m, 512,
"意外的 token '%s'（type=%d） | 附注: prev='%s' (line %d col %d) | next='%s' (line %d col %d)",
err_s, tok_type, pv_str, pv_line, pv_col, nx_str, nx_line, nx_col);
_g_parse_err_msg = buf_m
def _record_neg_literal_overflow(lit_text: str, lineno: int, col: int):
"""Record a combined "unary minus + integer literal" overflow (fail-fast; first only).
`-` is a separate unary operator, so a magnitude-valid literal (<= 2^64-1) may still be < -2^63
after negation. Message carries the literal text + allowed range [-2^63, 2^64-1] + fix hint.
"""
global _g_parse_err_line, _g_parse_err_col, _g_parse_err_msg
if _g_parse_err_line != 0:
return
_g_parse_err_line = lineno
_g_parse_err_col = col
buf: str = stdlib.malloc(512)
if buf is not None:
stdio.snprintf(buf, 512,
"[TPV] 负数整数字面量越界: 一元减作用于整数字面量后的结果超出 64 位"
"有符号可表示范围（允许范围 -9223372036854775808..18446744073709551615"
" = [-2^63, 2^64-1]；负数结果必须 >= -2^63）。字面量原文: '-%s'。"
"禁止静默回绕/截断，已终止编译。"
"修法: 把该负数改到 -9223372036854775808..-1 的范围内（本语言不存在"
"更宽的整数类型，如需更大数值请拆分为多个 64 位字表达）。", lit_text)
_g_parse_err_msg = buf
else:
_g_parse_err_msg = lit_text
# ============================================================
# Type descriptors const / volatile (soft keywords, annotation position only)
# ============================================================
def _IsDescriptorKeyword(ps: rawptr[Parser]) -> int:
"""Whether the current token is a type-descriptor keyword (const / volatile), 1 = yes / 0 = no.
Soft keywords consumed by _parse_type_ann only in annotation position; `t.attr.const()` as an
identifier is unaffected since attribute/call resolution looks at TokenType.Name, not kw_subtype.
"""
if _cur_type(ps) != TokenType.Name:
return 0
if _cur_kw(ps) == Keyword.Const or _cur_kw(ps) == Keyword.Volatile:
return 1
return 0
def _IsMagicAttributeCall(ps: rawptr[Parser]) -> int:
"""Test for an annotation-position `__attribute__(...)` prefix, 1 = yes / 0 = no.
`__attribute__` is a soft keyword (not in the keyword table): consumed here only in annotation
position immediately before '('. Same family as const/volatile: order-free and mixable.
"""
if _cur_type(ps) != TokenType.Name:
return 0
s: str = _cur_str(ps)
if s is None or string.strcmp(s, "__attribute__") != 0:
return 0
nxt: rawptr[Token] = ps.cur.next
if nxt is None or nxt.type != TokenType.Op or nxt.op_subtype != TokOp.LPar:
return 0
return 1
def _MakeDescriptorMarker(ps: rawptr[Parser]) -> rawptr[AST]:
"""Build the marker node Attribute(Name('t'), 'CConst'/'CVolatile') from the current descriptor keyword.
Field-identical to TPC's _ExpandTypeDescriptors text expansion (t.CConst / t.CVolatile) so both
sides share one recognition path; no second representation is introduced.
"""
attr_name: str = "CConst"
if _cur_kw(ps) == Keyword.Volatile:
attr_name = "CVolatile"
marker: rawptr[AST] = Attribute(ps.pool, Name(ps.pool, "t", ASTCtx.Load),
attr_name, ASTCtx.Load)
_set_pos(marker, _cur_lineno(ps), _cur_col(ps), 0, 0)
return marker
def _PrependDescriptorMarkers(ps: rawptr[Parser],
base: rawptr[AST],
mods: rawptr[list[rawptr[AST]]],
mod_count: int) -> rawptr[AST]:
"""Prepend descriptor markers to the leftmost operand of the type union (same shape as TPC text).
TPC expands `const volatile int | t.CPtr` to the left-associative `(((t.CConst | t.CVolatile) | int) | t.CPtr)`;
this function produces the identical tree rather than a differently-shaped one.
"""
slot_parent: rawptr[BinOp] = None # leftmost BitOr node: its left is the leftmost operand
slot_node: rawptr[AST] = base
cur: rawptr[AST] = base
while cur is not None and cur.kind() == ASTKind.BinOp:
b: rawptr[BinOp] = (rawptr[BinOp])(cur)
if b.op != OpKind.BitOr or b.left is None:
break
nxt: rawptr[AST] = b.left
if nxt.kind() == ASTKind.BinOp:
nb: rawptr[BinOp] = (rawptr[BinOp])(nxt)
if nb.op == OpKind.BitOr and nb.left is not None:
cur = nxt
continue
slot_parent = b
slot_node = nxt
break
# Descriptor chain left-associates in source order: `const volatile` -> (t.CConst | t.CVolatile)
chain: rawptr[AST] = mods.get(0)
i: int = 1
while i < mod_count:
chain = BinOp(ps.pool, chain, OpKind.BitOr, mods.get(i))
i += 1
newnode: rawptr[AST] = BinOp(ps.pool, chain, OpKind.BitOr, slot_node)
if slot_parent is None:
return newnode
slot_parent.left = newnode
return base
def _RecordDescriptorErr(detail: str, lineno: int, col: int):
"""Record an invalid type-descriptor form (fail-fast; first only).
Message carries the expected form + a correct example; file:line:col is filled later. Descriptors
are never silently ignored: missing type / non-type follower / bare use all abort compilation.
"""
global _g_parse_err_line, _g_parse_err_col, _g_parse_err_msg
if _g_parse_err_line != 0:
return
_g_parse_err_line = lineno
_g_parse_err_col = col
buf: str = stdlib.malloc(1024)
if buf is not None:
stdio.snprintf(buf, 1024,
"类型描述符非法（%s）。const/volatile 是只在类型位置生效的软关键字，"
"必须写成 `名字: const 类型` / `名字: volatile 类型`"
"（可组合 `名字: const volatile 类型`）。正确示例：a: const int / "
"a: volatile int / a: const volatile t.CInt32T / "
"def f(p: const rawptr[Foo]) -> const int。"
"（描述符不会被静默忽略：缺类型 / 后随非类型 / 裸用一律终止编译）", detail)
_g_parse_err_msg = buf
else:
_g_parse_err_msg = detail
# ============================================================
# virtual / novirtual modifier (soft keyword; statement start before class/def only), same
# representation as @t.CVTable / @t.NoVTable. Invalid forms fail loudly; `x = d.virtual` unaffected.
def _IsVirtualModifier(ps: rawptr[Parser]) -> int:
"""Whether the current token is a virtual / novirtual modifier (1 = yes / 0 = no).
Called only at statement start; identifier positions never reach this test, so `x = d.virtual`,
parameter names, for targets and dict keys are unaffected.
"""
if _cur_type(ps) != TokenType.Name:
return 0
if _cur_kw(ps) == Keyword.Virtual or _cur_kw(ps) == Keyword.Novirtual:
return 1
return 0
def _MakeVirtualMarker(ps: rawptr[Parser]) -> rawptr[AST]:
"""Build the marker node Attribute(Name('t'), 'CVTable'|'NoVTable') from the current virtual/novirtual token.
Field-identical to hand-written @t.CVTable / @t.NoVTable, so ClassGenBase._has_decorator needs no new test.
"""
attr_name: str = "CVTable"
if _cur_kw(ps) == Keyword.Novirtual:
attr_name = "NoVTable"
marker: rawptr[AST] = Attribute(ps.pool, Name(ps.pool, "t", ASTCtx.Load),
attr_name, ASTCtx.Load)
_set_pos(marker, _cur_lineno(ps), _cur_col(ps), 0, 0)
return marker
def _RecordVirtualErr(detail: str, lineno: int, col: int, extra: str):
"""Record an invalid virtual/novirtual form (fail-fast; first only); message carries allowed
positions + a correct example. Modifiers are never silently ignored.
Callers build detail/extra with f-string interpolation: this language's str is i8*, so `str + str`
degrades to raw pointer add and illegal IR; never concatenate strings with `+` in this file.
"""
global _g_parse_err_line, _g_parse_err_col, _g_parse_err_msg
if _g_parse_err_line != 0:
return
_g_parse_err_line = lineno
_g_parse_err_col = col
buf: str = stdlib.malloc(1536)
if buf is not None:
stdio.snprintf(buf, 1536,
"[TPV] virtual/novirtual 修饰符非法（%s）。可写位置：类级 "
"`virtual class C:` / `novirtual class C:`；方法级 "
"`virtual def m(self): ...` / `novirtual def m(self): ...`"
"（只能直接写在类体内：virtual class / class / novirtual class 的类体）。"
"正确示例：virtual class Shape: / novirtual class Node: / "
"class A: virtual def run(self) -> int: ... / "
"novirtual class B: novirtual def helper(self) -> int: ...。%s"
"（修饰符不会被静默忽略：非法形态一律终止编译）", detail, extra)
_g_parse_err_msg = buf
else:
_g_parse_err_msg = detail
def _ParseVirtualModified(ps: rawptr[Parser], parent: rawptr[AST]):
"""`virtual class` / `novirtual class` / `virtual def` / `novirtual def`.
Synthesizes a @t.CVTable / @t.NoVTable decorator (merged with existing decorators, appended last;
position-independent since _has_decorator matches by name) and hands off to _parse_classdef / _parse_funcdef.
"""
mod_lineno: int = _cur_lineno(ps)
mod_col: int = _cur_col(ps)
mod_name: str = "virtual"
if _cur_kw(ps) == Keyword.Novirtual:
mod_name = "novirtual"
marker: rawptr[AST] = _MakeVirtualMarker(ps)
_advance(ps) # consume the modifier
# The modifier must be followed by class / def (the def aliases fn/fun/func/function are accepted too,
# matching _is_funcdef_kw and TPC, whose fn->def expansion only hits a leading fn).
kw2: int = _cur_kw(ps)
is_class: int = 0
is_def: int = 0
if kw2 == Keyword.Class:
is_class = 1
elif kw2 == Keyword.Def or kw2 == Keyword.Fn or kw2 == Keyword.Fun \
or kw2 == Keyword.Func or kw2 == Keyword.Function:
is_def = 1
if is_class == 0 and is_def == 0:
nxt_s: str = _cur_str(ps)
if nxt_s is None:
nxt_s = "(none)"
if kw2 == Keyword.Struct:
# struct is a pure data type: no virtual / novirtual modifier allowed. A dedicated message
# (not the generic "follower not class/def") points at the fix: struct already implies @t.NoVTable.
_RecordStructErr(f"`{mod_name} struct` 非法：struct 禁止 virtual/novirtual 修饰",
mod_lineno, mod_col,
"struct 天然无虚表（解析期已合成 @t.NoVTable），"
"请直接写 `struct X:`；需要虚表语义请改用 class。")
else:
_RecordVirtualErr(f"`{mod_name}` 后随 '{nxt_s}'（非 class/def）",
mod_lineno, mod_col,
f"`{mod_name}` 只在 class/def 前生效；"
f"若此处本意是普通标识符名，请改名"
f"（该词在语句起始会被判为修饰符）。")
ps.error_count += 1
# Dispatch the remaining tokens to the generic statement path (no infinite loop, no error overwrite).
_parse_simple_stmt_into(ps, parent)
return
if is_def != 0 and ps.in_class_body == 0:
_RecordVirtualErr(f"{mod_name} def 出现在类体之外", mod_lineno, mod_col,
f"{mod_name} def 描述的是「类体内方法的虚/非虚」，"
f"模块级 / 函数体 / 分支体内没有可归属的类。")
ps.error_count += 1
# Merge existing decorators (@deco + virtual class/def) then hand off to the existing class/def parser
merged: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
if merged is None:
return
if ps.pending_decorators is not None:
dn: sizet = len(ps.pending_decorators)
for di in range(dn):
merged.append(ps.pending_decorators.get(di))
merged.append(marker)
ps.pending_decorators = merged
stmt: rawptr[AST] = None
if is_class != 0:
stmt = _parse_classdef(ps)
else:
stmt = _parse_funcdef(ps, 0)
if stmt is not None:
parent.append(stmt)
# ============================================================
# inline modifier (soft keyword): `inline def f(...)` -> synthesizes @t.attr.always_inline, same shape as
# virtual/novirtual (statement start before a function definition only); kept out of the keyword table.
# ============================================================
def _IsInlineModifier(ps: rawptr[Parser]) -> int:
"""Whether the current token is `inline` (soft keyword followed by a function definition), 1 = yes / 0 = no"""
if _cur_type(ps) != TokenType.Name:
return 0
s: str = _cur_str(ps)
if s is None or string.strcmp(s, "inline") != 0:
return 0
nxt: rawptr[Token] = ps.cur.next
if nxt is None or nxt.type != TokenType.Name:
return 0
nxt2: rawptr[Token] = nxt.next
if nxt2 is None:
return 0
# Followed by def (hard keyword): `inline def f(...)`
if nxt.kw_subtype == Keyword.Def:
return 1
# Followed by a function-definition soft-keyword alias (fn/fun/func/function): same rule as
# _is_funcdef_kw -- must be Name + '(' or '[T]', so `inline func` is not misread.
if nxt.kw_subtype == Keyword.Fn or nxt.kw_subtype == Keyword.Fun \
or nxt.kw_subtype == Keyword.Func or nxt.kw_subtype == Keyword.Function:
if nxt2.type == TokenType.Op \
and (nxt2.op_subtype == TokOp.LPar or nxt2.op_subtype == TokOp.Lsqb):
return 1
return 0
def _MakeInlineMarker(ps: rawptr[Parser]) -> rawptr[AST]:
"""Build the marker Attribute(Attribute(Name('t'),'attr'), 'always_inline') from the current inline token.
Field-identical to a hand-written @t.attr.always_inline. Deliberately not @c.Attribute(t.attr.x),
whose extractor requires `import c` and would silently skip inlining in files lacking it.
"""
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
t_name: rawptr[AST] = Name(ps.pool, "t", ASTCtx.Load)
attr_at: rawptr[AST] = Attribute(ps.pool, t_name, "attr", ASTCtx.Load)
marker: rawptr[AST] = Attribute(ps.pool, attr_at, "always_inline", ASTCtx.Load)
_set_pos(marker, lineno, col, 0, 0)
return marker
def _ParseInlineModified(ps: rawptr[Parser], parent: rawptr[AST]):
"""`inline def f(...)`: consume inline, synthesize @t.attr.always_inline and hand off to _parse_funcdef"""
marker: rawptr[AST] = _MakeInlineMarker(ps)
if marker is None:
return
_advance(ps) # consume the inline modifier
# Merge existing decorators (@deco + inline def), appending the marker last; position-independent
# (matched by name downstream). Same pending_decorators channel as _ParseVirtualModified.
merged: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
if merged is None:
return
if ps.pending_decorators is not None:
dn: sizet = len(ps.pending_decorators)
for di in range(dn):
merged.append(ps.pending_decorators.get(di))
merged.append(marker)
ps.pending_decorators = merged
stmt: rawptr[AST] = _parse_funcdef(ps, 0)
if stmt is not None:
parent.append(stmt)
def _IsStaticModifier(ps: rawptr[Parser]) -> int:
"""Whether the current token is a `static` modifier (soft keyword), 1 = yes / 0 = no.
Called only at statement start; matches when followed by (1) a function definition (`def` or the
fn/fun/func/function alias + '(' / '[') or (2) a variable declaration (`Name :`, i.e. AnnAssign).
Identifier positions never reach this test, so `x = d.static` / `f(static=1)` are unaffected.
"""
if _cur_type(ps) != TokenType.Name:
return 0
s: str = _cur_str(ps)
if s is None or string.strcmp(s, "static") != 0:
return 0
nxt: rawptr[Token] = ps.cur.next
if nxt is None or nxt.type != TokenType.Name:
return 0
# (1) followed by def (hard keyword): `static def f(...)`
if nxt.kw_subtype == Keyword.Def:
return 1
nxt2: rawptr[Token] = nxt.next
# (1') followed by a function-definition alias: must be Name + '(' or '[T]', same rule as
# _IsInlineModifier / _is_funcdef_kw, so `static func` is not misread.
if nxt.kw_subtype == Keyword.Fn or nxt.kw_subtype == Keyword.Fun \
or nxt.kw_subtype == Keyword.Func or nxt.kw_subtype == Keyword.Function:
if nxt2 is not None and nxt2.type == TokenType.Op \
and (nxt2.op_subtype == TokOp.LPar or nxt2.op_subtype == TokOp.Lsqb):
return 1
# (2) followed by a variable declaration `Name :` (AnnAssign)
if nxt2 is not None and nxt2.type == TokenType.Op and nxt2.op_subtype == TokOp.Colon:
return 1
return 0
def _IsStaticDescriptorKeyword(ps: rawptr[Parser]) -> int:
"""Whether the current token is an annotation-position `static` (soft keyword), 1 = yes / 0 = no.
This position (`name: static T`) is no longer supported: _parse_type_ann only detects it as misuse
and fails loudly, pointing at the prefix form `static name: T`; it is no longer desugared to t.CStatic.
Valid forms are statement-start `static name: T` and the `@t.CStatic` decorator. Reporting (not ignoring)
is mandatory: unconsumed, `static` degrades into an opaque "class" named static and errors far from the cause.
"""
if _cur_type(ps) != TokenType.Name:
return 0
s: str = _cur_str(ps)
if s is None or string.strcmp(s, "static") != 0:
return 0
return 1
def _MakeStaticMarker(ps: rawptr[Parser]) -> rawptr[AST]:
"""Build the marker node Attribute(Name('t'), 'CStatic').
Field-identical to the hand-written @t.CStatic decorator / `x: t.CStatic | T` marker, so downstream
(FuncGenDef storage-class detection, TypeGenResolve qualifier resolution) needs no new test.
"""
marker: rawptr[AST] = Attribute(ps.pool, Name(ps.pool, "t", ASTCtx.Load),
"CStatic", ASTCtx.Load)
_set_pos(marker, _cur_lineno(ps), _cur_col(ps), 0, 0)
return marker
def _ParseStaticModified(ps: rawptr[Parser], parent: rawptr[AST]) -> int:
"""Dispatch the `static` modifier: 1 = fully handled (function definition), 0 = only flagged
(variable declaration, caller continues with the generic statement dispatch).
Function: consume static and synthesize an @t.CStatic decorator, handed to _parse_funcdef.
Variable: set pending_static and consume static; the next _parse_type_ann prepends t.CStatic.
"""
nxt: rawptr[Token] = ps.cur.next
is_func: int = 0
if nxt is not None and nxt.type == TokenType.Name:
if nxt.kw_subtype == Keyword.Def:
is_func = 1
elif nxt.kw_subtype == Keyword.Fn or nxt.kw_subtype == Keyword.Fun \
or nxt.kw_subtype == Keyword.Func or nxt.kw_subtype == Keyword.Function:
nxt2: rawptr[Token] = nxt.next
if nxt2 is not None and nxt2.type == TokenType.Op \
and (nxt2.op_subtype == TokOp.LPar or nxt2.op_subtype == TokOp.Lsqb):
is_func = 1
if is_func == 0:
# Variable branch: only flag + consume static, then let the caller continue generic dispatch
# (AnnAssign parsing reads pending_static in _parse_type_ann).
ps.pending_static = 1
_advance(ps)
return 0
marker: rawptr[AST] = _MakeStaticMarker(ps)
if marker is None:
return 1
_advance(ps) # consume the static modifier
# Merge existing decorators (@deco + static def), appending the marker last; position-independent
# (matched by name downstream). Same pending_decorators channel as _ParseInlineModified.
merged: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
if merged is None:
return 1
if ps.pending_decorators is not None:
dn: sizet = len(ps.pending_decorators)
for di in range(dn):
merged.append(ps.pending_decorators.get(di))
merged.append(marker)
ps.pending_decorators = merged
stmt: rawptr[AST] = _parse_funcdef(ps, 0)
if stmt is not None:
parent.append(stmt)
return 1
def _IsBareDescriptor(ps: rawptr[Parser]) -> int:
"""Bare-descriptor test: the whole statement is a single descriptor run (const[: volatile]), 1 = yes / 0 = no.
Matches only when the descriptor run is followed directly by newline / EOF / semicolon; identifiers
(`const = 5` / `const(x)` / `t.attr.const()`) do not match (soft-keyword boundary).
"""
if _IsDescriptorKeyword(ps) == 0:
return 0
nxt: rawptr[Token] = ps.cur.next
while nxt is not None and nxt.type == TokenType.Name \
and (nxt.kw_subtype == Keyword.Const or nxt.kw_subtype == Keyword.Volatile):
nxt = nxt.next
if nxt is None:
return 1
if nxt.type == TokenType.NewLine or nxt.type == TokenType.EndMarker:
return 1
if nxt.type == TokenType.Op and nxt.op_subtype == TokOp.Semi:
return 1
return 0
def get_parse_err_line() -> int:
"""Line number of the latest parse error (0 = none)"""
return _g_parse_err_line
def get_parse_err_col() -> int:
"""Column of the latest parse error (0 = none)"""
return _g_parse_err_col
def get_parse_err_msg() -> str:
"""Message of the latest parse error (Python SyntaxError style)"""
return _g_parse_err_msg
def _cur_str(ps: rawptr[Parser]) -> str:
if ps.cur is None: return None
return ps.cur.str_val
def _cur_lineno(ps: rawptr[Parser]) -> int:
if ps.cur is None: return 0
return ps.cur.lineno
def _cur_col(ps: rawptr[Parser]) -> int:
if ps.cur is None: return 0
return ps.cur.col_offset
def _skip_nl(ps: rawptr[Parser]):
"""Skip TokenType.Nl token"""
while ps.cur is not None and ps.cur.type == TokenType.Nl:
_advance(ps)
def _skip_nls_and_newlines(ps: rawptr[Parser]):
"""Skip TokenType.Nl and TokenType.NewLine (used for line continuation)"""
while ps.cur is not None:
if ps.cur.type == TokenType.Nl or ps.cur.type == TokenType.NewLine:
_advance(ps)
else:
break
def _expect_type(ps: rawptr[Parser], ttype: int) -> int:
"""Expect the current token to be of the specified type, advance and return 1, otherwise return 0"""
if _cur_type(ps) == ttype:
_advance(ps)
return 1
_record_parse_err(ps)
ps.error_count += 1
return 0
def _expect_op(ps: rawptr[Parser], op_id: int) -> int:
"""Expecting the current one to be some operator"""
if _cur_type(ps) == TokenType.Op and _cur_op(ps) == op_id:
_advance(ps)
return 1
_record_parse_err(ps)
ps.error_count += 1
return 0
def _match_op(ps: rawptr[Parser], op_id: int) -> int:
"""Check but not consume the current token"""
_tick(ps)
if ps.aborted != 0:
# Convergence: if tokens remain, "match" and jump to the end so both
# `while not _match_op(...)` and `while _match_op(...)` end within <= 2 rounds
if ps.cur is not None:
ps.cur = None
return 1
return 0
if _cur_type(ps) == TokenType.Op and _cur_op(ps) == op_id:
return 1
return 0
def _accept_op(ps: rawptr[Parser], op_id: int) -> int:
"""If match, consume and return 1"""
if _match_op(ps, op_id):
_advance(ps)
return 1
return 0
# ============================================================
# Constant Construction Helper (New API: Constant directly accepts lineno/col)
# ============================================================
def _new_const_int(pool: rawptr[memhub.MemManager], val: int64,
lineno: int, col: int) -> rawptr[AST]:
return Constant(pool, CONST_INT, val, 0.0, None, lineno, col)
def _new_const_float(pool: rawptr[memhub.MemManager], val: double,
lineno: int, col: int) -> rawptr[AST]:
global _g_dbg_pts, _g_dbg_vals, _g_dbg_cnt
new_c: rawptr[Constant] = Constant(pool, CONST_FLOAT, 0, val, None, lineno, col)
# Store the numeric text in str_val (stdlib allocation, bypassing the pool overwrite of float_val);
# REG-SRC recovers the original value with string.atof(str_val).
lit_txt: str = stdlib.malloc(64)
if lit_txt is not None:
stdio.snprintf(lit_txt, 64, "%.17g", val)
new_c.str_val = lit_txt
# Record float node ptr->val into a stdlib table (value correct at construction; REG-SRC uses the
# table to bypass the pool overwrite). Reset when full: old entries belong to finished files.
if _g_dbg_pts is None:
_g_dbg_pts = stdlib.malloc(4096 * 8)
_g_dbg_vals = stdlib.malloc(4096 * 8)
_g_dbg_cnt = 0
elif _g_dbg_cnt >= 4096:
_g_dbg_cnt = 0
if _g_dbg_pts is not None and _g_dbg_vals is not None:
_g_dbg_pts[_g_dbg_cnt] = uint64(new_c)
_g_dbg_vals[_g_dbg_cnt] = val
_g_dbg_cnt += 1
return new_c
# Float-node construction-value table (stdlib allocated to avoid polluting the pool)
_g_dbg_pts: rawptr[uint64] = None
_g_dbg_vals: rawptr[double] = None
_g_dbg_cnt: int = 0
# Whether the latest _dbg_lookup_lit_val hit
_g_dbg_lit_found: int = 0
def _dbg_lookup_lit_val(ptr: uint64) -> double:
"""Look up the float-node construction-value table: return the original value (miss returns 0, found=0).
Used to bypass the pool allocation overwriting the AST float_val field (correct at construction).
"""
global _g_dbg_lit_found
_g_dbg_lit_found = 0
if _g_dbg_pts is None or _g_dbg_vals is None:
return 0.0
for i in range(_g_dbg_cnt):
if _g_dbg_pts[i] == ptr:
_g_dbg_lit_found = 1
return _g_dbg_vals[i]
return 0.0
def _dbg_lit_found() -> int:
"""Check whether the latest _dbg_lookup_lit_val hit"""
return _g_dbg_lit_found
def _new_const_str(pool: rawptr[memhub.MemManager], val: str,
lineno: int, col: int,
is_dquote: int = 1,
is_triple: int = 0,
is_char: int = 0) -> rawptr[AST]:
"""Create a string/character constant node
is_dquote: 0 = single quote, non-0 = double quote (default double quote)
is_triple: non-0 = triple-quoted string
is_char: non-0 = single-character candidate (lexer has confirmed that after escaping it's exactly 1 byte, including '0')
Semantic distinctions:
- 'x' (single quote, single character, not triple-quoted) → CONST_CHAR (i8 value)
- "x" (double quote, includes 0 terminator) → CONST_STR (i8* pointer, i8 x 2)
- 'ab' (single quotes, multiple characters) → CONST_STR (i8* pointer)
- '''x''' / \"\"\"x\"\"\" (triple quotes) → CONST_STR (i8* pointer)
- '' / "" (empty string) → CONST_STR (i8* pointer)
"""
# Single quotes, non-triple quotes, single-character candidates marked by lexer → CONST_CHAR
# Mark with is_char instead of strlen(val), because '0' becomes a NUL byte after escaping, strlen would return 0
if is_dquote == 0 and is_triple == 0 and is_char != 0:
# Single quote # Single character → CONST_CHAR, int_val stores the character's ASCII value
# val[0] is a legal value of 0 even if it's NUL
ch_val: int64 = val[0]
return Constant(pool, CONST_CHAR, ch_val, 0.0, val, lineno, col)
# Other situations → CONST_STR
return Constant(pool, CONST_STR, 0, 0.0, val, lineno, col)
def _new_const_bool(pool: rawptr[memhub.MemManager], val: int,
lineno: int, col: int) -> rawptr[AST]:
return Constant(pool, CONST_BOOL, val, 0.0, None, lineno, col)
def _new_const_none(pool: rawptr[memhub.MemManager],
lineno: int, col: int) -> rawptr[AST]:
return Constant(pool, CONST_NONE, 0, 0.0, None, lineno, col)
# ============================================================
# ctx Helper (Recursive Fixing Store Context)
# ============================================================
def _set_ctx(node: rawptr[AST], ctx: int):
"""Set ctx field of node (based on kind conversion)"""
if node is None: return
k: int = node.kind()
if k == ASTKind.Name:
nm: rawptr[Name] = (rawptr[Name])(node)
nm.ctx = ctx
elif k == ASTKind.Attribute:
at: rawptr[Attribute] = (rawptr[Attribute])(node)
at.ctx = ctx
elif k == ASTKind.Subscript:
sb: rawptr[Subscript] = (rawptr[Subscript])(node)
sb.ctx = ctx
elif k == ASTKind.Tuple:
tp: rawptr[Tuple] = (rawptr[Tuple])(node)
tp.ctx = ctx
elif k == ASTKind.List:
ls: rawptr[List] = (rawptr[List])(node)
ls.ctx = ctx
elif k == ASTKind.Starred:
st: rawptr[Starred] = (rawptr[Starred])(node)
st.ctx = ctx
def _fix_store_ctx(node: rawptr[AST]):
"""Fix ctx of store targets to be Store"""
if node is None: return
_set_ctx(node, ASTCtx.Store)
k: int = node.kind()
if k == ASTKind.Tuple or k == ASTKind.List:
tp_elts: rawptr[list[rawptr[AST]]] = None
if k == ASTKind.Tuple:
tp: rawptr[Tuple] = (rawptr[Tuple])(node)
tp_elts = tp.elts
else:
ls: rawptr[List] = (rawptr[List])(node)
tp_elts = ls.elts
if tp_elts is not None:
n: sizet = tp_elts.__len__()
for i in range(n):
cur: rawptr[AST] = tp_elts.get(i)
_fix_store_ctx(cur)
elif k == ASTKind.Starred:
st2: rawptr[Starred] = (rawptr[Starred])(node)
_fix_store_ctx(st2.value)
# ============================================================
# Expression Parser
# ============================================================
def _parse_expr(ps: rawptr[Parser]) -> rawptr[AST]:
"""expr: yield_expr | namedexpr_test"""
if _cur_kw(ps) == Keyword.Yield:
return _parse_yield(ps)
return _parse_namedexpr(ps)
def _parse_namedexpr(ps: rawptr[Parser]) -> rawptr[AST]:
"""namedexpr_test: test [':=' test]"""
left: rawptr[AST] = _parse_test(ps)
if left is None: return None
if _match_op(ps, TokOp.ColonEq):
_advance(ps)
right: rawptr[AST] = _parse_test(ps)
node: rawptr[AST] = NamedExpr(ps.pool, left, right)
_set_pos(node, left.lineno, left.col_offset, 0, 0)
return node
return left
def _IsVolatileCallPrefix(ps: rawptr[Parser]) -> int:
"""Test for a value-position `volatile f()` / `volatile S()` prefix, 1 = yes / 0 = no.
`volatile` is a soft keyword consumed only in value position before a call (Name followed by '(');
its use as an annotation descriptor (`a: volatile int`) or an identifier is unaffected.
"""
if _cur_kw(ps) != Keyword.Volatile:
return 0
nxt: rawptr[Token] = ps.cur.next
if nxt is None or nxt.type != TokenType.Name:
return 0
nxt2: rawptr[Token] = nxt.next
if nxt2 is None or nxt2.type != TokenType.Op or nxt2.op_subtype != TokOp.LPar:
return 0
return 1
def _parse_test(ps: rawptr[Parser]) -> rawptr[AST]:
"""test: or_test ['if' or_test 'else' test] | lambdef"""
if _cur_kw(ps) == Keyword.Lambda:
return _parse_lambda(ps)
# Value-position volatile call prefix: `a: int = volatile f()` / `volatile S()` -> consume volatile and
# mark the call node (result slot accessed volatile, struct writes too; LLVM call has no volatile qualifier).
if _IsVolatileCallPrefix(ps) != 0:
_advance(ps) # consume volatile
vinner: rawptr[AST] = _parse_test(ps)
if vinner is not None and vinner.kind() == ASTKind.Call:
(Call*(vinner)).IsVolatile = 1
return vinner
left: rawptr[AST] = _parse_or_test(ps)
if left is None: return None
if _cur_kw(ps) == Keyword.If:
_advance(ps)
test: rawptr[AST] = _parse_or_test(ps)
if _cur_kw(ps) == Keyword.Else:
_advance(ps)
orelse: rawptr[AST] = _parse_test(ps)
node: rawptr[AST] = IfExp(ps.pool, test, left, orelse)
_set_pos(node, left.lineno, left.col_offset, 0, 0)
return node
ps.error_count += 1
return left
return left
def _parse_lambda(ps: rawptr[Parser]) -> rawptr[AST]:
"""lambdef: 'lambda' [varargslist] ':' test"""
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps) # lambda
args: rawptr[AST] = None
if not _match_op(ps, TokOp.Colon):
args = _parse_varargslist(ps, 1)
if not _expect_op(ps, TokOp.Colon):
return None
body: rawptr[AST] = _parse_test(ps)
node: rawptr[AST] = Lambda(ps.pool, args, body)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_or_test(ps: rawptr[Parser]) -> rawptr[AST]:
"""or_test: and_test ('or' and_test)*"""
left: rawptr[AST] = _parse_and_test(ps)
if left is None: return None
values: rawptr[list[rawptr[AST]]] = None
while True:
# Supports both leading `or` ((A\n or B)) and trailing `or` (A or\n B) inside parens:
# each round skips Nl before checking, otherwise A or B\n or C stops on Nl after B.
_skip_nl(ps)
if _cur_kw(ps) != Keyword.Or:
break
if values is None:
values = list[rawptr[AST]](ps.pool, 8)
values.append(left)
_advance(ps)
_skip_nl(ps) # cross-line inside parens: a or\n b
right: rawptr[AST] = _parse_and_test(ps)
if right is None: break
values.append(right)
if values is not None:
node: rawptr[AST] = BoolOp(ps.pool, OpKind.Or, values)
_set_pos(node, left.lineno, left.col_offset, 0, 0)
return node
return left
def _parse_and_test(ps: rawptr[Parser]) -> rawptr[AST]:
"""and_test: not_test ('and' not_test)*"""
left: rawptr[AST] = _parse_not_test(ps)
if left is None: return None
values: rawptr[list[rawptr[AST]]] = None
while True:
# Supports both leading `and` ((A\n and B)) and trailing `and` (A and\n B) inside parens:
# each round skips Nl before checking, otherwise A and B\n and C stops on Nl after B.
_skip_nl(ps)
if _cur_kw(ps) != Keyword.And:
break
if values is None:
values = list[rawptr[AST]](ps.pool, 8)
values.append(left)
_advance(ps)
_skip_nl(ps) # cross-line inside parens: a and\n b
right: rawptr[AST] = _parse_not_test(ps)
if right is None: break
values.append(right)
if values is not None:
node: rawptr[AST] = BoolOp(ps.pool, OpKind.And, values)
_set_pos(node, left.lineno, left.col_offset, 0, 0)
return node
return left
def _parse_not_test(ps: rawptr[Parser]) -> rawptr[AST]:
"""not_test: 'not' not_test | comparison"""
if _cur_kw(ps) == Keyword.Not:
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps)
operand: rawptr[AST] = _parse_not_test(ps)
node: rawptr[AST] = UnaryOp(ps.pool, OpKind.Not, operand)
_set_pos(node, lineno, col, 0, 0)
return node
return _parse_comparison(ps)
def _parse_comparison(ps: rawptr[Parser]) -> rawptr[AST]:
"""comparison: bitor_expr (comp_op bitor_expr)*
Single comparison design: a < b → Compare(left=a, op=Lt, right=b)
Chained comparison a < b < c → BoolOp(And, [Compare(a,<,b), Compare(b,<,c)])
"""
left: rawptr[AST] = _parse_bitor(ps)
if left is None: return None
_skip_nl(ps) # leading comparison operator inside parens: (a\n < b)
op_id: int = _match_comp_op(ps)
if op_id == OpKind.NoneOp:
return left
_advance_comp_op(ps)
_skip_nl(ps) # cross-line inside parens: a ==\n b
right: rawptr[AST] = _parse_bitor(ps)
# Single comparison: construct ops and comparators list
ops: rawptr[list[int]] = list[int](ps.pool, 8)
ops.append(op_id)
comparators: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
comparators.append(right)
first_cmp: rawptr[AST] = Compare(ps.pool, left, ops, comparators)
_set_pos(first_cmp, left.lineno, left.col_offset, 0, 0)
# Check chained comparison
_skip_nl(ps) # leading chained comparison inside parens: (a < b\n < c)
next_op: int = _match_comp_op(ps)
if next_op == OpKind.NoneOp:
return first_cmp
# Single comparison with chained conditions: a < b < c → BoolOp(And, [Compare(a,<,b), Compare(b,<,c), ...])
values: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
values.append(first_cmp)
cur_left: rawptr[AST] = right
while next_op != OpKind.NoneOp:
_advance_comp_op(ps)
_skip_nl(ps) # cross-line chained comparison inside parens
cur_right: rawptr[AST] = _parse_bitor(ps)
ops2: rawptr[list[int]] = list[int](ps.pool, 8)
ops2.append(next_op)
comparators2: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
comparators2.append(cur_right)
cmp: rawptr[AST] = Compare(ps.pool, cur_left, ops2, comparators2)
_set_pos(cmp, cur_left.lineno, cur_left.col_offset, 0, 0)
values.append(cmp)
cur_left = cur_right
_skip_nl(ps) # leading chained comparison inside parens: (a < b\n < c)
next_op = _match_comp_op(ps)
node: rawptr[AST] = BoolOp(ps.pool, OpKind.And, values)
_set_pos(node, left.lineno, left.col_offset, 0, 0)
return node
def _match_comp_op(ps: rawptr[Parser]) -> int:
"""Check if the current one is a comparison operator, return OP_* or OpKind.NoneOp"""
if _cur_type(ps) == TokenType.Op:
op_id: int = _cur_op(ps)
if op_id == TokOp.EqEq: return OpKind.Eq
if op_id == TokOp.ExclaimEq: return OpKind.Ne
if op_id == TokOp.Less: return OpKind.Lt
if op_id == TokOp.LessEq: return OpKind.Le
if op_id == TokOp.Greater: return OpKind.Gt
if op_id == TokOp.GreaterEq: return OpKind.Ge
if _cur_kw(ps) == Keyword.Is:
if ps.cur.next is not None and ps.cur.next.type == TokenType.Name and ps.cur.next.kw_subtype == Keyword.Not:
return OpKind.IsNot
return OpKind.Is
if _cur_kw(ps) == Keyword.Not:
if ps.cur.next is not None and ps.cur.next.kw_subtype == Keyword.In:
return OpKind.NotIn
if _cur_kw(ps) == Keyword.In:
return OpKind.In
return OpKind.NoneOp
def _advance_comp_op(ps: rawptr[Parser]):
"""Consume comparison operator token"""
if _cur_type(ps) == TokenType.Op:
_advance(ps)
return
if _cur_kw(ps) == Keyword.Is:
_advance(ps)
if _cur_kw(ps) == Keyword.Not:
_advance(ps)
return
if _cur_kw(ps) == Keyword.Not:
_advance(ps)
if _cur_kw(ps) == Keyword.In:
_advance(ps)
return
if _cur_kw(ps) == Keyword.In:
_advance(ps)
def _parse_bitor(ps: rawptr[Parser]) -> rawptr[AST]:
left: rawptr[AST] = _parse_bitxor(ps)
if left is None: return None
while True:
_skip_nl(ps) # leading '|' inside parens: (a\n | b)
if _match_op(ps, TokOp.VBar) == 0:
break
_advance(ps)
_skip_nl(ps) # cross-line inside parens: a |\n b
right: rawptr[AST] = _parse_bitxor(ps)
node: rawptr[AST] = BinOp(ps.pool, left, OpKind.BitOr, right)
_set_pos(node, left.lineno, left.col_offset, 0, 0)
left = node
return left
def _parse_bitxor(ps: rawptr[Parser]) -> rawptr[AST]:
left: rawptr[AST] = _parse_bitand(ps)
if left is None: return None
while True:
_skip_nl(ps) # leading '^' inside parens: (a\n ^ b)
if _match_op(ps, TokOp.Caret) == 0:
break
_advance(ps)
_skip_nl(ps) # cross-line inside parens: a ^\n b
right: rawptr[AST] = _parse_bitand(ps)
node: rawptr[AST] = BinOp(ps.pool, left, OpKind.BitXor, right)
_set_pos(node, left.lineno, left.col_offset, 0, 0)
left = node
return left
def _parse_bitand(ps: rawptr[Parser]) -> rawptr[AST]:
left: rawptr[AST] = _parse_shift(ps)
if left is None: return None
while True:
_skip_nl(ps) # leading '&' inside parens: (a\n & b)
if _match_op(ps, TokOp.Amp) == 0:
break
_advance(ps)
_skip_nl(ps) # cross-line inside parens: a &\n b
right: rawptr[AST] = _parse_shift(ps)
node: rawptr[AST] = BinOp(ps.pool, left, OpKind.BitAnd, right)
_set_pos(node, left.lineno, left.col_offset, 0, 0)
left = node
return left
def _parse_shift(ps: rawptr[Parser]) -> rawptr[AST]:
left: rawptr[AST] = _parse_arith(ps)
if left is None: return None
while True:
_skip_nl(ps) # leading '<<'/'>>' inside parens: (a\n << b)
if _match_op(ps, TokOp.LtLt) == 0 and _match_op(ps, TokOp.GtGt) == 0:
break
op_token: int = _cur_op(ps)
_advance(ps)
_skip_nl(ps) # cross-line inside parens: a <<\n b
right: rawptr[AST] = _parse_arith(ps)
ast_op: int = OpKind.LShift
if op_token == TokOp.GtGt:
ast_op = OpKind.RShift
node: rawptr[AST] = BinOp(ps.pool, left, ast_op, right)
_set_pos(node, left.lineno, left.col_offset, 0, 0)
left = node
return left
def _parse_arith(ps: rawptr[Parser]) -> rawptr[AST]:
left: rawptr[AST] = _parse_term(ps)
if left is None: return None
while True:
_skip_nl(ps) # leading '+'/'-' inside parens: (a\n + b)
if _match_op(ps, TokOp.Plus) == 0 and _match_op(ps, TokOp.Minus) == 0:
break
op_token: int = _cur_op(ps)
_advance(ps)
_skip_nl(ps) # cross-line inside parens: a +\n b
right: rawptr[AST] = _parse_term(ps)
ast_op: int = OpKind.Add
if op_token == TokOp.Minus:
ast_op = OpKind.Sub
node: rawptr[AST] = BinOp(ps.pool, left, ast_op, right)
_set_pos(node, left.lineno, left.col_offset, 0, 0)
left = node
return left
# ============================================================
# C-style pointer suffix ('*' / '**') -- same source as TransPyC lib/vast _AtPtrSuffix
# ============================================================
# Start position (line/col) of the last consumed C-style pointer suffix ('*' / '**').
# _parse_paren matches it against ps.prev to tell the paren-less cast `(T*)x` from a
# plain group like `(a | b)`. Recorded as ints, not as a token pointer: comparing two
# token pointers whose declared pointer levels differ is rejected by the compare check.
_g_ptr_suffix_line: int = 0
_g_ptr_suffix_col: int = 0
def _ptr_suffix_end_op(op: int) -> int:
"""Whether the suffix-end token is a "no right operand" position (']' ',' ')' ':' '=').
Multiplication/power would lack a right operand there anyway (a pre-existing syntax error), so
treating '*'/'**' as a pointer suffix changes no valid expression (x[a * b] / a ** b unaffected).
"""
if op == TokOp.Rsqb or op == TokOp.Comma or op == TokOp.RPar \
or op == TokOp.Colon or op == TokOp.Equal:
return 1
return 0
def _at_ptr_suffix(ps: rawptr[Parser], base: rawptr[AST]) -> int:
"""Whether the current '*'/'**' run is a C-style pointer suffix applied to base.
Rule (same as vast): base is a type atom (Name/Attribute/Subscript) and, skipping consecutive
'*'/'**' and Nl, the next token is ']' / ',' / ')' / ':' / '='. Covers annotation and expression contexts.
"""
if base is None:
return 0
k: int = base.kind()
if k != ASTKind.Name and k != ASTKind.Attribute and k != ASTKind.Subscript:
return 0
nxt: rawptr[Token] = ps.cur
stars: int = 0
while nxt is not None:
if nxt.type == TokenType.Op \
and (nxt.op_subtype == TokOp.Star or nxt.op_subtype == TokOp.StarStar):
stars += 1
nxt = nxt.next
elif nxt.type == TokenType.Nl:
nxt = nxt.next
else:
break
if stars == 0 or nxt is None or nxt.type != TokenType.Op:
return 0
return _ptr_suffix_end_op(nxt.op_subtype)
def _make_ptr_type(ps: rawptr[Parser], base: rawptr[AST]) -> rawptr[AST]:
"""Build one pointer wrapper layer: base | t.CPtr."""
cptr: rawptr[AST] = Attribute(ps.pool, Name(ps.pool, "t", ASTCtx.Load),
"CPtr", ASTCtx.Load)
node: rawptr[AST] = BinOp(ps.pool, base, OpKind.BitOr, cptr)
_set_pos(node, base.lineno, base.col_offset, 0, 0)
return node
def _consume_ptr_suffix(ps: rawptr[Parser], base: rawptr[AST]) -> rawptr[AST]:
"""Consume consecutive C-style pointer suffixes ('*' = 1 layer, '**' = 2), expanding into a base | t.CPtr chain."""
global _g_ptr_suffix_line, _g_ptr_suffix_col
while True:
_skip_nl(ps)
if _match_op(ps, TokOp.StarStar):
_advance(ps)
_g_ptr_suffix_line = ps.prev.lineno
_g_ptr_suffix_col = ps.prev.col_offset
base = _make_ptr_type(ps, base)
base = _make_ptr_type(ps, base)
continue
if _match_op(ps, TokOp.Star):
_advance(ps)
_g_ptr_suffix_line = ps.prev.lineno
_g_ptr_suffix_col = ps.prev.col_offset
base = _make_ptr_type(ps, base)
continue
break
return base
def _at_cast_operand(ps: rawptr[Parser]) -> int:
"""Whether the current token may start the operand of a paren-less cast `(T*)x`.
'(' is deliberately absent: `(T*)(x)` is already handled by the trailer loop as a
cast call, and keeping it there leaves that path's AST untouched. Nl is not
skipped either: a line break after ')' starts a new statement, not an operand.
"""
if _cur_type(ps) == TokenType.Name:
return 1
if _cur_type(ps) == TokenType.Number:
return 1
if _cur_type(ps) == TokenType.String:
return 1
op: int = _cur_op(ps)
if op == TokOp.Lsqb or op == TokOp.LBrace or op == TokOp.Star or op == TokOp.Amp or op == TokOp.Minus or op == TokOp.Plus or op == TokOp.Tilde:
return 1
return 0
def _at_ptr_suffix_call(ps: rawptr[Parser], base: rawptr[AST]) -> int:
"""Whether the current '*'/'**' run is a `T*(x)` pointer-suffix cast call.
Rule (same as TPC lib/vast _AtPtrSuffixCall): (1) base is a type atom; (2) base and the star run are
adjacent (same line, no whitespace/newline); (3) the run, internally adjacent, is immediately followed
by '('. Required because `a * (b)` is valid multiplication, which a bare star rule would misread.
"""
if base is None:
return 0
k: int = base.kind()
if k != ASTKind.Name and k != ASTKind.Attribute and k != ASTKind.Subscript:
return 0
nxt: rawptr[Token] = ps.cur
pv: rawptr[Token] = ps.prev
if pv is None or nxt is None:
return 0
if pv.type == TokenType.Nl or pv.type == TokenType.NewLine:
return 0
if pv.end_lineno != nxt.lineno or pv.end_col_offset != nxt.col_offset:
return 0 # whitespace/newline between base and '*': not a C-style cast
stars: int = 0
prev: rawptr[Token] = None
last: rawptr[Token] = None
while nxt is not None:
if nxt.type == TokenType.Op \
and (nxt.op_subtype == TokOp.Star or nxt.op_subtype == TokOp.StarStar):
if prev is not None \
and (nxt.lineno != prev.end_lineno or nxt.col_offset != prev.end_col_offset):
return 0 # whitespace inside the star run: not adjacent, not a cast
stars += 1
prev = nxt
last = nxt
nxt = nxt.next
else:
break
if stars == 0 or last is None or nxt is None:
return 0
if nxt.type != TokenType.Op or nxt.op_subtype != TokOp.LPar:
return 0
if nxt.lineno != last.end_lineno or nxt.col_offset != last.end_col_offset:
return 0
return 1
def _consume_ptr_suffix_call(ps: rawptr[Parser], base: rawptr[AST]) -> rawptr[AST]:
"""Parse a recognized `T*(...)` pointer-suffix cast as a call.
Identical to the existing cast form `(T | t.CPtr)(...)`: a Call (func = the expanded base | t.CPtr
chain, args/keywords = the parenthesized arguments); no second cast representation is introduced.
"""
func: rawptr[AST] = _consume_ptr_suffix(ps, base)
return _parse_call(ps, func)
def _parse_term(ps: rawptr[Parser]) -> rawptr[AST]:
left: rawptr[AST] = _parse_factor(ps)
if left is None: return None
while True:
_skip_nl(ps) # leading '*'/'/'/'//'/'%'/'@' inside parens: (a\n * b)
if _match_op(ps, TokOp.Star) and _at_ptr_suffix(ps, left) != 0:
left = _consume_ptr_suffix(ps, left) # C-style pointer suffix, not multiplication
continue
op_token: int = 0
if _match_op(ps, TokOp.Star): op_token = TokOp.Star
elif _match_op(ps, TokOp.Slash): op_token = TokOp.Slash
elif _match_op(ps, TokOp.DSlash): op_token = TokOp.DSlash
elif _match_op(ps, TokOp.Percent): op_token = TokOp.Percent
elif _match_op(ps, TokOp.At): op_token = TokOp.At
if op_token == 0: break
_advance(ps)
_skip_nl(ps) # cross-line inside parens: a *\n b
right: rawptr[AST] = _parse_factor(ps)
ast_op: int = _binop_from_op(op_token)
node: rawptr[AST] = BinOp(ps.pool, left, ast_op, right)
_set_pos(node, left.lineno, left.col_offset, 0, 0)
left = node
return left
def _parse_factor(ps: rawptr[Parser]) -> rawptr[AST]:
"""factor: '+' factor | '-' factor | '~' factor | '*' factor | '&' factor | power"""
# A leading `*` in argument position was already parsed by _parse_call as a tightly bound Starred
# operand; return it as this level's factor, before the unary test (so `*p + n` => `(*p) + n`).
if ps.starred_seed is not None:
seed: rawptr[AST] = ps.starred_seed
ps.starred_seed = None
return seed
if (_match_op(ps, TokOp.Plus) or _match_op(ps, TokOp.Minus) or _match_op(ps, TokOp.Tilde)
or _match_op(ps, TokOp.Star) or _match_op(ps, TokOp.Amp)):
op_token: int = _cur_op(ps)
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps)
# Combined unary-minus + integer-literal overflow check: `-` is unary, so a magnitude-valid literal
# may become < -2^63 after negation. Only a unary minus directly on an integer constant is checked here.
if op_token == TokOp.Minus and _cur_type(ps) == TokenType.Number \
and ps.cur.is_float == 0 \
and _lex_int_neg_exceeds_i64(_cur_str(ps)) != 0:
_record_neg_literal_overflow(_cur_str(ps), lineno, col)
ps.error_count += 1
return None
operand: rawptr[AST] = _parse_factor(ps)
ast_op: int = OpKind.UAdd
if op_token == TokOp.Minus:
ast_op = OpKind.USub
elif op_token == TokOp.Tilde:
ast_op = OpKind.Invert
elif op_token == TokOp.Star:
ast_op = OpKind.Deref
elif op_token == TokOp.Amp:
ast_op = OpKind.AddrOf
node: rawptr[AST] = UnaryOp(ps.pool, ast_op, operand)
_set_pos(node, lineno, col, 0, 0)
return node
return _parse_power(ps)
def _parse_power(ps: rawptr[Parser]) -> rawptr[AST]:
"""power: await_expr ['**' factor] | atom_expr ['**' factor]"""
base: rawptr[AST] = _parse_atom_expr(ps)
if base is None: return None
if _match_op(ps, TokOp.StarStar) and _at_ptr_suffix(ps, base) != 0:
return _consume_ptr_suffix(ps, base) # C-style pointer suffix ('**' counts as 2 layers)
if _match_op(ps, TokOp.StarStar):
_advance(ps)
exp: rawptr[AST] = _parse_factor(ps)
node: rawptr[AST] = BinOp(ps.pool, base, OpKind.Pow, exp)
_set_pos(node, base.lineno, base.col_offset, 0, 0)
return node
return base
def _parse_atom_expr(ps: rawptr[Parser]) -> rawptr[AST]:
"""atom_expr: ['await'] atom trailer*"""
if _cur_kw(ps) == Keyword.Await:
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps)
inner: rawptr[AST] = _parse_atom_expr(ps)
node: rawptr[AST] = Await(ps.pool, inner)
_set_pos(node, lineno, col, 0, 0)
return node
base: rawptr[AST] = _parse_atom(ps)
if base is None: return None
# trailer loop
while True:
if (_match_op(ps, TokOp.Star) or _match_op(ps, TokOp.StarStar)) \
and _at_ptr_suffix_call(ps, base) != 0:
# `T*(x)` / `T**(x)` pointer-suffix cast: parsed as a cast call (same shape as
# `(T | t.CPtr)(x)`). Consumed in the trailer loop, so subsequent `.attr` / `[i]` / `(…)`
# are still attached by this same loop.
base = _consume_ptr_suffix_call(ps, base)
elif _match_op(ps, TokOp.Dot):
_advance(ps)
if _cur_type(ps) != TokenType.Name:
ps.error_count += 1
break
attr_name: str = _cur_str(ps)
attr_lineno: int = _cur_lineno(ps)
attr_col: int = _cur_col(ps)
_advance(ps)
ctx: int = ASTCtx.Load
if _match_op(ps, TokOp.Equal):
ctx = ASTCtx.Store
elif _match_op(ps, TokOp.PlusEq) or _match_op(ps, TokOp.MinusEq):
ctx = ASTCtx.Store
base = Attribute(ps.pool, base, attr_name, ctx)
_set_pos(base, attr_lineno, attr_col, 0, 0)
elif _match_op(ps, TokOp.LPar):
base = _parse_call(ps, base)
elif _match_op(ps, TokOp.Lsqb):
base = _parse_subscript(ps, base)
else:
break
return base
def _parse_call(ps: rawptr[Parser], func: rawptr[AST]) -> rawptr[AST]:
"""Parse function call trailer. Both args and keywords are lists."""
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps) # (
_skip_nls_and_newlines(ps) # Skip the line break after '('
args: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
keywords: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
if not _match_op(ps, TokOp.RPar):
while True:
# **expr
if _match_op(ps, TokOp.StarStar):
_advance(ps)
_skip_nls_and_newlines(ps)
val: rawptr[AST] = _parse_expr(ps)
kw: rawptr[AST] = KwNode(ps.pool, None, val)
_set_pos(kw, _cur_lineno(ps), _cur_col(ps), 0, 0)
keywords.append(kw)
# *expr
elif _match_op(ps, TokOp.Star):
_advance(ps)
_skip_nls_and_newlines(ps)
# Bind only a unary factor (preserving `*p[0]` = `*(p[0])`), put it on starred_seed and let
# _parse_expr continue the binary ops: `*p + n` => `(*p) + n`, not `*(p + n)`.
seed: rawptr[AST] = _parse_factor(ps)
if seed is None:
ps.error_count += 1
break
star: rawptr[AST] = Starred(ps.pool, seed, ASTCtx.Load)
_set_pos(star, seed.lineno, seed.col_offset, 0, 0)
ps.starred_seed = star
val = _parse_expr(ps)
args.append(val)
# keyword = expr
elif _cur_type(ps) == TokenType.Name:
if ps.cur.next is not None and ps.cur.next.type == TokenType.Op and ps.cur.next.op_subtype == TokOp.Equal:
kw_name: str = _cur_str(ps)
kw_lineno: int = _cur_lineno(ps)
kw_col: int = _cur_col(ps)
_advance(ps) # name
_advance(ps) # =
_skip_nls_and_newlines(ps)
val = _parse_expr(ps)
kw = KwNode(ps.pool, kw_name, val)
_set_pos(kw, kw_lineno, kw_col, 0, 0)
keywords.append(kw)
else:
val = _parse_expr(ps)
args.append(val)
else:
val = _parse_expr(ps)
args.append(val)
_skip_nls_and_newlines(ps) # Skip the line break after the parameter
if not _accept_op(ps, TokOp.Comma):
break
_skip_nls_and_newlines(ps) # Skip the line break after the comma
_skip_nls_and_newlines(ps) # Skip the line break before ')'
_expect_op(ps, TokOp.RPar)
node: rawptr[AST] = Call(ps.pool, func, args, keywords)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_subscript(ps: rawptr[Parser], value: rawptr[AST]) -> rawptr[AST]:
"""Parse subscript trailer."""
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps) # [
_skip_nl(ps)
slice_node: rawptr[AST] = _parse_slice(ps)
_expect_op(ps, TokOp.Rsqb)
ctx: int = ASTCtx.Load
if _match_op(ps, TokOp.Equal):
ctx = ASTCtx.Store
node: rawptr[AST] = Subscript(ps.pool, value, slice_node, ctx)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_slice(ps: rawptr[Parser]) -> rawptr[AST]:
"""Parse slice: [expr] ':' [expr] [':' [expr]] | expr | expr, expr, ..."""
lower: rawptr[AST] = None
# Empty subscript (`T[]`): `[` directly followed by `]` => slice is None (the type side treats it as
# an empty param list); calling _parse_expr here would report an unexpected token ']'.
if not _match_op(ps, TokOp.Colon) and not _match_op(ps, TokOp.Rsqb):
lower = _parse_expr(ps)
# Check if there’s a comma (tuple syntax: a[x, y, ...])
if _match_op(ps, TokOp.Comma):
elts: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
if lower is not None:
elts.append(lower)
while _accept_op(ps, TokOp.Comma):
if _match_op(ps, TokOp.Rsqb):
break # Trailing comma
elem: rawptr[AST] = _parse_expr(ps)
if elem is not None:
elts.append(elem)
lineno: int = 0
if lower is not None:
lineno = lower.lineno
node: rawptr[AST] = Tuple(ps.pool, elts, ASTCtx.Load)
_set_pos(node, lineno, 0, 0, 0)
return node
if _match_op(ps, TokOp.Colon):
_advance(ps)
upper: rawptr[AST] = None
if not _match_op(ps, TokOp.Colon) and not _match_op(ps, TokOp.Rsqb):
upper = _parse_expr(ps)
step: rawptr[AST] = None
if _accept_op(ps, TokOp.Colon):
if not _match_op(ps, TokOp.Rsqb):
step = _parse_expr(ps)
lineno: int = 0
if lower is not None:
lineno = lower.lineno
elif upper is not None:
lineno = upper.lineno
else:
lineno = _cur_lineno(ps)
node: rawptr[AST] = Slice(ps.pool, lower, upper, step)
_set_pos(node, lineno, 0, 0, 0)
return node
return lower
def _parse_atom(ps: rawptr[Parser]) -> rawptr[AST]:
"""atom: the smallest unit of expression"""
ttype: int = _cur_type(ps)
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
if ttype == TokenType.Name:
kw: int = _cur_kw(ps)
if kw == Keyword.True_:
_advance(ps)
return _new_const_bool(ps.pool, 1, lineno, col)
if kw == Keyword.False_:
_advance(ps)
return _new_const_bool(ps.pool, 0, lineno, col)
if kw == Keyword.None_:
_advance(ps)
return _new_const_none(ps.pool, lineno, col)
name_str: str = _cur_str(ps)
_advance(ps)
node: rawptr[AST] = Name(ps.pool, name_str, ASTCtx.Load)
_set_pos(node, lineno, col, 0, 0)
return node
if ttype == TokenType.Number:
tok: rawptr[Token] = ps.cur
_advance(ps)
if tok.is_float:
return _new_const_float(ps.pool, tok.float_val, lineno, col)
return _new_const_int(ps.pool, tok.int_val, lineno, col)
if ttype == TokenType.String:
sval: str = _cur_str(ps)
is_fstr: int = ps.cur.kw_subtype & 4
is_triple: int = ps.cur.kw_subtype & 8
is_dquote: int = ps.cur.kw_subtype & 16
is_char: int = ps.cur.kw_subtype & 32
_advance(ps)
node: rawptr[AST]
if is_fstr:
node = _parse_fstring(ps, sval, lineno, col)
else:
node = _new_const_str(ps.pool, sval, lineno, col, is_dquote, is_triple, is_char)
# Python implicit adjacent string literal concatenation: "a" "b" -> "ab".
# Nl is the in-paren newline token (allowed across lines); statement NewLine is not skipped.
while True:
_skip_nl(ps)
if _cur_type(ps) != TokenType.String:
break
n_sval: str = _cur_str(ps)
n_fstr: int = ps.cur.kw_subtype & 4
n_triple: int = ps.cur.kw_subtype & 8
n_dquote: int = ps.cur.kw_subtype & 16
n_char: int = ps.cur.kw_subtype & 32
_advance(ps)
n_node: rawptr[AST]
if n_fstr:
n_node = _parse_fstring(ps, n_sval, lineno, col)
else:
n_node = _new_const_str(ps.pool, n_sval, lineno, col, n_dquote, n_triple, n_char)
node = _concat_str_nodes(ps.pool, node, n_node)
return node
if _match_op(ps, TokOp.Ellipsis):
_advance(ps)
return _new_const_none(ps.pool, lineno, col)
if _match_op(ps, TokOp.LPar):
return _parse_paren(ps)
if _match_op(ps, TokOp.Lsqb):
return _parse_list(ps)
if _match_op(ps, TokOp.LBrace):
return _parse_brace(ps)
_record_parse_err(ps)
ps.error_count += 1
return None
def _parse_paren(ps: rawptr[Parser]) -> rawptr[AST]:
"""Parenthesis expressions: () | (expr) | (expr,) | (expr, ...) | (yield)"""
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps) # (
_skip_nl(ps) # Skip the line break after '(' (multi-line tuple)
if _match_op(ps, TokOp.RPar):
_advance(ps)
# Empty tuple
empty: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
node: rawptr[AST] = Tuple(ps.pool, empty, ASTCtx.Load)
_set_pos(node, lineno, col, 0, 0)
return node
# yield
if _cur_kw(ps) == Keyword.Yield:
y: rawptr[AST] = _parse_yield(ps)
_expect_op(ps, TokOp.RPar)
return y
first: rawptr[AST] = _parse_expr(ps)
if _match_op(ps, TokOp.Comma):
# Tuple
elts: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
elts.append(first)
while _accept_op(ps, TokOp.Comma):
_skip_nl(ps)
if _match_op(ps, TokOp.RPar):
break
e: rawptr[AST] = _parse_expr(ps)
elts.append(e)
_skip_nl(ps) # finish when the last element has no trailing comma (multi-line tuple)
_expect_op(ps, TokOp.RPar)
node = Tuple(ps.pool, elts, ASTCtx.Load)
_set_pos(node, lineno, col, 0, 0)
return node
# Paren-less C cast `(T*)x`: the inner expression ends with a pointer suffix
# (ps.prev is that suffix's '*'/'**' token) and the operand follows ')' directly.
# Same product as `(T*)(x)` / `T*(x)`: a Call whose func is `T | t.CPtr`.
is_ptr_cast: int = 0
if _g_ptr_suffix_line > 0 and _match_op(ps, TokOp.RPar) and ps.prev.lineno == _g_ptr_suffix_line and ps.prev.col_offset == _g_ptr_suffix_col:
is_ptr_cast = 1
_skip_nl(ps) # (expr\n) close a single-expression paren spanning lines
_expect_op(ps, TokOp.RPar)
# Check for generator expression
if _cur_kw(ps) == Keyword.For:
return _parse_comprehension(ps, first, ASTKind.GeneratorExp, lineno, col)
if is_ptr_cast != 0 and _at_cast_operand(ps) != 0:
operand: rawptr[AST] = _parse_factor(ps)
cast_args: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
cast_args.append(operand)
cast_kws: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
cast_node: rawptr[AST] = Call(ps.pool, first, cast_args, cast_kws)
_set_pos(cast_node, first.lineno, first.col_offset, 0, 0)
return cast_node
return first
def _parse_list(ps: rawptr[Parser]) -> rawptr[AST]:
"""List expressions: [] | [expr_list] | [expr_list comp_for]"""
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps) # [
_skip_nl(ps) # Skip the line break after '[' (multi-line list)
if _match_op(ps, TokOp.Rsqb):
_advance(ps)
empty: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
node: rawptr[AST] = List(ps.pool, empty, ASTCtx.Load)
_set_pos(node, lineno, col, 0, 0)
return node
first: rawptr[AST] = _parse_expr(ps)
if _cur_kw(ps) == Keyword.For:
node = _parse_comprehension(ps, first, ASTKind.ListComp, lineno, col)
_expect_op(ps, TokOp.Rsqb)
return node
elts: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
elts.append(first)
while _accept_op(ps, TokOp.Comma):
_skip_nl(ps)
if _match_op(ps, TokOp.Rsqb):
break
e: rawptr[AST] = _parse_expr(ps)
elts.append(e)
_skip_nl(ps) # finish when the last element has no trailing comma (multi-line list)
_expect_op(ps, TokOp.Rsqb)
node = List(ps.pool, elts, ASTCtx.Load)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_brace(ps: rawptr[Parser]) -> rawptr[AST]:
"""Brace expressions: {} | {set_items} | {dict_items} | {comp_for}"""
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps) # {
_skip_nl(ps) # Skip the line break after '{' (multi-line dict)
if _match_op(ps, TokOp.RBrace):
_advance(ps)
empty_k: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
empty_v: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
node: rawptr[AST] = Dict(ps.pool, empty_k, empty_v)
_set_pos(node, lineno, col, 0, 0)
return node
first: rawptr[AST] = _parse_expr(ps)
# dict: key: value
if _match_op(ps, TokOp.Colon):
_advance(ps)
val: rawptr[AST] = _parse_expr(ps)
if _cur_kw(ps) == Keyword.For:
# dict comprehension
gens: rawptr[list[rawptr[AST]]] = _parse_comp_for(ps)
node = DictComp(ps.pool, first, val, gens)
_set_pos(node, lineno, col, 0, 0)
_expect_op(ps, TokOp.RBrace)
return node
keys: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
values: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
keys.append(first)
values.append(val)
while _accept_op(ps, TokOp.Comma):
_skip_nl(ps)
if _match_op(ps, TokOp.RBrace):
break
k: rawptr[AST] = _parse_expr(ps)
_expect_op(ps, TokOp.Colon)
v: rawptr[AST] = _parse_expr(ps)
keys.append(k)
values.append(v)
_skip_nl(ps) # finish when the last element has no trailing comma
_expect_op(ps, TokOp.RBrace)
node = Dict(ps.pool, keys, values)
_set_pos(node, lineno, col, 0, 0)
return node
# set
if _cur_kw(ps) == Keyword.For:
node = _parse_comprehension(ps, first, ASTKind.SetComp, lineno, col)
_expect_op(ps, TokOp.RBrace)
return node
elts: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
elts.append(first)
while _accept_op(ps, TokOp.Comma):
_skip_nl(ps) # cross-line set literal inside braces
if _match_op(ps, TokOp.RBrace):
break
e: rawptr[AST] = _parse_expr(ps)
elts.append(e)
_skip_nl(ps) # finish when the last element has no trailing comma
_expect_op(ps, TokOp.RBrace)
node = Set(ps.pool, elts)
_set_pos(node, lineno, col, 0, 0)
return node
# ============================================================
# f-string parsing
# ============================================================
def _parse_sub_expr(ps: rawptr[Parser], src: str) -> rawptr[AST]:
"""Create a sub-lexer and parser to parse expressions (used for {expr} inside f-strings)"""
pool: rawptr[memhub.MemManager] = ps.pool
lx: rawptr[Lexer] = new_lexer(pool)
if lx is None: return None
_lexer_init(lx, src, pool)
tokens: rawptr[Token] = tokenize(lx)
if tokens is None: return None
sub_ps: rawptr[Parser] = new_parser(pool)
if sub_ps is None: return None
_parser_init(sub_ps, tokens, pool)
return _parse_test(sub_ps)
def _append_str_value(pool: rawptr[memhub.MemManager],
values: rawptr[list[rawptr[AST]]],
node: rawptr[AST]):
"""Flatten a string node into values (JoinedStr expands its values, everything else is appended)"""
if node is None:
return
if node.kind() == ASTKind.JoinedStr:
js_val: rawptr[JoinedStr] = (rawptr[JoinedStr])(node)
if js_val.values is not None:
n_v: sizet = js_val.values.__len__()
i_v: sizet
for i_v in range(n_v):
values.append(js_val.values.get(i_v))
else:
values.append(node)
def _concat_str_nodes(pool: rawptr[memhub.MemManager],
a: rawptr[AST], b: rawptr[AST]) -> rawptr[AST]:
"""Merge adjacent string literal nodes (Python implicit concatenation "a" "b" -> "ab").
Pure constant strings are concatenated as text; f-strings (JoinedStr/FormattedValue) are merged
into a JoinedStr values list.
"""
if a is None:
return b
if b is None:
return a
# Both are constant strings -> concatenate the text directly
if a.kind() == ASTKind.Constant and b.kind() == ASTKind.Constant:
ca_c: rawptr[Constant] = (rawptr[Constant])(a)
cb_c: rawptr[Constant] = (rawptr[Constant])(b)
if ca_c.const_kind == CONST_STR and cb_c.const_kind == CONST_STR \
and ca_c.str_val is not None and cb_c.str_val is not None:
la_c: sizet = string.strlen(ca_c.str_val)
lb_c: sizet = string.strlen(cb_c.str_val)
buf_c: str = pool.alloc(la_c + lb_c + 1)
if buf_c is not None:
string.memcpy(buf_c, ca_c.str_val, la_c)
string.memcpy(buf_c + la_c, cb_c.str_val, lb_c + 1)
return _new_const_str(pool, buf_c, a.lineno, a.col_offset, 0, 0, 0)
# Involves an f-string -> merge into a JoinedStr
values_c: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](pool, 8)
_append_str_value(pool, values_c, a)
_append_str_value(pool, values_c, b)
if values_c.__len__() == 1:
return values_c.get(0)
node_c: rawptr[AST] = JoinedStr(pool, values_c)
_set_pos(node_c, a.lineno, a.col_offset, 0, 0)
return node_c
def _parse_fstring(ps: rawptr[Parser], content: str,
lineno: int, col: int) -> rawptr[AST]:
"""Parse f-string content, building JoinedStr or FormattedValue nodes"""
pool: rawptr[memhub.MemManager] = ps.pool
values: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](pool, 8)
i: sizet = 0
content_len: sizet = string.strlen(content)
# Literal text buffer for f-string content
lit_buf: str = pool.alloc(content_len + 1)
if lit_buf is None: return None
lit_pos: sizet = 0
while i < content_len:
ch: char = content[i]
if ch == '{':
# {{ → Literal {
if i + 1 < content_len and content[i + 1] == '{':
lit_buf[lit_pos] = '{'
lit_pos += 1
i += 2
continue
# Refresh literal buffer
if lit_pos > 0:
lit_buf[lit_pos] = '\0'
lit_node: rawptr[AST] = _new_const_str(pool, lit_buf, lineno, col)
values.append(lit_node)
lit_buf = pool.alloc(content_len + 1)
if lit_buf is None: break
lit_pos = 0
# Find matching closing '}' while tracking nested '{}' and string literals inside the f-string
j: sizet = i + 1
depth: int = 1
conv_pos: sizet = 0
fmt_pos: sizet = 0
while j < content_len and depth > 0:
c: char = content[j]
if c == '\'' or c == '"':
# Skip nested string literals inside an expression block (PEP 701 same-quote nesting)
# so their characters do not confuse the {} depth / conversion / format delimiter scan
nq: char = c
j += 1
while j < content_len:
c2: char = content[j]
if c2 == '\\':
j += 2
continue
if c2 == nq:
j += 1
break
j += 1
continue
if c == '{':
depth += 1
elif c == '}':
depth -= 1
if depth == 0:
break
elif depth == 1:
if c == '!' and conv_pos == 0 and fmt_pos == 0:
if j + 1 < content_len:
nc: char = content[j + 1]
if nc == 's' or nc == 'r' or nc == 'a':
if j + 2 < content_len:
after: char = content[j + 2]
if after == ':' or after == '}':
conv_pos = j
elif j + 2 == content_len:
conv_pos = j
elif c == ':' and fmt_pos == 0:
fmt_pos = j
j += 1
if depth != 0:
_record_parse_err(ps)
ps.error_count += 1
break
# Extract expression text
expr_end: sizet = j
if conv_pos > 0:
expr_end = conv_pos
elif fmt_pos > 0:
expr_end = fmt_pos
expr_len: sizet = expr_end - i - 1
if expr_len == 0:
_record_parse_err(ps)
ps.error_count += 1
i = j + 1
continue
expr_buf: str = pool.alloc(expr_len + 1)
if expr_buf is None: break
string.memcpy(expr_buf, content + i + 1, expr_len)
expr_buf[expr_len] = '\0'
# Parse expression text
expr_node: rawptr[AST] = _parse_sub_expr(ps, expr_buf)
# Conversion specifier
conversion: int = -1
if conv_pos > 0:
conv_char: char = content[conv_pos + 1]
if conv_char == 's':
conversion = 115
elif conv_char == 'r':
conversion = 114
elif conv_char == 'a':
conversion = 97
# Format specifier
format_spec: rawptr[AST] = None
if fmt_pos > 0:
spec_len: sizet = j - fmt_pos - 1
if spec_len > 0:
spec_buf: str = pool.alloc(spec_len + 1)
if spec_buf is not None:
string.memcpy(spec_buf, content + fmt_pos + 1, spec_len)
spec_buf[spec_len] = '\0'
format_spec = _new_const_str(pool, spec_buf, lineno, col)
fv: rawptr[AST] = FormattedValue(pool, expr_node, conversion, format_spec)
_set_pos(fv, lineno, col, 0, 0)
values.append(fv)
i = j + 1
elif ch == '}':
if i + 1 < content_len and content[i + 1] == '}':
lit_buf[lit_pos] = '}'
lit_pos += 1
i += 2
else:
_record_parse_err(ps)
ps.error_count += 1
i += 1
else:
lit_buf[lit_pos] = ch
lit_pos += 1
i += 1
# Refresh remaining literal buffer
if lit_pos > 0:
lit_buf[lit_pos] = '\0'
lit_node2: rawptr[AST] = _new_const_str(pool, lit_buf, lineno, col)
values.append(lit_node2)
# If no content, return empty string
if values.__len__() == 0:
return _new_const_str(pool, "", lineno, col)
# If only one FormattedValue and no literal, return FormattedValue
if values.__len__() == 1:
first: rawptr[AST] = values.get(0)
if first.kind() == ASTKind.FormattedValue:
return first
node: rawptr[AST] = JoinedStr(pool, values)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_comprehension(ps: rawptr[Parser], elt: rawptr[AST],
vtype: int, lineno: int, col: int) -> rawptr[AST]:
"""General comprehension: elt comp_for"""
gens: rawptr[list[rawptr[AST]]] = _parse_comp_for(ps)
node: rawptr[AST] = None
if vtype == ASTKind.ListComp:
node = ListComp(ps.pool, elt, gens)
elif vtype == ASTKind.SetComp:
node = SetComp(ps.pool, elt, gens)
else:
node = GeneratorExp(ps.pool, elt, gens)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_comp_for(ps: rawptr[Parser]) -> rawptr[list[rawptr[AST]]]:
"""comp_for: ['async'] 'for' exprlist 'in' or_test [comp_if] (comp_for)*
Returns generators list."""
gens: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
while True:
is_async: int = 0
if _cur_kw(ps) == Keyword.Async:
is_async = 1
_advance(ps)
if _cur_kw(ps) != Keyword.For:
break
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps)
target: rawptr[AST] = _parse_target_list(ps)
if _cur_kw(ps) != Keyword.In:
ps.error_count += 1
break
_advance(ps)
iter_: rawptr[AST] = _parse_or_test(ps)
# Collect ifs to list
ifs: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
while _cur_kw(ps) == Keyword.If:
_advance(ps)
cond: rawptr[AST] = _parse_or_test(ps)
ifs.append(cond)
comp: rawptr[AST] = Comprehension(ps.pool, target, iter_, ifs, is_async)
_set_pos(comp, lineno, col, 0, 0)
gens.append(comp)
return gens
def _parse_target_list(ps: rawptr[Parser]) -> rawptr[AST]:
"""Parse assignment target list (comma-separated)"""
first: rawptr[AST] = _parse_target(ps)
if _match_op(ps, TokOp.Comma):
elts: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
elts.append(first)
while _accept_op(ps, TokOp.Comma):
if _cur_kw(ps) == Keyword.In:
break
e: rawptr[AST] = _parse_target(ps)
elts.append(e)
node: rawptr[AST] = Tuple(ps.pool, elts, ASTCtx.Store)
_set_pos(node, first.lineno, first.col_offset, 0, 0)
return node
return first
def _parse_target(ps: rawptr[Parser]) -> rawptr[AST]:
"""Parse assignment target"""
if _match_op(ps, TokOp.Star):
_advance(ps)
inner: rawptr[AST] = _parse_atom_expr(ps)
node: rawptr[AST] = Starred(ps.pool, inner, ASTCtx.Store)
_set_pos(node, inner.lineno, inner.col_offset, 0, 0)
return node
if _match_op(ps, TokOp.LPar):
_advance(ps)
_skip_nl(ps)
t_: rawptr[AST] = _parse_target_list(ps)
_skip_nl(ps)
_expect_op(ps, TokOp.RPar)
return t_
if _match_op(ps, TokOp.Lsqb):
_advance(ps)
_skip_nl(ps)
first: rawptr[AST] = _parse_target(ps)
elts: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
elts.append(first)
while _accept_op(ps, TokOp.Comma):
_skip_nl(ps)
if _match_op(ps, TokOp.Rsqb):
break
e: rawptr[AST] = _parse_target(ps)
elts.append(e)
_expect_op(ps, TokOp.Rsqb)
node = List(ps.pool, elts, ASTCtx.Store)
_set_pos(node, first.lineno, first.col_offset, 0, 0)
return node
# Normal atom_expr, ctx=Store
node = _parse_atom_expr(ps)
if node is not None:
_set_ctx(node, ASTCtx.Store)
return node
def _parse_yield(ps: rawptr[Parser]) -> rawptr[AST]:
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps) # yield
if _cur_kw(ps) == Keyword.From:
_advance(ps)
val: rawptr[AST] = _parse_expr(ps)
node: rawptr[AST] = YieldFrom(ps.pool, val)
_set_pos(node, lineno, col, 0, 0)
return node
val2: rawptr[AST] = None
if not _match_op(ps, TokOp.RPar) and _cur_type(ps) != TokenType.NewLine and _cur_type(ps) != TokenType.EndMarker:
if _cur_type(ps) == TokenType.Op and (_cur_op(ps) == TokOp.Comma or _cur_op(ps) == TokOp.Rsqb):
pass
else:
val2 = _parse_test(ps)
node = Yield(ps.pool, val2)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_varargslist(ps: rawptr[Parser], is_lambda: int = 0) -> rawptr[AST]:
"""Parse function parameter list
New API: args/defaults/kw_defaults are lists, vararg/kwarg are fields."""
args: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
vararg: rawptr[AST] = None
kwarg: rawptr[AST] = None
defaults: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
kw_defaults: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
while True:
_skip_nl(ps)
if _match_op(ps, TokOp.RPar):
break
# *args or * (kwonly separated)
if _match_op(ps, TokOp.Star):
_advance(ps)
_skip_nl(ps)
if _cur_type(ps) == TokenType.Name:
vararg = _parse_arg_def(ps, is_lambda)
if not _accept_op(ps, TokOp.Comma):
break
_skip_nl(ps)
continue
# **kwargs
if _match_op(ps, TokOp.StarStar):
_advance(ps)
_skip_nl(ps)
kwarg = _parse_arg_def(ps, is_lambda)
if not _accept_op(ps, TokOp.Comma):
break
_skip_nl(ps)
continue
# / Delimiter (posonly end)— New API: posonly/regular are not distinguished
if _match_op(ps, TokOp.Slash):
_advance(ps)
if not _accept_op(ps, TokOp.Comma):
break
_skip_nl(ps)
continue
# Normal parameter
a: rawptr[AST] = _parse_arg_def(ps, is_lambda)
if a is None: break
args.append(a)
if _match_op(ps, TokOp.Equal):
_advance(ps)
d: rawptr[AST] = _parse_test(ps)
defaults.append(d)
if not _accept_op(ps, TokOp.Comma):
break
_skip_nl(ps)
return Arguments(ps.pool, args, vararg, kwarg, defaults, kw_defaults)
def _parse_array_prefix_declarator(ps: rawptr[Parser]) -> rawptr[AST]:
"""C-style leading array declarator `[N]T` / `[]T` -> `t.CArray[T, N]` / `t.CArray[T, None]`.
Consumed only in annotation position (_parse_type_ann); returns None if the current token is not '['.
Desugars to the canonical array form shared with TPC / StubGen / DeclarationGenerator. Must be called
before _parse_test, otherwise `[N]` is parsed as a list literal and the element type then errors.
"""
if _match_op(ps, TokOp.Lsqb) == 0:
return None
lb_lineno: int = _cur_lineno(ps)
lb_col: int = _cur_col(ps)
_advance(ps) # consume '['
count_node: rawptr[AST] = None
if _match_op(ps, TokOp.Rsqb) == 0:
# Length expression (constant/constant expression; invalid lengths are rejected downstream)
count_node = _parse_test(ps)
if count_node is None:
_record_parse_err(ps)
return None
if _match_op(ps, TokOp.Rsqb) == 0:
_record_parse_err(ps) # missing ']'
ps.error_count += 1
return None
_advance(ps) # consume ']'
# Element type: recurse into _parse_type_ann, so `[N]const int` / `[N]int*` share one rule source
elem: rawptr[AST] = _parse_type_ann(ps)
if elem is None:
return None
if count_node is None:
count_node = _new_const_none(ps.pool, lb_lineno, lb_col)
# Element size must be explicitly 8 (list[T](pool, elem_size) addresses by byte stride; passing 2
# would write 8-byte pointers with a 2-byte stride -> heap overflow). Same arg as other list sites.
elts: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
elts.append(elem)
elts.append(count_node)
carray: rawptr[AST] = Subscript(ps.pool,
Attribute(ps.pool, Name(ps.pool, "t", ASTCtx.Load),
"CArray", ASTCtx.Load),
Tuple(ps.pool, elts, ASTCtx.Load),
ASTCtx.Load)
_set_pos(carray, lb_lineno, lb_col, 0, 0)
return carray
def _parse_type_ann(ps: rawptr[Parser]) -> rawptr[AST]:
"""Parse a type annotation: descriptor prefixes (const / volatile / __attribute__) and C-style pointer
suffixes (* / **), field-identical to TPC's expansion. `static` is invalid here; missing type / non-type
follower / bare use fail loudly. Reclaims the Mult/Pow junk _parse_test leaves and rebuilds t.CPtr chains.
"""
# Descriptor prefix: collect const / volatile markers (soft keywords, consumed only here)
mods: rawptr[list[rawptr[AST]]] = None
mod_count: int = 0
mod_lineno: int = _cur_lineno(ps)
mod_col: int = _cur_col(ps)
# `static` modifier (consumed at statement start): append t.CStatic as the first descriptor marker
# (pasted on the union's left later), then clear it so it applies only to this annotation.
if ps.pending_static != 0:
ps.pending_static = 0
if mods is None:
mods = list[rawptr[AST]](ps.pool, 8)
st_mk: rawptr[AST] = _MakeStaticMarker(ps)
if st_mk is not None:
mods.append(st_mk)
mod_count += 1
while _IsDescriptorKeyword(ps) != 0 or _IsMagicAttributeCall(ps) != 0 \
or _IsStaticDescriptorKeyword(ps) != 0:
if _IsStaticDescriptorKeyword(ps) != 0:
# Annotation-position `static` (`name: static T`) is no longer supported: storage-class markers
# are always prefixed. Fail loudly here (otherwise `static` becomes an opaque "static" class).
_RecordDescriptorErr("注解位置的 static 已不再支持：请写成前置形式"
" `static 名字: 类型 = 值`（等价 `名字: t.CStatic | 类型`）",
_cur_lineno(ps), _cur_col(ps))
_advance(ps)
continue
if mods is None:
# Element size is explicitly 8: list[T](pool, elem_size) addresses by byte stride, so passing 4
# would make the 2nd 8-byte pointer (e.g. for `const volatile`) write out of bounds.
mods = list[rawptr[AST]](ps.pool, 8)
if _IsMagicAttributeCall(ps) != 0:
# C-style `a: __attribute__(section(".text.wo"), aligned(16)) const int = ...`: same position
# and order-free mixing as const / volatile; each argument desugars to `t.attr.<name>` and joins
# the same markers list.
# Use only _parse_call for this call itself, never _parse_test: the latter is a general expression
# parser and would swallow the following `[]T` as a legal empty subscript, silently degrading the
# annotation to a bare element type with zero errors reported.
mka_lineno: int = _cur_lineno(ps)
mka_col: int = _cur_col(ps)
mka_fn: rawptr[AST] = Name(ps.pool, "__attribute__", ASTCtx.Load)
_set_pos(mka_fn, mka_lineno, mka_col, 0, 0)
_advance(ps) # consume `__attribute__`
ac_node: rawptr[AST] = _parse_call(ps, mka_fn) # parse only `( args )`
if ac_node is None:
break
if ac_node.kind() != ASTKind.Call:
break
ac: Call* = Call*(ac_node)
if ac.args is not None:
acn: sizet = len(ac.args)
for aci in range(acn):
mk: rawptr[AST] = _MakeTAttrMarker(ps, ac.args.get(aci))
if mk is not None:
mods.append(mk)
mod_count += 1
continue
mods.append(_MakeDescriptorMarker(ps))
mod_count += 1
_advance(ps)
# C-style leading array declarator `[N]T` / `[]T` (incl. `const [N]T`): desugar in place to
# `t.CArray[T, N]` / `t.CArray[T, None]` and paste the already-collected descriptors onto it.
if _match_op(ps, TokOp.Lsqb) != 0:
arr: rawptr[AST] = _parse_array_prefix_declarator(ps)
if arr is None:
return None
if mod_count > 0:
arr = _PrependDescriptorMarkers(ps, arr, mods, mod_count)
return arr
if mod_count > 0 and _cur_type(ps) != TokenType.Name:
# Missing type after a descriptor (`a: const` / `a: const = 1` / `def f(a: const)`):
# fail loudly -- never silently drop the descriptor.
_RecordDescriptorErr("描述符后缺少类型", mod_lineno, mod_col)
ps.error_count += 1
return None
before_ec: int = ps.error_count
base: rawptr[AST] = _parse_test(ps)
if base is None:
if mod_count > 0:
# Descriptor consumed but type parsing failed (e.g. `a: const = 1`): fail loudly keeping the descriptor error.
_RecordDescriptorErr("描述符后缺少类型（类型解析失败）", mod_lineno, mod_col)
_record_parse_err(ps) # annotation parse failed: record position (error_count already bumped in _parse_test)
return None
if mod_count > 0 and base.kind() == ASTKind.Constant:
# Non-type follower (`a: const 1` / `a: const "x"` / `a: const True`): fail loudly, since
# descriptors modify types only and constants are not types -- never silently degrade or drop.
_RecordDescriptorErr("描述符后随非类型（描述符只修饰类型）", mod_lineno, mod_col)
ps.error_count += 1
return None
ptr_level: int = 0
# Reclaim the junk nodes left by _parse_test consuming the * / ** suffixes
if base.kind() == ASTKind.BinOp:
gb: rawptr[BinOp] = (rawptr[BinOp])(base)
if gb.op == OpKind.Mult and gb.right is None:
ptr_level += 1
base = gb.left
elif gb.op == OpKind.Pow:
if gb.right is None:
ptr_level += 2
base = gb.left
elif gb.right.kind() == ASTKind.UnaryOp:
gu: rawptr[UnaryOp] = (rawptr[UnaryOp])(gb.right)
if gu.op == OpKind.Deref and gu.operand is None:
ptr_level += 3
base = gb.left
if ptr_level > 0:
ps.error_count = before_ec # reclaim succeeded: restore error count to avoid whole-file parse failure
# Consume remaining suffixes ('**' is one token counting as 2 layers, '*' as 1)
while _match_op(ps, TokOp.StarStar):
_advance(ps)
ptr_level += 2
while _match_op(ps, TokOp.Star):
_advance(ps)
ptr_level += 1
# Build n nested base | t.CPtr layers
for li_an in range(ptr_level):
cptr_an: rawptr[AST] = Attribute(ps.pool,
Name(ps.pool, "t", ASTCtx.Load),
"CPtr", ASTCtx.Load)
base = BinOp(ps.pool, base, OpKind.BitOr, cptr_an)
# Paste descriptors last (same shape as TPC text expansion; see _PrependDescriptorMarkers)
if mod_count > 0:
base = _PrependDescriptorMarkers(ps, base, mods, mod_count)
return base
def _parse_arg_def(ps: rawptr[Parser], is_lambda: int = 0) -> rawptr[AST]:
"""arg: TokenType.Name [':' annotation]"""
if _cur_type(ps) != TokenType.Name:
ps.error_count += 1
return None
name_str: str = _cur_str(ps)
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps)
annotation: rawptr[AST] = None
if is_lambda == 0:
if _accept_op(ps, TokOp.Colon):
annotation = _parse_type_ann(ps)
result: rawptr[AST] = Arg(ps.pool, name_str, annotation)
_set_pos(result, lineno, col, 0, 0)
return result
# ============================================================
# Statement parsing (Design: directly append to parent, do not return a list)
# ============================================================
def _parse_suite_into_children(ps: rawptr[Parser], parent: rawptr[AST],
owner_is_class: int = 0):
"""Parse statement suite, directly append to parent's children.
suite: TokenType.NewLine TokenType.Indent stmt+ TokenType.Dedent | simple_stmt
owner_is_class: whether this suite's direct owner is a ClassDef. During suite parsing ps.in_class_body
is set so _ParseVirtualModified can tell if `virtual def`/`novirtual def` is directly in a class body
(function/branch/loop bodies do not count). Default 0 keeps all non-class owners correct.
"""
prev_in_class: int = ps.in_class_body
ps.in_class_body = owner_is_class
_skip_nl(ps)
if _cur_type(ps) == TokenType.NewLine:
_advance(ps)
_skip_nl(ps)
if _cur_type(ps) != TokenType.Indent:
ps.error_count += 1
ps.in_class_body = prev_in_class
return
_advance(ps)
sidx: int = 0
while _cur_type(ps) != TokenType.Dedent and _cur_type(ps) != TokenType.EndMarker:
sidx += 1
# fail-fast anti-hang: over-budget steps (cyclic token list / cursor spinning) report and return
if ps.step_count > ps.step_limit:
_record_parse_err(ps)
return
# Decorator (class body methods can have decorators)
if _match_op(ps, TokOp.At):
ps.pending_decorators = _parse_decorators(ps)
before_cur: rawptr[Token] = ps.cur
_parse_statement_into(ps, parent)
# Defensive: if parse did not advance token, force advance to avoid infinite loop
if ps.cur is before_cur:
ps.error_count += 1
_advance(ps)
_skip_nl(ps)
_expect_type(ps, TokenType.Dedent)
ps.in_class_body = prev_in_class
return
# Single-line suite
_parse_simple_stmt_into(ps, parent)
ps.in_class_body = prev_in_class
def _parse_statement_into(ps: rawptr[Parser], parent: rawptr[AST]):
"""Parse statement (compound or simple), append to the parent."""
ttype: int = _cur_type(ps)
kw: int = _cur_kw(ps)
if ttype == TokenType.Name:
# define constant declaration: `define NAME[: T] = <literal>` (no parentheses). Must run before
# _parse_simple_stmt_into, otherwise `define` is parsed as an identifier expression and NAME is left
# as a stray token. `define NAME(...)` (macro function) is excluded by _is_define_const's lookahead.
if kw == Keyword.Define and _is_define_const(ps) != 0:
dc_stmt: rawptr[AST] = _parse_define_const(ps)
if dc_stmt is not None:
parent.append(dc_stmt)
return
# typedef alias declaration: `typedef T1 = T2` (statement-level reserved word, no lookahead since
# typedef is never an identifier in existing sources). Invalid forms fail loudly in _parse_typedef.
if _is_typedef(ps) != 0:
td_stmt: rawptr[AST] = _parse_typedef(ps)
if td_stmt is not None:
parent.append(td_stmt)
return
# export / extern variable declaration (soft keyword): `export NAME: T [= v]` / `extern NAME: T`
# -> a module-level AnnAssign with storage-class markers; function forms are excluded by lookahead.
if _is_export_extern_var(ps) != 0:
ee_stmt: rawptr[AST] = _parse_export_extern_var(ps)
if ee_stmt is not None:
parent.append(ee_stmt)
return
# virtual / novirtual modifier (soft keyword): only at statement start before class/def; synthesizes
# @t.CVTable / @t.NoVTable then proceeds with class/def parsing. Placed before the class/def dispatch.
if _IsVirtualModifier(ps) != 0:
_ParseVirtualModified(ps, parent)
return
# inline modifier (soft keyword): only at statement start before a function definition; synthesizes
# @t.always_inline then proceeds with def parsing (same pending_decorators channel as virtual).
if _IsInlineModifier(ps) != 0:
_ParseInlineModified(ps, parent)
return
# static modifier (soft keyword): statement start + static + (def | `Name :`). The function branch
# synthesizes @t.CStatic (returns); the variable branch flags pending_static for _parse_type_ann.
if _IsStaticModifier(ps) != 0:
if _ParseStaticModified(ps, parent) != 0:
return
# Bare descriptor use: the whole statement is only `const` / `volatile` (missing type) -> fail loudly.
# Matches only when followed by newline/EOF/semicolon (soft-keyword boundary), so `const = 5` /
# `t.attr.const()` do not match. After recording, continue the generic simple_stmt dispatch.
if _IsBareDescriptor(ps) != 0:
_RecordDescriptorErr("裸用描述符（整条语句只有描述符，缺类型）",
_cur_lineno(ps), _cur_col(ps))
ps.error_count += 1
if kw == Keyword.If or kw == Keyword.While or kw == Keyword.For or kw == Keyword.Try:
stmt: rawptr[AST] = _parse_compound(ps)
if stmt is not None:
parent.append(stmt)
return
if kw == Keyword.With or _is_funcdef_kw(ps, kw) != 0 or kw == Keyword.Class \
or kw == Keyword.Enum or kw == Keyword.Union or kw == Keyword.Renum \
or kw == Keyword.Struct:
stmt = _parse_compound(ps)
if stmt is not None:
parent.append(stmt)
return
if kw == Keyword.Match or kw == Keyword.Switch or kw == Keyword.Async:
stmt = _parse_compound(ps)
if stmt is not None:
parent.append(stmt)
return
_parse_simple_stmt_into(ps, parent)
def _parse_simple_stmt_into(ps: rawptr[Parser], parent: rawptr[AST]):
"""simple_stmt: small_stmt (';' small_stmt)* [';'] TokenType.NewLine"""
first: rawptr[AST] = _parse_small_stmt(ps)
if first is not None:
parent.append(first)
while _accept_op(ps, TokOp.Semi):
if _cur_type(ps) == TokenType.NewLine or _cur_type(ps) == TokenType.EndMarker:
break
s: rawptr[AST] = _parse_small_stmt(ps)
if s is None: break
parent.append(s)
# First skip Nl (in-paren newline, e.g. between multi-line call args; a legal token)
_skip_nl(ps)
# Statement-end check: simple_stmt must end with a newline/EOF. Stray tokens (e.g. adjacent
# expressions split into multiple statements, or a def missing its keyword) are a fail-fast syntax
# error, reported Python-style (SyntaxError at line/col).
if _cur_type(ps) != TokenType.NewLine and _cur_type(ps) != TokenType.EndMarker:
# Stray tokens (e.g. adjacent expressions split into statements, or a def missing its keyword) are a
# fail-fast syntax error. The message is no longer self-assembled: position/message go through the
# parse error channel with prev/next tokens appended, output Python-style by the caller's
# dump_parse_traceback (Traceback + File/line + source line + caret + SyntaxError).
# Locate: walk the list from the head to find cur, recording prev and next token context.
prev_tok: rawptr[Token] = None
walk_tok: rawptr[Token] = ps.tokens
while walk_tok is not None and walk_tok is not ps.cur:
prev_tok = walk_tok
walk_tok = walk_tok.next
pv_str: str = "(head)"
pv_line: int = 0
pv_col: int = 0
if prev_tok is not None:
pv_str = prev_tok.str_val
pv_line = prev_tok.lineno
pv_col = prev_tok.col_offset
nxt_tok: rawptr[Token] = ps.cur.next
nxt_str: str = "(none)"
nxt_line: int = 0
nxt_col: int = 0
if nxt_tok is not None:
nxt_str = nxt_tok.str_val
nxt_line = nxt_tok.lineno
nxt_col = nxt_tok.col_offset
_record_parse_err_ctx(ps, _cur_type(ps), pv_str, pv_line, pv_col,
nxt_str, nxt_line, nxt_col)
ps.error_count += 1
return
if _cur_type(ps) == TokenType.NewLine:
_advance(ps)
def _parse_small_stmt(ps: rawptr[Parser]) -> rawptr[AST]:
"""small_stmt: various simple statements"""
kw: int = _cur_kw(ps)
# fallthrough keyword (hard): legal only inside a switch case body ("fall to the next case").
# Legality is checked by the generator; writing it in a match or outside a switch is fail-fast.
if kw == Keyword.FallThrough:
ft_lineno: int = _cur_lineno(ps)
ft_col: int = _cur_col(ps)
_advance(ps)
ft_node: rawptr[AST] = FallThrough(ps.pool)
_set_pos(ft_node, ft_lineno, ft_col, 0, 0)
return ft_node
if kw == Keyword.Pass:
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps)
node: rawptr[AST] = Pass(ps.pool)
_set_pos(node, lineno, col, 0, 0)
return node
if kw == Keyword.Break:
lineno = _cur_lineno(ps)
col = _cur_col(ps)
_advance(ps)
node = Break(ps.pool)
_set_pos(node, lineno, col, 0, 0)
return node
if kw == Keyword.Continue:
lineno = _cur_lineno(ps)
col = _cur_col(ps)
_advance(ps)
node = Continue(ps.pool)
_set_pos(node, lineno, col, 0, 0)
return node
if kw == Keyword.Return:
return _parse_return(ps)
if kw == Keyword.Raise:
return _parse_raise(ps)
if kw == Keyword.Global:
return _parse_global(ps)
if kw == Keyword.Nonlocal:
return _parse_nonlocal(ps)
if kw == Keyword.Import:
return _parse_import(ps)
if kw == Keyword.From:
return _parse_from_import(ps)
if kw == Keyword.Del:
return _parse_del(ps)
if kw == Keyword.Assert:
return _parse_assert(ps)
if kw == Keyword.Yield:
return _parse_expr_stmt(ps, _parse_yield(ps))
# Expression statement / assignment / augassign / annassign
return _parse_expr_stmt(ps, None)
def _maybe_tuple(ps: rawptr[Parser], first: rawptr[AST]) -> rawptr[AST]:
"""If there is a comma after first, collect the comma-separated expressions into a Tuple."""
if first is None: return None
if not _match_op(ps, TokOp.Comma):
return first
lineno: int = first.lineno
col: int = first.col_offset
elts: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
elts.append(first)
_accept_op(ps, TokOp.Comma)
while True:
tt: int = _cur_type(ps)
if tt == TokenType.NewLine or tt == TokenType.EndMarker:
break
if tt == TokenType.Op:
op: int = _cur_op(ps)
if op == TokOp.Equal or op == TokOp.Semi:
break
e: rawptr[AST] = _parse_expr(ps)
if e is None: break
elts.append(e)
if not _accept_op(ps, TokOp.Comma):
break
node: rawptr[AST] = Tuple(ps.pool, elts, ASTCtx.Load)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_testlist(ps: rawptr[Parser]) -> rawptr[AST]:
"""Parse a comma-separated list of expressions, if there’s more than one, form a tuple."""
first: rawptr[AST] = _parse_expr(ps)
return _maybe_tuple(ps, first)
def _parse_expr_stmt(ps: rawptr[Parser], first: rawptr[AST]) -> rawptr[AST]:
"""Expression statement, handle assignment/augassign/annassign"""
if first is None:
first = _parse_testlist(ps)
else:
first = _maybe_tuple(ps, first)
if first is None:
return None
lineno: int = first.lineno
col: int = first.col_offset
# Assignment
if _match_op(ps, TokOp.Equal):
targets: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
targets.append(first)
_fix_store_ctx(first)
value: rawptr[AST] = None
accept_result: int = _accept_op(ps, TokOp.Equal)
while accept_result:
v: rawptr[AST] = _parse_testlist(ps)
if v is None: break
if value is not None:
targets.append(value)
_fix_store_ctx(value)
value = v
accept_result = _accept_op(ps, TokOp.Equal)
node: rawptr[AST] = Assign(ps.pool, targets, value)
_set_pos(node, lineno, col, 0, 0)
return node
# augassign
if _cur_type(ps) == TokenType.Op:
op_id: int = _cur_op(ps)
aug_op: int = _augop_from_op(op_id)
if aug_op != OpKind.NoneOp:
_advance(ps)
value2: rawptr[AST] = _parse_testlist(ps)
_fix_store_ctx(first)
node2: rawptr[AST] = AugAssign(ps.pool, first, aug_op, value2)
_set_pos(node2, lineno, col, 0, 0)
return node2
# annassign: target ':' annotation [= value]
if _match_op(ps, TokOp.Colon):
_advance(ps)
annotation: rawptr[AST] = _parse_type_ann(ps)
value3: rawptr[AST] = None
if _accept_op(ps, TokOp.Equal):
value3 = _parse_testlist(ps)
if ps.pool is None:
return None
_fix_store_ctx(first)
if ps.pool is None:
return None
node3: rawptr[AST] = AnnAssign(ps.pool, first, annotation, value3, 1)
_set_pos(node3, lineno, col, 0, 0)
return node3
# Expression statement
node4: rawptr[AST] = Expr(ps.pool, first)
_set_pos(node4, lineno, col, 0, 0)
return node4
def _parse_return(ps: rawptr[Parser]) -> rawptr[AST]:
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps)
val: rawptr[AST] = None
if _cur_type(ps) != TokenType.NewLine and _cur_type(ps) != TokenType.EndMarker and _cur_type(ps) != TokenType.Op:
val = _parse_testlist(ps)
elif _cur_type(ps) == TokenType.Op and _cur_op(ps) != TokOp.Semi:
val = _parse_testlist(ps)
node: rawptr[AST] = Return(ps.pool, val)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_raise(ps: rawptr[Parser]) -> rawptr[AST]:
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps)
exc: rawptr[AST] = None
cause: rawptr[AST] = None
if _cur_type(ps) != TokenType.NewLine and _cur_type(ps) != TokenType.EndMarker:
exc = _parse_test(ps)
if _cur_kw(ps) == Keyword.From:
_advance(ps)
cause = _parse_test(ps)
node: rawptr[AST] = Raise(ps.pool, exc, cause)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_global(ps: rawptr[Parser]) -> rawptr[AST]:
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps)
names: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
if _cur_type(ps) == TokenType.Name:
n: rawptr[AST] = Name(ps.pool, _cur_str(ps), ASTCtx.Store)
_set_pos(n, _cur_lineno(ps), _cur_col(ps), 0, 0)
names.append(n)
_advance(ps)
while _accept_op(ps, TokOp.Comma):
if _cur_type(ps) != TokenType.Name: break
n = Name(ps.pool, _cur_str(ps), ASTCtx.Store)
_set_pos(n, _cur_lineno(ps), _cur_col(ps), 0, 0)
names.append(n)
_advance(ps)
node: rawptr[AST] = Global(ps.pool, names)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_nonlocal(ps: rawptr[Parser]) -> rawptr[AST]:
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps)
names: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
if _cur_type(ps) == TokenType.Name:
n: rawptr[AST] = Name(ps.pool, _cur_str(ps), ASTCtx.Store)
_set_pos(n, _cur_lineno(ps), _cur_col(ps), 0, 0)
names.append(n)
_advance(ps)
while _accept_op(ps, TokOp.Comma):
if _cur_type(ps) != TokenType.Name: break
n = Name(ps.pool, _cur_str(ps), ASTCtx.Store)
_set_pos(n, _cur_lineno(ps), _cur_col(ps), 0, 0)
names.append(n)
_advance(ps)
node: rawptr[AST] = Nonlocal(ps.pool, names)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_import(ps: rawptr[Parser]) -> rawptr[AST]:
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps)
names: rawptr[list[rawptr[AST]]] = _parse_alias_list(ps)
node: rawptr[AST] = Import(ps.pool, names)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_from_import(ps: rawptr[Parser]) -> rawptr[AST]:
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps)
# Parse module (possibly with dots)
level: int = 0
while _match_op(ps, TokOp.Dot):
level += 1
_advance(ps)
while _match_op(ps, TokOp.Ellipsis):
level += 3
_advance(ps)
module_name: str = None
# The token type for keywords (like import) is also Name, so they must be excluded using kw_subtype,
# otherwise the `import` in `from . import` would be mistakenly treated as a module_name.
# Note: only `import` itself should be excluded. Module names can be keywords (like in `from .match import ...`,
# where match is a Viper match statement keyword), requiring NotKeyword would cause
# `from .match import` to fail parsing and the dependency scan would miss that module.
if _cur_type(ps) == TokenType.Name and _cur_kw(ps) != Keyword.Import:
module_name = _cur_str(ps)
_advance(ps)
while _match_op(ps, TokOp.Dot):
_advance(ps)
if _cur_type(ps) == TokenType.Name and _cur_kw(ps) == Keyword.NotKeyword:
next_name: str = _cur_str(ps)
mlen: int = string.strlen(module_name)
nlen: int = string.strlen(next_name)
new_mod: str = ps.pool.alloc(mlen + 1 + nlen + 1)
if new_mod is not None:
string.strcpy(new_mod, module_name)
new_mod[mlen] = '.'
string.strcpy(new_mod + mlen + 1, next_name)
module_name = new_mod
_advance(ps)
if _cur_kw(ps) != Keyword.Import:
ps.error_count += 1
return None
_advance(ps)
_skip_nl(ps)
# import *
if _match_op(ps, TokOp.Star):
_advance(ps)
star_alias: rawptr[AST] = Alias(ps.pool, "*", None)
_set_pos(star_alias, lineno, col, 0, 0)
star_names: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
star_names.append(star_alias)
node: rawptr[AST] = ImportFrom(ps.pool, module_name, star_names, level)
_set_pos(node, lineno, col, 0, 0)
return node
# ( names ) or names
if _match_op(ps, TokOp.LPar):
_advance(ps)
_skip_nl(ps)
names: rawptr[list[rawptr[AST]]] = _parse_alias_list(ps)
_skip_nl(ps)
_expect_op(ps, TokOp.RPar)
else:
names = _parse_alias_list(ps)
node: rawptr[AST] = ImportFrom(ps.pool, module_name, names, level)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_alias_list(ps: rawptr[Parser]) -> rawptr[list[rawptr[AST]]]:
"""Parse alias list."""
names: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
while True:
if _cur_type(ps) != TokenType.Name:
break
name_str: str = _cur_str(ps)
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps)
# dotted name
while _match_op(ps, TokOp.Dot):
_advance(ps)
if _cur_type(ps) == TokenType.Name:
next_n: str = _cur_str(ps)
nmlen: int = string.strlen(name_str)
nxtlen: int = string.strlen(next_n)
new_name: str = ps.pool.alloc(nmlen + 1 + nxtlen + 1)
if new_name is not None:
string.strcpy(new_name, name_str)
new_name[nmlen] = '.'
string.strcpy(new_name + nmlen + 1, next_n)
name_str = new_name
_advance(ps)
asname: str = None
if _cur_kw(ps) == Keyword.As:
_advance(ps)
if _cur_type(ps) == TokenType.Name:
asname = _cur_str(ps)
_advance(ps)
a: rawptr[AST] = Alias(ps.pool, name_str, asname)
_set_pos(a, lineno, col, 0, 0)
names.append(a)
if not _accept_op(ps, TokOp.Comma):
break
_skip_nl(ps)
return names
def _parse_del(ps: rawptr[Parser]) -> rawptr[AST]:
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps)
targets: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
while True:
t_: rawptr[AST] = _parse_atom_expr(ps)
if t_ is None: break
_set_ctx(t_, ASTCtx.Del)
targets.append(t_)
if not _accept_op(ps, TokOp.Comma):
break
node: rawptr[AST] = Delete(ps.pool, targets)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_assert(ps: rawptr[Parser]) -> rawptr[AST]:
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps)
test: rawptr[AST] = _parse_test(ps)
msg: rawptr[AST] = None
if _accept_op(ps, TokOp.Comma):
msg = _parse_test(ps)
node: rawptr[AST] = Assert(ps.pool, test, msg)
_set_pos(node, lineno, col, 0, 0)
return node
# ============================================================
# Compound statements
# ============================================================
def _parse_compound(ps: rawptr[Parser]) -> rawptr[AST]:
kw: int = _cur_kw(ps)
if kw == Keyword.If:
return _parse_if(ps)
if kw == Keyword.While:
return _parse_while(ps)
if kw == Keyword.For:
return _parse_for(ps, 0)
if kw == Keyword.Try:
return _parse_try(ps)
if kw == Keyword.With:
return _parse_with(ps, 0)
if _is_funcdef_kw(ps, kw) != 0:
return _parse_funcdef(ps, 0)
if kw == Keyword.Class:
return _parse_classdef(ps)
if kw == Keyword.Enum:
return _parse_enumdef(ps)
if kw == Keyword.Union:
return _parse_uniondef(ps)
if kw == Keyword.Renum:
return _parse_renumdef(ps)
if kw == Keyword.Struct:
return _parse_structdef(ps)
if kw == Keyword.Async:
_advance(ps)
if _is_funcdef_kw(ps, _cur_kw(ps)) != 0:
return _parse_funcdef(ps, 1)
if _cur_kw(ps) == Keyword.For:
return _parse_for(ps, 1)
if _cur_kw(ps) == Keyword.With:
return _parse_with(ps, 1)
ps.error_count += 1
return None
if kw == Keyword.Match:
return _parse_match(ps)
if kw == Keyword.Switch:
return _parse_switch(ps)
ps.error_count += 1
return None
def _parse_if(ps: rawptr[Parser]) -> rawptr[AST]:
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps) # if
test: rawptr[AST] = _parse_test(ps)
_expect_op(ps, TokOp.Colon)
# First construct with empty orelse, then fill it in later
empty: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
node: rawptr[If] = If(ps.pool, test, empty)
_parse_suite_into_children(ps, node)
orelse: rawptr[list[rawptr[AST]]] = _parse_if_tail(ps)
node.orelse = orelse
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_if_tail(ps: rawptr[Parser]) -> rawptr[list[rawptr[AST]]]:
"""Parse elif/else chain and return orelse list.
elif: orelse = list containing single If node (nested)
else: orelse = statement list
"""
if _cur_kw(ps) == Keyword.Elif:
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps)
elif_test: rawptr[AST] = _parse_test(ps)
_expect_op(ps, TokOp.Colon)
empty: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
elif_node: rawptr[If] = If(ps.pool, elif_test, empty)
_parse_suite_into_children(ps, elif_node)
elif_orelse: rawptr[list[rawptr[AST]]] = _parse_if_tail(ps)
elif_node.orelse = elif_orelse
_set_pos(elif_node, lineno, col, 0, 0)
orelse: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
orelse.append(elif_node)
return orelse
if _cur_kw(ps) == Keyword.Else:
_advance(ps)
_expect_op(ps, TokOp.Colon)
scratch: rawptr[Module] = Module(ps.pool)
_parse_suite_into_children(ps, scratch)
return scratch.children
return list[rawptr[AST]](ps.pool, 8) # empty list
# ============================================================
# Parsing Compound Statements (Continued)
# ============================================================
def _parse_while(ps: rawptr[Parser]) -> rawptr[AST]:
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps) # while
test: rawptr[AST] = _parse_test(ps)
_expect_op(ps, TokOp.Colon)
empty: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
node: rawptr[While] = While(ps.pool, test, empty)
_parse_suite_into_children(ps, node)
if _cur_kw(ps) == Keyword.Else:
_advance(ps)
_expect_op(ps, TokOp.Colon)
scratch: rawptr[Module] = Module(ps.pool)
_parse_suite_into_children(ps, scratch)
node.orelse = scratch.children
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_for(ps: rawptr[Parser], is_async: int) -> rawptr[AST]:
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps) # for
target: rawptr[AST] = _parse_target_list(ps)
if _cur_kw(ps) != Keyword.In:
ps.error_count += 1
return None
_advance(ps)
iter_: rawptr[AST] = _parse_or_test(ps)
_expect_op(ps, TokOp.Colon)
empty: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
node: rawptr[For] = For(ps.pool, target, iter_, empty, is_async)
_fix_store_ctx(target)
_parse_suite_into_children(ps, node)
if _cur_kw(ps) == Keyword.Else:
_advance(ps)
_expect_op(ps, TokOp.Colon)
scratch: rawptr[Module] = Module(ps.pool)
_parse_suite_into_children(ps, scratch)
node.orelse = scratch.children
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_try(ps: rawptr[Parser]) -> rawptr[AST]:
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps) # try
_expect_op(ps, TokOp.Colon)
empty: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
node: rawptr[Try] = Try(ps.pool, empty, empty, empty)
_parse_suite_into_children(ps, node)
# handlers (except)
handlers: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
while _cur_kw(ps) == Keyword.Except:
h: rawptr[AST] = _parse_except_handler(ps)
if h is not None:
handlers.append(h)
node.handlers = handlers
# else
if _cur_kw(ps) == Keyword.Else:
_advance(ps)
_expect_op(ps, TokOp.Colon)
scratch: rawptr[Module] = Module(ps.pool)
_parse_suite_into_children(ps, scratch)
node.orelse = scratch.children
# finally
if _cur_kw(ps) == Keyword.Finally:
_advance(ps)
_expect_op(ps, TokOp.Colon)
scratch2: rawptr[Module] = Module(ps.pool)
_parse_suite_into_children(ps, scratch2)
node.finalbody = scratch2.children
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_except_handler(ps: rawptr[Parser]) -> rawptr[AST]:
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps) # except
type_: rawptr[AST] = None
name: str = None
if _cur_op(ps) != TokOp.Colon:
type_ = _parse_test(ps)
if _cur_kw(ps) == Keyword.As:
_advance(ps)
if _cur_type(ps) == TokenType.Name:
name = _cur_str(ps)
_advance(ps)
_expect_op(ps, TokOp.Colon)
node: rawptr[AST] = ExceptHandler(ps.pool, type_, name)
_parse_suite_into_children(ps, node)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_with(ps: rawptr[Parser], is_async: int) -> rawptr[AST]:
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps) # with
items: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
# items
while True:
item: rawptr[AST] = _parse_with_item(ps)
if item is None: break
items.append(item)
if not _accept_op(ps, TokOp.Comma):
break
# Check if we can continue with a newline (with (...): form)
if _cur_type(ps) == TokenType.NewLine or _cur_type(ps) == TokenType.Nl:
break
_expect_op(ps, TokOp.Colon)
node: rawptr[With] = With(ps.pool, items, is_async)
_parse_suite_into_children(ps, node)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_with_item(ps: rawptr[Parser]) -> rawptr[AST]:
"""with_item: test ['as' expr]"""
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
context_expr: rawptr[AST] = _parse_or_test(ps)
if context_expr is None: return None
optional_vars: rawptr[AST] = None
if _cur_kw(ps) == Keyword.As:
_advance(ps)
optional_vars = _parse_target(ps)
_fix_store_ctx(optional_vars)
node: rawptr[AST] = WithItem(ps.pool, context_expr, optional_vars)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_funcdef(ps: rawptr[Parser], is_async: int) -> rawptr[AST]:
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
is_extern_fd: int = 0
is_export_fd: int = 0
is_define_fd: int = 0
if _cur_kw(ps) == Keyword.Extern:
# extern declaration: extern [def] A(...) [-> T] [optional ": pass"]
is_extern_fd = 1
_advance(ps) # extern
if _cur_kw(ps) == Keyword.Def:
_advance(ps)
elif _cur_kw(ps) == Keyword.Export:
# export declaration: export [def] A(...) [-> T] [optional ": pass"]
is_export_fd = 1
_advance(ps) # export
if _cur_kw(ps) == Keyword.Def:
_advance(ps)
elif _cur_kw(ps) == Keyword.Define:
# define macro function: syntax identical to def; t.CDefine is appended to the return annotation
# (StubGen keeps it as-is -> C #define / alwaysinline)
is_define_fd = 1
_advance(ps) # define
else:
_advance(ps) # def / fn / ...
if _cur_type(ps) != TokenType.Name:
ps.error_count += 1
return None
name: str = _cur_str(ps)
_advance(ps)
# PEP 695 generic function type parameters: def add[T](a: T, b: T) -- parsed and stored into
# type_params, consistent with the class parser. type_params maps to int as a fallback during translation.
type_params_fd: rawptr[list[str]] = None
if _accept_op(ps, TokOp.Lsqb):
type_params_fd = list[str](ps.pool, 8)
while not _match_op(ps, TokOp.Rsqb):
if _cur_type(ps) == TokenType.EndMarker:
break
if _cur_type(ps) == TokenType.Name:
tp_name_fd: str = _cur_str(ps)
_advance(ps)
# Skip the optional bound/constraint: [T: bound] or [T: (A, B)]
if _match_op(ps, TokOp.Colon):
_advance(ps)
_parse_test(ps)
type_params_fd.append(tp_name_fd)
if not _accept_op(ps, TokOp.Comma):
break
_expect_op(ps, TokOp.Rsqb)
args: rawptr[AST] = None
if _accept_op(ps, TokOp.LPar):
args = _parse_varargslist(ps, 0)
_expect_op(ps, TokOp.RPar)
returns: rawptr[AST] = None
if _accept_op(ps, TokOp.RArrow):
returns = _parse_type_ann(ps)
if is_extern_fd:
# extern declaration: rewrite the return annotation to <orig type> | t.State, reusing the existing
# State/external-C-function path (vpi annotation containing "State" -> bare name, no function body)
state_attr_fd: rawptr[AST] = Attribute(ps.pool,
Name(ps.pool, "t", ASTCtx.Load),
"State", ASTCtx.Load)
if returns != None:
returns = BinOp(ps.pool, returns, OpKind.BitOr, state_attr_fd)
else:
returns = state_attr_fd
elif is_export_fd:
# export declaration: rewrite the return annotation to <orig type> | t.CExport, reusing the existing
# CExport path (vpi annotation containing "CExport" -> bare name, no module SHA1 header)
export_attr_fd: rawptr[AST] = Attribute(ps.pool,
Name(ps.pool, "t", ASTCtx.Load),
"CExport", ASTCtx.Load)
if returns != None:
returns = BinOp(ps.pool, returns, OpKind.BitOr, export_attr_fd)
else:
returns = export_attr_fd
elif is_define_fd:
# define macro function: rewrite the return annotation to <orig type> | t.CDefine, reusing the
# existing CDefine path (StubGen keeps it as-is -> C #define / alwaysinline)
define_attr_fd: rawptr[AST] = Attribute(ps.pool,
Name(ps.pool, "t", ASTCtx.Load),
"CDefine", ASTCtx.Load)
if returns != None:
returns = BinOp(ps.pool, returns, OpKind.BitOr, define_attr_fd)
else:
returns = define_attr_fd
dec_list: rawptr[list[rawptr[AST]]] = ps.pending_decorators
ps.pending_decorators = None
node: rawptr[FunctionDef] = FunctionDef(ps.pool, name, args, dec_list, returns, is_async, type_params_fd)
if is_extern_fd:
node.flags = node.flags | FLAG_IS_EXTERN
if is_export_fd:
node.flags = node.flags | FLAG_IS_EXPORT
if is_define_fd:
node.flags = node.flags | FLAG_IS_DEFINE
if (is_extern_fd == 0 and is_export_fd == 0 and is_define_fd == 0) or _match_op(ps, TokOp.Colon):
_expect_op(ps, TokOp.Colon)
_parse_suite_into_children(ps, node)
else:
# extern/export without ': pass' (extern A(...) -> int): empty body, declaration only. The NewLine
# must be consumed here: the module/class statement loop only skips Nl, so a leftover NewLine would be
# parsed as an expression next round and fail parse_tokens.
if _cur_type(ps) == TokenType.NewLine:
_advance(ps)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_classdef(ps: rawptr[Parser]) -> rawptr[AST]:
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps) # class
if _cur_type(ps) != TokenType.Name:
ps.error_count += 1
return None
name: str = _cur_str(ps)
_advance(ps)
# Parse type parameters [T] or [T, U] (PEP 695 generic class syntax)
type_params: rawptr[list[str]] = None
if _accept_op(ps, TokOp.Lsqb):
type_params = list[str](ps.pool, 8)
while not _match_op(ps, TokOp.Rsqb):
if _cur_type(ps) == TokenType.EndMarker: break
if _cur_type(ps) == TokenType.Name:
tp_name: str = _cur_str(ps)
_advance(ps)
# Skip optional bound/constraint: [T: bound] or [T: (A, B)]
if _match_op(ps, TokOp.Colon):
_advance(ps)
_parse_test(ps)
type_params.append(tp_name)
if not _accept_op(ps, TokOp.Comma):
break
_expect_op(ps, TokOp.Rsqb)
bases: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
keywords: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
# bases / keywords
if _accept_op(ps, TokOp.LPar):
while not _match_op(ps, TokOp.RPar):
# Cross-line base list inside parens (class X(A,\n B):): skip the newline after a comma,
# otherwise cur=NL is not Name/Comma -> break -> _expect_op(RPar) reports an unexpected token.
_skip_nl(ps)
if _cur_type(ps) == TokenType.EndMarker: break
# base or keyword=value
if _cur_type(ps) == TokenType.Name:
# Parse as expression, if followed by '=', then as keyword=value
e: rawptr[AST] = _parse_or_test(ps)
if _match_op(ps, TokOp.Equal):
_advance(ps)
v: rawptr[AST] = _parse_test(ps)
kw_node: rawptr[AST] = KwNode(ps.pool, e, v)
keywords.append(kw_node)
else:
bases.append(e)
if not _accept_op(ps, TokOp.Comma):
break
_skip_nl(ps) # finish the newline after the last base (class X(A,\n B):)
_expect_op(ps, TokOp.RPar)
_expect_op(ps, TokOp.Colon)
dec_list: rawptr[list[rawptr[AST]]] = ps.pending_decorators
ps.pending_decorators = None
node: rawptr[ClassDef] = ClassDef(ps.pool, name, bases, keywords, dec_list, type_params)
# owner_is_class=1: ps.in_class_body=1 during the class body, so virtual/novirtual modifiers can tell
# whether a method-level modifier is directly in the class body (see _ParseVirtualModified).
_parse_suite_into_children(ps, node, 1)
_set_pos(node, lineno, col, 0, 0)
return node
# ============================================================
# enum sugar (fully equivalent to `class X(t.CEnum):`): `enum Name:` header + member lines
# (`A` / `A: T` / `A = 0`). The synthesized ClassDef is field-identical to _parse_classdef output, so
# downstream (ClassGenEnum._translate_enum_def -> EnumTable) has a single implementation path.
# ============================================================
def _enum_member_annassign(ps: rawptr[Parser], owner: rawptr[AST],
name_node: rawptr[AST],
lineno: int, col: int,
end_lineno: int, end_col: int) -> rawptr[AST]:
"""Rewrite a single member-name node into a `Name: t.State` AnnAssign (the uniform enum member form).
target=Name(id, Store), annotation=Attribute(Name('t'), 'State'), value=None, simple=1 --
field-identical to a hand-written `class X(t.CEnum): A: t.State`.
"""
ann: rawptr[AST] = Attribute(ps.pool,
Name(ps.pool, "t", ASTCtx.Load),
"State", ASTCtx.Load)
_set_pos(ann, lineno, col, end_lineno, end_col)
aa: rawptr[AST] = AnnAssign(ps.pool, name_node, ann, None, 1)
_fix_store_ctx(name_node)
_set_pos(aa, lineno, col, end_lineno, end_col)
aa.parent = owner # maintain the parent pointer like AST.append does
return aa
def _convert_enum_bare_members(ps: rawptr[Parser], node: rawptr[ClassDef]):
"""Rewrite bare-name / comma-grouped member lines in an enum body into `Name: t.State` AnnAssigns.
Member collection only recognizes Assign / AnnAssign Name targets, so bare names and comma groups
(`A, B, C`) were silently dropped; now `A` -> `A: t.State` and `A, B, C` -> three AnnAssigns. Valued
or annotated groups are not rewritten here and are caught by the registration-site fail-fast.
"""
children: rawptr[list[rawptr[AST]]] = node.children
if children is None:
return
cn: sizet = len(children)
# Rebuild the child list: one comma-group line -> several AnnAssigns. list has no insert, so rebuild
# entirely; the assignment form matches AST.append in base.vp.
node.children = list[rawptr[AST]](ps.pool, 8)
out: rawptr[list[rawptr[AST]]] = node.children
for i in range(cn):
stmt: rawptr[AST] = children.get(i)
handled: int = 0
if stmt is not None and stmt.kind() == ASTKind.Expr:
ex: rawptr[Expr] = (rawptr[Expr])(stmt)
if ex.value is not None and ex.value.kind() == ASTKind.Name:
out.append(_enum_member_annassign(ps, node, ex.value,
ex.lineno, ex.col_offset,
ex.end_lineno, ex.end_col_offset))
handled = 1
elif ex.value is not None and ex.value.kind() == ASTKind.Tuple:
tup: rawptr[Tuple] = (rawptr[Tuple])(ex.value)
if tup.elts is not None and len(tup.elts) >= 2:
all_names: int = 1
k: sizet = 0
while k < len(tup.elts):
el: rawptr[AST] = tup.elts.get(k)
if el is None or el.kind() != ASTKind.Name:
all_names = 0
break
k += 1
if all_names == 1:
k = 0
while k < len(tup.elts):
el2: rawptr[AST] = tup.elts.get(k)
out.append(_enum_member_annassign(ps, node, el2,
el2.lineno, el2.col_offset,
el2.end_lineno, el2.end_col_offset))
k += 1
handled = 1
if handled == 0 and stmt is not None:
out.append(stmt)
def _parse_define_const(ps: rawptr[Parser]) -> rawptr[AST]:
"""`define NAME[: T] = <literal>` -> an AnnAssign field-identical to `NAME: [T |] t.CDefine = <literal>`.
Same CDefine registration path as hand-written `NAME: t.CDefine = value`. With a type, T goes before
the union (`T | t.CDefine`), matching TPC's expansion. Must end with newline/EOF (fail-fast otherwise).
"""
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps) # define
if _cur_type(ps) != TokenType.Name:
ps.error_count += 1
_record_parse_err(ps)
return None
name: str = _cur_str(ps)
_advance(ps)
explicit_ann: rawptr[AST] = None
if _accept_op(ps, TokOp.Colon):
explicit_ann = _parse_type_ann(ps)
if explicit_ann is None:
return None
if _expect_op(ps, TokOp.Equal) == 0:
return None
value: rawptr[AST] = _parse_test(ps)
if value is None:
ps.error_count += 1
_record_parse_err(ps)
return None
cdefine_marker: rawptr[AST] = Attribute(ps.pool,
Name(ps.pool, "t", ASTCtx.Load),
"CDefine", ASTCtx.Load)
annotation: rawptr[AST] = cdefine_marker
if explicit_ann is not None:
annotation = BinOp(ps.pool, explicit_ann, OpKind.BitOr, cdefine_marker)
target: rawptr[AST] = Name(ps.pool, name, ASTCtx.Store)
_set_pos(target, lineno, col, 0, 0)
node: rawptr[AST] = AnnAssign(ps.pool, target, annotation, value, 1)
_set_pos(node, lineno, col, 0, 0)
_skip_nl(ps)
if _cur_type(ps) != TokenType.NewLine and _cur_type(ps) != TokenType.EndMarker:
_record_parse_err(ps)
ps.error_count += 1
return None
if _cur_type(ps) == TokenType.NewLine:
_advance(ps)
return node
def _parse_typedef(ps: rawptr[Parser]) -> rawptr[AST]:
"""`typedef T1 = T2` -> an AnnAssign field-identical to `T1: t.CTypedef = T2`.
Same typedef registration/lookup path as the hand-written form (CTypedef branch -> register_ctypedef);
no second implementation. Invalid forms (missing name / '=' / RHS) fail loudly; must end with newline/EOF.
"""
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps) # typedef
if _cur_type(ps) != TokenType.Name:
# Missing alias (e.g. `typedef = 1` / `typedef : x`): fail loudly.
_record_parse_err(ps)
ps.error_count += 1
return None
name: str = _cur_str(ps)
_advance(ps)
if _expect_op(ps, TokOp.Equal) == 0:
# Missing '=' (e.g. `typedef A`): fail loudly.
return None
value: rawptr[AST] = _parse_test(ps)
if value is None:
# Missing RHS (e.g. `typedef A = `): fail loudly.
ps.error_count += 1
_record_parse_err(ps)
return None
ctypedef_marker: rawptr[AST] = Attribute(ps.pool,
Name(ps.pool, "t", ASTCtx.Load),
"CTypedef", ASTCtx.Load)
target: rawptr[AST] = Name(ps.pool, name, ASTCtx.Store)
_set_pos(target, lineno, col, 0, 0)
node: rawptr[AST] = AnnAssign(ps.pool, target, ctypedef_marker, value, 1)
_set_pos(node, lineno, col, 0, 0)
_skip_nl(ps)
if _cur_type(ps) != TokenType.NewLine and _cur_type(ps) != TokenType.EndMarker:
_record_parse_err(ps)
ps.error_count += 1
return None
if _cur_type(ps) == TokenType.NewLine:
_advance(ps)
return node
def _parse_export_extern_var(ps: rawptr[Parser]) -> rawptr[AST]:
"""`export NAME: T [= v]` / `extern NAME: T` -> a module-level AnnAssign with storage-class markers.
export rewrites the annotation to `T | t.CExport` (same marker/path as `export def`: bare name, no
module SHA1 header) and allows an initializer; extern rewrites to `T | t.State` (declaration + bare
name) and forbids an initializer (`extern a: int = 1` fails loudly). Must end with newline/EOF.
"""
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
is_extern: int = 0
if _cur_kw(ps) == Keyword.Extern:
is_extern = 1
_advance(ps) # export / extern
if _cur_type(ps) != TokenType.Name:
# Missing variable name (e.g. `export : int`): fail loudly.
_record_parse_err(ps)
ps.error_count += 1
return None
name: str = _cur_str(ps)
_advance(ps)
if _expect_op(ps, TokOp.Colon) == 0:
# Missing ':' (e.g. `export x`): fail loudly.
return None
ann: rawptr[AST] = _parse_type_ann(ps)
if ann is None:
ps.error_count += 1
_record_parse_err(ps)
return None
# Storage-class marker: export -> t.CExport (bare-name export); extern -> t.State (declaration + bare name)
storage_attr: rawptr[AST] = Attribute(ps.pool,
Name(ps.pool, "t", ASTCtx.Load),
"State" if is_extern != 0 else "CExport",
ASTCtx.Load)
annotation: rawptr[AST] = BinOp(ps.pool, ann, OpKind.BitOr, storage_attr)
value: rawptr[AST] = None
if _match_op(ps, TokOp.Equal):
if is_extern != 0:
# An extern variable is externally defined: no initializer allowed in this module.
_RecordDescriptorErr("extern 变量声明不能带初始值（符号定义在外部）",
lineno, col)
ps.error_count += 1
return None
_advance(ps)
value = _parse_test(ps)
if value is None:
ps.error_count += 1
_record_parse_err(ps)
return None
target: rawptr[AST] = Name(ps.pool, name, ASTCtx.Store)
_set_pos(target, lineno, col, 0, 0)
node: rawptr[AST] = AnnAssign(ps.pool, target, annotation, value, 1)
_set_pos(node, lineno, col, 0, 0)
_skip_nl(ps)
if _cur_type(ps) != TokenType.NewLine and _cur_type(ps) != TokenType.EndMarker:
_record_parse_err(ps)
ps.error_count += 1
return None
if _cur_type(ps) == TokenType.NewLine:
_advance(ps)
return node
def _parse_enumdef(ps: rawptr[Parser]) -> rawptr[AST]:
"""enum NAME: sugar -> a ClassDef field-identical to `class NAME(t.CEnum):`.
Member forms (same as TPC's _ExpandEnumBlocks): bare `A` -> `A: t.State` (auto-numbered, i32 width);
comma group `A, B` -> several `A: t.State` / `B: t.State`; annotated `A: T` and valued `A = 0` as-is.
Base is always Attribute(Name('t'), 'CEnum'), field-identical to `class X(t.CEnum):`. Line/col inherited
from each member statement.
"""
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps) # enum
if _cur_type(ps) != TokenType.Name:
ps.error_count += 1
return None
name: str = _cur_str(ps)
_advance(ps)
bases: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
bases.append(Attribute(ps.pool, Name(ps.pool, "t", ASTCtx.Load),
"CEnum", ASTCtx.Load))
keywords: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
dec_list: rawptr[list[rawptr[AST]]] = ps.pending_decorators
ps.pending_decorators = None
_expect_op(ps, TokOp.Colon)
node: rawptr[ClassDef] = ClassDef(ps.pool, name, bases, keywords, dec_list, None)
_parse_suite_into_children(ps, node)
_convert_enum_bare_members(ps, node)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_uniondef(ps: rawptr[Parser]) -> rawptr[AST]:
"""union NAME: sugar -> a ClassDef field-identical to `class NAME(t.CUnion):`.
Members are parsed as a normal class body (AnnAssign). Base is always Attribute(Name('t'), 'CUnion'),
field-identical to `class X(t.CUnion):`; TPC detects unions via `issubclass(cls, t.CUnion)`, so both
sides share the existing union path (offset 0, size = widest member). No bare-name conversion.
"""
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps) # union
if _cur_type(ps) != TokenType.Name:
ps.error_count += 1
return None
name: str = _cur_str(ps)
_advance(ps)
bases: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
bases.append(Attribute(ps.pool, Name(ps.pool, "t", ASTCtx.Load),
"CUnion", ASTCtx.Load))
keywords: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
dec_list: rawptr[list[rawptr[AST]]] = ps.pending_decorators
ps.pending_decorators = None
_expect_op(ps, TokOp.Colon)
node: rawptr[ClassDef] = ClassDef(ps.pool, name, bases, keywords, dec_list, None)
_parse_suite_into_children(ps, node)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_renumdef(ps: rawptr[Parser]) -> rawptr[AST]:
"""renum NAME: sugar -> a ClassDef field-identical to `class NAME(t.REnum):`.
Members are parsed as a normal class body: a variant is a nested ClassDef (`class Some:` + field
annotations) and a payload-less variant is a plain assignment (`None_ = 0`). Base is always
Attribute(Name('t'), 'REnum'); TPC detects REnum via `issubclass(cls, t.REnum)`. No bare-name conversion.
"""
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps) # renum
if _cur_type(ps) != TokenType.Name:
ps.error_count += 1
return None
name: str = _cur_str(ps)
_advance(ps)
bases: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
bases.append(Attribute(ps.pool, Name(ps.pool, "t", ASTCtx.Load),
"REnum", ASTCtx.Load))
keywords: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
dec_list: rawptr[list[rawptr[AST]]] = ps.pending_decorators
ps.pending_decorators = None
_expect_op(ps, TokOp.Colon)
node: rawptr[ClassDef] = ClassDef(ps.pool, name, bases, keywords, dec_list, None)
_parse_suite_into_children(ps, node)
_set_pos(node, lineno, col, 0, 0)
return node
def _RecordStructErr(detail: str, lineno: int, col: int, extra: str):
"""Record an invalid struct form (fail-fast; first only), same reporting path as _RecordVirtualErr.
Messages always use f-string interpolation: this language's str is i8*, so `str + str` degrades to
raw pointer add and illegal IR.
"""
global _g_parse_err_line, _g_parse_err_col, _g_parse_err_msg
if _g_parse_err_line != 0:
return
_g_parse_err_line = lineno
_g_parse_err_col = col
buf: str = stdlib.malloc(1536)
if buf is not None:
stdio.snprintf(buf, 1536,
"[TPV] struct 定义非法（%s）。struct 与 class 同形，但**只能表示数据类型**："
"① 禁止 virtual / novirtual 修饰（struct 天然无虚表，已内置 @t.NoVTable）；"
"② 禁止在 struct 内定义 def（需要方法请改用 class）；"
"③ 允许在 REnum 内作为变体、也允许 struct 内再写 struct。"
"正确示例：struct Point: x: int / struct Point: x: int 换行 4 空格 y: int。%s"
"（struct 的约束不会被静默忽略：非法形态一律终止编译）", detail, extra)
_g_parse_err_msg = buf
else:
_g_parse_err_msg = detail
def _parse_structdef(ps: rawptr[Parser]) -> rawptr[AST]:
"""struct NAME: / struct NAME(Base): sugar -> a ClassDef field-identical to
`@t.NoVTable class NAME(t.CStruct):`.
Data-only: forces no vtable via the pseudo decorator @t.NoVTable, appends Attribute(Name('t'), 'CStruct')
after the user bases as a data-class marker, and forbids def in the body (fail-fast via _RecordStructErr).
Nestable in REnum and in another struct.
"""
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps) # struct
if _cur_type(ps) != TokenType.Name:
_RecordStructErr("`struct` 后缺少类型名", lineno, col,
"正确写法：struct Point:（类型名紧跟 struct）。")
ps.error_count += 1
return None
name: str = _cur_str(ps)
_advance(ps)
# Type parameters [T] / [T, U] (same as class, PEP 695 generics)
type_params: rawptr[list[str]] = None
if _accept_op(ps, TokOp.Lsqb):
type_params = list[str](ps.pool, 8)
while not _match_op(ps, TokOp.Rsqb):
if _cur_type(ps) == TokenType.EndMarker: break
if _cur_type(ps) == TokenType.Name:
tp_name: str = _cur_str(ps)
_advance(ps)
if _match_op(ps, TokOp.Colon):
_advance(ps)
_parse_test(ps)
type_params.append(tp_name)
if not _accept_op(ps, TokOp.Comma):
break
_expect_op(ps, TokOp.Rsqb)
bases: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
keywords: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
# bases / keywords (same parse path as _parse_classdef, including cross-line and keyword=value)
if _accept_op(ps, TokOp.LPar):
while not _match_op(ps, TokOp.RPar):
_skip_nl(ps)
if _cur_type(ps) == TokenType.EndMarker: break
if _cur_type(ps) == TokenType.Name:
e: rawptr[AST] = _parse_or_test(ps)
if _match_op(ps, TokOp.Equal):
_advance(ps)
v: rawptr[AST] = _parse_test(ps)
kw_node: rawptr[AST] = KwNode(ps.pool, e, v)
keywords.append(kw_node)
else:
bases.append(e)
if not _accept_op(ps, TokOp.Comma):
break
_skip_nl(ps)
_expect_op(ps, TokOp.RPar)
# Data-class marker base: appended after the user bases (real parent stays at bases[0])
bases.append(Attribute(ps.pool, Name(ps.pool, "t", ASTCtx.Load),
"CStruct", ASTCtx.Load))
_expect_op(ps, TokOp.Colon)
# Pseudo decorator @t.NoVTable: appended after the user decorators (same merge as _ParseVirtualModified)
dec_list: rawptr[list[rawptr[AST]]] = ps.pending_decorators
ps.pending_decorators = None
marker: rawptr[AST] = Attribute(ps.pool, Name(ps.pool, "t", ASTCtx.Load),
"NoVTable", ASTCtx.Load)
_set_pos(marker, lineno, col, 0, 0)
merged: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
if merged is not None:
if dec_list is not None:
dn_s: sizet = len(dec_list)
for di_s in range(dn_s):
merged.append(dec_list.get(di_s))
merged.append(marker)
node: rawptr[ClassDef] = ClassDef(ps.pool, name, bases, keywords, merged, type_params)
# owner_is_class=1: same as class (ps.in_class_body=1 during the class body)
_parse_suite_into_children(ps, node, 1)
# struct forbids def: check the direct children
chk: rawptr[list[rawptr[AST]]] = node.children
if chk is not None:
cn_s: sizet = len(chk)
for ci_s in range(cn_s):
it_s: rawptr[AST] = chk.get(ci_s)
if it_s is not None and it_s.kind() == ASTKind.FunctionDef:
fd_s: rawptr[FunctionDef] = (rawptr[FunctionDef])(it_s)
fname_s: str = "(anonymous)"
if fd_s is not None and fd_s.name is not None:
fname_s = fd_s.name
_RecordStructErr(f"struct '{name}' 内定义 def '{fname_s}'",
it_s.lineno, it_s.col_offset,
"struct 只表示数据类型（与 C 的 struct 同口径）；"
"若确实需要方法与虚表，请把该类型写成 class。")
ps.error_count += 1
break
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_match(ps: rawptr[Parser]) -> rawptr[AST]:
"""match statement: match subject ':' NewLine Indent case+ Dedent"""
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps) # match
subject: rawptr[AST] = _parse_test(ps)
_expect_op(ps, TokOp.Colon)
cases: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
node: rawptr[AST] = Match(ps.pool, subject, cases, 0)
# Parsing body: Same NewLine/Indent/Dedent pattern as _parse_suite_into_children,
# but parses case clauses into a cases list instead of normal statements into children
_skip_nl(ps)
if _cur_type(ps) == TokenType.NewLine:
_advance(ps)
_skip_nl(ps)
if _cur_type(ps) == TokenType.Indent:
_advance(ps)
while _cur_type(ps) != TokenType.Dedent and _cur_type(ps) != TokenType.EndMarker:
if _cur_kw(ps) == Keyword.Case:
case_node: rawptr[AST] = _parse_match_case(ps)
if case_node is not None:
cases.append(case_node)
else:
_advance(ps) # Skip non-case token to avoid infinite loop
_expect_type(ps, TokenType.Dedent)
_set_pos(node, lineno, col, 0, 0)
return node
# ============================================================
# switch statement (C switch semantics, desugared to Match(is_switch=1)): `switch <expr>:` with
# `case <constant | enum variant>:` / `default:` arms. C switch rules: no fall-through unless
# `fallthrough` is written in the case body. Case labels allow only constants / enum variants /
# multi-value `case 1 | 2:`, not structured patterns.
# ============================================================
def _is_switch_default_kw(ps: rawptr[Parser]) -> int:
"""`default` is a context keyword inside a switch suite: matches `default:` or
`default if <cond>:` (the latter supports a guard like `case _ if <cond>`)."""
if _cur_type(ps) != TokenType.Name:
return 0
s: str = _cur_str(ps)
if s is None or string.strcmp(s, "default") != 0:
return 0
nxt: rawptr[Token] = ps.cur.next
if nxt is None:
return 0
if nxt.type == TokenType.Op and nxt.op_subtype == TokOp.Colon:
return 1
if nxt.type == TokenType.Name and nxt.kw_subtype == Keyword.If:
return 1
return 0
def _RecordSwitchCaseErr(detail: str, lineno: int, col: int):
"""Record an invalid switch case label (fail-fast; first only)"""
global _g_parse_err_line, _g_parse_err_col, _g_parse_err_msg
if _g_parse_err_line != 0:
return
_g_parse_err_line = lineno
_g_parse_err_col = col
buf: str = stdlib.malloc(512)
if buf is not None:
stdio.snprintf(buf, 512,
"switch 的 case 标签非法（%s）。switch 只支持常量字面量或枚举变体标签"
"（可多值 `case 1 | 2:`），不接受序列/映射/捕获等结构化模式。"
"正确示例：case 1: / case \"a\": / case Color.Red: / case Opt.Some():", detail)
_g_parse_err_msg = buf
else:
_g_parse_err_msg = "switch 的 case 标签非法"
def _CheckSwitchCasePattern(ps: rawptr[Parser], pat: rawptr[AST]) -> int:
"""Validate a switch case label (only constants / enum variants / multi-value patterns allowed)"""
if pat is None:
return 0
k: int = pat.kind()
if k == ASTKind.MatchValue or k == ASTKind.MatchSingleton or k == ASTKind.MatchClass:
return 1
if k == ASTKind.MatchAs:
# Only the wildcard (`case _`, equivalent to default) is legal; as-bindings with a subpattern are not
ma_w: rawptr[MatchAs] = rawptr[MatchAs](pat)
if ma_w.pattern is None:
return 1
return 0
if k == ASTKind.MatchOr:
mo: rawptr[MatchOr] = rawptr[MatchOr](pat)
pats: rawptr[list[rawptr[AST]]] = mo.patterns
if pats is None or len(pats) == 0:
return 0
i: sizet = 0
n: sizet = len(pats)
while i < n:
sub: rawptr[AST] = pats.get(i)
if sub is None:
return 0
sk: int = sub.kind()
if sk != ASTKind.MatchValue and sk != ASTKind.MatchSingleton \
and sk != ASTKind.MatchClass:
return 0
i = i + 1
return 1
return 0
def _parse_switch_case(ps: rawptr[Parser]) -> rawptr[AST]:
"""switch `case <constant|enum variant> [if <cond>] ':' suite` (guard supported).
A bare name is a capture pattern (MatchAs) in match but a value reference in switch (like C's
`case NAME:`), so `MatchAs(pattern=None, name=X)` is normalized to `MatchValue(Name('X'))`; `_`
stays a wildcard. The guard is parsed like _parse_match_case and passed to MatchCase.guard.
"""
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps) # case
pattern: rawptr[AST] = _parse_pattern(ps)
if pattern is not None and pattern.kind() == ASTKind.MatchAs:
ma: rawptr[MatchAs] = rawptr[MatchAs](pattern)
if ma.pattern is None and ma.name is not None:
ref: rawptr[AST] = Name(ps.pool, ma.name, ASTCtx.Load)
_set_pos(ref, pattern.lineno, pattern.col_offset, 0, 0)
mv: rawptr[AST] = MatchValue(ps.pool, ref)
_set_pos(mv, pattern.lineno, pattern.col_offset, 0, 0)
pattern = mv
if _CheckSwitchCasePattern(ps, pattern) == 0:
_RecordSwitchCaseErr("case 标签不是常量/枚举变体", lineno, col)
ps.error_count += 1
# guard: `case <value> if <cond> ':'` (same form as _parse_match_case)
guard: rawptr[AST] = None
if _cur_kw(ps) == Keyword.If:
_advance(ps)
guard = _parse_or_test(ps)
_expect_op(ps, TokOp.Colon)
node: rawptr[AST] = MatchCase(ps.pool, pattern, guard)
_parse_suite_into_children(ps, node)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_switch_default(ps: rawptr[Parser]) -> rawptr[AST]:
"""switch `default [if <cond>] ':' suite` -- desugared to `case [<guard>]` (MatchAs wildcard).
`default` is switch's spelling of `case _`; it supports a guard like `case _ if <cond>`, passed
to MatchCase. Without if the behavior is unchanged (guard=None).
"""
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps) # default
guard: rawptr[AST] = None
if _cur_kw(ps) == Keyword.If:
_advance(ps)
guard = _parse_or_test(ps)
_expect_op(ps, TokOp.Colon)
wild: rawptr[AST] = MatchAs(ps.pool, None, None)
_set_pos(wild, lineno, col, 0, 0)
node: rawptr[AST] = MatchCase(ps.pool, wild, guard)
_parse_suite_into_children(ps, node)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_switch(ps: rawptr[Parser]) -> rawptr[AST]:
"""switch statement: switch subject ':' NewLine Indent (case|default)+ Dedent"""
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps) # switch
subject: rawptr[AST] = _parse_test(ps)
_expect_op(ps, TokOp.Colon)
cases: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
node: rawptr[AST] = Match(ps.pool, subject, cases, 1)
_skip_nl(ps)
if _cur_type(ps) == TokenType.NewLine:
_advance(ps)
_skip_nl(ps)
if _cur_type(ps) == TokenType.Indent:
_advance(ps)
while _cur_type(ps) != TokenType.Dedent and _cur_type(ps) != TokenType.EndMarker:
if _cur_kw(ps) == Keyword.Case:
case_node: rawptr[AST] = _parse_switch_case(ps)
if case_node is not None:
cases.append(case_node)
elif _is_switch_default_kw(ps) != 0:
def_node: rawptr[AST] = _parse_switch_default(ps)
if def_node is not None:
cases.append(def_node)
else:
_advance(ps) # Skip non-case token to avoid infinite loop
_expect_type(ps, TokenType.Dedent)
_set_pos(node, lineno, col, 0, 0)
return node
# ============================================================
# Match pattern parsing
# ============================================================
def _make_name_ast(pool: rawptr[memhub.MemManager], node: rawptr[Name]) -> rawptr[AST]:
# Analysis: "Convert Name* to AST* (Solving the TransPyC Type Inference Problem)"
return node
def _parse_pattern(ps: rawptr[Parser]) -> rawptr[AST]:
"""pattern: or_pattern ['as' NAME]
or_pattern: closed_pattern ('|' closed_pattern)*"""
pat: rawptr[AST] = _parse_closed_pattern(ps)
if pat is None: return None
# OR pattern
if _match_op(ps, TokOp.VBar):
patterns: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
patterns.append(pat)
while _accept_op(ps, TokOp.VBar):
p: rawptr[AST] = _parse_closed_pattern(ps)
if p is not None:
patterns.append(p)
or_node: rawptr[AST] = MatchOr(ps.pool, patterns)
_set_pos(or_node, pat.lineno, pat.col_offset, 0, 0)
pat = or_node
# AS pattern
if _cur_kw(ps) == Keyword.As:
_advance(ps)
if _cur_type(ps) != TokenType.Name:
ps.error_count += 1
return pat
name_str: str = _cur_str(ps)
_advance(ps)
as_node: rawptr[AST] = MatchAs(ps.pool, pat, name_str)
_set_pos(as_node, pat.lineno, pat.col_offset, 0, 0)
return as_node
return pat
def _parse_closed_pattern(ps: rawptr[Parser]) -> rawptr[AST]:
"""closed_pattern: literal | capture | wildcard | value | group | sequence | mapping | class"""
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
ttype: int = _cur_type(ps)
val: rawptr[AST]
node: rawptr[AST]
tok: rawptr[Token]
sval: str
# Number literal pattern
if ttype == TokenType.Number:
tok = ps.cur
_advance(ps)
if tok.is_float:
val = _new_const_float(ps.pool, tok.float_val, lineno, col)
else:
val = _new_const_int(ps.pool, tok.int_val, lineno, col)
node = MatchValue(ps.pool, val)
_set_pos(node, lineno, col, 0, 0)
return node
# String literal pattern
if ttype == TokenType.String:
sval = _cur_str(ps)
is_dquote_m: int = ps.cur.kw_subtype & 16
_advance(ps)
val = _new_const_str(ps.pool, sval, lineno, col, is_dquote_m)
node = MatchValue(ps.pool, val)
_set_pos(node, lineno, col, 0, 0)
return node
# Negative number literal pattern: -1, -1.5
if _match_op(ps, TokOp.Minus):
_advance(ps)
if _cur_type(ps) == TokenType.Number:
tok = ps.cur
# Combined unary-minus + integer-literal overflow check (same rule as _parse_factor):
# a negative literal with magnitude > 2^63 yields < -2^63 -> fail-fast, no silent wrap.
if tok.is_float == 0 and _lex_int_neg_exceeds_i64(_cur_str(ps)) != 0:
_record_neg_literal_overflow(_cur_str(ps), lineno, col)
ps.error_count += 1
return None
_advance(ps)
if tok.is_float:
val = _new_const_float(ps.pool, -tok.float_val, lineno, col)
else:
val = _new_const_int(ps.pool, -tok.int_val, lineno, col)
node = MatchValue(ps.pool, val)
_set_pos(node, lineno, col, 0, 0)
return node
ps.error_count += 1
return None
# Name: None/True/False/_ /capture/value/class
if ttype == TokenType.Name:
kw: int = _cur_kw(ps)
# None/True/False → MatchSingleton
if kw == Keyword.None_ or kw == Keyword.True_ or kw == Keyword.False_:
_advance(ps)
if kw == Keyword.None_:
val = _new_const_none(ps.pool, lineno, col)
elif kw == Keyword.True_:
val = _new_const_bool(ps.pool, 1, lineno, col)
else:
val = _new_const_bool(ps.pool, 0, lineno, col)
node = MatchSingleton(ps.pool, val)
_set_pos(node, lineno, col, 0, 0)
return node
name_str: str = _cur_str(ps)
# Wildcard pattern: _
if string.strcmp(name_str, "_") == 0:
_advance(ps)
node = MatchAs(ps.pool, None, None)
_set_pos(node, lineno, col, 0, 0)
return node
_advance(ps)
# Check dotted name (value pattern or class pattern)
if _match_op(ps, TokOp.Dot):
name_node: rawptr[Name] = Name(ps.pool, name_str, ASTCtx.Load)
_set_pos(name_node, lineno, col, 0, 0)
base: rawptr[AST] = _make_name_ast(ps.pool, name_node)
while _match_op(ps, TokOp.Dot):
_advance(ps)
if _cur_type(ps) != TokenType.Name:
ps.error_count += 1
break
attr_name: str = _cur_str(ps)
attr_lineno: int = _cur_lineno(ps)
attr_col: int = _cur_col(ps)
_advance(ps)
base = Attribute(ps.pool, base, attr_name, ASTCtx.Load)
_set_pos(base, attr_lineno, attr_col, 0, 0)
# class pattern: dotted.name(...)
if _match_op(ps, TokOp.LPar):
return _parse_class_pattern(ps, base, lineno, col)
# value pattern: dotted.name
node = MatchValue(ps.pool, base)
_set_pos(node, lineno, col, 0, 0)
return node
# class pattern: Name(...)
if _match_op(ps, TokOp.LPar):
cls_node: rawptr[Name] = Name(ps.pool, name_str, ASTCtx.Load)
_set_pos(cls_node, lineno, col, 0, 0)
cls: rawptr[AST] = _make_name_ast(ps.pool, cls_node)
return _parse_class_pattern(ps, cls, lineno, col)
# capture pattern: NAME
node = MatchAs(ps.pool, None, name_str)
_set_pos(node, lineno, col, 0, 0)
return node
# group pattern / sequence pattern (tuple): (...)
if _match_op(ps, TokOp.LPar):
_advance(ps)
inner: rawptr[AST] = _parse_pattern(ps)
if _accept_op(ps, TokOp.Comma):
# sequence pattern (tuple)
patterns: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
if inner is not None:
patterns.append(inner)
while not _match_op(ps, TokOp.RPar) and _cur_type(ps) != TokenType.EndMarker:
p: rawptr[AST] = _parse_pattern(ps)
if p is not None:
patterns.append(p)
if not _accept_op(ps, TokOp.Comma):
break
_expect_op(ps, TokOp.RPar)
node = MatchSequence(ps.pool, patterns)
_set_pos(node, lineno, col, 0, 0)
return node
_expect_op(ps, TokOp.RPar)
return inner # group pattern
# sequence pattern (list): [...]
if _match_op(ps, TokOp.Lsqb):
return _parse_sequence_pattern(ps, TokOp.Rsqb, lineno, col)
# mapping pattern: {...}
if _match_op(ps, TokOp.LBrace):
return _parse_mapping_pattern(ps, lineno, col)
ps.error_count += 1
return None
def _parse_sequence_pattern(ps: rawptr[Parser], close_op: int,
lineno: int, col: int) -> rawptr[AST]:
"""sequence_pattern: '[' [pattern_list] ']'"""
patterns: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
_advance(ps) # [
while not _match_op(ps, close_op) and _cur_type(ps) != TokenType.EndMarker:
# *name (star pattern)
if _match_op(ps, TokOp.Star):
_advance(ps)
if _cur_type(ps) == TokenType.Name:
star_name: str = _cur_str(ps)
_advance(ps)
star_node: rawptr[AST]
if string.strcmp(star_name, "_") == 0:
star_node = MatchStar(ps.pool, None)
else:
star_node = MatchStar(ps.pool, star_name)
_set_pos(star_node, lineno, col, 0, 0)
patterns.append(star_node)
else:
p: rawptr[AST] = _parse_pattern(ps)
if p is not None:
patterns.append(p)
if not _accept_op(ps, TokOp.Comma):
break
_expect_op(ps, close_op)
node: rawptr[AST] = MatchSequence(ps.pool, patterns)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_mapping_pattern(ps: rawptr[Parser],
lineno: int, col: int) -> rawptr[AST]:
"""mapping_pattern: '{' [items] '}'
items: key ':' pattern (',' key ':' pattern)* [',' '**' NAME] | '**' NAME"""
keys: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
patterns: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
rest: str = None
node: rawptr[AST]
key: rawptr[AST]
val_pat: rawptr[AST]
_advance(ps) # {
if _match_op(ps, TokOp.RBrace):
_advance(ps)
node = MatchMapping(ps.pool, keys, patterns, rest)
_set_pos(node, lineno, col, 0, 0)
return node
while _cur_type(ps) != TokenType.EndMarker:
if _match_op(ps, TokOp.StarStar):
_advance(ps)
if _cur_type(ps) == TokenType.Name:
rest = _cur_str(ps)
_advance(ps)
break
key = _parse_expr(ps)
_expect_op(ps, TokOp.Colon)
val_pat = _parse_pattern(ps)
if key is not None:
keys.append(key)
if val_pat is not None:
patterns.append(val_pat)
if not _accept_op(ps, TokOp.Comma):
break
_expect_op(ps, TokOp.RBrace)
node = MatchMapping(ps.pool, keys, patterns, rest)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_class_pattern(ps: rawptr[Parser], cls: rawptr[AST],
lineno: int, col: int) -> rawptr[AST]:
"""class_pattern: cls '(' [pattern_list] ')'
pattern_list: positional_pattern (',' positional_pattern)*
[',' keyword_pattern]* | keyword_pattern (',' keyword_pattern)*
keyword_pattern: NAME '=' pattern"""
patterns: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
kwd_attrs: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
kwd_patterns: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
p: rawptr[AST]
_advance(ps) # (
if not _match_op(ps, TokOp.RPar):
while _cur_type(ps) != TokenType.EndMarker:
# keyword pattern: NAME '=' pattern
if _cur_type(ps) == TokenType.Name and ps.cur is not None:
nxt: rawptr[Token] = ps.cur.next
if nxt is not None and nxt.type == TokenType.Op and nxt.op_subtype == TokOp.Equal:
attr_name: str = _cur_str(ps)
_advance(ps) # NAME
_advance(ps) # =
kwd_pat: rawptr[AST] = _parse_pattern(ps)
attr_n: rawptr[Name] = Name(ps.pool, attr_name, ASTCtx.Load)
_set_pos(attr_n, lineno, col, 0, 0)
attr_node: rawptr[AST] = _make_name_ast(ps.pool, attr_n)
kwd_attrs.append(attr_node)
if kwd_pat is not None:
kwd_patterns.append(kwd_pat)
else:
p = _parse_pattern(ps)
if p is not None:
patterns.append(p)
else:
p = _parse_pattern(ps)
if p is not None:
patterns.append(p)
if not _accept_op(ps, TokOp.Comma):
break
_expect_op(ps, TokOp.RPar)
node: rawptr[AST] = MatchClass(ps.pool, cls, patterns, kwd_attrs, kwd_patterns)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_match_case(ps: rawptr[Parser]) -> rawptr[AST]:
"""case pattern [if guard] ':' suite"""
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps) # case
guard: rawptr[AST] = None
pattern: rawptr[AST] = _parse_pattern(ps)
if _cur_kw(ps) == Keyword.If:
_advance(ps)
guard = _parse_or_test(ps)
_expect_op(ps, TokOp.Colon)
node: rawptr[AST] = MatchCase(ps.pool, pattern, guard)
_parse_suite_into_children(ps, node)
_set_pos(node, lineno, col, 0, 0)
return node
# ============================================================
# __attribute__ decorator desugar (C-style attribute syntax): `@__attribute__(section(".text.wo"),
# always_inline)` equals hand-written `@c.Attribute(t.attr.section(...), t.attr.always_inline)`.
# Arguments use unprefixed names, each desugared to `t.attr.<name>` (calls keep their args) with the
# outer call becoming `c.Attribute(...)`, so the result is field-identical and recognized as-is.
# Works for class and def; arguments already in `t.attr.X` / `c.Attribute(...)` form are kept.
# ============================================================
def _IsMagicAttributeCallee(node: AST*) -> int:
"""Whether the node is `__attribute__` (Name form, or the trailing X.__attribute__ form)"""
if node is None:
return 0
k: int = node.kind()
if k == ASTKind.Name:
nm: Name* = Name*(node)
if nm.id is not None and string.strcmp(nm.id, "__attribute__") == 0:
return 1
return 0
if k == ASTKind.Attribute:
at: Attribute* = Attribute*(node)
if at.attr is not None and string.strcmp(at.attr, "__attribute__") == 0:
return 1
return 0
def _MakeTAttrMarker(ps: rawptr[Parser], arg: AST*) -> AST*:
"""Desugar one __attribute__ argument into `t.attr.<name>` (calls keep their arguments).
`always_inline` (Name) -> `t.attr.always_inline`; `section(".text.wo")` (Call) -> `t.attr.section(...)`;
an argument already in `t.attr.X` form is returned as-is.
"""
if arg is None:
return None
k: int = arg.kind()
if k == ASTKind.Attribute:
at0: Attribute* = Attribute*(arg)
if at0.value is not None and at0.value.kind() == ASTKind.Attribute:
return arg
return arg
attr_root: AST* = Attribute(ps.pool, Name(ps.pool, "t", ASTCtx.Load),
"attr", ASTCtx.Load)
if k == ASTKind.Name:
nm2: Name* = Name*(arg)
if nm2.id is None:
return arg
mk: AST* = Attribute(ps.pool, attr_root, nm2.id, ASTCtx.Load)
_set_pos(mk, nm2.lineno, nm2.col_offset, 0, 0)
return mk
if k == ASTKind.Call:
cl2: Call* = Call*(arg)
if cl2.func is None or cl2.func.kind() != ASTKind.Name:
return arg
fnm: Name* = Name*(cl2.func)
if fnm.id is None:
return arg
inner: AST* = Attribute(ps.pool, attr_root, fnm.id, ASTCtx.Load)
_set_pos(inner, fnm.lineno, fnm.col_offset, 0, 0)
mk2: AST* = Call(ps.pool, inner, cl2.args, cl2.keywords)
_set_pos(mk2, cl2.lineno, cl2.col_offset, 0, 0)
return mk2
return arg
def _DesugarMagicAttribute(ps: rawptr[Parser], expr: AST*) -> AST*:
"""`@__attribute__(...)` -> `@c.Attribute(t.attr.x, ...)`; other decorators are returned as-is"""
if expr is None or expr.kind() != ASTKind.Call:
return expr
cl: Call* = Call*(expr)
if _IsMagicAttributeCallee(cl.func) == 0:
return expr
c_func: AST* = Attribute(ps.pool, Name(ps.pool, "c", ASTCtx.Load),
"Attribute", ASTCtx.Load)
new_args: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
if new_args is None:
return expr
if cl.args is not None:
an: sizet = len(cl.args)
for ai in range(an):
mk: AST* = _MakeTAttrMarker(ps, cl.args.get(ai))
if mk is not None:
new_args.append(mk)
new_call: AST* = Call(ps.pool, c_func, new_args, None)
_set_pos(new_call, cl.lineno, cl.col_offset, 0, 0)
return new_call
def _parse_decorators(ps: rawptr[Parser]) -> rawptr[list[rawptr[AST]]]:
"""Parse a decorator chain, returning a list."""
decs: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
while _match_op(ps, TokOp.At):
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps) # @
expr: rawptr[AST] = _parse_namedexpr(ps)
# C-style `@__attribute__(...)` desugars into the existing `@c.Attribute(t.attr.<x>...)`
expr = _DesugarMagicAttribute(ps, expr)
_skip_nl(ps)
if _cur_type(ps) == TokenType.NewLine:
_advance(ps)
decs.append(expr)
return decs
# ============================================================
# Module entry
# ============================================================
def _parse_module_body(ps: rawptr[Parser], module_node: rawptr[AST]):
"""Parse the module body, appending to module_node."""
_skip_nl(ps)
stmt_idx: int = 0
while _cur_type(ps) != TokenType.EndMarker:
stmt_idx += 1
# fail-fast anti-hang: over-budget steps (cyclic token list / cursor spinning) report and return
if ps.step_count > ps.step_limit:
_record_parse_err(ps)
return
# Decorator
if _match_op(ps, TokOp.At):
ps.pending_decorators = _parse_decorators(ps)
# Record current token pointer to check for infinite loop
before_cur: rawptr[Token] = ps.cur
_parse_statement_into(ps, module_node)
# Defense: If parsing doesn't advance the token (like when _parse_atom encounters an unrecognized token and returns None),
# force it to move forward to avoid an infinite loop in the main loop; accumulate error_count for the caller to diagnose
if ps.cur is before_cur:
ps.error_count += 1
_advance(ps)
_skip_nl(ps)
def parse_tokens(pool: rawptr[memhub.MemManager],
tokens: rawptr[Token]) -> rawptr[AST]:
"""Parse an AST Module from a token list."""
if pool is None or tokens is None:
return None
global _g_parse_err_line, _g_parse_err_col, _g_parse_err_msg
_g_parse_err_line = 0
_g_parse_err_col = 0
_g_parse_err_msg = None
# Lexer fail-fast takes priority: overflows recorded during tokenize (e.g. integer literal > 2^64-1)
# are forwarded to the parse error channel here; lexer has no independent fatal exit.
lex_line: int = get_lex_err_line()
if lex_line != 0:
_g_parse_err_line = lex_line
_g_parse_err_col = get_lex_err_col()
_g_parse_err_msg = get_lex_err_msg()
return None
ps: rawptr[Parser] = new_parser(pool)
if ps is None: return None
_parser_init(ps, tokens, pool)
# fail-fast anti-hang: count the token list first, detecting a cycle. A cyclic list (Token appended
# twice or aliased) would hang the statement loop; exceeding the cap before the tail means a cycle.
tok_count: sizet = 0
walk_t: rawptr[Token] = tokens
while walk_t is not None:
walk_t = walk_t.next
tok_count += 1
if tok_count > PARSER_MAX_TOKENS:
_g_parse_err_line = _cur_lineno(ps)
_g_parse_err_col = _cur_col(ps)
_g_parse_err_msg = "token 链表成环：lexer 输出的 token 链表无终止（_emit 重复追加同一 Token 或分配别名），已终止解析"
return None
# Budget: the watchdog counts "query current token" calls (well below 256 per token in valid parses,
# so it cannot trip falsely) while catching cycles/spins; it also bounds _advance and statement loops.
ps.step_limit = tok_count * 256 + 262144
module_node: rawptr[Module] = Module(pool)
_parse_module_body(ps, module_node)
# fail-fast: on a parse error (error_count > 0) return None so the caller aborts, avoiding silently
# compiling broken code (e.g. a function definition left without its def keyword).
if ps.error_count > 0:
# Fallback position/reason: the parser has 40+ `ps.error_count += 1` sites and not all call
# _record_*. Without a record, dump_parse_traceback can only print `File "..."` +
# `SyntaxError` with no line or reason. Fill in a locatable message from the current token so the
# report always has file:line:col + reason.
if _g_parse_err_line == 0:
_g_parse_err_line = _cur_lineno(ps)
_g_parse_err_col = _cur_col(ps)
err_s_e: str = _cur_str(ps)
if err_s_e is None:
err_s_e = "(none)"
buf_e: str = stdlib.malloc(256)
if buf_e is not None:
stdio.snprintf(buf_e, 256,
"语法错误：解析器已记录 %d 处错误但未记录具体原因；"
"出错位置附近的 token='%s'（type=%d）",
ps.error_count, err_s_e, _cur_type(ps));
_g_parse_err_msg = buf_e
else:
_g_parse_err_msg = "语法错误：解析器未记录具体原因"
return None
return module_node
Reference in new issue View Git Blame Copy Permalink