GVSDS/Viper_VRTE
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
6e3231d04c67eedffef3b1ab2b4264cd25d094ed
Viper_VRTE/ast/parser.vp
T
TermiNexus 6e3231d04c feat(parser): 支持 static 修饰符（软关键字，归一为 t.CStatic 标记） 
两种写法同源，最终都产出与手写 @t.CStatic / x: t.CStatic|T 逐字段同形的标记节点，故 TransPyV 侧零新增解析判据：

- static def f(...)：消费 static 后合成 @t.CStatic 装饰器（与 inline def 同一套 pending_decorators 通道，见 _ParseStaticModified）。

- static x: int = 0：置 Parser.pending_static，由 _parse_type_ann 在描述符序列首位插入 t.CStatic（= 手写 t.CStatic | int），随即清零只作用于紧随的一个注解。

- 软关键字边界：只在语句起始、且后随 def 或 Name: 时命中（_IsStaticModifier）；作标识符的位置（属性名/形参名/表达式内）不经过该判据。

语义（禁止跨模块使用）由 TransPyV 侧消费。
2026-09-27 12:58:44 +08:00

4378 lines
185 KiB
Plaintext
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import t, c
from stdint import *
import memhub
import string
import stdio
import stdlib
from .tokens import (Token, TokenType, Keyword, TokOp)
from .base import (
AST, ASTCtx, ASTKind, OpKind, ASTFlag,
_set_pos, _binop_from_op, _augop_from_op, _cmpop_from_op, _copy_str, _inherit_pos,
_init_ast,
CONST_INT, CONST_FLOAT, CONST_STR, CONST_BOOL, CONST_NONE, CONST_CHAR,
FLAG_IS_ASYNC, FLAG_IS_EXTERN, FLAG_IS_EXPORT, FLAG_IS_DEFINE,
)
from .exprs import (
BoolOp, BinOp, UnaryOp, Lambda, IfExp, Dict, Set,
ListComp, SetComp, DictComp, GeneratorExp,
Await, Yield, YieldFrom, FormattedValue, JoinedStr,
Constant, NamedExpr, Attribute, Subscript, Starred, Name,
List, Tuple, Slice, Call, Compare, OpNode,
)
from .stmts import (
Module, Expression, Interactive, FunctionType,
FunctionDef, ClassDef, Return, Delete, Assign, AugAssign, AnnAssign,
For, While, If, With, Raise, Try, Assert, Global, Nonlocal,
Pass, Break, Continue, Expr, Import, ImportFrom, Match,
)
from .astaux import (
ExceptHandler, Arguments, Arg, Keyword as KwNode, Alias, WithItem, Comprehension,
)
from .match import (
MatchCase, MatchValue, MatchSingleton, MatchSequence,
MatchMapping, MatchClass, MatchStar, MatchAs, MatchOr,
)
from .lexer import (
Lexer, _lexer_init, tokenize, new_lexer,
get_lex_err_line, get_lex_err_col, get_lex_err_msg,
_lex_int_neg_exceeds_i64,
)
# ============================================================
# Parser State
# ============================================================
# token 链表长度上限（成环检测的硬闸）：正常源码的 token 数远低于此（200KB 源文件
# 约数万 token）；走到这个数还没到尾即判为链表成环（见 parse_tokens）。
define PARSER_MAX_TOKENS = 8000000
class Parser:
tokens: rawptr[Token] # token linked list head
cur: rawptr[Token] # current token
prev: rawptr[Token] # 上一个被消费的 token（供 `T*(x)` 紧邻判据）
pool: rawptr[memhub.MemManager] # memory pool
error_count: int # error count
pending_decorators: rawptr[list[rawptr[AST]]] # pending decorators list
# `static` 修饰符（软关键字）挂起标记：语句起始消费 `static`（后随变量声明
# `Name :`）后置 1，由紧随的 `_parse_type_ann` 在类型联合最左插入 t.CStatic
# 标记（`static x: int = 0` → `x: t.CStatic | int = 0`），插入后立即清零，
# 只作用于紧随其后的这一个注解。函数分支不经此通道（改走合成 @t.CStatic
# 装饰器，与 inline def 同构，见 _ParseStaticModified）。
pending_static: int
# 当前正在解析的套件的**直接拥有者**是否 class（1=类体，0=其它）。
# 由 _parse_suite_into_children 按 owner_is_class 维护，供 virtual/novirtual
# 修饰符判"方法级修饰符是否直接写在类体内"（见 _ParseVirtualModified）。
in_class_body: int
# 步数预算（_advance 计数 / parse_tokens 预算）：语句主循环一旦超预算即
# fail-fast 报错返回，绝不无限循环（见 parse_tokens 的成环检测）。
step_count: sizet
step_limit: sizet
# 看门狗：对"当前 token 查询"的计数与收束标志（见 _tick）。
# tick_count 超 step_limit ⇒ aborted=1，解析各入口改返回收束值，
# 所有 while 循环常数轮内结束（任何原地打转都不可能存活）。
tick_count: sizet
aborted: int
def __new__(self, pool: rawptr[memhub.MemManager]):
ptr: rawptr[Parser] = pool.alloc(Parser.__sizeof__())
if ptr: string.memset(ptr, 0, Parser.__sizeof__())
return ptr
def new_parser(pool: rawptr[memhub.MemManager]) -> rawptr[Parser]:
"""Allocate from the pool and return the Parser (cross-module safe wrapper)"""
return Parser(pool)
def _parser_init(ps: rawptr[Parser], tokens: rawptr[Token],
pool: rawptr[memhub.MemManager]):
if ps is None: return
ps.tokens = tokens
ps.cur = tokens
ps.prev = None
ps.pool = pool
ps.error_count = 0
ps.pending_decorators = None
ps.pending_static = 0
ps.in_class_body = 0
ps.step_count = 0
# 默认预算取上限（parse_tokens 会按实际 token 数收紧）：保证不经过 parse_tokens
# 的入口（parse_expression 等）不会因预算为 0 而误报；同时仍有硬上界防死循环。
ps.step_limit = PARSER_MAX_TOKENS
ps.tick_count = 0
ps.aborted = 0
# ============================================================
# 解析看门狗（静态收口：杜绝"无输出卡死"）
#
# 任何"原地打转、一个 token 也不消费"的解析循环，都必然反复查询当前 token；因此把
# 「查询当前 token」的每个入口（_cur_type / _cur_op / _cur_kw / _match_op /
# _accept_op / _advance）都记一次 tick，超出预算即置 aborted + 记录解析错误，并让
# 这些入口返回**收束值**：
# _cur_type → EndMarker（结束 `while _cur_type != X` 类循环）
# _cur_op / _cur_kw → 0（结束 `while True` + 运算符不匹配则 break 类循环）
# _match_op / _accept_op → 还有 token 则返回 1 并一次性跳到末尾（结束
# `while not _match_op(...)` 与 `while _match_op(...)` 两类循环，≤2 轮）
# _advance → 直接跳到末尾
# 于是 parse_tokens 以 error_count > 0 返回 None → 调用方按 SyntaxError 报
# 文件:行:列 终止编译，**不再存在任何静默卡死路径**。
# ============================================================
def _tick(ps: rawptr[Parser]):
"""推进看门狗；超预算即置 aborted 并记录首个解析错误（含当前位置）"""
if ps is None: return
if ps.aborted != 0: return
ps.tick_count += 1
if ps.tick_count > ps.step_limit:
ps.aborted = 1
_record_parse_err(ps)
ps.error_count += 1
# ============================================================
# Token Navigation Helpers
# ============================================================
def _advance(ps: rawptr[Parser]):
"""Advance to the next token"""
_tick(ps)
if ps.aborted != 0:
ps.cur = None
return
if ps.cur is None: return
if ps.cur.next is not None:
ps.prev = ps.cur
ps.cur = ps.cur.next
ps.step_count += 1
def _cur_type(ps: rawptr[Parser]) -> int:
_tick(ps)
if ps.aborted != 0: return TokenType.EndMarker
if ps.cur is None: return TokenType.EndMarker
return ps.cur.type
def _cur_op(ps: rawptr[Parser]) -> int:
_tick(ps)
if ps.aborted != 0: return 0
if ps.cur is None: return 0
return ps.cur.op_subtype
def _cur_kw(ps: rawptr[Parser]) -> int:
_tick(ps)
if ps.aborted != 0: return 0
if ps.cur is None: return 0
return ps.cur.kw_subtype
def _is_funcdef_kw(ps: rawptr[Parser], kw: int) -> int:
"""判断当前 token 是否为函数定义关键字（def 及其别名 fn/fun/func/function，
以及 extern/export 声明关键字）
def 是硬关键字直接判定；fn/fun/func/function/extern/export 是软关键字，
仅当下一个 token 是 Name 且再下一个是 '('（或泛型 '[T]' 前的 '['）时
判定为函数定义。extern 还允许 'extern def A(...)' 形式（下一个是 def），
export 同样允许 'export def A(...)' 形式。
否则按普通标识符处理，兼容既有代码把 func 用作
字段/参数名（如 AST 节点 func 字段、Call.func 注解 `func: rawptr[AST]`）。
"""
if kw == Keyword.Def:
return 1
if kw == Keyword.Extern or kw == Keyword.Export:
# extern/export def A(...) 或 extern/export A(...)
# （extern/export 后是 def 或 Name+'(/'['）
nxt: rawptr[Token] = ps.cur.next
if nxt is None: return 0
if nxt.kw_subtype == Keyword.Def: return 1
if nxt.type != TokenType.Name: return 0
nxt2: rawptr[Token] = nxt.next
if nxt2 is None: return 0
if nxt2.type == TokenType.Op and (nxt2.op_subtype == TokOp.LPar or nxt2.op_subtype == TokOp.Lsqb):
return 1
return 0
if kw == Keyword.Fn or kw == Keyword.Fun \
or kw == Keyword.Func or kw == Keyword.Function \
or kw == Keyword.Define:
# 软关键字 lookahead: 下个是 Name，再下个是 '(' 或 '[T...]' 泛型参数（PEP 695）
# （define 宏函数与 def 语义一致，返回注解自动追加 t.CDefine）
nxt: rawptr[Token] = ps.cur.next
if nxt is None:
return 0
if nxt.type != TokenType.Name:
return 0
nxt2: rawptr[Token] = nxt.next
if nxt2 is None:
return 0
if nxt2.type == TokenType.Op \
and (nxt2.op_subtype == TokOp.LPar or nxt2.op_subtype == TokOp.Lsqb):
return 1
return 0
return 0
def _is_define_const(ps: rawptr[Parser]) -> int:
"""define 常量声明的软关键字 lookahead（1=是，0=否）。
`define NAME[: T] = <字面量>`：下一个 token 是 Name，再下一个是 '='（无注解）
或 ':'（带注解）。
- `define NAME(...)`（宏函数，`_is_funcdef_kw` 的既有形态）第二个 token 是
'(' / '[' ⇒ 此处不命中，宏函数语义逐字不变；
- 把 define 当标识符的写法（`define = 5` / `define(x)`）同样不命中；
- `define NAME: T`（**缺 `=`**）会命中（下一个是 ':'），随后在
`_parse_define_const` 里 `_expect_op(Equal)` 失败 ⇒ 响亮失败（非静默）。
"""
if _cur_kw(ps) != Keyword.Define: return 0
nxt: rawptr[Token] = ps.cur.next
if nxt is None or nxt.type != TokenType.Name: return 0
nxt2: rawptr[Token] = nxt.next
if nxt2 is None or nxt2.type != TokenType.Op: return 0
if nxt2.op_subtype == TokOp.Equal or nxt2.op_subtype == TokOp.Colon: return 1
return 0
def _is_typedef(ps: rawptr[Parser]) -> int:
"""typedef 类型别名声明的软关键字判定（1=是，0=否）。
`typedef T1 = T2` 与 `T1: t.CTypedef = T2` **完全等价**（同族同风格于
`enum`/`define`：展开为既有形态，落到同一条 typedef 注册/查找路径）。
`typedef` 是**语句级保留字**：全仓冲突普查确认它只在注释/文档字符串中出现，
没有任何变量/参数/字段/函数/类/模块以其命名 ⇒ 语句起始出现 `typedef` 即按
本语法处理；非法形态（`typedef = 1` / `typedef A` / `typedef A = ` 缺 RHS）
在 `_parse_typedef` 内 `_record_parse_err` **响亮失败**（非静默）。
"""
if _cur_kw(ps) != Keyword.Typedef:
return 0
return 1
def _is_match_stmt(ps: rawptr[Parser]) -> int:
"""match 是软关键字（CPython 3.10+）：lookahead 判断是否为 match 语句
match 后直接跟 ':'（变量注解，如 lexer.vp 的 `match: int = 1`）、
'='（赋值，如 `match = 0`）、换行/文件尾 → 按普通标识符；
否则扫描至行尾，在括号深度 0 处出现 ':' → match 语句（subject 后必须冒号）。
case/type 同理为软关键字（case 仅在 match 套件内消费，type 无语句分发）。
"""
nxt: rawptr[Token] = ps.cur.next
if nxt is None: return 0
if nxt.type == TokenType.NewLine or nxt.type == TokenType.EndMarker: return 0
if nxt.type == TokenType.Op:
if nxt.op_subtype == TokOp.Colon: return 0
if nxt.op_subtype == TokOp.Equal: return 0
walk: rawptr[Token] = nxt
depth: int = 0
while walk is not None:
if walk.type == TokenType.NewLine or walk.type == TokenType.EndMarker:
return 0
if walk.type == TokenType.Op:
wop: int = walk.op_subtype
if wop == TokOp.LPar or wop == TokOp.Lsqb or wop == TokOp.LBrace:
depth += 1
elif wop == TokOp.RPar or wop == TokOp.Rsqb or wop == TokOp.RBrace:
if depth > 0:
depth -= 1
elif wop == TokOp.Colon and depth == 0:
return 1
walk = walk.next
return 0
# 解析错误位置与消息（fail-fast 仿 Python SyntaxError 报告，首次错误记录）
_g_parse_err_line: int = 0
_g_parse_err_col: int = 0
_g_parse_err_msg: str = None
def _record_parse_err(ps: rawptr[Parser]):
"""记录首个解析错误的位置与消息（只记录第一次，后续错误不覆盖）"""
global _g_parse_err_line, _g_parse_err_col, _g_parse_err_msg
if _g_parse_err_line != 0:
return
_g_parse_err_line = _cur_lineno(ps)
_g_parse_err_col = _cur_col(ps)
err_s: str = _cur_str(ps)
if err_s is None:
err_s = "(none)"
# 消息文本用 stdlib 拷贝（token 的 str_val 在 pool 中，生命周期覆盖本次编译但保险起见独立存储）
buf_m: str = stdlib.malloc(128)
if buf_m is not None:
stdio.snprintf(buf_m, 128, "意外的 token '%s'", err_s)
_g_parse_err_msg = buf_m
else:
_g_parse_err_msg = err_s
def _record_parse_err_ctx(ps: rawptr[Parser], tok_type: int,
pv_str: str, pv_line: int, pv_col: int,
nx_str: str, nx_line: int, nx_col: int):
"""记录首个解析错误，并把 token 类型 + 前/后 token 作为附注并入消息
与 `_record_parse_err` 同源（只记首次、同位置口径），差别只在消息更全：
意外的 token 'x'（type=4）
附注: prev='union' (line 420 col 0) | next=':' (line 420 col 8)
`文件:行:列` + 源码行 + caret + `SyntaxError: <消息>` 由调用方的
dump_parse_traceback 统一补全（仿 fatal_error），解析器不再自拼单行报文。
"""
global _g_parse_err_line, _g_parse_err_col, _g_parse_err_msg
if _g_parse_err_line != 0:
return
_g_parse_err_line = _cur_lineno(ps)
_g_parse_err_col = _cur_col(ps)
err_s: str = _cur_str(ps)
if err_s is None:
err_s = "(none)"
buf_m: str = stdlib.malloc(512)
if buf_m is None:
_g_parse_err_msg = err_s
return
stdio.snprintf(buf_m, 512,
"意外的 token '%s'（type=%d）\n 附注: prev='%s' (line %d col %d) | next='%s' (line %d col %d)",
err_s, tok_type, pv_str, pv_line, pv_col, nx_str, nx_line, nx_col)
_g_parse_err_msg = buf_m
def _record_neg_literal_overflow(lit_text: str, lineno: int, col: int):
"""记录"一元减 + 整数字面量"的联合越界（fail-fast；只记首次）。
`-` 是独立的一元运算符、不属于字面量，故幅值合法（≤ 2^64-1）的字面量在取负
后可能 < -2^63。报文含：**字面量原文（含负号）** + 允许范围 [-2^63, 2^64-1]
+ 修法；`文件:行:列` 与源码行由 parse_tokens → dump_parse_traceback 补全
（与词法层 `_lex_record_overflow` 同一上报路径）。
"""
global _g_parse_err_line, _g_parse_err_col, _g_parse_err_msg
if _g_parse_err_line != 0:
return
_g_parse_err_line = lineno
_g_parse_err_col = col
buf: str = stdlib.malloc(512)
if buf is not None:
stdio.snprintf(buf, 512,
"[TPV] 负数整数字面量越界: 一元减作用于整数字面量后的结果超出 64 位"
"有符号可表示范围（允许范围 -9223372036854775808..18446744073709551615"
" = [-2^63, 2^64-1]；负数结果必须 >= -2^63）。字面量原文: '-%s'。"
"禁止静默回绕/截断，已终止编译。"
"修法: 把该负数改到 -9223372036854775808..-1 的范围内（本语言不存在"
"更宽的整数类型，如需更大数值请拆分为多个 64 位字表达）。", lit_text)
_g_parse_err_msg = buf
else:
_g_parse_err_msg = lit_text
# ============================================================
# 类型描述符 const / volatile（软关键字，仅注解位置生效）
# ============================================================
def _IsDescriptorKeyword(ps: rawptr[Parser]) -> int:
"""当前 token 是否为类型描述符关键字（const / volatile），1=是 / 0=否。
const/volatile 是**软关键字**：只在**注解位置**（`:` 后的参数/AnnAssign 注解、
`->` 后的返回注解）由 `_parse_type_ann` 消费；作标识符的写法
（`t.attr.const()`，TransPyV/Test/App/attr_test.vp:142）不在注解位置，
属性名/调用名解析只看 TokenType.Name，不看 kw_subtype，故不受影响。
"""
if _cur_type(ps) != TokenType.Name:
return 0
if _cur_kw(ps) == Keyword.Const or _cur_kw(ps) == Keyword.Volatile:
return 1
return 0
def _IsMagicAttributeCall(ps: rawptr[Parser]) -> int:
"""注解位置的 `__attribute__(...)` 前缀判定，1=是 / 0=否
`__attribute__` 是软关键字（不进关键字表）：只在**注解位置**、且紧跟 '(' 时
由本处消费；作普通标识符/装饰器名（`@__attribute__(...)` 走装饰器通道）都不受影响。
语义与 const / volatile 描述符同族：顺序自由、可混写。
"""
if _cur_type(ps) != TokenType.Name:
return 0
s: str = _cur_str(ps)
if s is None or string.strcmp(s, "__attribute__") != 0:
return 0
nxt: rawptr[Token] = ps.cur.next
if nxt is None or nxt.type != TokenType.Op or nxt.op_subtype != TokOp.LPar:
return 0
return 1
def _MakeDescriptorMarker(ps: rawptr[Parser]) -> rawptr[AST]:
"""由当前描述符关键字构造标记节点 Attribute(Name('t'), 'CConst'/'CVolatile')。
与 TPC 侧 `SymbolUtils._ExpandTypeDescriptors` 的文本展开产物
（`t.CConst` / `t.CVolatile`）**逐字段同形**：两侧落到同一条既有识别路径
（TPC：HandlesTypeMerge/TypeAnnotationResolver 的 t 模块类型限定符分支 →
DataConst/DataVolatile；TPV：注解解析的 t 模块属性分支），不新造第二套表示。
"""
attr_name: str = "CConst"
if _cur_kw(ps) == Keyword.Volatile:
attr_name = "CVolatile"
marker: rawptr[AST] = Attribute(ps.pool, Name(ps.pool, "t", ASTCtx.Load),
attr_name, ASTCtx.Load)
_set_pos(marker, _cur_lineno(ps), _cur_col(ps), 0, 0)
return marker
def _PrependDescriptorMarkers(ps: rawptr[Parser],
base: rawptr[AST],
mods: rawptr[list[rawptr[AST]]],
mod_count: int) -> rawptr[AST]:
"""把描述符标记贴到类型联合的**最左操作数**位置（与 TPC 文本展开同形）。
TPC 把 `const volatile int | t.CPtr` 展开为文本
`t.CConst | t.CVolatile | int | t.CPtr`，其解析树是左结合
`(((t.CConst | t.CVolatile) | int) | t.CPtr)`。本函数据此把描述符串整体
贴到 `base` 最左操作数的左侧，产出**同一棵树**（而非
`t.CConst | (t.CVolatile | int | t.CPtr)`——那会让两侧 AST 不同形）。
"""
slot_parent: rawptr[BinOp] = None # 最左 BitOr 节点：其 left 即"最左操作数"
slot_node: rawptr[AST] = base
cur: rawptr[AST] = base
while cur is not None and cur.kind() == ASTKind.BinOp:
b: rawptr[BinOp] = (rawptr[BinOp])(cur)
if b.op != OpKind.BitOr or b.left is None:
break
nxt: rawptr[AST] = b.left
if nxt.kind() == ASTKind.BinOp:
nb: rawptr[BinOp] = (rawptr[BinOp])(nxt)
if nb.op == OpKind.BitOr and nb.left is not None:
cur = nxt
continue
slot_parent = b
slot_node = nxt
break
# 描述符串按源码顺序左结合：`const volatile` → (t.CConst | t.CVolatile)
chain: rawptr[AST] = mods.get(0)
i: int = 1
while i < mod_count:
chain = BinOp(ps.pool, chain, OpKind.BitOr, mods.get(i))
i += 1
newnode: rawptr[AST] = BinOp(ps.pool, chain, OpKind.BitOr, slot_node)
if slot_parent is None:
return newnode
slot_parent.left = newnode
return base
def _RecordDescriptorErr(detail: str, lineno: int, col: int):
"""记录类型描述符非法形态（fail-fast；只记首次）。
报文含**期望形态 + 正确示例**；`文件:行:列` 与源码行 + `^` 列标记由
parse_tokens → dump_parse_traceback 补全（与 `_record_neg_literal_overflow`
同一上报路径）。描述符**绝不静默忽略**：缺类型 / 后随非类型 / 裸用都终止编译。
"""
global _g_parse_err_line, _g_parse_err_col, _g_parse_err_msg
if _g_parse_err_line != 0:
return
_g_parse_err_line = lineno
_g_parse_err_col = col
buf: str = stdlib.malloc(1024)
if buf is not None:
stdio.snprintf(buf, 1024,
"类型描述符非法（%s）。const/volatile 是只在类型位置生效的软关键字，"
"必须写成 `名字: const 类型` / `名字: volatile 类型`"
"（可组合 `名字: const volatile 类型`）。正确示例：a: const int / "
"a: volatile int / a: const volatile t.CInt32T / "
"def f(p: const rawptr[Foo]) -> const int。"
"（描述符不会被静默忽略：缺类型 / 后随非类型 / 裸用一律终止编译）", detail)
_g_parse_err_msg = buf
else:
_g_parse_err_msg = detail
# ============================================================
# virtual / novirtual 修饰符（软关键字，仅"语句起始 + 紧跟 class/def"生效）
# ============================================================
# 语法：`virtual class C:` / `novirtual class C:` / `virtual def m(...)` /
# `novirtual def m(...)`（修饰符写在 class/def **之前**）。
#
# 语义：与**手写装饰器同一套表示**——virtual → `@t.CVTable`，novirtual → `@t.NoVTable`：
# - 类级：CVTable 强制有虚表 / NoVTable 强制无虚表
# （ClassGenBase._detect_vtable_status）
# - 方法级：CVTable 进虚表槽 / NoVTable 不进虚表槽
# （ClassGenBase._should_method_be_virtual）
# 本层只"合成同名装饰器"，下游零新增判据，不新造第二套表示；位置/报文与 TPC 端
# （lib/vast/Parser.py:_ParseVirtualModified）逐条对齐。
#
# 软关键字边界（非保留字）：只在**语句起始**判定；作标识符的写法
# （`x = d.virtual` / 形参名 / for 目标 / 字典键）token 仍是 Name，那些位置只看
# TokenType.Name、不看 kw_subtype，故语义逐字不变。
#
# 非法形态**响亮失败**（不静默丢修饰符）：
# - 后随非 class/def；
# - `virtual def` / `novirtual def` 不直接写在类体内（模块级 / 函数体 / 分支体）。
# 组合与重复形态（由"后随必须是 class/def"这一条唯一判据统一挡住）：
# `virtual novirtual class` / `virtual virtual class` / `novirtual virtual def`
# 都因"修饰符后随的是另一个修饰符"而响亮失败，不会被静默折叠或忽略。
# `novirtual class` 内写 `virtual def` **合法**：与既有"@t.NoVTable 类 + 方法级
# @t.CVTable"语义完全一致（_detect_vtable_status → _has_any_func_cvtable ⇒ 该类
# 仍生成虚表，且槽里只有被标记的方法）。
def _IsVirtualModifier(ps: rawptr[Parser]) -> int:
"""当前 token 是否为 virtual / novirtual 修饰符（1=是 / 0=否）。
只在**语句起始**（_parse_statement_into 分派处）调用；作标识符的位置不经过
本判据，故 `x = d.virtual` / 形参名 / for 目标 / 字典键不受影响。
"""
if _cur_type(ps) != TokenType.Name:
return 0
if _cur_kw(ps) == Keyword.Virtual or _cur_kw(ps) == Keyword.Novirtual:
return 1
return 0
def _MakeVirtualMarker(ps: rawptr[Parser]) -> rawptr[AST]:
"""由当前 virtual/novirtual token 构造标记节点 Attribute(Name('t'), 'CVTable'|'NoVTable')。
与**手写** `@t.CVTable` / `@t.NoVTable` 装饰器逐字段同形（_parse_decorators 的
产物形态），故下游 `ClassGenBase._has_decorator` 零新增判据。
"""
attr_name: str = "CVTable"
if _cur_kw(ps) == Keyword.Novirtual:
attr_name = "NoVTable"
marker: rawptr[AST] = Attribute(ps.pool, Name(ps.pool, "t", ASTCtx.Load),
attr_name, ASTCtx.Load)
_set_pos(marker, _cur_lineno(ps), _cur_col(ps), 0, 0)
return marker
def _RecordVirtualErr(detail: str, lineno: int, col: int, extra: str):
"""记录 virtual/novirtual 修饰符非法形态（fail-fast；只记首次）。
报文含**允许出现的位置 + 正确示例 + 修法**；`文件:行:列` 与源码行 + `^` 列标记
由 parse_tokens → dump_parse_traceback 补全（与 _RecordDescriptorErr 同一上报
路径）。修饰符**绝不静默忽略**：后随非 class/def / 方法级不在类体内都终止编译。
两个入参（detail / extra）由调用方用 **f-string 插值**构建：本语言的 str 是
i8*，**没有** `str + str` 的拼接语义，`"a" + x` 会退化成裸指针 add ⇒ 非法 IR
（opt: invalid operand type for instruction）。故本文件所有含变量的报文一律
走 f-string / snprintf，禁止对 str 用 `+` 拼接。
"""
global _g_parse_err_line, _g_parse_err_col, _g_parse_err_msg
if _g_parse_err_line != 0:
return
_g_parse_err_line = lineno
_g_parse_err_col = col
buf: str = stdlib.malloc(1536)
if buf is not None:
stdio.snprintf(buf, 1536,
"[TPV] virtual/novirtual 修饰符非法（%s）。可写位置：类级 "
"`virtual class C:` / `novirtual class C:`；方法级 "
"`virtual def m(self): ...` / `novirtual def m(self): ...`"
"（只能直接写在类体内：virtual class / class / novirtual class 的类体）。"
"正确示例：virtual class Shape: / novirtual class Node: / "
"class A: virtual def run(self) -> int: ... / "
"novirtual class B: novirtual def helper(self) -> int: ...。%s"
"（修饰符不会被静默忽略：非法形态一律终止编译）", detail, extra)
_g_parse_err_msg = buf
else:
_g_parse_err_msg = detail
def _ParseVirtualModified(ps: rawptr[Parser], parent: rawptr[AST]):
"""`virtual class` / `novirtual class` / `virtual def` / `novirtual def`。
合成 `@t.CVTable` / `@t.NoVTable` 装饰器（与用户既有装饰器合并，标记追加在末尾；
位置无关——`_has_decorator` 按名判定）后交给 _parse_classdef / _parse_funcdef，
产出与**手写同名装饰器**逐字段同形的 AST，故下游零新增分支。
"""
mod_lineno: int = _cur_lineno(ps)
mod_col: int = _cur_col(ps)
mod_name: str = "virtual"
if _cur_kw(ps) == Keyword.Novirtual:
mod_name = "novirtual"
marker: rawptr[AST] = _MakeVirtualMarker(ps)
_advance(ps) # 消费修饰符
# 修饰符后必须是 class / def（def 的别名 fn/fun/func/function 同样接受：
# 与 _is_funcdef_kw 口径一致，也与 TPC 端逐条对齐——TPC 的 fn→def 文本展开
# 只命中行首 fn，`virtual fn m()` 不先展开，故两端都在本层显式接受别名）
kw2: int = _cur_kw(ps)
is_class: int = 0
is_def: int = 0
if kw2 == Keyword.Class:
is_class = 1
elif kw2 == Keyword.Def or kw2 == Keyword.Fn or kw2 == Keyword.Fun \
or kw2 == Keyword.Func or kw2 == Keyword.Function:
is_def = 1
if is_class == 0 and is_def == 0:
nxt_s: str = _cur_str(ps)
if nxt_s is None:
nxt_s = "(none)"
_RecordVirtualErr(f"`{mod_name}` 后随 '{nxt_s}'（非 class/def）",
mod_lineno, mod_col,
f"`{mod_name}` 只在 class/def 前生效；"
f"若此处本意是普通标识符名，请改名"
f"（该词在语句起始会被判为修饰符）。")
ps.error_count += 1
# 余下 token 交给通用语句分派消费（不制造死循环，也不覆盖首次错误）。
_parse_simple_stmt_into(ps, parent)
return
if is_def != 0 and ps.in_class_body == 0:
_RecordVirtualErr(f"{mod_name} def 出现在类体之外", mod_lineno, mod_col,
f"{mod_name} def 描述的是「类体内方法的虚/非虚」，"
f"模块级 / 函数体 / 分支体内没有可归属的类。")
ps.error_count += 1
# 合并既有装饰器（`@deco` + virtual class/def 组合）后交给既有 class/def 解析
merged: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
if merged is None:
return
if ps.pending_decorators is not None:
dn: sizet = len(ps.pending_decorators)
di: sizet = 0
while di < dn:
merged.append(ps.pending_decorators.get(di))
di += 1
merged.append(marker)
ps.pending_decorators = merged
stmt: rawptr[AST] = None
if is_class != 0:
stmt = _parse_classdef(ps)
else:
stmt = _parse_funcdef(ps, 0)
if stmt is not None:
parent.append(stmt)
# ============================================================
# inline 修饰符（软关键字）：`inline def f(...)` ⇒ 合成 @t.attr.always_inline
#
# 与 virtual/novirtual 完全同构：只在「语句起始 + inline + 紧跟函数定义」时生效，
# 其余位置的 inline 仍是普通标识符（故 `inline = 5` / 字段名 inline 等既有写法不受
# 影响）。合成的装饰器与**手写** `@t.attr.always_inline` 逐字段同形，下游
# FuncGenBase.extract_func_attrs → _append_llvm_attr 的 "always_inline" → LLVM
# "alwaysinline" 映射原样复用；StubGen 跨模块 alwaysinline 合并规则同样原样生效。
#
# 为什么不进关键字表：`inline` 在 TPV/TPC 既有源码里可能是普通标识符，进表会把
# 全局词法口径改掉；此处按软关键字文本判定，作用域只限"语句起始后随函数定义"。
# ============================================================
def _IsInlineModifier(ps: rawptr[Parser]) -> int:
"""当前 token 是否为 `inline`（软关键字，且后随函数定义），1=是 / 0=否"""
if _cur_type(ps) != TokenType.Name:
return 0
s: str = _cur_str(ps)
if s is None or string.strcmp(s, "inline") != 0:
return 0
nxt: rawptr[Token] = ps.cur.next
if nxt is None or nxt.type != TokenType.Name:
return 0
nxt2: rawptr[Token] = nxt.next
if nxt2 is None:
return 0
# 后随 def（硬关键字）：`inline def f(...)`
if nxt.kw_subtype == Keyword.Def:
return 1
# 后随函数定义软关键字别名（fn/fun/func/function）：与 _is_funcdef_kw 同口径
# ——必须是 Name + '(' 或 '[T]'，避免把 `inline func` 这种普通标识符组合误判。
if nxt.kw_subtype == Keyword.Fn or nxt.kw_subtype == Keyword.Fun \
or nxt.kw_subtype == Keyword.Func or nxt.kw_subtype == Keyword.Function:
if nxt2.type == TokenType.Op \
and (nxt2.op_subtype == TokOp.LPar or nxt2.op_subtype == TokOp.Lsqb):
return 1
return 0
def _MakeInlineMarker(ps: rawptr[Parser]) -> rawptr[AST]:
"""由当前 inline token 构造标记节点 Attribute(Attribute(Name('t'),'attr'), 'always_inline')
形态与**手写** `@t.attr.always_inline` 装饰器逐字段同形。取属性名的既有逻辑
（FuncGenBase._get_attr_name_from_node）对 Attribute 节点取最外层 attr，
故 t.attr.always_inline → "always_inline" → _append_llvm_attr 映射为 LLVM
"alwaysinline"。**刻意不用** `@c.Attribute(t.attr.x)` 形态：该形态的提取器
要求本源文件 `import c`，而 includes/asm.vp 之类文件没有 import，会静默不内联。
"""
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
t_name: rawptr[AST] = Name(ps.pool, "t", ASTCtx.Load)
attr_at: rawptr[AST] = Attribute(ps.pool, t_name, "attr", ASTCtx.Load)
marker: rawptr[AST] = Attribute(ps.pool, attr_at, "always_inline", ASTCtx.Load)
_set_pos(marker, lineno, col, 0, 0)
return marker
def _ParseInlineModified(ps: rawptr[Parser], parent: rawptr[AST]):
"""`inline def f(...)`：消费 inline，合成 @t.attr.always_inline 后交给 _parse_funcdef"""
marker: rawptr[AST] = _MakeInlineMarker(ps)
if marker is None:
return
_advance(ps) # 消费 inline 修饰符
# 合并既有装饰器（`@deco` + inline def 组合），标记追加在末尾；位置无关
# （下游按装饰器名判定）。与 _ParseVirtualModified 同一套 pending_decorators 通道。
merged: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
if merged is None:
return
if ps.pending_decorators is not None:
dn: sizet = len(ps.pending_decorators)
di: sizet = 0
while di < dn:
merged.append(ps.pending_decorators.get(di))
di += 1
merged.append(marker)
ps.pending_decorators = merged
stmt: rawptr[AST] = _parse_funcdef(ps, 0)
if stmt is not None:
parent.append(stmt)
def _IsStaticModifier(ps: rawptr[Parser]) -> int:
"""当前 token 是否为 `static` 修饰符（软关键字），1=是 / 0=否。
只在**语句起始**（_parse_statement_into 分派处）调用；后随
① 函数定义（`def`，或 fn/fun/func/function 别名 + '(' / '['）
② 变量声明（`Name :`，即 AnnAssign）
时命中。作标识符的位置（属性名 / 形参名 / 字典键 / 表达式内）不经过本判据，
故 `x = d.static`、`f(static=1)` 等写法不受影响（软关键字边界）。
语义（禁止跨模块使用）由 TransPyV 侧消费；本层只负责把两种写法归一，
且与手写 `@t.CStatic` / `x: t.CStatic | T` 的标记逐字段同形。
"""
if _cur_type(ps) != TokenType.Name:
return 0
s: str = _cur_str(ps)
if s is None or string.strcmp(s, "static") != 0:
return 0
nxt: rawptr[Token] = ps.cur.next
if nxt is None or nxt.type != TokenType.Name:
return 0
# ① 后随 def（硬关键字）：`static def f(...)`
if nxt.kw_subtype == Keyword.Def:
return 1
nxt2: rawptr[Token] = nxt.next
# ①' 后随函数定义软关键字别名：必须是 Name + '(' 或 '[T]'
#（与 _IsInlineModifier / _is_funcdef_kw 同口径，避免 `static func` 这类
# 普通标识符组合被误判）
if nxt.kw_subtype == Keyword.Fn or nxt.kw_subtype == Keyword.Fun \
or nxt.kw_subtype == Keyword.Func or nxt.kw_subtype == Keyword.Function:
if nxt2 is not None and nxt2.type == TokenType.Op \
and (nxt2.op_subtype == TokOp.LPar or nxt2.op_subtype == TokOp.Lsqb):
return 1
# ② 后随变量声明 `Name :`（AnnAssign）
if nxt2 is not None and nxt2.type == TokenType.Op and nxt2.op_subtype == TokOp.Colon:
return 1
return 0
def _MakeStaticMarker(ps: rawptr[Parser]) -> rawptr[AST]:
"""构造标记节点 Attribute(Name('t'), 'CStatic')
与**手写** `@t.CStatic` 装饰器 / `x: t.CStatic | T` 注解里的标记逐字段同形，
故下游（FuncGenDef 的存储类标记识别、TypeGenResolve 的存储类限定符判定）
零新增判据；两种写法同源。
"""
marker: rawptr[AST] = Attribute(ps.pool, Name(ps.pool, "t", ASTCtx.Load),
"CStatic", ASTCtx.Load)
_set_pos(marker, _cur_lineno(ps), _cur_col(ps), 0, 0)
return marker
def _ParseStaticModified(ps: rawptr[Parser], parent: rawptr[AST]) -> int:
"""`static` 修饰符分派：1=已完全处理（函数定义，调用方直接 return）；
0=仅置位（变量声明，调用方继续走通用语句分派）。
函数：消费 static 后合成 @t.CStatic 装饰器，交给 _parse_funcdef
（与 inline def 同一套 pending_decorators 通道）。
变量：置 pending_static 后消费 static，由紧随的 _parse_type_ann 把
t.CStatic 插到类型联合最左（= 手写 `x: t.CStatic | int`）。
"""
nxt: rawptr[Token] = ps.cur.next
is_func: int = 0
if nxt is not None and nxt.type == TokenType.Name:
if nxt.kw_subtype == Keyword.Def:
is_func = 1
elif nxt.kw_subtype == Keyword.Fn or nxt.kw_subtype == Keyword.Fun \
or nxt.kw_subtype == Keyword.Func or nxt.kw_subtype == Keyword.Function:
nxt2: rawptr[Token] = nxt.next
if nxt2 is not None and nxt2.type == TokenType.Op \
and (nxt2.op_subtype == TokOp.LPar or nxt2.op_subtype == TokOp.Lsqb):
is_func = 1
if is_func == 0:
# 变量分支：仅置位 + 消费 static，交调用方继续通用分派（AnnAssign 解析
# 会在 _parse_type_ann 里读取 pending_static）。
ps.pending_static = 1
_advance(ps)
return 0
marker: rawptr[AST] = _MakeStaticMarker(ps)
if marker is None:
return 1
_advance(ps) # 消费 static 修饰符
# 合并既有装饰器（`@deco` + `static def` 组合），标记追加在末尾；位置无关
# （下游按装饰器名判定）。与 _ParseInlineModified 同一套 pending_decorators 通道。
merged: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
if merged is None:
return 1
if ps.pending_decorators is not None:
dn: sizet = len(ps.pending_decorators)
di: sizet = 0
while di < dn:
merged.append(ps.pending_decorators.get(di))
di += 1
merged.append(marker)
ps.pending_decorators = merged
stmt: rawptr[AST] = _parse_funcdef(ps, 0)
if stmt is not None:
parent.append(stmt)
return 1
def _IsBareDescriptor(ps: rawptr[Parser]) -> int:
"""裸用描述符判定：整条语句只有一个描述符串（const[: volatile]），1=是 / 0=否。
只在"描述符串后直接换行 / 文件尾 / 分号"时命中；作普通标识符的写法
（`const = 5` / `const(x)` / `t.attr.const()`）不命中（软关键字边界）。
"""
if _IsDescriptorKeyword(ps) == 0:
return 0
nxt: rawptr[Token] = ps.cur.next
while nxt is not None and nxt.type == TokenType.Name \
and (nxt.kw_subtype == Keyword.Const or nxt.kw_subtype == Keyword.Volatile):
nxt = nxt.next
if nxt is None:
return 1
if nxt.type == TokenType.NewLine or nxt.type == TokenType.EndMarker:
return 1
if nxt.type == TokenType.Op and nxt.op_subtype == TokOp.Semi:
return 1
return 0
def get_parse_err_line() -> int:
"""获取最近一次解析错误的行号（0=无错误）"""
return _g_parse_err_line
def get_parse_err_col() -> int:
"""获取最近一次解析错误的列号（0=无错误）"""
return _g_parse_err_col
def get_parse_err_msg() -> str:
"""获取最近一次解析错误的错误消息（仿 Python SyntaxError 消息文本）"""
return _g_parse_err_msg
def _cur_str(ps: rawptr[Parser]) -> str:
if ps.cur is None: return None
return ps.cur.str_val
def _cur_lineno(ps: rawptr[Parser]) -> int:
if ps.cur is None: return 0
return ps.cur.lineno
def _cur_col(ps: rawptr[Parser]) -> int:
if ps.cur is None: return 0
return ps.cur.col_offset
def _skip_nl(ps: rawptr[Parser]):
"""Skip TokenType.Nl token"""
while ps.cur is not None and ps.cur.type == TokenType.Nl:
_advance(ps)
def _skip_nls_and_newlines(ps: rawptr[Parser]):
"""Skip TokenType.Nl and TokenType.NewLine (used for line continuation)"""
while ps.cur is not None:
if ps.cur.type == TokenType.Nl or ps.cur.type == TokenType.NewLine:
_advance(ps)
else:
break
def _expect_type(ps: rawptr[Parser], ttype: int) -> int:
"""Expect the current token to be of the specified type, advance and return 1, otherwise return 0"""
if _cur_type(ps) == ttype:
_advance(ps)
return 1
_record_parse_err(ps)
ps.error_count += 1
return 0
def _expect_op(ps: rawptr[Parser], op_id: int) -> int:
"""Expecting the current one to be some operator"""
if _cur_type(ps) == TokenType.Op and _cur_op(ps) == op_id:
_advance(ps)
return 1
_record_parse_err(ps)
ps.error_count += 1
return 0
def _match_op(ps: rawptr[Parser], op_id: int) -> int:
"""Check but not consume the current token"""
_tick(ps)
if ps.aborted != 0:
# 收束：仍有 token 时"匹配"并一次性跳到末尾，使
# `while not _match_op(...)` / `while _match_op(...)` 均在 ≤2 轮结束
if ps.cur is not None:
ps.cur = None
return 1
return 0
if _cur_type(ps) == TokenType.Op and _cur_op(ps) == op_id:
return 1
return 0
def _accept_op(ps: rawptr[Parser], op_id: int) -> int:
"""If match, consume and return 1"""
if _match_op(ps, op_id):
_advance(ps)
return 1
return 0
# ============================================================
# Constant Construction Helper (New API: Constant directly accepts lineno/col)
# ============================================================
def _new_const_int(pool: rawptr[memhub.MemManager], val: int64,
lineno: int, col: int) -> rawptr[AST]:
return Constant(pool, CONST_INT, val, 0.0, None, lineno, col)
def _new_const_float(pool: rawptr[memhub.MemManager], val: double,
lineno: int, col: int) -> rawptr[AST]:
global _g_dbg_pts, _g_dbg_vals, _g_dbg_cnt
new_c: rawptr[Constant] = Constant(pool, CONST_FLOAT, 0, val, None, lineno, col)
# 把数字文本存到 str_val（stdlib 分配，绕开 pool 对 float_val 字段的覆写）
# REG-SRC 用 string.atof(str_val) 恢复原始值
lit_txt: str = stdlib.malloc(64)
if lit_txt is not None:
stdio.snprintf(lit_txt, 64, "%.17g", val)
new_c.str_val = lit_txt
# 记录浮点节点 ptr→val 到 stdlib 表（构造时值正确，供 REG-SRC 查表绕开 pool 覆写）
# 表满时重置：旧记录对应已翻译完的文件，从当前节点重新记录
if _g_dbg_pts is None:
_g_dbg_pts = stdlib.malloc(4096 * 8)
_g_dbg_vals = stdlib.malloc(4096 * 8)
_g_dbg_cnt = 0
elif _g_dbg_cnt >= 4096:
_g_dbg_cnt = 0
if _g_dbg_pts is not None and _g_dbg_vals is not None:
_g_dbg_pts[_g_dbg_cnt] = uint64(new_c)
_g_dbg_vals[_g_dbg_cnt] = val
_g_dbg_cnt += 1
return new_c
# 浮点节点构造值表（stdlib 分配，避免污染 pool）
_g_dbg_pts: rawptr[uint64] = None
_g_dbg_vals: rawptr[double] = None
_g_dbg_cnt: int = 0
# 最近一次 _dbg_lookup_lit_val 是否命中
_g_dbg_lit_found: int = 0
def _dbg_lookup_lit_val(ptr: uint64) -> double:
"""查浮点节点构造值表：返回 ptr 节点的原始构造值（未命中返回 0 并置 found=0）
用于绕开 pool 分配对 AST float_val 字段的覆写（构造时值正确，读取时可能被覆写）。
"""
global _g_dbg_lit_found
_g_dbg_lit_found = 0
if _g_dbg_pts is None or _g_dbg_vals is None:
return 0.0
i: sizet = 0
while i < _g_dbg_cnt:
if _g_dbg_pts[i] == ptr:
_g_dbg_lit_found = 1
return _g_dbg_vals[i]
i += 1
return 0.0
def _dbg_lit_found() -> int:
"""检查最近一次 _dbg_lookup_lit_val 是否命中"""
return _g_dbg_lit_found
def _new_const_str(pool: rawptr[memhub.MemManager], val: str,
lineno: int, col: int,
is_dquote: int = 1,
is_triple: int = 0,
is_char: int = 0) -> rawptr[AST]:
"""Create a string/character constant node
is_dquote: 0 = single quote, non-0 = double quote (default double quote)
is_triple: non-0 = triple-quoted string
is_char: non-0 = single-character candidate (lexer has confirmed that after escaping it's exactly 1 byte, including '0')
Semantic distinctions:
- 'x' (single quote, single character, not triple-quoted) → CONST_CHAR (i8 value)
- "x" (double quote, includes 0 terminator) → CONST_STR (i8* pointer, i8 x 2)
- 'ab' (single quotes, multiple characters) → CONST_STR (i8* pointer)
- '''x''' / \"\"\"x\"\"\" (triple quotes) → CONST_STR (i8* pointer)
- '' / "" (empty string) → CONST_STR (i8* pointer)
"""
# Single quotes, non-triple quotes, single-character candidates marked by lexer → CONST_CHAR
# Mark with is_char instead of strlen(val), because '0' becomes a NUL byte after escaping, strlen would return 0
if is_dquote == 0 and is_triple == 0 and is_char != 0:
# Single quote # Single character → CONST_CHAR, int_val stores the character's ASCII value
# val[0] is a legal value of 0 even if it's NUL
ch_val: int64 = val[0]
return Constant(pool, CONST_CHAR, ch_val, 0.0, val, lineno, col)
# Other situations → CONST_STR
return Constant(pool, CONST_STR, 0, 0.0, val, lineno, col)
def _new_const_bool(pool: rawptr[memhub.MemManager], val: int,
lineno: int, col: int) -> rawptr[AST]:
return Constant(pool, CONST_BOOL, val, 0.0, None, lineno, col)
def _new_const_none(pool: rawptr[memhub.MemManager],
lineno: int, col: int) -> rawptr[AST]:
return Constant(pool, CONST_NONE, 0, 0.0, None, lineno, col)
# ============================================================
# ctx Helper (Recursive Fixing Store Context)
# ============================================================
def _set_ctx(node: rawptr[AST], ctx: int):
"""Set ctx field of node (based on kind conversion)"""
if node is None: return
k: int = node.kind()
if k == ASTKind.Name:
nm: rawptr[Name] = (rawptr[Name])(node)
nm.ctx = ctx
elif k == ASTKind.Attribute:
at: rawptr[Attribute] = (rawptr[Attribute])(node)
at.ctx = ctx
elif k == ASTKind.Subscript:
sb: rawptr[Subscript] = (rawptr[Subscript])(node)
sb.ctx = ctx
elif k == ASTKind.Tuple:
tp: rawptr[Tuple] = (rawptr[Tuple])(node)
tp.ctx = ctx
elif k == ASTKind.List:
ls: rawptr[List] = (rawptr[List])(node)
ls.ctx = ctx
elif k == ASTKind.Starred:
st: rawptr[Starred] = (rawptr[Starred])(node)
st.ctx = ctx
def _fix_store_ctx(node: rawptr[AST]):
"""Fix ctx of store targets to be Store"""
if node is None: return
_set_ctx(node, ASTCtx.Store)
k: int = node.kind()
if k == ASTKind.Tuple or k == ASTKind.List:
tp_elts: rawptr[list[rawptr[AST]]] = None
if k == ASTKind.Tuple:
tp: rawptr[Tuple] = (rawptr[Tuple])(node)
tp_elts = tp.elts
else:
ls: rawptr[List] = (rawptr[List])(node)
tp_elts = ls.elts
if tp_elts is not None:
n: sizet = tp_elts.__len__()
i: sizet = 0
while i < n:
cur: rawptr[AST] = tp_elts.get(i)
_fix_store_ctx(cur)
i += 1
elif k == ASTKind.Starred:
st2: rawptr[Starred] = (rawptr[Starred])(node)
_fix_store_ctx(st2.value)
# ============================================================
# Expression Parser
# ============================================================
def _parse_expr(ps: rawptr[Parser]) -> rawptr[AST]:
"""expr: yield_expr | namedexpr_test"""
if _cur_kw(ps) == Keyword.Yield:
return _parse_yield(ps)
return _parse_namedexpr(ps)
def _parse_namedexpr(ps: rawptr[Parser]) -> rawptr[AST]:
"""namedexpr_test: test [':=' test]"""
left: rawptr[AST] = _parse_test(ps)
if left is None: return None
if _match_op(ps, TokOp.ColonEq):
_advance(ps)
right: rawptr[AST] = _parse_test(ps)
node: rawptr[AST] = NamedExpr(ps.pool, left, right)
_set_pos(node, left.lineno, left.col_offset, 0, 0)
return node
return left
def _IsVolatileCallPrefix(ps: rawptr[Parser]) -> int:
"""值位置的 `volatile 函数()` / `volatile 结构体()` 前缀判定，1=是 / 0=否
`volatile` 是软关键字：只在"值位置 + 紧跟 调用（Name 后随 '('）"时消费，
作注解描述符（`a: volatile int`）与作标识符的写法都不受影响。
"""
if _cur_kw(ps) != Keyword.Volatile:
return 0
nxt: rawptr[Token] = ps.cur.next
if nxt is None or nxt.type != TokenType.Name:
return 0
nxt2: rawptr[Token] = nxt.next
if nxt2 is None or nxt2.type != TokenType.Op or nxt2.op_subtype != TokOp.LPar:
return 0
return 1
def _parse_test(ps: rawptr[Parser]) -> rawptr[AST]:
"""test: or_test ['if' or_test 'else' test] | lambdef"""
if _cur_kw(ps) == Keyword.Lambda:
return _parse_lambda(ps)
# 值位置 volatile 调用前缀：`a: int = volatile f()` / `volatile S()`
# ⇒ 消费 volatile 并把调用节点标记为 volatile（结果槽按 volatile 存取；
# 结构体构造的写入亦带 LLVM volatile）。LLVM 的 call 无 volatile 限定符，
# 故语义只落在内存存取上（见 Call.IsVolatile 注释）。
if _IsVolatileCallPrefix(ps) != 0:
_advance(ps) # 消费 volatile
vinner: rawptr[AST] = _parse_test(ps)
if vinner is not None and vinner.kind() == ast.ASTKind.Call:
(ast.Call*(vinner)).IsVolatile = 1
return vinner
left: rawptr[AST] = _parse_or_test(ps)
if left is None: return None
if _cur_kw(ps) == Keyword.If:
_advance(ps)
test: rawptr[AST] = _parse_or_test(ps)
if _cur_kw(ps) == Keyword.Else:
_advance(ps)
orelse: rawptr[AST] = _parse_test(ps)
node: rawptr[AST] = IfExp(ps.pool, test, left, orelse)
_set_pos(node, left.lineno, left.col_offset, 0, 0)
return node
ps.error_count += 1
return left
return left
def _parse_lambda(ps: rawptr[Parser]) -> rawptr[AST]:
"""lambdef: 'lambda' [varargslist] ':' test"""
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps) # lambda
args: rawptr[AST] = None
if not _match_op(ps, TokOp.Colon):
args = _parse_varargslist(ps, 1)
if not _expect_op(ps, TokOp.Colon):
return None
body: rawptr[AST] = _parse_test(ps)
node: rawptr[AST] = Lambda(ps.pool, args, body)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_or_test(ps: rawptr[Parser]) -> rawptr[AST]:
"""or_test: and_test ('or' and_test)*"""
left: rawptr[AST] = _parse_and_test(ps)
if left is None: return None
values: rawptr[list[rawptr[AST]]] = None
while True:
# 括号内行首 or（(A\n or B)）与行尾 or（A or\n B）都支持：
# 每轮先 _skip_nl 再检查 or，否则 A or B\n or C 在 B 后停在 Nl 上
# 循环退出，上层把续行 or 当非法 token（对齐 and_test / vast 实现）。
_skip_nl(ps)
if _cur_kw(ps) != Keyword.Or:
break
if values is None:
values = list[rawptr[AST]](ps.pool, 8)
values.append(left)
_advance(ps)
_skip_nl(ps) # 括号内跨行: a or\n b
right: rawptr[AST] = _parse_and_test(ps)
if right is None: break
values.append(right)
if values is not None:
node: rawptr[AST] = BoolOp(ps.pool, OpKind.Or, values)
_set_pos(node, left.lineno, left.col_offset, 0, 0)
return node
return left
def _parse_and_test(ps: rawptr[Parser]) -> rawptr[AST]:
"""and_test: not_test ('and' not_test)*"""
left: rawptr[AST] = _parse_not_test(ps)
if left is None: return None
values: rawptr[list[rawptr[AST]]] = None
while True:
# 括号内行首 and（(A\n and B)）与行尾 and（A and\n B）都支持：
# 每轮先 _skip_nl 再检查 and，否则 A and B\n and C 在 B 后停在
# Nl 上循环退出，上层把续行 and 当非法 token
# （ThreadingTest check 多行 and 链解析失败根因）。
_skip_nl(ps)
if _cur_kw(ps) != Keyword.And:
break
if values is None:
values = list[rawptr[AST]](ps.pool, 8)
values.append(left)
_advance(ps)
_skip_nl(ps) # 括号内跨行: a and\n b
right: rawptr[AST] = _parse_not_test(ps)
if right is None: break
values.append(right)
if values is not None:
node: rawptr[AST] = BoolOp(ps.pool, OpKind.And, values)
_set_pos(node, left.lineno, left.col_offset, 0, 0)
return node
return left
def _parse_not_test(ps: rawptr[Parser]) -> rawptr[AST]:
"""not_test: 'not' not_test | comparison"""
if _cur_kw(ps) == Keyword.Not:
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps)
operand: rawptr[AST] = _parse_not_test(ps)
node: rawptr[AST] = UnaryOp(ps.pool, OpKind.Not, operand)
_set_pos(node, lineno, col, 0, 0)
return node
return _parse_comparison(ps)
def _parse_comparison(ps: rawptr[Parser]) -> rawptr[AST]:
"""comparison: bitor_expr (comp_op bitor_expr)*
Single comparison design: a < b → Compare(left=a, op=Lt, right=b)
Chained comparison a < b < c → BoolOp(And, [Compare(a,<,b), Compare(b,<,c)])
"""
left: rawptr[AST] = _parse_bitor(ps)
if left is None: return None
_skip_nl(ps) # 括号内行首比较运算符: (a\n < b)
op_id: int = _match_comp_op(ps)
if op_id == OpKind.NoneOp:
return left
_advance_comp_op(ps)
_skip_nl(ps) # 括号内跨行: a ==\n b
right: rawptr[AST] = _parse_bitor(ps)
# Single comparison: construct ops and comparators list
ops: rawptr[list[int]] = list[int](ps.pool, 8)
ops.append(op_id)
comparators: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
comparators.append(right)
first_cmp: rawptr[AST] = Compare(ps.pool, left, ops, comparators)
_set_pos(first_cmp, left.lineno, left.col_offset, 0, 0)
# Check chained comparison
_skip_nl(ps) # 括号内行首链式比较: (a < b\n < c)
next_op: int = _match_comp_op(ps)
if next_op == OpKind.NoneOp:
return first_cmp
# Single comparison with chained conditions: a < b < c → BoolOp(And, [Compare(a,<,b), Compare(b,<,c), ...])
values: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
values.append(first_cmp)
cur_left: rawptr[AST] = right
while next_op != OpKind.NoneOp:
_advance_comp_op(ps)
_skip_nl(ps) # 括号内跨行链式比较
cur_right: rawptr[AST] = _parse_bitor(ps)
ops2: rawptr[list[int]] = list[int](ps.pool, 8)
ops2.append(next_op)
comparators2: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
comparators2.append(cur_right)
cmp: rawptr[AST] = Compare(ps.pool, cur_left, ops2, comparators2)
_set_pos(cmp, cur_left.lineno, cur_left.col_offset, 0, 0)
values.append(cmp)
cur_left = cur_right
_skip_nl(ps) # 括号内行首链式比较: (a < b\n < c)
next_op = _match_comp_op(ps)
node: rawptr[AST] = BoolOp(ps.pool, OpKind.And, values)
_set_pos(node, left.lineno, left.col_offset, 0, 0)
return node
def _match_comp_op(ps: rawptr[Parser]) -> int:
"""Check if the current one is a comparison operator, return OP_* or OpKind.NoneOp"""
if _cur_type(ps) == TokenType.Op:
op_id: int = _cur_op(ps)
if op_id == TokOp.EqEq: return OpKind.Eq
if op_id == TokOp.ExclaimEq: return OpKind.Ne
if op_id == TokOp.Less: return OpKind.Lt
if op_id == TokOp.LessEq: return OpKind.Le
if op_id == TokOp.Greater: return OpKind.Gt
if op_id == TokOp.GreaterEq: return OpKind.Ge
if _cur_kw(ps) == Keyword.Is:
if ps.cur.next is not None and ps.cur.next.type == TokenType.Name and ps.cur.next.kw_subtype == Keyword.Not:
return OpKind.IsNot
return OpKind.Is
if _cur_kw(ps) == Keyword.Not:
if ps.cur.next is not None and ps.cur.next.kw_subtype == Keyword.In:
return OpKind.NotIn
if _cur_kw(ps) == Keyword.In:
return OpKind.In
return OpKind.NoneOp
def _advance_comp_op(ps: rawptr[Parser]):
"""Consume comparison operator token"""
if _cur_type(ps) == TokenType.Op:
_advance(ps)
return
if _cur_kw(ps) == Keyword.Is:
_advance(ps)
if _cur_kw(ps) == Keyword.Not:
_advance(ps)
return
if _cur_kw(ps) == Keyword.Not:
_advance(ps)
if _cur_kw(ps) == Keyword.In:
_advance(ps)
return
if _cur_kw(ps) == Keyword.In:
_advance(ps)
def _parse_bitor(ps: rawptr[Parser]) -> rawptr[AST]:
left: rawptr[AST] = _parse_bitxor(ps)
if left is None: return None
while True:
_skip_nl(ps) # 括号内行首 '|': (a\n | b)
if _match_op(ps, TokOp.VBar) == 0:
break
_advance(ps)
_skip_nl(ps) # 括号内跨行: a |\n b
right: rawptr[AST] = _parse_bitxor(ps)
node: rawptr[AST] = BinOp(ps.pool, left, OpKind.BitOr, right)
_set_pos(node, left.lineno, left.col_offset, 0, 0)
left = node
return left
def _parse_bitxor(ps: rawptr[Parser]) -> rawptr[AST]:
left: rawptr[AST] = _parse_bitand(ps)
if left is None: return None
while True:
_skip_nl(ps) # 括号内行首 '^': (a\n ^ b)
if _match_op(ps, TokOp.Caret) == 0:
break
_advance(ps)
_skip_nl(ps) # 括号内跨行: a ^\n b
right: rawptr[AST] = _parse_bitand(ps)
node: rawptr[AST] = BinOp(ps.pool, left, OpKind.BitXor, right)
_set_pos(node, left.lineno, left.col_offset, 0, 0)
left = node
return left
def _parse_bitand(ps: rawptr[Parser]) -> rawptr[AST]:
left: rawptr[AST] = _parse_shift(ps)
if left is None: return None
while True:
_skip_nl(ps) # 括号内行首 '&': (a\n & b)
if _match_op(ps, TokOp.Amp) == 0:
break
_advance(ps)
_skip_nl(ps) # 括号内跨行: a &\n b
right: rawptr[AST] = _parse_shift(ps)
node: rawptr[AST] = BinOp(ps.pool, left, OpKind.BitAnd, right)
_set_pos(node, left.lineno, left.col_offset, 0, 0)
left = node
return left
def _parse_shift(ps: rawptr[Parser]) -> rawptr[AST]:
left: rawptr[AST] = _parse_arith(ps)
if left is None: return None
while True:
_skip_nl(ps) # 括号内行首 '<<'/'>>': (a\n << b)
if _match_op(ps, TokOp.LtLt) == 0 and _match_op(ps, TokOp.GtGt) == 0:
break
op_token: int = _cur_op(ps)
_advance(ps)
_skip_nl(ps) # 括号内跨行: a <<\n b
right: rawptr[AST] = _parse_arith(ps)
ast_op: int = OpKind.LShift
if op_token == TokOp.GtGt:
ast_op = OpKind.RShift
node: rawptr[AST] = BinOp(ps.pool, left, ast_op, right)
_set_pos(node, left.lineno, left.col_offset, 0, 0)
left = node
return left
def _parse_arith(ps: rawptr[Parser]) -> rawptr[AST]:
left: rawptr[AST] = _parse_term(ps)
if left is None: return None
while True:
_skip_nl(ps) # 括号内行首 '+'/'-': (a\n + b)
if _match_op(ps, TokOp.Plus) == 0 and _match_op(ps, TokOp.Minus) == 0:
break
op_token: int = _cur_op(ps)
_advance(ps)
_skip_nl(ps) # 括号内跨行: a +\n b
right: rawptr[AST] = _parse_term(ps)
ast_op: int = OpKind.Add
if op_token == TokOp.Minus:
ast_op = OpKind.Sub
node: rawptr[AST] = BinOp(ps.pool, left, ast_op, right)
_set_pos(node, left.lineno, left.col_offset, 0, 0)
left = node
return left
# ============================================================
# C 风格指针后缀（'*' / '**'）—— 与 TransPyC lib/vast 的 _AtPtrSuffix 同源
# ============================================================
# Start position (line/col) of the last consumed C-style pointer suffix ('*' / '**').
# _parse_paren matches it against ps.prev to tell the paren-less cast `(T*)x` from a
# plain group like `(a | b)`. Recorded as ints, not as a token pointer: comparing two
# token pointers whose declared pointer levels differ is rejected by the compare check.
_g_ptr_suffix_line: int = 0
_g_ptr_suffix_col: int = 0
def _ptr_suffix_end_op(op: int) -> int:
"""后缀末尾 token 是否为“无右操作数”位置（']' ',' ')' ':' '='）。
这些位置后接 '*'/'**' 时，乘法/幂本就缺右操作数（原语法错误），
故按指针后缀处理不会改变任何合法表达式的语义（x[a * b] / a ** b 等不受影响）。
"""
if op == TokOp.Rsqb or op == TokOp.Comma or op == TokOp.RPar \
or op == TokOp.Colon or op == TokOp.Equal:
return 1
return 0
def _at_ptr_suffix(ps: rawptr[Parser], base: rawptr[AST]) -> int:
"""判定当前 '*'/'**' 串是否为作用于 base 的 C 风格指针后缀。
判据（与 vast 一致）：base 是类型原子（Name/Attribute/Subscript），
且从当前 '*'/'**' 起跨过连续 '*'/'**' 与 Nl 后，紧跟 ']' / ',' / ')' / ':' / '='。
该判据同时覆盖注解与表达式上下文（如 list[ast.AST*] / list[ast.AST*]*）。
"""
if base is None:
return 0
k: int = base.kind()
if k != ASTKind.Name and k != ASTKind.Attribute and k != ASTKind.Subscript:
return 0
nxt: rawptr[Token] = ps.cur
stars: int = 0
while nxt is not None:
if nxt.type == TokenType.Op \
and (nxt.op_subtype == TokOp.Star or nxt.op_subtype == TokOp.StarStar):
stars += 1
nxt = nxt.next
elif nxt.type == TokenType.Nl:
nxt = nxt.next
else:
break
if stars == 0 or nxt is None or nxt.type != TokenType.Op:
return 0
return _ptr_suffix_end_op(nxt.op_subtype)
def _make_ptr_type(ps: rawptr[Parser], base: rawptr[AST]) -> rawptr[AST]:
"""构造一层指针包装：base | t.CPtr。"""
cptr: rawptr[AST] = Attribute(ps.pool, Name(ps.pool, "t", ASTCtx.Load),
"CPtr", ASTCtx.Load)
node: rawptr[AST] = BinOp(ps.pool, base, OpKind.BitOr, cptr)
_set_pos(node, base.lineno, base.col_offset, 0, 0)
return node
def _consume_ptr_suffix(ps: rawptr[Parser], base: rawptr[AST]) -> rawptr[AST]:
"""消费连续的 C 风格指针后缀（'*' 一层 / '**' 两层），展开为 base | t.CPtr 链。"""
global _g_ptr_suffix_line, _g_ptr_suffix_col
while True:
_skip_nl(ps)
if _match_op(ps, TokOp.StarStar):
_advance(ps)
_g_ptr_suffix_line = ps.prev.lineno
_g_ptr_suffix_col = ps.prev.col_offset
base = _make_ptr_type(ps, base)
base = _make_ptr_type(ps, base)
continue
if _match_op(ps, TokOp.Star):
_advance(ps)
_g_ptr_suffix_line = ps.prev.lineno
_g_ptr_suffix_col = ps.prev.col_offset
base = _make_ptr_type(ps, base)
continue
break
return base
def _at_cast_operand(ps: rawptr[Parser]) -> int:
"""Whether the current token may start the operand of a paren-less cast `(T*)x`.
'(' is deliberately absent: `(T*)(x)` is already handled by the trailer loop as a
cast call, and keeping it there leaves that path's AST untouched. Nl is not
skipped either: a line break after ')' starts a new statement, not an operand.
"""
if _cur_type(ps) == TokenType.Name:
return 1
if _cur_type(ps) == TokenType.Number:
return 1
if _cur_type(ps) == TokenType.String:
return 1
op: int = _cur_op(ps)
if op == TokOp.Lsqb or op == TokOp.LBrace or op == TokOp.Star or op == TokOp.Amp or op == TokOp.Minus or op == TokOp.Plus or op == TokOp.Tilde:
return 1
return 0
def _at_ptr_suffix_call(ps: rawptr[Parser], base: rawptr[AST]) -> int:
"""判定当前 '*'/'**' 串是否为 `T*(x)` 形式的指针后缀**强转调用**。
判据（与 TPC lib/vast 的 _AtPtrSuffixCall 同源）：
1. base 是类型原子（Name/Attribute/Subscript）；
2. base 与 '*'/'**' 串**紧贴**（token 行列相邻、同行，无空白/换行）；
3. '*'/'**' 串（串内逐个 token 紧贴）之后**紧贴**一个 '('。
即整段写作**无空白**的 `T*(x)`（C 风格强转写法）。为什么 '(' 不进
_ptr_suffix_end_op 表：`a * (b)` 是合法乘法（'(' 可作乘法右操作数起点），
无条件计入会误伤乘法。故叠加"前驱是类型原子 + 两侧紧贴"；`a * (b + c)` /
`n * (m - 1)` 等有空格者仍按乘法解析。
"""
if base is None:
return 0
k: int = base.kind()
if k != ASTKind.Name and k != ASTKind.Attribute and k != ASTKind.Subscript:
return 0
nxt: rawptr[Token] = ps.cur
pv: rawptr[Token] = ps.prev
if pv is None or nxt is None:
return 0
if pv.type == TokenType.Nl or pv.type == TokenType.NewLine:
return 0
if pv.end_lineno != nxt.lineno or pv.end_col_offset != nxt.col_offset:
return 0 # base 与 '*'/'**' 之间有空白/换行：非 C 风格强转写法
stars: int = 0
prev: rawptr[Token] = None
last: rawptr[Token] = None
while nxt is not None:
if nxt.type == TokenType.Op \
and (nxt.op_subtype == TokOp.Star or nxt.op_subtype == TokOp.StarStar):
if prev is not None \
and (nxt.lineno != prev.end_lineno or nxt.col_offset != prev.end_col_offset):
return 0 # 星串内部出现空白：非紧邻，不判为强转
stars += 1
prev = nxt
last = nxt
nxt = nxt.next
else:
break
if stars == 0 or last is None or nxt is None:
return 0
if nxt.type != TokenType.Op or nxt.op_subtype != TokOp.LPar:
return 0
if nxt.lineno != last.end_lineno or nxt.col_offset != last.end_col_offset:
return 0
return 1
def _consume_ptr_suffix_call(ps: rawptr[Parser], base: rawptr[AST]) -> rawptr[AST]:
"""把已认定为 `T*(...)` 的指针后缀强转解析为**调用**。
产物与既有强转形态 `(T | t.CPtr)(...)` 逐字同形：同为 Call
（func = base 展开的 `base | t.CPtr` 链，args/keywords = 括号实参），
不新造第二套 cast 表示。
"""
func: rawptr[AST] = _consume_ptr_suffix(ps, base)
return _parse_call(ps, func)
def _parse_term(ps: rawptr[Parser]) -> rawptr[AST]:
left: rawptr[AST] = _parse_factor(ps)
if left is None: return None
while True:
_skip_nl(ps) # 括号内行首 '*'/'/'/'//'/'%'/'@': (a\n * b)
if _match_op(ps, TokOp.Star) and _at_ptr_suffix(ps, left) != 0:
left = _consume_ptr_suffix(ps, left) # C 风格指针后缀，不按乘法处理
continue
op_token: int = 0
if _match_op(ps, TokOp.Star): op_token = TokOp.Star
elif _match_op(ps, TokOp.Slash): op_token = TokOp.Slash
elif _match_op(ps, TokOp.DSlash): op_token = TokOp.DSlash
elif _match_op(ps, TokOp.Percent): op_token = TokOp.Percent
elif _match_op(ps, TokOp.At): op_token = TokOp.At
if op_token == 0: break
_advance(ps)
_skip_nl(ps) # 括号内跨行: a *\n b
right: rawptr[AST] = _parse_factor(ps)
ast_op: int = _binop_from_op(op_token)
node: rawptr[AST] = BinOp(ps.pool, left, ast_op, right)
_set_pos(node, left.lineno, left.col_offset, 0, 0)
left = node
return left
def _parse_factor(ps: rawptr[Parser]) -> rawptr[AST]:
"""factor: '+' factor | '-' factor | '~' factor | '*' factor | '&' factor | power"""
if (_match_op(ps, TokOp.Plus) or _match_op(ps, TokOp.Minus) or _match_op(ps, TokOp.Tilde)
or _match_op(ps, TokOp.Star) or _match_op(ps, TokOp.Amp)):
op_token: int = _cur_op(ps)
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps)
# 一元减 + 整数字面量的**联合越界判定**：`-` 是一元运算符、不属于字面量，
# 故幅值合法（≤ 2^64-1，词法层已放行）的字面量取负后可能 < -2^63。
# 只有"直接作用于整数常量"的一元减在此判定：`1 - 2` / `a - b` 是二元减
# （经 _parse_arith，不进入本分支）；`-x`（变量）操作数不是常量，不判定。
if op_token == TokOp.Minus and _cur_type(ps) == TokenType.Number \
and ps.cur.is_float == 0 \
and _lex_int_neg_exceeds_i64(_cur_str(ps)) != 0:
_record_neg_literal_overflow(_cur_str(ps), lineno, col)
ps.error_count += 1
return None
operand: rawptr[AST] = _parse_factor(ps)
ast_op: int = OpKind.UAdd
if op_token == TokOp.Minus:
ast_op = OpKind.USub
elif op_token == TokOp.Tilde:
ast_op = OpKind.Invert
elif op_token == TokOp.Star:
ast_op = OpKind.Deref
elif op_token == TokOp.Amp:
ast_op = OpKind.AddrOf
node: rawptr[AST] = UnaryOp(ps.pool, ast_op, operand)
_set_pos(node, lineno, col, 0, 0)
return node
return _parse_power(ps)
def _parse_power(ps: rawptr[Parser]) -> rawptr[AST]:
"""power: await_expr ['**' factor] | atom_expr ['**' factor]"""
base: rawptr[AST] = _parse_atom_expr(ps)
if base is None: return None
if _match_op(ps, TokOp.StarStar) and _at_ptr_suffix(ps, base) != 0:
return _consume_ptr_suffix(ps, base) # C 风格指针后缀（'**' 计 2 层）
if _match_op(ps, TokOp.StarStar):
_advance(ps)
exp: rawptr[AST] = _parse_factor(ps)
node: rawptr[AST] = BinOp(ps.pool, base, OpKind.Pow, exp)
_set_pos(node, base.lineno, base.col_offset, 0, 0)
return node
return base
def _parse_atom_expr(ps: rawptr[Parser]) -> rawptr[AST]:
"""atom_expr: ['await'] atom trailer*"""
if _cur_kw(ps) == Keyword.Await:
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps)
inner: rawptr[AST] = _parse_atom_expr(ps)
node: rawptr[AST] = Await(ps.pool, inner)
_set_pos(node, lineno, col, 0, 0)
return node
base: rawptr[AST] = _parse_atom(ps)
if base is None: return None
# trailer loop
while True:
if (_match_op(ps, TokOp.Star) or _match_op(ps, TokOp.StarStar)) \
and _at_ptr_suffix_call(ps, base) != 0:
# `T*(x)` / `T**(x)` 指针后缀强转：解析为 cast 调用（与
# `(T | t.CPtr)(x)` 同形）。在 trailer 循环里消费，故其后的
# `.attr` / `[i]` / `(…)` 仍由本循环继续挂接。
base = _consume_ptr_suffix_call(ps, base)
elif _match_op(ps, TokOp.Dot):
_advance(ps)
if _cur_type(ps) != TokenType.Name:
ps.error_count += 1
break
attr_name: str = _cur_str(ps)
attr_lineno: int = _cur_lineno(ps)
attr_col: int = _cur_col(ps)
_advance(ps)
ctx: int = ASTCtx.Load
if _match_op(ps, TokOp.Equal):
ctx = ASTCtx.Store
elif _match_op(ps, TokOp.PlusEq) or _match_op(ps, TokOp.MinusEq):
ctx = ASTCtx.Store
base = Attribute(ps.pool, base, attr_name, ctx)
_set_pos(base, attr_lineno, attr_col, 0, 0)
elif _match_op(ps, TokOp.LPar):
base = _parse_call(ps, base)
elif _match_op(ps, TokOp.Lsqb):
base = _parse_subscript(ps, base)
else:
break
return base
def _parse_call(ps: rawptr[Parser], func: rawptr[AST]) -> rawptr[AST]:
"""Parse function call trailer. Both args and keywords are lists."""
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps) # (
_skip_nls_and_newlines(ps) # Skip the line break after '('
args: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
keywords: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
if not _match_op(ps, TokOp.RPar):
while True:
# **expr
if _match_op(ps, TokOp.StarStar):
_advance(ps)
_skip_nls_and_newlines(ps)
val: rawptr[AST] = _parse_expr(ps)
kw: rawptr[AST] = KwNode(ps.pool, None, val)
_set_pos(kw, _cur_lineno(ps), _cur_col(ps), 0, 0)
keywords.append(kw)
# *expr
elif _match_op(ps, TokOp.Star):
_advance(ps)
_skip_nls_and_newlines(ps)
val = _parse_expr(ps)
star: rawptr[AST] = Starred(ps.pool, val, ASTCtx.Load)
_set_pos(star, val.lineno, val.col_offset, 0, 0)
args.append(star)
# keyword = expr
elif _cur_type(ps) == TokenType.Name:
if ps.cur.next is not None and ps.cur.next.type == TokenType.Op and ps.cur.next.op_subtype == TokOp.Equal:
kw_name: str = _cur_str(ps)
kw_lineno: int = _cur_lineno(ps)
kw_col: int = _cur_col(ps)
_advance(ps) # name
_advance(ps) # =
_skip_nls_and_newlines(ps)
val = _parse_expr(ps)
kw = KwNode(ps.pool, kw_name, val)
_set_pos(kw, kw_lineno, kw_col, 0, 0)
keywords.append(kw)
else:
val = _parse_expr(ps)
args.append(val)
else:
val = _parse_expr(ps)
args.append(val)
_skip_nls_and_newlines(ps) # Skip the line break after the parameter
if not _accept_op(ps, TokOp.Comma):
break
_skip_nls_and_newlines(ps) # Skip the line break after the comma
_skip_nls_and_newlines(ps) # Skip the line break before ')'
_expect_op(ps, TokOp.RPar)
node: rawptr[AST] = Call(ps.pool, func, args, keywords)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_subscript(ps: rawptr[Parser], value: rawptr[AST]) -> rawptr[AST]:
"""Parse subscript trailer."""
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps) # [
_skip_nl(ps)
slice_node: rawptr[AST] = _parse_slice(ps)
_expect_op(ps, TokOp.Rsqb)
ctx: int = ASTCtx.Load
if _match_op(ps, TokOp.Equal):
ctx = ASTCtx.Store
node: rawptr[AST] = Subscript(ps.pool, value, slice_node, ctx)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_slice(ps: rawptr[Parser]) -> rawptr[AST]:
"""Parse slice: [expr] ':' [expr] [':' [expr]] | expr | expr, expr, ..."""
lower: rawptr[AST] = None
# 空下标（`T[]`，类型注解位置的无签名函数指针 t.Callable[]）：`[` 后直接 `]`
# ⇒ slice 为 None（由类型解析侧按"空参数表"处理），不能调 _parse_expr，
# 否则报"意外的 token ']'"。
if not _match_op(ps, TokOp.Colon) and not _match_op(ps, TokOp.Rsqb):
lower = _parse_expr(ps)
# Check if there’s a comma (tuple syntax: a[x, y, ...])
if _match_op(ps, TokOp.Comma):
elts: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
if lower is not None:
elts.append(lower)
while _accept_op(ps, TokOp.Comma):
if _match_op(ps, TokOp.Rsqb):
break # Trailing comma
elem: rawptr[AST] = _parse_expr(ps)
if elem is not None:
elts.append(elem)
lineno: int = 0
if lower is not None:
lineno = lower.lineno
node: rawptr[AST] = Tuple(ps.pool, elts, ASTCtx.Load)
_set_pos(node, lineno, 0, 0, 0)
return node
if _match_op(ps, TokOp.Colon):
_advance(ps)
upper: rawptr[AST] = None
if not _match_op(ps, TokOp.Colon) and not _match_op(ps, TokOp.Rsqb):
upper = _parse_expr(ps)
step: rawptr[AST] = None
if _accept_op(ps, TokOp.Colon):
if not _match_op(ps, TokOp.Rsqb):
step = _parse_expr(ps)
lineno: int = 0
if lower is not None:
lineno = lower.lineno
elif upper is not None:
lineno = upper.lineno
else:
lineno = _cur_lineno(ps)
node: rawptr[AST] = Slice(ps.pool, lower, upper, step)
_set_pos(node, lineno, 0, 0, 0)
return node
return lower
def _parse_atom(ps: rawptr[Parser]) -> rawptr[AST]:
"""atom: the smallest unit of expression"""
ttype: int = _cur_type(ps)
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
if ttype == TokenType.Name:
kw: int = _cur_kw(ps)
if kw == Keyword.True_:
_advance(ps)
return _new_const_bool(ps.pool, 1, lineno, col)
if kw == Keyword.False_:
_advance(ps)
return _new_const_bool(ps.pool, 0, lineno, col)
if kw == Keyword.None_:
_advance(ps)
return _new_const_none(ps.pool, lineno, col)
name_str: str = _cur_str(ps)
_advance(ps)
node: rawptr[AST] = Name(ps.pool, name_str, ASTCtx.Load)
_set_pos(node, lineno, col, 0, 0)
return node
if ttype == TokenType.Number:
tok: rawptr[Token] = ps.cur
_advance(ps)
if tok.is_float:
return _new_const_float(ps.pool, tok.float_val, lineno, col)
return _new_const_int(ps.pool, tok.int_val, lineno, col)
if ttype == TokenType.String:
sval: str = _cur_str(ps)
is_fstr: int = ps.cur.kw_subtype & 4
is_triple: int = ps.cur.kw_subtype & 8
is_dquote: int = ps.cur.kw_subtype & 16
is_char: int = ps.cur.kw_subtype & 32
_advance(ps)
node: rawptr[AST]
if is_fstr:
node = _parse_fstring(ps, sval, lineno, col)
else:
node = _new_const_str(ps.pool, sval, lineno, col, is_dquote, is_triple, is_char)
# Python 相邻字符串字面量隐式拼接："a" "b" → "ab"
# Nl 是括号内换行 token（允许括号内跨行拼接）；语句级换行是 NewLine，不跳过
while True:
_skip_nl(ps)
if _cur_type(ps) != TokenType.String:
break
n_sval: str = _cur_str(ps)
n_fstr: int = ps.cur.kw_subtype & 4
n_triple: int = ps.cur.kw_subtype & 8
n_dquote: int = ps.cur.kw_subtype & 16
n_char: int = ps.cur.kw_subtype & 32
_advance(ps)
n_node: rawptr[AST]
if n_fstr:
n_node = _parse_fstring(ps, n_sval, lineno, col)
else:
n_node = _new_const_str(ps.pool, n_sval, lineno, col, n_dquote, n_triple, n_char)
node = _concat_str_nodes(ps.pool, node, n_node)
return node
if _match_op(ps, TokOp.Ellipsis):
_advance(ps)
return _new_const_none(ps.pool, lineno, col)
if _match_op(ps, TokOp.LPar):
return _parse_paren(ps)
if _match_op(ps, TokOp.Lsqb):
return _parse_list(ps)
if _match_op(ps, TokOp.LBrace):
return _parse_brace(ps)
_record_parse_err(ps)
ps.error_count += 1
return None
def _parse_paren(ps: rawptr[Parser]) -> rawptr[AST]:
"""Parenthesis expressions: () | (expr) | (expr,) | (expr, ...) | (yield)"""
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps) # (
_skip_nl(ps) # Skip the line break after '(' (multi-line tuple)
if _match_op(ps, TokOp.RPar):
_advance(ps)
# Empty tuple
empty: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
node: rawptr[AST] = Tuple(ps.pool, empty, ASTCtx.Load)
_set_pos(node, lineno, col, 0, 0)
return node
# yield
if _cur_kw(ps) == Keyword.Yield:
y: rawptr[AST] = _parse_yield(ps)
_expect_op(ps, TokOp.RPar)
return y
first: rawptr[AST] = _parse_expr(ps)
if _match_op(ps, TokOp.Comma):
# Tuple
elts: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
elts.append(first)
while _accept_op(ps, TokOp.Comma):
_skip_nl(ps)
if _match_op(ps, TokOp.RPar):
break
e: rawptr[AST] = _parse_expr(ps)
elts.append(e)
_skip_nl(ps) # 末元素无尾逗号时收尾（多行元组）
_expect_op(ps, TokOp.RPar)
node = Tuple(ps.pool, elts, ASTCtx.Load)
_set_pos(node, lineno, col, 0, 0)
return node
# Paren-less C cast `(T*)x`: the inner expression ends with a pointer suffix
# (ps.prev is that suffix's '*'/'**' token) and the operand follows ')' directly.
# Same product as `(T*)(x)` / `T*(x)`: a Call whose func is `T | t.CPtr`.
is_ptr_cast: int = 0
if _g_ptr_suffix_line > 0 and _match_op(ps, TokOp.RPar) and ps.prev.lineno == _g_ptr_suffix_line and ps.prev.col_offset == _g_ptr_suffix_col:
is_ptr_cast = 1
_skip_nl(ps) # (expr\n) 单表达式括号跨行收尾
_expect_op(ps, TokOp.RPar)
# Check for generator expression
if _cur_kw(ps) == Keyword.For:
return _parse_comprehension(ps, first, ASTKind.GeneratorExp, lineno, col)
if is_ptr_cast != 0 and _at_cast_operand(ps) != 0:
operand: rawptr[AST] = _parse_factor(ps)
cast_args: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
cast_args.append(operand)
cast_kws: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
cast_node: rawptr[AST] = Call(ps.pool, first, cast_args, cast_kws)
_set_pos(cast_node, first.lineno, first.col_offset, 0, 0)
return cast_node
return first
def _parse_list(ps: rawptr[Parser]) -> rawptr[AST]:
"""List expressions: [] | [expr_list] | [expr_list comp_for]"""
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps) # [
_skip_nl(ps) # Skip the line break after '[' (multi-line list)
if _match_op(ps, TokOp.Rsqb):
_advance(ps)
empty: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
node: rawptr[AST] = List(ps.pool, empty, ASTCtx.Load)
_set_pos(node, lineno, col, 0, 0)
return node
first: rawptr[AST] = _parse_expr(ps)
if _cur_kw(ps) == Keyword.For:
node = _parse_comprehension(ps, first, ASTKind.ListComp, lineno, col)
_expect_op(ps, TokOp.Rsqb)
return node
elts: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
elts.append(first)
while _accept_op(ps, TokOp.Comma):
_skip_nl(ps)
if _match_op(ps, TokOp.Rsqb):
break
e: rawptr[AST] = _parse_expr(ps)
elts.append(e)
_skip_nl(ps) # 末元素无尾逗号时收尾（base64.vp 多行列表）
_expect_op(ps, TokOp.Rsqb)
node = List(ps.pool, elts, ASTCtx.Load)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_brace(ps: rawptr[Parser]) -> rawptr[AST]:
"""Brace expressions: {} | {set_items} | {dict_items} | {comp_for}"""
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps) # {
_skip_nl(ps) # Skip the line break after '{' (multi-line dict)
if _match_op(ps, TokOp.RBrace):
_advance(ps)
empty_k: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
empty_v: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
node: rawptr[AST] = Dict(ps.pool, empty_k, empty_v)
_set_pos(node, lineno, col, 0, 0)
return node
first: rawptr[AST] = _parse_expr(ps)
# dict: key: value
if _match_op(ps, TokOp.Colon):
_advance(ps)
val: rawptr[AST] = _parse_expr(ps)
if _cur_kw(ps) == Keyword.For:
# dict comprehension
gens: rawptr[list[rawptr[AST]]] = _parse_comp_for(ps)
node = DictComp(ps.pool, first, val, gens)
_set_pos(node, lineno, col, 0, 0)
_expect_op(ps, TokOp.RBrace)
return node
keys: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
values: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
keys.append(first)
values.append(val)
while _accept_op(ps, TokOp.Comma):
_skip_nl(ps)
if _match_op(ps, TokOp.RBrace):
break
k: rawptr[AST] = _parse_expr(ps)
_expect_op(ps, TokOp.Colon)
v: rawptr[AST] = _parse_expr(ps)
keys.append(k)
values.append(v)
_skip_nl(ps) # 末元素无尾逗号时收尾
_expect_op(ps, TokOp.RBrace)
node = Dict(ps.pool, keys, values)
_set_pos(node, lineno, col, 0, 0)
return node
# set
if _cur_kw(ps) == Keyword.For:
node = _parse_comprehension(ps, first, ASTKind.SetComp, lineno, col)
_expect_op(ps, TokOp.RBrace)
return node
elts: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
elts.append(first)
while _accept_op(ps, TokOp.Comma):
_skip_nl(ps) # 括号内跨行集合字面量
if _match_op(ps, TokOp.RBrace):
break
e: rawptr[AST] = _parse_expr(ps)
elts.append(e)
_skip_nl(ps) # 末元素无尾逗号时收尾
_expect_op(ps, TokOp.RBrace)
node = Set(ps.pool, elts)
_set_pos(node, lineno, col, 0, 0)
return node
# ============================================================
# f-string parsing
# ============================================================
def _parse_sub_expr(ps: rawptr[Parser], src: str) -> rawptr[AST]:
"""Create a sub-lexer and parser to parse expressions (used for {expr} inside f-strings)"""
pool: rawptr[memhub.MemManager] = ps.pool
lx: rawptr[Lexer] = new_lexer(pool)
if lx is None: return None
_lexer_init(lx, src, pool)
tokens: rawptr[Token] = tokenize(lx)
if tokens is None: return None
sub_ps: rawptr[Parser] = new_parser(pool)
if sub_ps is None: return None
_parser_init(sub_ps, tokens, pool)
return _parse_test(sub_ps)
def _append_str_value(pool: rawptr[memhub.MemManager],
values: rawptr[list[rawptr[AST]]],
node: rawptr[AST]):
"""将字符串节点展开加入 values（JoinedStr 展开其 values，其余直接追加）"""
if node is None:
return
if node.kind() == ASTKind.JoinedStr:
js_val: rawptr[JoinedStr] = (rawptr[JoinedStr])(node)
if js_val.values is not None:
n_v: sizet = js_val.values.__len__()
i_v: sizet
for i_v in range(n_v):
values.append(js_val.values.get(i_v))
else:
values.append(node)
def _concat_str_nodes(pool: rawptr[memhub.MemManager],
a: rawptr[AST], b: rawptr[AST]) -> rawptr[AST]:
"""合并相邻字符串字面量节点（Python 隐式拼接 "a" "b" → "ab"）
纯常量字符串直接拼接文本；涉及 f-string（JoinedStr/FormattedValue）时
合并到 JoinedStr 的 values 列表。
"""
if a is None:
return b
if b is None:
return a
# 两个都是常量字符串 → 直接拼接文本
if a.kind() == ASTKind.Constant and b.kind() == ASTKind.Constant:
ca_c: rawptr[Constant] = (rawptr[Constant])(a)
cb_c: rawptr[Constant] = (rawptr[Constant])(b)
if ca_c.const_kind == CONST_STR and cb_c.const_kind == CONST_STR \
and ca_c.str_val is not None and cb_c.str_val is not None:
la_c: sizet = string.strlen(ca_c.str_val)
lb_c: sizet = string.strlen(cb_c.str_val)
buf_c: str = pool.alloc(la_c + lb_c + 1)
if buf_c is not None:
string.memcpy(buf_c, ca_c.str_val, la_c)
string.memcpy(buf_c + la_c, cb_c.str_val, lb_c + 1)
return _new_const_str(pool, buf_c, a.lineno, a.col_offset, 0, 0, 0)
# 涉及 f-string → 合并到 JoinedStr
values_c: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](pool, 8)
_append_str_value(pool, values_c, a)
_append_str_value(pool, values_c, b)
if values_c.__len__() == 1:
return values_c.get(0)
node_c: rawptr[AST] = JoinedStr(pool, values_c)
_set_pos(node_c, a.lineno, a.col_offset, 0, 0)
return node_c
def _parse_fstring(ps: rawptr[Parser], content: str,
lineno: int, col: int) -> rawptr[AST]:
"""Parse f-string content, building JoinedStr or FormattedValue nodes"""
pool: rawptr[memhub.MemManager] = ps.pool
values: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](pool, 8)
i: sizet = 0
content_len: sizet = string.strlen(content)
# Literal text buffer for f-string content
lit_buf: str = pool.alloc(content_len + 1)
if lit_buf is None: return None
lit_pos: sizet = 0
while i < content_len:
ch: char = content[i]
if ch == '{':
# {{ → Literal {
if i + 1 < content_len and content[i + 1] == '{':
lit_buf[lit_pos] = '{'
lit_pos += 1
i += 2
continue
# Refresh literal buffer
if lit_pos > 0:
lit_buf[lit_pos] = '\0'
lit_node: rawptr[AST] = _new_const_str(pool, lit_buf, lineno, col)
values.append(lit_node)
lit_buf = pool.alloc(content_len + 1)
if lit_buf is None: break
lit_pos = 0
# Find matching closing '}' while tracking nested '{}' and string literals inside the f-string
j: sizet = i + 1
depth: int = 1
conv_pos: sizet = 0
fmt_pos: sizet = 0
while j < content_len and depth > 0:
c: char = content[j]
if c == '\'' or c == '"':
# 跳过表达式块内的嵌套字符串字面量（PEP 701 同引号嵌套），
# 避免其内部字符干扰 {} 深度/转换/格式分隔符定位
nq: char = c
j += 1
while j < content_len:
c2: char = content[j]
if c2 == '\\':
j += 2
continue
if c2 == nq:
j += 1
break
j += 1
continue
if c == '{':
depth += 1
elif c == '}':
depth -= 1
if depth == 0:
break
elif depth == 1:
if c == '!' and conv_pos == 0 and fmt_pos == 0:
if j + 1 < content_len:
nc: char = content[j + 1]
if nc == 's' or nc == 'r' or nc == 'a':
if j + 2 < content_len:
after: char = content[j + 2]
if after == ':' or after == '}':
conv_pos = j
elif j + 2 == content_len:
conv_pos = j
elif c == ':' and fmt_pos == 0:
fmt_pos = j
j += 1
if depth != 0:
_record_parse_err(ps)
ps.error_count += 1
break
# Extract expression text
expr_end: sizet = j
if conv_pos > 0:
expr_end = conv_pos
elif fmt_pos > 0:
expr_end = fmt_pos
expr_len: sizet = expr_end - i - 1
if expr_len == 0:
_record_parse_err(ps)
ps.error_count += 1
i = j + 1
continue
expr_buf: str = pool.alloc(expr_len + 1)
if expr_buf is None: break
string.memcpy(expr_buf, content + i + 1, expr_len)
expr_buf[expr_len] = '\0'
# Parse expression text
expr_node: rawptr[AST] = _parse_sub_expr(ps, expr_buf)
# Conversion specifier
conversion: int = -1
if conv_pos > 0:
conv_char: char = content[conv_pos + 1]
if conv_char == 's':
conversion = 115
elif conv_char == 'r':
conversion = 114
elif conv_char == 'a':
conversion = 97
# Format specifier
format_spec: rawptr[AST] = None
if fmt_pos > 0:
spec_len: sizet = j - fmt_pos - 1
if spec_len > 0:
spec_buf: str = pool.alloc(spec_len + 1)
if spec_buf is not None:
string.memcpy(spec_buf, content + fmt_pos + 1, spec_len)
spec_buf[spec_len] = '\0'
format_spec = _new_const_str(pool, spec_buf, lineno, col)
fv: rawptr[AST] = FormattedValue(pool, expr_node, conversion, format_spec)
_set_pos(fv, lineno, col, 0, 0)
values.append(fv)
i = j + 1
elif ch == '}':
if i + 1 < content_len and content[i + 1] == '}':
lit_buf[lit_pos] = '}'
lit_pos += 1
i += 2
else:
_record_parse_err(ps)
ps.error_count += 1
i += 1
else:
lit_buf[lit_pos] = ch
lit_pos += 1
i += 1
# Refresh remaining literal buffer
if lit_pos > 0:
lit_buf[lit_pos] = '\0'
lit_node2: rawptr[AST] = _new_const_str(pool, lit_buf, lineno, col)
values.append(lit_node2)
# If no content, return empty string
if values.__len__() == 0:
return _new_const_str(pool, "", lineno, col)
# If only one FormattedValue and no literal, return FormattedValue
if values.__len__() == 1:
first: rawptr[AST] = values.get(0)
if first.kind() == ASTKind.FormattedValue:
return first
node: rawptr[AST] = JoinedStr(pool, values)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_comprehension(ps: rawptr[Parser], elt: rawptr[AST],
vtype: int, lineno: int, col: int) -> rawptr[AST]:
"""General comprehension: elt comp_for"""
gens: rawptr[list[rawptr[AST]]] = _parse_comp_for(ps)
node: rawptr[AST] = None
if vtype == ASTKind.ListComp:
node = ListComp(ps.pool, elt, gens)
elif vtype == ASTKind.SetComp:
node = SetComp(ps.pool, elt, gens)
else:
node = GeneratorExp(ps.pool, elt, gens)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_comp_for(ps: rawptr[Parser]) -> rawptr[list[rawptr[AST]]]:
"""comp_for: ['async'] 'for' exprlist 'in' or_test [comp_if] (comp_for)*
Returns generators list。"""
gens: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
while True:
is_async: int = 0
if _cur_kw(ps) == Keyword.Async:
is_async = 1
_advance(ps)
if _cur_kw(ps) != Keyword.For:
break
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps)
target: rawptr[AST] = _parse_target_list(ps)
if _cur_kw(ps) != Keyword.In:
ps.error_count += 1
break
_advance(ps)
iter_: rawptr[AST] = _parse_or_test(ps)
# Collect ifs to list
ifs: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
while _cur_kw(ps) == Keyword.If:
_advance(ps)
cond: rawptr[AST] = _parse_or_test(ps)
ifs.append(cond)
comp: rawptr[AST] = Comprehension(ps.pool, target, iter_, ifs, is_async)
_set_pos(comp, lineno, col, 0, 0)
gens.append(comp)
return gens
def _parse_target_list(ps: rawptr[Parser]) -> rawptr[AST]:
"""Parse assignment target list (comma-separated)"""
first: rawptr[AST] = _parse_target(ps)
if _match_op(ps, TokOp.Comma):
elts: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
elts.append(first)
while _accept_op(ps, TokOp.Comma):
if _cur_kw(ps) == Keyword.In:
break
e: rawptr[AST] = _parse_target(ps)
elts.append(e)
node: rawptr[AST] = Tuple(ps.pool, elts, ASTCtx.Store)
_set_pos(node, first.lineno, first.col_offset, 0, 0)
return node
return first
def _parse_target(ps: rawptr[Parser]) -> rawptr[AST]:
"""Parse assignment target"""
if _match_op(ps, TokOp.Star):
_advance(ps)
inner: rawptr[AST] = _parse_atom_expr(ps)
node: rawptr[AST] = Starred(ps.pool, inner, ASTCtx.Store)
_set_pos(node, inner.lineno, inner.col_offset, 0, 0)
return node
if _match_op(ps, TokOp.LPar):
_advance(ps)
_skip_nl(ps)
t_: rawptr[AST] = _parse_target_list(ps)
_skip_nl(ps)
_expect_op(ps, TokOp.RPar)
return t_
if _match_op(ps, TokOp.Lsqb):
_advance(ps)
_skip_nl(ps)
first: rawptr[AST] = _parse_target(ps)
elts: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
elts.append(first)
while _accept_op(ps, TokOp.Comma):
_skip_nl(ps)
if _match_op(ps, TokOp.Rsqb):
break
e: rawptr[AST] = _parse_target(ps)
elts.append(e)
_expect_op(ps, TokOp.Rsqb)
node = List(ps.pool, elts, ASTCtx.Store)
_set_pos(node, first.lineno, first.col_offset, 0, 0)
return node
# Normal atom_expr, ctx=Store
node = _parse_atom_expr(ps)
if node is not None:
_set_ctx(node, ASTCtx.Store)
return node
def _parse_yield(ps: rawptr[Parser]) -> rawptr[AST]:
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps) # yield
if _cur_kw(ps) == Keyword.From:
_advance(ps)
val: rawptr[AST] = _parse_expr(ps)
node: rawptr[AST] = YieldFrom(ps.pool, val)
_set_pos(node, lineno, col, 0, 0)
return node
val2: rawptr[AST] = None
if not _match_op(ps, TokOp.RPar) and _cur_type(ps) != TokenType.NewLine and _cur_type(ps) != TokenType.EndMarker:
if _cur_type(ps) == TokenType.Op and (_cur_op(ps) == TokOp.Comma or _cur_op(ps) == TokOp.Rsqb):
pass
else:
val2 = _parse_test(ps)
node = Yield(ps.pool, val2)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_varargslist(ps: rawptr[Parser], is_lambda: int = 0) -> rawptr[AST]:
"""Parse function parameter list
New API: args/defaults/kw_defaults are lists, vararg/kwarg are fields."""
args: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
vararg: rawptr[AST] = None
kwarg: rawptr[AST] = None
defaults: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
kw_defaults: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
while True:
_skip_nl(ps)
if _match_op(ps, TokOp.RPar):
break
# *args 或 * (kwonly separated)
if _match_op(ps, TokOp.Star):
_advance(ps)
_skip_nl(ps)
if _cur_type(ps) == TokenType.Name:
vararg = _parse_arg_def(ps, is_lambda)
if not _accept_op(ps, TokOp.Comma):
break
_skip_nl(ps)
continue
# **kwargs
if _match_op(ps, TokOp.StarStar):
_advance(ps)
_skip_nl(ps)
kwarg = _parse_arg_def(ps, is_lambda)
if not _accept_op(ps, TokOp.Comma):
break
_skip_nl(ps)
continue
# / Delimiter (posonly end)— New API: posonly/regular are not distinguished
if _match_op(ps, TokOp.Slash):
_advance(ps)
if not _accept_op(ps, TokOp.Comma):
break
_skip_nl(ps)
continue
# Normal parameter
a: rawptr[AST] = _parse_arg_def(ps, is_lambda)
if a is None: break
args.append(a)
if _match_op(ps, TokOp.Equal):
_advance(ps)
d: rawptr[AST] = _parse_test(ps)
defaults.append(d)
if not _accept_op(ps, TokOp.Comma):
break
_skip_nl(ps)
return Arguments(ps.pool, args, vararg, kwarg, defaults, kw_defaults)
def _parse_array_prefix_declarator(ps: rawptr[Parser]) -> rawptr[AST]:
"""C 风格前置数组声明符 `[N]T` / `[]T` → `t.CArray[T, N]` / `t.CArray[T, None]`
只在注解位置（_parse_type_ann）消费；当前 token 不是 '[' 时返回 None。
脱糖目标与 TPC / StubGen / DeclarationGenerator 的规范数组形态同形
（`t.CArray[E, N]`；无长度按 `t.CArray[E, None]` = 指针模式），即把
`coeffs: const [8]double` 就地等价成文档给出的修法
`coeffs: const t.CArray[double, 8]`（Debugger/type-descriptors.md §3 第 6 例）。
必须在 `_parse_test` 之前调用：否则 `[N]` 会被解析成 list 字面量，随后在元素
类型标识符处报"意外的 token"（includes/binascii.vp `HEX_VALS: [256]int` 现象）。
"""
if _match_op(ps, TokOp.Lsqb) == 0:
return None
lb_lineno: int = _cur_lineno(ps)
lb_col: int = _cur_col(ps)
_advance(ps) # 消费 '['
count_node: rawptr[AST] = None
if _match_op(ps, TokOp.Rsqb) == 0:
# 长度表达式（常量/常量表达式；非法长度由下游 ArrayDims 判据拒绝）
count_node = _parse_test(ps)
if count_node is None:
_record_parse_err(ps)
return None
if _match_op(ps, TokOp.Rsqb) == 0:
_record_parse_err(ps) # 缺少 ']'
ps.error_count += 1
return None
_advance(ps) # 消费 ']'
# 元素类型：递归 _parse_type_ann，故 `[N]const int` / `[N]int*` 走同一规则源
elem: rawptr[AST] = _parse_type_ann(ps)
if elem is None:
return None
if count_node is None:
count_node = _new_const_none(ps.pool, lb_lineno, lb_col)
# 元素大小必须显式传 8（list[T](pool, elem_size) 按字节步长寻址；
# 传 2 会让 8 字节指针按 2 字节步长写入 → 堆越界，ASan 在 _set_parent_list
# 抛 access-violation）。与文件内其它 list[rawptr[AST]] 站点同参。
elts: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
elts.append(elem)
elts.append(count_node)
carray: rawptr[AST] = Subscript(ps.pool,
Attribute(ps.pool, Name(ps.pool, "t", ASTCtx.Load),
"CArray", ASTCtx.Load),
Tuple(ps.pool, elts, ASTCtx.Load),
ASTCtx.Load)
_set_pos(carray, lb_lineno, lb_col, 0, 0)
return carray
def _parse_type_ann(ps: rawptr[Parser]) -> rawptr[AST]:
"""解析类型注解，支持类型描述符前缀（const / volatile）与 C 风格指针后缀（* / **）。
类型描述符（软关键字，只在注解位置生效；与 TPC 侧
`SymbolUtils._ExpandTypeDescriptors` 的展开产物逐字段同形）：
const int → t.CConst | int
volatile int → t.CVolatile | int
const volatile int → t.CConst | t.CVolatile | int（顺序任意，两限定符都累积）
const t.CInt32T → t.CConst | t.CInt32T
const rawptr[Foo] → t.CConst | rawptr[Foo]
const int | t.CPtr → t.CConst | int | t.CPtr（粘贴在联合的最左操作数：与
TPC 文本展开 `t.CConst | int | t.CPtr` 的解析树同形）
非法形态**响亮失败**（非静默，不丢描述符）：缺类型（`a: const`）/
后随非类型（`a: const 1`）/ 裸用（语句只有 `const`，见 _is_bare_descriptor）。
C 风格指针后缀：
int* → rawptr[int] （i32*）
int** → (rawptr[int]) | t.CPtr （i32**）
int****** → 6 层嵌套 （i32******）
_parse_test 会把后缀 '*' 消费为乘法（Mult）、'**' 消费为幂（Pow），
右操作数解析失败时产生 BinOp(op, right=None) 垃圾节点并 error_count+1
（'int***' 形态为 BinOp(Pow, UnaryOp(Deref, None))，StarStar+Star）。
本函数回收该形态并恢复 error_count，再按层数嵌套构造 BinOp(BitOr, t.CPtr)。
"""
# 类型描述符前缀：收集 const / volatile 标记（软关键字，仅本位置消费）
mods: rawptr[list[rawptr[AST]]] = None
mod_count: int = 0
mod_lineno: int = _cur_lineno(ps)
mod_col: int = _cur_col(ps)
# `static` 修饰符（语句起始消费，`static x: int = 0`）：把 t.CStatic 标记
# 追加为描述符序列的**第一个**（后续由 _PrependDescriptorMarkers 贴到类型
# 联合最左），与手写 `x: t.CStatic | int` 逐字段同形；随即清零，只作用于
# 紧随其后的这一个注解（下一个注解不会继承）。
if ps.pending_static != 0:
ps.pending_static = 0
if mods is None:
mods = list[rawptr[AST]](ps.pool, 8)
st_mk: rawptr[AST] = _MakeStaticMarker(ps)
if st_mk is not None:
mods.append(st_mk)
mod_count += 1
while _IsDescriptorKeyword(ps) != 0 or _IsMagicAttributeCall(ps) != 0:
if mods is None:
# 元素大小显式传 8：list[T](pool, elem_size) 按字节步长寻址，传 4 会让
# 两个及以上描述符（如 `const volatile`）的第 2 个 8 字节指针越位写入。
mods = list[rawptr[AST]](ps.pool, 8)
if _IsMagicAttributeCall(ps) != 0:
# C 风格 `a: __attribute__(section(".text.wo"), aligned(16)) const int = ...`：
# 与 const / volatile 同位置、同顺序自由、可混写；每个实参 desugar 成
# `t.attr.<name>`（带调用则保留实参）后并入同一 markers 列表，由
# _PrependDescriptorMarkers 贴到类型联合最左（与 TPC 文本展开同形）。
ac_node: rawptr[AST] = _parse_test(ps)
if ac_node is None:
break
if ac_node.kind() != ast.ASTKind.Call:
break
ac: ast.Call* = ast.Call*(ac_node)
if ac.args is not None:
acn: sizet = len(ac.args)
aci: sizet = 0
while aci < acn:
mk: rawptr[AST] = _MakeTAttrMarker(ps, ac.args.get(aci))
if mk is not None:
mods.append(mk)
mod_count += 1
aci += 1
continue
mods.append(_MakeDescriptorMarker(ps))
mod_count += 1
_advance(ps)
# C 风格前置数组声明符 `[N]T` / `[]T`（含 `const [N]T`）：就地脱糖为
# `t.CArray[T, N]` / `t.CArray[T, None]` 后直接返回。描述符此时已收集完毕，
# 就地贴到数组节点上（`t.CConst | t.CArray[T, N]`，与文档修法同形）。
# 该形态后不接指针后缀；全仓用法只有 `[N]T` / `[]T` 两种。
if _match_op(ps, TokOp.Lsqb) != 0:
arr: rawptr[AST] = _parse_array_prefix_declarator(ps)
if arr is None:
return None
if mod_count > 0:
arr = _PrependDescriptorMarkers(ps, arr, mods, mod_count)
return arr
if mod_count > 0 and _cur_type(ps) != TokenType.Name:
# 描述符后缺类型（`a: const` / `a: const = 1` / `def f(a: const)`）：
# 响亮失败——**不**把描述符静默丢掉。
_RecordDescriptorErr("描述符后缺少类型", mod_lineno, mod_col)
ps.error_count += 1
return None
before_ec: int = ps.error_count
base: rawptr[AST] = _parse_test(ps)
if base is None:
if mod_count > 0:
# 描述符已消费但类型解析失败（如 `a: const = 1`）：响亮失败保描述符报错。
_RecordDescriptorErr("描述符后缺少类型（类型解析失败）", mod_lineno, mod_col)
_record_parse_err(ps) # 注解解析失败：记录具体位置（error_count 已在 _parse_test 内增加）
return None
if mod_count > 0 and base.kind() == ASTKind.Constant:
# 后随非类型（`a: const 1` / `a: const "x"` / `a: const True`）：响亮失败
# （描述符只修饰类型，常量不是类型——不得静默降级/丢弃）。
_RecordDescriptorErr("描述符后随非类型（描述符只修饰类型）", mod_lineno, mod_col)
ps.error_count += 1
return None
ptr_level: int = 0
# 回收垃圾节点（_parse_test 对后缀 * / ** 的消费产物）
if base.kind() == ASTKind.BinOp:
gb: rawptr[BinOp] = (rawptr[BinOp])(base)
if gb.op == OpKind.Mult and gb.right is None:
ptr_level += 1
base = gb.left
elif gb.op == OpKind.Pow:
if gb.right is None:
ptr_level += 2
base = gb.left
elif gb.right.kind() == ASTKind.UnaryOp:
gu: rawptr[UnaryOp] = (rawptr[UnaryOp])(gb.right)
if gu.op == OpKind.Deref and gu.operand is None:
ptr_level += 3
base = gb.left
if ptr_level > 0:
ps.error_count = before_ec # 回收成功，恢复错误计数避免整文件解析失败
# 循环消费残留后缀（'**' 是词法单 token 计 2 层，'*' 计 1 层）
while _match_op(ps, TokOp.StarStar):
_advance(ps)
ptr_level += 2
while _match_op(ps, TokOp.Star):
_advance(ps)
ptr_level += 1
# 嵌套构造 rawptr[base]（n 层）
li_an: int = 0
while li_an < ptr_level:
cptr_an: rawptr[AST] = Attribute(ps.pool,
Name(ps.pool, "t", ASTCtx.Load),
"CPtr", ASTCtx.Load)
base = BinOp(ps.pool, base, OpKind.BitOr, cptr_an)
li_an += 1
# 描述符最后粘贴：与 TPC 文本展开同形（见 _PrependDescriptorMarkers）
if mod_count > 0:
base = _PrependDescriptorMarkers(ps, base, mods, mod_count)
return base
def _parse_arg_def(ps: rawptr[Parser], is_lambda: int = 0) -> rawptr[AST]:
"""arg: TokenType.Name [':' annotation]"""
if _cur_type(ps) != TokenType.Name:
ps.error_count += 1
return None
name_str: str = _cur_str(ps)
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps)
annotation: rawptr[AST] = None
if is_lambda == 0:
if _accept_op(ps, TokOp.Colon):
annotation = _parse_type_ann(ps)
result: rawptr[AST] = Arg(ps.pool, name_str, annotation)
_set_pos(result, lineno, col, 0, 0)
return result
# ============================================================
# Statement parsing (Design: directly append to parent, do not return a list)
# ============================================================
def _parse_suite_into_children(ps: rawptr[Parser], parent: rawptr[AST],
owner_is_class: int = 0):
"""Parse statement suite, directly append to parent's children.
suite: TokenType.NewLine TokenType.Indent stmt+ TokenType.Dedent | simple_stmt
owner_is_class：本套件的**直接拥有者**是否 ClassDef。套件解析期间置
ps.in_class_body，供 _ParseVirtualModified 判 `virtual def`/`novirtual def`
是否"直接写在类体内"（函数体 / 分支体 / 循环体都不算）。默认 0 ⇒ 除类体外的
所有套件拥有者天然正确，无需逐个改动调用点。
"""
prev_in_class: int = ps.in_class_body
ps.in_class_body = owner_is_class
_skip_nl(ps)
if _cur_type(ps) == TokenType.NewLine:
_advance(ps)
_skip_nl(ps)
if _cur_type(ps) != TokenType.Indent:
ps.error_count += 1
ps.in_class_body = prev_in_class
return
_advance(ps)
sidx: int = 0
while _cur_type(ps) != TokenType.Dedent and _cur_type(ps) != TokenType.EndMarker:
sidx += 1
# fail-fast 防卡死：步数超预算（token 链表成环 / 游标原地打转）立即报错返回
if ps.step_count > ps.step_limit:
_record_parse_err(ps)
return
# Decorator (class body methods can have decorators)
if _match_op(ps, TokOp.At):
ps.pending_decorators = _parse_decorators(ps)
before_cur: rawptr[Token] = ps.cur
_parse_statement_into(ps, parent)
# Defensive: if parse did not advance token, force advance to avoid infinite loop
if ps.cur is before_cur:
ps.error_count += 1
_advance(ps)
_skip_nl(ps)
_expect_type(ps, TokenType.Dedent)
ps.in_class_body = prev_in_class
return
# Single-line suite
_parse_simple_stmt_into(ps, parent)
ps.in_class_body = prev_in_class
def _parse_statement_into(ps: rawptr[Parser], parent: rawptr[AST]):
"""Parse statement (compound or simple), append to the parent."""
ttype: int = _cur_type(ps)
kw: int = _cur_kw(ps)
if ttype == TokenType.Name:
# define 常量声明：`define NAME[: T] = <字面量>`（**无括号**）。
# 必须早于 _parse_simple_stmt_into 分派——否则 `define` 被当普通标识符
# 解析成表达式语句，随后的 NAME 沦为残留 token（虽响亮失败但定位错误）。
# `define NAME(...)`（宏函数）被 _is_define_const 的 lookahead 排除，
# 仍走下方 _is_funcdef_kw 分支，语义逐字不变。
if kw == Keyword.Define and _is_define_const(ps) != 0:
dc_stmt: rawptr[AST] = _parse_define_const(ps)
if dc_stmt is not None:
parent.append(dc_stmt)
return
# typedef 类型别名声明：`typedef T1 = T2`（语句级保留字，无 lookahead——
# typedef 不在任何既有源码中作标识符）。非法形态在 _parse_typedef 内
# 响亮失败（_record_parse_err），不静默。
if _is_typedef(ps) != 0:
td_stmt: rawptr[AST] = _parse_typedef(ps)
if td_stmt is not None:
parent.append(td_stmt)
return
# virtual / novirtual 修饰符（软关键字）：只在"语句起始 + 紧跟 class/def"
# 时生效；合成既有装饰器 @t.CVTable / @t.NoVTable 后照常走 class/def 解析
# （见 _ParseVirtualModified）。放在 class/def 分派之前：修饰符必须先被消费，
# 后面的 class/def 才能落到 _is_funcdef_kw / Keyword.Class 分支。
# 与软关键字 fn→def 的关系：两种写法都接受（def 的别名 fn/fun/func/function），
# 与 TPC 端（先做 fn→def 文本展开、再判修饰符）结果一致。
if _IsVirtualModifier(ps) != 0:
_ParseVirtualModified(ps, parent)
return
# inline 修饰符（软关键字）：只在"语句起始 + inline + 紧跟函数定义"时生效，
# 合成既有装饰器 @t.always_inline 后照常走 def 解析（见 _ParseInlineModified）。
# 与 virtual/novirtual 同一位置、同一套 pending_decorators 通道。
if _IsInlineModifier(ps) != 0:
_ParseInlineModified(ps, parent)
return
# static 修饰符（软关键字）：语句起始 + static + （def | `Name :`）。
# 函数分支消费 static 并合成 @t.CStatic 装饰器（同 inline，已处理→return）；
# 变量分支仅消费 static 并置 pending_static（返回 0），继续走下方通用分派
# ——AnnAssign 解析经 _parse_type_ann 把 t.CStatic 插到类型联合最左。
# 语义（禁止跨模块使用）由 TransPyV 侧消费；此处只归一两种书写形态。
if _IsStaticModifier(ps) != 0:
if _ParseStaticModified(ps, parent) != 0:
return
# 类型描述符裸用：整条语句只有 `const` / `volatile`（缺类型）⇒ 响亮失败。
# 只在"描述符串后直接换行/文件尾/分号"时命中（软关键字边界），故
# `const = 5`（作标识符的赋值）/ `t.attr.const()` 等写法不命中、语义不变。
# 记录自定义报文后**继续走通用 simple_stmt 分派**：既消费 token（不制造
# 死循环），也不覆盖首次错误（_record_parse_err 只记首次错误）。
if _IsBareDescriptor(ps) != 0:
_RecordDescriptorErr("裸用描述符（整条语句只有描述符，缺类型）",
_cur_lineno(ps), _cur_col(ps))
ps.error_count += 1
if kw == Keyword.If or kw == Keyword.While or kw == Keyword.For or kw == Keyword.Try:
stmt: rawptr[AST] = _parse_compound(ps)
if stmt is not None:
parent.append(stmt)
return
if kw == Keyword.With or _is_funcdef_kw(ps, kw) != 0 or kw == Keyword.Class \
or kw == Keyword.Enum or kw == Keyword.Union:
stmt = _parse_compound(ps)
if stmt is not None:
parent.append(stmt)
return
if (kw == Keyword.Match and _is_match_stmt(ps) != 0) or kw == Keyword.Async:
stmt = _parse_compound(ps)
if stmt is not None:
parent.append(stmt)
return
_parse_simple_stmt_into(ps, parent)
def _parse_simple_stmt_into(ps: rawptr[Parser], parent: rawptr[AST]):
"""simple_stmt: small_stmt (';' small_stmt)* [';'] TokenType.NewLine"""
first: rawptr[AST] = _parse_small_stmt(ps)
if first is not None:
parent.append(first)
while _accept_op(ps, TokOp.Semi):
if _cur_type(ps) == TokenType.NewLine or _cur_type(ps) == TokenType.EndMarker:
break
s: rawptr[AST] = _parse_small_stmt(ps)
if s is None: break
parent.append(s)
# 先跳过 Nl（括号内换行，如多行函数调用参数间的换行，属合法 token）
_skip_nl(ps)
# 语句结束检查：simple_stmt 必须以换行/文件尾结束。
# 残留 token（如相邻表达式 `func12344444 avg3(...)` 被静默拆成多条语句、
# `def` 缺失的关键字函数定义残留）是语法错误（fail-fast），禁止静默，
# 并仿 Python 报告错误位置（SyntaxError at line/col）。
if _cur_type(ps) != TokenType.NewLine and _cur_type(ps) != TokenType.EndMarker:
# 残留 token（如相邻表达式 `func12344444 avg3(...)` 被静默拆成多条语句、
# `def` 缺失的关键字函数定义残留）是语法错误（fail-fast），禁止静默。
# 报文**不再由解析器自拼单行**：位置与消息走解析错误通道，前/后 token
# 作为附注并入消息，由调用方 dump_parse_traceback 以 Python 风格统一输出
# （Traceback + File/行 + 源码行 + caret + SyntaxError）。
# 定位：从链表头找 cur，记录前一个 token 与下一个 token 的上下文
prev_tok: rawptr[Token] = None
walk_tok: rawptr[Token] = ps.tokens
while walk_tok is not None and walk_tok is not ps.cur:
prev_tok = walk_tok
walk_tok = walk_tok.next
pv_str: str = "(head)"
pv_line: int = 0
pv_col: int = 0
if prev_tok is not None:
pv_str = prev_tok.str_val
pv_line = prev_tok.lineno
pv_col = prev_tok.col_offset
nxt_tok: rawptr[Token] = ps.cur.next
nxt_str: str = "(none)"
nxt_line: int = 0
nxt_col: int = 0
if nxt_tok is not None:
nxt_str = nxt_tok.str_val
nxt_line = nxt_tok.lineno
nxt_col = nxt_tok.col_offset
_record_parse_err_ctx(ps, _cur_type(ps), pv_str, pv_line, pv_col,
nxt_str, nxt_line, nxt_col)
ps.error_count += 1
return
if _cur_type(ps) == TokenType.NewLine:
_advance(ps)
def _parse_small_stmt(ps: rawptr[Parser]) -> rawptr[AST]:
"""small_stmt: various simple statements"""
kw: int = _cur_kw(ps)
if kw == Keyword.Pass:
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps)
node: rawptr[AST] = Pass(ps.pool)
_set_pos(node, lineno, col, 0, 0)
return node
if kw == Keyword.Break:
lineno = _cur_lineno(ps)
col = _cur_col(ps)
_advance(ps)
node = Break(ps.pool)
_set_pos(node, lineno, col, 0, 0)
return node
if kw == Keyword.Continue:
lineno = _cur_lineno(ps)
col = _cur_col(ps)
_advance(ps)
node = Continue(ps.pool)
_set_pos(node, lineno, col, 0, 0)
return node
if kw == Keyword.Return:
return _parse_return(ps)
if kw == Keyword.Raise:
return _parse_raise(ps)
if kw == Keyword.Global:
return _parse_global(ps)
if kw == Keyword.Nonlocal:
return _parse_nonlocal(ps)
if kw == Keyword.Import:
return _parse_import(ps)
if kw == Keyword.From:
return _parse_from_import(ps)
if kw == Keyword.Del:
return _parse_del(ps)
if kw == Keyword.Assert:
return _parse_assert(ps)
if kw == Keyword.Yield:
return _parse_expr_stmt(ps, _parse_yield(ps))
# Expression statement / assignment / augassign / annassign
return _parse_expr_stmt(ps, None)
def _maybe_tuple(ps: rawptr[Parser], first: rawptr[AST]) -> rawptr[AST]:
"""If there is a comma after first, collect the comma-separated expressions into a Tuple."""
if first is None: return None
if not _match_op(ps, TokOp.Comma):
return first
lineno: int = first.lineno
col: int = first.col_offset
elts: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
elts.append(first)
_accept_op(ps, TokOp.Comma)
while True:
tt: int = _cur_type(ps)
if tt == TokenType.NewLine or tt == TokenType.EndMarker:
break
if tt == TokenType.Op:
op: int = _cur_op(ps)
if op == TokOp.Equal or op == TokOp.Semi:
break
e: rawptr[AST] = _parse_expr(ps)
if e is None: break
elts.append(e)
if not _accept_op(ps, TokOp.Comma):
break
node: rawptr[AST] = Tuple(ps.pool, elts, ASTCtx.Load)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_testlist(ps: rawptr[Parser]) -> rawptr[AST]:
"""Parse a comma-separated list of expressions, if there’s more than one, form a tuple."""
first: rawptr[AST] = _parse_expr(ps)
return _maybe_tuple(ps, first)
def _parse_expr_stmt(ps: rawptr[Parser], first: rawptr[AST]) -> rawptr[AST]:
"""Expression statement, handle assignment/augassign/annassign"""
if first is None:
first = _parse_testlist(ps)
else:
first = _maybe_tuple(ps, first)
if first is None:
return None
lineno: int = first.lineno
col: int = first.col_offset
# Assignment
if _match_op(ps, TokOp.Equal):
targets: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
targets.append(first)
_fix_store_ctx(first)
value: rawptr[AST] = None
accept_result: int = _accept_op(ps, TokOp.Equal)
while accept_result:
v: rawptr[AST] = _parse_testlist(ps)
if v is None: break
if value is not None:
targets.append(value)
_fix_store_ctx(value)
value = v
accept_result = _accept_op(ps, TokOp.Equal)
node: rawptr[AST] = Assign(ps.pool, targets, value)
_set_pos(node, lineno, col, 0, 0)
return node
# augassign
if _cur_type(ps) == TokenType.Op:
op_id: int = _cur_op(ps)
aug_op: int = _augop_from_op(op_id)
if aug_op != OpKind.NoneOp:
_advance(ps)
value2: rawptr[AST] = _parse_testlist(ps)
_fix_store_ctx(first)
node2: rawptr[AST] = AugAssign(ps.pool, first, aug_op, value2)
_set_pos(node2, lineno, col, 0, 0)
return node2
# annassign: target ':' annotation [= value]
if _match_op(ps, TokOp.Colon):
_advance(ps)
annotation: rawptr[AST] = _parse_type_ann(ps)
value3: rawptr[AST] = None
if _accept_op(ps, TokOp.Equal):
value3 = _parse_testlist(ps)
if ps.pool is None:
return None
_fix_store_ctx(first)
if ps.pool is None:
return None
node3: rawptr[AST] = AnnAssign(ps.pool, first, annotation, value3, 1)
_set_pos(node3, lineno, col, 0, 0)
return node3
# Expression statement
node4: rawptr[AST] = Expr(ps.pool, first)
_set_pos(node4, lineno, col, 0, 0)
return node4
def _parse_return(ps: rawptr[Parser]) -> rawptr[AST]:
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps)
val: rawptr[AST] = None
if _cur_type(ps) != TokenType.NewLine and _cur_type(ps) != TokenType.EndMarker and _cur_type(ps) != TokenType.Op:
val = _parse_testlist(ps)
elif _cur_type(ps) == TokenType.Op and _cur_op(ps) != TokOp.Semi:
val = _parse_testlist(ps)
node: rawptr[AST] = Return(ps.pool, val)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_raise(ps: rawptr[Parser]) -> rawptr[AST]:
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps)
exc: rawptr[AST] = None
cause: rawptr[AST] = None
if _cur_type(ps) != TokenType.NewLine and _cur_type(ps) != TokenType.EndMarker:
exc = _parse_test(ps)
if _cur_kw(ps) == Keyword.From:
_advance(ps)
cause = _parse_test(ps)
node: rawptr[AST] = Raise(ps.pool, exc, cause)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_global(ps: rawptr[Parser]) -> rawptr[AST]:
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps)
names: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
if _cur_type(ps) == TokenType.Name:
n: rawptr[AST] = Name(ps.pool, _cur_str(ps), ASTCtx.Store)
_set_pos(n, _cur_lineno(ps), _cur_col(ps), 0, 0)
names.append(n)
_advance(ps)
while _accept_op(ps, TokOp.Comma):
if _cur_type(ps) != TokenType.Name: break
n = Name(ps.pool, _cur_str(ps), ASTCtx.Store)
_set_pos(n, _cur_lineno(ps), _cur_col(ps), 0, 0)
names.append(n)
_advance(ps)
node: rawptr[AST] = Global(ps.pool, names)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_nonlocal(ps: rawptr[Parser]) -> rawptr[AST]:
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps)
names: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
if _cur_type(ps) == TokenType.Name:
n: rawptr[AST] = Name(ps.pool, _cur_str(ps), ASTCtx.Store)
_set_pos(n, _cur_lineno(ps), _cur_col(ps), 0, 0)
names.append(n)
_advance(ps)
while _accept_op(ps, TokOp.Comma):
if _cur_type(ps) != TokenType.Name: break
n = Name(ps.pool, _cur_str(ps), ASTCtx.Store)
_set_pos(n, _cur_lineno(ps), _cur_col(ps), 0, 0)
names.append(n)
_advance(ps)
node: rawptr[AST] = Nonlocal(ps.pool, names)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_import(ps: rawptr[Parser]) -> rawptr[AST]:
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps)
names: rawptr[list[rawptr[AST]]] = _parse_alias_list(ps)
node: rawptr[AST] = Import(ps.pool, names)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_from_import(ps: rawptr[Parser]) -> rawptr[AST]:
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps)
# Parse module (possibly with dots)
level: int = 0
while _match_op(ps, TokOp.Dot):
level += 1
_advance(ps)
while _match_op(ps, TokOp.Ellipsis):
level += 3
_advance(ps)
module_name: str = None
# The token type for keywords (like import) is also Name, so they must be excluded using kw_subtype,
# otherwise the `import` in `from . import` would be mistakenly treated as a module_name.
# Note: only `import` itself should be excluded. Module names can be keywords (like in `from .match import ...`,
# where match is a Viper match statement keyword), requiring NotKeyword would cause
# `from .match import` to fail parsing and the dependency scan would miss that module.
if _cur_type(ps) == TokenType.Name and _cur_kw(ps) != Keyword.Import:
module_name = _cur_str(ps)
_advance(ps)
while _match_op(ps, TokOp.Dot):
_advance(ps)
if _cur_type(ps) == TokenType.Name and _cur_kw(ps) == Keyword.NotKeyword:
next_name: str = _cur_str(ps)
mlen: int = string.strlen(module_name)
nlen: int = string.strlen(next_name)
new_mod: str = ps.pool.alloc(mlen + 1 + nlen + 1)
if new_mod is not None:
string.strcpy(new_mod, module_name)
new_mod[mlen] = '.'
string.strcpy(new_mod + mlen + 1, next_name)
module_name = new_mod
_advance(ps)
if _cur_kw(ps) != Keyword.Import:
ps.error_count += 1
return None
_advance(ps)
_skip_nl(ps)
# import *
if _match_op(ps, TokOp.Star):
_advance(ps)
star_alias: rawptr[AST] = Alias(ps.pool, "*", None)
_set_pos(star_alias, lineno, col, 0, 0)
star_names: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
star_names.append(star_alias)
node: rawptr[AST] = ImportFrom(ps.pool, module_name, star_names, level)
_set_pos(node, lineno, col, 0, 0)
return node
# ( names ) or names
if _match_op(ps, TokOp.LPar):
_advance(ps)
_skip_nl(ps)
names: rawptr[list[rawptr[AST]]] = _parse_alias_list(ps)
_skip_nl(ps)
_expect_op(ps, TokOp.RPar)
else:
names = _parse_alias_list(ps)
node: rawptr[AST] = ImportFrom(ps.pool, module_name, names, level)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_alias_list(ps: rawptr[Parser]) -> rawptr[list[rawptr[AST]]]:
"""Parse alias list."""
names: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
while True:
if _cur_type(ps) != TokenType.Name:
break
name_str: str = _cur_str(ps)
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps)
# dotted name
while _match_op(ps, TokOp.Dot):
_advance(ps)
if _cur_type(ps) == TokenType.Name:
next_n: str = _cur_str(ps)
nmlen: int = string.strlen(name_str)
nxtlen: int = string.strlen(next_n)
new_name: str = ps.pool.alloc(nmlen + 1 + nxtlen + 1)
if new_name is not None:
string.strcpy(new_name, name_str)
new_name[nmlen] = '.'
string.strcpy(new_name + nmlen + 1, next_n)
name_str = new_name
_advance(ps)
asname: str = None
if _cur_kw(ps) == Keyword.As:
_advance(ps)
if _cur_type(ps) == TokenType.Name:
asname = _cur_str(ps)
_advance(ps)
a: rawptr[AST] = Alias(ps.pool, name_str, asname)
_set_pos(a, lineno, col, 0, 0)
names.append(a)
if not _accept_op(ps, TokOp.Comma):
break
_skip_nl(ps)
return names
def _parse_del(ps: rawptr[Parser]) -> rawptr[AST]:
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps)
targets: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
while True:
t_: rawptr[AST] = _parse_atom_expr(ps)
if t_ is None: break
_set_ctx(t_, ASTCtx.Del)
targets.append(t_)
if not _accept_op(ps, TokOp.Comma):
break
node: rawptr[AST] = Delete(ps.pool, targets)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_assert(ps: rawptr[Parser]) -> rawptr[AST]:
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps)
test: rawptr[AST] = _parse_test(ps)
msg: rawptr[AST] = None
if _accept_op(ps, TokOp.Comma):
msg = _parse_test(ps)
node: rawptr[AST] = Assert(ps.pool, test, msg)
_set_pos(node, lineno, col, 0, 0)
return node
# ============================================================
# Compound statements
# ============================================================
def _parse_compound(ps: rawptr[Parser]) -> rawptr[AST]:
kw: int = _cur_kw(ps)
if kw == Keyword.If:
return _parse_if(ps)
if kw == Keyword.While:
return _parse_while(ps)
if kw == Keyword.For:
return _parse_for(ps, 0)
if kw == Keyword.Try:
return _parse_try(ps)
if kw == Keyword.With:
return _parse_with(ps, 0)
if _is_funcdef_kw(ps, kw) != 0:
return _parse_funcdef(ps, 0)
if kw == Keyword.Class:
return _parse_classdef(ps)
if kw == Keyword.Enum:
return _parse_enumdef(ps)
if kw == Keyword.Union:
return _parse_uniondef(ps)
if kw == Keyword.Async:
_advance(ps)
if _is_funcdef_kw(ps, _cur_kw(ps)) != 0:
return _parse_funcdef(ps, 1)
if _cur_kw(ps) == Keyword.For:
return _parse_for(ps, 1)
if _cur_kw(ps) == Keyword.With:
return _parse_with(ps, 1)
ps.error_count += 1
return None
if kw == Keyword.Match and _is_match_stmt(ps) != 0:
return _parse_match(ps)
ps.error_count += 1
return None
def _parse_if(ps: rawptr[Parser]) -> rawptr[AST]:
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps) # if
test: rawptr[AST] = _parse_test(ps)
_expect_op(ps, TokOp.Colon)
# First construct with empty orelse, then fill it in later
empty: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
node: rawptr[If] = If(ps.pool, test, empty)
_parse_suite_into_children(ps, node)
orelse: rawptr[list[rawptr[AST]]] = _parse_if_tail(ps)
node.orelse = orelse
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_if_tail(ps: rawptr[Parser]) -> rawptr[list[rawptr[AST]]]:
"""Parse elif/else chain and return orelse list.
elif: orelse = list containing single If node (nested)
else: orelse = statement list
"""
if _cur_kw(ps) == Keyword.Elif:
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps)
elif_test: rawptr[AST] = _parse_test(ps)
_expect_op(ps, TokOp.Colon)
empty: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
elif_node: rawptr[If] = If(ps.pool, elif_test, empty)
_parse_suite_into_children(ps, elif_node)
elif_orelse: rawptr[list[rawptr[AST]]] = _parse_if_tail(ps)
elif_node.orelse = elif_orelse
_set_pos(elif_node, lineno, col, 0, 0)
orelse: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
orelse.append(elif_node)
return orelse
if _cur_kw(ps) == Keyword.Else:
_advance(ps)
_expect_op(ps, TokOp.Colon)
scratch: rawptr[Module] = Module(ps.pool)
_parse_suite_into_children(ps, scratch)
return scratch.children
return list[rawptr[AST]](ps.pool, 8) # 空 list
# ============================================================
# Parsing Compound Statements (Continued)
# ============================================================
def _parse_while(ps: rawptr[Parser]) -> rawptr[AST]:
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps) # while
test: rawptr[AST] = _parse_test(ps)
_expect_op(ps, TokOp.Colon)
empty: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
node: rawptr[While] = While(ps.pool, test, empty)
_parse_suite_into_children(ps, node)
if _cur_kw(ps) == Keyword.Else:
_advance(ps)
_expect_op(ps, TokOp.Colon)
scratch: rawptr[Module] = Module(ps.pool)
_parse_suite_into_children(ps, scratch)
node.orelse = scratch.children
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_for(ps: rawptr[Parser], is_async: int) -> rawptr[AST]:
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps) # for
target: rawptr[AST] = _parse_target_list(ps)
if _cur_kw(ps) != Keyword.In:
ps.error_count += 1
return None
_advance(ps)
iter_: rawptr[AST] = _parse_or_test(ps)
_expect_op(ps, TokOp.Colon)
empty: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
node: rawptr[For] = For(ps.pool, target, iter_, empty, is_async)
_fix_store_ctx(target)
_parse_suite_into_children(ps, node)
if _cur_kw(ps) == Keyword.Else:
_advance(ps)
_expect_op(ps, TokOp.Colon)
scratch: rawptr[Module] = Module(ps.pool)
_parse_suite_into_children(ps, scratch)
node.orelse = scratch.children
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_try(ps: rawptr[Parser]) -> rawptr[AST]:
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps) # try
_expect_op(ps, TokOp.Colon)
empty: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
node: rawptr[Try] = Try(ps.pool, empty, empty, empty)
_parse_suite_into_children(ps, node)
# handlers (except)
handlers: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
while _cur_kw(ps) == Keyword.Except:
h: rawptr[AST] = _parse_except_handler(ps)
if h is not None:
handlers.append(h)
node.handlers = handlers
# else
if _cur_kw(ps) == Keyword.Else:
_advance(ps)
_expect_op(ps, TokOp.Colon)
scratch: rawptr[Module] = Module(ps.pool)
_parse_suite_into_children(ps, scratch)
node.orelse = scratch.children
# finally
if _cur_kw(ps) == Keyword.Finally:
_advance(ps)
_expect_op(ps, TokOp.Colon)
scratch2: rawptr[Module] = Module(ps.pool)
_parse_suite_into_children(ps, scratch2)
node.finalbody = scratch2.children
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_except_handler(ps: rawptr[Parser]) -> rawptr[AST]:
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps) # except
type_: rawptr[AST] = None
name: str = None
if _cur_op(ps) != TokOp.Colon:
type_ = _parse_test(ps)
if _cur_kw(ps) == Keyword.As:
_advance(ps)
if _cur_type(ps) == TokenType.Name:
name = _cur_str(ps)
_advance(ps)
_expect_op(ps, TokOp.Colon)
node: rawptr[AST] = ExceptHandler(ps.pool, type_, name)
_parse_suite_into_children(ps, node)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_with(ps: rawptr[Parser], is_async: int) -> rawptr[AST]:
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps) # with
items: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
# items
while True:
item: rawptr[AST] = _parse_with_item(ps)
if item is None: break
items.append(item)
if not _accept_op(ps, TokOp.Comma):
break
# Check if we can continue with a newline (with (...): form)
if _cur_type(ps) == TokenType.NewLine or _cur_type(ps) == TokenType.Nl:
break
_expect_op(ps, TokOp.Colon)
node: rawptr[With] = With(ps.pool, items, is_async)
_parse_suite_into_children(ps, node)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_with_item(ps: rawptr[Parser]) -> rawptr[AST]:
"""with_item: test ['as' expr]"""
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
context_expr: rawptr[AST] = _parse_or_test(ps)
if context_expr is None: return None
optional_vars: rawptr[AST] = None
if _cur_kw(ps) == Keyword.As:
_advance(ps)
optional_vars = _parse_target(ps)
_fix_store_ctx(optional_vars)
node: rawptr[AST] = WithItem(ps.pool, context_expr, optional_vars)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_funcdef(ps: rawptr[Parser], is_async: int) -> rawptr[AST]:
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
is_extern_fd: int = 0
is_export_fd: int = 0
is_define_fd: int = 0
if _cur_kw(ps) == Keyword.Extern:
# extern 声明：extern [def] A(...) [-> T] [": pass" 可选]
is_extern_fd = 1
_advance(ps) # extern
if _cur_kw(ps) == Keyword.Def:
_advance(ps)
elif _cur_kw(ps) == Keyword.Export:
# export 声明：export [def] A(...) [-> T] [": pass" 可选]
is_export_fd = 1
_advance(ps) # export
if _cur_kw(ps) == Keyword.Def:
_advance(ps)
elif _cur_kw(ps) == Keyword.Define:
# define 宏函数：语法与 def 完全一致，返回注解自动追加 t.CDefine
# 标记为宏函数（StubGen 保留原样 → C #define / alwaysinline 内联）
is_define_fd = 1
_advance(ps) # define
else:
_advance(ps) # def / fn / ...
if _cur_type(ps) != TokenType.Name:
ps.error_count += 1
return None
name: str = _cur_str(ps)
_advance(ps)
# PEP 695 泛型函数类型参数: def add[T](a: T, b: T) — 解析并存入 type_params。
# （与 class 解析器一致；之前缺失此处理导致 _expect_op(Colon) 失败、函数体被
# 误提升到模块级——b = Box(42) 被当作模块级全局注册，函数内赋值误报
# "需用 global 声明"。翻译时 type_params 映射到 int 兜底。）
type_params_fd: rawptr[list[str]] = None
if _accept_op(ps, TokOp.Lsqb):
type_params_fd = list[str](ps.pool, 8)
while not _match_op(ps, TokOp.Rsqb):
if _cur_type(ps) == TokenType.EndMarker:
break
if _cur_type(ps) == TokenType.Name:
tp_name_fd: str = _cur_str(ps)
_advance(ps)
# 跳过可选 bound/约束: [T: bound] 或 [T: (A, B)]
if _match_op(ps, TokOp.Colon):
_advance(ps)
_parse_test(ps)
type_params_fd.append(tp_name_fd)
if not _accept_op(ps, TokOp.Comma):
break
_expect_op(ps, TokOp.Rsqb)
args: rawptr[AST] = None
if _accept_op(ps, TokOp.LPar):
args = _parse_varargslist(ps, 0)
_expect_op(ps, TokOp.RPar)
returns: rawptr[AST] = None
if _accept_op(ps, TokOp.RArrow):
returns = _parse_type_ann(ps)
if is_extern_fd:
# extern 声明：返回注解改写为 <原类型> | t.State，复用既有
# State/外部 C 函数识别路径（vpi 注解含 "State" → 定义端裸名 + 不生成函数体）
state_attr_fd: rawptr[AST] = Attribute(ps.pool,
Name(ps.pool, "t", ASTCtx.Load),
"State", ASTCtx.Load)
if returns != None:
returns = BinOp(ps.pool, returns, OpKind.BitOr, state_attr_fd)
else:
returns = state_attr_fd
elif is_export_fd:
# export 声明：返回注解改写为 <原类型> | t.CExport，复用既有
# CExport 识别路径（vpi 注解含 "CExport" → 定义端裸名 + 不加模块 SHA1 头）
export_attr_fd: rawptr[AST] = Attribute(ps.pool,
Name(ps.pool, "t", ASTCtx.Load),
"CExport", ASTCtx.Load)
if returns != None:
returns = BinOp(ps.pool, returns, OpKind.BitOr, export_attr_fd)
else:
returns = export_attr_fd
elif is_define_fd:
# define 宏函数：返回注解改写为 <原类型> | t.CDefine，复用既有
# CDefine 宏函数识别路径（StubGen 保留原样 → C #define / alwaysinline）
define_attr_fd: rawptr[AST] = Attribute(ps.pool,
Name(ps.pool, "t", ASTCtx.Load),
"CDefine", ASTCtx.Load)
if returns != None:
returns = BinOp(ps.pool, returns, OpKind.BitOr, define_attr_fd)
else:
returns = define_attr_fd
dec_list: rawptr[list[rawptr[AST]]] = ps.pending_decorators
ps.pending_decorators = None
node: rawptr[FunctionDef] = FunctionDef(ps.pool, name, args, dec_list, returns, is_async, type_params_fd)
if is_extern_fd:
node.flags = node.flags | FLAG_IS_EXTERN
if is_export_fd:
node.flags = node.flags | FLAG_IS_EXPORT
if is_define_fd:
node.flags = node.flags | FLAG_IS_DEFINE
if (is_extern_fd == 0 and is_export_fd == 0 and is_define_fd == 0) or _match_op(ps, TokOp.Colon):
_expect_op(ps, TokOp.Colon)
_parse_suite_into_children(ps, node)
else:
# extern/export 无 ': pass' 形式（extern A(...) -> int）：body 为空，仅声明。
# 必须消费本行 NewLine：模块级/类体语句循环只 _skip_nl（Nl），
# 残留 NewLine 会被下一轮当表达式解析（_parse_atom → error_count++）
# → parse_tokens 返回 None（TPV 编译 extern 文件报 7 个扫描失败的根因）
if _cur_type(ps) == TokenType.NewLine:
_advance(ps)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_classdef(ps: rawptr[Parser]) -> rawptr[AST]:
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps) # class
if _cur_type(ps) != TokenType.Name:
ps.error_count += 1
return None
name: str = _cur_str(ps)
_advance(ps)
# Parse type parameters [T] or [T, U] (PEP 695 generic class syntax)
type_params: rawptr[list[str]] = None
if _accept_op(ps, TokOp.Lsqb):
type_params = list[str](ps.pool, 8)
while not _match_op(ps, TokOp.Rsqb):
if _cur_type(ps) == TokenType.EndMarker: break
if _cur_type(ps) == TokenType.Name:
tp_name: str = _cur_str(ps)
_advance(ps)
# Skip optional bound/constraint: [T: bound] or [T: (A, B)]
if _match_op(ps, TokOp.Colon):
_advance(ps)
_parse_test(ps)
type_params.append(tp_name)
if not _accept_op(ps, TokOp.Comma):
break
_expect_op(ps, TokOp.Rsqb)
bases: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
keywords: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
# bases / keywords
if _accept_op(ps, TokOp.LPar):
while not _match_op(ps, TokOp.RPar):
# 基类括号内跨行（class X(A,\n B):）：逗号后换行在此跳过，
# 否则 cur=NL 非 Name/Comma → break → _expect_op(RPar) 报
# "意外的 token"（Generator.vp L34 多继承跨行根因）
_skip_nl(ps)
if _cur_type(ps) == TokenType.EndMarker: break
# base 或 keyword=value
if _cur_type(ps) == TokenType.Name:
# Parse as expression, if followed by '=', then as keyword=value
e: rawptr[AST] = _parse_or_test(ps)
if _match_op(ps, TokOp.Equal):
_advance(ps)
v: rawptr[AST] = _parse_test(ps)
kw_node: rawptr[AST] = KwNode(ps.pool, e, v)
keywords.append(kw_node)
else:
bases.append(e)
if not _accept_op(ps, TokOp.Comma):
break
_skip_nl(ps) # 末基类后换行收尾（class X(A,\n B):）
_expect_op(ps, TokOp.RPar)
_expect_op(ps, TokOp.Colon)
dec_list: rawptr[list[rawptr[AST]]] = ps.pending_decorators
ps.pending_decorators = None
node: rawptr[ClassDef] = ClassDef(ps.pool, name, bases, keywords, dec_list, type_params)
# owner_is_class=1：类体期间 ps.in_class_body=1，供 virtual/novirtual 修饰符判
# "方法级修饰符是否直接写在类体内"（见 _ParseVirtualModified / _parse_suite_into_children）
_parse_suite_into_children(ps, node, 1)
_set_pos(node, lineno, col, 0, 0)
return node
# ============================================================
# enum 语法糖（与 class X(t.CEnum): 完全等价）
#
# 规格：`enum Name:` 头行 + 成员行（`A` / `A: T` / `A = 0`）。
# 语义与手写 `class Name(t.CEnum):` **完全等价**——本函数合成的 ClassDef
# 与 _parse_classdef 的产物逐字段同形，故下游（ClassGenDef 分派 →
# ClassGenEnum._translate_enum_def → EnumTable 编译期常量）只有**同一条**
# 实现路径，不存在第二套枚举实现。
# ============================================================
def _enum_member_annassign(ps: rawptr[Parser], owner: rawptr[AST],
name_node: rawptr[AST],
lineno: int, col: int,
end_lineno: int, end_col: int) -> rawptr[AST]:
"""把单个成员名节点改写为 `Name: t.State` 的 AnnAssign（enum 成员统一形态）。
target=Name(id, Store)、annotation=Attribute(Name('t'), 'State')、
value=None、simple=1 —— 与手写 `class X(t.CEnum): A: t.State` 逐字段同形。
"""
ann: rawptr[AST] = Attribute(ps.pool,
Name(ps.pool, "t", ASTCtx.Load),
"State", ASTCtx.Load)
_set_pos(ann, lineno, col, end_lineno, end_col)
aa: rawptr[AST] = AnnAssign(ps.pool, name_node, ann, None, 1)
_fix_store_ctx(name_node)
_set_pos(aa, lineno, col, end_lineno, end_col)
aa.parent = owner # 与 AST.append 一致地维护父指针
return aa
def _convert_enum_bare_members(ps: rawptr[Parser], node: rawptr[ClassDef]):
"""把 enum 体里的裸名 / 逗号分组成员行就地改写为 `Name: t.State` 的 AnnAssign。
成员收集（ClassGenEnum._translate_enum_def 只认 Assign / AnnAssign 的 Name
目标；TPC 侧 HandlesClassDef._RegisterEnumMembers 同口径）**不认**裸名语句
（表达式语句 `Expr(Name)`）与逗号分组语句（表达式语句 `Expr(Tuple)`）——
两者此前都会被**静默漏掉**（成员凭空消失，直到访问处 NameError / 别处误报）。
两种成员**行**形态（与 TPC 侧 SymbolUtils._ExpandEnumBlocks 同口径）：
- `A` → `A: t.State`（整条语句替换为一个 AnnAssign）
- `A, B, C` → `A: t.State` / `B: t.State` / `C: t.State`（三条 AnnAssign）
逗号组要求"全是简单标识符且 ≥2 个"（尾随逗号在 AST 上与正常组不可区分，按
正常组处理）；带值逗号组 `A, B = 1`、带注解逗号组 `A, B: T`、单名尾随逗号
`A,` 与其余语句**不**在此改写——它们由注册处的"源码可见成员名数 ≠ 实际登记
成员数"fail-fast 拦住（ClassGenEnum._translate_enum_def），不再静默丢成员。
缩进/空行/注释的容忍由 _parse_suite_into_children 承担（本函数只看语句种类）。
"""
children: rawptr[list[rawptr[AST]]] = node.children
if children is None:
return
cn: sizet = len(children)
# 重建子表：逗号组一行 → 多条 AnnAssign。list 无 insert，故整体重建；赋值形态
# 与 AST.append（base.vp:358 `self.children = list[rawptr[AST]](...)`）逐字同形。
node.children = list[rawptr[AST]](ps.pool, 8)
out: rawptr[list[rawptr[AST]]] = node.children
i: sizet = 0
while i < cn:
stmt: rawptr[AST] = children.get(i)
handled: int = 0
if stmt is not None and stmt.kind() == ASTKind.Expr:
ex: rawptr[Expr] = (rawptr[Expr])(stmt)
if ex.value is not None and ex.value.kind() == ASTKind.Name:
out.append(_enum_member_annassign(ps, node, ex.value,
ex.lineno, ex.col_offset,
ex.end_lineno, ex.end_col_offset))
handled = 1
elif ex.value is not None and ex.value.kind() == ASTKind.Tuple:
tup: rawptr[Tuple] = (rawptr[Tuple])(ex.value)
if tup.elts is not None and len(tup.elts) >= 2:
all_names: int = 1
k: sizet = 0
while k < len(tup.elts):
el: rawptr[AST] = tup.elts.get(k)
if el is None or el.kind() != ASTKind.Name:
all_names = 0
break
k += 1
if all_names == 1:
k = 0
while k < len(tup.elts):
el2: rawptr[AST] = tup.elts.get(k)
out.append(_enum_member_annassign(ps, node, el2,
el2.lineno, el2.col_offset,
el2.end_lineno, el2.end_col_offset))
k += 1
handled = 1
if handled == 0 and stmt is not None:
out.append(stmt)
i += 1
def _parse_define_const(ps: rawptr[Parser]) -> rawptr[AST]:
"""`define NAME[: T] = <字面量>` → 与 `NAME: [T |] t.CDefine = <字面量>`
逐字段同形的 AnnAssign（define 常量声明语法糖）。
- 与手写 `NAME: t.CDefine = 值` 落到**同一条** CDefine 注册路径
（AssignGen 的 is_cdefine_annotation 分支）；不新增第二套常量实现。
- 带类型时 T 写在联合**前面**（`T | t.CDefine`）：与 TPC 侧
`SymbolUtils._ExpandDefineConsts` 的展开产物同形，使"值必须能放进 T"
这条判据有据可依（TPV 的 is_cdefine_annotation 与顺序无关，
仅取"含 CDefine"这一事实）。
- 行号/列号取自 define 关键字所在位置，报错回显定位到真实源码行。
- 语句必须换行/文件尾结束（与 _parse_simple_stmt_into 同口径）；
残留 token 属语法错误（_record_parse_err，fail-fast，不静默）。
"""
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps) # define
if _cur_type(ps) != TokenType.Name:
ps.error_count += 1
_record_parse_err(ps)
return None
name: str = _cur_str(ps)
_advance(ps)
explicit_ann: rawptr[AST] = None
if _accept_op(ps, TokOp.Colon):
explicit_ann = _parse_type_ann(ps)
if explicit_ann is None:
return None
if _expect_op(ps, TokOp.Equal) == 0:
return None
value: rawptr[AST] = _parse_test(ps)
if value is None:
ps.error_count += 1
_record_parse_err(ps)
return None
cdefine_marker: rawptr[AST] = Attribute(ps.pool,
Name(ps.pool, "t", ASTCtx.Load),
"CDefine", ASTCtx.Load)
annotation: rawptr[AST] = cdefine_marker
if explicit_ann is not None:
annotation = BinOp(ps.pool, explicit_ann, OpKind.BitOr, cdefine_marker)
target: rawptr[AST] = Name(ps.pool, name, ASTCtx.Store)
_set_pos(target, lineno, col, 0, 0)
node: rawptr[AST] = AnnAssign(ps.pool, target, annotation, value, 1)
_set_pos(node, lineno, col, 0, 0)
_skip_nl(ps)
if _cur_type(ps) != TokenType.NewLine and _cur_type(ps) != TokenType.EndMarker:
_record_parse_err(ps)
ps.error_count += 1
return None
if _cur_type(ps) == TokenType.NewLine:
_advance(ps)
return node
def _parse_typedef(ps: rawptr[Parser]) -> rawptr[AST]:
"""`typedef T1 = T2` → 与 `T1: t.CTypedef = T2` 逐字段同形的 AnnAssign。
- 与手写 `T1: t.CTypedef = T2` 落到**同一条** typedef 注册/查找路径
（AssignGenMixin.translate_annassign 的 CTypedef 分支 → ScanCTypedefDeclarations
/ TypeGenCType.register_ctypedef）；**不新增第二套 typedef 实现**。
- 注解为 `Attribute(Name('t'), 'CTypedef')`，与 TPC 侧展开产物
`SymbolUtils._ExpandTypedefs` 同形（`t.CTypedef`）。
- RHS 即"类型表达式"（写在值位，按 `_parse_test` 解析）：裸名 `int32` /
`t.X` / 并集 `char | t.CPtr` / 泛型 `array[E, N]` / 链式 `A = B`。
- 行号/列号取自 typedef 关键字所在位置，报错回显定位到真实源码行。
- 非法形态（缺名字 / 缺 `=` / 缺 RHS）⇒ `_record_parse_err` **响亮失败**
（fail-fast，不静默）；语句必须换行/文件尾结束（与 _parse_define_const 同口径）。
"""
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps) # typedef
if _cur_type(ps) != TokenType.Name:
# 缺别名（如 `typedef = 1` / `typedef : x`）：响亮失败。
_record_parse_err(ps)
ps.error_count += 1
return None
name: str = _cur_str(ps)
_advance(ps)
if _expect_op(ps, TokOp.Equal) == 0:
# 缺 `=`（如 `typedef A`）：响亮失败。
return None
value: rawptr[AST] = _parse_test(ps)
if value is None:
# 缺 RHS（如 `typedef A = `）：响亮失败。
ps.error_count += 1
_record_parse_err(ps)
return None
ctypedef_marker: rawptr[AST] = Attribute(ps.pool,
Name(ps.pool, "t", ASTCtx.Load),
"CTypedef", ASTCtx.Load)
target: rawptr[AST] = Name(ps.pool, name, ASTCtx.Store)
_set_pos(target, lineno, col, 0, 0)
node: rawptr[AST] = AnnAssign(ps.pool, target, ctypedef_marker, value, 1)
_set_pos(node, lineno, col, 0, 0)
_skip_nl(ps)
if _cur_type(ps) != TokenType.NewLine and _cur_type(ps) != TokenType.EndMarker:
_record_parse_err(ps)
ps.error_count += 1
return None
if _cur_type(ps) == TokenType.NewLine:
_advance(ps)
return node
def _parse_enumdef(ps: rawptr[Parser]) -> rawptr[AST]:
"""enum NAME: 语法糖 → 与 `class NAME(t.CEnum):` 逐字段同形的 ClassDef。
成员书写形态（与 TPC 侧 SymbolUtils._ExpandEnumBlocks 同口径）：
- 裸名 `A` → `A: t.State`（默认编号 0..N，宽度默认 i32）
- 逗号组 `A, B` → 同行拆成多条 `A: t.State` / `B: t.State`（≥2 个简单标识符）
- 带注解 `A: T` → 原样（自定义具体类型，参与"最宽成员类型"宽度推导）
- 带值 `A = 0` → 原样（显式编号）
基类固定为 Attribute(Name('t'), 'CEnum')，与手写 `class X(t.CEnum):` 的
bases[0] 逐字段同形（同一 _parse_classdef 基类解析产物）。行号/列号原样继承
自各成员语句，故报错回显（如"枚举成员 'P' 不能是指针类型"）定位到真实源码行。
"""
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps) # enum
if _cur_type(ps) != TokenType.Name:
ps.error_count += 1
return None
name: str = _cur_str(ps)
_advance(ps)
bases: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
bases.append(Attribute(ps.pool, Name(ps.pool, "t", ASTCtx.Load),
"CEnum", ASTCtx.Load))
keywords: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
dec_list: rawptr[list[rawptr[AST]]] = ps.pending_decorators
ps.pending_decorators = None
_expect_op(ps, TokOp.Colon)
node: rawptr[ClassDef] = ClassDef(ps.pool, name, bases, keywords, dec_list, None)
_parse_suite_into_children(ps, node)
_convert_enum_bare_members(ps, node)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_uniondef(ps: rawptr[Parser]) -> rawptr[AST]:
"""union NAME: 语法糖 → 与 `class NAME(t.CUnion):` 逐字段同形的 ClassDef。
成员按普通类体（AnnAssign）解析：`d: double` / `u: uint64`。基类固定为
Attribute(Name('t'), 'CUnion')，与手写 `class X(t.CUnion):` 的 bases[0]
逐字段同形（同一 _parse_classdef 基类解析产物）——TPC 侧正是靠
`issubclass(cls, t.CUnion)` 判 union（HandlesClassDef._ClassifyBase），
故两侧落到同一条既有的 union 语义路径（成员偏移 0、尺寸取最宽成员）。
行号/列号继承自语句本身，报错回显定位到真实源码行。
与 `enum NAME:` 的差异：不做裸名成员转换（union 成员必须带注解/类型）。
"""
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps) # union
if _cur_type(ps) != TokenType.Name:
ps.error_count += 1
return None
name: str = _cur_str(ps)
_advance(ps)
bases: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
bases.append(Attribute(ps.pool, Name(ps.pool, "t", ASTCtx.Load),
"CUnion", ASTCtx.Load))
keywords: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
dec_list: rawptr[list[rawptr[AST]]] = ps.pending_decorators
ps.pending_decorators = None
_expect_op(ps, TokOp.Colon)
node: rawptr[ClassDef] = ClassDef(ps.pool, name, bases, keywords, dec_list, None)
_parse_suite_into_children(ps, node)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_match(ps: rawptr[Parser]) -> rawptr[AST]:
"""match statement: match subject ':' NewLine Indent case+ Dedent"""
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps) # match
subject: rawptr[AST] = _parse_test(ps)
_expect_op(ps, TokOp.Colon)
cases: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
node: rawptr[AST] = Match(ps.pool, subject, cases)
# Parsing body: Same NewLine/Indent/Dedent pattern as _parse_suite_into_children,
# but parses case clauses into a cases list instead of normal statements into children
_skip_nl(ps)
if _cur_type(ps) == TokenType.NewLine:
_advance(ps)
_skip_nl(ps)
if _cur_type(ps) == TokenType.Indent:
_advance(ps)
while _cur_type(ps) != TokenType.Dedent and _cur_type(ps) != TokenType.EndMarker:
if _cur_kw(ps) == Keyword.Case:
case_node: rawptr[AST] = _parse_match_case(ps)
if case_node is not None:
cases.append(case_node)
else:
_advance(ps) # Skip non-case token to avoid infinite loop
_expect_type(ps, TokenType.Dedent)
_set_pos(node, lineno, col, 0, 0)
return node
# ============================================================
# Match pattern parsing
# ============================================================
def _make_name_ast(pool: rawptr[memhub.MemManager], node: rawptr[Name]) -> rawptr[AST]:
# Analysis: "Convert Name* to AST* (Solving the TransPyC Type Inference Problem)"
return node
def _parse_pattern(ps: rawptr[Parser]) -> rawptr[AST]:
"""pattern: or_pattern ['as' NAME]
or_pattern: closed_pattern ('|' closed_pattern)*"""
pat: rawptr[AST] = _parse_closed_pattern(ps)
if pat is None: return None
# OR pattern
if _match_op(ps, TokOp.VBar):
patterns: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
patterns.append(pat)
while _accept_op(ps, TokOp.VBar):
p: rawptr[AST] = _parse_closed_pattern(ps)
if p is not None:
patterns.append(p)
or_node: rawptr[AST] = MatchOr(ps.pool, patterns)
_set_pos(or_node, pat.lineno, pat.col_offset, 0, 0)
pat = or_node
# AS pattern
if _cur_kw(ps) == Keyword.As:
_advance(ps)
if _cur_type(ps) != TokenType.Name:
ps.error_count += 1
return pat
name_str: str = _cur_str(ps)
_advance(ps)
as_node: rawptr[AST] = MatchAs(ps.pool, pat, name_str)
_set_pos(as_node, pat.lineno, pat.col_offset, 0, 0)
return as_node
return pat
def _parse_closed_pattern(ps: rawptr[Parser]) -> rawptr[AST]:
"""closed_pattern: literal | capture | wildcard | value | group | sequence | mapping | class"""
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
ttype: int = _cur_type(ps)
val: rawptr[AST]
node: rawptr[AST]
tok: rawptr[Token]
sval: str
# Number literal pattern
if ttype == TokenType.Number:
tok = ps.cur
_advance(ps)
if tok.is_float:
val = _new_const_float(ps.pool, tok.float_val, lineno, col)
else:
val = _new_const_int(ps.pool, tok.int_val, lineno, col)
node = MatchValue(ps.pool, val)
_set_pos(node, lineno, col, 0, 0)
return node
# String literal pattern
if ttype == TokenType.String:
sval = _cur_str(ps)
is_dquote_m: int = ps.cur.kw_subtype & 16
_advance(ps)
val = _new_const_str(ps.pool, sval, lineno, col, is_dquote_m)
node = MatchValue(ps.pool, val)
_set_pos(node, lineno, col, 0, 0)
return node
# Negative number literal pattern: -1, -1.5
if _match_op(ps, TokOp.Minus):
_advance(ps)
if _cur_type(ps) == TokenType.Number:
tok = ps.cur
# 一元减 + 整数字面量的联合越界判定（与 _parse_factor 同口径）：
# 幅值 > 2^63 的负数字面量（如 -18446744073709551615、-9223372036854775809）
# 结果 < -2^63 ⇒ fail-fast，不静默回绕。
if tok.is_float == 0 and _lex_int_neg_exceeds_i64(_cur_str(ps)) != 0:
_record_neg_literal_overflow(_cur_str(ps), lineno, col)
ps.error_count += 1
return None
_advance(ps)
if tok.is_float:
val = _new_const_float(ps.pool, -tok.float_val, lineno, col)
else:
val = _new_const_int(ps.pool, -tok.int_val, lineno, col)
node = MatchValue(ps.pool, val)
_set_pos(node, lineno, col, 0, 0)
return node
ps.error_count += 1
return None
# Name: None/True/False/_ /capture/value/class
if ttype == TokenType.Name:
kw: int = _cur_kw(ps)
# None/True/False → MatchSingleton
if kw == Keyword.None_ or kw == Keyword.True_ or kw == Keyword.False_:
_advance(ps)
if kw == Keyword.None_:
val = _new_const_none(ps.pool, lineno, col)
elif kw == Keyword.True_:
val = _new_const_bool(ps.pool, 1, lineno, col)
else:
val = _new_const_bool(ps.pool, 0, lineno, col)
node = MatchSingleton(ps.pool, val)
_set_pos(node, lineno, col, 0, 0)
return node
name_str: str = _cur_str(ps)
# Wildcard pattern: _
if string.strcmp(name_str, "_") == 0:
_advance(ps)
node = MatchAs(ps.pool, None, None)
_set_pos(node, lineno, col, 0, 0)
return node
_advance(ps)
# Check dotted name（value pattern 或 class pattern）
if _match_op(ps, TokOp.Dot):
name_node: rawptr[Name] = Name(ps.pool, name_str, ASTCtx.Load)
_set_pos(name_node, lineno, col, 0, 0)
base: rawptr[AST] = _make_name_ast(ps.pool, name_node)
while _match_op(ps, TokOp.Dot):
_advance(ps)
if _cur_type(ps) != TokenType.Name:
ps.error_count += 1
break
attr_name: str = _cur_str(ps)
attr_lineno: int = _cur_lineno(ps)
attr_col: int = _cur_col(ps)
_advance(ps)
base = Attribute(ps.pool, base, attr_name, ASTCtx.Load)
_set_pos(base, attr_lineno, attr_col, 0, 0)
# class pattern: dotted.name(...)
if _match_op(ps, TokOp.LPar):
return _parse_class_pattern(ps, base, lineno, col)
# value pattern: dotted.name
node = MatchValue(ps.pool, base)
_set_pos(node, lineno, col, 0, 0)
return node
# class pattern: Name(...)
if _match_op(ps, TokOp.LPar):
cls_node: rawptr[Name] = Name(ps.pool, name_str, ASTCtx.Load)
_set_pos(cls_node, lineno, col, 0, 0)
cls: rawptr[AST] = _make_name_ast(ps.pool, cls_node)
return _parse_class_pattern(ps, cls, lineno, col)
# capture pattern: NAME
node = MatchAs(ps.pool, None, name_str)
_set_pos(node, lineno, col, 0, 0)
return node
# group pattern / sequence pattern (tuple): (...)
if _match_op(ps, TokOp.LPar):
_advance(ps)
inner: rawptr[AST] = _parse_pattern(ps)
if _accept_op(ps, TokOp.Comma):
# sequence pattern (tuple)
patterns: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
if inner is not None:
patterns.append(inner)
while not _match_op(ps, TokOp.RPar) and _cur_type(ps) != TokenType.EndMarker:
p: rawptr[AST] = _parse_pattern(ps)
if p is not None:
patterns.append(p)
if not _accept_op(ps, TokOp.Comma):
break
_expect_op(ps, TokOp.RPar)
node = MatchSequence(ps.pool, patterns)
_set_pos(node, lineno, col, 0, 0)
return node
_expect_op(ps, TokOp.RPar)
return inner # group pattern
# sequence pattern (list): [...]
if _match_op(ps, TokOp.Lsqb):
return _parse_sequence_pattern(ps, TokOp.Rsqb, lineno, col)
# mapping pattern: {...}
if _match_op(ps, TokOp.LBrace):
return _parse_mapping_pattern(ps, lineno, col)
ps.error_count += 1
return None
def _parse_sequence_pattern(ps: rawptr[Parser], close_op: int,
lineno: int, col: int) -> rawptr[AST]:
"""sequence_pattern: '[' [pattern_list] ']'"""
patterns: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
_advance(ps) # [
while not _match_op(ps, close_op) and _cur_type(ps) != TokenType.EndMarker:
# *name (star pattern)
if _match_op(ps, TokOp.Star):
_advance(ps)
if _cur_type(ps) == TokenType.Name:
star_name: str = _cur_str(ps)
_advance(ps)
star_node: rawptr[AST]
if string.strcmp(star_name, "_") == 0:
star_node = MatchStar(ps.pool, None)
else:
star_node = MatchStar(ps.pool, star_name)
_set_pos(star_node, lineno, col, 0, 0)
patterns.append(star_node)
else:
p: rawptr[AST] = _parse_pattern(ps)
if p is not None:
patterns.append(p)
if not _accept_op(ps, TokOp.Comma):
break
_expect_op(ps, close_op)
node: rawptr[AST] = MatchSequence(ps.pool, patterns)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_mapping_pattern(ps: rawptr[Parser],
lineno: int, col: int) -> rawptr[AST]:
"""mapping_pattern: '{' [items] '}'
items: key ':' pattern (',' key ':' pattern)* [',' '**' NAME] | '**' NAME"""
keys: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
patterns: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
rest: str = None
node: rawptr[AST]
key: rawptr[AST]
val_pat: rawptr[AST]
_advance(ps) # {
if _match_op(ps, TokOp.RBrace):
_advance(ps)
node = MatchMapping(ps.pool, keys, patterns, rest)
_set_pos(node, lineno, col, 0, 0)
return node
while _cur_type(ps) != TokenType.EndMarker:
if _match_op(ps, TokOp.StarStar):
_advance(ps)
if _cur_type(ps) == TokenType.Name:
rest = _cur_str(ps)
_advance(ps)
break
key = _parse_expr(ps)
_expect_op(ps, TokOp.Colon)
val_pat = _parse_pattern(ps)
if key is not None:
keys.append(key)
if val_pat is not None:
patterns.append(val_pat)
if not _accept_op(ps, TokOp.Comma):
break
_expect_op(ps, TokOp.RBrace)
node = MatchMapping(ps.pool, keys, patterns, rest)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_class_pattern(ps: rawptr[Parser], cls: rawptr[AST],
lineno: int, col: int) -> rawptr[AST]:
"""class_pattern: cls '(' [pattern_list] ')'
pattern_list: positional_pattern (',' positional_pattern)*
[',' keyword_pattern]* | keyword_pattern (',' keyword_pattern)*
keyword_pattern: NAME '=' pattern"""
patterns: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
kwd_attrs: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
kwd_patterns: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
p: rawptr[AST]
_advance(ps) # (
if not _match_op(ps, TokOp.RPar):
while _cur_type(ps) != TokenType.EndMarker:
# keyword pattern: NAME '=' pattern
if _cur_type(ps) == TokenType.Name and ps.cur is not None:
nxt: rawptr[Token] = ps.cur.next
if nxt is not None and nxt.type == TokenType.Op and nxt.op_subtype == TokOp.Equal:
attr_name: str = _cur_str(ps)
_advance(ps) # NAME
_advance(ps) # =
kwd_pat: rawptr[AST] = _parse_pattern(ps)
attr_n: rawptr[Name] = Name(ps.pool, attr_name, ASTCtx.Load)
_set_pos(attr_n, lineno, col, 0, 0)
attr_node: rawptr[AST] = _make_name_ast(ps.pool, attr_n)
kwd_attrs.append(attr_node)
if kwd_pat is not None:
kwd_patterns.append(kwd_pat)
else:
p = _parse_pattern(ps)
if p is not None:
patterns.append(p)
else:
p = _parse_pattern(ps)
if p is not None:
patterns.append(p)
if not _accept_op(ps, TokOp.Comma):
break
_expect_op(ps, TokOp.RPar)
node: rawptr[AST] = MatchClass(ps.pool, cls, patterns, kwd_attrs, kwd_patterns)
_set_pos(node, lineno, col, 0, 0)
return node
def _parse_match_case(ps: rawptr[Parser]) -> rawptr[AST]:
"""case pattern [if guard] ':' suite"""
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps) # case
guard: rawptr[AST] = None
pattern: rawptr[AST] = _parse_pattern(ps)
if _cur_kw(ps) == Keyword.If:
_advance(ps)
guard = _parse_or_test(ps)
_expect_op(ps, TokOp.Colon)
node: rawptr[AST] = MatchCase(ps.pool, pattern, guard)
_parse_suite_into_children(ps, node)
_set_pos(node, lineno, col, 0, 0)
return node
# ============================================================
# __attribute__ 装饰器 desugar（C 风格属性书写法）
#
# @__attribute__(section(".text.wo"), always_inline)
# def f(): ...
#
# 等价于手写 `@c.Attribute(t.attr.section(".text.wo"), t.attr.always_inline)`：
# 实参用**无前缀**属性名，逐个 desugar 为 `t.attr.<name>`（带调用时保留实参），
# 外层换成 `c.Attribute(...)`。产出的装饰器与手写形态逐字段同形 ⇒
# ClassGenBase._has_decorator / FuncGenBase.extract_func_attrs / _append_llvm_attr
# 原样识别（零新增判据）；也即同时适用于 class 与 def（装饰器链共用本函数）。
# 容错：实参已是 `t.attr.X` / `c.Attribute(...)` 形态时原样保留（允许混写）。
# ============================================================
def _IsMagicAttributeCallee(node: ast.AST*) -> int:
"""节点是否为 `__attribute__`（Name 形态，或 X.__attribute__ 的尾段形态）"""
if node is None:
return 0
k: int = node.kind()
if k == ast.ASTKind.Name:
nm: ast.Name* = ast.Name*(node)
if nm.id is not None and string.strcmp(nm.id, "__attribute__") == 0:
return 1
return 0
if k == ast.ASTKind.Attribute:
at: ast.Attribute* = ast.Attribute*(node)
if at.attr is not None and string.strcmp(at.attr, "__attribute__") == 0:
return 1
return 0
def _MakeTAttrMarker(ps: rawptr[Parser], arg: ast.AST*) -> ast.AST*:
"""把一个 __attribute__ 实参 desugar 为 `t.attr.<name>`（带调用则保留实参）
- `always_inline`（Name） → `t.attr.always_inline`
- `section(".text.wo")`（Call）→ `t.attr.section(".text.wo")`
- 已含 `t.attr.X` 形态（value 也是 Attribute）→ 原样返回
"""
if arg is None:
return None
k: int = arg.kind()
if k == ast.ASTKind.Attribute:
at0: ast.Attribute* = ast.Attribute*(arg)
if at0.value is not None and at0.value.kind() == ast.ASTKind.Attribute:
return arg
return arg
attr_root: ast.AST* = Attribute(ps.pool, Name(ps.pool, "t", ASTCtx.Load),
"attr", ASTCtx.Load)
if k == ast.ASTKind.Name:
nm2: ast.Name* = ast.Name*(arg)
if nm2.id is None:
return arg
mk: ast.AST* = Attribute(ps.pool, attr_root, nm2.id, ASTCtx.Load)
_set_pos(mk, nm2.lineno, nm2.col_offset, 0, 0)
return mk
if k == ast.ASTKind.Call:
cl2: ast.Call* = ast.Call*(arg)
if cl2.func is None or cl2.func.kind() != ast.ASTKind.Name:
return arg
fnm: ast.Name* = ast.Name*(cl2.func)
if fnm.id is None:
return arg
inner: ast.AST* = Attribute(ps.pool, attr_root, fnm.id, ASTCtx.Load)
_set_pos(inner, fnm.lineno, fnm.col_offset, 0, 0)
mk2: ast.AST* = Call(ps.pool, inner, cl2.args, cl2.keywords)
_set_pos(mk2, cl2.lineno, cl2.col_offset, 0, 0)
return mk2
return arg
def _DesugarMagicAttribute(ps: rawptr[Parser], expr: ast.AST*) -> ast.AST*:
"""`@__attribute__(...)` → `@c.Attribute(t.attr.x, ...)`；其它装饰器原样返回"""
if expr is None or expr.kind() != ast.ASTKind.Call:
return expr
cl: ast.Call* = ast.Call*(expr)
if _IsMagicAttributeCallee(cl.func) == 0:
return expr
c_func: ast.AST* = Attribute(ps.pool, Name(ps.pool, "c", ASTCtx.Load),
"Attribute", ASTCtx.Load)
new_args: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
if new_args is None:
return expr
if cl.args is not None:
an: sizet = len(cl.args)
ai: sizet = 0
while ai < an:
mk: ast.AST* = _MakeTAttrMarker(ps, cl.args.get(ai))
if mk is not None:
new_args.append(mk)
ai += 1
new_call: ast.AST* = Call(ps.pool, c_func, new_args, None)
_set_pos(new_call, cl.lineno, cl.col_offset, 0, 0)
return new_call
def _parse_decorators(ps: rawptr[Parser]) -> rawptr[list[rawptr[AST]]]:
"""解析装饰器链，返回 list。"""
decs: rawptr[list[rawptr[AST]]] = list[rawptr[AST]](ps.pool, 8)
while _match_op(ps, TokOp.At):
lineno: int = _cur_lineno(ps)
col: int = _cur_col(ps)
_advance(ps) # @
expr: rawptr[AST] = _parse_namedexpr(ps)
# C 风格 `@__attribute__(...)` desugar 成既有的 `@c.Attribute(t.attr.<x>...)`
expr = _DesugarMagicAttribute(ps, expr)
_skip_nl(ps)
if _cur_type(ps) == TokenType.NewLine:
_advance(ps)
decs.append(expr)
return decs
# ============================================================
# Module entry
# ============================================================
def _parse_module_body(ps: rawptr[Parser], module_node: rawptr[AST]):
"""解析模块体，append 到 module_node。"""
_skip_nl(ps)
stmt_idx: int = 0
while _cur_type(ps) != TokenType.EndMarker:
stmt_idx += 1
# fail-fast 防卡死：步数超预算（token 链表成环 / 游标原地打转）立即报错返回
if ps.step_count > ps.step_limit:
_record_parse_err(ps)
return
# Decorator
if _match_op(ps, TokOp.At):
ps.pending_decorators = _parse_decorators(ps)
# Record current token pointer to check for infinite loop
before_cur: rawptr[Token] = ps.cur
_parse_statement_into(ps, module_node)
# Defense: If parsing doesn't advance the token (like when _parse_atom encounters an unrecognized token and returns None),
# force it to move forward to avoid an infinite loop in the main loop; accumulate error_count for the caller to diagnose
if ps.cur is before_cur:
ps.error_count += 1
_advance(ps)
_skip_nl(ps)
def parse_tokens(pool: rawptr[memhub.MemManager],
tokens: rawptr[Token]) -> rawptr[AST]:
"""Parse an AST Module from a token list."""
if pool is None or tokens is None:
return None
global _g_parse_err_line, _g_parse_err_col, _g_parse_err_msg
_g_parse_err_line = 0
_g_parse_err_col = 0
_g_parse_err_msg = None
# 词法层 fail-fast 优先上报：tokenize 期间记录的溢出（如整数字面量 > 2^64-1）在此
# 转写到解析错误通道，复用 dump_parse_traceback 的 "文件:行:列 + 源码行 +
# SyntaxError: 消息" 报告路径（并终止编译）。lexer 无独立 fatal 出口，这是它
# 唯一的对外上报点。
lex_line: int = get_lex_err_line()
if lex_line != 0:
_g_parse_err_line = lex_line
_g_parse_err_col = get_lex_err_col()
_g_parse_err_msg = get_lex_err_msg()
return None
ps: rawptr[Parser] = new_parser(pool)
if ps is None: return None
_parser_init(ps, tokens, pool)
# fail-fast 防卡死：先统计 token 链表长度，顺带检测成环。
# 背景：lexer 的 token 链表若出现环（_emit 追加了同一个 Token 指针两次，或分配
# 别名把两个 token 落在同一地址），语句主循环永远等不到 EndMarker → 编译进程
# 静默卡死（无输出、无报错）。此处 walk 到尾得到 token 数作为步数预算；超过上限
# 仍未到尾即判为成环，立即报错返回（绝不进入无限解析）。
tok_count: sizet = 0
walk_t: rawptr[Token] = tokens
while walk_t is not None:
walk_t = walk_t.next
tok_count += 1
if tok_count > PARSER_MAX_TOKENS:
_g_parse_err_line = _cur_lineno(ps)
_g_parse_err_col = _cur_col(ps)
_g_parse_err_msg = "token 链表成环：lexer 输出的 token 链表无终止（_emit 重复追加同一 Token 或分配别名），已终止解析"
return None
# 预算：看门狗按"当前 token 查询"计数（合法解析每 token 的查询次数远小于 256，
# 不可能触顶），成环/原地打转必触发；它同时是 _advance 步数与语句循环的上界。
ps.step_limit = tok_count * 256 + 262144
module_node: rawptr[Module] = Module(pool)
_parse_module_body(ps, module_node)
# fail-fast: 解析出错（error_count > 0）时返回 None，由调用方（TPV 等）报错终止，
# 避免错误代码静默编译（如 `func12344444 avg3(...)` 缺 def 关键字的函数定义残留）。
if ps.error_count > 0: return None
return module_node
Reference in new issue View Git Blame Copy Permalink