diff --git a/ast/exprs.vp b/ast/exprs.vp index bc72ab0..381fe36 100644 --- a/ast/exprs.vp +++ b/ast/exprs.vp @@ -82,11 +82,16 @@ class Constant(ExprNode): float_val: double str_val: str const_kind: int + # 显式整型宽度后缀（_int/_uint）：int_width>0 = 声明宽度，0 = 未指定； + # int_unsigned=1 表示 uint 形式。追加在末尾，保持既有字段偏移不变（经 __sizeof__ 分配并 memset）。 + int_width: int + int_unsigned: int def __new__(self, pool: memhub.MemManager*, const_kind: int, int_val: int64, float_val: double, str_val: str, - lineno: int, col: int): + lineno: int, col: int, + int_width: int = 0, int_unsigned: int = 0): ptr: Constant* = pool.alloc(Constant.__sizeof__()) if ptr: string.memset(ptr, 0, Constant.__sizeof__()) @@ -95,11 +100,14 @@ class Constant(ExprNode): def __init__(self, pool: memhub.MemManager*, const_kind: int, int_val: int64, float_val: double, str_val: str, - lineno: int, col: int): + lineno: int, col: int, + int_width: int = 0, int_unsigned: int = 0): _init_ast(self, pool, lineno, col) self.const_kind = const_kind self.int_val = int_val self.float_val = float_val + self.int_width = int_width + self.int_unsigned = int_unsigned if str_val is not None: self.str_val = _copy_str(pool, str_val) else: diff --git a/ast/lexer.vp b/ast/lexer.vp index 40cd688..cba2c02 100644 --- a/ast/lexer.vp +++ b/ast/lexer.vp @@ -1,1131 +1,1191 @@ -from stdint import * -import memhub -import string -import stdio -import stdlib -from .tokens import ( - Token, TokenType, TokOp, _init_tables, _kw_lookup, _op_head, - OpEntry, new_token, token_set_str, token_set_str_literal, -) - - -# ============================================================ -# Integer literal overflow (lexer-layer fail-fast) -# -# Integer constants are int64 in the token/AST (tokens.vp `int_val` / exprs.vp `int_val`). -# The old implementation used `string.atoll` (string.vp:292) to, under int64, do `num = num * 10 + digit` -# digit-by-digit accumulation => when the value > 2^64-1 it **silently wraps** (e.g. 2^64 wraps to 0, 2^64-1 wraps to -1). -# This implementation, in `_emit_number`, first performs a pure-text magnitude check on **integer** literals -# (`_lex_int_exceeds_u64`, using the same criteria as AssignGenMixin._int_text_exceeds_64: by -# radix, counting significant digits / comparing length + lexicographic order against the decimal text of 2^64-1, **without landing in any fixed-width -# container and without wrapping before the comparison**), so exceeding 2^64-1 records a fail-fast (no more silent wrapping). -# The legal range 0..2^64-1 is still stored into int64 as a 64-bit bit pattern (the shape of the results on the existing legal path is unchanged, -# e.g. 0xFFFFFFFFFFFFFFFF -> int64 -1, 0xA5A5A5A5A5A5A5A5 keeps its 64-bit bit pattern). -# The criteria for each radix (2/8/10/16) match TPC's (Python int(x, 0), arbitrary precision): the accepted interval is likewise -# [-2^63, 2^64-1] (literals themselves are unsigned; the negative side is expressed by unary minus), and anything beyond is rejected. -# -# Reporting channel: the lexer has no separate fatal exit, so it records into a module-level triple (`_lex_record_overflow`), -# which the parser's parse_tokens reads and transcribes into _g_parse_err_* (position + message), reusing the existing -# Traceback/SyntaxError report path (the "file:line:col + source line" is filled in by that path, which also terminates compilation). -# This covers all source literals (file-level parsing necessarily goes through parse_tokens); parse_expression is used only for compiler- -# synthesized short type strings and carries no source literals, so no separate channel is provided. -# ============================================================ -_g_lex_err_line: int = 0 -_g_lex_err_col: int = 0 -_g_lex_err_msg: str = None - - -def get_lex_err_line() -> int: - """Line number of the latest lexer fail-fast (0 = none). - - Consumed by parser.parse_tokens and forwarded to the parse error channel. - """ - return _g_lex_err_line - - -def get_lex_err_col() -> int: - """Column of the latest lexer fail-fast (0-based; -1 = no column info)""" - return _g_lex_err_col - - -def get_lex_err_msg() -> str: - """Message of the latest lexer fail-fast (literal / required width / limit / fix hint)""" - return _g_lex_err_msg - - -def _lex_reset_err(): - """Clear the lexer fail-fast triple (called at each tokenize entry to avoid stale errors)""" - global _g_lex_err_line, _g_lex_err_col, _g_lex_err_msg - _g_lex_err_line = 0 - _g_lex_err_col = 0 - _g_lex_err_msg = None - - -def _lex_record_overflow(text: str, lineno: int, col: int): - """Record an integer literal overflow (fail-fast; first record only); message carries literal - + required width (>= 65 bits, limit 2^64-1) + fix hint (file:line:col filled later by parse_tokens). - """ - global _g_lex_err_line, _g_lex_err_col, _g_lex_err_msg - if _g_lex_err_line != 0: - return - _g_lex_err_line = lineno - _g_lex_err_col = col - msg_buf: str = stdlib.malloc(512) - if msg_buf is not None: - stdio.snprintf(msg_buf, 512, - "[TPV] integer literal overflow: literal '%s' exceeds the 64-bit representable range" - " (the bit-pattern interpretation upper bound is 18446744073709551615 = 2^64-1," - " the literal needs a width of at least 65 bits)." - " Silent wrap-around/truncation is forbidden; compilation aborted." - " Fix: change the literal into the 64-bit bit-pattern range 0..18446744073709551615;" - " this language has no wider integer type, for larger values split into multiple 64-bit words.", text) - _g_lex_err_msg = msg_buf - else: - _g_lex_err_msg = text - - -def _lex_int_exceeds_u64(text: str) -> int: - """True if the integer literal magnitude exceeds the 64-bit limit 2^64-1 (1 = yes, 0 = no). - Pure text compare, no wrap-around: decimal length+lexicographic vs "18446744073709551615", - 0x > 16, 0o > 22 (or 22 leading > '1'), 0b > 64 (literal text carries no sign). - """ - if text is None: - return 0 - tl: sizet = string.strlen(text) - if tl == 0: - return 0 - # Base detection (prefix letter case-insensitive) - base: int = 10 - start: sizet = 0 - if tl >= 2 and text[0] == '0': - pfx: char = text[1] - if pfx == 'x' or pfx == 'X': - base = 16 - start = 2 - elif pfx == 'o' or pfx == 'O': - base = 8 - start = 2 - elif pfx == 'b' or pfx == 'B': - base = 2 - start = 2 - # Count significant digits (skip '_' and leading zeros) and remember the first one - sig: int = 0 - first: char = '0' - j: sizet = start - while j < tl: - ch: char = text[j] - j += 1 - if ch == '_': - continue - ok_digit: int = 0 - if base == 16: - if (ch >= '0' and ch <= '9') or (ch >= 'a' and ch <= 'f') or (ch >= 'A' and ch <= 'F'): - ok_digit = 1 - elif base == 10: - if ch >= '0' and ch <= '9': - ok_digit = 1 - elif base == 8: - if ch >= '0' and ch <= '7': - ok_digit = 1 - else: - if ch == '0' or ch == '1': - ok_digit = 1 - if ok_digit == 0: - break - if sig == 0: - if ch == '0': - continue # leading zeros not counted - first = ch - sig += 1 - if sig == 0: - return 0 # value 0 - if base == 16: - return 1 if sig > 16 else 0 - if base == 2: - return 1 if sig > 64 else 0 - if base == 8: - if sig > 22: - return 1 - if sig == 22 and first > '1': - return 1 - return 0 - # Decimal: compare significant-digit count against the 2^64-1 text length - mlen: sizet = 20 - if sig > mlen: - return 1 - if sig < mlen: - return 0 - # Equal length (20 digits): lexicographic compare against '18446744073709551615' - max_text: str = "18446744073709551615" - d: sizet = 0 - k: sizet = start - lead: int = 1 - while k < tl and d < mlen: - c: char = text[k] - k += 1 - if c == '_': - continue - if c < '0' or c > '9': - return 0 - if lead != 0 and c == '0': - continue - lead = 0 - mc: char = max_text[d] - if c > mc: - return 1 - if c < mc: - return 0 - d += 1 - return 0 - - - -def _lex_mag_digit_val(ch: char, base: int) -> int: - """Digit char -> positional value (hex accepts a-f/A-F; other bases are '0'..base-1).""" - if base == 16: - return _hex_val(ch) - return int(ch) - 48 - - -def _lex_mag_gt_text(text: str, base: int, start: sizet, sig: int, max_text: str) -> int: - """True if the literal's significant digits exceed the same-length max text (1 = yes). - Both sides skip '_'/leading zeros, compared by positional value (hex via _hex_val); pure text. - """ - d: sizet = 0 - k: sizet = start - tl: sizet = string.strlen(text) - lead: int = 1 - while k < tl and d < sizet(sig): - c: char = text[k] - k += 1 - if c == '_': - continue - if base == 16: - if not ((c >= '0' and c <= '9') or (c >= 'a' and c <= 'f') or (c >= 'A' and c <= 'F')): - break - elif base == 10: - if c < '0' or c > '9': - break - elif base == 8: - if c < '0' or c > '7': - break - else: - if c != '0' and c != '1': - break - if lead != 0 and c == '0': - continue - lead = 0 - cv: int = _lex_mag_digit_val(c, base) - mv: int = _lex_mag_digit_val(max_text[d], base) - if cv > mv: - return 1 - if cv < mv: - return 0 - d += 1 - return 0 - - -def _lex_int_neg_exceeds_i64(text: str) -> int: - """True if the magnitude is strictly greater than 2^63 (1 = yes). - For unary-minus + literal overflow: a magnitude-valid literal may become < -2^63 after - negation; exactly 2^63 is legal (INT64_MIN). Pure text compare, no wrap-around. - """ - if text is None: - return 0 - tl: sizet = string.strlen(text) - if tl == 0: - return 0 - base: int = 10 - start: sizet = 0 - if tl >= 2 and text[0] == '0': - pfx: char = text[1] - if pfx == 'x' or pfx == 'X': - base = 16 - start = 2 - elif pfx == 'o' or pfx == 'O': - base = 8 - start = 2 - elif pfx == 'b' or pfx == 'B': - base = 2 - start = 2 - sig: int = 0 - j: sizet = start - while j < tl: - ch: char = text[j] - j += 1 - if ch == '_': - continue - ok_digit: int = 0 - if base == 16: - if (ch >= '0' and ch <= '9') or (ch >= 'a' and ch <= 'f') or (ch >= 'A' and ch <= 'F'): - ok_digit = 1 - elif base == 10: - if ch >= '0' and ch <= '9': - ok_digit = 1 - elif base == 8: - if ch >= '0' and ch <= '7': - ok_digit = 1 - else: - if ch == '0' or ch == '1': - ok_digit = 1 - if ok_digit == 0: - break - if sig == 0 and ch == '0': - continue # leading zeros not counted - sig += 1 - if sig == 0: - return 0 # value 0 - if base == 16: - if sig > 16: return 1 - if sig < 16: return 0 - return _lex_mag_gt_text(text, 16, start, sig, "8000000000000000") - if base == 2: - if sig > 64: return 1 - if sig < 64: return 0 - return _lex_mag_gt_text(text, 2, start, sig, - "1000000000000000000000000000000000000000000000000000000000000000") - if base == 8: - if sig > 22: return 1 - if sig < 22: return 0 - return _lex_mag_gt_text(text, 8, start, sig, "1000000000000000000000") - if sig > 19: return 1 - if sig < 19: return 0 - return _lex_mag_gt_text(text, 10, start, sig, "9223372036854775808") - - -# Maximum Indentation Stack Depth -define LEXER_MAX_INDENT = 256 - - -# ============================================================ -# Lexer State Structure -# ============================================================ -class Lexer: - src: str # Source code (NUL-terminated) - pos: sizet # Current position - len: sizet # Source code length - lineno: int # Current line number (1-based) - col: int # Current column number (0-based) - pool: memhub.MemManager* # Memory pool - indent_stack: int* # Indent stack (int array, allocated from pool) - indent_top: int # Stack top (-1 for empty stack) - paren_depth: int # Paren depth (>0 for no TokenType.NewLine) - tokens_head: Token* # token list head - tokens_tail: Token* # token list tail - at_line_start: int # 1=line start (need indent) - error_count: int # Error count - - def __new__(self, pool: memhub.MemManager*): - ptr: Lexer* = pool.alloc(Lexer.__sizeof__()) - if ptr == None: - # Pool exhausted: fail-fast instead of returning NULL (downstream would treat NULL as a valid Lexer) - stdio.printf("[AST] Lexer allocation failed: memory pool exhausted (need=%zu bytes)\n", - sizet(Lexer.__sizeof__())) - stdio.fflush(None) - stdlib.exit(1) - string.memset(ptr, 0, Lexer.__sizeof__()) - return ptr - - -def new_lexer(pool: memhub.MemManager*) -> Lexer*: - """Allocate from the pool and return a Lexer (cross-module safe wrapper)""" - return Lexer(pool) - - -def _lexer_init(lx: Lexer*, src: str, pool: memhub.MemManager*): - """Initialize Lexer""" - if lx == None: return - lx.src = src - lx.pos = 0 - lx.len = string.strlen(src) if src != None else 0 - lx.lineno = 1 - lx.col = 0 - lx.pool = pool - # Allocate indentation stack - lx.indent_stack = pool.alloc(LEXER_MAX_INDENT * 8) - if lx.indent_stack == None: - stdio.printf("[AST] Lexer.indent_stack allocation failed: memory pool exhausted (need=%zu bytes)\n", - sizet(LEXER_MAX_INDENT * 8)) - stdio.fflush(None) - stdlib.exit(1) - string.memset(lx.indent_stack, 0, LEXER_MAX_INDENT * 8) - lx.indent_top = -1 - # Push 0 (base indent) - lx.indent_stack[0] = 0 - lx.indent_top = 0 - lx.paren_depth = 0 - lx.tokens_head = None - lx.tokens_tail = None - lx.at_line_start = 1 - lx.error_count = 0 - _init_tables(pool) - - -# ============================================================ -# Auxiliary Functions -# ============================================================ - -def _peek(lx: Lexer*, offset: sizet) -> char: - """Peek character at position pos+offset, return '\0' if out of bounds""" - p: sizet = lx.pos + offset - if p >= lx.len: return '\0' - return lx.src[p] - - -def _peek_cur(lx: Lexer*) -> char: - if lx.pos >= lx.len: return '\0' - return lx.src[lx.pos] - - -def _advance(lx: Lexer*) -> char: - """Advance one character, return the character""" - if lx.pos >= lx.len: return '\0' - ch: char = lx.src[lx.pos] - lx.pos += 1 - if ch == '\n': - lx.lineno += 1 - lx.col = 0 - else: - lx.col += 1 - return ch - - -def _is_alpha(ch: char) -> int: - if ch >= 'a' and ch <= 'z': return 1 - if ch >= 'A' and ch <= 'Z': return 1 - if ch == '_': return 1 - return 0 - - -def _is_digit(ch: char) -> int: - if ch >= '0' and ch <= '9': return 1 - return 0 - - -def _is_alnum(ch: char) -> int: - if _is_alpha(ch): return 1 - if _is_digit(ch): return 1 - return 0 - - -def _is_hex(ch: char) -> int: - if ch >= '0' and ch <= '9': return 1 - if ch >= 'a' and ch <= 'f': return 1 - if ch >= 'A' and ch <= 'F': return 1 - return 0 - - -def _hex_val(ch: char) -> int: - """Hex char -> value 0-15 (for \xNN escape parsing)""" - if ch >= '0' and ch <= '9': return int(ch) - 48 - if ch >= 'a' and ch <= 'f': return int(ch) - 87 - if ch >= 'A' and ch <= 'F': return int(ch) - 55 - return 0 - - -def _is_space(ch: char) -> int: - if ch == ' ': return 1 - if ch == '\t': return 1 - if ch == '\f': return 1 - if ch == '\v': return 1 - if ch == '\r': return 1 - return 0 - - -def _emit(lx: Lexer*, tok: Token*): - """Append token to token list""" - if tok == None: return - if lx.tokens_head == None: - lx.tokens_head = tok - lx.tokens_tail = tok - else: - lx.tokens_tail.next = tok - lx.tokens_tail = tok - - -def _make_op_token(lx: Lexer*, op_id: int, length: int, - lineno: int, col: int) -> Token*: - tok: Token* = new_token(lx.pool, TokenType.Op, lineno, col) - if tok == None: return None - tok.op_subtype = op_id - tok.end_col_offset = col + length - return tok - - -# ============================================================ -# Indent Handling -# ============================================================ - -def _handle_indent(lx: Lexer*): - """Handle line start indent, generate TokenType.Indent/TokenType.Dedent. - Skip empty lines and comment lines.""" - # Skip empty lines and comment lines - while lx.pos < lx.len: - # Save line start position for non-empty lines - line_start_pos: sizet = lx.pos - line_start_col: int = lx.col - # Skip whitespace, not across lines - while lx.pos < lx.len: - ch: char = lx.src[lx.pos] - if ch == ' ' or ch == '\t' or ch == '\f' or ch == '\v': - lx.pos += 1 - lx.col += 1 - elif ch == '\r': - # CRLF line end \r, skip \n to trigger empty line processing - lx.pos += 1 - else: - break - if lx.pos >= lx.len: - break - ch = lx.src[lx.pos] - if ch == '\n': - # Empty line, skip - lx.pos += 1 - lx.lineno += 1 - lx.col = 0 - continue - if ch == '\r': - # CRLF line end \r, skip \n to trigger empty line processing - lx.pos += 1 - if lx.pos < lx.len and lx.src[lx.pos] == '\n': - lx.pos += 1 - lx.lineno += 1 - lx.col = 0 - continue - if ch == '#': - # Comment line, skip to end of line - while lx.pos < lx.len and lx.src[lx.pos] != '\n': - lx.pos += 1 - continue - # Non-empty line: backtrack to line start to allow indent calculation - lx.pos = line_start_pos - lx.col = line_start_col - break - - if lx.pos >= lx.len: - # End of file, no indent to handle - lx.at_line_start = 0 - return - - # Calculate current line indent - indent: int = 0 - save_pos: sizet = lx.pos - save_col: int = lx.col - while lx.pos < lx.len: - ch: char = lx.src[lx.pos] - if ch == ' ': - indent += 1 - lx.pos += 1 - lx.col += 1 - elif ch == '\t': - indent += 8 - (indent % 8) - lx.pos += 1 - lx.col += 1 - else: - break - - # If line end (empty line), skip indent calculation - if lx.pos < lx.len: - ch = lx.src[lx.pos] - if ch == '\n' or ch == '\r' or ch == '#': - # Empty line or comment line, backtrack position to allow loop to handle indent - lx.pos = save_pos - lx.col = save_col - lx.at_line_start = 0 - return - - # Compare indent with top of indent stack - top_indent: int = lx.indent_stack[lx.indent_top] - if indent > top_indent: - # TokenType.Indent - if lx.indent_top + 1 >= LEXER_MAX_INDENT: - lx.error_count += 1 - return - lx.indent_top += 1 - lx.indent_stack[lx.indent_top] = indent - tok: Token* = new_token(lx.pool, TokenType.Indent, lx.lineno, 0) - _emit(lx, tok) - else: - while indent < lx.indent_stack[lx.indent_top] and lx.indent_top > 0: - lx.indent_top -= 1 - tok: Token* = new_token(lx.pool, TokenType.Dedent, lx.lineno, 0) - _emit(lx, tok) - lx.at_line_start = 0 - - -# ============================================================ -# Identifier/Keyword Handling -# ============================================================ - -def _read_name(lx: Lexer*): - start: sizet = lx.pos - start_lineno: int = lx.lineno - start_col: int = lx.col - while lx.pos < lx.len: - ch: char = lx.src[lx.pos] - if _is_alnum(ch): - lx.pos += 1 - lx.col += 1 - else: - break - length: sizet = lx.pos - start - tok: Token* = new_token(lx.pool, TokenType.Name, start_lineno, start_col) - if tok == None: return - token_set_str(lx.pool, tok, lx.src, start, length) - tok.end_lineno = lx.lineno - tok.end_col_offset = start_col + int(length) - # Look up keyword if any - kw_id: int = _kw_lookup(tok.str_val) - if kw_id: tok.kw_subtype = kw_id - _emit(lx, tok) - - -# ============================================================ -# Number Handling -# ============================================================ - -def _read_number(lx: Lexer*): - start: sizet = lx.pos - start_lineno: int = lx.lineno - start_col: int = lx.col - is_float: int = 0 - is_complex: int = 0 - - ch0: char = lx.src[lx.pos] - # Handle 0x/0o/0b prefix - if ch0 == '0' and lx.pos + 1 < lx.len: - ch1: char = lx.src[lx.pos + 1] - if ch1 == 'x' or ch1 == 'X': - lx.pos += 2 - lx.col += 2 - while lx.pos < lx.len: - ch: char = lx.src[lx.pos] - if _is_hex(ch) or ch == '_': - lx.pos += 1 - lx.col += 1 - else: - break - _emit_number(lx, start, start_lineno, start_col, 0, 0, 0) - return - if ch1 == 'o' or ch1 == 'O': - lx.pos += 2 - lx.col += 2 - while lx.pos < lx.len: - ch: char = lx.src[lx.pos] - if ch >= '0' and ch <= '7': - lx.pos += 1 - lx.col += 1 - elif ch == '_': - lx.pos += 1 - lx.col += 1 - else: - break - _emit_number(lx, start, start_lineno, start_col, 0, 0, 0) - return - if ch1 == 'b' or ch1 == 'B': - lx.pos += 2 - lx.col += 2 - while lx.pos < lx.len: - ch: char = lx.src[lx.pos] - if ch == '0' or ch == '1' or ch == '_': - lx.pos += 1 - lx.col += 1 - else: - break - _emit_number(lx, start, start_lineno, start_col, 0, 0, 0) - return - - # Decimal integer/float - while lx.pos < lx.len: - ch: char = lx.src[lx.pos] - if _is_digit(ch) or ch == '_': - lx.pos += 1 - lx.col += 1 - else: - break - - # Decimal part - if lx.pos < lx.len and lx.src[lx.pos] == '.': - is_float = 1 - lx.pos += 1 - lx.col += 1 - while lx.pos < lx.len: - ch: char = lx.src[lx.pos] - if _is_digit(ch) or ch == '_': - lx.pos += 1 - lx.col += 1 - else: - break - - # Exponent part - if lx.pos < lx.len: - ch: char = lx.src[lx.pos] - if ch == 'e' or ch == 'E': - is_float = 1 - lx.pos += 1 - lx.col += 1 - if lx.pos < lx.len: - ch = lx.src[lx.pos] - if ch == '+' or ch == '-': - lx.pos += 1 - lx.col += 1 - while lx.pos < lx.len: - ch: char = lx.src[lx.pos] - if _is_digit(ch) or ch == '_': - lx.pos += 1 - lx.col += 1 - else: - break - - # Complex suffix part - if lx.pos < lx.len and lx.src[lx.pos] == 'j': - is_complex = 1 - is_float = 1 - lx.pos += 1 - lx.col += 1 - - _emit_number(lx, start, start_lineno, start_col, is_float, is_complex, 0) - - -def _emit_number(lx: Lexer*, start: sizet, - start_lineno: int, start_col: int, - is_float: int, is_complex: int, is_hex: int): - length: sizet = lx.pos - start - tok: Token* = new_token(lx.pool, TokenType.Number, start_lineno, start_col) - if tok == None: return - token_set_str(lx.pool, tok, lx.src, start, length) - tok.is_float = is_float - tok.is_complex = is_complex - tok.end_lineno = lx.lineno - tok.end_col_offset = start_col + int(length) - # Parse number - if is_float: - tok.float_val = string.atof(tok.str_val) - else: - # Integer literal magnitude > 2^64-1: lexer fail-fast (pure text, no silent wrap). - # Range 0..2^64-1 is still stored into int64 as a 64-bit bit pattern (unchanged shape). - if _lex_int_exceeds_u64(tok.str_val) != 0: - _lex_record_overflow(tok.str_val, start_lineno, start_col) - tok.int_val = string.atoll(tok.str_val) - _emit(lx, tok) - - -# ============================================================ -# String Handling -# ============================================================ - -def _read_string(lx: Lexer*): - """Read string literal. - Supports prefixes: r, b, u, f, rb, br, rf, fr (case insensitive) - Supports quotes: ', ", ''', - Later we need to support string prefixes like i16/i24""" - start_lineno: int = lx.lineno - start_col: int = lx.col - prefix_start: sizet = lx.pos - - # Read prefix (r/b/u/f) - is_raw: int = 0 - is_bytes: int = 0 - is_fstring: int = 0 - while lx.pos < lx.len: - ch: char = lx.src[lx.pos] - if ch == 'r' or ch == 'R': - is_raw = 1 - lx.pos += 1 - lx.col += 1 - elif ch == 'b' or ch == 'B': - is_bytes = 1 - lx.pos += 1 - lx.col += 1 - elif ch == 'u' or ch == 'U': - lx.pos += 1 - lx.col += 1 - elif ch == 'f' or ch == 'F': - is_fstring = 1 - lx.pos += 1 - lx.col += 1 - else: - break - - if lx.pos >= lx.len: - lx.error_count += 1 - return - - # Determine quote type - quote: char = lx.src[lx.pos] - if quote != '\'' and quote != '"': - lx.error_count += 1 - return - - # Check triple quote - is_triple: int = 0 - if lx.pos + 2 < lx.len: - if lx.src[lx.pos] == quote and lx.src[lx.pos + 1] == quote and lx.src[lx.pos + 2] == quote: - is_triple = 1 - - content_start: sizet = lx.pos - if is_triple: - lx.pos += 3 - lx.col += 3 - else: - lx.pos += 1 - lx.col += 1 - - # Read string content - content_buf_start: sizet = lx.pos - # f-string {} expression block nesting depth (for PEP 701 same-quote nested strings) - fstr_depth: int = 0 - while lx.pos < lx.len: - ch = lx.src[lx.pos] - if is_triple: - if ch == quote: - if lx.pos + 2 < lx.len: - if lx.src[lx.pos + 1] == quote and lx.src[lx.pos + 2] == quote: - break - elif lx.pos + 2 == lx.len: - break - if ch == '\\': - lx.pos += 1 - lx.col += 1 - if lx.pos < lx.len: - if lx.src[lx.pos] == '\n': - lx.lineno += 1 - lx.col = 0 - lx.pos += 1 - lx.col += 1 - continue - if ch == '\n': - lx.lineno += 1 - lx.col = 0 - lx.pos += 1 - lx.col += 1 - else: - if is_fstring: - # {{ and }} are escaped literals, not counted in depth - if ch == '{' and lx.pos + 1 < lx.len and lx.src[lx.pos + 1] == '{': - lx.pos += 2 - lx.col += 2 - continue - if ch == '}' and lx.pos + 1 < lx.len and lx.src[lx.pos + 1] == '}': - lx.pos += 2 - lx.col += 2 - continue - if ch == '{': - fstr_depth += 1 - lx.pos += 1 - lx.col += 1 - continue - if ch == '}': - if fstr_depth > 0: - fstr_depth -= 1 - lx.pos += 1 - lx.col += 1 - continue - if fstr_depth > 0 and (ch == '\'' or ch == '"'): - # Nested string literals allowed inside {} blocks (PEP 701 same-quote nesting): - # skip to the matching quote; inner quotes do not terminate the outer f-string. - nq: char = ch - lx.pos += 1 - lx.col += 1 - while lx.pos < lx.len: - c2: char = lx.src[lx.pos] - if c2 == '\\': - lx.pos += 2 - lx.col += 2 - continue - if c2 == nq: - lx.pos += 1 - lx.col += 1 - break - if c2 == '\n': - # single-line string cannot span lines - lx.error_count += 1 - return - lx.pos += 1 - lx.col += 1 - continue - if ch == quote: - break - if ch == '\n': - # Single quote string cannot span multiple lines (unless continued) - lx.error_count += 1 - return - if ch == '\\' and not is_raw: - lx.pos += 1 - lx.col += 1 - if lx.pos < lx.len: - lx.pos += 1 - lx.col += 1 - continue - lx.pos += 1 - lx.col += 1 - - content_end: sizet = lx.pos - # Skip the closing quote - if is_triple: - if lx.pos + 2 < lx.len: - lx.pos += 3 - lx.col += 3 - else: - lx.pos = lx.len - else: - if lx.pos < lx.len: - lx.pos += 1 - lx.col += 1 - - # Create token - tok: Token* = new_token(lx.pool, TokenType.String, start_lineno, start_col) - if tok == None: return - content_len: sizet = content_end - content_buf_start - # esc_len: Actual bytes after escaping (used to check single character) - esc_len: sizet = content_len - - # Handle escape sequences (non-raw strings) - if is_raw == 0 and content_len > 0: - # Allocate conversion buffer (max case: each char is plain, no extension) - conv_buf: str = lx.pool.alloc(content_len + 1) - if conv_buf == None: - token_set_str(lx.pool, tok, lx.src, content_buf_start, content_len) - else: - wpos: sizet = 0 - rpos: sizet = 0 - while rpos < content_len: - ch: char = lx.src[content_buf_start + rpos] - if ch == '\\' and rpos + 1 < content_len: - rpos += 1 - next_ch: char = lx.src[content_buf_start + rpos] - if next_ch == 'n': - conv_buf[wpos] = '\n' - elif next_ch == 't': - conv_buf[wpos] = '\t' - elif next_ch == 'r': - conv_buf[wpos] = '\r' - elif next_ch == 'f': - conv_buf[wpos] = '\f' - elif next_ch == 'v': - conv_buf[wpos] = '\v' - elif next_ch == 'a': - conv_buf[wpos] = '\a' - elif next_ch == 'b': - conv_buf[wpos] = '\b' - elif next_ch == '\\': - conv_buf[wpos] = '\\' - elif next_ch == '\'': - conv_buf[wpos] = '\'' - elif next_ch == '"': - conv_buf[wpos] = '"' - elif next_ch == '0': - conv_buf[wpos] = '\0' - elif next_ch == 'x' and rpos + 2 < content_len: - # \xNN hex escape (matches Python): '\\x1b' -> single byte 0x1B. - # Must yield esc_len == 1 so '\xNN' becomes CONST_CHAR, not CONST_STR. - h1: char = lx.src[content_buf_start + rpos + 1] - h2: char = lx.src[content_buf_start + rpos + 2] - if _is_hex(h1) != 0 and _is_hex(h2) != 0: - hex_v: int = (_hex_val(h1) << 4) | _hex_val(h2) - conv_buf[wpos] = char(hex_v) - wpos += 1 - rpos += 3 - continue - # Invalid hex (\x followed by fewer than 2 hex digits): keep as-is - conv_buf[wpos] = '\\' - wpos += 1 - conv_buf[wpos] = next_ch - else: - # Unknown escape sequence, keep original (includes \xNN) - conv_buf[wpos] = '\\' - wpos += 1 - conv_buf[wpos] = next_ch - wpos += 1 - rpos += 1 - else: - conv_buf[wpos] = ch - wpos += 1 - rpos += 1 - conv_buf[wpos] = '\0' - token_set_str_literal(lx.pool, tok, conv_buf) - esc_len = wpos - else: - token_set_str(lx.pool, tok, lx.src, content_buf_start, content_len) - tok.end_lineno = lx.lineno - tok.end_col_offset = lx.col - # flags field: bit0=raw, bit1=bytes, bit2=fstring, bit3=triple, bit4=dquote, bit5=single char candidate - if is_raw: tok.kw_subtype = tok.kw_subtype | 1 - if is_bytes: tok.kw_subtype = tok.kw_subtype | 2 - if is_fstring: tok.kw_subtype = tok.kw_subtype | 4 - if is_triple: tok.kw_subtype = tok.kw_subtype | 8 - # bit4 (16) = double quote=single quote - if quote == '"': tok.kw_subtype = tok.kw_subtype | 16 - # bit5 (32) = single-character candidate (not triple quotes, not f-string, exactly 1 byte after escaping) - # Used for CONST_CHAR checks to prevent '0' and other NUL bytes from being mistaken as empty strings by strlen - if is_triple == 0 and is_fstring == 0 and esc_len == 1: - tok.kw_subtype = tok.kw_subtype | 32 - _emit(lx, tok) - - -# ============================================================ -# Operator Matching -# ============================================================ - -def _read_operator(lx: Lexer*): - """Greedy match operator""" - start_lineno: int = lx.lineno - start_col: int = lx.col - remaining: sizet = lx.len - lx.pos - - # Traverse operator table, find longest match - best_entry: OpEntry* = None - best_len: int = 0 - cur: OpEntry* = _op_head - while cur != None: - if cur.length <= int(remaining): - # Compare cur.name with src[pos:pos+cur.length] - Matched: int = 1 - for i in range(sizet(cur.length)): - if lx.src[lx.pos + i] != cur.name[i]: - Matched = 0 - break - if Matched != 0: - if cur.length > best_len: - best_len = cur.length - best_entry = cur - cur = cur.next - - if best_entry == None: - # Unknown character, report error and skip - lx.error_count += 1 - lx.pos += 1 - lx.col += 1 - return - - # Update parenthesis depth - if best_entry.op_id == TokOp.LPar or best_entry.op_id == TokOp.Lsqb or best_entry.op_id == TokOp.LBrace: - lx.paren_depth += 1 - elif best_entry.op_id == TokOp.RPar or best_entry.op_id == TokOp.Rsqb or best_entry.op_id == TokOp.RBrace: - if lx.paren_depth > 0: - lx.paren_depth -= 1 - - tok: Token* = _make_op_token(lx, best_entry.op_id, best_entry.length, - start_lineno, start_col) - if tok == None: return - token_set_str(lx.pool, tok, lx.src, lx.pos, sizet(best_len)) - lx.pos += sizet(best_len) - lx.col += best_len - _emit(lx, tok) - - -# ============================================================ -# Main Lexical Analysis Entry Point -# ============================================================ - -def tokenize(lx: Lexer*) -> Token*: - """Execute tokenization, return token list head""" - if lx == None: return None - while lx.pos < lx.len: - # Handle line start indent - if lx.at_line_start and lx.paren_depth == 0: - _handle_indent(lx) - if lx.pos >= lx.len: - break - - ch: char = _peek_cur(lx) - if ch == '\0': - break - - # New line - if ch == '\n': - if lx.paren_depth > 0: - # # In bi brackets, generates TokenType.Nl - tok: Token* = new_token(lx.pool, TokenType.Nl, lx.lineno, lx.col) - _emit(lx, tok) - else: - # Logical new line - tok: Token* = new_token(lx.pool, TokenType.NewLine, lx.lineno, lx.col) - _emit(lx, tok) - lx.at_line_start = 1 - lx.pos += 1 - lx.lineno += 1 - lx.col = 0 - continue - - # Backslash continuation line - # Support LF and CRLF line endings: under CRLF `\` is followed by '\r' (not '\n'), - # so checking only '\n' fails and the backslash continuation breaks. - if ch == '\\' and lx.pos + 1 < lx.len: - if lx.src[lx.pos + 1] == '\n': - lx.pos += 2 - lx.lineno += 1 - lx.col = 0 - continue - if lx.src[lx.pos + 1] == '\r': - # \ + \r\n: skip \, \r, \n - lx.pos += 2 - if lx.pos < lx.len and lx.src[lx.pos] == '\n': - lx.pos += 1 - lx.lineno += 1 - lx.col = 0 - continue - - # Whitespace - if _is_space(ch): - lx.pos += 1 - lx.col += 1 - continue - - # Comment - if ch == '#': - while lx.pos < lx.len and lx.src[lx.pos] != '\n': - lx.pos += 1 - continue - - # String prefixes (like r"...") or strings - if ch == '\'' or ch == '"': - _read_string(lx) - continue - # A string prefix letter followed by a quote (must be checked before _is_alpha, otherwise f/r/b/u will be treated as TokenType.Name) - if (ch == 'r' or ch == 'R' or ch == 'b' or ch == 'B' or - ch == 'u' or ch == 'U' or ch == 'f' or ch == 'F'): - if lx.pos + 1 < lx.len: - ch1: char = lx.src[lx.pos + 1] - if ch1 == '\'' or ch1 == '"': - _read_string(lx) - continue - # Double prefixes like rb, br, rf, fr - if lx.pos + 2 < lx.len: - ch2: char = lx.src[lx.pos + 2] - if ((ch1 == 'r' or ch1 == 'R' or ch1 == 'b' or ch1 == 'B' or - ch1 == 'f' or ch1 == 'F' or ch1 == 'u' or ch1 == 'U') and - (ch2 == '\'' or ch2 == '"')): - _read_string(lx) - continue - - # Identifier/keyword - if _is_alpha(ch): - _read_name(lx) - continue - - # Number - if _is_digit(ch): - _read_number(lx) - continue - - # Operator - _read_operator(lx) - - # End of file: generate TokenType.NewLine (if there's content) - if lx.tokens_tail != None: - if lx.tokens_tail.type != TokenType.NewLine and lx.tokens_tail.type != TokenType.Nl: - tok: Token* = new_token(lx.pool, TokenType.NewLine, lx.lineno, lx.col) - _emit(lx, tok) - - # Generate remaining remaining TokenType.Dedent - while lx.indent_top > 0: - lx.indent_top -= 1 - tok: Token* = new_token(lx.pool, TokenType.Dedent, lx.lineno, 0) - _emit(lx, tok) - - # TokenType.EndMarker - tok: Token* = new_token(lx.pool, TokenType.EndMarker, lx.lineno, 0) - _emit(lx, tok) - - return lx.tokens_head +from stdint import * +import memhub +import string +import stdio +import stdlib +from .tokens import ( + Token, TokenType, TokOp, _init_tables, _kw_lookup, _op_head, + OpEntry, new_token, token_set_str, token_set_str_literal, +) + + +# ============================================================ +# Integer literal overflow (lexer-layer fail-fast) +# +# Integer constants are int64 in the token/AST (tokens.vp `int_val` / exprs.vp `int_val`). +# The old implementation used `string.atoll` (string.vp:292) to, under int64, do `num = num * 10 + digit` +# digit-by-digit accumulation => when the value > 2^64-1 it **silently wraps** (e.g. 2^64 wraps to 0, 2^64-1 wraps to -1). +# This implementation, in `_emit_number`, first performs a pure-text magnitude check on **integer** literals +# (`_lex_int_exceeds_u64`, using the same criteria as AssignGenMixin._int_text_exceeds_64: by +# radix, counting significant digits / comparing length + lexicographic order against the decimal text of 2^64-1, **without landing in any fixed-width +# container and without wrapping before the comparison**), so exceeding 2^64-1 records a fail-fast (no more silent wrapping). +# The legal range 0..2^64-1 is still stored into int64 as a 64-bit bit pattern (the shape of the results on the existing legal path is unchanged, +# e.g. 0xFFFFFFFFFFFFFFFF -> int64 -1, 0xA5A5A5A5A5A5A5A5 keeps its 64-bit bit pattern). +# The criteria for each radix (2/8/10/16) match TPC's (Python int(x, 0), arbitrary precision): the accepted interval is likewise +# [-2^63, 2^64-1] (literals themselves are unsigned; the negative side is expressed by unary minus), and anything beyond is rejected. +# +# Reporting channel: the lexer has no separate fatal exit, so it records into a module-level triple (`_lex_record_overflow`), +# which the parser's parse_tokens reads and transcribes into _g_parse_err_* (position + message), reusing the existing +# Traceback/SyntaxError report path (the "file:line:col + source line" is filled in by that path, which also terminates compilation). +# This covers all source literals (file-level parsing necessarily goes through parse_tokens); parse_expression is used only for compiler- +# synthesized short type strings and carries no source literals, so no separate channel is provided. +# ============================================================ +_g_lex_err_line: int = 0 +_g_lex_err_col: int = 0 +_g_lex_err_msg: str = None + + +def get_lex_err_line() -> int: + """Line number of the latest lexer fail-fast (0 = none). + + Consumed by parser.parse_tokens and forwarded to the parse error channel. + """ + return _g_lex_err_line + + +def get_lex_err_col() -> int: + """Column of the latest lexer fail-fast (0-based; -1 = no column info)""" + return _g_lex_err_col + + +def get_lex_err_msg() -> str: + """Message of the latest lexer fail-fast (literal / required width / limit / fix hint)""" + return _g_lex_err_msg + + +def _lex_reset_err(): + """Clear the lexer fail-fast triple (called at each tokenize entry to avoid stale errors)""" + global _g_lex_err_line, _g_lex_err_col, _g_lex_err_msg + _g_lex_err_line = 0 + _g_lex_err_col = 0 + _g_lex_err_msg = None + + +def _lex_record_overflow(text: str, lineno: int, col: int): + """Record an integer literal overflow (fail-fast; first record only); message carries literal + + required width (>= 65 bits, limit 2^64-1) + fix hint (file:line:col filled later by parse_tokens). + """ + global _g_lex_err_line, _g_lex_err_col, _g_lex_err_msg + if _g_lex_err_line != 0: + return + _g_lex_err_line = lineno + _g_lex_err_col = col + msg_buf: str = stdlib.malloc(512) + if msg_buf is not None: + stdio.snprintf(msg_buf, 512, + "[VPC] integer literal overflow: literal '%s' exceeds the 64-bit representable range" + " (the bit-pattern interpretation upper bound is 18446744073709551615 = 2^64-1," + " the literal needs a width of at least 65 bits)." + " Silent wrap-around/truncation is forbidden; compilation aborted." + " Fix: change the literal into the 64-bit bit-pattern range 0..18446744073709551615;" + " this language has no wider integer type, for larger values split into multiple 64-bit words.", text) + _g_lex_err_msg = msg_buf + else: + _g_lex_err_msg = text + + +def _lex_int_exceeds_u64(text: str) -> int: + """True if the integer literal magnitude exceeds the 64-bit limit 2^64-1 (1 = yes, 0 = no). + Pure text compare, no wrap-around: decimal length+lexicographic vs "18446744073709551615", + 0x > 16, 0o > 22 (or 22 leading > '1'), 0b > 64 (literal text carries no sign). + """ + if text is None: + return 0 + tl: sizet = string.strlen(text) + if tl == 0: + return 0 + # Base detection (prefix letter case-insensitive) + base: int = 10 + start: sizet = 0 + if tl >= 2 and text[0] == '0': + pfx: char = text[1] + if pfx == 'x' or pfx == 'X': + base = 16 + start = 2 + elif pfx == 'o' or pfx == 'O': + base = 8 + start = 2 + elif pfx == 'b' or pfx == 'B': + base = 2 + start = 2 + # Count significant digits (skip '_' and leading zeros) and remember the first one + sig: int = 0 + first: char = '0' + j: sizet = start + while j < tl: + ch: char = text[j] + j += 1 + if ch == '_': + continue + ok_digit: int = 0 + if base == 16: + if (ch >= '0' and ch <= '9') or (ch >= 'a' and ch <= 'f') or (ch >= 'A' and ch <= 'F'): + ok_digit = 1 + elif base == 10: + if ch >= '0' and ch <= '9': + ok_digit = 1 + elif base == 8: + if ch >= '0' and ch <= '7': + ok_digit = 1 + else: + if ch == '0' or ch == '1': + ok_digit = 1 + if ok_digit == 0: + break + if sig == 0: + if ch == '0': + continue # leading zeros not counted + first = ch + sig += 1 + if sig == 0: + return 0 # value 0 + if base == 16: + return 1 if sig > 16 else 0 + if base == 2: + return 1 if sig > 64 else 0 + if base == 8: + if sig > 22: + return 1 + if sig == 22 and first > '1': + return 1 + return 0 + # Decimal: compare significant-digit count against the 2^64-1 text length + mlen: sizet = 20 + if sig > mlen: + return 1 + if sig < mlen: + return 0 + # Equal length (20 digits): lexicographic compare against '18446744073709551615' + max_text: str = "18446744073709551615" + d: sizet = 0 + k: sizet = start + lead: int = 1 + while k < tl and d < mlen: + c: char = text[k] + k += 1 + if c == '_': + continue + if c < '0' or c > '9': + return 0 + if lead != 0 and c == '0': + continue + lead = 0 + mc: char = max_text[d] + if c > mc: + return 1 + if c < mc: + return 0 + d += 1 + return 0 + + + +def _lex_mag_digit_val(ch: char, base: int) -> int: + """Digit char -> positional value (hex accepts a-f/A-F; other bases are '0'..base-1).""" + if base == 16: + return _hex_val(ch) + return int(ch) - 48 + + +def _lex_mag_gt_text(text: str, base: int, start: sizet, sig: int, max_text: str) -> int: + """True if the literal's significant digits exceed the same-length max text (1 = yes). + Both sides skip '_'/leading zeros, compared by positional value (hex via _hex_val); pure text. + """ + d: sizet = 0 + k: sizet = start + tl: sizet = string.strlen(text) + lead: int = 1 + while k < tl and d < sizet(sig): + c: char = text[k] + k += 1 + if c == '_': + continue + if base == 16: + if not ((c >= '0' and c <= '9') or (c >= 'a' and c <= 'f') or (c >= 'A' and c <= 'F')): + break + elif base == 10: + if c < '0' or c > '9': + break + elif base == 8: + if c < '0' or c > '7': + break + else: + if c != '0' and c != '1': + break + if lead != 0 and c == '0': + continue + lead = 0 + cv: int = _lex_mag_digit_val(c, base) + mv: int = _lex_mag_digit_val(max_text[d], base) + if cv > mv: + return 1 + if cv < mv: + return 0 + d += 1 + return 0 + + +def _lex_int_neg_exceeds_i64(text: str) -> int: + """True if the magnitude is strictly greater than 2^63 (1 = yes). + For unary-minus + literal overflow: a magnitude-valid literal may become < -2^63 after + negation; exactly 2^63 is legal (INT64_MIN). Pure text compare, no wrap-around. + """ + if text is None: + return 0 + tl: sizet = string.strlen(text) + if tl == 0: + return 0 + base: int = 10 + start: sizet = 0 + if tl >= 2 and text[0] == '0': + pfx: char = text[1] + if pfx == 'x' or pfx == 'X': + base = 16 + start = 2 + elif pfx == 'o' or pfx == 'O': + base = 8 + start = 2 + elif pfx == 'b' or pfx == 'B': + base = 2 + start = 2 + sig: int = 0 + j: sizet = start + while j < tl: + ch: char = text[j] + j += 1 + if ch == '_': + continue + ok_digit: int = 0 + if base == 16: + if (ch >= '0' and ch <= '9') or (ch >= 'a' and ch <= 'f') or (ch >= 'A' and ch <= 'F'): + ok_digit = 1 + elif base == 10: + if ch >= '0' and ch <= '9': + ok_digit = 1 + elif base == 8: + if ch >= '0' and ch <= '7': + ok_digit = 1 + else: + if ch == '0' or ch == '1': + ok_digit = 1 + if ok_digit == 0: + break + if sig == 0 and ch == '0': + continue # leading zeros not counted + sig += 1 + if sig == 0: + return 0 # value 0 + if base == 16: + if sig > 16: return 1 + if sig < 16: return 0 + return _lex_mag_gt_text(text, 16, start, sig, "8000000000000000") + if base == 2: + if sig > 64: return 1 + if sig < 64: return 0 + return _lex_mag_gt_text(text, 2, start, sig, + "1000000000000000000000000000000000000000000000000000000000000000") + if base == 8: + if sig > 22: return 1 + if sig < 22: return 0 + return _lex_mag_gt_text(text, 8, start, sig, "1000000000000000000000") + if sig > 19: return 1 + if sig < 19: return 0 + return _lex_mag_gt_text(text, 10, start, sig, "9223372036854775808") + + +# Maximum Indentation Stack Depth +define LEXER_MAX_INDENT = 256 + + +# ============================================================ +# Lexer State Structure +# ============================================================ +class Lexer: + src: str # Source code (NUL-terminated) + pos: sizet # Current position + len: sizet # Source code length + lineno: int # Current line number (1-based) + col: int # Current column number (0-based) + pool: memhub.MemManager* # Memory pool + indent_stack: int* # Indent stack (int array, allocated from pool) + indent_top: int # Stack top (-1 for empty stack) + paren_depth: int # Paren depth (>0 for no TokenType.NewLine) + tokens_head: Token* # token list head + tokens_tail: Token* # token list tail + at_line_start: int # 1=line start (need indent) + error_count: int # Error count + + def __new__(self, pool: memhub.MemManager*): + ptr: Lexer* = pool.alloc(Lexer.__sizeof__()) + if ptr == None: + # Pool exhausted: fail-fast instead of returning NULL (downstream would treat NULL as a valid Lexer) + stdio.printf("[AST] Lexer allocation failed: memory pool exhausted (need=%zu bytes)\n", + sizet(Lexer.__sizeof__())) + stdio.fflush(None) + stdlib.exit(1) + string.memset(ptr, 0, Lexer.__sizeof__()) + return ptr + + +def new_lexer(pool: memhub.MemManager*) -> Lexer*: + """Allocate from the pool and return a Lexer (cross-module safe wrapper)""" + return Lexer(pool) + + +def _lexer_init(lx: Lexer*, src: str, pool: memhub.MemManager*): + """Initialize Lexer""" + if lx == None: return + lx.src = src + lx.pos = 0 + lx.len = string.strlen(src) if src != None else 0 + lx.lineno = 1 + lx.col = 0 + lx.pool = pool + # Allocate indentation stack + lx.indent_stack = pool.alloc(LEXER_MAX_INDENT * 8) + if lx.indent_stack == None: + stdio.printf("[AST] Lexer.indent_stack allocation failed: memory pool exhausted (need=%zu bytes)\n", + sizet(LEXER_MAX_INDENT * 8)) + stdio.fflush(None) + stdlib.exit(1) + string.memset(lx.indent_stack, 0, LEXER_MAX_INDENT * 8) + lx.indent_top = -1 + # Push 0 (base indent) + lx.indent_stack[0] = 0 + lx.indent_top = 0 + lx.paren_depth = 0 + lx.tokens_head = None + lx.tokens_tail = None + lx.at_line_start = 1 + lx.error_count = 0 + _init_tables(pool) + + +# ============================================================ +# Auxiliary Functions +# ============================================================ + +def _peek(lx: Lexer*, offset: sizet) -> char: + """Peek character at position pos+offset, return '\0' if out of bounds""" + p: sizet = lx.pos + offset + if p >= lx.len: return '\0' + return lx.src[p] + + +def _peek_cur(lx: Lexer*) -> char: + if lx.pos >= lx.len: return '\0' + return lx.src[lx.pos] + + +def _advance(lx: Lexer*) -> char: + """Advance one character, return the character""" + if lx.pos >= lx.len: return '\0' + ch: char = lx.src[lx.pos] + lx.pos += 1 + if ch == '\n': + lx.lineno += 1 + lx.col = 0 + else: + lx.col += 1 + return ch + + +def _is_alpha(ch: char) -> int: + if ch >= 'a' and ch <= 'z': return 1 + if ch >= 'A' and ch <= 'Z': return 1 + if ch == '_': return 1 + return 0 + + +def _is_digit(ch: char) -> int: + if ch >= '0' and ch <= '9': return 1 + return 0 + + +def _is_alnum(ch: char) -> int: + if _is_alpha(ch): return 1 + if _is_digit(ch): return 1 + return 0 + + +def _is_hex(ch: char) -> int: + if ch >= '0' and ch <= '9': return 1 + if ch >= 'a' and ch <= 'f': return 1 + if ch >= 'A' and ch <= 'F': return 1 + return 0 + + +def _hex_val(ch: char) -> int: + """Hex char -> value 0-15 (for \xNN escape parsing)""" + if ch >= '0' and ch <= '9': return int(ch) - 48 + if ch >= 'a' and ch <= 'f': return int(ch) - 87 + if ch >= 'A' and ch <= 'F': return int(ch) - 55 + return 0 + + +def _is_space(ch: char) -> int: + if ch == ' ': return 1 + if ch == '\t': return 1 + if ch == '\f': return 1 + if ch == '\v': return 1 + if ch == '\r': return 1 + return 0 + + +def _emit(lx: Lexer*, tok: Token*): + """Append token to token list""" + if tok == None: return + if lx.tokens_head == None: + lx.tokens_head = tok + lx.tokens_tail = tok + else: + lx.tokens_tail.next = tok + lx.tokens_tail = tok + + +def _make_op_token(lx: Lexer*, op_id: int, length: int, + lineno: int, col: int) -> Token*: + tok: Token* = new_token(lx.pool, TokenType.Op, lineno, col) + if tok == None: return None + tok.op_subtype = op_id + tok.end_col_offset = col + length + return tok + + +# ============================================================ +# Indent Handling +# ============================================================ + +def _handle_indent(lx: Lexer*): + """Handle line start indent, generate TokenType.Indent/TokenType.Dedent. + Skip empty lines and comment lines.""" + # Skip empty lines and comment lines + while lx.pos < lx.len: + # Save line start position for non-empty lines + line_start_pos: sizet = lx.pos + line_start_col: int = lx.col + # Skip whitespace, not across lines + while lx.pos < lx.len: + ch: char = lx.src[lx.pos] + if ch == ' ' or ch == '\t' or ch == '\f' or ch == '\v': + lx.pos += 1 + lx.col += 1 + elif ch == '\r': + # CRLF line end \r, skip \n to trigger empty line processing + lx.pos += 1 + else: + break + if lx.pos >= lx.len: + break + ch = lx.src[lx.pos] + if ch == '\n': + # Empty line, skip + lx.pos += 1 + lx.lineno += 1 + lx.col = 0 + continue + if ch == '\r': + # CRLF line end \r, skip \n to trigger empty line processing + lx.pos += 1 + if lx.pos < lx.len and lx.src[lx.pos] == '\n': + lx.pos += 1 + lx.lineno += 1 + lx.col = 0 + continue + if ch == '#': + # Comment line, skip to end of line + while lx.pos < lx.len and lx.src[lx.pos] != '\n': + lx.pos += 1 + continue + # Non-empty line: backtrack to line start to allow indent calculation + lx.pos = line_start_pos + lx.col = line_start_col + break + + if lx.pos >= lx.len: + # End of file, no indent to handle + lx.at_line_start = 0 + return + + # Calculate current line indent + indent: int = 0 + save_pos: sizet = lx.pos + save_col: int = lx.col + while lx.pos < lx.len: + ch: char = lx.src[lx.pos] + if ch == ' ': + indent += 1 + lx.pos += 1 + lx.col += 1 + elif ch == '\t': + indent += 8 - (indent % 8) + lx.pos += 1 + lx.col += 1 + else: + break + + # If line end (empty line), skip indent calculation + if lx.pos < lx.len: + ch = lx.src[lx.pos] + if ch == '\n' or ch == '\r' or ch == '#': + # Empty line or comment line, backtrack position to allow loop to handle indent + lx.pos = save_pos + lx.col = save_col + lx.at_line_start = 0 + return + + # Compare indent with top of indent stack + top_indent: int = lx.indent_stack[lx.indent_top] + if indent > top_indent: + # TokenType.Indent + if lx.indent_top + 1 >= LEXER_MAX_INDENT: + lx.error_count += 1 + return + lx.indent_top += 1 + lx.indent_stack[lx.indent_top] = indent + tok: Token* = new_token(lx.pool, TokenType.Indent, lx.lineno, 0) + _emit(lx, tok) + else: + while indent < lx.indent_stack[lx.indent_top] and lx.indent_top > 0: + lx.indent_top -= 1 + tok: Token* = new_token(lx.pool, TokenType.Dedent, lx.lineno, 0) + _emit(lx, tok) + lx.at_line_start = 0 + + +# ============================================================ +# Identifier/Keyword Handling +# ============================================================ + +def _read_name(lx: Lexer*): + start: sizet = lx.pos + start_lineno: int = lx.lineno + start_col: int = lx.col + while lx.pos < lx.len: + ch: char = lx.src[lx.pos] + if _is_alnum(ch): + lx.pos += 1 + lx.col += 1 + else: + break + length: sizet = lx.pos - start + tok: Token* = new_token(lx.pool, TokenType.Name, start_lineno, start_col) + if tok == None: return + token_set_str(lx.pool, tok, lx.src, start, length) + tok.end_lineno = lx.lineno + tok.end_col_offset = start_col + int(length) + # Look up keyword if any + kw_id: int = _kw_lookup(tok.str_val) + if kw_id: tok.kw_subtype = kw_id + _emit(lx, tok) + + +# ============================================================ +# Number Handling +# ============================================================ + +def _read_number(lx: Lexer*): + start: sizet = lx.pos + start_lineno: int = lx.lineno + start_col: int = lx.col + is_float: int = 0 + is_complex: int = 0 + + ch0: char = lx.src[lx.pos] + # Handle 0x/0o/0b prefix + if ch0 == '0' and lx.pos + 1 < lx.len: + ch1: char = lx.src[lx.pos + 1] + if ch1 == 'x' or ch1 == 'X': + lx.pos += 2 + lx.col += 2 + while lx.pos < lx.len: + ch: char = lx.src[lx.pos] + if _is_hex(ch) or ch == '_': + lx.pos += 1 + lx.col += 1 + else: + break + _emit_number(lx, start, start_lineno, start_col, 0, 0, 0) + return + if ch1 == 'o' or ch1 == 'O': + lx.pos += 2 + lx.col += 2 + while lx.pos < lx.len: + ch: char = lx.src[lx.pos] + if ch >= '0' and ch <= '7': + lx.pos += 1 + lx.col += 1 + elif ch == '_': + lx.pos += 1 + lx.col += 1 + else: + break + _emit_number(lx, start, start_lineno, start_col, 0, 0, 0) + return + if ch1 == 'b' or ch1 == 'B': + lx.pos += 2 + lx.col += 2 + while lx.pos < lx.len: + ch: char = lx.src[lx.pos] + if ch == '0' or ch == '1' or ch == '_': + lx.pos += 1 + lx.col += 1 + else: + break + _emit_number(lx, start, start_lineno, start_col, 0, 0, 0) + return + + # Decimal integer/float + while lx.pos < lx.len: + ch: char = lx.src[lx.pos] + if _is_digit(ch) or ch == '_': + lx.pos += 1 + lx.col += 1 + else: + break + + # Decimal part + if lx.pos < lx.len and lx.src[lx.pos] == '.': + is_float = 1 + lx.pos += 1 + lx.col += 1 + while lx.pos < lx.len: + ch: char = lx.src[lx.pos] + if _is_digit(ch) or ch == '_': + lx.pos += 1 + lx.col += 1 + else: + break + + # Exponent part + if lx.pos < lx.len: + ch: char = lx.src[lx.pos] + if ch == 'e' or ch == 'E': + is_float = 1 + lx.pos += 1 + lx.col += 1 + if lx.pos < lx.len: + ch = lx.src[lx.pos] + if ch == '+' or ch == '-': + lx.pos += 1 + lx.col += 1 + while lx.pos < lx.len: + ch: char = lx.src[lx.pos] + if _is_digit(ch) or ch == '_': + lx.pos += 1 + lx.col += 1 + else: + break + + # Complex suffix part + if lx.pos < lx.len and lx.src[lx.pos] == 'j': + is_complex = 1 + is_float = 1 + lx.pos += 1 + lx.col += 1 + + _emit_number(lx, start, start_lineno, start_col, is_float, is_complex, 0) + + +# _suffix_is - 源码位置 p 处是否以 s 开头（长度 n，带边界检查） +def _suffix_is(lx: Lexer*, p: sizet, s: str, n: int) -> int: + """Bound-checked prefix compare of lx.src at p against s (n chars); 1=match / 0=no.""" + if p + sizet(n) > lx.len: return 0 + return 1 if string.strncmp(lx.src + p, s, n) == 0 else 0 + + +# _read_width_suffix - 解析整型字面量的显式宽度后缀（_int/_uint/_i/_u）。 +# 命中则推进 lx.pos/col，返回 +N（有符号）/ -N（无符号）；未命中返回 0 且不动位置。 +# N 取 1..64（字面量值本身是 64 位）；后缀之后不得直接跟标识符字符。 +def _read_width_suffix(lx: Lexer*) -> int: + """Explicit integer width suffix: +N signed / -N unsigned / 0 none (advances on hit).""" + if lx.pos >= lx.len: return 0 + if lx.src[lx.pos] != '_': return 0 + p: sizet = lx.pos + 1 + is_u: int = 0 + if _suffix_is(lx, p, "uint", 4) != 0: + p += 4 + is_u = 1 + elif _suffix_is(lx, p, "int", 3) != 0: + p += 3 + elif _suffix_is(lx, p, "u", 1) != 0: + p += 1 + is_u = 1 + elif _suffix_is(lx, p, "i", 1) != 0: + p += 1 + else: + return 0 + nbits: int = 0 + ndig: int = 0 + while p < lx.len: + ch: char = lx.src[p] + if ch < '0' or ch > '9': break + nbits = nbits * 10 + (int(ch) - 48) + ndig += 1 + p += 1 + if ndig == 0 or nbits <= 0 or nbits > 64: return 0 + if p < lx.len: + ch2: char = lx.src[p] + if (ch2 >= '0' and ch2 <= '9') or (ch2 >= 'a' and ch2 <= 'z') \ + or (ch2 >= 'A' and ch2 <= 'Z') or ch2 == '_': + return 0 + adv: sizet = p - lx.pos + lx.pos = p + lx.col += int(adv) + if is_u != 0: return -nbits + return nbits + + +def _emit_number(lx: Lexer*, start: sizet, + start_lineno: int, start_col: int, + is_float: int, is_complex: int, is_hex: int): + # 显式宽度后缀（仅整型字面量）：_int/_uint/_i/_u + int_w: int = 0 + int_u: int = 0 + if is_float == 0 and is_complex == 0: + sw: int = _read_width_suffix(lx) + if sw > 0: int_w = sw + elif sw < 0: + int_w = -sw + int_u = 1 + length: sizet = lx.pos - start + tok: Token* = new_token(lx.pool, TokenType.Number, start_lineno, start_col) + if tok == None: return + token_set_str(lx.pool, tok, lx.src, start, length) + tok.is_float = is_float + tok.is_complex = is_complex + tok.int_width = int_w + tok.int_unsigned = int_u + tok.end_lineno = lx.lineno + tok.end_col_offset = start_col + int(length) + # Parse number + if is_float: + tok.float_val = string.atof(tok.str_val) + else: + # Integer literal magnitude > 2^64-1: lexer fail-fast (pure text, no silent wrap). + # Range 0..2^64-1 is still stored into int64 as a 64-bit bit pattern (unchanged shape). + if _lex_int_exceeds_u64(tok.str_val) != 0: + _lex_record_overflow(tok.str_val, start_lineno, start_col) + tok.int_val = string.atoll(tok.str_val) + _emit(lx, tok) + + +# ============================================================ +# String Handling +# ============================================================ + +def _read_string(lx: Lexer*): + """Read string literal. + Supports prefixes: r, b, u, f, rb, br, rf, fr (case insensitive) + Supports quotes: ', ", ''', + Later we need to support string prefixes like i16/i24""" + start_lineno: int = lx.lineno + start_col: int = lx.col + prefix_start: sizet = lx.pos + + # Read prefix (r/b/u/f) + is_raw: int = 0 + is_bytes: int = 0 + is_fstring: int = 0 + while lx.pos < lx.len: + ch: char = lx.src[lx.pos] + if ch == 'r' or ch == 'R': + is_raw = 1 + lx.pos += 1 + lx.col += 1 + elif ch == 'b' or ch == 'B': + is_bytes = 1 + lx.pos += 1 + lx.col += 1 + elif ch == 'u' or ch == 'U': + lx.pos += 1 + lx.col += 1 + elif ch == 'f' or ch == 'F': + is_fstring = 1 + lx.pos += 1 + lx.col += 1 + else: + break + + if lx.pos >= lx.len: + lx.error_count += 1 + return + + # Determine quote type + quote: char = lx.src[lx.pos] + if quote != '\'' and quote != '"': + lx.error_count += 1 + return + + # Check triple quote + is_triple: int = 0 + if lx.pos + 2 < lx.len: + if lx.src[lx.pos] == quote and lx.src[lx.pos + 1] == quote and lx.src[lx.pos + 2] == quote: + is_triple = 1 + + content_start: sizet = lx.pos + if is_triple: + lx.pos += 3 + lx.col += 3 + else: + lx.pos += 1 + lx.col += 1 + + # Read string content + content_buf_start: sizet = lx.pos + # f-string {} expression block nesting depth (for PEP 701 same-quote nested strings) + fstr_depth: int = 0 + while lx.pos < lx.len: + ch = lx.src[lx.pos] + if is_triple: + if ch == quote: + if lx.pos + 2 < lx.len: + if lx.src[lx.pos + 1] == quote and lx.src[lx.pos + 2] == quote: + break + elif lx.pos + 2 == lx.len: + break + if ch == '\\': + lx.pos += 1 + lx.col += 1 + if lx.pos < lx.len: + if lx.src[lx.pos] == '\n': + lx.lineno += 1 + lx.col = 0 + lx.pos += 1 + lx.col += 1 + continue + if ch == '\n': + lx.lineno += 1 + lx.col = 0 + lx.pos += 1 + lx.col += 1 + else: + if is_fstring: + # {{ and }} are escaped literals, not counted in depth + if ch == '{' and lx.pos + 1 < lx.len and lx.src[lx.pos + 1] == '{': + lx.pos += 2 + lx.col += 2 + continue + if ch == '}' and lx.pos + 1 < lx.len and lx.src[lx.pos + 1] == '}': + lx.pos += 2 + lx.col += 2 + continue + if ch == '{': + fstr_depth += 1 + lx.pos += 1 + lx.col += 1 + continue + if ch == '}': + if fstr_depth > 0: + fstr_depth -= 1 + lx.pos += 1 + lx.col += 1 + continue + if fstr_depth > 0 and (ch == '\'' or ch == '"'): + # Nested string literals allowed inside {} blocks (PEP 701 same-quote nesting): + # skip to the matching quote; inner quotes do not terminate the outer f-string. + nq: char = ch + lx.pos += 1 + lx.col += 1 + while lx.pos < lx.len: + c2: char = lx.src[lx.pos] + if c2 == '\\': + lx.pos += 2 + lx.col += 2 + continue + if c2 == nq: + lx.pos += 1 + lx.col += 1 + break + if c2 == '\n': + # single-line string cannot span lines + lx.error_count += 1 + return + lx.pos += 1 + lx.col += 1 + continue + if ch == quote: + break + if ch == '\n': + # Single quote string cannot span multiple lines (unless continued) + lx.error_count += 1 + return + if ch == '\\' and not is_raw: + lx.pos += 1 + lx.col += 1 + if lx.pos < lx.len: + lx.pos += 1 + lx.col += 1 + continue + lx.pos += 1 + lx.col += 1 + + content_end: sizet = lx.pos + # Skip the closing quote + if is_triple: + if lx.pos + 2 < lx.len: + lx.pos += 3 + lx.col += 3 + else: + lx.pos = lx.len + else: + if lx.pos < lx.len: + lx.pos += 1 + lx.col += 1 + + # Create token + tok: Token* = new_token(lx.pool, TokenType.String, start_lineno, start_col) + if tok == None: return + content_len: sizet = content_end - content_buf_start + # esc_len: Actual bytes after escaping (used to check single character) + esc_len: sizet = content_len + + # Handle escape sequences (non-raw strings) + if is_raw == 0 and content_len > 0: + # Allocate conversion buffer (max case: each char is plain, no extension) + conv_buf: str = lx.pool.alloc(content_len + 1) + if conv_buf == None: + token_set_str(lx.pool, tok, lx.src, content_buf_start, content_len) + else: + wpos: sizet = 0 + rpos: sizet = 0 + while rpos < content_len: + ch: char = lx.src[content_buf_start + rpos] + if ch == '\\' and rpos + 1 < content_len: + rpos += 1 + next_ch: char = lx.src[content_buf_start + rpos] + if next_ch == 'n': + conv_buf[wpos] = '\n' + elif next_ch == 't': + conv_buf[wpos] = '\t' + elif next_ch == 'r': + conv_buf[wpos] = '\r' + elif next_ch == 'f': + conv_buf[wpos] = '\f' + elif next_ch == 'v': + conv_buf[wpos] = '\v' + elif next_ch == 'a': + conv_buf[wpos] = '\a' + elif next_ch == 'b': + conv_buf[wpos] = '\b' + elif next_ch == '\\': + conv_buf[wpos] = '\\' + elif next_ch == '\'': + conv_buf[wpos] = '\'' + elif next_ch == '"': + conv_buf[wpos] = '"' + elif next_ch == '0': + conv_buf[wpos] = '\0' + elif next_ch == 'x' and rpos + 2 < content_len: + # \xNN hex escape (matches Python): '\\x1b' -> single byte 0x1B. + # Must yield esc_len == 1 so '\xNN' becomes CONST_CHAR, not CONST_STR. + h1: char = lx.src[content_buf_start + rpos + 1] + h2: char = lx.src[content_buf_start + rpos + 2] + if _is_hex(h1) != 0 and _is_hex(h2) != 0: + hex_v: int = (_hex_val(h1) << 4) | _hex_val(h2) + conv_buf[wpos] = char(hex_v) + wpos += 1 + rpos += 3 + continue + # Invalid hex (\x followed by fewer than 2 hex digits): keep as-is + conv_buf[wpos] = '\\' + wpos += 1 + conv_buf[wpos] = next_ch + else: + # Unknown escape sequence, keep original (includes \xNN) + conv_buf[wpos] = '\\' + wpos += 1 + conv_buf[wpos] = next_ch + wpos += 1 + rpos += 1 + else: + conv_buf[wpos] = ch + wpos += 1 + rpos += 1 + conv_buf[wpos] = '\0' + token_set_str_literal(lx.pool, tok, conv_buf) + esc_len = wpos + else: + token_set_str(lx.pool, tok, lx.src, content_buf_start, content_len) + tok.end_lineno = lx.lineno + tok.end_col_offset = lx.col + # flags field: bit0=raw, bit1=bytes, bit2=fstring, bit3=triple, bit4=dquote, bit5=single char candidate + if is_raw: tok.kw_subtype = tok.kw_subtype | 1 + if is_bytes: tok.kw_subtype = tok.kw_subtype | 2 + if is_fstring: tok.kw_subtype = tok.kw_subtype | 4 + if is_triple: tok.kw_subtype = tok.kw_subtype | 8 + # bit4 (16) = double quote=single quote + if quote == '"': tok.kw_subtype = tok.kw_subtype | 16 + # bit5 (32) = single-character candidate (not triple quotes, not f-string, exactly 1 byte after escaping) + # Used for CONST_CHAR checks to prevent '0' and other NUL bytes from being mistaken as empty strings by strlen + if is_triple == 0 and is_fstring == 0 and esc_len == 1: + tok.kw_subtype = tok.kw_subtype | 32 + _emit(lx, tok) + + +# ============================================================ +# Operator Matching +# ============================================================ + +def _read_operator(lx: Lexer*): + """Greedy match operator""" + start_lineno: int = lx.lineno + start_col: int = lx.col + remaining: sizet = lx.len - lx.pos + + # Traverse operator table, find longest match + best_entry: OpEntry* = None + best_len: int = 0 + cur: OpEntry* = _op_head + while cur != None: + if cur.length <= int(remaining): + # Compare cur.name with src[pos:pos+cur.length] + Matched: int = 1 + for i in range(int(sizet(cur.length))): + if lx.src[lx.pos + i] != cur.name[i]: + Matched = 0 + break + if Matched != 0: + if cur.length > best_len: + best_len = cur.length + best_entry = cur + cur = cur.next + + if best_entry == None: + # Unknown character, report error and skip + lx.error_count += 1 + lx.pos += 1 + lx.col += 1 + return + + # Update parenthesis depth + if best_entry.op_id == TokOp.LPar or best_entry.op_id == TokOp.Lsqb or best_entry.op_id == TokOp.LBrace: + lx.paren_depth += 1 + elif best_entry.op_id == TokOp.RPar or best_entry.op_id == TokOp.Rsqb or best_entry.op_id == TokOp.RBrace: + if lx.paren_depth > 0: + lx.paren_depth -= 1 + + tok: Token* = _make_op_token(lx, best_entry.op_id, best_entry.length, + start_lineno, start_col) + if tok == None: return + token_set_str(lx.pool, tok, lx.src, lx.pos, sizet(best_len)) + lx.pos += sizet(best_len) + lx.col += best_len + _emit(lx, tok) + + +# ============================================================ +# Main Lexical Analysis Entry Point +# ============================================================ + +def tokenize(lx: Lexer*) -> Token*: + """Execute tokenization, return token list head""" + if lx == None: return None + while lx.pos < lx.len: + # Handle line start indent + if lx.at_line_start and lx.paren_depth == 0: + _handle_indent(lx) + if lx.pos >= lx.len: + break + + ch: char = _peek_cur(lx) + if ch == '\0': + break + + # New line + if ch == '\n': + if lx.paren_depth > 0: + # # In bi brackets, generates TokenType.Nl + tok: Token* = new_token(lx.pool, TokenType.Nl, lx.lineno, lx.col) + _emit(lx, tok) + else: + # Logical new line + tok: Token* = new_token(lx.pool, TokenType.NewLine, lx.lineno, lx.col) + _emit(lx, tok) + lx.at_line_start = 1 + lx.pos += 1 + lx.lineno += 1 + lx.col = 0 + continue + + # Backslash continuation line + # Support LF and CRLF line endings: under CRLF `\` is followed by '\r' (not '\n'), + # so checking only '\n' fails and the backslash continuation breaks. + if ch == '\\' and lx.pos + 1 < lx.len: + if lx.src[lx.pos + 1] == '\n': + lx.pos += 2 + lx.lineno += 1 + lx.col = 0 + continue + if lx.src[lx.pos + 1] == '\r': + # \ + \r\n: skip \, \r, \n + lx.pos += 2 + if lx.pos < lx.len and lx.src[lx.pos] == '\n': + lx.pos += 1 + lx.lineno += 1 + lx.col = 0 + continue + + # Whitespace + if _is_space(ch): + lx.pos += 1 + lx.col += 1 + continue + + # Comment + if ch == '#': + while lx.pos < lx.len and lx.src[lx.pos] != '\n': + lx.pos += 1 + continue + + # String prefixes (like r"...") or strings + if ch == '\'' or ch == '"': + _read_string(lx) + continue + # A string prefix letter followed by a quote (must be checked before _is_alpha, otherwise f/r/b/u will be treated as TokenType.Name) + if (ch == 'r' or ch == 'R' or ch == 'b' or ch == 'B' or + ch == 'u' or ch == 'U' or ch == 'f' or ch == 'F'): + if lx.pos + 1 < lx.len: + ch1: char = lx.src[lx.pos + 1] + if ch1 == '\'' or ch1 == '"': + _read_string(lx) + continue + # Double prefixes like rb, br, rf, fr + if lx.pos + 2 < lx.len: + ch2: char = lx.src[lx.pos + 2] + if ((ch1 == 'r' or ch1 == 'R' or ch1 == 'b' or ch1 == 'B' or + ch1 == 'f' or ch1 == 'F' or ch1 == 'u' or ch1 == 'U') and + (ch2 == '\'' or ch2 == '"')): + _read_string(lx) + continue + + # Identifier/keyword + if _is_alpha(ch): + _read_name(lx) + continue + + # Number + if _is_digit(ch): + _read_number(lx) + continue + + # Operator + _read_operator(lx) + + # End of file: generate TokenType.NewLine (if there's content) + if lx.tokens_tail != None: + if lx.tokens_tail.type != TokenType.NewLine and lx.tokens_tail.type != TokenType.Nl: + tok: Token* = new_token(lx.pool, TokenType.NewLine, lx.lineno, lx.col) + _emit(lx, tok) + + # Generate remaining remaining TokenType.Dedent + while lx.indent_top > 0: + lx.indent_top -= 1 + tok: Token* = new_token(lx.pool, TokenType.Dedent, lx.lineno, 0) + _emit(lx, tok) + + # TokenType.EndMarker + tok: Token* = new_token(lx.pool, TokenType.EndMarker, lx.lineno, 0) + _emit(lx, tok) + + return lx.tokens_head diff --git a/ast/parser.vp b/ast/parser.vp index e5c8bba..a6cb02c 100644 --- a/ast/parser.vp +++ b/ast/parser.vp @@ -1,4949 +1,4950 @@ -from stdint import * -import memhub -import string -import stdio -import stdlib -from .tokens import (Token, TokenType, Keyword, TokOp) -from .base import ( - AST, ASTCtx, ASTKind, OpKind, ASTFlag, - _set_pos, _binop_from_op, _augop_from_op, _cmpop_from_op, _copy_str, _inherit_pos, - _init_ast, - CONST_INT, CONST_FLOAT, CONST_STR, CONST_BOOL, CONST_NONE, CONST_CHAR, - FLAG_IS_ASYNC, FLAG_IS_EXTERN, FLAG_IS_EXPORT, FLAG_IS_DEFINE, -) -from .exprs import ( - BoolOp, BinOp, UnaryOp, Lambda, IfExp, Dict, Set, - ListComp, SetComp, DictComp, GeneratorExp, - Await, Yield, YieldFrom, FormattedValue, JoinedStr, - Constant, NamedExpr, Attribute, Subscript, Starred, Name, - List, Tuple, Slice, Call, Compare, OpNode, New, - NEW_SRC_DEFAULT, NEW_SRC_STACK, NEW_SRC_HEAP, NEW_SRC_MEMGR, - NEW_LIFE_DEFAULT, NEW_LIFE_SLIFE, NEW_LIFE_HLIFE, -) -from .stmts import ( - Module, Expression, Interactive, FunctionType, - FunctionDef, ClassDef, Return, Delete, Assign, AugAssign, AnnAssign, - For, While, If, With, Raise, Try, Assert, Global, Nonlocal, - Pass, Break, Continue, FallThrough, Expr, Import, ImportFrom, Match, -) -from .astaux import ( - ExceptHandler, Arguments, Arg, Keyword as KwNode, Alias, WithItem, Comprehension, -) -from .match import ( - MatchCase, MatchValue, MatchSingleton, MatchSequence, - MatchMapping, MatchClass, MatchStar, MatchAs, MatchOr, -) -from .lexer import ( - Lexer, _lexer_init, tokenize, new_lexer, - get_lex_err_line, get_lex_err_col, get_lex_err_msg, - _lex_int_neg_exceeds_i64, -) - - -# ============================================================ -# Parser State -# ============================================================ - -# Hard cap on the token linked-list length (cycle detection): normal sources have far fewer -# tokens, so reaching this count without the tail means the list is cyclic (see parse_tokens). -define PARSER_MAX_TOKENS = 8000000 - - -class Parser: - tokens: Token* # token linked list head - cur: Token* # current token - prev: Token* # previous consumed token (for the `T*(x)` adjacency test) - pool: memhub.MemManager* # memory pool - error_count: int # error count - pending_decorators: list[AST*]* # pending decorators list - # Pending parsed left operand for a leading `*` in argument position (see _parse_call / _parse_power). - starred_seed: AST* - # `static` modifier (soft keyword) pending flag: set after consuming `static` before a `Name :` - # declaration; the next _parse_type_ann prepends t.CStatic and clears it (see _ParseStaticModified). - pending_static: int - # Whether the suite currently being parsed is directly owned by a class (1 = class body, 0 = other), - # maintained by _parse_suite_into_children for the virtual/novirtual class-body check. - in_class_body: int - # Step budget (_advance count / parse_tokens budget): the statement loop fails fast once exceeded, - # never looping forever (see the cycle detection in parse_tokens). - step_count: sizet - step_limit: sizet - # Watchdog: counter and convergence flag for "query current token" (see _tick). When tick_count - # exceeds step_limit, aborted=1, entries return converged values and every loop ends in bounded rounds. - tick_count: sizet - aborted: int - - def __new__(self, pool: memhub.MemManager*): - ptr: Parser* = pool.alloc(Parser.__sizeof__()) - if ptr: string.memset(ptr, 0, Parser.__sizeof__()) - return ptr - - -def new_parser(pool: memhub.MemManager*) -> Parser*: - """Allocate from the pool and return the Parser (cross-module safe wrapper)""" - return Parser(pool) - - -def _parser_init(ps: Parser*, tokens: Token*, - pool: memhub.MemManager*): - if ps is None: return - ps.tokens = tokens - ps.cur = tokens - ps.prev = None - ps.pool = pool - ps.error_count = 0 - ps.pending_decorators = None - ps.starred_seed = None - ps.pending_static = 0 - ps.in_class_body = 0 - ps.step_count = 0 - # Default budget is the cap (parse_tokens later tightens it to the real token count) so entries that - # bypass parse_tokens (parse_expression etc.) do not misreport at 0, while a hard bound still prevents loops. - ps.step_limit = PARSER_MAX_TOKENS - ps.tick_count = 0 - ps.aborted = 0 - - -# ============================================================ -# Parser watchdog (static convergence: no silent hang) -# -# Any loop that spins without consuming a token must repeatedly query the current token, so every query -# entry (_cur_type / _cur_op / _cur_kw / _match_op / _accept_op / _advance) ticks the watchdog; once the -# budget is exceeded it sets aborted, records a parse error and returns a converged value: -# _cur_type -> EndMarker, _cur_op / _cur_kw -> 0, _match_op / _accept_op -> 1 (jump to end), -# _advance -> jump to end. parse_tokens then returns None with error_count > 0 and the caller reports -# file:line:col SyntaxError; there is no silent-hang path left. -# ============================================================ - -def _tick(ps: Parser*): - """Tick the watchdog; over budget sets aborted and records the first parse error (with position)""" - if ps is None: return - if ps.aborted != 0: return - ps.tick_count += 1 - if ps.tick_count > ps.step_limit: - ps.aborted = 1 - _record_parse_err(ps) - ps.error_count += 1 - - -# ============================================================ -# Token Navigation Helpers -# ============================================================ - -def _advance(ps: Parser*): - """Advance to the next token""" - _tick(ps) - if ps.aborted != 0: - ps.cur = None - return - if ps.cur is None: return - if ps.cur.next is not None: - ps.prev = ps.cur - ps.cur = ps.cur.next - ps.step_count += 1 - - -def _cur_type(ps: Parser*) -> int: - _tick(ps) - if ps.aborted != 0: return TokenType.EndMarker - if ps.cur is None: return TokenType.EndMarker - return ps.cur.type - - -def _cur_op(ps: Parser*) -> int: - _tick(ps) - if ps.aborted != 0: return 0 - if ps.cur is None: return 0 - return ps.cur.op_subtype - - -def _cur_kw(ps: Parser*) -> int: - _tick(ps) - if ps.aborted != 0: return 0 - if ps.cur is None: return 0 - return ps.cur.kw_subtype - - -def _is_funcdef_kw(ps: Parser*, kw: int) -> int: - """Whether the current token is a function-definition keyword (def and its aliases - fn/fun/func/function, plus extern/export). - - def is a hard keyword. fn/fun/func/function/extern/export are soft: they count only when the next - token is a Name followed by '(' or a generic '['; extern/export also accept ` def A(...)`. - Otherwise they are ordinary identifiers (e.g. the AST `func` field or `func: AST*`). - """ - if kw == Keyword.Def: - return 1 - if kw == Keyword.Extern or kw == Keyword.Export: - # extern/export def A(...) or extern/export A(...) - # (after extern/export comes def or Name + '(' / '[') - nxt: Token* = ps.cur.next - if nxt is None: return 0 - if nxt.kw_subtype == Keyword.Def: return 1 - if nxt.type != TokenType.Name: return 0 - nxt2: Token* = nxt.next - if nxt2 is None: return 0 - if nxt2.type == TokenType.Op and (nxt2.op_subtype == TokOp.LPar or nxt2.op_subtype == TokOp.Lsqb): - return 1 - return 0 - if kw == Keyword.Fn or kw == Keyword.Fun \ - or kw == Keyword.Func or kw == Keyword.Function \ - or kw == Keyword.Define: - # Soft-keyword lookahead: next is a Name, the one after is '(' or a generic '[T...]' (PEP 695) - # (the define macro function matches def, with t.CDefine appended to the return annotation) - nxt: Token* = ps.cur.next - if nxt is None: - return 0 - if nxt.type != TokenType.Name: - return 0 - nxt2: Token* = nxt.next - if nxt2 is None: - return 0 - if nxt2.type == TokenType.Op \ - and (nxt2.op_subtype == TokOp.LPar or nxt2.op_subtype == TokOp.Lsqb): - return 1 - return 0 - return 0 - - -def _is_define_const(ps: Parser*) -> int: - """Soft-keyword lookahead for a define constant declaration (1 = yes, 0 = no). - - `define NAME[: T] = `: next is a Name, then '=' (no annotation) or ':' (annotated). - `define NAME(...)` and `define` used as an identifier do not match. `define NAME: T` (missing '=') - does match and then fails loudly in _parse_define_const's _expect_op(Equal). - """ - if _cur_kw(ps) != Keyword.Define: return 0 - nxt: Token* = ps.cur.next - if nxt is None or nxt.type != TokenType.Name: return 0 - nxt2: Token* = nxt.next - if nxt2 is None or nxt2.type != TokenType.Op: return 0 - if nxt2.op_subtype == TokOp.Equal or nxt2.op_subtype == TokOp.Colon: return 1 - return 0 - - -def _is_typedef(ps: Parser*) -> int: - """Soft-keyword test for a typedef alias declaration (1 = yes, 0 = no). - - `typedef T1 = T2` is fully equivalent to `T1: t.CTypedef = T2`. `typedef` is a statement-level - reserved word (never an identifier in the repo); invalid forms fail loudly in _parse_typedef. - """ - if _cur_kw(ps) != Keyword.Typedef: - return 0 - return 1 - - -def _is_export_extern_var(ps: Parser*) -> int: - """Soft-keyword test for `export NAME :` / `extern NAME :` variable declarations (1 = yes, 0 = no). - - export/extern also serve function declarations (see _is_funcdef_kw); the third token distinguishes - them: '(' or '[' is a function, ':' is a variable. Other forms do not match; invalid forms fail loudly. - """ - kw: int = _cur_kw(ps) - if kw != Keyword.Export and kw != Keyword.Extern: - return 0 - nxt: Token* = ps.cur.next - if nxt is None: - return 0 - if nxt.type != TokenType.Name: - return 0 - # The name must be a non-keyword identifier (excludes forms like `export def f()`) - if nxt.kw_subtype != Keyword.NotKeyword: - return 0 - nxt2: Token* = nxt.next - if nxt2 is None: - return 0 - if nxt2.type != TokenType.Op: - return 0 - if nxt2.op_subtype != TokOp.Colon: - return 0 - return 1 - - -def _is_named_soft(ps: Parser*, name: str) -> int: - """Whether the current token is the given soft-keyword identifier (Name, not a hard keyword).""" - if _cur_type(ps) != TokenType.Name: - return 0 - if _cur_kw(ps) != Keyword.NotKeyword: - return 0 - s: str = _cur_str(ps) - if s is None: - return 0 - if string.strcmp(s, name) == 0: - return 1 - return 0 - - -def _is_cif_stmt(ps: Parser*) -> int: - """Whether the current token starts a `cif` statement (soft keyword; 1 = yes / 0 = no). - - `cif EXPR:` is a compile-time conditional statement. An assignment/annotation/attribute form - (`cif = ...` / `cif: T = ...` / `cif += ...` / `cif.attr`) is an ordinary identifier use and - must not be treated as the keyword. - """ - if _is_named_soft(ps, "cif") == 0: - return 0 - nxt: Token* = ps.cur.next - if nxt is None: - return 0 - if nxt.type == TokenType.Op: - o: int = nxt.op_subtype - if o == TokOp.Equal or o == TokOp.Colon or o == TokOp.ColonEq or o == TokOp.Dot \ - or o == TokOp.PlusEq or o == TokOp.MinusEq or o == TokOp.StarEq \ - or o == TokOp.SlashEq or o == TokOp.DSlashEq or o == TokOp.PercentEq \ - or o == TokOp.AtEq or o == TokOp.AmpEq or o == TokOp.VBarEq \ - or o == TokOp.CaretEq or o == TokOp.GtGtEq or o == TokOp.LtLtEq \ - or o == TokOp.StarEqEq: - return 0 - return 1 - - -# Parse error position and message (fail-fast, Python SyntaxError style, first error only) -_g_parse_err_line: int = 0 -_g_parse_err_col: int = 0 -_g_parse_err_msg: str = None - - -def _record_parse_err(ps: Parser*): - """Record the position and message of the first parse error (later errors do not overwrite it)""" - global _g_parse_err_line, _g_parse_err_col, _g_parse_err_msg - if _g_parse_err_line != 0: - return - _g_parse_err_line = _cur_lineno(ps) - _g_parse_err_col = _cur_col(ps) - err_s: str = _cur_str(ps) - if err_s is None: - err_s = "(none)" - # Copy the message text via stdlib (token.str_val lives in the pool; stored independently to be safe) - buf_m: str = stdlib.malloc(128) - if buf_m is not None: - stdio.snprintf(buf_m, 128, "unexpected token '%s'", err_s) - _g_parse_err_msg = buf_m - else: - _g_parse_err_msg = err_s - - -def _record_parse_err_ctx(ps: Parser*, tok_type: int, - pv_str: str, pv_line: int, pv_col: int, - nx_str: str, nx_line: int, nx_col: int): - """Record the first parse error with token type and prev/next tokens appended to the message. - - Same source as _record_parse_err (first only); the message must stay a single line since logs - inline it. file:line:col and the caret are filled later by the caller's dump_parse_traceback. - """ - global _g_parse_err_line, _g_parse_err_col, _g_parse_err_msg - if _g_parse_err_line != 0: - return - _g_parse_err_line = _cur_lineno(ps) - _g_parse_err_col = _cur_col(ps) - err_s: str = _cur_str(ps) - if err_s is None: - err_s = "(none)" - buf_m: str = stdlib.malloc(512) - if buf_m is None: - _g_parse_err_msg = err_s - return - stdio.snprintf(buf_m, 512, - "unexpected token '%s' (type=%d) | note: prev='%s' (line %d col %d) | next='%s' (line %d col %d)", - err_s, tok_type, pv_str, pv_line, pv_col, nx_str, nx_line, nx_col); - _g_parse_err_msg = buf_m - - -def _record_neg_literal_overflow(lit_text: str, lineno: int, col: int): - """Record a combined "unary minus + integer literal" overflow (fail-fast; first only). - - `-` is a separate unary operator, so a magnitude-valid literal (<= 2^64-1) may still be < -2^63 - after negation. Message carries the literal text + allowed range [-2^63, 2^64-1] + fix hint. - """ - global _g_parse_err_line, _g_parse_err_col, _g_parse_err_msg - if _g_parse_err_line != 0: - return - _g_parse_err_line = lineno - _g_parse_err_col = col - buf: str = stdlib.malloc(512) - if buf is not None: - stdio.snprintf(buf, 512, - "[TPV] negative integer literal out of range: unary minus applied to an integer literal exceeds the" - " signed 64-bit representable range (allowed range -9223372036854775808..18446744073709551615" - " = [-2^63, 2^64-1]; a negative result must be >= -2^63). Literal text: '-%s'." - " Silent wrap-around/truncation is forbidden; compilation aborted." - " Fix: change the negative number into -9223372036854775808..-1 (this language has no" - " wider integer type, for larger values split into multiple 64-bit words).", lit_text) - _g_parse_err_msg = buf - else: - _g_parse_err_msg = lit_text - - -# ============================================================ -# Type descriptors const / volatile (soft keywords, annotation position only) -# ============================================================ -def _IsDescriptorKeyword(ps: Parser*) -> int: - """Whether the current token is a type-descriptor keyword (const / volatile), 1 = yes / 0 = no. - - Soft keywords consumed by _parse_type_ann only in annotation position; `t.attr.const()` as an - identifier is unaffected since attribute/call resolution looks at TokenType.Name, not kw_subtype. - """ - if _cur_type(ps) != TokenType.Name: - return 0 - if _cur_kw(ps) == Keyword.Const or _cur_kw(ps) == Keyword.Volatile: - return 1 - return 0 - - -def _IsMagicAttributeCall(ps: Parser*) -> int: - """Test for an annotation-position `__attribute__(...)` prefix, 1 = yes / 0 = no. - - `__attribute__` is a soft keyword (not in the keyword table): consumed here only in annotation - position immediately before '('. Same family as const/volatile: order-free and mixable. - """ - if _cur_type(ps) != TokenType.Name: - return 0 - s: str = _cur_str(ps) - if s is None or string.strcmp(s, "__attribute__") != 0: - return 0 - nxt: Token* = ps.cur.next - if nxt is None or nxt.type != TokenType.Op or nxt.op_subtype != TokOp.LPar: - return 0 - return 1 - - -def _MakeDescriptorMarker(ps: Parser*) -> AST*: - """Build the marker node Attribute(Name('t'), 'CConst'/'CVolatile') from the current descriptor keyword. - - Field-identical to TPC's _ExpandTypeDescriptors text expansion (t.CConst / t.CVolatile) so both - sides share one recognition path; no second representation is introduced. - """ - attr_name: str = "CConst" - if _cur_kw(ps) == Keyword.Volatile: - attr_name = "CVolatile" - marker: AST* = Attribute(ps.pool, Name(ps.pool, "t", ASTCtx.Load), - attr_name, ASTCtx.Load) - _set_pos(marker, _cur_lineno(ps), _cur_col(ps), 0, 0) - return marker - - -def _PrependDescriptorMarkers(ps: Parser*, - base: AST*, - mods: list[AST*]*, - mod_count: int) -> AST*: - """Prepend descriptor markers to the leftmost operand of the type union (same shape as TPC text). - - TPC expands `const volatile int | t.CPtr` to the left-associative `(((t.CConst | t.CVolatile) | int) | t.CPtr)`; - this function produces the identical tree rather than a differently-shaped one. - """ - slot_parent: BinOp* = None # leftmost BitOr node: its left is the leftmost operand - slot_node: AST* = base - cur: AST* = base - while cur is not None and cur.kind() == ASTKind.BinOp: - b: BinOp* = (BinOp*)(cur) - if b.op != OpKind.BitOr or b.left is None: - break - nxt: AST* = b.left - if nxt.kind() == ASTKind.BinOp: - nb: BinOp* = (BinOp*)(nxt) - if nb.op == OpKind.BitOr and nb.left is not None: - cur = nxt - continue - slot_parent = b - slot_node = nxt - break - # Descriptor chain left-associates in source order: `const volatile` -> (t.CConst | t.CVolatile) - chain: AST* = mods.get(0) - i: int = 1 - while i < mod_count: - chain = BinOp(ps.pool, chain, OpKind.BitOr, mods.get(i)) - i += 1 - newnode: AST* = BinOp(ps.pool, chain, OpKind.BitOr, slot_node) - if slot_parent is None: - return newnode - slot_parent.left = newnode - return base - - -def _RecordDescriptorErr(detail: str, lineno: int, col: int): - """Record an invalid type-descriptor form (fail-fast; first only). - - Message carries the expected form + a correct example; file:line:col is filled later. Descriptors - are never silently ignored: missing type / non-type follower / bare use all abort compilation. - """ - global _g_parse_err_line, _g_parse_err_col, _g_parse_err_msg - if _g_parse_err_line != 0: - return - _g_parse_err_line = lineno - _g_parse_err_col = col - buf: str = stdlib.malloc(1024) - if buf is not None: - stdio.snprintf(buf, 1024, - "invalid type descriptor (%s). const/volatile are soft keywords valid only in type position;" - " they must be written as `name: const type` / `name: volatile type`" - " (combinable as `name: const volatile type`). Correct examples: a: const int / " - "a: volatile int / a: const volatile t.CInt32T / " - "def f(p: const rawptr[Foo]) -> const int." - " (descriptors are never silently ignored: missing type / non-type follower / bare use all abort compilation)", detail) - _g_parse_err_msg = buf - else: - _g_parse_err_msg = detail - - -# ============================================================ -# virtual / novirtual modifier (soft keyword; statement start before class/def only), same -# representation as @t.CVTable / @t.NoVTable. Invalid forms fail loudly; `x = d.virtual` unaffected. -def _IsVirtualModifier(ps: Parser*) -> int: - """Whether the current token is a virtual / novirtual modifier (1 = yes / 0 = no). - - Called only at statement start; identifier positions never reach this test, so `x = d.virtual`, - parameter names, for targets and dict keys are unaffected. - """ - if _cur_type(ps) != TokenType.Name: - return 0 - if _cur_kw(ps) == Keyword.Virtual or _cur_kw(ps) == Keyword.Novirtual: - return 1 - return 0 - - -def _MakeVirtualMarker(ps: Parser*) -> AST*: - """Build the marker node Attribute(Name('t'), 'CVTable'|'NoVTable') from the current virtual/novirtual token. - - Field-identical to hand-written @t.CVTable / @t.NoVTable, so ClassGenBase._has_decorator needs no new test. - """ - attr_name: str = "CVTable" - if _cur_kw(ps) == Keyword.Novirtual: - attr_name = "NoVTable" - marker: AST* = Attribute(ps.pool, Name(ps.pool, "t", ASTCtx.Load), - attr_name, ASTCtx.Load) - _set_pos(marker, _cur_lineno(ps), _cur_col(ps), 0, 0) - return marker - - -def _RecordVirtualErr(detail: str, lineno: int, col: int, extra: str): - """Record an invalid virtual/novirtual form (fail-fast; first only); message carries allowed - positions + a correct example. Modifiers are never silently ignored. - - Callers build detail/extra with f-string interpolation: this language's str is i8*, so `str + str` - degrades to raw pointer add and illegal IR; never concatenate strings with `+` in this file. - """ - global _g_parse_err_line, _g_parse_err_col, _g_parse_err_msg - if _g_parse_err_line != 0: - return - _g_parse_err_line = lineno - _g_parse_err_col = col - buf: str = stdlib.malloc(1536) - if buf is not None: - stdio.snprintf(buf, 1536, - "[TPV] invalid virtual/novirtual modifier (%s). Valid positions: class level " - "`virtual class C:` / `novirtual class C:`; method level " - "`virtual def m(self): ...` / `novirtual def m(self): ...`" - " (only directly inside a class body: the body of virtual class / class / novirtual class)." - " Correct examples: virtual class Shape: / novirtual class Node: / " - "class A: virtual def run(self) -> int: ... / " - "novirtual class B: novirtual def helper(self) -> int: ....%s" - " (modifiers are never silently ignored: illegal forms all abort compilation)", detail, extra) - _g_parse_err_msg = buf - else: - _g_parse_err_msg = detail - - -def _ParseVirtualModified(ps: Parser*, parent: AST*): - """`virtual class` / `novirtual class` / `virtual def` / `novirtual def`. - - Synthesizes a @t.CVTable / @t.NoVTable decorator (merged with existing decorators, appended last; - position-independent since _has_decorator matches by name) and hands off to _parse_classdef / _parse_funcdef. - """ - mod_lineno: int = _cur_lineno(ps) - mod_col: int = _cur_col(ps) - mod_name: str = "virtual" - if _cur_kw(ps) == Keyword.Novirtual: - mod_name = "novirtual" - marker: AST* = _MakeVirtualMarker(ps) - _advance(ps) # consume the modifier - # The modifier must be followed by class / def (the def aliases fn/fun/func/function are accepted too, - # matching _is_funcdef_kw and TPC, whose fn->def expansion only hits a leading fn). - kw2: int = _cur_kw(ps) - is_class: int = 0 - is_def: int = 0 - if kw2 == Keyword.Class: - is_class = 1 - elif kw2 == Keyword.Def or kw2 == Keyword.Fn or kw2 == Keyword.Fun \ - or kw2 == Keyword.Func or kw2 == Keyword.Function: - is_def = 1 - if is_class == 0 and is_def == 0: - nxt_s: str = _cur_str(ps) - if nxt_s is None: - nxt_s = "(none)" - if kw2 == Keyword.Struct: - # struct is a pure data type: no virtual / novirtual modifier allowed. A dedicated message - # (not the generic "follower not class/def") points at the fix: struct already implies @t.NoVTable. - _RecordStructErr(f"`{mod_name} struct` is invalid: struct forbids the virtual/novirtual modifier", - mod_lineno, mod_col, - "struct has no vtable by nature (a @t.NoVTable is synthesized at parse time);" - " write `struct X:` directly; if vtable semantics are needed, use class instead.") - else: - _RecordVirtualErr(f"`{mod_name}` is followed by '{nxt_s}' (not class/def)", - mod_lineno, mod_col, - f"`{mod_name}` takes effect only before class/def;" - f" if a plain identifier name was intended here, rename it" - f" (this word is treated as a modifier at statement start).") - ps.error_count += 1 - # Dispatch the remaining tokens to the generic statement path (no infinite loop, no error overwrite). - _parse_simple_stmt_into(ps, parent) - return - if is_def != 0 and ps.in_class_body == 0: - _RecordVirtualErr(f"{mod_name} def appears outside a class body", mod_lineno, mod_col, - f"{mod_name} def describes the 'virtual/non-virtual' of a method inside a class body;" - f" at module level / function body / branch body there is no class to attach to.") - ps.error_count += 1 - # Merge existing decorators (@deco + virtual class/def) then hand off to the existing class/def parser - merged: list[AST*]* = list[AST*](ps.pool, 8) - if merged is None: - return - if ps.pending_decorators is not None: - dn: sizet = len(ps.pending_decorators) - for di in range(dn): - merged.append(ps.pending_decorators.get(di)) - merged.append(marker) - ps.pending_decorators = merged - stmt: AST* = None - if is_class != 0: - stmt = _parse_classdef(ps) - else: - stmt = _parse_funcdef(ps, 0) - if stmt is not None: - parent.append(stmt) - - -# ============================================================ -# inline modifier (soft keyword): `inline def f(...)` -> synthesizes @t.attr.always_inline, same shape as -# virtual/novirtual (statement start before a function definition only); kept out of the keyword table. -# ============================================================ -def _IsInlineModifier(ps: Parser*) -> int: - """Whether the current token is `inline` (soft keyword followed by a function definition), 1 = yes / 0 = no""" - if _cur_type(ps) != TokenType.Name: - return 0 - s: str = _cur_str(ps) - if s is None or string.strcmp(s, "inline") != 0: - return 0 - nxt: Token* = ps.cur.next - if nxt is None or nxt.type != TokenType.Name: - return 0 - nxt2: Token* = nxt.next - if nxt2 is None: - return 0 - # Followed by def (hard keyword): `inline def f(...)` - if nxt.kw_subtype == Keyword.Def: - return 1 - # Followed by a function-definition soft-keyword alias (fn/fun/func/function): same rule as - # _is_funcdef_kw -- must be Name + '(' or '[T]', so `inline func` is not misread. - if nxt.kw_subtype == Keyword.Fn or nxt.kw_subtype == Keyword.Fun \ - or nxt.kw_subtype == Keyword.Func or nxt.kw_subtype == Keyword.Function: - if nxt2.type == TokenType.Op \ - and (nxt2.op_subtype == TokOp.LPar or nxt2.op_subtype == TokOp.Lsqb): - return 1 - return 0 - - -def _MakeInlineMarker(ps: Parser*) -> AST*: - """Build the marker Attribute(Attribute(Name('t'),'attr'), 'always_inline') from the current inline token. - - Field-identical to a hand-written @t.attr.always_inline. Deliberately not @c.Attribute(t.attr.x), - whose extractor requires `import c` and would silently skip inlining in files lacking it. - """ - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - t_name: AST* = Name(ps.pool, "t", ASTCtx.Load) - attr_at: AST* = Attribute(ps.pool, t_name, "attr", ASTCtx.Load) - marker: AST* = Attribute(ps.pool, attr_at, "always_inline", ASTCtx.Load) - _set_pos(marker, lineno, col, 0, 0) - return marker - - -def _ParseInlineModified(ps: Parser*, parent: AST*): - """`inline def f(...)`: consume inline, synthesize @t.attr.always_inline and hand off to _parse_funcdef""" - marker: AST* = _MakeInlineMarker(ps) - if marker is None: - return - _advance(ps) # consume the inline modifier - # Merge existing decorators (@deco + inline def), appending the marker last; position-independent - # (matched by name downstream). Same pending_decorators channel as _ParseVirtualModified. - merged: list[AST*]* = list[AST*](ps.pool, 8) - if merged is None: - return - if ps.pending_decorators is not None: - dn: sizet = len(ps.pending_decorators) - for di in range(dn): - merged.append(ps.pending_decorators.get(di)) - merged.append(marker) - ps.pending_decorators = merged - stmt: AST* = _parse_funcdef(ps, 0) - if stmt is not None: - parent.append(stmt) - - -def _IsStaticModifier(ps: Parser*) -> int: - """Whether the current token is a `static` modifier (soft keyword), 1 = yes / 0 = no. - - Called only at statement start; matches when followed by (1) a function definition (`def` or the - fn/fun/func/function alias + '(' / '[') or (2) a variable declaration (`Name :`, i.e. AnnAssign). - Identifier positions never reach this test, so `x = d.static` / `f(static=1)` are unaffected. - """ - if _cur_type(ps) != TokenType.Name: - return 0 - s: str = _cur_str(ps) - if s is None or string.strcmp(s, "static") != 0: - return 0 - nxt: Token* = ps.cur.next - if nxt is None or nxt.type != TokenType.Name: - return 0 - # (1) followed by def (hard keyword): `static def f(...)` - if nxt.kw_subtype == Keyword.Def: - return 1 - nxt2: Token* = nxt.next - # (1') followed by a function-definition alias: must be Name + '(' or '[T]', same rule as - # _IsInlineModifier / _is_funcdef_kw, so `static func` is not misread. - if nxt.kw_subtype == Keyword.Fn or nxt.kw_subtype == Keyword.Fun \ - or nxt.kw_subtype == Keyword.Func or nxt.kw_subtype == Keyword.Function: - if nxt2 is not None and nxt2.type == TokenType.Op \ - and (nxt2.op_subtype == TokOp.LPar or nxt2.op_subtype == TokOp.Lsqb): - return 1 - # (2) followed by a variable declaration `Name :` (AnnAssign) - if nxt2 is not None and nxt2.type == TokenType.Op and nxt2.op_subtype == TokOp.Colon: - return 1 - return 0 - - -def _IsStaticDescriptorKeyword(ps: Parser*) -> int: - """Whether the current token is an annotation-position `static` (soft keyword), 1 = yes / 0 = no. - - This position (`name: static T`) is no longer supported: _parse_type_ann only detects it as misuse - and fails loudly, pointing at the prefix form `static name: T`; it is no longer desugared to t.CStatic. - Valid forms are statement-start `static name: T` and the `@t.CStatic` decorator. Reporting (not ignoring) - is mandatory: unconsumed, `static` degrades into an opaque "class" named static and errors far from the cause. - """ - if _cur_type(ps) != TokenType.Name: - return 0 - s: str = _cur_str(ps) - if s is None or string.strcmp(s, "static") != 0: - return 0 - return 1 - - -def _MakeStaticMarker(ps: Parser*) -> AST*: - """Build the marker node Attribute(Name('t'), 'CStatic'). - - Field-identical to the hand-written @t.CStatic decorator / `x: t.CStatic | T` marker, so downstream - (FuncGenDef storage-class detection, TypeGenResolve qualifier resolution) needs no new test. - """ - marker: AST* = Attribute(ps.pool, Name(ps.pool, "t", ASTCtx.Load), - "CStatic", ASTCtx.Load) - _set_pos(marker, _cur_lineno(ps), _cur_col(ps), 0, 0) - return marker - - -def _ParseStaticModified(ps: Parser*, parent: AST*) -> int: - """Dispatch the `static` modifier: 1 = fully handled (function definition), 0 = only flagged - (variable declaration, caller continues with the generic statement dispatch). - - Function: consume static and synthesize an @t.CStatic decorator, handed to _parse_funcdef. - Variable: set pending_static and consume static; the next _parse_type_ann prepends t.CStatic. - """ - nxt: Token* = ps.cur.next - is_func: int = 0 - if nxt is not None and nxt.type == TokenType.Name: - if nxt.kw_subtype == Keyword.Def: - is_func = 1 - elif nxt.kw_subtype == Keyword.Fn or nxt.kw_subtype == Keyword.Fun \ - or nxt.kw_subtype == Keyword.Func or nxt.kw_subtype == Keyword.Function: - nxt2: Token* = nxt.next - if nxt2 is not None and nxt2.type == TokenType.Op \ - and (nxt2.op_subtype == TokOp.LPar or nxt2.op_subtype == TokOp.Lsqb): - is_func = 1 - if is_func == 0: - # Variable branch: only flag + consume static, then let the caller continue generic dispatch - # (AnnAssign parsing reads pending_static in _parse_type_ann). - ps.pending_static = 1 - _advance(ps) - return 0 - marker: AST* = _MakeStaticMarker(ps) - if marker is None: - return 1 - _advance(ps) # consume the static modifier - # Merge existing decorators (@deco + static def), appending the marker last; position-independent - # (matched by name downstream). Same pending_decorators channel as _ParseInlineModified. - merged: list[AST*]* = list[AST*](ps.pool, 8) - if merged is None: - return 1 - if ps.pending_decorators is not None: - dn: sizet = len(ps.pending_decorators) - for di in range(dn): - merged.append(ps.pending_decorators.get(di)) - merged.append(marker) - ps.pending_decorators = merged - stmt: AST* = _parse_funcdef(ps, 0) - if stmt is not None: - parent.append(stmt) - return 1 - - -def _IsBareDescriptor(ps: Parser*) -> int: - """Bare-descriptor test: the whole statement is a single descriptor run (const[: volatile]), 1 = yes / 0 = no. - - Matches only when the descriptor run is followed directly by newline / EOF / semicolon; identifiers - (`const = 5` / `const(x)` / `t.attr.const()`) do not match (soft-keyword boundary). - """ - if _IsDescriptorKeyword(ps) == 0: - return 0 - nxt: Token* = ps.cur.next - while nxt is not None and nxt.type == TokenType.Name \ - and (nxt.kw_subtype == Keyword.Const or nxt.kw_subtype == Keyword.Volatile): - nxt = nxt.next - if nxt is None: - return 1 - if nxt.type == TokenType.NewLine or nxt.type == TokenType.EndMarker: - return 1 - if nxt.type == TokenType.Op and nxt.op_subtype == TokOp.Semi: - return 1 - return 0 - - -def get_parse_err_line() -> int: - """Line number of the latest parse error (0 = none)""" - return _g_parse_err_line - - -def get_parse_err_col() -> int: - """Column of the latest parse error (0 = none)""" - return _g_parse_err_col - - -def get_parse_err_msg() -> str: - """Message of the latest parse error (Python SyntaxError style)""" - return _g_parse_err_msg - - -def _cur_str(ps: Parser*) -> str: - if ps.cur is None: return None - return ps.cur.str_val - - -def _cur_lineno(ps: Parser*) -> int: - if ps.cur is None: return 0 - return ps.cur.lineno - - -def _cur_col(ps: Parser*) -> int: - if ps.cur is None: return 0 - return ps.cur.col_offset - - -def _skip_nl(ps: Parser*): - """Skip TokenType.Nl token""" - while ps.cur is not None and ps.cur.type == TokenType.Nl: - _advance(ps) - - -def _skip_nls_and_newlines(ps: Parser*): - """Skip TokenType.Nl and TokenType.NewLine (used for line continuation)""" - while ps.cur is not None: - if ps.cur.type == TokenType.Nl or ps.cur.type == TokenType.NewLine: - _advance(ps) - else: - break - - -def _expect_type(ps: Parser*, ttype: int) -> int: - """Expect the current token to be of the specified type, advance and return 1, otherwise return 0""" - if _cur_type(ps) == ttype: - _advance(ps) - return 1 - _record_parse_err(ps) - ps.error_count += 1 - return 0 - - -def _expect_op(ps: Parser*, op_id: int) -> int: - """Expecting the current one to be some operator""" - if _cur_type(ps) == TokenType.Op and _cur_op(ps) == op_id: - _advance(ps) - return 1 - _record_parse_err(ps) - ps.error_count += 1 - return 0 - - -def _match_op(ps: Parser*, op_id: int) -> int: - """Check but not consume the current token""" - _tick(ps) - if ps.aborted != 0: - # Convergence: if tokens remain, "match" and jump to the end so both - # `while not _match_op(...)` and `while _match_op(...)` end within <= 2 rounds - if ps.cur is not None: - ps.cur = None - return 1 - return 0 - if _cur_type(ps) == TokenType.Op and _cur_op(ps) == op_id: - return 1 - return 0 - - -def _accept_op(ps: Parser*, op_id: int) -> int: - """If match, consume and return 1""" - if _match_op(ps, op_id): - _advance(ps) - return 1 - return 0 - - -# ============================================================ -# Constant Construction Helper (New API: Constant directly accepts lineno/col) -# ============================================================ - -def _new_const_int(pool: memhub.MemManager*, val: int64, - lineno: int, col: int) -> AST*: - return Constant(pool, CONST_INT, val, 0.0, None, lineno, col) - - -def _new_const_float(pool: memhub.MemManager*, val: double, - lineno: int, col: int) -> AST*: - global _g_dbg_pts, _g_dbg_vals, _g_dbg_cnt - new_c: Constant* = Constant(pool, CONST_FLOAT, 0, val, None, lineno, col) - # Store the numeric text in str_val (stdlib allocation, bypassing the pool overwrite of float_val); - # REG-SRC recovers the original value with string.atof(str_val). - lit_txt: str = stdlib.malloc(64) - if lit_txt is not None: - stdio.snprintf(lit_txt, 64, "%.17g", val) - new_c.str_val = lit_txt - # Record float node ptr->val into a stdlib table (value correct at construction; REG-SRC uses the - # table to bypass the pool overwrite). Reset when full: old entries belong to finished files. - if _g_dbg_pts is None: - _g_dbg_pts = stdlib.malloc(4096 * 8) - _g_dbg_vals = stdlib.malloc(4096 * 8) - _g_dbg_cnt = 0 - elif _g_dbg_cnt >= 4096: - _g_dbg_cnt = 0 - if _g_dbg_pts is not None and _g_dbg_vals is not None: - _g_dbg_pts[_g_dbg_cnt] = uint64(new_c) - _g_dbg_vals[_g_dbg_cnt] = val - _g_dbg_cnt += 1 - return new_c - - -# Float-node construction-value table (stdlib allocated to avoid polluting the pool) -_g_dbg_pts: uint64* = None -_g_dbg_vals: double* = None -_g_dbg_cnt: int = 0 -# Whether the latest _dbg_lookup_lit_val hit -_g_dbg_lit_found: int = 0 - - -def _dbg_lookup_lit_val(ptr: uint64) -> double: - """Look up the float-node construction-value table: return the original value (miss returns 0, found=0). - - Used to bypass the pool allocation overwriting the AST float_val field (correct at construction). - """ - global _g_dbg_lit_found - _g_dbg_lit_found = 0 - if _g_dbg_pts is None or _g_dbg_vals is None: - return 0.0 - for i in range(_g_dbg_cnt): - if _g_dbg_pts[i] == ptr: - _g_dbg_lit_found = 1 - return _g_dbg_vals[i] - return 0.0 - - -def _dbg_lit_found() -> int: - """Check whether the latest _dbg_lookup_lit_val hit""" - return _g_dbg_lit_found - - -def _new_const_str(pool: memhub.MemManager*, val: str, - lineno: int, col: int, - is_dquote: int = 1, - is_triple: int = 0, - is_char: int = 0) -> AST*: - """Create a string/character constant node - - is_dquote: 0 = single quote, non-0 = double quote (default double quote) - is_triple: non-0 = triple-quoted string - is_char: non-0 = single-character candidate (lexer has confirmed that after escaping it's exactly 1 byte, including '0') - Semantic distinctions: - - 'x' (single quote, single character, not triple-quoted) → CONST_CHAR (i8 value) - - "x" (double quote, includes 0 terminator) → CONST_STR (i8* pointer, i8 x 2) - - 'ab' (single quotes, multiple characters) → CONST_STR (i8* pointer) - - '''x''' / \"\"\"x\"\"\" (triple quotes) → CONST_STR (i8* pointer) - - '' / "" (empty string) → CONST_STR (i8* pointer) - """ - # Single quotes, non-triple quotes, single-character candidates marked by lexer → CONST_CHAR - # Mark with is_char instead of strlen(val), because '0' becomes a NUL byte after escaping, strlen would return 0 - if is_dquote == 0 and is_triple == 0 and is_char != 0: - # Single quote # Single character → CONST_CHAR, int_val stores the character's ASCII value - # val[0] is a legal value of 0 even if it's NUL - ch_val: int64 = val[0] - return Constant(pool, CONST_CHAR, ch_val, 0.0, val, lineno, col) - # Other situations → CONST_STR - return Constant(pool, CONST_STR, 0, 0.0, val, lineno, col) - - -def _new_const_bool(pool: memhub.MemManager*, val: int, - lineno: int, col: int) -> AST*: - return Constant(pool, CONST_BOOL, val, 0.0, None, lineno, col) - - -def _new_const_none(pool: memhub.MemManager*, - lineno: int, col: int) -> AST*: - return Constant(pool, CONST_NONE, 0, 0.0, None, lineno, col) - - -# ============================================================ -# ctx Helper (Recursive Fixing Store Context) -# ============================================================ - -def _set_ctx(node: AST*, ctx: int): - """Set ctx field of node (based on kind conversion)""" - if node is None: return - k: int = node.kind() - if k == ASTKind.Name: - nm: Name* = (Name*)(node) - nm.ctx = ctx - elif k == ASTKind.Attribute: - at: Attribute* = (Attribute*)(node) - at.ctx = ctx - elif k == ASTKind.Subscript: - sb: Subscript* = (Subscript*)(node) - sb.ctx = ctx - elif k == ASTKind.Tuple: - tp: Tuple* = (Tuple*)(node) - tp.ctx = ctx - elif k == ASTKind.List: - ls: List* = (List*)(node) - ls.ctx = ctx - elif k == ASTKind.Starred: - st: Starred* = (Starred*)(node) - st.ctx = ctx - - -def _fix_store_ctx(node: AST*): - """Fix ctx of store targets to be Store""" - if node is None: return - _set_ctx(node, ASTCtx.Store) - k: int = node.kind() - if k == ASTKind.Tuple or k == ASTKind.List: - tp_elts: list[AST*]* = None - if k == ASTKind.Tuple: - tp: Tuple* = (Tuple*)(node) - tp_elts = tp.elts - else: - ls: List* = (List*)(node) - tp_elts = ls.elts - if tp_elts is not None: - n: sizet = tp_elts.__len__() - for i in range(n): - cur: AST* = tp_elts.get(i) - _fix_store_ctx(cur) - elif k == ASTKind.Starred: - st2: Starred* = (Starred*)(node) - _fix_store_ctx(st2.value) - - -# ============================================================ -# Expression Parser -# ============================================================ - -def _parse_expr(ps: Parser*) -> AST*: - """expr: yield_expr | namedexpr_test""" - if _cur_kw(ps) == Keyword.Yield: - return _parse_yield(ps) - return _parse_namedexpr(ps) - - -def _parse_namedexpr(ps: Parser*) -> AST*: - """namedexpr_test: test [':=' test]""" - left: AST* = _parse_test(ps) - if left is None: return None - if _match_op(ps, TokOp.ColonEq): - _advance(ps) - right: AST* = _parse_test(ps) - node: AST* = NamedExpr(ps.pool, left, right) - _set_pos(node, left.lineno, left.col_offset, 0, 0) - return node - return left - - -def _IsVolatileCallPrefix(ps: Parser*) -> int: - """Test for a value-position `volatile f()` / `volatile S()` prefix, 1 = yes / 0 = no. - - `volatile` is a soft keyword consumed only in value position before a call (Name followed by '('); - its use as an annotation descriptor (`a: volatile int`) or an identifier is unaffected. - """ - if _cur_kw(ps) != Keyword.Volatile: - return 0 - nxt: Token* = ps.cur.next - if nxt is None or nxt.type != TokenType.Name: - return 0 - nxt2: Token* = nxt.next - if nxt2 is None or nxt2.type != TokenType.Op or nxt2.op_subtype != TokOp.LPar: - return 0 - return 1 - - -def _parse_test(ps: Parser*) -> AST*: - """test: or_test ['if' or_test 'else' test] | lambdef""" - if _cur_kw(ps) == Keyword.Lambda: - return _parse_lambda(ps) - # Value-position volatile call prefix: `a: int = volatile f()` / `volatile S()` -> consume volatile and - # mark the call node (result slot accessed volatile, struct writes too; LLVM call has no volatile qualifier). - if _IsVolatileCallPrefix(ps) != 0: - _advance(ps) # consume volatile - vinner: AST* = _parse_test(ps) - if vinner is not None and vinner.kind() == ASTKind.Call: - (Call*(vinner)).IsVolatile = 1 - return vinner - left: AST* = _parse_or_test(ps) - if left is None: return None - if _cur_kw(ps) == Keyword.If: - _advance(ps) - test: AST* = _parse_or_test(ps) - if _cur_kw(ps) == Keyword.Else: - _advance(ps) - orelse: AST* = _parse_test(ps) - node: AST* = IfExp(ps.pool, test, left, orelse) - _set_pos(node, left.lineno, left.col_offset, 0, 0) - return node - ps.error_count += 1 - return left - return left - - -def _parse_lambda(ps: Parser*) -> AST*: - """lambdef: 'lambda' [varargslist] ':' test""" - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - _advance(ps) # lambda - args: AST* = None - if not _match_op(ps, TokOp.Colon): - args = _parse_varargslist(ps, 1) - if not _expect_op(ps, TokOp.Colon): - return None - body: AST* = _parse_test(ps) - node: AST* = Lambda(ps.pool, args, body) - _set_pos(node, lineno, col, 0, 0) - return node - - -def _parse_or_test(ps: Parser*) -> AST*: - """or_test: and_test ('or' and_test)*""" - left: AST* = _parse_and_test(ps) - if left is None: return None - values: list[AST*]* = None - while True: - # Supports both leading `or` ((A\n or B)) and trailing `or` (A or\n B) inside parens: - # each round skips Nl before checking, otherwise A or B\n or C stops on Nl after B. - _skip_nl(ps) - if _cur_kw(ps) != Keyword.Or: - break - if values is None: - values = list[AST*](ps.pool, 8) - values.append(left) - _advance(ps) - _skip_nl(ps) # cross-line inside parens: a or\n b - right: AST* = _parse_and_test(ps) - if right is None: break - values.append(right) - if values is not None: - node: AST* = BoolOp(ps.pool, OpKind.Or, values) - _set_pos(node, left.lineno, left.col_offset, 0, 0) - return node - return left - - -def _parse_and_test(ps: Parser*) -> AST*: - """and_test: not_test ('and' not_test)*""" - left: AST* = _parse_not_test(ps) - if left is None: return None - values: list[AST*]* = None - while True: - # Supports both leading `and` ((A\n and B)) and trailing `and` (A and\n B) inside parens: - # each round skips Nl before checking, otherwise A and B\n and C stops on Nl after B. - _skip_nl(ps) - if _cur_kw(ps) != Keyword.And: - break - if values is None: - values = list[AST*](ps.pool, 8) - values.append(left) - _advance(ps) - _skip_nl(ps) # cross-line inside parens: a and\n b - right: AST* = _parse_not_test(ps) - if right is None: break - values.append(right) - if values is not None: - node: AST* = BoolOp(ps.pool, OpKind.And, values) - _set_pos(node, left.lineno, left.col_offset, 0, 0) - return node - return left - - -def _parse_not_test(ps: Parser*) -> AST*: - """not_test: 'not' not_test | comparison""" - if _cur_kw(ps) == Keyword.Not: - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - _advance(ps) - operand: AST* = _parse_not_test(ps) - node: AST* = UnaryOp(ps.pool, OpKind.Not, operand) - _set_pos(node, lineno, col, 0, 0) - return node - return _parse_comparison(ps) - - -def _parse_comparison(ps: Parser*) -> AST*: - """comparison: bitor_expr (comp_op bitor_expr)* - - Single comparison design: a < b → Compare(left=a, op=Lt, right=b) - Chained comparison a < b < c → BoolOp(And, [Compare(a,<,b), Compare(b,<,c)]) - """ - left: AST* = _parse_bitor(ps) - if left is None: return None - - _skip_nl(ps) # leading comparison operator inside parens: (a\n < b) - op_id: int = _match_comp_op(ps) - if op_id == OpKind.NoneOp: - return left - _advance_comp_op(ps) - _skip_nl(ps) # cross-line inside parens: a ==\n b - right: AST* = _parse_bitor(ps) - # Single comparison: construct ops and comparators list - ops: list[int]* = list[int](ps.pool, 8) - ops.append(op_id) - comparators: list[AST*]* = list[AST*](ps.pool, 8) - comparators.append(right) - first_cmp: AST* = Compare(ps.pool, left, ops, comparators) - _set_pos(first_cmp, left.lineno, left.col_offset, 0, 0) - - # Check chained comparison - _skip_nl(ps) # leading chained comparison inside parens: (a < b\n < c) - next_op: int = _match_comp_op(ps) - if next_op == OpKind.NoneOp: - return first_cmp - - # Single comparison with chained conditions: a < b < c → BoolOp(And, [Compare(a,<,b), Compare(b,<,c), ...]) - values: list[AST*]* = list[AST*](ps.pool, 8) - values.append(first_cmp) - cur_left: AST* = right - while next_op != OpKind.NoneOp: - _advance_comp_op(ps) - _skip_nl(ps) # cross-line chained comparison inside parens - cur_right: AST* = _parse_bitor(ps) - ops2: list[int]* = list[int](ps.pool, 8) - ops2.append(next_op) - comparators2: list[AST*]* = list[AST*](ps.pool, 8) - comparators2.append(cur_right) - cmp: AST* = Compare(ps.pool, cur_left, ops2, comparators2) - _set_pos(cmp, cur_left.lineno, cur_left.col_offset, 0, 0) - values.append(cmp) - cur_left = cur_right - _skip_nl(ps) # leading chained comparison inside parens: (a < b\n < c) - next_op = _match_comp_op(ps) - node: AST* = BoolOp(ps.pool, OpKind.And, values) - _set_pos(node, left.lineno, left.col_offset, 0, 0) - return node - - -def _match_comp_op(ps: Parser*) -> int: - """Check if the current one is a comparison operator, return OP_* or OpKind.NoneOp""" - if _cur_type(ps) == TokenType.Op: - op_id: int = _cur_op(ps) - if op_id == TokOp.EqEq: return OpKind.Eq - if op_id == TokOp.ExclaimEq: return OpKind.Ne - if op_id == TokOp.Less: return OpKind.Lt - if op_id == TokOp.LessEq: return OpKind.Le - if op_id == TokOp.Greater: return OpKind.Gt - if op_id == TokOp.GreaterEq: return OpKind.Ge - if _cur_kw(ps) == Keyword.Is: - if ps.cur.next is not None and ps.cur.next.type == TokenType.Name and ps.cur.next.kw_subtype == Keyword.Not: - return OpKind.IsNot - return OpKind.Is - if _cur_kw(ps) == Keyword.Not: - if ps.cur.next is not None and ps.cur.next.kw_subtype == Keyword.In: - return OpKind.NotIn - if _cur_kw(ps) == Keyword.In: - return OpKind.In - return OpKind.NoneOp - - -def _advance_comp_op(ps: Parser*): - """Consume comparison operator token""" - if _cur_type(ps) == TokenType.Op: - _advance(ps) - return - if _cur_kw(ps) == Keyword.Is: - _advance(ps) - if _cur_kw(ps) == Keyword.Not: - _advance(ps) - return - if _cur_kw(ps) == Keyword.Not: - _advance(ps) - if _cur_kw(ps) == Keyword.In: - _advance(ps) - return - if _cur_kw(ps) == Keyword.In: - _advance(ps) - - -def _parse_bitor(ps: Parser*) -> AST*: - left: AST* = _parse_bitxor(ps) - if left is None: return None - while True: - _skip_nl(ps) # leading '|' inside parens: (a\n | b) - if _match_op(ps, TokOp.VBar) == 0: - break - _advance(ps) - _skip_nl(ps) # cross-line inside parens: a |\n b - right: AST* = _parse_bitxor(ps) - node: AST* = BinOp(ps.pool, left, OpKind.BitOr, right) - _set_pos(node, left.lineno, left.col_offset, 0, 0) - left = node - return left - - -def _parse_bitxor(ps: Parser*) -> AST*: - left: AST* = _parse_bitand(ps) - if left is None: return None - while True: - _skip_nl(ps) # leading '^' inside parens: (a\n ^ b) - if _match_op(ps, TokOp.Caret) == 0: - break - _advance(ps) - _skip_nl(ps) # cross-line inside parens: a ^\n b - right: AST* = _parse_bitand(ps) - node: AST* = BinOp(ps.pool, left, OpKind.BitXor, right) - _set_pos(node, left.lineno, left.col_offset, 0, 0) - left = node - return left - - -def _parse_bitand(ps: Parser*) -> AST*: - left: AST* = _parse_shift(ps) - if left is None: return None - while True: - _skip_nl(ps) # leading '&' inside parens: (a\n & b) - if _match_op(ps, TokOp.Amp) == 0: - break - _advance(ps) - _skip_nl(ps) # cross-line inside parens: a &\n b - right: AST* = _parse_shift(ps) - node: AST* = BinOp(ps.pool, left, OpKind.BitAnd, right) - _set_pos(node, left.lineno, left.col_offset, 0, 0) - left = node - return left - - -def _parse_shift(ps: Parser*) -> AST*: - left: AST* = _parse_arith(ps) - if left is None: return None - while True: - _skip_nl(ps) # leading '<<'/'>>' inside parens: (a\n << b) - if _match_op(ps, TokOp.LtLt) == 0 and _match_op(ps, TokOp.GtGt) == 0: - break - op_token: int = _cur_op(ps) - _advance(ps) - _skip_nl(ps) # cross-line inside parens: a <<\n b - right: AST* = _parse_arith(ps) - ast_op: int = OpKind.LShift - if op_token == TokOp.GtGt: - ast_op = OpKind.RShift - node: AST* = BinOp(ps.pool, left, ast_op, right) - _set_pos(node, left.lineno, left.col_offset, 0, 0) - left = node - return left - - -def _parse_arith(ps: Parser*) -> AST*: - left: AST* = _parse_term(ps) - if left is None: return None - while True: - _skip_nl(ps) # leading '+'/'-' inside parens: (a\n + b) - if _match_op(ps, TokOp.Plus) == 0 and _match_op(ps, TokOp.Minus) == 0: - break - op_token: int = _cur_op(ps) - _advance(ps) - _skip_nl(ps) # cross-line inside parens: a +\n b - right: AST* = _parse_term(ps) - ast_op: int = OpKind.Add - if op_token == TokOp.Minus: - ast_op = OpKind.Sub - node: AST* = BinOp(ps.pool, left, ast_op, right) - _set_pos(node, left.lineno, left.col_offset, 0, 0) - left = node - return left - - -# ============================================================ -# C-style pointer suffix ('*' / '**') -- same source as TransPyC lib/vast _AtPtrSuffix -# ============================================================ -# Start position (line/col) of the last consumed C-style pointer suffix ('*' / '**'). -# _parse_paren matches it against ps.prev to tell the paren-less cast `(T*)x` from a -# plain group like `(a | b)`. Recorded as ints, not as a token pointer: comparing two -# token pointers whose declared pointer levels differ is rejected by the compare check. -_g_ptr_suffix_line: int = 0 -_g_ptr_suffix_col: int = 0 - - -def _ptr_suffix_end_op(op: int) -> int: - """Whether the suffix-end token is a "no right operand" position (']' ',' ')' ':' '='). - - Multiplication/power would lack a right operand there anyway (a pre-existing syntax error), so - treating '*'/'**' as a pointer suffix changes no valid expression (x[a * b] / a ** b unaffected). - """ - if op == TokOp.Rsqb or op == TokOp.Comma or op == TokOp.RPar \ - or op == TokOp.Colon or op == TokOp.Equal: - return 1 - return 0 - - -def _at_ptr_suffix(ps: Parser*, base: AST*) -> int: - """Whether the current '*'/'**' run is a C-style pointer suffix applied to base. - - Rule (same as vast): base is a type atom (Name/Attribute/Subscript) and, skipping consecutive - '*'/'**' and Nl, the next token is ']' / ',' / ')' / ':' / '='. Covers annotation and expression contexts. - """ - if base is None: - return 0 - k: int = base.kind() - if k != ASTKind.Name and k != ASTKind.Attribute and k != ASTKind.Subscript: - return 0 - nxt: Token* = ps.cur - stars: int = 0 - while nxt is not None: - if nxt.type == TokenType.Op \ - and (nxt.op_subtype == TokOp.Star or nxt.op_subtype == TokOp.StarStar): - stars += 1 - nxt = nxt.next - elif nxt.type == TokenType.Nl: - nxt = nxt.next - else: - break - if stars == 0 or nxt is None or nxt.type != TokenType.Op: - return 0 - return _ptr_suffix_end_op(nxt.op_subtype) - - -def _make_ptr_type(ps: Parser*, base: AST*) -> AST*: - """Build one pointer wrapper layer: base | t.CPtr.""" - cptr: AST* = Attribute(ps.pool, Name(ps.pool, "t", ASTCtx.Load), - "CPtr", ASTCtx.Load) - node: AST* = BinOp(ps.pool, base, OpKind.BitOr, cptr) - _set_pos(node, base.lineno, base.col_offset, 0, 0) - return node - - -def _consume_ptr_suffix(ps: Parser*, base: AST*) -> AST*: - """Consume consecutive C-style pointer suffixes ('*' = 1 layer, '**' = 2), expanding into a base | t.CPtr chain.""" - global _g_ptr_suffix_line, _g_ptr_suffix_col - while True: - _skip_nl(ps) - if _match_op(ps, TokOp.StarStar): - _advance(ps) - _g_ptr_suffix_line = ps.prev.lineno - _g_ptr_suffix_col = ps.prev.col_offset - base = _make_ptr_type(ps, base) - base = _make_ptr_type(ps, base) - continue - if _match_op(ps, TokOp.Star): - _advance(ps) - _g_ptr_suffix_line = ps.prev.lineno - _g_ptr_suffix_col = ps.prev.col_offset - base = _make_ptr_type(ps, base) - continue - break - return base - - -def _at_cast_operand(ps: Parser*) -> int: - """Whether the current token may start the operand of a paren-less cast `(T*)x`. - - '(' is deliberately absent: `(T*)(x)` is already handled by the trailer loop as a - cast call, and keeping it there leaves that path's AST untouched. Nl is not - skipped either: a line break after ')' starts a new statement, not an operand. - """ - if _cur_type(ps) == TokenType.Name: - return 1 - if _cur_type(ps) == TokenType.Number: - return 1 - if _cur_type(ps) == TokenType.String: - return 1 - op: int = _cur_op(ps) - if op == TokOp.Lsqb or op == TokOp.LBrace or op == TokOp.Star or op == TokOp.Amp or op == TokOp.Minus or op == TokOp.Plus or op == TokOp.Tilde: - return 1 - return 0 - - -def _at_ptr_suffix_call(ps: Parser*, base: AST*) -> int: - """Whether the current '*'/'**' run is a `T*(x)` pointer-suffix cast call. - - Rule (same as TPC lib/vast _AtPtrSuffixCall): (1) base is a type atom; (2) base and the star run are - adjacent (same line, no whitespace/newline); (3) the run, internally adjacent, is immediately followed - by '('. Required because `a * (b)` is valid multiplication, which a bare star rule would misread. - """ - if base is None: - return 0 - k: int = base.kind() - if k != ASTKind.Name and k != ASTKind.Attribute and k != ASTKind.Subscript: - return 0 - nxt: Token* = ps.cur - pv: Token* = ps.prev - if pv is None or nxt is None: - return 0 - if pv.type == TokenType.Nl or pv.type == TokenType.NewLine: - return 0 - if pv.end_lineno != nxt.lineno or pv.end_col_offset != nxt.col_offset: - return 0 # whitespace/newline between base and '*': not a C-style cast - stars: int = 0 - prev: Token* = None - last: Token* = None - while nxt is not None: - if nxt.type == TokenType.Op \ - and (nxt.op_subtype == TokOp.Star or nxt.op_subtype == TokOp.StarStar): - if prev is not None \ - and (nxt.lineno != prev.end_lineno or nxt.col_offset != prev.end_col_offset): - return 0 # whitespace inside the star run: not adjacent, not a cast - stars += 1 - prev = nxt - last = nxt - nxt = nxt.next - else: - break - if stars == 0 or last is None or nxt is None: - return 0 - if nxt.type != TokenType.Op or nxt.op_subtype != TokOp.LPar: - return 0 - if nxt.lineno != last.end_lineno or nxt.col_offset != last.end_col_offset: - return 0 - return 1 - - -def _consume_ptr_suffix_call(ps: Parser*, base: AST*) -> AST*: - """Parse a recognized `T*(...)` pointer-suffix cast as a call. - - Identical to the existing cast form `(T | t.CPtr)(...)`: a Call (func = the expanded base | t.CPtr - chain, args/keywords = the parenthesized arguments); no second cast representation is introduced. - """ - func: AST* = _consume_ptr_suffix(ps, base) - return _parse_call(ps, func) - - -def _parse_term(ps: Parser*) -> AST*: - left: AST* = _parse_factor(ps) - if left is None: return None - while True: - _skip_nl(ps) # leading '*'/'/'/'//'/'%'/'@' inside parens: (a\n * b) - if _match_op(ps, TokOp.Star) and _at_ptr_suffix(ps, left) != 0: - left = _consume_ptr_suffix(ps, left) # C-style pointer suffix, not multiplication - continue - op_token: int = 0 - if _match_op(ps, TokOp.Star): op_token = TokOp.Star - elif _match_op(ps, TokOp.Slash): op_token = TokOp.Slash - elif _match_op(ps, TokOp.DSlash): op_token = TokOp.DSlash - elif _match_op(ps, TokOp.Percent): op_token = TokOp.Percent - elif _match_op(ps, TokOp.At): op_token = TokOp.At - if op_token == 0: break - _advance(ps) - _skip_nl(ps) # cross-line inside parens: a *\n b - right: AST* = _parse_factor(ps) - ast_op: int = _binop_from_op(op_token) - node: AST* = BinOp(ps.pool, left, ast_op, right) - _set_pos(node, left.lineno, left.col_offset, 0, 0) - left = node - return left - - -def _parse_factor(ps: Parser*) -> AST*: - """factor: '+' factor | '-' factor | '~' factor | '*' factor | '&' factor | power""" - # A leading `*` in argument position was already parsed by _parse_call as a tightly bound Starred - # operand; return it as this level's factor, before the unary test (so `*p + n` => `(*p) + n`). - if ps.starred_seed is not None: - seed: AST* = ps.starred_seed - ps.starred_seed = None - return seed - if (_match_op(ps, TokOp.Plus) or _match_op(ps, TokOp.Minus) or _match_op(ps, TokOp.Tilde) - or _match_op(ps, TokOp.Star) or _match_op(ps, TokOp.Amp)): - op_token: int = _cur_op(ps) - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - _advance(ps) - # Combined unary-minus + integer-literal overflow check: `-` is unary, so a magnitude-valid literal - # may become < -2^63 after negation. Only a unary minus directly on an integer constant is checked here. - if op_token == TokOp.Minus and _cur_type(ps) == TokenType.Number \ - and ps.cur.is_float == 0 \ - and _lex_int_neg_exceeds_i64(_cur_str(ps)) != 0: - _record_neg_literal_overflow(_cur_str(ps), lineno, col) - ps.error_count += 1 - return None - operand: AST* = _parse_factor(ps) - ast_op: int = OpKind.UAdd - if op_token == TokOp.Minus: - ast_op = OpKind.USub - elif op_token == TokOp.Tilde: - ast_op = OpKind.Invert - elif op_token == TokOp.Star: - ast_op = OpKind.Deref - elif op_token == TokOp.Amp: - ast_op = OpKind.AddrOf - node: AST* = UnaryOp(ps.pool, ast_op, operand) - _set_pos(node, lineno, col, 0, 0) - return node - return _parse_power(ps) - - -def _parse_power(ps: Parser*) -> AST*: - """power: await_expr ['**' factor] | atom_expr ['**' factor]""" - base: AST* = _parse_atom_expr(ps) - if base is None: return None - if _match_op(ps, TokOp.StarStar) and _at_ptr_suffix(ps, base) != 0: - return _consume_ptr_suffix(ps, base) # C-style pointer suffix ('**' counts as 2 layers) - if _match_op(ps, TokOp.StarStar): - _advance(ps) - exp: AST* = _parse_factor(ps) - node: AST* = BinOp(ps.pool, base, OpKind.Pow, exp) - _set_pos(node, base.lineno, base.col_offset, 0, 0) - return node - return base - - -def _parse_atom_expr(ps: Parser*) -> AST*: - """atom_expr: ['await'] atom trailer*""" - if _cur_kw(ps) == Keyword.Await: - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - _advance(ps) - inner: AST* = _parse_atom_expr(ps) - node: AST* = Await(ps.pool, inner) - _set_pos(node, lineno, col, 0, 0) - return node - base: AST* = _parse_atom(ps) - if base is None: return None - # trailer loop - while True: - if (_match_op(ps, TokOp.Star) or _match_op(ps, TokOp.StarStar)) \ - and _at_ptr_suffix_call(ps, base) != 0: - # `T*(x)` / `T**(x)` pointer-suffix cast: parsed as a cast call (same shape as - # `(T | t.CPtr)(x)`). Consumed in the trailer loop, so subsequent `.attr` / `[i]` / `(…)` - # are still attached by this same loop. - base = _consume_ptr_suffix_call(ps, base) - elif _match_op(ps, TokOp.Dot): - _advance(ps) - if _cur_type(ps) != TokenType.Name: - ps.error_count += 1 - break - attr_name: str = _cur_str(ps) - attr_lineno: int = _cur_lineno(ps) - attr_col: int = _cur_col(ps) - _advance(ps) - ctx: int = ASTCtx.Load - if _match_op(ps, TokOp.Equal): - ctx = ASTCtx.Store - elif _match_op(ps, TokOp.PlusEq) or _match_op(ps, TokOp.MinusEq): - ctx = ASTCtx.Store - base = Attribute(ps.pool, base, attr_name, ctx) - _set_pos(base, attr_lineno, attr_col, 0, 0) - elif _match_op(ps, TokOp.LPar): - base = _parse_call(ps, base) - elif _match_op(ps, TokOp.Lsqb): - base = _parse_subscript(ps, base) - else: - break - return base - - -def _parse_call(ps: Parser*, func: AST*) -> AST*: - """Parse function call trailer. Both args and keywords are lists.""" - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - _advance(ps) # ( - _skip_nls_and_newlines(ps) # Skip the line break after '(' - args: list[AST*]* = list[AST*](ps.pool, 8) - keywords: list[AST*]* = list[AST*](ps.pool, 8) - - if not _match_op(ps, TokOp.RPar): - while True: - # **expr - if _match_op(ps, TokOp.StarStar): - _advance(ps) - _skip_nls_and_newlines(ps) - val: AST* = _parse_expr(ps) - kw: AST* = KwNode(ps.pool, None, val) - _set_pos(kw, _cur_lineno(ps), _cur_col(ps), 0, 0) - keywords.append(kw) - # *expr - elif _match_op(ps, TokOp.Star): - _advance(ps) - _skip_nls_and_newlines(ps) - # Bind only a unary factor (preserving `*p[0]` = `*(p[0])`), put it on starred_seed and let - # _parse_expr continue the binary ops: `*p + n` => `(*p) + n`, not `*(p + n)`. - seed: AST* = _parse_factor(ps) - if seed is None: - ps.error_count += 1 - break - star: AST* = Starred(ps.pool, seed, ASTCtx.Load) - _set_pos(star, seed.lineno, seed.col_offset, 0, 0) - ps.starred_seed = star - val = _parse_expr(ps) - args.append(val) - # keyword = expr - elif _cur_type(ps) == TokenType.Name: - if ps.cur.next is not None and ps.cur.next.type == TokenType.Op and ps.cur.next.op_subtype == TokOp.Equal: - kw_name: str = _cur_str(ps) - kw_lineno: int = _cur_lineno(ps) - kw_col: int = _cur_col(ps) - _advance(ps) # name - _advance(ps) # = - _skip_nls_and_newlines(ps) - val = _parse_expr(ps) - kw = KwNode(ps.pool, kw_name, val) - _set_pos(kw, kw_lineno, kw_col, 0, 0) - keywords.append(kw) - else: - val = _parse_expr(ps) - args.append(val) - else: - val = _parse_expr(ps) - args.append(val) - _skip_nls_and_newlines(ps) # Skip the line break after the parameter - if not _accept_op(ps, TokOp.Comma): - break - _skip_nls_and_newlines(ps) # Skip the line break after the comma - _skip_nls_and_newlines(ps) # Skip the line break before ')' - _expect_op(ps, TokOp.RPar) - node: AST* = Call(ps.pool, func, args, keywords) - _set_pos(node, lineno, col, 0, 0) - return node - - -def _parse_subscript(ps: Parser*, value: AST*) -> AST*: - """Parse subscript trailer.""" - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - _advance(ps) # [ - _skip_nl(ps) - slice_node: AST* = _parse_slice(ps) - _expect_op(ps, TokOp.Rsqb) - ctx: int = ASTCtx.Load - if _match_op(ps, TokOp.Equal): - ctx = ASTCtx.Store - node: AST* = Subscript(ps.pool, value, slice_node, ctx) - _set_pos(node, lineno, col, 0, 0) - return node - - -def _parse_slice(ps: Parser*) -> AST*: - """Parse slice: [expr] ':' [expr] [':' [expr]] | expr | expr, expr, ...""" - lower: AST* = None - # Empty subscript (`T[]`): `[` directly followed by `]` => slice is None (the type side treats it as - # an empty param list); calling _parse_expr here would report an unexpected token ']'. - if not _match_op(ps, TokOp.Colon) and not _match_op(ps, TokOp.Rsqb): - lower = _parse_expr(ps) - - # Check if there’s a comma (tuple syntax: a[x, y, ...]) - if _match_op(ps, TokOp.Comma): - elts: list[AST*]* = list[AST*](ps.pool, 8) - if lower is not None: - elts.append(lower) - while _accept_op(ps, TokOp.Comma): - if _match_op(ps, TokOp.Rsqb): - break # Trailing comma - elem: AST* = _parse_expr(ps) - if elem is not None: - elts.append(elem) - lineno: int = 0 - if lower is not None: - lineno = lower.lineno - node: AST* = Tuple(ps.pool, elts, ASTCtx.Load) - _set_pos(node, lineno, 0, 0, 0) - return node - - if _match_op(ps, TokOp.Colon): - _advance(ps) - upper: AST* = None - if not _match_op(ps, TokOp.Colon) and not _match_op(ps, TokOp.Rsqb): - upper = _parse_expr(ps) - step: AST* = None - if _accept_op(ps, TokOp.Colon): - if not _match_op(ps, TokOp.Rsqb): - step = _parse_expr(ps) - lineno: int = 0 - if lower is not None: - lineno = lower.lineno - elif upper is not None: - lineno = upper.lineno - else: - lineno = _cur_lineno(ps) - node: AST* = Slice(ps.pool, lower, upper, step) - _set_pos(node, lineno, 0, 0, 0) - return node - return lower - - -def _is_new_expr(ps: Parser*) -> int: - """是否 `new` 描述符表达式（软关键字判定）。 - - `new` 是软关键字：只有紧跟一个**普通名**（描述符 hlife/slife/stack/heap/memgr 或类型名 - `T(args)`）时才当关键字。`new = 1`、`new.x`、`new(1)`、`for new in ...` 都仍是标识符。 - """ - if _is_named_soft(ps, "new") == 0: return 0 - nxt: Token* = ps.cur.next - if nxt is None: return 0 - if nxt.type != TokenType.Name: return 0 - if nxt.kw_subtype != Keyword.NotKeyword: return 0 - return 1 - - -def _parse_new_expr(ps: Parser*) -> AST*: - """`new [Descriptor...] T(args)`：描述符顺序无关、各轴最多一次（冲突即报错）。 - - 结果恒为 `T*`；来源缺省 = heap（D3），生命期缺省由来源决定（语义侧落地）。 - """ - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - _advance(ps) # 吃掉 `new` - src: int = NEW_SRC_DEFAULT - life: int = NEW_LIFE_DEFAULT - memgr_arg: AST* = None - while True: - if _is_named_soft(ps, "hlife") != 0: - if life != NEW_LIFE_DEFAULT: - _record_parse_err(ps) - ps.error_count += 1 - return None - life = NEW_LIFE_HLIFE - _advance(ps) - continue - if _is_named_soft(ps, "slife") != 0: - if life != NEW_LIFE_DEFAULT: - _record_parse_err(ps) - ps.error_count += 1 - return None - life = NEW_LIFE_SLIFE - _advance(ps) - continue - if _is_named_soft(ps, "stack") != 0: - if src != NEW_SRC_DEFAULT: - _record_parse_err(ps) - ps.error_count += 1 - return None - src = NEW_SRC_STACK - _advance(ps) - continue - if _is_named_soft(ps, "heap") != 0: - if src != NEW_SRC_DEFAULT: - _record_parse_err(ps) - ps.error_count += 1 - return None - src = NEW_SRC_HEAP - _advance(ps) - continue - if _is_named_soft(ps, "memgr") != 0: - if src != NEW_SRC_DEFAULT: - _record_parse_err(ps) - ps.error_count += 1 - return None - _advance(ps) # memgr - if _expect_op(ps, TokOp.LPar) == 0: return None - _skip_nl(ps) - memgr_arg = _parse_test(ps) - if memgr_arg is None: return None - if _expect_op(ps, TokOp.RPar) == 0: return None - src = NEW_SRC_MEMGR - continue - break - # 类型 + 可选实参：类型用**注解规则**解析 —— 这样前导数组声明符 `[N]T`、指针后缀 `T*`、 - # 泛型下标 `list[int]`、描述符前缀都与注解位置同口径；`T(args)` 的实参括号会被 `_parse_test` - # 自身吃成 Call，故这里按"是不是 Call"分流，行为与旧版一致。 - callee: AST* = _parse_type_ann(ps) - if callee is None: - _record_parse_err(ps) - ps.error_count += 1 - return None - callee_kind: int = callee.kind() - func_node: AST* = callee - args_node: list[AST*]* = None - if callee_kind == ASTKind.Call: - cl: Call* = Call*(callee) - func_node = cl.func - args_node = cl.args - elif callee_kind != ASTKind.Name and callee_kind != ASTKind.Attribute and callee_kind != ASTKind.Subscript: - _record_parse_err(ps) - ps.error_count += 1 - return None - node: New* = New(ps.pool, src, life, memgr_arg, func_node, args_node) - _set_pos(node, lineno, col, 0, 0) - return node - - -def _parse_atom(ps: Parser*) -> AST*: - """atom: the smallest unit of expression""" - ttype: int = _cur_type(ps) - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - - if ttype == TokenType.Name: - # M2.4 `new` 描述符分配（软关键字）：仅当后面确实跟着描述符/类型调用时才当关键字 - if _is_new_expr(ps) != 0: return _parse_new_expr(ps) - kw: int = _cur_kw(ps) - if kw == Keyword.True_: - _advance(ps) - return _new_const_bool(ps.pool, 1, lineno, col) - if kw == Keyword.False_: - _advance(ps) - return _new_const_bool(ps.pool, 0, lineno, col) - if kw == Keyword.None_: - _advance(ps) - return _new_const_none(ps.pool, lineno, col) - name_str: str = _cur_str(ps) - _advance(ps) - node: AST* = Name(ps.pool, name_str, ASTCtx.Load) - _set_pos(node, lineno, col, 0, 0) - return node - - if ttype == TokenType.Number: - tok: Token* = ps.cur - _advance(ps) - if tok.is_float: - return _new_const_float(ps.pool, tok.float_val, lineno, col) - return _new_const_int(ps.pool, tok.int_val, lineno, col) - - if ttype == TokenType.String: - sval: str = _cur_str(ps) - is_fstr: int = ps.cur.kw_subtype & 4 - is_triple: int = ps.cur.kw_subtype & 8 - is_dquote: int = ps.cur.kw_subtype & 16 - is_char: int = ps.cur.kw_subtype & 32 - _advance(ps) - node: AST* - if is_fstr: - node = _parse_fstring(ps, sval, lineno, col) - else: - node = _new_const_str(ps.pool, sval, lineno, col, is_dquote, is_triple, is_char) - # Python implicit adjacent string literal concatenation: "a" "b" -> "ab". - # Nl is the in-paren newline token (allowed across lines); statement NewLine is not skipped. - while True: - _skip_nl(ps) - if _cur_type(ps) != TokenType.String: - break - n_sval: str = _cur_str(ps) - n_fstr: int = ps.cur.kw_subtype & 4 - n_triple: int = ps.cur.kw_subtype & 8 - n_dquote: int = ps.cur.kw_subtype & 16 - n_char: int = ps.cur.kw_subtype & 32 - _advance(ps) - n_node: AST* - if n_fstr: - n_node = _parse_fstring(ps, n_sval, lineno, col) - else: - n_node = _new_const_str(ps.pool, n_sval, lineno, col, n_dquote, n_triple, n_char) - node = _concat_str_nodes(ps.pool, node, n_node) - return node - - if _match_op(ps, TokOp.Ellipsis): - _advance(ps) - return _new_const_none(ps.pool, lineno, col) - - if _match_op(ps, TokOp.LPar): - return _parse_paren(ps) - if _match_op(ps, TokOp.Lsqb): - return _parse_list(ps) - if _match_op(ps, TokOp.LBrace): - return _parse_brace(ps) - - _record_parse_err(ps) - ps.error_count += 1 - return None - - -def _parse_paren(ps: Parser*) -> AST*: - """Parenthesis expressions: () | (expr) | (expr,) | (expr, ...) | (yield)""" - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - _advance(ps) # ( - _skip_nl(ps) # Skip the line break after '(' (multi-line tuple) - if _match_op(ps, TokOp.RPar): - _advance(ps) - # Empty tuple - empty: list[AST*]* = list[AST*](ps.pool, 8) - node: AST* = Tuple(ps.pool, empty, ASTCtx.Load) - _set_pos(node, lineno, col, 0, 0) - return node - # yield - if _cur_kw(ps) == Keyword.Yield: - y: AST* = _parse_yield(ps) - _expect_op(ps, TokOp.RPar) - return y - first: AST* = _parse_expr(ps) - if _match_op(ps, TokOp.Comma): - # Tuple - elts: list[AST*]* = list[AST*](ps.pool, 8) - elts.append(first) - while _accept_op(ps, TokOp.Comma): - _skip_nl(ps) - if _match_op(ps, TokOp.RPar): - break - e: AST* = _parse_expr(ps) - elts.append(e) - _skip_nl(ps) # finish when the last element has no trailing comma (multi-line tuple) - _expect_op(ps, TokOp.RPar) - node = Tuple(ps.pool, elts, ASTCtx.Load) - _set_pos(node, lineno, col, 0, 0) - return node - # Paren-less C cast `(T*)x`: the inner expression ends with a pointer suffix - # (ps.prev is that suffix's '*'/'**' token) and the operand follows ')' directly. - # Same product as `(T*)(x)` / `T*(x)`: a Call whose func is `T | t.CPtr`. - is_ptr_cast: int = 0 - if _g_ptr_suffix_line > 0 and _match_op(ps, TokOp.RPar) and ps.prev.lineno == _g_ptr_suffix_line and ps.prev.col_offset == _g_ptr_suffix_col: - is_ptr_cast = 1 - _skip_nl(ps) # (expr\n) close a single-expression paren spanning lines - _expect_op(ps, TokOp.RPar) - # Check for generator expression - if _cur_kw(ps) == Keyword.For: - return _parse_comprehension(ps, first, ASTKind.GeneratorExp, lineno, col) - if is_ptr_cast != 0 and _at_cast_operand(ps) != 0: - operand: AST* = _parse_factor(ps) - cast_args: list[AST*]* = list[AST*](ps.pool, 8) - cast_args.append(operand) - cast_kws: list[AST*]* = list[AST*](ps.pool, 8) - cast_node: AST* = Call(ps.pool, first, cast_args, cast_kws) - _set_pos(cast_node, first.lineno, first.col_offset, 0, 0) - return cast_node - return first - - -def _parse_list(ps: Parser*) -> AST*: - """List expressions: [] | [expr_list] | [expr_list comp_for]""" - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - _advance(ps) # [ - _skip_nl(ps) # Skip the line break after '[' (multi-line list) - if _match_op(ps, TokOp.Rsqb): - _advance(ps) - empty: list[AST*]* = list[AST*](ps.pool, 8) - node: AST* = List(ps.pool, empty, ASTCtx.Load) - _set_pos(node, lineno, col, 0, 0) - return node - first: AST* = _parse_expr(ps) - if _cur_kw(ps) == Keyword.For: - node = _parse_comprehension(ps, first, ASTKind.ListComp, lineno, col) - _expect_op(ps, TokOp.Rsqb) - return node - elts: list[AST*]* = list[AST*](ps.pool, 8) - elts.append(first) - while _accept_op(ps, TokOp.Comma): - _skip_nl(ps) - if _match_op(ps, TokOp.Rsqb): - break - e: AST* = _parse_expr(ps) - elts.append(e) - _skip_nl(ps) # finish when the last element has no trailing comma (multi-line list) - _expect_op(ps, TokOp.Rsqb) - node = List(ps.pool, elts, ASTCtx.Load) - _set_pos(node, lineno, col, 0, 0) - return node - - -def _parse_brace(ps: Parser*) -> AST*: - """Brace expressions: {} | {set_items} | {dict_items} | {comp_for}""" - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - _advance(ps) # { - _skip_nl(ps) # Skip the line break after '{' (multi-line dict) - if _match_op(ps, TokOp.RBrace): - _advance(ps) - empty_k: list[AST*]* = list[AST*](ps.pool, 8) - empty_v: list[AST*]* = list[AST*](ps.pool, 8) - node: AST* = Dict(ps.pool, empty_k, empty_v) - _set_pos(node, lineno, col, 0, 0) - return node - first: AST* = _parse_expr(ps) - # dict: key: value - if _match_op(ps, TokOp.Colon): - _advance(ps) - val: AST* = _parse_expr(ps) - if _cur_kw(ps) == Keyword.For: - # dict comprehension - gens: list[AST*]* = _parse_comp_for(ps) - node = DictComp(ps.pool, first, val, gens) - _set_pos(node, lineno, col, 0, 0) - _expect_op(ps, TokOp.RBrace) - return node - keys: list[AST*]* = list[AST*](ps.pool, 8) - values: list[AST*]* = list[AST*](ps.pool, 8) - keys.append(first) - values.append(val) - while _accept_op(ps, TokOp.Comma): - _skip_nl(ps) - if _match_op(ps, TokOp.RBrace): - break - k: AST* = _parse_expr(ps) - _expect_op(ps, TokOp.Colon) - v: AST* = _parse_expr(ps) - keys.append(k) - values.append(v) - _skip_nl(ps) # finish when the last element has no trailing comma - _expect_op(ps, TokOp.RBrace) - node = Dict(ps.pool, keys, values) - _set_pos(node, lineno, col, 0, 0) - return node - # set - if _cur_kw(ps) == Keyword.For: - node = _parse_comprehension(ps, first, ASTKind.SetComp, lineno, col) - _expect_op(ps, TokOp.RBrace) - return node - elts: list[AST*]* = list[AST*](ps.pool, 8) - elts.append(first) - while _accept_op(ps, TokOp.Comma): - _skip_nl(ps) # cross-line set literal inside braces - if _match_op(ps, TokOp.RBrace): - break - e: AST* = _parse_expr(ps) - elts.append(e) - _skip_nl(ps) # finish when the last element has no trailing comma - _expect_op(ps, TokOp.RBrace) - node = Set(ps.pool, elts) - _set_pos(node, lineno, col, 0, 0) - return node - - -# ============================================================ -# f-string parsing -# ============================================================ - -def _parse_sub_expr(ps: Parser*, src: str) -> AST*: - """Create a sub-lexer and parser to parse expressions (used for {expr} inside f-strings)""" - pool: memhub.MemManager* = ps.pool - lx: Lexer* = new_lexer(pool) - if lx is None: return None - _lexer_init(lx, src, pool) - tokens: Token* = tokenize(lx) - if tokens is None: return None - sub_ps: Parser* = new_parser(pool) - if sub_ps is None: return None - _parser_init(sub_ps, tokens, pool) - return _parse_test(sub_ps) - - -def _append_str_value(pool: memhub.MemManager*, - values: list[AST*]*, - node: AST*): - """Flatten a string node into values (JoinedStr expands its values, everything else is appended)""" - if node is None: - return - if node.kind() == ASTKind.JoinedStr: - js_val: JoinedStr* = (JoinedStr*)(node) - if js_val.values is not None: - n_v: sizet = js_val.values.__len__() - i_v: sizet - for i_v in range(n_v): - values.append(js_val.values.get(i_v)) - else: - values.append(node) - - -def _concat_str_nodes(pool: memhub.MemManager*, - a: AST*, b: AST*) -> AST*: - """Merge adjacent string literal nodes (Python implicit concatenation "a" "b" -> "ab"). - - Pure constant strings are concatenated as text; f-strings (JoinedStr/FormattedValue) are merged - into a JoinedStr values list. - """ - if a is None: - return b - if b is None: - return a - # Both are constant strings -> concatenate the text directly - if a.kind() == ASTKind.Constant and b.kind() == ASTKind.Constant: - ca_c: Constant* = (Constant*)(a) - cb_c: Constant* = (Constant*)(b) - if ca_c.const_kind == CONST_STR and cb_c.const_kind == CONST_STR \ - and ca_c.str_val is not None and cb_c.str_val is not None: - la_c: sizet = string.strlen(ca_c.str_val) - lb_c: sizet = string.strlen(cb_c.str_val) - buf_c: str = pool.alloc(la_c + lb_c + 1) - if buf_c is not None: - string.memcpy(buf_c, ca_c.str_val, la_c) - string.memcpy(buf_c + la_c, cb_c.str_val, lb_c + 1) - return _new_const_str(pool, buf_c, a.lineno, a.col_offset, 0, 0, 0) - # Involves an f-string -> merge into a JoinedStr - values_c: list[AST*]* = list[AST*](pool, 8) - _append_str_value(pool, values_c, a) - _append_str_value(pool, values_c, b) - if values_c.__len__() == 1: - return values_c.get(0) - node_c: AST* = JoinedStr(pool, values_c) - _set_pos(node_c, a.lineno, a.col_offset, 0, 0) - return node_c - - -def _parse_fstring(ps: Parser*, content: str, - lineno: int, col: int) -> AST*: - """Parse f-string content, building JoinedStr or FormattedValue nodes""" - pool: memhub.MemManager* = ps.pool - values: list[AST*]* = list[AST*](pool, 8) - i: sizet = 0 - content_len: sizet = string.strlen(content) - # Literal text buffer for f-string content - lit_buf: str = pool.alloc(content_len + 1) - if lit_buf is None: return None - lit_pos: sizet = 0 - - while i < content_len: - ch: char = content[i] - if ch == '{': - # {{ → Literal { - if i + 1 < content_len and content[i + 1] == '{': - lit_buf[lit_pos] = '{' - lit_pos += 1 - i += 2 - continue - # Refresh literal buffer - if lit_pos > 0: - lit_buf[lit_pos] = '\0' - lit_node: AST* = _new_const_str(pool, lit_buf, lineno, col) - values.append(lit_node) - lit_buf = pool.alloc(content_len + 1) - if lit_buf is None: break - lit_pos = 0 - # Find matching closing '}' while tracking nested '{}' and string literals inside the f-string - j: sizet = i + 1 - depth: int = 1 - conv_pos: sizet = 0 - fmt_pos: sizet = 0 - while j < content_len and depth > 0: - c: char = content[j] - if c == '\'' or c == '"': - # Skip nested string literals inside an expression block (PEP 701 same-quote nesting) - # so their characters do not confuse the {} depth / conversion / format delimiter scan - nq: char = c - j += 1 - while j < content_len: - c2: char = content[j] - if c2 == '\\': - j += 2 - continue - if c2 == nq: - j += 1 - break - j += 1 - continue - if c == '{': - depth += 1 - elif c == '}': - depth -= 1 - if depth == 0: - break - elif depth == 1: - if c == '!' and conv_pos == 0 and fmt_pos == 0: - if j + 1 < content_len: - nc: char = content[j + 1] - if nc == 's' or nc == 'r' or nc == 'a': - if j + 2 < content_len: - after: char = content[j + 2] - if after == ':' or after == '}': - conv_pos = j - elif j + 2 == content_len: - conv_pos = j - elif c == ':' and fmt_pos == 0: - fmt_pos = j - j += 1 - if depth != 0: - _record_parse_err(ps) - ps.error_count += 1 - break - # Extract expression text - expr_end: sizet = j - if conv_pos > 0: - expr_end = conv_pos - elif fmt_pos > 0: - expr_end = fmt_pos - expr_len: sizet = expr_end - i - 1 - if expr_len == 0: - _record_parse_err(ps) - ps.error_count += 1 - i = j + 1 - continue - expr_buf: str = pool.alloc(expr_len + 1) - if expr_buf is None: break - string.memcpy(expr_buf, content + i + 1, expr_len) - expr_buf[expr_len] = '\0' - # Parse expression text - expr_node: AST* = _parse_sub_expr(ps, expr_buf) - # Conversion specifier - conversion: int = -1 - if conv_pos > 0: - conv_char: char = content[conv_pos + 1] - if conv_char == 's': - conversion = 115 - elif conv_char == 'r': - conversion = 114 - elif conv_char == 'a': - conversion = 97 - # Format specifier - format_spec: AST* = None - if fmt_pos > 0: - spec_len: sizet = j - fmt_pos - 1 - if spec_len > 0: - spec_buf: str = pool.alloc(spec_len + 1) - if spec_buf is not None: - string.memcpy(spec_buf, content + fmt_pos + 1, spec_len) - spec_buf[spec_len] = '\0' - format_spec = _new_const_str(pool, spec_buf, lineno, col) - fv: AST* = FormattedValue(pool, expr_node, conversion, format_spec) - _set_pos(fv, lineno, col, 0, 0) - values.append(fv) - i = j + 1 - elif ch == '}': - if i + 1 < content_len and content[i + 1] == '}': - lit_buf[lit_pos] = '}' - lit_pos += 1 - i += 2 - else: - _record_parse_err(ps) - ps.error_count += 1 - i += 1 - else: - lit_buf[lit_pos] = ch - lit_pos += 1 - i += 1 - - # Refresh remaining literal buffer - if lit_pos > 0: - lit_buf[lit_pos] = '\0' - lit_node2: AST* = _new_const_str(pool, lit_buf, lineno, col) - values.append(lit_node2) - - # If no content, return empty string - if values.__len__() == 0: - return _new_const_str(pool, "", lineno, col) - # If only one FormattedValue and no literal, return FormattedValue - if values.__len__() == 1: - first: AST* = values.get(0) - if first.kind() == ASTKind.FormattedValue: - return first - node: AST* = JoinedStr(pool, values) - _set_pos(node, lineno, col, 0, 0) - return node - - -def _parse_comprehension(ps: Parser*, elt: AST*, - vtype: int, lineno: int, col: int) -> AST*: - """General comprehension: elt comp_for""" - gens: list[AST*]* = _parse_comp_for(ps) - node: AST* = None - if vtype == ASTKind.ListComp: - node = ListComp(ps.pool, elt, gens) - elif vtype == ASTKind.SetComp: - node = SetComp(ps.pool, elt, gens) - else: - node = GeneratorExp(ps.pool, elt, gens) - _set_pos(node, lineno, col, 0, 0) - return node - - -def _parse_comp_for(ps: Parser*) -> list[AST*]*: - """comp_for: ['async'] 'for' exprlist 'in' or_test [comp_if] (comp_for)* - Returns generators list.""" - gens: list[AST*]* = list[AST*](ps.pool, 8) - while True: - is_async: int = 0 - if _cur_kw(ps) == Keyword.Async: - is_async = 1 - _advance(ps) - if _cur_kw(ps) != Keyword.For: - break - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - _advance(ps) - target: AST* = _parse_target_list(ps) - if _cur_kw(ps) != Keyword.In: - ps.error_count += 1 - break - _advance(ps) - iter_: AST* = _parse_or_test(ps) - # Collect ifs to list - ifs: list[AST*]* = list[AST*](ps.pool, 8) - while _cur_kw(ps) == Keyword.If: - _advance(ps) - cond: AST* = _parse_or_test(ps) - ifs.append(cond) - comp: AST* = Comprehension(ps.pool, target, iter_, ifs, is_async) - _set_pos(comp, lineno, col, 0, 0) - gens.append(comp) - return gens - - -def _parse_target_list(ps: Parser*) -> AST*: - """Parse assignment target list (comma-separated)""" - first: AST* = _parse_target(ps) - if _match_op(ps, TokOp.Comma): - elts: list[AST*]* = list[AST*](ps.pool, 8) - elts.append(first) - while _accept_op(ps, TokOp.Comma): - if _cur_kw(ps) == Keyword.In: - break - e: AST* = _parse_target(ps) - elts.append(e) - node: AST* = Tuple(ps.pool, elts, ASTCtx.Store) - _set_pos(node, first.lineno, first.col_offset, 0, 0) - return node - return first - - -def _parse_target(ps: Parser*) -> AST*: - """Parse assignment target""" - if _match_op(ps, TokOp.Star): - _advance(ps) - inner: AST* = _parse_atom_expr(ps) - node: AST* = Starred(ps.pool, inner, ASTCtx.Store) - _set_pos(node, inner.lineno, inner.col_offset, 0, 0) - return node - if _match_op(ps, TokOp.LPar): - _advance(ps) - _skip_nl(ps) - t_: AST* = _parse_target_list(ps) - _skip_nl(ps) - _expect_op(ps, TokOp.RPar) - return t_ - if _match_op(ps, TokOp.Lsqb): - _advance(ps) - _skip_nl(ps) - first: AST* = _parse_target(ps) - elts: list[AST*]* = list[AST*](ps.pool, 8) - elts.append(first) - while _accept_op(ps, TokOp.Comma): - _skip_nl(ps) - if _match_op(ps, TokOp.Rsqb): - break - e: AST* = _parse_target(ps) - elts.append(e) - _expect_op(ps, TokOp.Rsqb) - node = List(ps.pool, elts, ASTCtx.Store) - _set_pos(node, first.lineno, first.col_offset, 0, 0) - return node - # Normal atom_expr, ctx=Store - node = _parse_atom_expr(ps) - if node is not None: - _set_ctx(node, ASTCtx.Store) - return node - - -def _parse_yield(ps: Parser*) -> AST*: - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - _advance(ps) # yield - if _cur_kw(ps) == Keyword.From: - _advance(ps) - val: AST* = _parse_expr(ps) - node: AST* = YieldFrom(ps.pool, val) - _set_pos(node, lineno, col, 0, 0) - return node - val2: AST* = None - if not _match_op(ps, TokOp.RPar) and _cur_type(ps) != TokenType.NewLine and _cur_type(ps) != TokenType.EndMarker: - if _cur_type(ps) == TokenType.Op and (_cur_op(ps) == TokOp.Comma or _cur_op(ps) == TokOp.Rsqb): - pass - else: - val2 = _parse_test(ps) - node = Yield(ps.pool, val2) - _set_pos(node, lineno, col, 0, 0) - return node - - -def _parse_varargslist(ps: Parser*, is_lambda: int = 0) -> AST*: - """Parse function parameter list - New API: args/defaults/kw_defaults are lists, vararg/kwarg are fields.""" - args: list[AST*]* = list[AST*](ps.pool, 8) - vararg: AST* = None - kwarg: AST* = None - defaults: list[AST*]* = list[AST*](ps.pool, 8) - kw_defaults: list[AST*]* = list[AST*](ps.pool, 8) - - while True: - _skip_nl(ps) - if _match_op(ps, TokOp.RPar): - break - # *args or * (kwonly separated) - if _match_op(ps, TokOp.Star): - _advance(ps) - _skip_nl(ps) - if _cur_type(ps) == TokenType.Name: - vararg = _parse_arg_def(ps, is_lambda) - if not _accept_op(ps, TokOp.Comma): - break - _skip_nl(ps) - continue - # **kwargs - if _match_op(ps, TokOp.StarStar): - _advance(ps) - _skip_nl(ps) - kwarg = _parse_arg_def(ps, is_lambda) - if not _accept_op(ps, TokOp.Comma): - break - _skip_nl(ps) - continue - # / Delimiter (posonly end)— New API: posonly/regular are not distinguished - if _match_op(ps, TokOp.Slash): - _advance(ps) - if not _accept_op(ps, TokOp.Comma): - break - _skip_nl(ps) - continue - # Normal parameter - a: AST* = _parse_arg_def(ps, is_lambda) - if a is None: break - args.append(a) - if _match_op(ps, TokOp.Equal): - _advance(ps) - d: AST* = _parse_test(ps) - defaults.append(d) - if not _accept_op(ps, TokOp.Comma): - break - _skip_nl(ps) - return Arguments(ps.pool, args, vararg, kwarg, defaults, kw_defaults) - - -def _parse_array_prefix_declarator(ps: Parser*) -> AST*: - """C-style leading array declarator `[N]T` / `[]T` -> `t.CArray[T, N]` / `t.CArray[T, None]`. - - Consumed only in annotation position (_parse_type_ann); returns None if the current token is not '['. - Desugars to the canonical array form shared with TPC / StubGen / DeclarationGenerator. Must be called - before _parse_test, otherwise `[N]` is parsed as a list literal and the element type then errors. - """ - if _match_op(ps, TokOp.Lsqb) == 0: - return None - lb_lineno: int = _cur_lineno(ps) - lb_col: int = _cur_col(ps) - _advance(ps) # consume '[' - count_node: AST* = None - if _match_op(ps, TokOp.Rsqb) == 0: - # Length expression (constant/constant expression; invalid lengths are rejected downstream) - count_node = _parse_test(ps) - if count_node is None: - _record_parse_err(ps) - return None - if _match_op(ps, TokOp.Rsqb) == 0: - _record_parse_err(ps) # missing ']' - ps.error_count += 1 - return None - _advance(ps) # consume ']' - # Element type: recurse into _parse_type_ann, so `[N]const int` / `[N]int*` share one rule source - elem: AST* = _parse_type_ann(ps) - if elem is None: - return None - if count_node is None: - count_node = _new_const_none(ps.pool, lb_lineno, lb_col) - # Element size must be explicitly 8 (list[T](pool, elem_size) addresses by byte stride; passing 2 - # would write 8-byte pointers with a 2-byte stride -> heap overflow). Same arg as other list sites. - elts: list[AST*]* = list[AST*](ps.pool, 8) - elts.append(elem) - elts.append(count_node) - carray: AST* = Subscript(ps.pool, - Attribute(ps.pool, Name(ps.pool, "t", ASTCtx.Load), - "CArray", ASTCtx.Load), - Tuple(ps.pool, elts, ASTCtx.Load), - ASTCtx.Load) - _set_pos(carray, lb_lineno, lb_col, 0, 0) - return carray - - -def _IsCTypedefAnnotation(ann: AST*) -> int: - """1 if the annotation is a typedef marker (bare CTypedef / t.CTypedef / a union containing it). - - Used to parse a typedef's right-hand side as a **type**: `X: t.CTypedef = const uint16*` - synthesizes descriptors and the `*` suffix through _parse_type_ann (the annotation rule), - so both sides agree. TPC's vast Parser._IsCTypedefAnnotation uses the same name-based test. - """ - if ann is None: return 0 - kd: int = ann.kind() - if kd == ASTKind.Attribute: - at: Attribute* = Attribute*(ann) - if at.attr is not None and string.strcmp(at.attr, "CTypedef") == 0: return 1 - return 0 - if kd == ASTKind.Name: - nm: Name* = Name*(ann) - if nm.id is not None and string.strcmp(nm.id, "CTypedef") == 0: return 1 - return 0 - if kd == ASTKind.BinOp: - bo: BinOp* = BinOp*(ann) - if bo.left is not None and _IsCTypedefAnnotation(bo.left) != 0: return 1 - if bo.right is not None and _IsCTypedefAnnotation(bo.right) != 0: return 1 - return 0 - return 0 - - -def _parse_type_ann(ps: Parser*) -> AST*: - """Parse a type annotation: descriptor prefixes (const / volatile / __attribute__) and C-style pointer - suffixes (* / **), field-identical to TPC's expansion. `static` is invalid here; missing type / non-type - follower / bare use fail loudly. Reclaims the Mult/Pow junk _parse_test leaves and rebuilds t.CPtr chains. - """ - # Value-reference parameter prefix: `&T` (ref mode) / `&temp T` (temp mode). Recorded as a t.CRef / - # t.CTempRef union member so the Gen layer can register call-site auto-address. This is the annotation - # position only; the expression-position `&x` (AddrOf) stays in _parse_factor and is left untouched. - # 回收伪错（见下 ptr_level 分支）时要连**位置/消息**一起回退，故这里读写解析错误通道。 - global _g_parse_err_line, _g_parse_err_col, _g_parse_err_msg - ref_marker: str = None - if _accept_op(ps, TokOp.Amp) != 0: - ref_marker = "CRef" - if _cur_type(ps) == TokenType.Name: - nm_ref: str = _cur_str(ps) - if nm_ref is not None and string.strcmp(nm_ref, "temp") == 0: - # `temp` is a soft keyword only when a type name follows (e.g. `&temp int`). - nxt_ref: Token* = ps.cur.next - if nxt_ref is not None and nxt_ref.type == TokenType.Name: - _advance(ps) - ref_marker = "CTempRef" - # Descriptor prefix: collect const / volatile markers (soft keywords, consumed only here) - mods: list[AST*]* = None - mod_count: int = 0 - mod_lineno: int = _cur_lineno(ps) - mod_col: int = _cur_col(ps) - # `static` modifier (consumed at statement start): append t.CStatic as the first descriptor marker - # (pasted on the union's left later), then clear it so it applies only to this annotation. - if ps.pending_static != 0: - ps.pending_static = 0 - if mods is None: - mods = list[AST*](ps.pool, 8) - st_mk: AST* = _MakeStaticMarker(ps) - if st_mk is not None: - mods.append(st_mk) - mod_count += 1 - while _IsDescriptorKeyword(ps) != 0 or _IsMagicAttributeCall(ps) != 0 \ - or _IsStaticDescriptorKeyword(ps) != 0: - if _IsStaticDescriptorKeyword(ps) != 0: - # Annotation-position `static` (`name: static T`) is no longer supported: storage-class markers - # are always prefixed. Fail loudly here (otherwise `static` becomes an opaque "static" class). - _RecordDescriptorErr("static in annotation position is no longer supported: use the prefix form" - " `static name: type = value` (equivalent to `name: t.CStatic | type`)", - _cur_lineno(ps), _cur_col(ps)) - _advance(ps) - continue - if mods is None: - # Element size is explicitly 8: list[T](pool, elem_size) addresses by byte stride, so passing 4 - # would make the 2nd 8-byte pointer (e.g. for `const volatile`) write out of bounds. - mods = list[AST*](ps.pool, 8) - if _IsMagicAttributeCall(ps) != 0: - # C-style `a: __attribute__(section(".text.wo"), aligned(16)) const int = ...`: same position - # and order-free mixing as const / volatile; each argument desugars to `t.attr.` and joins - # the same markers list. - # Use only _parse_call for this call itself, never _parse_test: the latter is a general expression - # parser and would swallow the following `[]T` as a legal empty subscript, silently degrading the - # annotation to a bare element type with zero errors reported. - mka_lineno: int = _cur_lineno(ps) - mka_col: int = _cur_col(ps) - mka_fn: AST* = Name(ps.pool, "__attribute__", ASTCtx.Load) - _set_pos(mka_fn, mka_lineno, mka_col, 0, 0) - _advance(ps) # consume `__attribute__` - ac_node: AST* = _parse_call(ps, mka_fn) # parse only `( args )` - if ac_node is None: - break - if ac_node.kind() != ASTKind.Call: - break - ac: Call* = Call*(ac_node) - if ac.args is not None: - acn: sizet = len(ac.args) - for aci in range(acn): - mk: AST* = _MakeTAttrMarker(ps, ac.args.get(aci)) - if mk is not None: - mods.append(mk) - mod_count += 1 - continue - mods.append(_MakeDescriptorMarker(ps)) - mod_count += 1 - _advance(ps) - # C-style leading array declarator `[N]T` / `[]T` (incl. `const [N]T`): desugar in place to - # `t.CArray[T, N]` / `t.CArray[T, None]` and paste the already-collected descriptors onto it. - if _match_op(ps, TokOp.Lsqb) != 0: - arr: AST* = _parse_array_prefix_declarator(ps) - if arr is None: - return None - if mod_count > 0: - arr = _PrependDescriptorMarkers(ps, arr, mods, mod_count) - return arr - if mod_count > 0 and _cur_type(ps) != TokenType.Name: - # Missing type after a descriptor (`a: const` / `a: const = 1` / `def f(a: const)`): - # fail loudly -- never silently drop the descriptor. - _RecordDescriptorErr("missing type after descriptor", mod_lineno, mod_col) - ps.error_count += 1 - return None - before_ec: int = ps.error_count - # 错误记录快照：_parse_test 会把"末尾 '*' 无右操作数"记成一处语法错误（见下方 ptr_level 回收）， - # 回收时连位置/消息一起回退，否则后面真正的语法错误会报出这个**过期**位置。 - err_line_before: int = _g_parse_err_line - err_col_before: int = _g_parse_err_col - err_msg_before: str = _g_parse_err_msg - base: AST* = _parse_test(ps) - if base is None: - if mod_count > 0: - # Descriptor consumed but type parsing failed (e.g. `a: const = 1`): fail loudly keeping the descriptor error. - _RecordDescriptorErr("missing type after descriptor (type parse failed)", mod_lineno, mod_col) - _record_parse_err(ps) # annotation parse failed: record position (error_count already bumped in _parse_test) - return None - if mod_count > 0 and base.kind() == ASTKind.Constant: - # Non-type follower (`a: const 1` / `a: const "x"` / `a: const True`): fail loudly, since - # descriptors modify types only and constants are not types -- never silently degrade or drop. - _RecordDescriptorErr("descriptor followed by a non-type (descriptors modify types only)", mod_lineno, mod_col) - ps.error_count += 1 - return None - ptr_level: int = 0 - # Reclaim the junk nodes left by _parse_test consuming the * / ** suffixes - if base.kind() == ASTKind.BinOp: - gb: BinOp* = (BinOp*)(base) - if gb.op == OpKind.Mult and gb.right is None: - ptr_level += 1 - base = gb.left - elif gb.op == OpKind.Pow: - if gb.right is None: - ptr_level += 2 - base = gb.left - elif gb.right.kind() == ASTKind.UnaryOp: - gu: UnaryOp* = (UnaryOp*)(gb.right) - if gu.op == OpKind.Deref and gu.operand is None: - ptr_level += 3 - base = gb.left - if ptr_level > 0: - ps.error_count = before_ec # reclaim succeeded: restore error count to avoid whole-file parse failure - # 位置/消息同样要回退：这次回收解决的只有刚记下的那一处伪错（末尾 '*' 被当成乘法、右操作数缺失）。 - # 只回退计数不回退位置，过期位置就会顶掉后面真正的语法错误（现场：EscapeTest 第 13 行 - # `new stack char` 被报成第 6 行 `a: char*` 的 'unexpected token (none)'，排查方向被完全带偏）。 - # 判据用"记录是否在本次解析中改变"：若快照本来就非 0（更早的真错误），当前值未变，回退是空操作。 - if _g_parse_err_line != err_line_before: - _g_parse_err_line = err_line_before - _g_parse_err_col = err_col_before - _g_parse_err_msg = err_msg_before - # Consume remaining suffixes ('**' is one token counting as 2 layers, '*' as 1) - while _match_op(ps, TokOp.StarStar): - _advance(ps) - ptr_level += 2 - while _match_op(ps, TokOp.Star): - _advance(ps) - ptr_level += 1 - # Build n nested base | t.CPtr layers - for li_an in range(ptr_level): - cptr_an: AST* = Attribute(ps.pool, - Name(ps.pool, "t", ASTCtx.Load), - "CPtr", ASTCtx.Load) - base = BinOp(ps.pool, base, OpKind.BitOr, cptr_an) - # Paste descriptors last (same shape as TPC text expansion; see _PrependDescriptorMarkers) - if mod_count > 0: - base = _PrependDescriptorMarkers(ps, base, mods, mod_count) - # Append the value-reference marker as the outermost union member: `base | t.CRef` / `base | t.CTempRef`. - if ref_marker is not None: - ref_marker_node: AST* = Attribute(ps.pool, - Name(ps.pool, "t", ASTCtx.Load), - ref_marker, ASTCtx.Load) - base = BinOp(ps.pool, base, OpKind.BitOr, ref_marker_node) - return base - - -def _parse_arg_def(ps: Parser*, is_lambda: int = 0) -> AST*: - """arg: TokenType.Name [':' annotation]""" - if _cur_type(ps) != TokenType.Name: - ps.error_count += 1 - return None - name_str: str = _cur_str(ps) - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - _advance(ps) - annotation: AST* = None - if is_lambda == 0: - if _accept_op(ps, TokOp.Colon): - annotation = _parse_type_ann(ps) - result: AST* = Arg(ps.pool, name_str, annotation) - _set_pos(result, lineno, col, 0, 0) - return result - - -# ============================================================ -# Statement parsing (Design: directly append to parent, do not return a list) -# ============================================================ - -def _parse_suite_into_children(ps: Parser*, parent: AST*, - owner_is_class: int = 0): - """Parse statement suite, directly append to parent's children. - suite: TokenType.NewLine TokenType.Indent stmt+ TokenType.Dedent | simple_stmt - - owner_is_class: whether this suite's direct owner is a ClassDef. During suite parsing ps.in_class_body - is set so _ParseVirtualModified can tell if `virtual def`/`novirtual def` is directly in a class body - (function/branch/loop bodies do not count). Default 0 keeps all non-class owners correct. - """ - prev_in_class: int = ps.in_class_body - ps.in_class_body = owner_is_class - _skip_nl(ps) - if _cur_type(ps) == TokenType.NewLine: - _advance(ps) - _skip_nl(ps) - if _cur_type(ps) != TokenType.Indent: - ps.error_count += 1 - ps.in_class_body = prev_in_class - return - _advance(ps) - sidx: int = 0 - while _cur_type(ps) != TokenType.Dedent and _cur_type(ps) != TokenType.EndMarker: - sidx += 1 - # fail-fast anti-hang: over-budget steps (cyclic token list / cursor spinning) report and return - if ps.step_count > ps.step_limit: - _record_parse_err(ps) - return - # Decorator (class body methods can have decorators) - if _match_op(ps, TokOp.At): - ps.pending_decorators = _parse_decorators(ps) - before_cur: Token* = ps.cur - _parse_statement_into(ps, parent) - # Defensive: if parse did not advance token, force advance to avoid infinite loop - if ps.cur is before_cur: - ps.error_count += 1 - _advance(ps) - _skip_nl(ps) - _expect_type(ps, TokenType.Dedent) - ps.in_class_body = prev_in_class - return - # Single-line suite - _parse_simple_stmt_into(ps, parent) - ps.in_class_body = prev_in_class - - -def _parse_statement_into(ps: Parser*, parent: AST*): - """Parse statement (compound or simple), append to the parent.""" - ttype: int = _cur_type(ps) - kw: int = _cur_kw(ps) - if ttype == TokenType.Name: - # cif: compile-time conditional statement (soft keyword). Desugars to If(test=c.@CIf(EXPR)). - if _is_cif_stmt(ps) != 0: - cif_stmt: AST* = _parse_cif(ps) - if cif_stmt is not None: - parent.append(cif_stmt) - return - # celif/celse are valid only inside a cif chain; a bare occurrence is fail-fast. Skip the line. - if _is_named_soft(ps, "celif") != 0 or _is_named_soft(ps, "celse") != 0: - _RecordCifChainErr("celif/celse must be part of a single cif chain (they cannot stand alone, nor appear after the chain's celse)", - _cur_lineno(ps), _cur_col(ps)) - ps.error_count += 1 - while _cur_type(ps) != TokenType.NewLine and _cur_type(ps) != TokenType.EndMarker: - _advance(ps) - return - # define constant declaration: `define NAME[: T] = ` (no parentheses). Must run before - # _parse_simple_stmt_into, otherwise `define` is parsed as an identifier expression and NAME is left - # as a stray token. `define NAME(...)` (macro function) is excluded by _is_define_const's lookahead. - if kw == Keyword.Define and _is_define_const(ps) != 0: - dc_stmt: AST* = _parse_define_const(ps) - if dc_stmt is not None: - parent.append(dc_stmt) - return - # typedef alias declaration: `typedef T1 = T2` (statement-level reserved word, no lookahead since - # typedef is never an identifier in existing sources). Invalid forms fail loudly in _parse_typedef. - if _is_typedef(ps) != 0: - td_stmt: AST* = _parse_typedef(ps) - if td_stmt is not None: - parent.append(td_stmt) - return - # export / extern variable declaration (soft keyword): `export NAME: T [= v]` / `extern NAME: T` - # -> a module-level AnnAssign with storage-class markers; function forms are excluded by lookahead. - if _is_export_extern_var(ps) != 0: - ee_stmt: AST* = _parse_export_extern_var(ps) - if ee_stmt is not None: - parent.append(ee_stmt) - return - # virtual / novirtual modifier (soft keyword): only at statement start before class/def; synthesizes - # @t.CVTable / @t.NoVTable then proceeds with class/def parsing. Placed before the class/def dispatch. - if _IsVirtualModifier(ps) != 0: - _ParseVirtualModified(ps, parent) - return - # inline modifier (soft keyword): only at statement start before a function definition; synthesizes - # @t.always_inline then proceeds with def parsing (same pending_decorators channel as virtual). - if _IsInlineModifier(ps) != 0: - _ParseInlineModified(ps, parent) - return - # static modifier (soft keyword): statement start + static + (def | `Name :`). The function branch - # synthesizes @t.CStatic (returns); the variable branch flags pending_static for _parse_type_ann. - if _IsStaticModifier(ps) != 0: - if _ParseStaticModified(ps, parent) != 0: - return - # Bare descriptor use: the whole statement is only `const` / `volatile` (missing type) -> fail loudly. - # Matches only when followed by newline/EOF/semicolon (soft-keyword boundary), so `const = 5` / - # `t.attr.const()` do not match. After recording, continue the generic simple_stmt dispatch. - if _IsBareDescriptor(ps) != 0: - _RecordDescriptorErr("bare descriptor use (the whole statement is only a descriptor, missing type)", - _cur_lineno(ps), _cur_col(ps)) - ps.error_count += 1 - if kw == Keyword.If or kw == Keyword.While or kw == Keyword.For or kw == Keyword.Try: - stmt: AST* = _parse_compound(ps) - if stmt is not None: - parent.append(stmt) - return - if kw == Keyword.With or _is_funcdef_kw(ps, kw) != 0 or kw == Keyword.Class \ - or kw == Keyword.Enum or kw == Keyword.Union or kw == Keyword.Renum \ - or kw == Keyword.Struct: - stmt = _parse_compound(ps) - if stmt is not None: - parent.append(stmt) - return - if kw == Keyword.Match or kw == Keyword.Switch or kw == Keyword.Async: - stmt = _parse_compound(ps) - if stmt is not None: - parent.append(stmt) - return - _parse_simple_stmt_into(ps, parent) - - -def _parse_simple_stmt_into(ps: Parser*, parent: AST*): - """simple_stmt: small_stmt (';' small_stmt)* [';'] TokenType.NewLine""" - first: AST* = _parse_small_stmt(ps) - if first is not None: - parent.append(first) - while _accept_op(ps, TokOp.Semi): - if _cur_type(ps) == TokenType.NewLine or _cur_type(ps) == TokenType.EndMarker: - break - s: AST* = _parse_small_stmt(ps) - if s is None: break - parent.append(s) - # First skip Nl (in-paren newline, e.g. between multi-line call args; a legal token) - _skip_nl(ps) - # Statement-end check: simple_stmt must end with a newline/EOF. Stray tokens (e.g. adjacent - # expressions split into multiple statements, or a def missing its keyword) are a fail-fast syntax - # error, reported Python-style (SyntaxError at line/col). - if _cur_type(ps) != TokenType.NewLine and _cur_type(ps) != TokenType.EndMarker: - # Stray tokens (e.g. adjacent expressions split into statements, or a def missing its keyword) are a - # fail-fast syntax error. The message is no longer self-assembled: position/message go through the - # parse error channel with prev/next tokens appended, output Python-style by the caller's - # dump_parse_traceback (Traceback + File/line + source line + caret + SyntaxError). - # Locate: walk the list from the head to find cur, recording prev and next token context. - prev_tok: Token* = None - walk_tok: Token* = ps.tokens - while walk_tok is not None and walk_tok is not ps.cur: - prev_tok = walk_tok - walk_tok = walk_tok.next - pv_str: str = "(head)" - pv_line: int = 0 - pv_col: int = 0 - if prev_tok is not None: - pv_str = prev_tok.str_val - pv_line = prev_tok.lineno - pv_col = prev_tok.col_offset - nxt_tok: Token* = ps.cur.next - nxt_str: str = "(none)" - nxt_line: int = 0 - nxt_col: int = 0 - if nxt_tok is not None: - nxt_str = nxt_tok.str_val - nxt_line = nxt_tok.lineno - nxt_col = nxt_tok.col_offset - _record_parse_err_ctx(ps, _cur_type(ps), pv_str, pv_line, pv_col, - nxt_str, nxt_line, nxt_col) - ps.error_count += 1 - return - if _cur_type(ps) == TokenType.NewLine: - _advance(ps) - - -def _parse_small_stmt(ps: Parser*) -> AST*: - """small_stmt: various simple statements""" - kw: int = _cur_kw(ps) - # fallthrough keyword (hard): legal only inside a switch case body ("fall to the next case"). - # Legality is checked by the generator; writing it in a match or outside a switch is fail-fast. - if kw == Keyword.FallThrough: - ft_lineno: int = _cur_lineno(ps) - ft_col: int = _cur_col(ps) - _advance(ps) - ft_node: AST* = FallThrough(ps.pool) - _set_pos(ft_node, ft_lineno, ft_col, 0, 0) - return ft_node - if kw == Keyword.Pass: - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - _advance(ps) - node: AST* = Pass(ps.pool) - _set_pos(node, lineno, col, 0, 0) - return node - if kw == Keyword.Break: - lineno = _cur_lineno(ps) - col = _cur_col(ps) - _advance(ps) - node = Break(ps.pool) - _set_pos(node, lineno, col, 0, 0) - return node - if kw == Keyword.Continue: - lineno = _cur_lineno(ps) - col = _cur_col(ps) - _advance(ps) - node = Continue(ps.pool) - _set_pos(node, lineno, col, 0, 0) - return node - if kw == Keyword.Return: - return _parse_return(ps) - if kw == Keyword.Raise: - return _parse_raise(ps) - if kw == Keyword.Global: - return _parse_global(ps) - if kw == Keyword.Nonlocal: - return _parse_nonlocal(ps) - if kw == Keyword.Import: - return _parse_import(ps) - if kw == Keyword.From: - return _parse_from_import(ps) - if kw == Keyword.Del: - return _parse_del(ps) - if kw == Keyword.Assert: - return _parse_assert(ps) - if kw == Keyword.Yield: - return _parse_expr_stmt(ps, _parse_yield(ps)) - # Expression statement / assignment / augassign / annassign - return _parse_expr_stmt(ps, None) - - -def _maybe_tuple(ps: Parser*, first: AST*) -> AST*: - """If there is a comma after first, collect the comma-separated expressions into a Tuple.""" - if first is None: return None - if not _match_op(ps, TokOp.Comma): - return first - lineno: int = first.lineno - col: int = first.col_offset - elts: list[AST*]* = list[AST*](ps.pool, 8) - elts.append(first) - _accept_op(ps, TokOp.Comma) - while True: - tt: int = _cur_type(ps) - if tt == TokenType.NewLine or tt == TokenType.EndMarker: - break - if tt == TokenType.Op: - op: int = _cur_op(ps) - if op == TokOp.Equal or op == TokOp.Semi: - break - e: AST* = _parse_expr(ps) - if e is None: break - elts.append(e) - if not _accept_op(ps, TokOp.Comma): - break - node: AST* = Tuple(ps.pool, elts, ASTCtx.Load) - _set_pos(node, lineno, col, 0, 0) - return node - - -def _parse_testlist(ps: Parser*) -> AST*: - """Parse a comma-separated list of expressions, if there’s more than one, form a tuple.""" - first: AST* = _parse_expr(ps) - return _maybe_tuple(ps, first) - - -def _parse_expr_stmt(ps: Parser*, first: AST*) -> AST*: - """Expression statement, handle assignment/augassign/annassign""" - if first is None: - first = _parse_testlist(ps) - else: - first = _maybe_tuple(ps, first) - if first is None: - return None - lineno: int = first.lineno - col: int = first.col_offset - # Assignment - if _match_op(ps, TokOp.Equal): - targets: list[AST*]* = list[AST*](ps.pool, 8) - targets.append(first) - _fix_store_ctx(first) - value: AST* = None - accept_result: int = _accept_op(ps, TokOp.Equal) - while accept_result: - v: AST* = _parse_testlist(ps) - if v is None: break - if value is not None: - targets.append(value) - _fix_store_ctx(value) - value = v - accept_result = _accept_op(ps, TokOp.Equal) - node: AST* = Assign(ps.pool, targets, value) - _set_pos(node, lineno, col, 0, 0) - return node - # augassign - if _cur_type(ps) == TokenType.Op: - op_id: int = _cur_op(ps) - aug_op: int = _augop_from_op(op_id) - if aug_op != OpKind.NoneOp: - _advance(ps) - value2: AST* = _parse_testlist(ps) - _fix_store_ctx(first) - node2: AST* = AugAssign(ps.pool, first, aug_op, value2) - _set_pos(node2, lineno, col, 0, 0) - return node2 - # annassign: target ':' annotation [= value] - if _match_op(ps, TokOp.Colon): - _advance(ps) - annotation: AST* = _parse_type_ann(ps) - value3: AST* = None - if _accept_op(ps, TokOp.Equal): - # A typedef's right-hand side is a **type** (`const str` / `uint16*`), not a general - # expression: parse it with the annotation rule so descriptors and C-style pointer - # suffixes behave exactly as in annotation position (same as TPC's vast). - if _IsCTypedefAnnotation(annotation) != 0: - value3 = _parse_type_ann(ps) - else: - value3 = _parse_testlist(ps) - if ps.pool is None: - return None - _fix_store_ctx(first) - if ps.pool is None: - return None - node3: AST* = AnnAssign(ps.pool, first, annotation, value3, 1) - _set_pos(node3, lineno, col, 0, 0) - return node3 - # Expression statement - node4: AST* = Expr(ps.pool, first) - _set_pos(node4, lineno, col, 0, 0) - return node4 - - -def _parse_return(ps: Parser*) -> AST*: - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - _advance(ps) - val: AST* = None - if _cur_type(ps) != TokenType.NewLine and _cur_type(ps) != TokenType.EndMarker and _cur_type(ps) != TokenType.Op: - val = _parse_testlist(ps) - elif _cur_type(ps) == TokenType.Op and _cur_op(ps) != TokOp.Semi: - val = _parse_testlist(ps) - node: AST* = Return(ps.pool, val) - _set_pos(node, lineno, col, 0, 0) - return node - - -def _parse_raise(ps: Parser*) -> AST*: - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - _advance(ps) - exc: AST* = None - cause: AST* = None - if _cur_type(ps) != TokenType.NewLine and _cur_type(ps) != TokenType.EndMarker: - exc = _parse_test(ps) - if _cur_kw(ps) == Keyword.From: - _advance(ps) - cause = _parse_test(ps) - node: AST* = Raise(ps.pool, exc, cause) - _set_pos(node, lineno, col, 0, 0) - return node - - -def _parse_global(ps: Parser*) -> AST*: - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - _advance(ps) - names: list[AST*]* = list[AST*](ps.pool, 8) - if _cur_type(ps) == TokenType.Name: - n: AST* = Name(ps.pool, _cur_str(ps), ASTCtx.Store) - _set_pos(n, _cur_lineno(ps), _cur_col(ps), 0, 0) - names.append(n) - _advance(ps) - while _accept_op(ps, TokOp.Comma): - if _cur_type(ps) != TokenType.Name: break - n = Name(ps.pool, _cur_str(ps), ASTCtx.Store) - _set_pos(n, _cur_lineno(ps), _cur_col(ps), 0, 0) - names.append(n) - _advance(ps) - node: AST* = Global(ps.pool, names) - _set_pos(node, lineno, col, 0, 0) - return node - - -def _parse_nonlocal(ps: Parser*) -> AST*: - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - _advance(ps) - names: list[AST*]* = list[AST*](ps.pool, 8) - if _cur_type(ps) == TokenType.Name: - n: AST* = Name(ps.pool, _cur_str(ps), ASTCtx.Store) - _set_pos(n, _cur_lineno(ps), _cur_col(ps), 0, 0) - names.append(n) - _advance(ps) - while _accept_op(ps, TokOp.Comma): - if _cur_type(ps) != TokenType.Name: break - n = Name(ps.pool, _cur_str(ps), ASTCtx.Store) - _set_pos(n, _cur_lineno(ps), _cur_col(ps), 0, 0) - names.append(n) - _advance(ps) - node: AST* = Nonlocal(ps.pool, names) - _set_pos(node, lineno, col, 0, 0) - return node - - -def _parse_import(ps: Parser*) -> AST*: - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - _advance(ps) - names: list[AST*]* = _parse_alias_list(ps) - node: AST* = Import(ps.pool, names) - _set_pos(node, lineno, col, 0, 0) - return node - - -def _parse_from_import(ps: Parser*) -> AST*: - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - _advance(ps) - # Parse module (possibly with dots) - level: int = 0 - while _match_op(ps, TokOp.Dot): - level += 1 - _advance(ps) - while _match_op(ps, TokOp.Ellipsis): - level += 3 - _advance(ps) - module_name: str = None - # The token type for keywords (like import) is also Name, so they must be excluded using kw_subtype, - # otherwise the `import` in `from . import` would be mistakenly treated as a module_name. - # Note: only `import` itself should be excluded. Module names can be keywords (like in `from .match import ...`, - # where match is a Viper match statement keyword), requiring NotKeyword would cause - # `from .match import` to fail parsing and the dependency scan would miss that module. - if _cur_type(ps) == TokenType.Name and _cur_kw(ps) != Keyword.Import: - module_name = _cur_str(ps) - _advance(ps) - while _match_op(ps, TokOp.Dot): - _advance(ps) - if _cur_type(ps) == TokenType.Name and _cur_kw(ps) == Keyword.NotKeyword: - next_name: str = _cur_str(ps) - mlen: int = int(string.strlen(module_name)) # 显式 cast：避免 64→32 隐式窄化 - nlen: int = int(string.strlen(next_name)) # 显式 cast：避免 64→32 隐式窄化 - new_mod: str = ps.pool.alloc(mlen + 1 + nlen + 1) - if new_mod is not None: - string.strcpy(new_mod, module_name) - new_mod[mlen] = '.' - string.strcpy(new_mod + mlen + 1, next_name) - module_name = new_mod - _advance(ps) - if _cur_kw(ps) != Keyword.Import: - ps.error_count += 1 - return None - _advance(ps) - _skip_nl(ps) - # import * - if _match_op(ps, TokOp.Star): - _advance(ps) - star_alias: AST* = Alias(ps.pool, "*", None) - _set_pos(star_alias, lineno, col, 0, 0) - star_names: list[AST*]* = list[AST*](ps.pool, 8) - star_names.append(star_alias) - node: AST* = ImportFrom(ps.pool, module_name, star_names, level) - _set_pos(node, lineno, col, 0, 0) - return node - # ( names ) or names - if _match_op(ps, TokOp.LPar): - _advance(ps) - _skip_nl(ps) - names: list[AST*]* = _parse_alias_list(ps) - _skip_nl(ps) - _expect_op(ps, TokOp.RPar) - else: - names = _parse_alias_list(ps) - node: AST* = ImportFrom(ps.pool, module_name, names, level) - _set_pos(node, lineno, col, 0, 0) - return node - - -def _parse_alias_list(ps: Parser*) -> list[AST*]*: - """Parse alias list.""" - names: list[AST*]* = list[AST*](ps.pool, 8) - while True: - if _cur_type(ps) != TokenType.Name: - break - name_str: str = _cur_str(ps) - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - _advance(ps) - # dotted name - while _match_op(ps, TokOp.Dot): - _advance(ps) - if _cur_type(ps) == TokenType.Name: - next_n: str = _cur_str(ps) - nmlen: int = int(string.strlen(name_str)) # 显式 cast：避免 64→32 隐式窄化 - nxtlen: int = int(string.strlen(next_n)) # 显式 cast：避免 64→32 隐式窄化 - new_name: str = ps.pool.alloc(nmlen + 1 + nxtlen + 1) - if new_name is not None: - string.strcpy(new_name, name_str) - new_name[nmlen] = '.' - string.strcpy(new_name + nmlen + 1, next_n) - name_str = new_name - _advance(ps) - asname: str = None - if _cur_kw(ps) == Keyword.As: - _advance(ps) - if _cur_type(ps) == TokenType.Name: - asname = _cur_str(ps) - _advance(ps) - a: AST* = Alias(ps.pool, name_str, asname) - _set_pos(a, lineno, col, 0, 0) - names.append(a) - if not _accept_op(ps, TokOp.Comma): - break - _skip_nl(ps) - return names - - -def _parse_del(ps: Parser*) -> AST*: - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - _advance(ps) - targets: list[AST*]* = list[AST*](ps.pool, 8) - while True: - t_: AST* = _parse_atom_expr(ps) - if t_ is None: break - _set_ctx(t_, ASTCtx.Del) - targets.append(t_) - if not _accept_op(ps, TokOp.Comma): - break - node: AST* = Delete(ps.pool, targets) - _set_pos(node, lineno, col, 0, 0) - return node - - -def _parse_assert(ps: Parser*) -> AST*: - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - _advance(ps) - test: AST* = _parse_test(ps) - msg: AST* = None - if _accept_op(ps, TokOp.Comma): - msg = _parse_test(ps) - node: AST* = Assert(ps.pool, test, msg) - _set_pos(node, lineno, col, 0, 0) - return node - - -# ============================================================ -# Compound statements -# ============================================================ - -def _parse_compound(ps: Parser*) -> AST*: - kw: int = _cur_kw(ps) - if kw == Keyword.If: - return _parse_if(ps) - if kw == Keyword.While: - return _parse_while(ps) - if kw == Keyword.For: - return _parse_for(ps, 0) - if kw == Keyword.Try: - return _parse_try(ps) - if kw == Keyword.With: - return _parse_with(ps, 0) - if _is_funcdef_kw(ps, kw) != 0: - return _parse_funcdef(ps, 0) - if kw == Keyword.Class: - return _parse_classdef(ps) - if kw == Keyword.Enum: - return _parse_enumdef(ps) - if kw == Keyword.Union: - return _parse_uniondef(ps) - if kw == Keyword.Renum: - return _parse_renumdef(ps) - if kw == Keyword.Struct: - return _parse_structdef(ps) - if kw == Keyword.Async: - _advance(ps) - if _is_funcdef_kw(ps, _cur_kw(ps)) != 0: - return _parse_funcdef(ps, 1) - if _cur_kw(ps) == Keyword.For: - return _parse_for(ps, 1) - if _cur_kw(ps) == Keyword.With: - return _parse_with(ps, 1) - ps.error_count += 1 - return None - if kw == Keyword.Match: - return _parse_match(ps) - if kw == Keyword.Switch: - return _parse_switch(ps) - ps.error_count += 1 - return None - - -# ============================================================ -# cif / celif / celse compile-time conditional statements -# ============================================================ -# `cif EXPR:` is a soft-keyword statement desugared at parse time into -# If(test=c.@CIf(EXPR), body, orelse). `celif`/`celse` extend the same chain and are recognized only -# inside _parse_cif_tail. The marker attribute name '@CIf' is internal-only spelling (a leading '@' -# cannot be produced by the lexer), so a user-written c.CIf(...) never matches it and is rejected. - - -def _MakeCifMarker(ps: Parser*, test: AST*, lineno: int, col: int) -> AST*: - """Build the internal marker call c.@CIf(test) that wraps a cif/celif condition.""" - attr: AST* = Attribute(ps.pool, Name(ps.pool, "c", ASTCtx.Load), "@CIf", ASTCtx.Load) - args: list[AST*]* = list[AST*](ps.pool, 4) - if test is not None: args.append(test) - kw: list[AST*]* = list[AST*](ps.pool, 1) - marker: AST* = Call(ps.pool, attr, args, kw) - _set_pos(marker, lineno, col, 0, 0) - return marker - - -def _RecordCifChainErr(detail: str, lineno: int, col: int): - """Record an invalid cif/celif/celse chain (fail-fast; first only). Message states the fix. - - A single cif chain is cif ... celif ... celse: only `cif` opens a chain, and the chain must be - terminated by `celse`; a plain else/elif cannot close it and celif/celse cannot stand alone. - """ - global _g_parse_err_line, _g_parse_err_col, _g_parse_err_msg - if _g_parse_err_line != 0: - return - _g_parse_err_line = lineno - _g_parse_err_col = col - buf: str = stdlib.malloc(1024) - if buf is not None: - stdio.snprintf(buf, 1024, - "invalid cif chain (%s). cif/celif/celse form a single compile-time conditional chain:" - " only `cif` opens a chain, `celif` extends the current cif chain, and the chain must be" - " terminated by `celse` (a plain else/elif cannot close a cif chain, and celif/celse cannot" - " stand alone). Correct form: cif : ... celif : ... celse: ...", detail) - _g_parse_err_msg = buf - else: - _g_parse_err_msg = detail - - -def _parse_cif(ps: Parser*) -> AST*: - """Parse a `cif EXPR: ...` statement and its celif/celse chain (desugars to nested If nodes).""" - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - _advance(ps) # cif - test: AST* = _parse_test(ps) - _expect_op(ps, TokOp.Colon) - empty: list[AST*]* = list[AST*](ps.pool, 8) - node: If* = If(ps.pool, _MakeCifMarker(ps, test, lineno, col), empty) - _parse_suite_into_children(ps, node) - node.orelse = _parse_cif_tail(ps) - _set_pos(node, lineno, col, 0, 0) - return node - - -def _parse_cif_tail(ps: Parser*) -> list[AST*]*: - """Parse the celif/celse continuation of a cif chain and return the orelse list. - - celif: orelse = [nested If(test=c.@CIf(EXPR), ...)] (same chain, recurses). - celse: orelse = statement list (a plain else; no marker). - A plain elif/else after a cif chain is fail-fast (a cif chain must end with celse). - """ - if _is_named_soft(ps, "celif") != 0: - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - _advance(ps) # celif - test: AST* = _parse_test(ps) - _expect_op(ps, TokOp.Colon) - empty: list[AST*]* = list[AST*](ps.pool, 8) - elif_node: If* = If(ps.pool, _MakeCifMarker(ps, test, lineno, col), empty) - _parse_suite_into_children(ps, elif_node) - elif_node.orelse = _parse_cif_tail(ps) - _set_pos(elif_node, lineno, col, 0, 0) - orelse: list[AST*]* = list[AST*](ps.pool, 8) - orelse.append(elif_node) - return orelse - if _is_named_soft(ps, "celse") != 0: - _advance(ps) # celse - _expect_op(ps, TokOp.Colon) - scratch: Module* = Module(ps.pool) - _parse_suite_into_children(ps, scratch) - return scratch.children - # A plain elif/else cannot close a cif chain; consume and discard its clause to keep parsing. - if _cur_kw(ps) == Keyword.Elif or _cur_kw(ps) == Keyword.Else: - _RecordCifChainErr("a plain elif/else cannot follow a cif/celif branch; the cif chain must be terminated by celse", - _cur_lineno(ps), _cur_col(ps)) - ps.error_count += 1 - is_elif: int = 0 - if _cur_kw(ps) == Keyword.Elif: is_elif = 1 - _advance(ps) # elif / else - if is_elif != 0: _parse_test(ps) - _expect_op(ps, TokOp.Colon) - scratch2: Module* = Module(ps.pool) - _parse_suite_into_children(ps, scratch2) - return list[AST*](ps.pool, 8) - return list[AST*](ps.pool, 8) - - -def _parse_if(ps: Parser*) -> AST*: - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - _advance(ps) # if - test: AST* = _parse_test(ps) - _expect_op(ps, TokOp.Colon) - # First construct with empty orelse, then fill it in later - empty: list[AST*]* = list[AST*](ps.pool, 8) - node: If* = If(ps.pool, test, empty) - _parse_suite_into_children(ps, node) - orelse: list[AST*]* = _parse_if_tail(ps) - node.orelse = orelse - _set_pos(node, lineno, col, 0, 0) - return node - - -def _parse_if_tail(ps: Parser*) -> list[AST*]*: - """Parse elif/else chain and return orelse list. - - elif: orelse = list containing single If node (nested) - else: orelse = statement list - """ - # celif/celse belong to a cif chain only; after a plain if they are fail-fast (and discarded). - if _is_named_soft(ps, "celif") != 0 or _is_named_soft(ps, "celse") != 0: - _RecordCifChainErr("celif/celse follow a plain if statement (not a cif chain); a compile-time chain must start with cif", - _cur_lineno(ps), _cur_col(ps)) - ps.error_count += 1 - is_celif: int = _is_named_soft(ps, "celif") - _advance(ps) # celif / celse - if is_celif != 0: _parse_test(ps) - _expect_op(ps, TokOp.Colon) - scratch_ce: Module* = Module(ps.pool) - _parse_suite_into_children(ps, scratch_ce) - return list[AST*](ps.pool, 8) - if _cur_kw(ps) == Keyword.Elif: - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - _advance(ps) - elif_test: AST* = _parse_test(ps) - _expect_op(ps, TokOp.Colon) - empty: list[AST*]* = list[AST*](ps.pool, 8) - elif_node: If* = If(ps.pool, elif_test, empty) - _parse_suite_into_children(ps, elif_node) - elif_orelse: list[AST*]* = _parse_if_tail(ps) - elif_node.orelse = elif_orelse - _set_pos(elif_node, lineno, col, 0, 0) - orelse: list[AST*]* = list[AST*](ps.pool, 8) - orelse.append(elif_node) - return orelse - if _cur_kw(ps) == Keyword.Else: - _advance(ps) - _expect_op(ps, TokOp.Colon) - scratch: Module* = Module(ps.pool) - _parse_suite_into_children(ps, scratch) - return scratch.children - return list[AST*](ps.pool, 8) # empty list - - -# ============================================================ -# Parsing Compound Statements (Continued) -# ============================================================ - -def _parse_while(ps: Parser*) -> AST*: - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - _advance(ps) # while - test: AST* = _parse_test(ps) - _expect_op(ps, TokOp.Colon) - empty: list[AST*]* = list[AST*](ps.pool, 8) - node: While* = While(ps.pool, test, empty) - _parse_suite_into_children(ps, node) - if _cur_kw(ps) == Keyword.Else: - _advance(ps) - _expect_op(ps, TokOp.Colon) - scratch: Module* = Module(ps.pool) - _parse_suite_into_children(ps, scratch) - node.orelse = scratch.children - _set_pos(node, lineno, col, 0, 0) - return node - - -def _parse_for(ps: Parser*, is_async: int) -> AST*: - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - _advance(ps) # for - target: AST* = _parse_target_list(ps) - if _cur_kw(ps) != Keyword.In: - ps.error_count += 1 - return None - _advance(ps) - iter_: AST* = _parse_or_test(ps) - _expect_op(ps, TokOp.Colon) - empty: list[AST*]* = list[AST*](ps.pool, 8) - node: For* = For(ps.pool, target, iter_, empty, is_async) - _fix_store_ctx(target) - _parse_suite_into_children(ps, node) - if _cur_kw(ps) == Keyword.Else: - _advance(ps) - _expect_op(ps, TokOp.Colon) - scratch: Module* = Module(ps.pool) - _parse_suite_into_children(ps, scratch) - node.orelse = scratch.children - _set_pos(node, lineno, col, 0, 0) - return node - - -def _parse_try(ps: Parser*) -> AST*: - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - _advance(ps) # try - _expect_op(ps, TokOp.Colon) - empty: list[AST*]* = list[AST*](ps.pool, 8) - node: Try* = Try(ps.pool, empty, empty, empty) - _parse_suite_into_children(ps, node) - # handlers (except) - handlers: list[AST*]* = list[AST*](ps.pool, 8) - while _cur_kw(ps) == Keyword.Except: - h: AST* = _parse_except_handler(ps) - if h is not None: - handlers.append(h) - node.handlers = handlers - # else - if _cur_kw(ps) == Keyword.Else: - _advance(ps) - _expect_op(ps, TokOp.Colon) - scratch: Module* = Module(ps.pool) - _parse_suite_into_children(ps, scratch) - node.orelse = scratch.children - # finally - if _cur_kw(ps) == Keyword.Finally: - _advance(ps) - _expect_op(ps, TokOp.Colon) - scratch2: Module* = Module(ps.pool) - _parse_suite_into_children(ps, scratch2) - node.finalbody = scratch2.children - _set_pos(node, lineno, col, 0, 0) - return node - - -def _parse_except_handler(ps: Parser*) -> AST*: - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - _advance(ps) # except - type_: AST* = None - name: str = None - if _cur_op(ps) != TokOp.Colon: - type_ = _parse_test(ps) - if _cur_kw(ps) == Keyword.As: - _advance(ps) - if _cur_type(ps) == TokenType.Name: - name = _cur_str(ps) - _advance(ps) - _expect_op(ps, TokOp.Colon) - node: AST* = ExceptHandler(ps.pool, type_, name) - _parse_suite_into_children(ps, node) - _set_pos(node, lineno, col, 0, 0) - return node - - -def _parse_with(ps: Parser*, is_async: int) -> AST*: - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - _advance(ps) # with - items: list[AST*]* = list[AST*](ps.pool, 8) - # items - while True: - item: AST* = _parse_with_item(ps) - if item is None: break - items.append(item) - if not _accept_op(ps, TokOp.Comma): - break - # Check if we can continue with a newline (with (...): form) - if _cur_type(ps) == TokenType.NewLine or _cur_type(ps) == TokenType.Nl: - break - _expect_op(ps, TokOp.Colon) - node: With* = With(ps.pool, items, is_async) - _parse_suite_into_children(ps, node) - _set_pos(node, lineno, col, 0, 0) - return node - - -def _parse_with_item(ps: Parser*) -> AST*: - """with_item: test ['as' expr]""" - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - context_expr: AST* = _parse_or_test(ps) - if context_expr is None: return None - optional_vars: AST* = None - if _cur_kw(ps) == Keyword.As: - _advance(ps) - optional_vars = _parse_target(ps) - _fix_store_ctx(optional_vars) - node: AST* = WithItem(ps.pool, context_expr, optional_vars) - _set_pos(node, lineno, col, 0, 0) - return node - - -def _parse_funcdef(ps: Parser*, is_async: int) -> AST*: - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - is_extern_fd: int = 0 - is_export_fd: int = 0 - is_define_fd: int = 0 - if _cur_kw(ps) == Keyword.Extern: - # extern declaration: extern [def] A(...) [-> T] [optional ": pass"] - is_extern_fd = 1 - _advance(ps) # extern - if _cur_kw(ps) == Keyword.Def: - _advance(ps) - elif _cur_kw(ps) == Keyword.Export: - # export declaration: export [def] A(...) [-> T] [optional ": pass"] - is_export_fd = 1 - _advance(ps) # export - if _cur_kw(ps) == Keyword.Def: - _advance(ps) - elif _cur_kw(ps) == Keyword.Define: - # define macro function: syntax identical to def; t.CDefine is appended to the return annotation - # (StubGen keeps it as-is -> C #define / alwaysinline) - is_define_fd = 1 - _advance(ps) # define - else: - _advance(ps) # def / fn / ... - if _cur_type(ps) != TokenType.Name: - ps.error_count += 1 - return None - name: str = _cur_str(ps) - _advance(ps) - # PEP 695 generic function type parameters: def add[T](a: T, b: T) -- parsed and stored into - # type_params, consistent with the class parser. type_params maps to int as a fallback during translation. - type_params_fd: list[str]* = None - if _accept_op(ps, TokOp.Lsqb): - type_params_fd = list[str](ps.pool, 8) - while not _match_op(ps, TokOp.Rsqb): - if _cur_type(ps) == TokenType.EndMarker: - break - if _cur_type(ps) == TokenType.Name: - tp_name_fd: str = _cur_str(ps) - _advance(ps) - # Skip the optional bound/constraint: [T: bound] or [T: (A, B)] - if _match_op(ps, TokOp.Colon): - _advance(ps) - _parse_test(ps) - type_params_fd.append(tp_name_fd) - if not _accept_op(ps, TokOp.Comma): - break - _expect_op(ps, TokOp.Rsqb) - args: AST* = None - if _accept_op(ps, TokOp.LPar): - args = _parse_varargslist(ps, 0) - _expect_op(ps, TokOp.RPar) - returns: AST* = None - if _accept_op(ps, TokOp.RArrow): - returns = _parse_type_ann(ps) - if is_extern_fd: - # extern declaration: rewrite the return annotation to | t.State, reusing the existing - # State/external-C-function path (vpi annotation containing "State" -> bare name, no function body) - state_attr_fd: AST* = Attribute(ps.pool, - Name(ps.pool, "t", ASTCtx.Load), - "State", ASTCtx.Load) - if returns != None: - returns = BinOp(ps.pool, returns, OpKind.BitOr, state_attr_fd) - else: - returns = state_attr_fd - elif is_export_fd: - # export declaration: rewrite the return annotation to | t.CExport, reusing the existing - # CExport path (vpi annotation containing "CExport" -> bare name, no module SHA1 header) - export_attr_fd: AST* = Attribute(ps.pool, - Name(ps.pool, "t", ASTCtx.Load), - "CExport", ASTCtx.Load) - if returns != None: - returns = BinOp(ps.pool, returns, OpKind.BitOr, export_attr_fd) - else: - returns = export_attr_fd - elif is_define_fd: - # define macro function: rewrite the return annotation to | t.CDefine, reusing the - # existing CDefine path (StubGen keeps it as-is -> C #define / alwaysinline) - define_attr_fd: AST* = Attribute(ps.pool, - Name(ps.pool, "t", ASTCtx.Load), - "CDefine", ASTCtx.Load) - if returns != None: - returns = BinOp(ps.pool, returns, OpKind.BitOr, define_attr_fd) - else: - returns = define_attr_fd - dec_list: list[AST*]* = ps.pending_decorators - ps.pending_decorators = None - node: FunctionDef* = FunctionDef(ps.pool, name, args, dec_list, returns, is_async, type_params_fd) - if is_extern_fd: - node.flags = node.flags | FLAG_IS_EXTERN - if is_export_fd: - node.flags = node.flags | FLAG_IS_EXPORT - if is_define_fd: - node.flags = node.flags | FLAG_IS_DEFINE - if (is_extern_fd == 0 and is_export_fd == 0 and is_define_fd == 0) or _match_op(ps, TokOp.Colon): - _expect_op(ps, TokOp.Colon) - _parse_suite_into_children(ps, node) - else: - # extern/export without ': pass' (extern A(...) -> int): empty body, declaration only. The NewLine - # must be consumed here: the module/class statement loop only skips Nl, so a leftover NewLine would be - # parsed as an expression next round and fail parse_tokens. - if _cur_type(ps) == TokenType.NewLine: - _advance(ps) - _set_pos(node, lineno, col, 0, 0) - return node - - -def _parse_classdef(ps: Parser*) -> AST*: - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - _advance(ps) # class - if _cur_type(ps) != TokenType.Name: - ps.error_count += 1 - return None - name: str = _cur_str(ps) - _advance(ps) - # Parse type parameters [T] or [T, U] (PEP 695 generic class syntax) - type_params: list[str]* = None - if _accept_op(ps, TokOp.Lsqb): - type_params = list[str](ps.pool, 8) - while not _match_op(ps, TokOp.Rsqb): - if _cur_type(ps) == TokenType.EndMarker: break - if _cur_type(ps) == TokenType.Name: - tp_name: str = _cur_str(ps) - _advance(ps) - # Skip optional bound/constraint: [T: bound] or [T: (A, B)] - if _match_op(ps, TokOp.Colon): - _advance(ps) - _parse_test(ps) - type_params.append(tp_name) - if not _accept_op(ps, TokOp.Comma): - break - _expect_op(ps, TokOp.Rsqb) - bases: list[AST*]* = list[AST*](ps.pool, 8) - keywords: list[AST*]* = list[AST*](ps.pool, 8) - # bases / keywords - if _accept_op(ps, TokOp.LPar): - while not _match_op(ps, TokOp.RPar): - # Cross-line base list inside parens (class X(A,\n B):): skip the newline after a comma, - # otherwise cur=NL is not Name/Comma -> break -> _expect_op(RPar) reports an unexpected token. - _skip_nl(ps) - if _cur_type(ps) == TokenType.EndMarker: break - # base or keyword=value - if _cur_type(ps) == TokenType.Name: - # Parse as expression, if followed by '=', then as keyword=value - e: AST* = _parse_or_test(ps) - if _match_op(ps, TokOp.Equal): - _advance(ps) - v: AST* = _parse_test(ps) - kw_node: AST* = KwNode(ps.pool, e, v) - keywords.append(kw_node) - else: - bases.append(e) - if not _accept_op(ps, TokOp.Comma): - break - _skip_nl(ps) # finish the newline after the last base (class X(A,\n B):) - _expect_op(ps, TokOp.RPar) - _expect_op(ps, TokOp.Colon) - dec_list: list[AST*]* = ps.pending_decorators - ps.pending_decorators = None - node: ClassDef* = ClassDef(ps.pool, name, bases, keywords, dec_list, type_params) - # owner_is_class=1: ps.in_class_body=1 during the class body, so virtual/novirtual modifiers can tell - # whether a method-level modifier is directly in the class body (see _ParseVirtualModified). - _parse_suite_into_children(ps, node, 1) - _set_pos(node, lineno, col, 0, 0) - return node - - -# ============================================================ -# enum sugar (fully equivalent to `class X(t.CEnum):`): `enum Name:` header + member lines -# (`A` / `A: T` / `A = 0`). The synthesized ClassDef is field-identical to _parse_classdef output, so -# downstream (ClassGenEnum._translate_enum_def -> EnumTable) has a single implementation path. -# ============================================================ - -def _enum_member_annassign(ps: Parser*, owner: AST*, - name_node: AST*, - lineno: int, col: int, - end_lineno: int, end_col: int) -> AST*: - """Rewrite a single member-name node into a `Name: t.State` AnnAssign (the uniform enum member form). - - target=Name(id, Store), annotation=Attribute(Name('t'), 'State'), value=None, simple=1 -- - field-identical to a hand-written `class X(t.CEnum): A: t.State`. - """ - ann: AST* = Attribute(ps.pool, - Name(ps.pool, "t", ASTCtx.Load), - "State", ASTCtx.Load) - _set_pos(ann, lineno, col, end_lineno, end_col) - aa: AST* = AnnAssign(ps.pool, name_node, ann, None, 1) - _fix_store_ctx(name_node) - _set_pos(aa, lineno, col, end_lineno, end_col) - aa.parent = owner # maintain the parent pointer like AST.append does - return aa - - -def _convert_enum_bare_members(ps: Parser*, node: ClassDef*): - """Rewrite bare-name / comma-grouped member lines in an enum body into `Name: t.State` AnnAssigns. - - Member collection only recognizes Assign / AnnAssign Name targets, so bare names and comma groups - (`A, B, C`) were silently dropped; now `A` -> `A: t.State` and `A, B, C` -> three AnnAssigns. Valued - or annotated groups are not rewritten here and are caught by the registration-site fail-fast. - """ - children: list[AST*]* = node.children - if children is None: - return - cn: sizet = len(children) - # Rebuild the child list: one comma-group line -> several AnnAssigns. list has no insert, so rebuild - # entirely; the assignment form matches AST.append in base.vp. - node.children = list[AST*](ps.pool, 8) - out: list[AST*]* = node.children - for i in range(cn): - stmt: AST* = children.get(i) - handled: int = 0 - if stmt is not None and stmt.kind() == ASTKind.Expr: - ex: Expr* = (Expr*)(stmt) - if ex.value is not None and ex.value.kind() == ASTKind.Name: - out.append(_enum_member_annassign(ps, node, ex.value, - ex.lineno, ex.col_offset, - ex.end_lineno, ex.end_col_offset)) - handled = 1 - elif ex.value is not None and ex.value.kind() == ASTKind.Tuple: - tup: Tuple* = (Tuple*)(ex.value) - if tup.elts is not None and len(tup.elts) >= 2: - all_names: int = 1 - k: sizet = 0 - while k < len(tup.elts): - el: AST* = tup.elts.get(k) - if el is None or el.kind() != ASTKind.Name: - all_names = 0 - break - k += 1 - if all_names == 1: - for k in range(len(tup.elts)): - el2: AST* = tup.elts.get(k) - out.append(_enum_member_annassign(ps, node, el2, - el2.lineno, el2.col_offset, - el2.end_lineno, el2.end_col_offset)) - handled = 1 - if handled == 0 and stmt is not None: - out.append(stmt) - - -def _parse_define_const(ps: Parser*) -> AST*: - """`define NAME[: T] = ` -> an AnnAssign field-identical to `NAME: [T |] t.CDefine = `. - - Same CDefine registration path as hand-written `NAME: t.CDefine = value`. With a type, T goes before - the union (`T | t.CDefine`), matching TPC's expansion. Must end with newline/EOF (fail-fast otherwise). - """ - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - _advance(ps) # define - if _cur_type(ps) != TokenType.Name: - ps.error_count += 1 - _record_parse_err(ps) - return None - name: str = _cur_str(ps) - _advance(ps) - explicit_ann: AST* = None - if _accept_op(ps, TokOp.Colon): - explicit_ann = _parse_type_ann(ps) - if explicit_ann is None: - return None - if _expect_op(ps, TokOp.Equal) == 0: - return None - value: AST* = _parse_test(ps) - if value is None: - ps.error_count += 1 - _record_parse_err(ps) - return None - cdefine_marker: AST* = Attribute(ps.pool, - Name(ps.pool, "t", ASTCtx.Load), - "CDefine", ASTCtx.Load) - annotation: AST* = cdefine_marker - if explicit_ann is not None: - annotation = BinOp(ps.pool, explicit_ann, OpKind.BitOr, cdefine_marker) - target: AST* = Name(ps.pool, name, ASTCtx.Store) - _set_pos(target, lineno, col, 0, 0) - node: AST* = AnnAssign(ps.pool, target, annotation, value, 1) - _set_pos(node, lineno, col, 0, 0) - _skip_nl(ps) - if _cur_type(ps) != TokenType.NewLine and _cur_type(ps) != TokenType.EndMarker: - _record_parse_err(ps) - ps.error_count += 1 - return None - if _cur_type(ps) == TokenType.NewLine: - _advance(ps) - return node - - -def _parse_typedef(ps: Parser*) -> AST*: - """`typedef T1 = T2` -> an AnnAssign field-identical to `T1: t.CTypedef = T2`. - - Same typedef registration/lookup path as the hand-written form (CTypedef branch -> register_ctypedef); - no second implementation. Invalid forms (missing name / '=' / RHS) fail loudly; must end with newline/EOF. - """ - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - _advance(ps) # typedef - if _cur_type(ps) != TokenType.Name: - # Missing alias (e.g. `typedef = 1` / `typedef : x`): fail loudly. - _record_parse_err(ps) - ps.error_count += 1 - return None - name: str = _cur_str(ps) - _advance(ps) - if _expect_op(ps, TokOp.Equal) == 0: - # Missing '=' (e.g. `typedef A`): fail loudly. - return None - # The sugar's RHS is the same type position as `T1: t.CTypedef = T2`: parse it with the - # annotation rule so `typedef X = const str` / `= uint16*` match the hand-written form. - value: AST* = _parse_type_ann(ps) - if value is None: - # Missing RHS (e.g. `typedef A = `): fail loudly. - ps.error_count += 1 - _record_parse_err(ps) - return None - ctypedef_marker: AST* = Attribute(ps.pool, - Name(ps.pool, "t", ASTCtx.Load), - "CTypedef", ASTCtx.Load) - target: AST* = Name(ps.pool, name, ASTCtx.Store) - _set_pos(target, lineno, col, 0, 0) - node: AST* = AnnAssign(ps.pool, target, ctypedef_marker, value, 1) - _set_pos(node, lineno, col, 0, 0) - _skip_nl(ps) - if _cur_type(ps) != TokenType.NewLine and _cur_type(ps) != TokenType.EndMarker: - _record_parse_err(ps) - ps.error_count += 1 - return None - if _cur_type(ps) == TokenType.NewLine: - _advance(ps) - return node - - -def _parse_export_extern_var(ps: Parser*) -> AST*: - """`export NAME: T [= v]` / `extern NAME: T` -> a module-level AnnAssign with storage-class markers. - - export rewrites the annotation to `T | t.CExport` (same marker/path as `export def`: bare name, no - module SHA1 header) and allows an initializer; extern rewrites to `T | t.State` (declaration + bare - name) and forbids an initializer (`extern a: int = 1` fails loudly). Must end with newline/EOF. - """ - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - is_extern: int = 0 - if _cur_kw(ps) == Keyword.Extern: - is_extern = 1 - _advance(ps) # export / extern - if _cur_type(ps) != TokenType.Name: - # Missing variable name (e.g. `export : int`): fail loudly. - _record_parse_err(ps) - ps.error_count += 1 - return None - name: str = _cur_str(ps) - _advance(ps) - if _expect_op(ps, TokOp.Colon) == 0: - # Missing ':' (e.g. `export x`): fail loudly. - return None - ann: AST* = _parse_type_ann(ps) - if ann is None: - ps.error_count += 1 - _record_parse_err(ps) - return None - # Storage-class marker: export -> t.CExport (bare-name export); extern -> t.State (declaration + bare name) - storage_attr: AST* = Attribute(ps.pool, - Name(ps.pool, "t", ASTCtx.Load), - "State" if is_extern != 0 else "CExport", - ASTCtx.Load) - annotation: AST* = BinOp(ps.pool, ann, OpKind.BitOr, storage_attr) - value: AST* = None - if _match_op(ps, TokOp.Equal): - if is_extern != 0: - # An extern variable is externally defined: no initializer allowed in this module. - _RecordDescriptorErr("extern variable declaration cannot have an initializer (the symbol is defined externally)", - lineno, col) - ps.error_count += 1 - return None - _advance(ps) - value = _parse_test(ps) - if value is None: - ps.error_count += 1 - _record_parse_err(ps) - return None - target: AST* = Name(ps.pool, name, ASTCtx.Store) - _set_pos(target, lineno, col, 0, 0) - node: AST* = AnnAssign(ps.pool, target, annotation, value, 1) - _set_pos(node, lineno, col, 0, 0) - _skip_nl(ps) - if _cur_type(ps) != TokenType.NewLine and _cur_type(ps) != TokenType.EndMarker: - _record_parse_err(ps) - ps.error_count += 1 - return None - if _cur_type(ps) == TokenType.NewLine: - _advance(ps) - return node - - -def _parse_enumdef(ps: Parser*) -> AST*: - """enum NAME: sugar -> a ClassDef field-identical to `class NAME(t.CEnum):`. - - Member forms (same as TPC's _ExpandEnumBlocks): bare `A` -> `A: t.State` (auto-numbered, i32 width); - comma group `A, B` -> several `A: t.State` / `B: t.State`; annotated `A: T` and valued `A = 0` as-is. - Base is always Attribute(Name('t'), 'CEnum'), field-identical to `class X(t.CEnum):`. Line/col inherited - from each member statement. - """ - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - _advance(ps) # enum - if _cur_type(ps) != TokenType.Name: - ps.error_count += 1 - return None - name: str = _cur_str(ps) - _advance(ps) - bases: list[AST*]* = list[AST*](ps.pool, 8) - bases.append(Attribute(ps.pool, Name(ps.pool, "t", ASTCtx.Load), - "CEnum", ASTCtx.Load)) - keywords: list[AST*]* = list[AST*](ps.pool, 8) - dec_list: list[AST*]* = ps.pending_decorators - ps.pending_decorators = None - _expect_op(ps, TokOp.Colon) - node: ClassDef* = ClassDef(ps.pool, name, bases, keywords, dec_list, None) - _parse_suite_into_children(ps, node) - _convert_enum_bare_members(ps, node) - _set_pos(node, lineno, col, 0, 0) - return node - - -def _parse_uniondef(ps: Parser*) -> AST*: - """union NAME: sugar -> a ClassDef field-identical to `class NAME(t.CUnion):`. - - Members are parsed as a normal class body (AnnAssign). Base is always Attribute(Name('t'), 'CUnion'), - field-identical to `class X(t.CUnion):`; TPC detects unions via `issubclass(cls, t.CUnion)`, so both - sides share the existing union path (offset 0, size = widest member). No bare-name conversion. - """ - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - _advance(ps) # union - if _cur_type(ps) != TokenType.Name: - ps.error_count += 1 - return None - name: str = _cur_str(ps) - _advance(ps) - bases: list[AST*]* = list[AST*](ps.pool, 8) - bases.append(Attribute(ps.pool, Name(ps.pool, "t", ASTCtx.Load), - "CUnion", ASTCtx.Load)) - keywords: list[AST*]* = list[AST*](ps.pool, 8) - dec_list: list[AST*]* = ps.pending_decorators - ps.pending_decorators = None - _expect_op(ps, TokOp.Colon) - node: ClassDef* = ClassDef(ps.pool, name, bases, keywords, dec_list, None) - _parse_suite_into_children(ps, node) - _set_pos(node, lineno, col, 0, 0) - return node - - -def _parse_renumdef(ps: Parser*) -> AST*: - """renum NAME: sugar -> a ClassDef field-identical to `class NAME(t.REnum):`. - - Members are parsed as a normal class body: a variant is a nested ClassDef (`class Some:` + field - annotations) and a payload-less variant is a plain assignment (`None_ = 0`). Base is always - Attribute(Name('t'), 'REnum'); TPC detects REnum via `issubclass(cls, t.REnum)`. No bare-name conversion. - """ - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - _advance(ps) # renum - if _cur_type(ps) != TokenType.Name: - ps.error_count += 1 - return None - name: str = _cur_str(ps) - _advance(ps) - bases: list[AST*]* = list[AST*](ps.pool, 8) - bases.append(Attribute(ps.pool, Name(ps.pool, "t", ASTCtx.Load), - "REnum", ASTCtx.Load)) - keywords: list[AST*]* = list[AST*](ps.pool, 8) - dec_list: list[AST*]* = ps.pending_decorators - ps.pending_decorators = None - _expect_op(ps, TokOp.Colon) - node: ClassDef* = ClassDef(ps.pool, name, bases, keywords, dec_list, None) - _parse_suite_into_children(ps, node) - _set_pos(node, lineno, col, 0, 0) - return node - - -def _RecordStructErr(detail: str, lineno: int, col: int, extra: str): - """Record an invalid struct form (fail-fast; first only), same reporting path as _RecordVirtualErr. - - Messages always use f-string interpolation: this language's str is i8*, so `str + str` degrades to - raw pointer add and illegal IR. - """ - global _g_parse_err_line, _g_parse_err_col, _g_parse_err_msg - if _g_parse_err_line != 0: - return - _g_parse_err_line = lineno - _g_parse_err_col = col - buf: str = stdlib.malloc(1536) - if buf is not None: - stdio.snprintf(buf, 1536, - "[TPV] invalid struct definition (%s). A struct has the same shape as a class but **can only represent a data type**:" - "(1) the virtual / novirtual modifier is forbidden (a struct has no vtable by nature, @t.NoVTable is built in);" - "(2) defining a def inside a struct is forbidden (use class if methods are needed);" - "(3) it may appear as a variant inside REnum, and a struct may contain another struct." - " Correct example: struct Point: x: int / struct Point: x: int, then a newline with 4-space indented y: int.%s" - " (struct constraints are never silently ignored: illegal forms all abort compilation)", detail, extra) - _g_parse_err_msg = buf - else: - _g_parse_err_msg = detail - - -def _parse_structdef(ps: Parser*) -> AST*: - """struct NAME: / struct NAME(Base): sugar -> a ClassDef field-identical to - `@t.NoVTable class NAME(t.CStruct):`. - - Data-only: forces no vtable via the pseudo decorator @t.NoVTable, appends Attribute(Name('t'), 'CStruct') - after the user bases as a data-class marker, and forbids def in the body (fail-fast via _RecordStructErr). - Nestable in REnum and in another struct. - """ - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - _advance(ps) # struct - if _cur_type(ps) != TokenType.Name: - _RecordStructErr("missing type name after `struct`", lineno, col, - "Correct form: struct Point: (the type name follows struct immediately).") - ps.error_count += 1 - return None - name: str = _cur_str(ps) - _advance(ps) - # Type parameters [T] / [T, U] (same as class, PEP 695 generics) - type_params: list[str]* = None - if _accept_op(ps, TokOp.Lsqb): - type_params = list[str](ps.pool, 8) - while not _match_op(ps, TokOp.Rsqb): - if _cur_type(ps) == TokenType.EndMarker: break - if _cur_type(ps) == TokenType.Name: - tp_name: str = _cur_str(ps) - _advance(ps) - if _match_op(ps, TokOp.Colon): - _advance(ps) - _parse_test(ps) - type_params.append(tp_name) - if not _accept_op(ps, TokOp.Comma): - break - _expect_op(ps, TokOp.Rsqb) - bases: list[AST*]* = list[AST*](ps.pool, 8) - keywords: list[AST*]* = list[AST*](ps.pool, 8) - # bases / keywords (same parse path as _parse_classdef, including cross-line and keyword=value) - if _accept_op(ps, TokOp.LPar): - while not _match_op(ps, TokOp.RPar): - _skip_nl(ps) - if _cur_type(ps) == TokenType.EndMarker: break - if _cur_type(ps) == TokenType.Name: - e: AST* = _parse_or_test(ps) - if _match_op(ps, TokOp.Equal): - _advance(ps) - v: AST* = _parse_test(ps) - kw_node: AST* = KwNode(ps.pool, e, v) - keywords.append(kw_node) - else: - bases.append(e) - if not _accept_op(ps, TokOp.Comma): - break - _skip_nl(ps) - _expect_op(ps, TokOp.RPar) - # Data-class marker base: appended after the user bases (real parent stays at bases[0]) - bases.append(Attribute(ps.pool, Name(ps.pool, "t", ASTCtx.Load), - "CStruct", ASTCtx.Load)) - _expect_op(ps, TokOp.Colon) - # Pseudo decorator @t.NoVTable: appended after the user decorators (same merge as _ParseVirtualModified) - dec_list: list[AST*]* = ps.pending_decorators - ps.pending_decorators = None - marker: AST* = Attribute(ps.pool, Name(ps.pool, "t", ASTCtx.Load), - "NoVTable", ASTCtx.Load) - _set_pos(marker, lineno, col, 0, 0) - merged: list[AST*]* = list[AST*](ps.pool, 8) - if merged is not None: - if dec_list is not None: - dn_s: sizet = len(dec_list) - for di_s in range(dn_s): - merged.append(dec_list.get(di_s)) - merged.append(marker) - node: ClassDef* = ClassDef(ps.pool, name, bases, keywords, merged, type_params) - # owner_is_class=1: same as class (ps.in_class_body=1 during the class body) - _parse_suite_into_children(ps, node, 1) - # struct forbids def: check the direct children - chk: list[AST*]* = node.children - if chk is not None: - cn_s: sizet = len(chk) - for ci_s in range(cn_s): - it_s: AST* = chk.get(ci_s) - if it_s is not None and it_s.kind() == ASTKind.FunctionDef: - fd_s: FunctionDef* = (FunctionDef*)(it_s) - fname_s: str = "(anonymous)" - if fd_s is not None and fd_s.name is not None: - fname_s = fd_s.name - _RecordStructErr(f"def '{fname_s}' defined inside struct '{name}'", - it_s.lineno, it_s.col_offset, - "a struct represents a data type only (same as C's struct);" - " if methods and a vtable are really needed, write the type as a class.") - ps.error_count += 1 - break - _set_pos(node, lineno, col, 0, 0) - return node - - -def _parse_match(ps: Parser*) -> AST*: - """match statement: match subject ':' NewLine Indent case+ Dedent""" - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - _advance(ps) # match - subject: AST* = _parse_test(ps) - _expect_op(ps, TokOp.Colon) - cases: list[AST*]* = list[AST*](ps.pool, 8) - node: AST* = Match(ps.pool, subject, cases, 0) - # Parsing body: Same NewLine/Indent/Dedent pattern as _parse_suite_into_children, - # but parses case clauses into a cases list instead of normal statements into children - _skip_nl(ps) - if _cur_type(ps) == TokenType.NewLine: - _advance(ps) - _skip_nl(ps) - if _cur_type(ps) == TokenType.Indent: - _advance(ps) - while _cur_type(ps) != TokenType.Dedent and _cur_type(ps) != TokenType.EndMarker: - if _cur_kw(ps) == Keyword.Case: - case_node: AST* = _parse_match_case(ps) - if case_node is not None: - cases.append(case_node) - else: - _advance(ps) # Skip non-case token to avoid infinite loop - _expect_type(ps, TokenType.Dedent) - _set_pos(node, lineno, col, 0, 0) - return node - - -# ============================================================ -# switch statement (C switch semantics, desugared to Match(is_switch=1)): `switch :` with -# `case :` / `default:` arms. C switch rules: no fall-through unless -# `fallthrough` is written in the case body. Case labels allow only constants / enum variants / -# multi-value `case 1 | 2:`, not structured patterns. -# ============================================================ - -def _is_switch_default_kw(ps: Parser*) -> int: - """`default` is a context keyword inside a switch suite: matches `default:` or - `default if :` (the latter supports a guard like `case _ if `).""" - if _cur_type(ps) != TokenType.Name: - return 0 - s: str = _cur_str(ps) - if s is None or string.strcmp(s, "default") != 0: - return 0 - nxt: Token* = ps.cur.next - if nxt is None: - return 0 - if nxt.type == TokenType.Op and nxt.op_subtype == TokOp.Colon: - return 1 - if nxt.type == TokenType.Name and nxt.kw_subtype == Keyword.If: - return 1 - return 0 - - -def _RecordSwitchCaseErr(detail: str, lineno: int, col: int): - """Record an invalid switch case label (fail-fast; first only)""" - global _g_parse_err_line, _g_parse_err_col, _g_parse_err_msg - if _g_parse_err_line != 0: - return - _g_parse_err_line = lineno - _g_parse_err_col = col - buf: str = stdlib.malloc(512) - if buf is not None: - stdio.snprintf(buf, 512, - "invalid switch case label (%s). switch supports only constant literals or enum variant labels" - " (multi-value `case 1 | 2:` allowed); it does not accept structural patterns such as sequences/mappings/captures." - " Correct examples: case 1: / case \"a\": / case Color.Red: / case Opt.Some():", detail) - _g_parse_err_msg = buf - else: - _g_parse_err_msg = "invalid switch case label" - - -def _CheckSwitchCasePattern(ps: Parser*, pat: AST*) -> int: - """Validate a switch case label (only constants / enum variants / multi-value patterns allowed)""" - if pat is None: - return 0 - k: int = pat.kind() - if k == ASTKind.MatchValue or k == ASTKind.MatchSingleton or k == ASTKind.MatchClass: - return 1 - if k == ASTKind.MatchAs: - # Only the wildcard (`case _`, equivalent to default) is legal; as-bindings with a subpattern are not - ma_w: MatchAs* = MatchAs*(pat) - if ma_w.pattern is None: - return 1 - return 0 - if k == ASTKind.MatchOr: - mo: MatchOr* = MatchOr*(pat) - pats: list[AST*]* = mo.patterns - if pats is None or len(pats) == 0: - return 0 - i: sizet = 0 - n: sizet = len(pats) - while i < n: - sub: AST* = pats.get(i) - if sub is None: - return 0 - sk: int = sub.kind() - if sk != ASTKind.MatchValue and sk != ASTKind.MatchSingleton \ - and sk != ASTKind.MatchClass: - return 0 - i = i + 1 - return 1 - return 0 - - -def _parse_switch_case(ps: Parser*) -> AST*: - """switch `case [if ] ':' suite` (guard supported). - - A bare name is a capture pattern (MatchAs) in match but a value reference in switch (like C's - `case NAME:`), so `MatchAs(pattern=None, name=X)` is normalized to `MatchValue(Name('X'))`; `_` - stays a wildcard. The guard is parsed like _parse_match_case and passed to MatchCase.guard. - """ - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - _advance(ps) # case - pattern: AST* = _parse_pattern(ps) - if pattern is not None and pattern.kind() == ASTKind.MatchAs: - ma: MatchAs* = MatchAs*(pattern) - if ma.pattern is None and ma.name is not None: - ref: AST* = Name(ps.pool, ma.name, ASTCtx.Load) - _set_pos(ref, pattern.lineno, pattern.col_offset, 0, 0) - mv: AST* = MatchValue(ps.pool, ref) - _set_pos(mv, pattern.lineno, pattern.col_offset, 0, 0) - pattern = mv - if _CheckSwitchCasePattern(ps, pattern) == 0: - _RecordSwitchCaseErr("case label is not a constant/enum variant", lineno, col) - ps.error_count += 1 - # guard: `case if ':'` (same form as _parse_match_case) - guard: AST* = None - if _cur_kw(ps) == Keyword.If: - _advance(ps) - guard = _parse_or_test(ps) - _expect_op(ps, TokOp.Colon) - node: AST* = MatchCase(ps.pool, pattern, guard) - _parse_suite_into_children(ps, node) - _set_pos(node, lineno, col, 0, 0) - return node - - -def _parse_switch_default(ps: Parser*) -> AST*: - """switch `default [if ] ':' suite` -- desugared to `case []` (MatchAs wildcard). - - `default` is switch's spelling of `case _`; it supports a guard like `case _ if `, passed - to MatchCase. Without if the behavior is unchanged (guard=None). - """ - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - _advance(ps) # default - guard: AST* = None - if _cur_kw(ps) == Keyword.If: - _advance(ps) - guard = _parse_or_test(ps) - _expect_op(ps, TokOp.Colon) - wild: AST* = MatchAs(ps.pool, None, None) - _set_pos(wild, lineno, col, 0, 0) - node: AST* = MatchCase(ps.pool, wild, guard) - _parse_suite_into_children(ps, node) - _set_pos(node, lineno, col, 0, 0) - return node - - -def _parse_switch(ps: Parser*) -> AST*: - """switch statement: switch subject ':' NewLine Indent (case|default)+ Dedent""" - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - _advance(ps) # switch - subject: AST* = _parse_test(ps) - _expect_op(ps, TokOp.Colon) - cases: list[AST*]* = list[AST*](ps.pool, 8) - node: AST* = Match(ps.pool, subject, cases, 1) - _skip_nl(ps) - if _cur_type(ps) == TokenType.NewLine: - _advance(ps) - _skip_nl(ps) - if _cur_type(ps) == TokenType.Indent: - _advance(ps) - while _cur_type(ps) != TokenType.Dedent and _cur_type(ps) != TokenType.EndMarker: - if _cur_kw(ps) == Keyword.Case: - case_node: AST* = _parse_switch_case(ps) - if case_node is not None: - cases.append(case_node) - elif _is_switch_default_kw(ps) != 0: - def_node: AST* = _parse_switch_default(ps) - if def_node is not None: - cases.append(def_node) - else: - _advance(ps) # Skip non-case token to avoid infinite loop - _expect_type(ps, TokenType.Dedent) - _set_pos(node, lineno, col, 0, 0) - return node - - -# ============================================================ -# Match pattern parsing -# ============================================================ - -def _make_name_ast(pool: memhub.MemManager*, node: Name*) -> AST*: - # Analysis: "Convert Name* to AST* (Solving the TransPyC Type Inference Problem)" - return node - - -def _parse_pattern(ps: Parser*) -> AST*: - """pattern: or_pattern ['as' NAME] - or_pattern: closed_pattern ('|' closed_pattern)*""" - pat: AST* = _parse_closed_pattern(ps) - if pat is None: return None - # OR pattern - if _match_op(ps, TokOp.VBar): - patterns: list[AST*]* = list[AST*](ps.pool, 8) - patterns.append(pat) - while _accept_op(ps, TokOp.VBar): - p: AST* = _parse_closed_pattern(ps) - if p is not None: - patterns.append(p) - or_node: AST* = MatchOr(ps.pool, patterns) - _set_pos(or_node, pat.lineno, pat.col_offset, 0, 0) - pat = or_node - # AS pattern - if _cur_kw(ps) == Keyword.As: - _advance(ps) - if _cur_type(ps) != TokenType.Name: - ps.error_count += 1 - return pat - name_str: str = _cur_str(ps) - _advance(ps) - as_node: AST* = MatchAs(ps.pool, pat, name_str) - _set_pos(as_node, pat.lineno, pat.col_offset, 0, 0) - return as_node - return pat - - -def _parse_closed_pattern(ps: Parser*) -> AST*: - """closed_pattern: literal | capture | wildcard | value | group | sequence | mapping | class""" - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - ttype: int = _cur_type(ps) - val: AST* - node: AST* - tok: Token* - sval: str - - # Number literal pattern - if ttype == TokenType.Number: - tok = ps.cur - _advance(ps) - if tok.is_float: - val = _new_const_float(ps.pool, tok.float_val, lineno, col) - else: - val = _new_const_int(ps.pool, tok.int_val, lineno, col) - node = MatchValue(ps.pool, val) - _set_pos(node, lineno, col, 0, 0) - return node - - # String literal pattern - if ttype == TokenType.String: - sval = _cur_str(ps) - is_dquote_m: int = ps.cur.kw_subtype & 16 - _advance(ps) - val = _new_const_str(ps.pool, sval, lineno, col, is_dquote_m) - node = MatchValue(ps.pool, val) - _set_pos(node, lineno, col, 0, 0) - return node - - # Negative number literal pattern: -1, -1.5 - if _match_op(ps, TokOp.Minus): - _advance(ps) - if _cur_type(ps) == TokenType.Number: - tok = ps.cur - # Combined unary-minus + integer-literal overflow check (same rule as _parse_factor): - # a negative literal with magnitude > 2^63 yields < -2^63 -> fail-fast, no silent wrap. - if tok.is_float == 0 and _lex_int_neg_exceeds_i64(_cur_str(ps)) != 0: - _record_neg_literal_overflow(_cur_str(ps), lineno, col) - ps.error_count += 1 - return None - _advance(ps) - if tok.is_float: - val = _new_const_float(ps.pool, -tok.float_val, lineno, col) - else: - val = _new_const_int(ps.pool, -tok.int_val, lineno, col) - node = MatchValue(ps.pool, val) - _set_pos(node, lineno, col, 0, 0) - return node - ps.error_count += 1 - return None - - # Name: None/True/False/_ /capture/value/class - if ttype == TokenType.Name: - kw: int = _cur_kw(ps) - # None/True/False → MatchSingleton - if kw == Keyword.None_ or kw == Keyword.True_ or kw == Keyword.False_: - _advance(ps) - if kw == Keyword.None_: - val = _new_const_none(ps.pool, lineno, col) - elif kw == Keyword.True_: - val = _new_const_bool(ps.pool, 1, lineno, col) - else: - val = _new_const_bool(ps.pool, 0, lineno, col) - node = MatchSingleton(ps.pool, val) - _set_pos(node, lineno, col, 0, 0) - return node - name_str: str = _cur_str(ps) - # Wildcard pattern: _ - if string.strcmp(name_str, "_") == 0: - _advance(ps) - node = MatchAs(ps.pool, None, None) - _set_pos(node, lineno, col, 0, 0) - return node - _advance(ps) - # Check dotted name (value pattern or class pattern) - if _match_op(ps, TokOp.Dot): - name_node: Name* = Name(ps.pool, name_str, ASTCtx.Load) - _set_pos(name_node, lineno, col, 0, 0) - base: AST* = _make_name_ast(ps.pool, name_node) - while _match_op(ps, TokOp.Dot): - _advance(ps) - if _cur_type(ps) != TokenType.Name: - ps.error_count += 1 - break - attr_name: str = _cur_str(ps) - attr_lineno: int = _cur_lineno(ps) - attr_col: int = _cur_col(ps) - _advance(ps) - base = Attribute(ps.pool, base, attr_name, ASTCtx.Load) - _set_pos(base, attr_lineno, attr_col, 0, 0) - # class pattern: dotted.name(...) - if _match_op(ps, TokOp.LPar): - return _parse_class_pattern(ps, base, lineno, col) - # value pattern: dotted.name - node = MatchValue(ps.pool, base) - _set_pos(node, lineno, col, 0, 0) - return node - # class pattern: Name(...) - if _match_op(ps, TokOp.LPar): - cls_node: Name* = Name(ps.pool, name_str, ASTCtx.Load) - _set_pos(cls_node, lineno, col, 0, 0) - cls: AST* = _make_name_ast(ps.pool, cls_node) - return _parse_class_pattern(ps, cls, lineno, col) - # capture pattern: NAME - node = MatchAs(ps.pool, None, name_str) - _set_pos(node, lineno, col, 0, 0) - return node - - # group pattern / sequence pattern (tuple): (...) - if _match_op(ps, TokOp.LPar): - _advance(ps) - inner: AST* = _parse_pattern(ps) - if _accept_op(ps, TokOp.Comma): - # sequence pattern (tuple) - patterns: list[AST*]* = list[AST*](ps.pool, 8) - if inner is not None: - patterns.append(inner) - while not _match_op(ps, TokOp.RPar) and _cur_type(ps) != TokenType.EndMarker: - p: AST* = _parse_pattern(ps) - if p is not None: - patterns.append(p) - if not _accept_op(ps, TokOp.Comma): - break - _expect_op(ps, TokOp.RPar) - node = MatchSequence(ps.pool, patterns) - _set_pos(node, lineno, col, 0, 0) - return node - _expect_op(ps, TokOp.RPar) - return inner # group pattern - - # sequence pattern (list): [...] - if _match_op(ps, TokOp.Lsqb): - return _parse_sequence_pattern(ps, TokOp.Rsqb, lineno, col) - - # mapping pattern: {...} - if _match_op(ps, TokOp.LBrace): - return _parse_mapping_pattern(ps, lineno, col) - - ps.error_count += 1 - return None - - -def _parse_sequence_pattern(ps: Parser*, close_op: int, - lineno: int, col: int) -> AST*: - """sequence_pattern: '[' [pattern_list] ']'""" - patterns: list[AST*]* = list[AST*](ps.pool, 8) - _advance(ps) # [ - while not _match_op(ps, close_op) and _cur_type(ps) != TokenType.EndMarker: - # *name (star pattern) - if _match_op(ps, TokOp.Star): - _advance(ps) - if _cur_type(ps) == TokenType.Name: - star_name: str = _cur_str(ps) - _advance(ps) - star_node: AST* - if string.strcmp(star_name, "_") == 0: - star_node = MatchStar(ps.pool, None) - else: - star_node = MatchStar(ps.pool, star_name) - _set_pos(star_node, lineno, col, 0, 0) - patterns.append(star_node) - else: - p: AST* = _parse_pattern(ps) - if p is not None: - patterns.append(p) - if not _accept_op(ps, TokOp.Comma): - break - _expect_op(ps, close_op) - node: AST* = MatchSequence(ps.pool, patterns) - _set_pos(node, lineno, col, 0, 0) - return node - - -def _parse_mapping_pattern(ps: Parser*, - lineno: int, col: int) -> AST*: - """mapping_pattern: '{' [items] '}' - items: key ':' pattern (',' key ':' pattern)* [',' '**' NAME] | '**' NAME""" - keys: list[AST*]* = list[AST*](ps.pool, 8) - patterns: list[AST*]* = list[AST*](ps.pool, 8) - rest: str = None - node: AST* - key: AST* - val_pat: AST* - _advance(ps) # { - if _match_op(ps, TokOp.RBrace): - _advance(ps) - node = MatchMapping(ps.pool, keys, patterns, rest) - _set_pos(node, lineno, col, 0, 0) - return node - while _cur_type(ps) != TokenType.EndMarker: - if _match_op(ps, TokOp.StarStar): - _advance(ps) - if _cur_type(ps) == TokenType.Name: - rest = _cur_str(ps) - _advance(ps) - break - key = _parse_expr(ps) - _expect_op(ps, TokOp.Colon) - val_pat = _parse_pattern(ps) - if key is not None: - keys.append(key) - if val_pat is not None: - patterns.append(val_pat) - if not _accept_op(ps, TokOp.Comma): - break - _expect_op(ps, TokOp.RBrace) - node = MatchMapping(ps.pool, keys, patterns, rest) - _set_pos(node, lineno, col, 0, 0) - return node - - -def _parse_class_pattern(ps: Parser*, cls: AST*, - lineno: int, col: int) -> AST*: - """class_pattern: cls '(' [pattern_list] ')' - pattern_list: positional_pattern (',' positional_pattern)* - [',' keyword_pattern]* | keyword_pattern (',' keyword_pattern)* - keyword_pattern: NAME '=' pattern""" - patterns: list[AST*]* = list[AST*](ps.pool, 8) - kwd_attrs: list[AST*]* = list[AST*](ps.pool, 8) - kwd_patterns: list[AST*]* = list[AST*](ps.pool, 8) - p: AST* - _advance(ps) # ( - if not _match_op(ps, TokOp.RPar): - while _cur_type(ps) != TokenType.EndMarker: - # keyword pattern: NAME '=' pattern - if _cur_type(ps) == TokenType.Name and ps.cur is not None: - nxt: Token* = ps.cur.next - if nxt is not None and nxt.type == TokenType.Op and nxt.op_subtype == TokOp.Equal: - attr_name: str = _cur_str(ps) - _advance(ps) # NAME - _advance(ps) # = - kwd_pat: AST* = _parse_pattern(ps) - attr_n: Name* = Name(ps.pool, attr_name, ASTCtx.Load) - _set_pos(attr_n, lineno, col, 0, 0) - attr_node: AST* = _make_name_ast(ps.pool, attr_n) - kwd_attrs.append(attr_node) - if kwd_pat is not None: - kwd_patterns.append(kwd_pat) - else: - p = _parse_pattern(ps) - if p is not None: - patterns.append(p) - else: - p = _parse_pattern(ps) - if p is not None: - patterns.append(p) - if not _accept_op(ps, TokOp.Comma): - break - _expect_op(ps, TokOp.RPar) - node: AST* = MatchClass(ps.pool, cls, patterns, kwd_attrs, kwd_patterns) - _set_pos(node, lineno, col, 0, 0) - return node - - -def _parse_match_case(ps: Parser*) -> AST*: - """case pattern [if guard] ':' suite""" - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - _advance(ps) # case - guard: AST* = None - pattern: AST* = _parse_pattern(ps) - if _cur_kw(ps) == Keyword.If: - _advance(ps) - guard = _parse_or_test(ps) - _expect_op(ps, TokOp.Colon) - node: AST* = MatchCase(ps.pool, pattern, guard) - _parse_suite_into_children(ps, node) - _set_pos(node, lineno, col, 0, 0) - return node - - -# ============================================================ -# __attribute__ decorator desugar (C-style attribute syntax): `@__attribute__(section(".text.wo"), -# always_inline)` equals hand-written `@c.Attribute(t.attr.section(...), t.attr.always_inline)`. -# Arguments use unprefixed names, each desugared to `t.attr.` (calls keep their args) with the -# outer call becoming `c.Attribute(...)`, so the result is field-identical and recognized as-is. -# Works for class and def; arguments already in `t.attr.X` / `c.Attribute(...)` form are kept. -# ============================================================ -def _IsMagicAttributeCallee(node: AST*) -> int: - """Whether the node is `__attribute__` (Name form, or the trailing X.__attribute__ form)""" - if node is None: - return 0 - k: int = node.kind() - if k == ASTKind.Name: - nm: Name* = Name*(node) - if nm.id is not None and string.strcmp(nm.id, "__attribute__") == 0: - return 1 - return 0 - if k == ASTKind.Attribute: - at: Attribute* = Attribute*(node) - if at.attr is not None and string.strcmp(at.attr, "__attribute__") == 0: - return 1 - return 0 - - -def _MakeTAttrMarker(ps: Parser*, arg: AST*) -> AST*: - """Desugar one __attribute__ argument into `t.attr.` (calls keep their arguments). - - `always_inline` (Name) -> `t.attr.always_inline`; `section(".text.wo")` (Call) -> `t.attr.section(...)`; - an argument already in `t.attr.X` form is returned as-is. - """ - if arg is None: - return None - k: int = arg.kind() - if k == ASTKind.Attribute: - at0: Attribute* = Attribute*(arg) - if at0.value is not None and at0.value.kind() == ASTKind.Attribute: - return arg - return arg - attr_root: AST* = Attribute(ps.pool, Name(ps.pool, "t", ASTCtx.Load), - "attr", ASTCtx.Load) - if k == ASTKind.Name: - nm2: Name* = Name*(arg) - if nm2.id is None: - return arg - mk: AST* = Attribute(ps.pool, attr_root, nm2.id, ASTCtx.Load) - _set_pos(mk, nm2.lineno, nm2.col_offset, 0, 0) - return mk - if k == ASTKind.Call: - cl2: Call* = Call*(arg) - if cl2.func is None or cl2.func.kind() != ASTKind.Name: - return arg - fnm: Name* = Name*(cl2.func) - if fnm.id is None: - return arg - inner: AST* = Attribute(ps.pool, attr_root, fnm.id, ASTCtx.Load) - _set_pos(inner, fnm.lineno, fnm.col_offset, 0, 0) - mk2: AST* = Call(ps.pool, inner, cl2.args, cl2.keywords) - _set_pos(mk2, cl2.lineno, cl2.col_offset, 0, 0) - return mk2 - return arg - - -def _DesugarMagicAttribute(ps: Parser*, expr: AST*) -> AST*: - """`@__attribute__(...)` -> `@c.Attribute(t.attr.x, ...)`; other decorators are returned as-is""" - if expr is None or expr.kind() != ASTKind.Call: - return expr - cl: Call* = Call*(expr) - if _IsMagicAttributeCallee(cl.func) == 0: - return expr - c_func: AST* = Attribute(ps.pool, Name(ps.pool, "c", ASTCtx.Load), - "Attribute", ASTCtx.Load) - new_args: list[AST*]* = list[AST*](ps.pool, 8) - if new_args is None: - return expr - if cl.args is not None: - an: sizet = len(cl.args) - for ai in range(an): - mk: AST* = _MakeTAttrMarker(ps, cl.args.get(ai)) - if mk is not None: - new_args.append(mk) - new_call: AST* = Call(ps.pool, c_func, new_args, None) - _set_pos(new_call, cl.lineno, cl.col_offset, 0, 0) - return new_call - - -def _parse_decorators(ps: Parser*) -> list[AST*]*: - """Parse a decorator chain, returning a list.""" - decs: list[AST*]* = list[AST*](ps.pool, 8) - while _match_op(ps, TokOp.At): - lineno: int = _cur_lineno(ps) - col: int = _cur_col(ps) - _advance(ps) # @ - expr: AST* = _parse_namedexpr(ps) - # C-style `@__attribute__(...)` desugars into the existing `@c.Attribute(t.attr....)` - expr = _DesugarMagicAttribute(ps, expr) - _skip_nl(ps) - if _cur_type(ps) == TokenType.NewLine: - _advance(ps) - decs.append(expr) - return decs - - -# ============================================================ -# Module entry -# ============================================================ - -def _parse_module_body(ps: Parser*, module_node: AST*): - """Parse the module body, appending to module_node.""" - _skip_nl(ps) - stmt_idx: int = 0 - while _cur_type(ps) != TokenType.EndMarker: - stmt_idx += 1 - # fail-fast anti-hang: over-budget steps (cyclic token list / cursor spinning) report and return - if ps.step_count > ps.step_limit: - _record_parse_err(ps) - return - # Decorator - if _match_op(ps, TokOp.At): - ps.pending_decorators = _parse_decorators(ps) - # Record current token pointer to check for infinite loop - before_cur: Token* = ps.cur - _parse_statement_into(ps, module_node) - # Defense: If parsing doesn't advance the token (like when _parse_atom encounters an unrecognized token and returns None), - # force it to move forward to avoid an infinite loop in the main loop; accumulate error_count for the caller to diagnose - if ps.cur is before_cur: - ps.error_count += 1 - _advance(ps) - _skip_nl(ps) - - -def parse_tokens(pool: memhub.MemManager*, - tokens: Token*) -> AST*: - """Parse an AST Module from a token list.""" - if pool is None or tokens is None: - return None - global _g_parse_err_line, _g_parse_err_col, _g_parse_err_msg - _g_parse_err_line = 0 - _g_parse_err_col = 0 - _g_parse_err_msg = None - # Lexer fail-fast takes priority: overflows recorded during tokenize (e.g. integer literal > 2^64-1) - # are forwarded to the parse error channel here; lexer has no independent fatal exit. - lex_line: int = get_lex_err_line() - if lex_line != 0: - _g_parse_err_line = lex_line - _g_parse_err_col = get_lex_err_col() - _g_parse_err_msg = get_lex_err_msg() - return None - ps: Parser* = new_parser(pool) - if ps is None: return None - _parser_init(ps, tokens, pool) - # fail-fast anti-hang: count the token list first, detecting a cycle. A cyclic list (Token appended - # twice or aliased) would hang the statement loop; exceeding the cap before the tail means a cycle. - tok_count: sizet = 0 - walk_t: Token* = tokens - while walk_t is not None: - walk_t = walk_t.next - tok_count += 1 - if tok_count > PARSER_MAX_TOKENS: - _g_parse_err_line = _cur_lineno(ps) - _g_parse_err_col = _cur_col(ps) - _g_parse_err_msg = "token list is cyclic: the token list emitted by the lexer has no terminator (_emit appended the same Token twice or aliased it); parsing aborted" - return None - # Budget: the watchdog counts "query current token" calls (well below 256 per token in valid parses, - # so it cannot trip falsely) while catching cycles/spins; it also bounds _advance and statement loops. - ps.step_limit = tok_count * 256 + 262144 - module_node: Module* = Module(pool) - _parse_module_body(ps, module_node) - # fail-fast: on a parse error (error_count > 0) return None so the caller aborts, avoiding silently - # compiling broken code (e.g. a function definition left without its def keyword). - if ps.error_count > 0: - # Fallback position/reason: the parser has 40+ `ps.error_count += 1` sites and not all call - # _record_*. Without a record, dump_parse_traceback can only print `File "..."` + - # `SyntaxError` with no line or reason. Fill in a locatable message from the current token so the - # report always has file:line:col + reason. - if _g_parse_err_line == 0: - _g_parse_err_line = _cur_lineno(ps) - _g_parse_err_col = _cur_col(ps) - err_s_e: str = _cur_str(ps) - if err_s_e is None: - err_s_e = "(none)" - buf_e: str = stdlib.malloc(256) - if buf_e is not None: - stdio.snprintf(buf_e, 256, - "syntax error: the parser recorded %d error(s) but no specific reason;" - " token near the error position='%s' (type=%d)", - ps.error_count, err_s_e, _cur_type(ps)); - _g_parse_err_msg = buf_e - else: - _g_parse_err_msg = "syntax error: the parser recorded no specific reason" - return None - return module_node +from stdint import * +import memhub +import string +import stdio +import stdlib +from .tokens import (Token, TokenType, Keyword, TokOp) +from .base import ( + AST, ASTCtx, ASTKind, OpKind, ASTFlag, + _set_pos, _binop_from_op, _augop_from_op, _cmpop_from_op, _copy_str, _inherit_pos, + _init_ast, + CONST_INT, CONST_FLOAT, CONST_STR, CONST_BOOL, CONST_NONE, CONST_CHAR, + FLAG_IS_ASYNC, FLAG_IS_EXTERN, FLAG_IS_EXPORT, FLAG_IS_DEFINE, +) +from .exprs import ( + BoolOp, BinOp, UnaryOp, Lambda, IfExp, Dict, Set, + ListComp, SetComp, DictComp, GeneratorExp, + Await, Yield, YieldFrom, FormattedValue, JoinedStr, + Constant, NamedExpr, Attribute, Subscript, Starred, Name, + List, Tuple, Slice, Call, Compare, OpNode, New, + NEW_SRC_DEFAULT, NEW_SRC_STACK, NEW_SRC_HEAP, NEW_SRC_MEMGR, + NEW_LIFE_DEFAULT, NEW_LIFE_SLIFE, NEW_LIFE_HLIFE, +) +from .stmts import ( + Module, Expression, Interactive, FunctionType, + FunctionDef, ClassDef, Return, Delete, Assign, AugAssign, AnnAssign, + For, While, If, With, Raise, Try, Assert, Global, Nonlocal, + Pass, Break, Continue, FallThrough, Expr, Import, ImportFrom, Match, +) +from .astaux import ( + ExceptHandler, Arguments, Arg, Keyword as KwNode, Alias, WithItem, Comprehension, +) +from .match import ( + MatchCase, MatchValue, MatchSingleton, MatchSequence, + MatchMapping, MatchClass, MatchStar, MatchAs, MatchOr, +) +from .lexer import ( + Lexer, _lexer_init, tokenize, new_lexer, + get_lex_err_line, get_lex_err_col, get_lex_err_msg, + _lex_int_neg_exceeds_i64, +) + + +# ============================================================ +# Parser State +# ============================================================ + +# Hard cap on the token linked-list length (cycle detection): normal sources have far fewer +# tokens, so reaching this count without the tail means the list is cyclic (see parse_tokens). +define PARSER_MAX_TOKENS = 8000000 + + +class Parser: + tokens: Token* # token linked list head + cur: Token* # current token + prev: Token* # previous consumed token (for the `T*(x)` adjacency test) + pool: memhub.MemManager* # memory pool + error_count: int # error count + pending_decorators: list[AST*]* # pending decorators list + # Pending parsed left operand for a leading `*` in argument position (see _parse_call / _parse_power). + starred_seed: AST* + # `static` modifier (soft keyword) pending flag: set after consuming `static` before a `Name :` + # declaration; the next _parse_type_ann prepends t.CStatic and clears it (see _ParseStaticModified). + pending_static: int + # Whether the suite currently being parsed is directly owned by a class (1 = class body, 0 = other), + # maintained by _parse_suite_into_children for the virtual/novirtual class-body check. + in_class_body: int + # Step budget (_advance count / parse_tokens budget): the statement loop fails fast once exceeded, + # never looping forever (see the cycle detection in parse_tokens). + step_count: sizet + step_limit: sizet + # Watchdog: counter and convergence flag for "query current token" (see _tick). When tick_count + # exceeds step_limit, aborted=1, entries return converged values and every loop ends in bounded rounds. + tick_count: sizet + aborted: int + + def __new__(self, pool: memhub.MemManager*): + ptr: Parser* = pool.alloc(Parser.__sizeof__()) + if ptr: string.memset(ptr, 0, Parser.__sizeof__()) + return ptr + + +def new_parser(pool: memhub.MemManager*) -> Parser*: + """Allocate from the pool and return the Parser (cross-module safe wrapper)""" + return Parser(pool) + + +def _parser_init(ps: Parser*, tokens: Token*, + pool: memhub.MemManager*): + if ps is None: return + ps.tokens = tokens + ps.cur = tokens + ps.prev = None + ps.pool = pool + ps.error_count = 0 + ps.pending_decorators = None + ps.starred_seed = None + ps.pending_static = 0 + ps.in_class_body = 0 + ps.step_count = 0 + # Default budget is the cap (parse_tokens later tightens it to the real token count) so entries that + # bypass parse_tokens (parse_expression etc.) do not misreport at 0, while a hard bound still prevents loops. + ps.step_limit = PARSER_MAX_TOKENS + ps.tick_count = 0 + ps.aborted = 0 + + +# ============================================================ +# Parser watchdog (static convergence: no silent hang) +# +# Any loop that spins without consuming a token must repeatedly query the current token, so every query +# entry (_cur_type / _cur_op / _cur_kw / _match_op / _accept_op / _advance) ticks the watchdog; once the +# budget is exceeded it sets aborted, records a parse error and returns a converged value: +# _cur_type -> EndMarker, _cur_op / _cur_kw -> 0, _match_op / _accept_op -> 1 (jump to end), +# _advance -> jump to end. parse_tokens then returns None with error_count > 0 and the caller reports +# file:line:col SyntaxError; there is no silent-hang path left. +# ============================================================ + +def _tick(ps: Parser*): + """Tick the watchdog; over budget sets aborted and records the first parse error (with position)""" + if ps is None: return + if ps.aborted != 0: return + ps.tick_count += 1 + if ps.tick_count > ps.step_limit: + ps.aborted = 1 + _record_parse_err(ps) + ps.error_count += 1 + + +# ============================================================ +# Token Navigation Helpers +# ============================================================ + +def _advance(ps: Parser*): + """Advance to the next token""" + _tick(ps) + if ps.aborted != 0: + ps.cur = None + return + if ps.cur is None: return + if ps.cur.next is not None: + ps.prev = ps.cur + ps.cur = ps.cur.next + ps.step_count += 1 + + +def _cur_type(ps: Parser*) -> int: + _tick(ps) + if ps.aborted != 0: return TokenType.EndMarker + if ps.cur is None: return TokenType.EndMarker + return ps.cur.type + + +def _cur_op(ps: Parser*) -> int: + _tick(ps) + if ps.aborted != 0: return 0 + if ps.cur is None: return 0 + return ps.cur.op_subtype + + +def _cur_kw(ps: Parser*) -> int: + _tick(ps) + if ps.aborted != 0: return 0 + if ps.cur is None: return 0 + return ps.cur.kw_subtype + + +def _is_funcdef_kw(ps: Parser*, kw: int) -> int: + """Whether the current token is a function-definition keyword (def and its aliases + fn/fun/func/function, plus extern/export). + + def is a hard keyword. fn/fun/func/function/extern/export are soft: they count only when the next + token is a Name followed by '(' or a generic '['; extern/export also accept ` def A(...)`. + Otherwise they are ordinary identifiers (e.g. the AST `func` field or `func: AST*`). + """ + if kw == Keyword.Def: + return 1 + if kw == Keyword.Extern or kw == Keyword.Export: + # extern/export def A(...) or extern/export A(...) + # (after extern/export comes def or Name + '(' / '[') + nxt: Token* = ps.cur.next + if nxt is None: return 0 + if nxt.kw_subtype == Keyword.Def: return 1 + if nxt.type != TokenType.Name: return 0 + nxt2: Token* = nxt.next + if nxt2 is None: return 0 + if nxt2.type == TokenType.Op and (nxt2.op_subtype == TokOp.LPar or nxt2.op_subtype == TokOp.Lsqb): + return 1 + return 0 + if kw == Keyword.Fn or kw == Keyword.Fun \ + or kw == Keyword.Func or kw == Keyword.Function \ + or kw == Keyword.Define: + # Soft-keyword lookahead: next is a Name, the one after is '(' or a generic '[T...]' (PEP 695) + # (the define macro function matches def, with t.CDefine appended to the return annotation) + nxt: Token* = ps.cur.next + if nxt is None: + return 0 + if nxt.type != TokenType.Name: + return 0 + nxt2: Token* = nxt.next + if nxt2 is None: + return 0 + if nxt2.type == TokenType.Op \ + and (nxt2.op_subtype == TokOp.LPar or nxt2.op_subtype == TokOp.Lsqb): + return 1 + return 0 + return 0 + + +def _is_define_const(ps: Parser*) -> int: + """Soft-keyword lookahead for a define constant declaration (1 = yes, 0 = no). + + `define NAME[: T] = `: next is a Name, then '=' (no annotation) or ':' (annotated). + `define NAME(...)` and `define` used as an identifier do not match. `define NAME: T` (missing '=') + does match and then fails loudly in _parse_define_const's _expect_op(Equal). + """ + if _cur_kw(ps) != Keyword.Define: return 0 + nxt: Token* = ps.cur.next + if nxt is None or nxt.type != TokenType.Name: return 0 + nxt2: Token* = nxt.next + if nxt2 is None or nxt2.type != TokenType.Op: return 0 + if nxt2.op_subtype == TokOp.Equal or nxt2.op_subtype == TokOp.Colon: return 1 + return 0 + + +def _is_typedef(ps: Parser*) -> int: + """Soft-keyword test for a typedef alias declaration (1 = yes, 0 = no). + + `typedef T1 = T2` is fully equivalent to `T1: t.CTypedef = T2`. `typedef` is a statement-level + reserved word (never an identifier in the repo); invalid forms fail loudly in _parse_typedef. + """ + if _cur_kw(ps) != Keyword.Typedef: + return 0 + return 1 + + +def _is_export_extern_var(ps: Parser*) -> int: + """Soft-keyword test for `export NAME :` / `extern NAME :` variable declarations (1 = yes, 0 = no). + + export/extern also serve function declarations (see _is_funcdef_kw); the third token distinguishes + them: '(' or '[' is a function, ':' is a variable. Other forms do not match; invalid forms fail loudly. + """ + kw: int = _cur_kw(ps) + if kw != Keyword.Export and kw != Keyword.Extern: + return 0 + nxt: Token* = ps.cur.next + if nxt is None: + return 0 + if nxt.type != TokenType.Name: + return 0 + # The name must be a non-keyword identifier (excludes forms like `export def f()`) + if nxt.kw_subtype != Keyword.NotKeyword: + return 0 + nxt2: Token* = nxt.next + if nxt2 is None: + return 0 + if nxt2.type != TokenType.Op: + return 0 + if nxt2.op_subtype != TokOp.Colon: + return 0 + return 1 + + +def _is_named_soft(ps: Parser*, name: str) -> int: + """Whether the current token is the given soft-keyword identifier (Name, not a hard keyword).""" + if _cur_type(ps) != TokenType.Name: + return 0 + if _cur_kw(ps) != Keyword.NotKeyword: + return 0 + s: str = _cur_str(ps) + if s is None: + return 0 + if string.strcmp(s, name) == 0: + return 1 + return 0 + + +def _is_cif_stmt(ps: Parser*) -> int: + """Whether the current token starts a `cif` statement (soft keyword; 1 = yes / 0 = no). + + `cif EXPR:` is a compile-time conditional statement. An assignment/annotation/attribute form + (`cif = ...` / `cif: T = ...` / `cif += ...` / `cif.attr`) is an ordinary identifier use and + must not be treated as the keyword. + """ + if _is_named_soft(ps, "cif") == 0: + return 0 + nxt: Token* = ps.cur.next + if nxt is None: + return 0 + if nxt.type == TokenType.Op: + o: int = nxt.op_subtype + if o == TokOp.Equal or o == TokOp.Colon or o == TokOp.ColonEq or o == TokOp.Dot \ + or o == TokOp.PlusEq or o == TokOp.MinusEq or o == TokOp.StarEq \ + or o == TokOp.SlashEq or o == TokOp.DSlashEq or o == TokOp.PercentEq \ + or o == TokOp.AtEq or o == TokOp.AmpEq or o == TokOp.VBarEq \ + or o == TokOp.CaretEq or o == TokOp.GtGtEq or o == TokOp.LtLtEq \ + or o == TokOp.StarEqEq: + return 0 + return 1 + + +# Parse error position and message (fail-fast, Python SyntaxError style, first error only) +_g_parse_err_line: int = 0 +_g_parse_err_col: int = 0 +_g_parse_err_msg: str = None + + +def _record_parse_err(ps: Parser*): + """Record the position and message of the first parse error (later errors do not overwrite it)""" + global _g_parse_err_line, _g_parse_err_col, _g_parse_err_msg + if _g_parse_err_line != 0: + return + _g_parse_err_line = _cur_lineno(ps) + _g_parse_err_col = _cur_col(ps) + err_s: str = _cur_str(ps) + if err_s is None: + err_s = "(none)" + # Copy the message text via stdlib (token.str_val lives in the pool; stored independently to be safe) + buf_m: str = stdlib.malloc(128) + if buf_m is not None: + stdio.snprintf(buf_m, 128, "unexpected token '%s'", err_s) + _g_parse_err_msg = buf_m + else: + _g_parse_err_msg = err_s + + +def _record_parse_err_ctx(ps: Parser*, tok_type: int, + pv_str: str, pv_line: int, pv_col: int, + nx_str: str, nx_line: int, nx_col: int): + """Record the first parse error with token type and prev/next tokens appended to the message. + + Same source as _record_parse_err (first only); the message must stay a single line since logs + inline it. file:line:col and the caret are filled later by the caller's dump_parse_traceback. + """ + global _g_parse_err_line, _g_parse_err_col, _g_parse_err_msg + if _g_parse_err_line != 0: + return + _g_parse_err_line = _cur_lineno(ps) + _g_parse_err_col = _cur_col(ps) + err_s: str = _cur_str(ps) + if err_s is None: + err_s = "(none)" + buf_m: str = stdlib.malloc(512) + if buf_m is None: + _g_parse_err_msg = err_s + return + stdio.snprintf(buf_m, 512, + "unexpected token '%s' (type=%d) | note: prev='%s' (line %d col %d) | next='%s' (line %d col %d)", + err_s, tok_type, pv_str, pv_line, pv_col, nx_str, nx_line, nx_col); + _g_parse_err_msg = buf_m + + +def _record_neg_literal_overflow(lit_text: str, lineno: int, col: int): + """Record a combined "unary minus + integer literal" overflow (fail-fast; first only). + + `-` is a separate unary operator, so a magnitude-valid literal (<= 2^64-1) may still be < -2^63 + after negation. Message carries the literal text + allowed range [-2^63, 2^64-1] + fix hint. + """ + global _g_parse_err_line, _g_parse_err_col, _g_parse_err_msg + if _g_parse_err_line != 0: + return + _g_parse_err_line = lineno + _g_parse_err_col = col + buf: str = stdlib.malloc(512) + if buf is not None: + stdio.snprintf(buf, 512, + "[VPC] negative integer literal out of range: unary minus applied to an integer literal exceeds the" + " signed 64-bit representable range (allowed range -9223372036854775808..18446744073709551615" + " = [-2^63, 2^64-1]; a negative result must be >= -2^63). Literal text: '-%s'." + " Silent wrap-around/truncation is forbidden; compilation aborted." + " Fix: change the negative number into -9223372036854775808..-1 (this language has no" + " wider integer type, for larger values split into multiple 64-bit words).", lit_text) + _g_parse_err_msg = buf + else: + _g_parse_err_msg = lit_text + + +# ============================================================ +# Type descriptors const / volatile (soft keywords, annotation position only) +# ============================================================ +def _IsDescriptorKeyword(ps: Parser*) -> int: + """Whether the current token is a type-descriptor keyword (const / volatile), 1 = yes / 0 = no. + + Soft keywords consumed by _parse_type_ann only in annotation position; `t.attr.const()` as an + identifier is unaffected since attribute/call resolution looks at TokenType.Name, not kw_subtype. + """ + if _cur_type(ps) != TokenType.Name: + return 0 + if _cur_kw(ps) == Keyword.Const or _cur_kw(ps) == Keyword.Volatile: + return 1 + return 0 + + +def _IsMagicAttributeCall(ps: Parser*) -> int: + """Test for an annotation-position `__attribute__(...)` prefix, 1 = yes / 0 = no. + + `__attribute__` is a soft keyword (not in the keyword table): consumed here only in annotation + position immediately before '('. Same family as const/volatile: order-free and mixable. + """ + if _cur_type(ps) != TokenType.Name: + return 0 + s: str = _cur_str(ps) + if s is None or string.strcmp(s, "__attribute__") != 0: + return 0 + nxt: Token* = ps.cur.next + if nxt is None or nxt.type != TokenType.Op or nxt.op_subtype != TokOp.LPar: + return 0 + return 1 + + +def _MakeDescriptorMarker(ps: Parser*) -> AST*: + """Build the marker node Attribute(Name('t'), 'CConst'/'CVolatile') from the current descriptor keyword. + + Field-identical to TPC's _ExpandTypeDescriptors text expansion (t.CConst / t.CVolatile) so both + sides share one recognition path; no second representation is introduced. + """ + attr_name: str = "CConst" + if _cur_kw(ps) == Keyword.Volatile: + attr_name = "CVolatile" + marker: AST* = Attribute(ps.pool, Name(ps.pool, "t", ASTCtx.Load), + attr_name, ASTCtx.Load) + _set_pos(marker, _cur_lineno(ps), _cur_col(ps), 0, 0) + return marker + + +def _PrependDescriptorMarkers(ps: Parser*, + base: AST*, + mods: list[AST*]*, + mod_count: int) -> AST*: + """Prepend descriptor markers to the leftmost operand of the type union (same shape as TPC text). + + TPC expands `const volatile int | t.CPtr` to the left-associative `(((t.CConst | t.CVolatile) | int) | t.CPtr)`; + this function produces the identical tree rather than a differently-shaped one. + """ + slot_parent: BinOp* = None # leftmost BitOr node: its left is the leftmost operand + slot_node: AST* = base + cur: AST* = base + while cur is not None and cur.kind() == ASTKind.BinOp: + b: BinOp* = (BinOp*)(cur) + if b.op != OpKind.BitOr or b.left is None: + break + nxt: AST* = b.left + if nxt.kind() == ASTKind.BinOp: + nb: BinOp* = (BinOp*)(nxt) + if nb.op == OpKind.BitOr and nb.left is not None: + cur = nxt + continue + slot_parent = b + slot_node = nxt + break + # Descriptor chain left-associates in source order: `const volatile` -> (t.CConst | t.CVolatile) + chain: AST* = mods.get(0) + i: int = 1 + while i < mod_count: + chain = BinOp(ps.pool, chain, OpKind.BitOr, mods.get(i)) + i += 1 + newnode: AST* = BinOp(ps.pool, chain, OpKind.BitOr, slot_node) + if slot_parent is None: + return newnode + slot_parent.left = newnode + return base + + +def _RecordDescriptorErr(detail: str, lineno: int, col: int): + """Record an invalid type-descriptor form (fail-fast; first only). + + Message carries the expected form + a correct example; file:line:col is filled later. Descriptors + are never silently ignored: missing type / non-type follower / bare use all abort compilation. + """ + global _g_parse_err_line, _g_parse_err_col, _g_parse_err_msg + if _g_parse_err_line != 0: + return + _g_parse_err_line = lineno + _g_parse_err_col = col + buf: str = stdlib.malloc(1024) + if buf is not None: + stdio.snprintf(buf, 1024, + "invalid type descriptor (%s). const/volatile are soft keywords valid only in type position;" + " they must be written as `name: const type` / `name: volatile type`" + " (combinable as `name: const volatile type`). Correct examples: a: const int / " + "a: volatile int / a: const volatile t.CInt32T / " + "def f(p: const rawptr[Foo]) -> const int." + " (descriptors are never silently ignored: missing type / non-type follower / bare use all abort compilation)", detail) + _g_parse_err_msg = buf + else: + _g_parse_err_msg = detail + + +# ============================================================ +# virtual / novirtual modifier (soft keyword; statement start before class/def only), same +# representation as @t.CVTable / @t.NoVTable. Invalid forms fail loudly; `x = d.virtual` unaffected. +def _IsVirtualModifier(ps: Parser*) -> int: + """Whether the current token is a virtual / novirtual modifier (1 = yes / 0 = no). + + Called only at statement start; identifier positions never reach this test, so `x = d.virtual`, + parameter names, for targets and dict keys are unaffected. + """ + if _cur_type(ps) != TokenType.Name: + return 0 + if _cur_kw(ps) == Keyword.Virtual or _cur_kw(ps) == Keyword.Novirtual: + return 1 + return 0 + + +def _MakeVirtualMarker(ps: Parser*) -> AST*: + """Build the marker node Attribute(Name('t'), 'CVTable'|'NoVTable') from the current virtual/novirtual token. + + Field-identical to hand-written @t.CVTable / @t.NoVTable, so ClassGenBase._has_decorator needs no new test. + """ + attr_name: str = "CVTable" + if _cur_kw(ps) == Keyword.Novirtual: + attr_name = "NoVTable" + marker: AST* = Attribute(ps.pool, Name(ps.pool, "t", ASTCtx.Load), + attr_name, ASTCtx.Load) + _set_pos(marker, _cur_lineno(ps), _cur_col(ps), 0, 0) + return marker + + +def _RecordVirtualErr(detail: str, lineno: int, col: int, extra: str): + """Record an invalid virtual/novirtual form (fail-fast; first only); message carries allowed + positions + a correct example. Modifiers are never silently ignored. + + Callers build detail/extra with f-string interpolation: this language's str is i8*, so `str + str` + degrades to raw pointer add and illegal IR; never concatenate strings with `+` in this file. + """ + global _g_parse_err_line, _g_parse_err_col, _g_parse_err_msg + if _g_parse_err_line != 0: + return + _g_parse_err_line = lineno + _g_parse_err_col = col + buf: str = stdlib.malloc(1536) + if buf is not None: + stdio.snprintf(buf, 1536, + "[VPC] invalid virtual/novirtual modifier (%s). Valid positions: class level " + "`virtual class C:` / `novirtual class C:`; method level " + "`virtual def m(self): ...` / `novirtual def m(self): ...`" + " (only directly inside a class body: the body of virtual class / class / novirtual class)." + " Correct examples: virtual class Shape: / novirtual class Node: / " + "class A: virtual def run(self) -> int: ... / " + "novirtual class B: novirtual def helper(self) -> int: ....%s" + " (modifiers are never silently ignored: illegal forms all abort compilation)", detail, extra) + _g_parse_err_msg = buf + else: + _g_parse_err_msg = detail + + +def _ParseVirtualModified(ps: Parser*, parent: AST*): + """`virtual class` / `novirtual class` / `virtual def` / `novirtual def`. + + Synthesizes a @t.CVTable / @t.NoVTable decorator (merged with existing decorators, appended last; + position-independent since _has_decorator matches by name) and hands off to _parse_classdef / _parse_funcdef. + """ + mod_lineno: int = _cur_lineno(ps) + mod_col: int = _cur_col(ps) + mod_name: str = "virtual" + if _cur_kw(ps) == Keyword.Novirtual: + mod_name = "novirtual" + marker: AST* = _MakeVirtualMarker(ps) + _advance(ps) # consume the modifier + # The modifier must be followed by class / def (the def aliases fn/fun/func/function are accepted too, + # matching _is_funcdef_kw and TPC, whose fn->def expansion only hits a leading fn). + kw2: int = _cur_kw(ps) + is_class: int = 0 + is_def: int = 0 + if kw2 == Keyword.Class: + is_class = 1 + elif kw2 == Keyword.Def or kw2 == Keyword.Fn or kw2 == Keyword.Fun \ + or kw2 == Keyword.Func or kw2 == Keyword.Function: + is_def = 1 + if is_class == 0 and is_def == 0: + nxt_s: str = _cur_str(ps) + if nxt_s is None: + nxt_s = "(none)" + if kw2 == Keyword.Struct: + # struct is a pure data type: no virtual / novirtual modifier allowed. A dedicated message + # (not the generic "follower not class/def") points at the fix: struct already implies @t.NoVTable. + _RecordStructErr(f"`{mod_name} struct` is invalid: struct forbids the virtual/novirtual modifier", + mod_lineno, mod_col, + "struct has no vtable by nature (a @t.NoVTable is synthesized at parse time);" + " write `struct X:` directly; if vtable semantics are needed, use class instead.") + else: + _RecordVirtualErr(f"`{mod_name}` is followed by '{nxt_s}' (not class/def)", + mod_lineno, mod_col, + f"`{mod_name}` takes effect only before class/def;" + f" if a plain identifier name was intended here, rename it" + f" (this word is treated as a modifier at statement start).") + ps.error_count += 1 + # Dispatch the remaining tokens to the generic statement path (no infinite loop, no error overwrite). + _parse_simple_stmt_into(ps, parent) + return + if is_def != 0 and ps.in_class_body == 0: + _RecordVirtualErr(f"{mod_name} def appears outside a class body", mod_lineno, mod_col, + f"{mod_name} def describes the 'virtual/non-virtual' of a method inside a class body;" + f" at module level / function body / branch body there is no class to attach to.") + ps.error_count += 1 + # Merge existing decorators (@deco + virtual class/def) then hand off to the existing class/def parser + merged: list[AST*]* = list[AST*](ps.pool, 8) + if merged is None: + return + if ps.pending_decorators is not None: + dn: sizet = len(ps.pending_decorators) + for di in range(int(dn)): + merged.append(ps.pending_decorators.get(di)) + merged.append(marker) + ps.pending_decorators = merged + stmt: AST* = None + if is_class != 0: + stmt = _parse_classdef(ps) + else: + stmt = _parse_funcdef(ps, 0) + if stmt is not None: + parent.append(stmt) + + +# ============================================================ +# inline modifier (soft keyword): `inline def f(...)` -> synthesizes @t.attr.always_inline, same shape as +# virtual/novirtual (statement start before a function definition only); kept out of the keyword table. +# ============================================================ +def _IsInlineModifier(ps: Parser*) -> int: + """Whether the current token is `inline` (soft keyword followed by a function definition), 1 = yes / 0 = no""" + if _cur_type(ps) != TokenType.Name: + return 0 + s: str = _cur_str(ps) + if s is None or string.strcmp(s, "inline") != 0: + return 0 + nxt: Token* = ps.cur.next + if nxt is None or nxt.type != TokenType.Name: + return 0 + nxt2: Token* = nxt.next + if nxt2 is None: + return 0 + # Followed by def (hard keyword): `inline def f(...)` + if nxt.kw_subtype == Keyword.Def: + return 1 + # Followed by a function-definition soft-keyword alias (fn/fun/func/function): same rule as + # _is_funcdef_kw -- must be Name + '(' or '[T]', so `inline func` is not misread. + if nxt.kw_subtype == Keyword.Fn or nxt.kw_subtype == Keyword.Fun \ + or nxt.kw_subtype == Keyword.Func or nxt.kw_subtype == Keyword.Function: + if nxt2.type == TokenType.Op \ + and (nxt2.op_subtype == TokOp.LPar or nxt2.op_subtype == TokOp.Lsqb): + return 1 + return 0 + + +def _MakeInlineMarker(ps: Parser*) -> AST*: + """Build the marker Attribute(Attribute(Name('t'),'attr'), 'always_inline') from the current inline token. + + Field-identical to a hand-written @t.attr.always_inline. Deliberately not @c.Attribute(t.attr.x), + whose extractor requires `import c` and would silently skip inlining in files lacking it. + """ + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + t_name: AST* = Name(ps.pool, "t", ASTCtx.Load) + attr_at: AST* = Attribute(ps.pool, t_name, "attr", ASTCtx.Load) + marker: AST* = Attribute(ps.pool, attr_at, "always_inline", ASTCtx.Load) + _set_pos(marker, lineno, col, 0, 0) + return marker + + +def _ParseInlineModified(ps: Parser*, parent: AST*): + """`inline def f(...)`: consume inline, synthesize @t.attr.always_inline and hand off to _parse_funcdef""" + marker: AST* = _MakeInlineMarker(ps) + if marker is None: + return + _advance(ps) # consume the inline modifier + # Merge existing decorators (@deco + inline def), appending the marker last; position-independent + # (matched by name downstream). Same pending_decorators channel as _ParseVirtualModified. + merged: list[AST*]* = list[AST*](ps.pool, 8) + if merged is None: + return + if ps.pending_decorators is not None: + dn: sizet = len(ps.pending_decorators) + for di in range(int(dn)): + merged.append(ps.pending_decorators.get(di)) + merged.append(marker) + ps.pending_decorators = merged + stmt: AST* = _parse_funcdef(ps, 0) + if stmt is not None: + parent.append(stmt) + + +def _IsStaticModifier(ps: Parser*) -> int: + """Whether the current token is a `static` modifier (soft keyword), 1 = yes / 0 = no. + + Called only at statement start; matches when followed by (1) a function definition (`def` or the + fn/fun/func/function alias + '(' / '[') or (2) a variable declaration (`Name :`, i.e. AnnAssign). + Identifier positions never reach this test, so `x = d.static` / `f(static=1)` are unaffected. + """ + if _cur_type(ps) != TokenType.Name: + return 0 + s: str = _cur_str(ps) + if s is None or string.strcmp(s, "static") != 0: + return 0 + nxt: Token* = ps.cur.next + if nxt is None or nxt.type != TokenType.Name: + return 0 + # (1) followed by def (hard keyword): `static def f(...)` + if nxt.kw_subtype == Keyword.Def: + return 1 + nxt2: Token* = nxt.next + # (1') followed by a function-definition alias: must be Name + '(' or '[T]', same rule as + # _IsInlineModifier / _is_funcdef_kw, so `static func` is not misread. + if nxt.kw_subtype == Keyword.Fn or nxt.kw_subtype == Keyword.Fun \ + or nxt.kw_subtype == Keyword.Func or nxt.kw_subtype == Keyword.Function: + if nxt2 is not None and nxt2.type == TokenType.Op \ + and (nxt2.op_subtype == TokOp.LPar or nxt2.op_subtype == TokOp.Lsqb): + return 1 + # (2) followed by a variable declaration `Name :` (AnnAssign) + if nxt2 is not None and nxt2.type == TokenType.Op and nxt2.op_subtype == TokOp.Colon: + return 1 + return 0 + + +def _IsStaticDescriptorKeyword(ps: Parser*) -> int: + """Whether the current token is an annotation-position `static` (soft keyword), 1 = yes / 0 = no. + + This position (`name: static T`) is no longer supported: _parse_type_ann only detects it as misuse + and fails loudly, pointing at the prefix form `static name: T`; it is no longer desugared to t.CStatic. + Valid forms are statement-start `static name: T` and the `@t.CStatic` decorator. Reporting (not ignoring) + is mandatory: unconsumed, `static` degrades into an opaque "class" named static and errors far from the cause. + """ + if _cur_type(ps) != TokenType.Name: + return 0 + s: str = _cur_str(ps) + if s is None or string.strcmp(s, "static") != 0: + return 0 + return 1 + + +def _MakeStaticMarker(ps: Parser*) -> AST*: + """Build the marker node Attribute(Name('t'), 'CStatic'). + + Field-identical to the hand-written @t.CStatic decorator / `x: t.CStatic | T` marker, so downstream + (FuncGenDef storage-class detection, TypeGenResolve qualifier resolution) needs no new test. + """ + marker: AST* = Attribute(ps.pool, Name(ps.pool, "t", ASTCtx.Load), + "CStatic", ASTCtx.Load) + _set_pos(marker, _cur_lineno(ps), _cur_col(ps), 0, 0) + return marker + + +def _ParseStaticModified(ps: Parser*, parent: AST*) -> int: + """Dispatch the `static` modifier: 1 = fully handled (function definition), 0 = only flagged + (variable declaration, caller continues with the generic statement dispatch). + + Function: consume static and synthesize an @t.CStatic decorator, handed to _parse_funcdef. + Variable: set pending_static and consume static; the next _parse_type_ann prepends t.CStatic. + """ + nxt: Token* = ps.cur.next + is_func: int = 0 + if nxt is not None and nxt.type == TokenType.Name: + if nxt.kw_subtype == Keyword.Def: + is_func = 1 + elif nxt.kw_subtype == Keyword.Fn or nxt.kw_subtype == Keyword.Fun \ + or nxt.kw_subtype == Keyword.Func or nxt.kw_subtype == Keyword.Function: + nxt2: Token* = nxt.next + if nxt2 is not None and nxt2.type == TokenType.Op \ + and (nxt2.op_subtype == TokOp.LPar or nxt2.op_subtype == TokOp.Lsqb): + is_func = 1 + if is_func == 0: + # Variable branch: only flag + consume static, then let the caller continue generic dispatch + # (AnnAssign parsing reads pending_static in _parse_type_ann). + ps.pending_static = 1 + _advance(ps) + return 0 + marker: AST* = _MakeStaticMarker(ps) + if marker is None: + return 1 + _advance(ps) # consume the static modifier + # Merge existing decorators (@deco + static def), appending the marker last; position-independent + # (matched by name downstream). Same pending_decorators channel as _ParseInlineModified. + merged: list[AST*]* = list[AST*](ps.pool, 8) + if merged is None: + return 1 + if ps.pending_decorators is not None: + dn: sizet = len(ps.pending_decorators) + for di in range(int(dn)): + merged.append(ps.pending_decorators.get(di)) + merged.append(marker) + ps.pending_decorators = merged + stmt: AST* = _parse_funcdef(ps, 0) + if stmt is not None: + parent.append(stmt) + return 1 + + +def _IsBareDescriptor(ps: Parser*) -> int: + """Bare-descriptor test: the whole statement is a single descriptor run (const[: volatile]), 1 = yes / 0 = no. + + Matches only when the descriptor run is followed directly by newline / EOF / semicolon; identifiers + (`const = 5` / `const(x)` / `t.attr.const()`) do not match (soft-keyword boundary). + """ + if _IsDescriptorKeyword(ps) == 0: + return 0 + nxt: Token* = ps.cur.next + while nxt is not None and nxt.type == TokenType.Name \ + and (nxt.kw_subtype == Keyword.Const or nxt.kw_subtype == Keyword.Volatile): + nxt = nxt.next + if nxt is None: + return 1 + if nxt.type == TokenType.NewLine or nxt.type == TokenType.EndMarker: + return 1 + if nxt.type == TokenType.Op and nxt.op_subtype == TokOp.Semi: + return 1 + return 0 + + +def get_parse_err_line() -> int: + """Line number of the latest parse error (0 = none)""" + return _g_parse_err_line + + +def get_parse_err_col() -> int: + """Column of the latest parse error (0 = none)""" + return _g_parse_err_col + + +def get_parse_err_msg() -> str: + """Message of the latest parse error (Python SyntaxError style)""" + return _g_parse_err_msg + + +def _cur_str(ps: Parser*) -> str: + if ps.cur is None: return None + return ps.cur.str_val + + +def _cur_lineno(ps: Parser*) -> int: + if ps.cur is None: return 0 + return ps.cur.lineno + + +def _cur_col(ps: Parser*) -> int: + if ps.cur is None: return 0 + return ps.cur.col_offset + + +def _skip_nl(ps: Parser*): + """Skip TokenType.Nl token""" + while ps.cur is not None and ps.cur.type == TokenType.Nl: + _advance(ps) + + +def _skip_nls_and_newlines(ps: Parser*): + """Skip TokenType.Nl and TokenType.NewLine (used for line continuation)""" + while ps.cur is not None: + if ps.cur.type == TokenType.Nl or ps.cur.type == TokenType.NewLine: + _advance(ps) + else: + break + + +def _expect_type(ps: Parser*, ttype: int) -> int: + """Expect the current token to be of the specified type, advance and return 1, otherwise return 0""" + if _cur_type(ps) == ttype: + _advance(ps) + return 1 + _record_parse_err(ps) + ps.error_count += 1 + return 0 + + +def _expect_op(ps: Parser*, op_id: int) -> int: + """Expecting the current one to be some operator""" + if _cur_type(ps) == TokenType.Op and _cur_op(ps) == op_id: + _advance(ps) + return 1 + _record_parse_err(ps) + ps.error_count += 1 + return 0 + + +def _match_op(ps: Parser*, op_id: int) -> int: + """Check but not consume the current token""" + _tick(ps) + if ps.aborted != 0: + # Convergence: if tokens remain, "match" and jump to the end so both + # `while not _match_op(...)` and `while _match_op(...)` end within <= 2 rounds + if ps.cur is not None: + ps.cur = None + return 1 + return 0 + if _cur_type(ps) == TokenType.Op and _cur_op(ps) == op_id: + return 1 + return 0 + + +def _accept_op(ps: Parser*, op_id: int) -> int: + """If match, consume and return 1""" + if _match_op(ps, op_id): + _advance(ps) + return 1 + return 0 + + +# ============================================================ +# Constant Construction Helper (New API: Constant directly accepts lineno/col) +# ============================================================ + +def _new_const_int(pool: memhub.MemManager*, val: int64, + lineno: int, col: int, + int_width: int = 0, int_unsigned: int = 0) -> AST*: + return Constant(pool, CONST_INT, val, 0.0, None, lineno, col, int_width, int_unsigned) + + +def _new_const_float(pool: memhub.MemManager*, val: double, + lineno: int, col: int) -> AST*: + global _g_dbg_pts, _g_dbg_vals, _g_dbg_cnt + new_c: Constant* = Constant(pool, CONST_FLOAT, 0, val, None, lineno, col) + # Store the numeric text in str_val (stdlib allocation, bypassing the pool overwrite of float_val); + # REG-SRC recovers the original value with string.atof(str_val). + lit_txt: str = stdlib.malloc(64) + if lit_txt is not None: + stdio.snprintf(lit_txt, 64, "%.17g", val) + new_c.str_val = lit_txt + # Record float node ptr->val into a stdlib table (value correct at construction; REG-SRC uses the + # table to bypass the pool overwrite). Reset when full: old entries belong to finished files. + if _g_dbg_pts is None: + _g_dbg_pts = stdlib.malloc(4096 * 8) + _g_dbg_vals = stdlib.malloc(4096 * 8) + _g_dbg_cnt = 0 + elif _g_dbg_cnt >= 4096: + _g_dbg_cnt = 0 + if _g_dbg_pts is not None and _g_dbg_vals is not None: + _g_dbg_pts[_g_dbg_cnt] = uint64(new_c) + _g_dbg_vals[_g_dbg_cnt] = val + _g_dbg_cnt += 1 + return new_c + + +# Float-node construction-value table (stdlib allocated to avoid polluting the pool) +_g_dbg_pts: uint64* = None +_g_dbg_vals: double* = None +_g_dbg_cnt: int = 0 +# Whether the latest _dbg_lookup_lit_val hit +_g_dbg_lit_found: int = 0 + + +def _dbg_lookup_lit_val(ptr: uint64) -> double: + """Look up the float-node construction-value table: return the original value (miss returns 0, found=0). + + Used to bypass the pool allocation overwriting the AST float_val field (correct at construction). + """ + global _g_dbg_lit_found + _g_dbg_lit_found = 0 + if _g_dbg_pts is None or _g_dbg_vals is None: + return 0.0 + for i in range(int(_g_dbg_cnt)): + if _g_dbg_pts[i] == ptr: + _g_dbg_lit_found = 1 + return _g_dbg_vals[i] + return 0.0 + + +def _dbg_lit_found() -> int: + """Check whether the latest _dbg_lookup_lit_val hit""" + return _g_dbg_lit_found + + +def _new_const_str(pool: memhub.MemManager*, val: str, + lineno: int, col: int, + is_dquote: int = 1, + is_triple: int = 0, + is_char: int = 0) -> AST*: + """Create a string/character constant node + + is_dquote: 0 = single quote, non-0 = double quote (default double quote) + is_triple: non-0 = triple-quoted string + is_char: non-0 = single-character candidate (lexer has confirmed that after escaping it's exactly 1 byte, including '0') + Semantic distinctions: + - 'x' (single quote, single character, not triple-quoted) → CONST_CHAR (i8 value) + - "x" (double quote, includes 0 terminator) → CONST_STR (i8* pointer, i8 x 2) + - 'ab' (single quotes, multiple characters) → CONST_STR (i8* pointer) + - '''x''' / \"\"\"x\"\"\" (triple quotes) → CONST_STR (i8* pointer) + - '' / "" (empty string) → CONST_STR (i8* pointer) + """ + # Single quotes, non-triple quotes, single-character candidates marked by lexer → CONST_CHAR + # Mark with is_char instead of strlen(val), because '0' becomes a NUL byte after escaping, strlen would return 0 + if is_dquote == 0 and is_triple == 0 and is_char != 0: + # Single quote # Single character → CONST_CHAR, int_val stores the character's ASCII value + # val[0] is a legal value of 0 even if it's NUL + ch_val: int64 = val[0] + return Constant(pool, CONST_CHAR, ch_val, 0.0, val, lineno, col) + # Other situations → CONST_STR + return Constant(pool, CONST_STR, 0, 0.0, val, lineno, col) + + +def _new_const_bool(pool: memhub.MemManager*, val: int, + lineno: int, col: int) -> AST*: + return Constant(pool, CONST_BOOL, val, 0.0, None, lineno, col) + + +def _new_const_none(pool: memhub.MemManager*, + lineno: int, col: int) -> AST*: + return Constant(pool, CONST_NONE, 0, 0.0, None, lineno, col) + + +# ============================================================ +# ctx Helper (Recursive Fixing Store Context) +# ============================================================ + +def _set_ctx(node: AST*, ctx: int): + """Set ctx field of node (based on kind conversion)""" + if node is None: return + k: int = node.kind() + if k == ASTKind.Name: + nm: Name* = (Name*)(node) + nm.ctx = ctx + elif k == ASTKind.Attribute: + at: Attribute* = (Attribute*)(node) + at.ctx = ctx + elif k == ASTKind.Subscript: + sb: Subscript* = (Subscript*)(node) + sb.ctx = ctx + elif k == ASTKind.Tuple: + tp: Tuple* = (Tuple*)(node) + tp.ctx = ctx + elif k == ASTKind.List: + ls: List* = (List*)(node) + ls.ctx = ctx + elif k == ASTKind.Starred: + st: Starred* = (Starred*)(node) + st.ctx = ctx + + +def _fix_store_ctx(node: AST*): + """Fix ctx of store targets to be Store""" + if node is None: return + _set_ctx(node, ASTCtx.Store) + k: int = node.kind() + if k == ASTKind.Tuple or k == ASTKind.List: + tp_elts: list[AST*]* = None + if k == ASTKind.Tuple: + tp: Tuple* = (Tuple*)(node) + tp_elts = tp.elts + else: + ls: List* = (List*)(node) + tp_elts = ls.elts + if tp_elts is not None: + n: sizet = tp_elts.__len__() + for i in range(int(n)): + cur: AST* = tp_elts.get(i) + _fix_store_ctx(cur) + elif k == ASTKind.Starred: + st2: Starred* = (Starred*)(node) + _fix_store_ctx(st2.value) + + +# ============================================================ +# Expression Parser +# ============================================================ + +def _parse_expr(ps: Parser*) -> AST*: + """expr: yield_expr | namedexpr_test""" + if _cur_kw(ps) == Keyword.Yield: + return _parse_yield(ps) + return _parse_namedexpr(ps) + + +def _parse_namedexpr(ps: Parser*) -> AST*: + """namedexpr_test: test [':=' test]""" + left: AST* = _parse_test(ps) + if left is None: return None + if _match_op(ps, TokOp.ColonEq): + _advance(ps) + right: AST* = _parse_test(ps) + node: AST* = NamedExpr(ps.pool, left, right) + _set_pos(node, left.lineno, left.col_offset, 0, 0) + return node + return left + + +def _IsVolatileCallPrefix(ps: Parser*) -> int: + """Test for a value-position `volatile f()` / `volatile S()` prefix, 1 = yes / 0 = no. + + `volatile` is a soft keyword consumed only in value position before a call (Name followed by '('); + its use as an annotation descriptor (`a: volatile int`) or an identifier is unaffected. + """ + if _cur_kw(ps) != Keyword.Volatile: + return 0 + nxt: Token* = ps.cur.next + if nxt is None or nxt.type != TokenType.Name: + return 0 + nxt2: Token* = nxt.next + if nxt2 is None or nxt2.type != TokenType.Op or nxt2.op_subtype != TokOp.LPar: + return 0 + return 1 + + +def _parse_test(ps: Parser*) -> AST*: + """test: or_test ['if' or_test 'else' test] | lambdef""" + if _cur_kw(ps) == Keyword.Lambda: + return _parse_lambda(ps) + # Value-position volatile call prefix: `a: int = volatile f()` / `volatile S()` -> consume volatile and + # mark the call node (result slot accessed volatile, struct writes too; LLVM call has no volatile qualifier). + if _IsVolatileCallPrefix(ps) != 0: + _advance(ps) # consume volatile + vinner: AST* = _parse_test(ps) + if vinner is not None and vinner.kind() == ASTKind.Call: + (Call*(vinner)).IsVolatile = 1 + return vinner + left: AST* = _parse_or_test(ps) + if left is None: return None + if _cur_kw(ps) == Keyword.If: + _advance(ps) + test: AST* = _parse_or_test(ps) + if _cur_kw(ps) == Keyword.Else: + _advance(ps) + orelse: AST* = _parse_test(ps) + node: AST* = IfExp(ps.pool, test, left, orelse) + _set_pos(node, left.lineno, left.col_offset, 0, 0) + return node + ps.error_count += 1 + return left + return left + + +def _parse_lambda(ps: Parser*) -> AST*: + """lambdef: 'lambda' [varargslist] ':' test""" + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + _advance(ps) # lambda + args: AST* = None + if not _match_op(ps, TokOp.Colon): + args = _parse_varargslist(ps, 1) + if not _expect_op(ps, TokOp.Colon): + return None + body: AST* = _parse_test(ps) + node: AST* = Lambda(ps.pool, args, body) + _set_pos(node, lineno, col, 0, 0) + return node + + +def _parse_or_test(ps: Parser*) -> AST*: + """or_test: and_test ('or' and_test)*""" + left: AST* = _parse_and_test(ps) + if left is None: return None + values: list[AST*]* = None + while True: + # Supports both leading `or` ((A\n or B)) and trailing `or` (A or\n B) inside parens: + # each round skips Nl before checking, otherwise A or B\n or C stops on Nl after B. + _skip_nl(ps) + if _cur_kw(ps) != Keyword.Or: + break + if values is None: + values = list[AST*](ps.pool, 8) + values.append(left) + _advance(ps) + _skip_nl(ps) # cross-line inside parens: a or\n b + right: AST* = _parse_and_test(ps) + if right is None: break + values.append(right) + if values is not None: + node: AST* = BoolOp(ps.pool, OpKind.Or, values) + _set_pos(node, left.lineno, left.col_offset, 0, 0) + return node + return left + + +def _parse_and_test(ps: Parser*) -> AST*: + """and_test: not_test ('and' not_test)*""" + left: AST* = _parse_not_test(ps) + if left is None: return None + values: list[AST*]* = None + while True: + # Supports both leading `and` ((A\n and B)) and trailing `and` (A and\n B) inside parens: + # each round skips Nl before checking, otherwise A and B\n and C stops on Nl after B. + _skip_nl(ps) + if _cur_kw(ps) != Keyword.And: + break + if values is None: + values = list[AST*](ps.pool, 8) + values.append(left) + _advance(ps) + _skip_nl(ps) # cross-line inside parens: a and\n b + right: AST* = _parse_not_test(ps) + if right is None: break + values.append(right) + if values is not None: + node: AST* = BoolOp(ps.pool, OpKind.And, values) + _set_pos(node, left.lineno, left.col_offset, 0, 0) + return node + return left + + +def _parse_not_test(ps: Parser*) -> AST*: + """not_test: 'not' not_test | comparison""" + if _cur_kw(ps) == Keyword.Not: + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + _advance(ps) + operand: AST* = _parse_not_test(ps) + node: AST* = UnaryOp(ps.pool, OpKind.Not, operand) + _set_pos(node, lineno, col, 0, 0) + return node + return _parse_comparison(ps) + + +def _parse_comparison(ps: Parser*) -> AST*: + """comparison: bitor_expr (comp_op bitor_expr)* + + Single comparison design: a < b → Compare(left=a, op=Lt, right=b) + Chained comparison a < b < c → BoolOp(And, [Compare(a,<,b), Compare(b,<,c)]) + """ + left: AST* = _parse_bitor(ps) + if left is None: return None + + _skip_nl(ps) # leading comparison operator inside parens: (a\n < b) + op_id: int = _match_comp_op(ps) + if op_id == OpKind.NoneOp: + return left + _advance_comp_op(ps) + _skip_nl(ps) # cross-line inside parens: a ==\n b + right: AST* = _parse_bitor(ps) + # Single comparison: construct ops and comparators list + ops: list[int]* = list[int](ps.pool, 8) + ops.append(op_id) + comparators: list[AST*]* = list[AST*](ps.pool, 8) + comparators.append(right) + first_cmp: AST* = Compare(ps.pool, left, ops, comparators) + _set_pos(first_cmp, left.lineno, left.col_offset, 0, 0) + + # Check chained comparison + _skip_nl(ps) # leading chained comparison inside parens: (a < b\n < c) + next_op: int = _match_comp_op(ps) + if next_op == OpKind.NoneOp: + return first_cmp + + # Single comparison with chained conditions: a < b < c → BoolOp(And, [Compare(a,<,b), Compare(b,<,c), ...]) + values: list[AST*]* = list[AST*](ps.pool, 8) + values.append(first_cmp) + cur_left: AST* = right + while next_op != OpKind.NoneOp: + _advance_comp_op(ps) + _skip_nl(ps) # cross-line chained comparison inside parens + cur_right: AST* = _parse_bitor(ps) + ops2: list[int]* = list[int](ps.pool, 8) + ops2.append(next_op) + comparators2: list[AST*]* = list[AST*](ps.pool, 8) + comparators2.append(cur_right) + cmp: AST* = Compare(ps.pool, cur_left, ops2, comparators2) + _set_pos(cmp, cur_left.lineno, cur_left.col_offset, 0, 0) + values.append(cmp) + cur_left = cur_right + _skip_nl(ps) # leading chained comparison inside parens: (a < b\n < c) + next_op = _match_comp_op(ps) + node: AST* = BoolOp(ps.pool, OpKind.And, values) + _set_pos(node, left.lineno, left.col_offset, 0, 0) + return node + + +def _match_comp_op(ps: Parser*) -> int: + """Check if the current one is a comparison operator, return OP_* or OpKind.NoneOp""" + if _cur_type(ps) == TokenType.Op: + op_id: int = _cur_op(ps) + if op_id == TokOp.EqEq: return OpKind.Eq + if op_id == TokOp.ExclaimEq: return OpKind.Ne + if op_id == TokOp.Less: return OpKind.Lt + if op_id == TokOp.LessEq: return OpKind.Le + if op_id == TokOp.Greater: return OpKind.Gt + if op_id == TokOp.GreaterEq: return OpKind.Ge + if _cur_kw(ps) == Keyword.Is: + if ps.cur.next is not None and ps.cur.next.type == TokenType.Name and ps.cur.next.kw_subtype == Keyword.Not: + return OpKind.IsNot + return OpKind.Is + if _cur_kw(ps) == Keyword.Not: + if ps.cur.next is not None and ps.cur.next.kw_subtype == Keyword.In: + return OpKind.NotIn + if _cur_kw(ps) == Keyword.In: + return OpKind.In + return OpKind.NoneOp + + +def _advance_comp_op(ps: Parser*): + """Consume comparison operator token""" + if _cur_type(ps) == TokenType.Op: + _advance(ps) + return + if _cur_kw(ps) == Keyword.Is: + _advance(ps) + if _cur_kw(ps) == Keyword.Not: + _advance(ps) + return + if _cur_kw(ps) == Keyword.Not: + _advance(ps) + if _cur_kw(ps) == Keyword.In: + _advance(ps) + return + if _cur_kw(ps) == Keyword.In: + _advance(ps) + + +def _parse_bitor(ps: Parser*) -> AST*: + left: AST* = _parse_bitxor(ps) + if left is None: return None + while True: + _skip_nl(ps) # leading '|' inside parens: (a\n | b) + if _match_op(ps, TokOp.VBar) == 0: + break + _advance(ps) + _skip_nl(ps) # cross-line inside parens: a |\n b + right: AST* = _parse_bitxor(ps) + node: AST* = BinOp(ps.pool, left, OpKind.BitOr, right) + _set_pos(node, left.lineno, left.col_offset, 0, 0) + left = node + return left + + +def _parse_bitxor(ps: Parser*) -> AST*: + left: AST* = _parse_bitand(ps) + if left is None: return None + while True: + _skip_nl(ps) # leading '^' inside parens: (a\n ^ b) + if _match_op(ps, TokOp.Caret) == 0: + break + _advance(ps) + _skip_nl(ps) # cross-line inside parens: a ^\n b + right: AST* = _parse_bitand(ps) + node: AST* = BinOp(ps.pool, left, OpKind.BitXor, right) + _set_pos(node, left.lineno, left.col_offset, 0, 0) + left = node + return left + + +def _parse_bitand(ps: Parser*) -> AST*: + left: AST* = _parse_shift(ps) + if left is None: return None + while True: + _skip_nl(ps) # leading '&' inside parens: (a\n & b) + if _match_op(ps, TokOp.Amp) == 0: + break + _advance(ps) + _skip_nl(ps) # cross-line inside parens: a &\n b + right: AST* = _parse_shift(ps) + node: AST* = BinOp(ps.pool, left, OpKind.BitAnd, right) + _set_pos(node, left.lineno, left.col_offset, 0, 0) + left = node + return left + + +def _parse_shift(ps: Parser*) -> AST*: + left: AST* = _parse_arith(ps) + if left is None: return None + while True: + _skip_nl(ps) # leading '<<'/'>>' inside parens: (a\n << b) + if _match_op(ps, TokOp.LtLt) == 0 and _match_op(ps, TokOp.GtGt) == 0: + break + op_token: int = _cur_op(ps) + _advance(ps) + _skip_nl(ps) # cross-line inside parens: a <<\n b + right: AST* = _parse_arith(ps) + ast_op: int = OpKind.LShift + if op_token == TokOp.GtGt: + ast_op = OpKind.RShift + node: AST* = BinOp(ps.pool, left, ast_op, right) + _set_pos(node, left.lineno, left.col_offset, 0, 0) + left = node + return left + + +def _parse_arith(ps: Parser*) -> AST*: + left: AST* = _parse_term(ps) + if left is None: return None + while True: + _skip_nl(ps) # leading '+'/'-' inside parens: (a\n + b) + if _match_op(ps, TokOp.Plus) == 0 and _match_op(ps, TokOp.Minus) == 0: + break + op_token: int = _cur_op(ps) + _advance(ps) + _skip_nl(ps) # cross-line inside parens: a +\n b + right: AST* = _parse_term(ps) + ast_op: int = OpKind.Add + if op_token == TokOp.Minus: + ast_op = OpKind.Sub + node: AST* = BinOp(ps.pool, left, ast_op, right) + _set_pos(node, left.lineno, left.col_offset, 0, 0) + left = node + return left + + +# ============================================================ +# C-style pointer suffix ('*' / '**') -- same source as TransPyC lib/vast _AtPtrSuffix +# ============================================================ +# Start position (line/col) of the last consumed C-style pointer suffix ('*' / '**'). +# _parse_paren matches it against ps.prev to tell the paren-less cast `(T*)x` from a +# plain group like `(a | b)`. Recorded as ints, not as a token pointer: comparing two +# token pointers whose declared pointer levels differ is rejected by the compare check. +_g_ptr_suffix_line: int = 0 +_g_ptr_suffix_col: int = 0 + + +def _ptr_suffix_end_op(op: int) -> int: + """Whether the suffix-end token is a "no right operand" position (']' ',' ')' ':' '='). + + Multiplication/power would lack a right operand there anyway (a pre-existing syntax error), so + treating '*'/'**' as a pointer suffix changes no valid expression (x[a * b] / a ** b unaffected). + """ + if op == TokOp.Rsqb or op == TokOp.Comma or op == TokOp.RPar \ + or op == TokOp.Colon or op == TokOp.Equal: + return 1 + return 0 + + +def _at_ptr_suffix(ps: Parser*, base: AST*) -> int: + """Whether the current '*'/'**' run is a C-style pointer suffix applied to base. + + Rule (same as vast): base is a type atom (Name/Attribute/Subscript) and, skipping consecutive + '*'/'**' and Nl, the next token is ']' / ',' / ')' / ':' / '='. Covers annotation and expression contexts. + """ + if base is None: + return 0 + k: int = base.kind() + if k != ASTKind.Name and k != ASTKind.Attribute and k != ASTKind.Subscript: + return 0 + nxt: Token* = ps.cur + stars: int = 0 + while nxt is not None: + if nxt.type == TokenType.Op \ + and (nxt.op_subtype == TokOp.Star or nxt.op_subtype == TokOp.StarStar): + stars += 1 + nxt = nxt.next + elif nxt.type == TokenType.Nl: + nxt = nxt.next + else: + break + if stars == 0 or nxt is None or nxt.type != TokenType.Op: + return 0 + return _ptr_suffix_end_op(nxt.op_subtype) + + +def _make_ptr_type(ps: Parser*, base: AST*) -> AST*: + """Build one pointer wrapper layer: base | t.CPtr.""" + cptr: AST* = Attribute(ps.pool, Name(ps.pool, "t", ASTCtx.Load), + "CPtr", ASTCtx.Load) + node: AST* = BinOp(ps.pool, base, OpKind.BitOr, cptr) + _set_pos(node, base.lineno, base.col_offset, 0, 0) + return node + + +def _consume_ptr_suffix(ps: Parser*, base: AST*) -> AST*: + """Consume consecutive C-style pointer suffixes ('*' = 1 layer, '**' = 2), expanding into a base | t.CPtr chain.""" + global _g_ptr_suffix_line, _g_ptr_suffix_col + while True: + _skip_nl(ps) + if _match_op(ps, TokOp.StarStar): + _advance(ps) + _g_ptr_suffix_line = ps.prev.lineno + _g_ptr_suffix_col = ps.prev.col_offset + base = _make_ptr_type(ps, base) + base = _make_ptr_type(ps, base) + continue + if _match_op(ps, TokOp.Star): + _advance(ps) + _g_ptr_suffix_line = ps.prev.lineno + _g_ptr_suffix_col = ps.prev.col_offset + base = _make_ptr_type(ps, base) + continue + break + return base + + +def _at_cast_operand(ps: Parser*) -> int: + """Whether the current token may start the operand of a paren-less cast `(T*)x`. + + '(' is deliberately absent: `(T*)(x)` is already handled by the trailer loop as a + cast call, and keeping it there leaves that path's AST untouched. Nl is not + skipped either: a line break after ')' starts a new statement, not an operand. + """ + if _cur_type(ps) == TokenType.Name: + return 1 + if _cur_type(ps) == TokenType.Number: + return 1 + if _cur_type(ps) == TokenType.String: + return 1 + op: int = _cur_op(ps) + if op == TokOp.Lsqb or op == TokOp.LBrace or op == TokOp.Star or op == TokOp.Amp or op == TokOp.Minus or op == TokOp.Plus or op == TokOp.Tilde: + return 1 + return 0 + + +def _at_ptr_suffix_call(ps: Parser*, base: AST*) -> int: + """Whether the current '*'/'**' run is a `T*(x)` pointer-suffix cast call. + + Rule (same as TPC lib/vast _AtPtrSuffixCall): (1) base is a type atom; (2) base and the star run are + adjacent (same line, no whitespace/newline); (3) the run, internally adjacent, is immediately followed + by '('. Required because `a * (b)` is valid multiplication, which a bare star rule would misread. + """ + if base is None: + return 0 + k: int = base.kind() + if k != ASTKind.Name and k != ASTKind.Attribute and k != ASTKind.Subscript: + return 0 + nxt: Token* = ps.cur + pv: Token* = ps.prev + if pv is None or nxt is None: + return 0 + if pv.type == TokenType.Nl or pv.type == TokenType.NewLine: + return 0 + if pv.end_lineno != nxt.lineno or pv.end_col_offset != nxt.col_offset: + return 0 # whitespace/newline between base and '*': not a C-style cast + stars: int = 0 + prev: Token* = None + last: Token* = None + while nxt is not None: + if nxt.type == TokenType.Op \ + and (nxt.op_subtype == TokOp.Star or nxt.op_subtype == TokOp.StarStar): + if prev is not None \ + and (nxt.lineno != prev.end_lineno or nxt.col_offset != prev.end_col_offset): + return 0 # whitespace inside the star run: not adjacent, not a cast + stars += 1 + prev = nxt + last = nxt + nxt = nxt.next + else: + break + if stars == 0 or last is None or nxt is None: + return 0 + if nxt.type != TokenType.Op or nxt.op_subtype != TokOp.LPar: + return 0 + if nxt.lineno != last.end_lineno or nxt.col_offset != last.end_col_offset: + return 0 + return 1 + + +def _consume_ptr_suffix_call(ps: Parser*, base: AST*) -> AST*: + """Parse a recognized `T*(...)` pointer-suffix cast as a call. + + Identical to the existing cast form `(T | t.CPtr)(...)`: a Call (func = the expanded base | t.CPtr + chain, args/keywords = the parenthesized arguments); no second cast representation is introduced. + """ + func: AST* = _consume_ptr_suffix(ps, base) + return _parse_call(ps, func) + + +def _parse_term(ps: Parser*) -> AST*: + left: AST* = _parse_factor(ps) + if left is None: return None + while True: + _skip_nl(ps) # leading '*'/'/'/'//'/'%'/'@' inside parens: (a\n * b) + if _match_op(ps, TokOp.Star) and _at_ptr_suffix(ps, left) != 0: + left = _consume_ptr_suffix(ps, left) # C-style pointer suffix, not multiplication + continue + op_token: int = 0 + if _match_op(ps, TokOp.Star): op_token = TokOp.Star + elif _match_op(ps, TokOp.Slash): op_token = TokOp.Slash + elif _match_op(ps, TokOp.DSlash): op_token = TokOp.DSlash + elif _match_op(ps, TokOp.Percent): op_token = TokOp.Percent + elif _match_op(ps, TokOp.At): op_token = TokOp.At + if op_token == 0: break + _advance(ps) + _skip_nl(ps) # cross-line inside parens: a *\n b + right: AST* = _parse_factor(ps) + ast_op: int = _binop_from_op(op_token) + node: AST* = BinOp(ps.pool, left, ast_op, right) + _set_pos(node, left.lineno, left.col_offset, 0, 0) + left = node + return left + + +def _parse_factor(ps: Parser*) -> AST*: + """factor: '+' factor | '-' factor | '~' factor | '*' factor | '&' factor | power""" + # A leading `*` in argument position was already parsed by _parse_call as a tightly bound Starred + # operand; return it as this level's factor, before the unary test (so `*p + n` => `(*p) + n`). + if ps.starred_seed is not None: + seed: AST* = ps.starred_seed + ps.starred_seed = None + return seed + if (_match_op(ps, TokOp.Plus) or _match_op(ps, TokOp.Minus) or _match_op(ps, TokOp.Tilde) + or _match_op(ps, TokOp.Star) or _match_op(ps, TokOp.Amp)): + op_token: int = _cur_op(ps) + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + _advance(ps) + # Combined unary-minus + integer-literal overflow check: `-` is unary, so a magnitude-valid literal + # may become < -2^63 after negation. Only a unary minus directly on an integer constant is checked here. + if op_token == TokOp.Minus and _cur_type(ps) == TokenType.Number \ + and ps.cur.is_float == 0 \ + and _lex_int_neg_exceeds_i64(_cur_str(ps)) != 0: + _record_neg_literal_overflow(_cur_str(ps), lineno, col) + ps.error_count += 1 + return None + operand: AST* = _parse_factor(ps) + ast_op: int = OpKind.UAdd + if op_token == TokOp.Minus: + ast_op = OpKind.USub + elif op_token == TokOp.Tilde: + ast_op = OpKind.Invert + elif op_token == TokOp.Star: + ast_op = OpKind.Deref + elif op_token == TokOp.Amp: + ast_op = OpKind.AddrOf + node: AST* = UnaryOp(ps.pool, ast_op, operand) + _set_pos(node, lineno, col, 0, 0) + return node + return _parse_power(ps) + + +def _parse_power(ps: Parser*) -> AST*: + """power: await_expr ['**' factor] | atom_expr ['**' factor]""" + base: AST* = _parse_atom_expr(ps) + if base is None: return None + if _match_op(ps, TokOp.StarStar) and _at_ptr_suffix(ps, base) != 0: + return _consume_ptr_suffix(ps, base) # C-style pointer suffix ('**' counts as 2 layers) + if _match_op(ps, TokOp.StarStar): + _advance(ps) + exp: AST* = _parse_factor(ps) + node: AST* = BinOp(ps.pool, base, OpKind.Pow, exp) + _set_pos(node, base.lineno, base.col_offset, 0, 0) + return node + return base + + +def _parse_atom_expr(ps: Parser*) -> AST*: + """atom_expr: ['await'] atom trailer*""" + if _cur_kw(ps) == Keyword.Await: + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + _advance(ps) + inner: AST* = _parse_atom_expr(ps) + node: AST* = Await(ps.pool, inner) + _set_pos(node, lineno, col, 0, 0) + return node + base: AST* = _parse_atom(ps) + if base is None: return None + # trailer loop + while True: + if (_match_op(ps, TokOp.Star) or _match_op(ps, TokOp.StarStar)) \ + and _at_ptr_suffix_call(ps, base) != 0: + # `T*(x)` / `T**(x)` pointer-suffix cast: parsed as a cast call (same shape as + # `(T | t.CPtr)(x)`). Consumed in the trailer loop, so subsequent `.attr` / `[i]` / `(…)` + # are still attached by this same loop. + base = _consume_ptr_suffix_call(ps, base) + elif _match_op(ps, TokOp.Dot): + _advance(ps) + if _cur_type(ps) != TokenType.Name: + ps.error_count += 1 + break + attr_name: str = _cur_str(ps) + attr_lineno: int = _cur_lineno(ps) + attr_col: int = _cur_col(ps) + _advance(ps) + ctx: int = ASTCtx.Load + if _match_op(ps, TokOp.Equal): + ctx = ASTCtx.Store + elif _match_op(ps, TokOp.PlusEq) or _match_op(ps, TokOp.MinusEq): + ctx = ASTCtx.Store + base = Attribute(ps.pool, base, attr_name, ctx) + _set_pos(base, attr_lineno, attr_col, 0, 0) + elif _match_op(ps, TokOp.LPar): + base = _parse_call(ps, base) + elif _match_op(ps, TokOp.Lsqb): + base = _parse_subscript(ps, base) + else: + break + return base + + +def _parse_call(ps: Parser*, func: AST*) -> AST*: + """Parse function call trailer. Both args and keywords are lists.""" + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + _advance(ps) # ( + _skip_nls_and_newlines(ps) # Skip the line break after '(' + args: list[AST*]* = list[AST*](ps.pool, 8) + keywords: list[AST*]* = list[AST*](ps.pool, 8) + + if not _match_op(ps, TokOp.RPar): + while True: + # **expr + if _match_op(ps, TokOp.StarStar): + _advance(ps) + _skip_nls_and_newlines(ps) + val: AST* = _parse_expr(ps) + kw: AST* = KwNode(ps.pool, None, val) + _set_pos(kw, _cur_lineno(ps), _cur_col(ps), 0, 0) + keywords.append(kw) + # *expr + elif _match_op(ps, TokOp.Star): + _advance(ps) + _skip_nls_and_newlines(ps) + # Bind only a unary factor (preserving `*p[0]` = `*(p[0])`), put it on starred_seed and let + # _parse_expr continue the binary ops: `*p + n` => `(*p) + n`, not `*(p + n)`. + seed: AST* = _parse_factor(ps) + if seed is None: + ps.error_count += 1 + break + star: AST* = Starred(ps.pool, seed, ASTCtx.Load) + _set_pos(star, seed.lineno, seed.col_offset, 0, 0) + ps.starred_seed = star + val = _parse_expr(ps) + args.append(val) + # keyword = expr + elif _cur_type(ps) == TokenType.Name: + if ps.cur.next is not None and ps.cur.next.type == TokenType.Op and ps.cur.next.op_subtype == TokOp.Equal: + kw_name: str = _cur_str(ps) + kw_lineno: int = _cur_lineno(ps) + kw_col: int = _cur_col(ps) + _advance(ps) # name + _advance(ps) # = + _skip_nls_and_newlines(ps) + val = _parse_expr(ps) + kw = KwNode(ps.pool, kw_name, val) + _set_pos(kw, kw_lineno, kw_col, 0, 0) + keywords.append(kw) + else: + val = _parse_expr(ps) + args.append(val) + else: + val = _parse_expr(ps) + args.append(val) + _skip_nls_and_newlines(ps) # Skip the line break after the parameter + if not _accept_op(ps, TokOp.Comma): + break + _skip_nls_and_newlines(ps) # Skip the line break after the comma + _skip_nls_and_newlines(ps) # Skip the line break before ')' + _expect_op(ps, TokOp.RPar) + node: AST* = Call(ps.pool, func, args, keywords) + _set_pos(node, lineno, col, 0, 0) + return node + + +def _parse_subscript(ps: Parser*, value: AST*) -> AST*: + """Parse subscript trailer.""" + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + _advance(ps) # [ + _skip_nl(ps) + slice_node: AST* = _parse_slice(ps) + _expect_op(ps, TokOp.Rsqb) + ctx: int = ASTCtx.Load + if _match_op(ps, TokOp.Equal): + ctx = ASTCtx.Store + node: AST* = Subscript(ps.pool, value, slice_node, ctx) + _set_pos(node, lineno, col, 0, 0) + return node + + +def _parse_slice(ps: Parser*) -> AST*: + """Parse slice: [expr] ':' [expr] [':' [expr]] | expr | expr, expr, ...""" + lower: AST* = None + # Empty subscript (`T[]`): `[` directly followed by `]` => slice is None (the type side treats it as + # an empty param list); calling _parse_expr here would report an unexpected token ']'. + if not _match_op(ps, TokOp.Colon) and not _match_op(ps, TokOp.Rsqb): + lower = _parse_expr(ps) + + # Check if there’s a comma (tuple syntax: a[x, y, ...]) + if _match_op(ps, TokOp.Comma): + elts: list[AST*]* = list[AST*](ps.pool, 8) + if lower is not None: + elts.append(lower) + while _accept_op(ps, TokOp.Comma): + if _match_op(ps, TokOp.Rsqb): + break # Trailing comma + elem: AST* = _parse_expr(ps) + if elem is not None: + elts.append(elem) + lineno: int = 0 + if lower is not None: + lineno = lower.lineno + node: AST* = Tuple(ps.pool, elts, ASTCtx.Load) + _set_pos(node, lineno, 0, 0, 0) + return node + + if _match_op(ps, TokOp.Colon): + _advance(ps) + upper: AST* = None + if not _match_op(ps, TokOp.Colon) and not _match_op(ps, TokOp.Rsqb): + upper = _parse_expr(ps) + step: AST* = None + if _accept_op(ps, TokOp.Colon): + if not _match_op(ps, TokOp.Rsqb): + step = _parse_expr(ps) + lineno: int = 0 + if lower is not None: + lineno = lower.lineno + elif upper is not None: + lineno = upper.lineno + else: + lineno = _cur_lineno(ps) + node: AST* = Slice(ps.pool, lower, upper, step) + _set_pos(node, lineno, 0, 0, 0) + return node + return lower + + +def _is_new_expr(ps: Parser*) -> int: + """是否 `new` 描述符表达式（软关键字判定）。 + + `new` 是软关键字：只有紧跟一个**普通名**（描述符 hlife/slife/stack/heap/memgr 或类型名 + `T(args)`）时才当关键字。`new = 1`、`new.x`、`new(1)`、`for new in ...` 都仍是标识符。 + """ + if _is_named_soft(ps, "new") == 0: return 0 + nxt: Token* = ps.cur.next + if nxt is None: return 0 + if nxt.type != TokenType.Name: return 0 + if nxt.kw_subtype != Keyword.NotKeyword: return 0 + return 1 + + +def _parse_new_expr(ps: Parser*) -> AST*: + """`new [Descriptor...] T(args)`：描述符顺序无关、各轴最多一次（冲突即报错）。 + + 结果恒为 `T*`；来源缺省 = heap（D3），生命期缺省由来源决定（语义侧落地）。 + """ + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + _advance(ps) # 吃掉 `new` + src: int = NEW_SRC_DEFAULT + life: int = NEW_LIFE_DEFAULT + memgr_arg: AST* = None + while True: + if _is_named_soft(ps, "hlife") != 0: + if life != NEW_LIFE_DEFAULT: + _record_parse_err(ps) + ps.error_count += 1 + return None + life = NEW_LIFE_HLIFE + _advance(ps) + continue + if _is_named_soft(ps, "slife") != 0: + if life != NEW_LIFE_DEFAULT: + _record_parse_err(ps) + ps.error_count += 1 + return None + life = NEW_LIFE_SLIFE + _advance(ps) + continue + if _is_named_soft(ps, "stack") != 0: + if src != NEW_SRC_DEFAULT: + _record_parse_err(ps) + ps.error_count += 1 + return None + src = NEW_SRC_STACK + _advance(ps) + continue + if _is_named_soft(ps, "heap") != 0: + if src != NEW_SRC_DEFAULT: + _record_parse_err(ps) + ps.error_count += 1 + return None + src = NEW_SRC_HEAP + _advance(ps) + continue + if _is_named_soft(ps, "memgr") != 0: + if src != NEW_SRC_DEFAULT: + _record_parse_err(ps) + ps.error_count += 1 + return None + _advance(ps) # memgr + if _expect_op(ps, TokOp.LPar) == 0: return None + _skip_nl(ps) + memgr_arg = _parse_test(ps) + if memgr_arg is None: return None + if _expect_op(ps, TokOp.RPar) == 0: return None + src = NEW_SRC_MEMGR + continue + break + # 类型 + 可选实参：类型用**注解规则**解析 —— 这样前导数组声明符 `[N]T`、指针后缀 `T*`、 + # 泛型下标 `list[int]`、描述符前缀都与注解位置同口径；`T(args)` 的实参括号会被 `_parse_test` + # 自身吃成 Call，故这里按"是不是 Call"分流，行为与旧版一致。 + callee: AST* = _parse_type_ann(ps) + if callee is None: + _record_parse_err(ps) + ps.error_count += 1 + return None + callee_kind: int = callee.kind() + func_node: AST* = callee + args_node: list[AST*]* = None + if callee_kind == ASTKind.Call: + cl: Call* = Call*(callee) + func_node = cl.func + args_node = cl.args + elif callee_kind != ASTKind.Name and callee_kind != ASTKind.Attribute and callee_kind != ASTKind.Subscript: + _record_parse_err(ps) + ps.error_count += 1 + return None + node: New* = New(ps.pool, src, life, memgr_arg, func_node, args_node) + _set_pos(node, lineno, col, 0, 0) + return node + + +def _parse_atom(ps: Parser*) -> AST*: + """atom: the smallest unit of expression""" + ttype: int = _cur_type(ps) + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + + if ttype == TokenType.Name: + # M2.4 `new` 描述符分配（软关键字）：仅当后面确实跟着描述符/类型调用时才当关键字 + if _is_new_expr(ps) != 0: return _parse_new_expr(ps) + kw: int = _cur_kw(ps) + if kw == Keyword.True_: + _advance(ps) + return _new_const_bool(ps.pool, 1, lineno, col) + if kw == Keyword.False_: + _advance(ps) + return _new_const_bool(ps.pool, 0, lineno, col) + if kw == Keyword.None_: + _advance(ps) + return _new_const_none(ps.pool, lineno, col) + name_str: str = _cur_str(ps) + _advance(ps) + node: AST* = Name(ps.pool, name_str, ASTCtx.Load) + _set_pos(node, lineno, col, 0, 0) + return node + + if ttype == TokenType.Number: + tok: Token* = ps.cur + _advance(ps) + if tok.is_float: + return _new_const_float(ps.pool, tok.float_val, lineno, col) + return _new_const_int(ps.pool, tok.int_val, lineno, col, tok.int_width, tok.int_unsigned) + + if ttype == TokenType.String: + sval: str = _cur_str(ps) + is_fstr: int = ps.cur.kw_subtype & 4 + is_triple: int = ps.cur.kw_subtype & 8 + is_dquote: int = ps.cur.kw_subtype & 16 + is_char: int = ps.cur.kw_subtype & 32 + _advance(ps) + node: AST* + if is_fstr: + node = _parse_fstring(ps, sval, lineno, col) + else: + node = _new_const_str(ps.pool, sval, lineno, col, is_dquote, is_triple, is_char) + # Python implicit adjacent string literal concatenation: "a" "b" -> "ab". + # Nl is the in-paren newline token (allowed across lines); statement NewLine is not skipped. + while True: + _skip_nl(ps) + if _cur_type(ps) != TokenType.String: + break + n_sval: str = _cur_str(ps) + n_fstr: int = ps.cur.kw_subtype & 4 + n_triple: int = ps.cur.kw_subtype & 8 + n_dquote: int = ps.cur.kw_subtype & 16 + n_char: int = ps.cur.kw_subtype & 32 + _advance(ps) + n_node: AST* + if n_fstr: + n_node = _parse_fstring(ps, n_sval, lineno, col) + else: + n_node = _new_const_str(ps.pool, n_sval, lineno, col, n_dquote, n_triple, n_char) + node = _concat_str_nodes(ps.pool, node, n_node) + return node + + if _match_op(ps, TokOp.Ellipsis): + _advance(ps) + return _new_const_none(ps.pool, lineno, col) + + if _match_op(ps, TokOp.LPar): + return _parse_paren(ps) + if _match_op(ps, TokOp.Lsqb): + return _parse_list(ps) + if _match_op(ps, TokOp.LBrace): + return _parse_brace(ps) + + _record_parse_err(ps) + ps.error_count += 1 + return None + + +def _parse_paren(ps: Parser*) -> AST*: + """Parenthesis expressions: () | (expr) | (expr,) | (expr, ...) | (yield)""" + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + _advance(ps) # ( + _skip_nl(ps) # Skip the line break after '(' (multi-line tuple) + if _match_op(ps, TokOp.RPar): + _advance(ps) + # Empty tuple + empty: list[AST*]* = list[AST*](ps.pool, 8) + node: AST* = Tuple(ps.pool, empty, ASTCtx.Load) + _set_pos(node, lineno, col, 0, 0) + return node + # yield + if _cur_kw(ps) == Keyword.Yield: + y: AST* = _parse_yield(ps) + _expect_op(ps, TokOp.RPar) + return y + first: AST* = _parse_expr(ps) + if _match_op(ps, TokOp.Comma): + # Tuple + elts: list[AST*]* = list[AST*](ps.pool, 8) + elts.append(first) + while _accept_op(ps, TokOp.Comma): + _skip_nl(ps) + if _match_op(ps, TokOp.RPar): + break + e: AST* = _parse_expr(ps) + elts.append(e) + _skip_nl(ps) # finish when the last element has no trailing comma (multi-line tuple) + _expect_op(ps, TokOp.RPar) + node = Tuple(ps.pool, elts, ASTCtx.Load) + _set_pos(node, lineno, col, 0, 0) + return node + # Paren-less C cast `(T*)x`: the inner expression ends with a pointer suffix + # (ps.prev is that suffix's '*'/'**' token) and the operand follows ')' directly. + # Same product as `(T*)(x)` / `T*(x)`: a Call whose func is `T | t.CPtr`. + is_ptr_cast: int = 0 + if _g_ptr_suffix_line > 0 and _match_op(ps, TokOp.RPar) and ps.prev.lineno == _g_ptr_suffix_line and ps.prev.col_offset == _g_ptr_suffix_col: + is_ptr_cast = 1 + _skip_nl(ps) # (expr\n) close a single-expression paren spanning lines + _expect_op(ps, TokOp.RPar) + # Check for generator expression + if _cur_kw(ps) == Keyword.For: + return _parse_comprehension(ps, first, ASTKind.GeneratorExp, lineno, col) + if is_ptr_cast != 0 and _at_cast_operand(ps) != 0: + operand: AST* = _parse_factor(ps) + cast_args: list[AST*]* = list[AST*](ps.pool, 8) + cast_args.append(operand) + cast_kws: list[AST*]* = list[AST*](ps.pool, 8) + cast_node: AST* = Call(ps.pool, first, cast_args, cast_kws) + _set_pos(cast_node, first.lineno, first.col_offset, 0, 0) + return cast_node + return first + + +def _parse_list(ps: Parser*) -> AST*: + """List expressions: [] | [expr_list] | [expr_list comp_for]""" + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + _advance(ps) # [ + _skip_nl(ps) # Skip the line break after '[' (multi-line list) + if _match_op(ps, TokOp.Rsqb): + _advance(ps) + empty: list[AST*]* = list[AST*](ps.pool, 8) + node: AST* = List(ps.pool, empty, ASTCtx.Load) + _set_pos(node, lineno, col, 0, 0) + return node + first: AST* = _parse_expr(ps) + if _cur_kw(ps) == Keyword.For: + node = _parse_comprehension(ps, first, ASTKind.ListComp, lineno, col) + _expect_op(ps, TokOp.Rsqb) + return node + elts: list[AST*]* = list[AST*](ps.pool, 8) + elts.append(first) + while _accept_op(ps, TokOp.Comma): + _skip_nl(ps) + if _match_op(ps, TokOp.Rsqb): + break + e: AST* = _parse_expr(ps) + elts.append(e) + _skip_nl(ps) # finish when the last element has no trailing comma (multi-line list) + _expect_op(ps, TokOp.Rsqb) + node = List(ps.pool, elts, ASTCtx.Load) + _set_pos(node, lineno, col, 0, 0) + return node + + +def _parse_brace(ps: Parser*) -> AST*: + """Brace expressions: {} | {set_items} | {dict_items} | {comp_for}""" + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + _advance(ps) # { + _skip_nl(ps) # Skip the line break after '{' (multi-line dict) + if _match_op(ps, TokOp.RBrace): + _advance(ps) + empty_k: list[AST*]* = list[AST*](ps.pool, 8) + empty_v: list[AST*]* = list[AST*](ps.pool, 8) + node: AST* = Dict(ps.pool, empty_k, empty_v) + _set_pos(node, lineno, col, 0, 0) + return node + first: AST* = _parse_expr(ps) + # dict: key: value + if _match_op(ps, TokOp.Colon): + _advance(ps) + val: AST* = _parse_expr(ps) + if _cur_kw(ps) == Keyword.For: + # dict comprehension + gens: list[AST*]* = _parse_comp_for(ps) + node = DictComp(ps.pool, first, val, gens) + _set_pos(node, lineno, col, 0, 0) + _expect_op(ps, TokOp.RBrace) + return node + keys: list[AST*]* = list[AST*](ps.pool, 8) + values: list[AST*]* = list[AST*](ps.pool, 8) + keys.append(first) + values.append(val) + while _accept_op(ps, TokOp.Comma): + _skip_nl(ps) + if _match_op(ps, TokOp.RBrace): + break + k: AST* = _parse_expr(ps) + _expect_op(ps, TokOp.Colon) + v: AST* = _parse_expr(ps) + keys.append(k) + values.append(v) + _skip_nl(ps) # finish when the last element has no trailing comma + _expect_op(ps, TokOp.RBrace) + node = Dict(ps.pool, keys, values) + _set_pos(node, lineno, col, 0, 0) + return node + # set + if _cur_kw(ps) == Keyword.For: + node = _parse_comprehension(ps, first, ASTKind.SetComp, lineno, col) + _expect_op(ps, TokOp.RBrace) + return node + elts: list[AST*]* = list[AST*](ps.pool, 8) + elts.append(first) + while _accept_op(ps, TokOp.Comma): + _skip_nl(ps) # cross-line set literal inside braces + if _match_op(ps, TokOp.RBrace): + break + e: AST* = _parse_expr(ps) + elts.append(e) + _skip_nl(ps) # finish when the last element has no trailing comma + _expect_op(ps, TokOp.RBrace) + node = Set(ps.pool, elts) + _set_pos(node, lineno, col, 0, 0) + return node + + +# ============================================================ +# f-string parsing +# ============================================================ + +def _parse_sub_expr(ps: Parser*, src: str) -> AST*: + """Create a sub-lexer and parser to parse expressions (used for {expr} inside f-strings)""" + pool: memhub.MemManager* = ps.pool + lx: Lexer* = new_lexer(pool) + if lx is None: return None + _lexer_init(lx, src, pool) + tokens: Token* = tokenize(lx) + if tokens is None: return None + sub_ps: Parser* = new_parser(pool) + if sub_ps is None: return None + _parser_init(sub_ps, tokens, pool) + return _parse_test(sub_ps) + + +def _append_str_value(pool: memhub.MemManager*, + values: list[AST*]*, + node: AST*): + """Flatten a string node into values (JoinedStr expands its values, everything else is appended)""" + if node is None: + return + if node.kind() == ASTKind.JoinedStr: + js_val: JoinedStr* = (JoinedStr*)(node) + if js_val.values is not None: + n_v: sizet = js_val.values.__len__() + i_v: sizet + for i_v in range(int(n_v)): + values.append(js_val.values.get(i_v)) + else: + values.append(node) + + +def _concat_str_nodes(pool: memhub.MemManager*, + a: AST*, b: AST*) -> AST*: + """Merge adjacent string literal nodes (Python implicit concatenation "a" "b" -> "ab"). + + Pure constant strings are concatenated as text; f-strings (JoinedStr/FormattedValue) are merged + into a JoinedStr values list. + """ + if a is None: + return b + if b is None: + return a + # Both are constant strings -> concatenate the text directly + if a.kind() == ASTKind.Constant and b.kind() == ASTKind.Constant: + ca_c: Constant* = (Constant*)(a) + cb_c: Constant* = (Constant*)(b) + if ca_c.const_kind == CONST_STR and cb_c.const_kind == CONST_STR \ + and ca_c.str_val is not None and cb_c.str_val is not None: + la_c: sizet = string.strlen(ca_c.str_val) + lb_c: sizet = string.strlen(cb_c.str_val) + buf_c: str = pool.alloc(la_c + lb_c + 1) + if buf_c is not None: + string.memcpy(buf_c, ca_c.str_val, la_c) + string.memcpy(buf_c + la_c, cb_c.str_val, lb_c + 1) + return _new_const_str(pool, buf_c, a.lineno, a.col_offset, 0, 0, 0) + # Involves an f-string -> merge into a JoinedStr + values_c: list[AST*]* = list[AST*](pool, 8) + _append_str_value(pool, values_c, a) + _append_str_value(pool, values_c, b) + if values_c.__len__() == 1: + return values_c.get(0) + node_c: AST* = JoinedStr(pool, values_c) + _set_pos(node_c, a.lineno, a.col_offset, 0, 0) + return node_c + + +def _parse_fstring(ps: Parser*, content: str, + lineno: int, col: int) -> AST*: + """Parse f-string content, building JoinedStr or FormattedValue nodes""" + pool: memhub.MemManager* = ps.pool + values: list[AST*]* = list[AST*](pool, 8) + i: sizet = 0 + content_len: sizet = string.strlen(content) + # Literal text buffer for f-string content + lit_buf: str = pool.alloc(content_len + 1) + if lit_buf is None: return None + lit_pos: sizet = 0 + + while i < content_len: + ch: char = content[i] + if ch == '{': + # {{ → Literal { + if i + 1 < content_len and content[i + 1] == '{': + lit_buf[lit_pos] = '{' + lit_pos += 1 + i += 2 + continue + # Refresh literal buffer + if lit_pos > 0: + lit_buf[lit_pos] = '\0' + lit_node: AST* = _new_const_str(pool, lit_buf, lineno, col) + values.append(lit_node) + lit_buf = pool.alloc(content_len + 1) + if lit_buf is None: break + lit_pos = 0 + # Find matching closing '}' while tracking nested '{}' and string literals inside the f-string + j: sizet = i + 1 + depth: int = 1 + conv_pos: sizet = 0 + fmt_pos: sizet = 0 + while j < content_len and depth > 0: + c: char = content[j] + if c == '\'' or c == '"': + # Skip nested string literals inside an expression block (PEP 701 same-quote nesting) + # so their characters do not confuse the {} depth / conversion / format delimiter scan + nq: char = c + j += 1 + while j < content_len: + c2: char = content[j] + if c2 == '\\': + j += 2 + continue + if c2 == nq: + j += 1 + break + j += 1 + continue + if c == '{': + depth += 1 + elif c == '}': + depth -= 1 + if depth == 0: + break + elif depth == 1: + if c == '!' and conv_pos == 0 and fmt_pos == 0: + if j + 1 < content_len: + nc: char = content[j + 1] + if nc == 's' or nc == 'r' or nc == 'a': + if j + 2 < content_len: + after: char = content[j + 2] + if after == ':' or after == '}': + conv_pos = j + elif j + 2 == content_len: + conv_pos = j + elif c == ':' and fmt_pos == 0: + fmt_pos = j + j += 1 + if depth != 0: + _record_parse_err(ps) + ps.error_count += 1 + break + # Extract expression text + expr_end: sizet = j + if conv_pos > 0: + expr_end = conv_pos + elif fmt_pos > 0: + expr_end = fmt_pos + expr_len: sizet = expr_end - i - 1 + if expr_len == 0: + _record_parse_err(ps) + ps.error_count += 1 + i = j + 1 + continue + expr_buf: str = pool.alloc(expr_len + 1) + if expr_buf is None: break + string.memcpy(expr_buf, content + i + 1, expr_len) + expr_buf[expr_len] = '\0' + # Parse expression text + expr_node: AST* = _parse_sub_expr(ps, expr_buf) + # Conversion specifier + conversion: int = -1 + if conv_pos > 0: + conv_char: char = content[conv_pos + 1] + if conv_char == 's': + conversion = 115 + elif conv_char == 'r': + conversion = 114 + elif conv_char == 'a': + conversion = 97 + # Format specifier + format_spec: AST* = None + if fmt_pos > 0: + spec_len: sizet = j - fmt_pos - 1 + if spec_len > 0: + spec_buf: str = pool.alloc(spec_len + 1) + if spec_buf is not None: + string.memcpy(spec_buf, content + fmt_pos + 1, spec_len) + spec_buf[spec_len] = '\0' + format_spec = _new_const_str(pool, spec_buf, lineno, col) + fv: AST* = FormattedValue(pool, expr_node, conversion, format_spec) + _set_pos(fv, lineno, col, 0, 0) + values.append(fv) + i = j + 1 + elif ch == '}': + if i + 1 < content_len and content[i + 1] == '}': + lit_buf[lit_pos] = '}' + lit_pos += 1 + i += 2 + else: + _record_parse_err(ps) + ps.error_count += 1 + i += 1 + else: + lit_buf[lit_pos] = ch + lit_pos += 1 + i += 1 + + # Refresh remaining literal buffer + if lit_pos > 0: + lit_buf[lit_pos] = '\0' + lit_node2: AST* = _new_const_str(pool, lit_buf, lineno, col) + values.append(lit_node2) + + # If no content, return empty string + if values.__len__() == 0: + return _new_const_str(pool, "", lineno, col) + # If only one FormattedValue and no literal, return FormattedValue + if values.__len__() == 1: + first: AST* = values.get(0) + if first.kind() == ASTKind.FormattedValue: + return first + node: AST* = JoinedStr(pool, values) + _set_pos(node, lineno, col, 0, 0) + return node + + +def _parse_comprehension(ps: Parser*, elt: AST*, + vtype: int, lineno: int, col: int) -> AST*: + """General comprehension: elt comp_for""" + gens: list[AST*]* = _parse_comp_for(ps) + node: AST* = None + if vtype == ASTKind.ListComp: + node = ListComp(ps.pool, elt, gens) + elif vtype == ASTKind.SetComp: + node = SetComp(ps.pool, elt, gens) + else: + node = GeneratorExp(ps.pool, elt, gens) + _set_pos(node, lineno, col, 0, 0) + return node + + +def _parse_comp_for(ps: Parser*) -> list[AST*]*: + """comp_for: ['async'] 'for' exprlist 'in' or_test [comp_if] (comp_for)* + Returns generators list.""" + gens: list[AST*]* = list[AST*](ps.pool, 8) + while True: + is_async: int = 0 + if _cur_kw(ps) == Keyword.Async: + is_async = 1 + _advance(ps) + if _cur_kw(ps) != Keyword.For: + break + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + _advance(ps) + target: AST* = _parse_target_list(ps) + if _cur_kw(ps) != Keyword.In: + ps.error_count += 1 + break + _advance(ps) + iter_: AST* = _parse_or_test(ps) + # Collect ifs to list + ifs: list[AST*]* = list[AST*](ps.pool, 8) + while _cur_kw(ps) == Keyword.If: + _advance(ps) + cond: AST* = _parse_or_test(ps) + ifs.append(cond) + comp: AST* = Comprehension(ps.pool, target, iter_, ifs, is_async) + _set_pos(comp, lineno, col, 0, 0) + gens.append(comp) + return gens + + +def _parse_target_list(ps: Parser*) -> AST*: + """Parse assignment target list (comma-separated)""" + first: AST* = _parse_target(ps) + if _match_op(ps, TokOp.Comma): + elts: list[AST*]* = list[AST*](ps.pool, 8) + elts.append(first) + while _accept_op(ps, TokOp.Comma): + if _cur_kw(ps) == Keyword.In: + break + e: AST* = _parse_target(ps) + elts.append(e) + node: AST* = Tuple(ps.pool, elts, ASTCtx.Store) + _set_pos(node, first.lineno, first.col_offset, 0, 0) + return node + return first + + +def _parse_target(ps: Parser*) -> AST*: + """Parse assignment target""" + if _match_op(ps, TokOp.Star): + _advance(ps) + inner: AST* = _parse_atom_expr(ps) + node: AST* = Starred(ps.pool, inner, ASTCtx.Store) + _set_pos(node, inner.lineno, inner.col_offset, 0, 0) + return node + if _match_op(ps, TokOp.LPar): + _advance(ps) + _skip_nl(ps) + t_: AST* = _parse_target_list(ps) + _skip_nl(ps) + _expect_op(ps, TokOp.RPar) + return t_ + if _match_op(ps, TokOp.Lsqb): + _advance(ps) + _skip_nl(ps) + first: AST* = _parse_target(ps) + elts: list[AST*]* = list[AST*](ps.pool, 8) + elts.append(first) + while _accept_op(ps, TokOp.Comma): + _skip_nl(ps) + if _match_op(ps, TokOp.Rsqb): + break + e: AST* = _parse_target(ps) + elts.append(e) + _expect_op(ps, TokOp.Rsqb) + node = List(ps.pool, elts, ASTCtx.Store) + _set_pos(node, first.lineno, first.col_offset, 0, 0) + return node + # Normal atom_expr, ctx=Store + node = _parse_atom_expr(ps) + if node is not None: + _set_ctx(node, ASTCtx.Store) + return node + + +def _parse_yield(ps: Parser*) -> AST*: + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + _advance(ps) # yield + if _cur_kw(ps) == Keyword.From: + _advance(ps) + val: AST* = _parse_expr(ps) + node: AST* = YieldFrom(ps.pool, val) + _set_pos(node, lineno, col, 0, 0) + return node + val2: AST* = None + if not _match_op(ps, TokOp.RPar) and _cur_type(ps) != TokenType.NewLine and _cur_type(ps) != TokenType.EndMarker: + if _cur_type(ps) == TokenType.Op and (_cur_op(ps) == TokOp.Comma or _cur_op(ps) == TokOp.Rsqb): + pass + else: + val2 = _parse_test(ps) + node = Yield(ps.pool, val2) + _set_pos(node, lineno, col, 0, 0) + return node + + +def _parse_varargslist(ps: Parser*, is_lambda: int = 0) -> AST*: + """Parse function parameter list + New API: args/defaults/kw_defaults are lists, vararg/kwarg are fields.""" + args: list[AST*]* = list[AST*](ps.pool, 8) + vararg: AST* = None + kwarg: AST* = None + defaults: list[AST*]* = list[AST*](ps.pool, 8) + kw_defaults: list[AST*]* = list[AST*](ps.pool, 8) + + while True: + _skip_nl(ps) + if _match_op(ps, TokOp.RPar): + break + # *args or * (kwonly separated) + if _match_op(ps, TokOp.Star): + _advance(ps) + _skip_nl(ps) + if _cur_type(ps) == TokenType.Name: + vararg = _parse_arg_def(ps, is_lambda) + if not _accept_op(ps, TokOp.Comma): + break + _skip_nl(ps) + continue + # **kwargs + if _match_op(ps, TokOp.StarStar): + _advance(ps) + _skip_nl(ps) + kwarg = _parse_arg_def(ps, is_lambda) + if not _accept_op(ps, TokOp.Comma): + break + _skip_nl(ps) + continue + # / Delimiter (posonly end)— New API: posonly/regular are not distinguished + if _match_op(ps, TokOp.Slash): + _advance(ps) + if not _accept_op(ps, TokOp.Comma): + break + _skip_nl(ps) + continue + # Normal parameter + a: AST* = _parse_arg_def(ps, is_lambda) + if a is None: break + args.append(a) + if _match_op(ps, TokOp.Equal): + _advance(ps) + d: AST* = _parse_test(ps) + defaults.append(d) + if not _accept_op(ps, TokOp.Comma): + break + _skip_nl(ps) + return Arguments(ps.pool, args, vararg, kwarg, defaults, kw_defaults) + + +def _parse_array_prefix_declarator(ps: Parser*) -> AST*: + """C-style leading array declarator `[N]T` / `[]T` -> `t.CArray[T, N]` / `t.CArray[T, None]`. + + Consumed only in annotation position (_parse_type_ann); returns None if the current token is not '['. + Desugars to the canonical array form shared with TPC / StubGen / DeclarationGenerator. Must be called + before _parse_test, otherwise `[N]` is parsed as a list literal and the element type then errors. + """ + if _match_op(ps, TokOp.Lsqb) == 0: + return None + lb_lineno: int = _cur_lineno(ps) + lb_col: int = _cur_col(ps) + _advance(ps) # consume '[' + count_node: AST* = None + if _match_op(ps, TokOp.Rsqb) == 0: + # Length expression (constant/constant expression; invalid lengths are rejected downstream) + count_node = _parse_test(ps) + if count_node is None: + _record_parse_err(ps) + return None + if _match_op(ps, TokOp.Rsqb) == 0: + _record_parse_err(ps) # missing ']' + ps.error_count += 1 + return None + _advance(ps) # consume ']' + # Element type: recurse into _parse_type_ann, so `[N]const int` / `[N]int*` share one rule source + elem: AST* = _parse_type_ann(ps) + if elem is None: + return None + if count_node is None: + count_node = _new_const_none(ps.pool, lb_lineno, lb_col) + # Element size must be explicitly 8 (list[T](pool, elem_size) addresses by byte stride; passing 2 + # would write 8-byte pointers with a 2-byte stride -> heap overflow). Same arg as other list sites. + elts: list[AST*]* = list[AST*](ps.pool, 8) + elts.append(elem) + elts.append(count_node) + carray: AST* = Subscript(ps.pool, + Attribute(ps.pool, Name(ps.pool, "t", ASTCtx.Load), + "CArray", ASTCtx.Load), + Tuple(ps.pool, elts, ASTCtx.Load), + ASTCtx.Load) + _set_pos(carray, lb_lineno, lb_col, 0, 0) + return carray + + +def _IsCTypedefAnnotation(ann: AST*) -> int: + """1 if the annotation is a typedef marker (bare CTypedef / t.CTypedef / a union containing it). + + Used to parse a typedef's right-hand side as a **type**: `X: t.CTypedef = const uint16*` + synthesizes descriptors and the `*` suffix through _parse_type_ann (the annotation rule), + so both sides agree. TPC's vast Parser._IsCTypedefAnnotation uses the same name-based test. + """ + if ann is None: return 0 + kd: int = ann.kind() + if kd == ASTKind.Attribute: + at: Attribute* = Attribute*(ann) + if at.attr is not None and string.strcmp(at.attr, "CTypedef") == 0: return 1 + return 0 + if kd == ASTKind.Name: + nm: Name* = Name*(ann) + if nm.id is not None and string.strcmp(nm.id, "CTypedef") == 0: return 1 + return 0 + if kd == ASTKind.BinOp: + bo: BinOp* = BinOp*(ann) + if bo.left is not None and _IsCTypedefAnnotation(bo.left) != 0: return 1 + if bo.right is not None and _IsCTypedefAnnotation(bo.right) != 0: return 1 + return 0 + return 0 + + +def _parse_type_ann(ps: Parser*) -> AST*: + """Parse a type annotation: descriptor prefixes (const / volatile / __attribute__) and C-style pointer + suffixes (* / **), field-identical to TPC's expansion. `static` is invalid here; missing type / non-type + follower / bare use fail loudly. Reclaims the Mult/Pow junk _parse_test leaves and rebuilds t.CPtr chains. + """ + # Value-reference parameter prefix: `&T` (ref mode) / `&temp T` (temp mode). Recorded as a t.CRef / + # t.CTempRef union member so the Gen layer can register call-site auto-address. This is the annotation + # position only; the expression-position `&x` (AddrOf) stays in _parse_factor and is left untouched. + # 回收伪错（见下 ptr_level 分支）时要连**位置/消息**一起回退，故这里读写解析错误通道。 + global _g_parse_err_line, _g_parse_err_col, _g_parse_err_msg + ref_marker: str = None + if _accept_op(ps, TokOp.Amp) != 0: + ref_marker = "CRef" + if _cur_type(ps) == TokenType.Name: + nm_ref: str = _cur_str(ps) + if nm_ref is not None and string.strcmp(nm_ref, "temp") == 0: + # `temp` is a soft keyword only when a type name follows (e.g. `&temp int`). + nxt_ref: Token* = ps.cur.next + if nxt_ref is not None and nxt_ref.type == TokenType.Name: + _advance(ps) + ref_marker = "CTempRef" + # Descriptor prefix: collect const / volatile markers (soft keywords, consumed only here) + mods: list[AST*]* = None + mod_count: int = 0 + mod_lineno: int = _cur_lineno(ps) + mod_col: int = _cur_col(ps) + # `static` modifier (consumed at statement start): append t.CStatic as the first descriptor marker + # (pasted on the union's left later), then clear it so it applies only to this annotation. + if ps.pending_static != 0: + ps.pending_static = 0 + if mods is None: + mods = list[AST*](ps.pool, 8) + st_mk: AST* = _MakeStaticMarker(ps) + if st_mk is not None: + mods.append(st_mk) + mod_count += 1 + while _IsDescriptorKeyword(ps) != 0 or _IsMagicAttributeCall(ps) != 0 \ + or _IsStaticDescriptorKeyword(ps) != 0: + if _IsStaticDescriptorKeyword(ps) != 0: + # Annotation-position `static` (`name: static T`) is no longer supported: storage-class markers + # are always prefixed. Fail loudly here (otherwise `static` becomes an opaque "static" class). + _RecordDescriptorErr("static in annotation position is no longer supported: use the prefix form" + " `static name: type = value` (equivalent to `name: t.CStatic | type`)", + _cur_lineno(ps), _cur_col(ps)) + _advance(ps) + continue + if mods is None: + # Element size is explicitly 8: list[T](pool, elem_size) addresses by byte stride, so passing 4 + # would make the 2nd 8-byte pointer (e.g. for `const volatile`) write out of bounds. + mods = list[AST*](ps.pool, 8) + if _IsMagicAttributeCall(ps) != 0: + # C-style `a: __attribute__(section(".text.wo"), aligned(16)) const int = ...`: same position + # and order-free mixing as const / volatile; each argument desugars to `t.attr.` and joins + # the same markers list. + # Use only _parse_call for this call itself, never _parse_test: the latter is a general expression + # parser and would swallow the following `[]T` as a legal empty subscript, silently degrading the + # annotation to a bare element type with zero errors reported. + mka_lineno: int = _cur_lineno(ps) + mka_col: int = _cur_col(ps) + mka_fn: AST* = Name(ps.pool, "__attribute__", ASTCtx.Load) + _set_pos(mka_fn, mka_lineno, mka_col, 0, 0) + _advance(ps) # consume `__attribute__` + ac_node: AST* = _parse_call(ps, mka_fn) # parse only `( args )` + if ac_node is None: + break + if ac_node.kind() != ASTKind.Call: + break + ac: Call* = Call*(ac_node) + if ac.args is not None: + acn: sizet = len(ac.args) + for aci in range(int(acn)): + mk: AST* = _MakeTAttrMarker(ps, ac.args.get(aci)) + if mk is not None: + mods.append(mk) + mod_count += 1 + continue + mods.append(_MakeDescriptorMarker(ps)) + mod_count += 1 + _advance(ps) + # C-style leading array declarator `[N]T` / `[]T` (incl. `const [N]T`): desugar in place to + # `t.CArray[T, N]` / `t.CArray[T, None]` and paste the already-collected descriptors onto it. + if _match_op(ps, TokOp.Lsqb) != 0: + arr: AST* = _parse_array_prefix_declarator(ps) + if arr is None: + return None + if mod_count > 0: + arr = _PrependDescriptorMarkers(ps, arr, mods, mod_count) + return arr + if mod_count > 0 and _cur_type(ps) != TokenType.Name: + # Missing type after a descriptor (`a: const` / `a: const = 1` / `def f(a: const)`): + # fail loudly -- never silently drop the descriptor. + _RecordDescriptorErr("missing type after descriptor", mod_lineno, mod_col) + ps.error_count += 1 + return None + before_ec: int = ps.error_count + # 错误记录快照：_parse_test 会把"末尾 '*' 无右操作数"记成一处语法错误（见下方 ptr_level 回收）， + # 回收时连位置/消息一起回退，否则后面真正的语法错误会报出这个**过期**位置。 + err_line_before: int = _g_parse_err_line + err_col_before: int = _g_parse_err_col + err_msg_before: str = _g_parse_err_msg + base: AST* = _parse_test(ps) + if base is None: + if mod_count > 0: + # Descriptor consumed but type parsing failed (e.g. `a: const = 1`): fail loudly keeping the descriptor error. + _RecordDescriptorErr("missing type after descriptor (type parse failed)", mod_lineno, mod_col) + _record_parse_err(ps) # annotation parse failed: record position (error_count already bumped in _parse_test) + return None + if mod_count > 0 and base.kind() == ASTKind.Constant: + # Non-type follower (`a: const 1` / `a: const "x"` / `a: const True`): fail loudly, since + # descriptors modify types only and constants are not types -- never silently degrade or drop. + _RecordDescriptorErr("descriptor followed by a non-type (descriptors modify types only)", mod_lineno, mod_col) + ps.error_count += 1 + return None + ptr_level: int = 0 + # Reclaim the junk nodes left by _parse_test consuming the * / ** suffixes + if base.kind() == ASTKind.BinOp: + gb: BinOp* = (BinOp*)(base) + if gb.op == OpKind.Mult and gb.right is None: + ptr_level += 1 + base = gb.left + elif gb.op == OpKind.Pow: + if gb.right is None: + ptr_level += 2 + base = gb.left + elif gb.right.kind() == ASTKind.UnaryOp: + gu: UnaryOp* = (UnaryOp*)(gb.right) + if gu.op == OpKind.Deref and gu.operand is None: + ptr_level += 3 + base = gb.left + if ptr_level > 0: + ps.error_count = before_ec # reclaim succeeded: restore error count to avoid whole-file parse failure + # 位置/消息同样要回退：这次回收解决的只有刚记下的那一处伪错（末尾 '*' 被当成乘法、右操作数缺失）。 + # 只回退计数不回退位置，过期位置就会顶掉后面真正的语法错误（现场：EscapeTest 第 13 行 + # `new stack char` 被报成第 6 行 `a: char*` 的 'unexpected token (none)'，排查方向被完全带偏）。 + # 判据用"记录是否在本次解析中改变"：若快照本来就非 0（更早的真错误），当前值未变，回退是空操作。 + if _g_parse_err_line != err_line_before: + _g_parse_err_line = err_line_before + _g_parse_err_col = err_col_before + _g_parse_err_msg = err_msg_before + # Consume remaining suffixes ('**' is one token counting as 2 layers, '*' as 1) + while _match_op(ps, TokOp.StarStar): + _advance(ps) + ptr_level += 2 + while _match_op(ps, TokOp.Star): + _advance(ps) + ptr_level += 1 + # Build n nested base | t.CPtr layers + for li_an in range(int(ptr_level)): + cptr_an: AST* = Attribute(ps.pool, + Name(ps.pool, "t", ASTCtx.Load), + "CPtr", ASTCtx.Load) + base = BinOp(ps.pool, base, OpKind.BitOr, cptr_an) + # Paste descriptors last (same shape as TPC text expansion; see _PrependDescriptorMarkers) + if mod_count > 0: + base = _PrependDescriptorMarkers(ps, base, mods, mod_count) + # Append the value-reference marker as the outermost union member: `base | t.CRef` / `base | t.CTempRef`. + if ref_marker is not None: + ref_marker_node: AST* = Attribute(ps.pool, + Name(ps.pool, "t", ASTCtx.Load), + ref_marker, ASTCtx.Load) + base = BinOp(ps.pool, base, OpKind.BitOr, ref_marker_node) + return base + + +def _parse_arg_def(ps: Parser*, is_lambda: int = 0) -> AST*: + """arg: TokenType.Name [':' annotation]""" + if _cur_type(ps) != TokenType.Name: + ps.error_count += 1 + return None + name_str: str = _cur_str(ps) + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + _advance(ps) + annotation: AST* = None + if is_lambda == 0: + if _accept_op(ps, TokOp.Colon): + annotation = _parse_type_ann(ps) + result: AST* = Arg(ps.pool, name_str, annotation) + _set_pos(result, lineno, col, 0, 0) + return result + + +# ============================================================ +# Statement parsing (Design: directly append to parent, do not return a list) +# ============================================================ + +def _parse_suite_into_children(ps: Parser*, parent: AST*, + owner_is_class: int = 0): + """Parse statement suite, directly append to parent's children. + suite: TokenType.NewLine TokenType.Indent stmt+ TokenType.Dedent | simple_stmt + + owner_is_class: whether this suite's direct owner is a ClassDef. During suite parsing ps.in_class_body + is set so _ParseVirtualModified can tell if `virtual def`/`novirtual def` is directly in a class body + (function/branch/loop bodies do not count). Default 0 keeps all non-class owners correct. + """ + prev_in_class: int = ps.in_class_body + ps.in_class_body = owner_is_class + _skip_nl(ps) + if _cur_type(ps) == TokenType.NewLine: + _advance(ps) + _skip_nl(ps) + if _cur_type(ps) != TokenType.Indent: + ps.error_count += 1 + ps.in_class_body = prev_in_class + return + _advance(ps) + sidx: int = 0 + while _cur_type(ps) != TokenType.Dedent and _cur_type(ps) != TokenType.EndMarker: + sidx += 1 + # fail-fast anti-hang: over-budget steps (cyclic token list / cursor spinning) report and return + if ps.step_count > ps.step_limit: + _record_parse_err(ps) + return + # Decorator (class body methods can have decorators) + if _match_op(ps, TokOp.At): + ps.pending_decorators = _parse_decorators(ps) + before_cur: Token* = ps.cur + _parse_statement_into(ps, parent) + # Defensive: if parse did not advance token, force advance to avoid infinite loop + if ps.cur is before_cur: + ps.error_count += 1 + _advance(ps) + _skip_nl(ps) + _expect_type(ps, TokenType.Dedent) + ps.in_class_body = prev_in_class + return + # Single-line suite + _parse_simple_stmt_into(ps, parent) + ps.in_class_body = prev_in_class + + +def _parse_statement_into(ps: Parser*, parent: AST*): + """Parse statement (compound or simple), append to the parent.""" + ttype: int = _cur_type(ps) + kw: int = _cur_kw(ps) + if ttype == TokenType.Name: + # cif: compile-time conditional statement (soft keyword). Desugars to If(test=c.@CIf(EXPR)). + if _is_cif_stmt(ps) != 0: + cif_stmt: AST* = _parse_cif(ps) + if cif_stmt is not None: + parent.append(cif_stmt) + return + # celif/celse are valid only inside a cif chain; a bare occurrence is fail-fast. Skip the line. + if _is_named_soft(ps, "celif") != 0 or _is_named_soft(ps, "celse") != 0: + _RecordCifChainErr("celif/celse must be part of a single cif chain (they cannot stand alone, nor appear after the chain's celse)", + _cur_lineno(ps), _cur_col(ps)) + ps.error_count += 1 + while _cur_type(ps) != TokenType.NewLine and _cur_type(ps) != TokenType.EndMarker: + _advance(ps) + return + # define constant declaration: `define NAME[: T] = ` (no parentheses). Must run before + # _parse_simple_stmt_into, otherwise `define` is parsed as an identifier expression and NAME is left + # as a stray token. `define NAME(...)` (macro function) is excluded by _is_define_const's lookahead. + if kw == Keyword.Define and _is_define_const(ps) != 0: + dc_stmt: AST* = _parse_define_const(ps) + if dc_stmt is not None: + parent.append(dc_stmt) + return + # typedef alias declaration: `typedef T1 = T2` (statement-level reserved word, no lookahead since + # typedef is never an identifier in existing sources). Invalid forms fail loudly in _parse_typedef. + if _is_typedef(ps) != 0: + td_stmt: AST* = _parse_typedef(ps) + if td_stmt is not None: + parent.append(td_stmt) + return + # export / extern variable declaration (soft keyword): `export NAME: T [= v]` / `extern NAME: T` + # -> a module-level AnnAssign with storage-class markers; function forms are excluded by lookahead. + if _is_export_extern_var(ps) != 0: + ee_stmt: AST* = _parse_export_extern_var(ps) + if ee_stmt is not None: + parent.append(ee_stmt) + return + # virtual / novirtual modifier (soft keyword): only at statement start before class/def; synthesizes + # @t.CVTable / @t.NoVTable then proceeds with class/def parsing. Placed before the class/def dispatch. + if _IsVirtualModifier(ps) != 0: + _ParseVirtualModified(ps, parent) + return + # inline modifier (soft keyword): only at statement start before a function definition; synthesizes + # @t.always_inline then proceeds with def parsing (same pending_decorators channel as virtual). + if _IsInlineModifier(ps) != 0: + _ParseInlineModified(ps, parent) + return + # static modifier (soft keyword): statement start + static + (def | `Name :`). The function branch + # synthesizes @t.CStatic (returns); the variable branch flags pending_static for _parse_type_ann. + if _IsStaticModifier(ps) != 0: + if _ParseStaticModified(ps, parent) != 0: + return + # Bare descriptor use: the whole statement is only `const` / `volatile` (missing type) -> fail loudly. + # Matches only when followed by newline/EOF/semicolon (soft-keyword boundary), so `const = 5` / + # `t.attr.const()` do not match. After recording, continue the generic simple_stmt dispatch. + if _IsBareDescriptor(ps) != 0: + _RecordDescriptorErr("bare descriptor use (the whole statement is only a descriptor, missing type)", + _cur_lineno(ps), _cur_col(ps)) + ps.error_count += 1 + if kw == Keyword.If or kw == Keyword.While or kw == Keyword.For or kw == Keyword.Try: + stmt: AST* = _parse_compound(ps) + if stmt is not None: + parent.append(stmt) + return + if kw == Keyword.With or _is_funcdef_kw(ps, kw) != 0 or kw == Keyword.Class \ + or kw == Keyword.Enum or kw == Keyword.Union or kw == Keyword.Renum \ + or kw == Keyword.Struct: + stmt = _parse_compound(ps) + if stmt is not None: + parent.append(stmt) + return + if kw == Keyword.Match or kw == Keyword.Switch or kw == Keyword.Async: + stmt = _parse_compound(ps) + if stmt is not None: + parent.append(stmt) + return + _parse_simple_stmt_into(ps, parent) + + +def _parse_simple_stmt_into(ps: Parser*, parent: AST*): + """simple_stmt: small_stmt (';' small_stmt)* [';'] TokenType.NewLine""" + first: AST* = _parse_small_stmt(ps) + if first is not None: + parent.append(first) + while _accept_op(ps, TokOp.Semi): + if _cur_type(ps) == TokenType.NewLine or _cur_type(ps) == TokenType.EndMarker: + break + s: AST* = _parse_small_stmt(ps) + if s is None: break + parent.append(s) + # First skip Nl (in-paren newline, e.g. between multi-line call args; a legal token) + _skip_nl(ps) + # Statement-end check: simple_stmt must end with a newline/EOF. Stray tokens (e.g. adjacent + # expressions split into multiple statements, or a def missing its keyword) are a fail-fast syntax + # error, reported Python-style (SyntaxError at line/col). + if _cur_type(ps) != TokenType.NewLine and _cur_type(ps) != TokenType.EndMarker: + # Stray tokens (e.g. adjacent expressions split into statements, or a def missing its keyword) are a + # fail-fast syntax error. The message is no longer self-assembled: position/message go through the + # parse error channel with prev/next tokens appended, output Python-style by the caller's + # dump_parse_traceback (Traceback + File/line + source line + caret + SyntaxError). + # Locate: walk the list from the head to find cur, recording prev and next token context. + prev_tok: Token* = None + walk_tok: Token* = ps.tokens + while walk_tok is not None and walk_tok is not ps.cur: + prev_tok = walk_tok + walk_tok = walk_tok.next + pv_str: str = "(head)" + pv_line: int = 0 + pv_col: int = 0 + if prev_tok is not None: + pv_str = prev_tok.str_val + pv_line = prev_tok.lineno + pv_col = prev_tok.col_offset + nxt_tok: Token* = ps.cur.next + nxt_str: str = "(none)" + nxt_line: int = 0 + nxt_col: int = 0 + if nxt_tok is not None: + nxt_str = nxt_tok.str_val + nxt_line = nxt_tok.lineno + nxt_col = nxt_tok.col_offset + _record_parse_err_ctx(ps, _cur_type(ps), pv_str, pv_line, pv_col, + nxt_str, nxt_line, nxt_col) + ps.error_count += 1 + return + if _cur_type(ps) == TokenType.NewLine: + _advance(ps) + + +def _parse_small_stmt(ps: Parser*) -> AST*: + """small_stmt: various simple statements""" + kw: int = _cur_kw(ps) + # fallthrough keyword (hard): legal only inside a switch case body ("fall to the next case"). + # Legality is checked by the generator; writing it in a match or outside a switch is fail-fast. + if kw == Keyword.FallThrough: + ft_lineno: int = _cur_lineno(ps) + ft_col: int = _cur_col(ps) + _advance(ps) + ft_node: AST* = FallThrough(ps.pool) + _set_pos(ft_node, ft_lineno, ft_col, 0, 0) + return ft_node + if kw == Keyword.Pass: + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + _advance(ps) + node: AST* = Pass(ps.pool) + _set_pos(node, lineno, col, 0, 0) + return node + if kw == Keyword.Break: + lineno = _cur_lineno(ps) + col = _cur_col(ps) + _advance(ps) + node = Break(ps.pool) + _set_pos(node, lineno, col, 0, 0) + return node + if kw == Keyword.Continue: + lineno = _cur_lineno(ps) + col = _cur_col(ps) + _advance(ps) + node = Continue(ps.pool) + _set_pos(node, lineno, col, 0, 0) + return node + if kw == Keyword.Return: + return _parse_return(ps) + if kw == Keyword.Raise: + return _parse_raise(ps) + if kw == Keyword.Global: + return _parse_global(ps) + if kw == Keyword.Nonlocal: + return _parse_nonlocal(ps) + if kw == Keyword.Import: + return _parse_import(ps) + if kw == Keyword.From: + return _parse_from_import(ps) + if kw == Keyword.Del: + return _parse_del(ps) + if kw == Keyword.Assert: + return _parse_assert(ps) + if kw == Keyword.Yield: + return _parse_expr_stmt(ps, _parse_yield(ps)) + # Expression statement / assignment / augassign / annassign + return _parse_expr_stmt(ps, None) + + +def _maybe_tuple(ps: Parser*, first: AST*) -> AST*: + """If there is a comma after first, collect the comma-separated expressions into a Tuple.""" + if first is None: return None + if not _match_op(ps, TokOp.Comma): + return first + lineno: int = first.lineno + col: int = first.col_offset + elts: list[AST*]* = list[AST*](ps.pool, 8) + elts.append(first) + _accept_op(ps, TokOp.Comma) + while True: + tt: int = _cur_type(ps) + if tt == TokenType.NewLine or tt == TokenType.EndMarker: + break + if tt == TokenType.Op: + op: int = _cur_op(ps) + if op == TokOp.Equal or op == TokOp.Semi: + break + e: AST* = _parse_expr(ps) + if e is None: break + elts.append(e) + if not _accept_op(ps, TokOp.Comma): + break + node: AST* = Tuple(ps.pool, elts, ASTCtx.Load) + _set_pos(node, lineno, col, 0, 0) + return node + + +def _parse_testlist(ps: Parser*) -> AST*: + """Parse a comma-separated list of expressions, if there’s more than one, form a tuple.""" + first: AST* = _parse_expr(ps) + return _maybe_tuple(ps, first) + + +def _parse_expr_stmt(ps: Parser*, first: AST*) -> AST*: + """Expression statement, handle assignment/augassign/annassign""" + if first is None: + first = _parse_testlist(ps) + else: + first = _maybe_tuple(ps, first) + if first is None: + return None + lineno: int = first.lineno + col: int = first.col_offset + # Assignment + if _match_op(ps, TokOp.Equal): + targets: list[AST*]* = list[AST*](ps.pool, 8) + targets.append(first) + _fix_store_ctx(first) + value: AST* = None + accept_result: int = _accept_op(ps, TokOp.Equal) + while accept_result: + v: AST* = _parse_testlist(ps) + if v is None: break + if value is not None: + targets.append(value) + _fix_store_ctx(value) + value = v + accept_result = _accept_op(ps, TokOp.Equal) + node: AST* = Assign(ps.pool, targets, value) + _set_pos(node, lineno, col, 0, 0) + return node + # augassign + if _cur_type(ps) == TokenType.Op: + op_id: int = _cur_op(ps) + aug_op: int = _augop_from_op(op_id) + if aug_op != OpKind.NoneOp: + _advance(ps) + value2: AST* = _parse_testlist(ps) + _fix_store_ctx(first) + node2: AST* = AugAssign(ps.pool, first, aug_op, value2) + _set_pos(node2, lineno, col, 0, 0) + return node2 + # annassign: target ':' annotation [= value] + if _match_op(ps, TokOp.Colon): + _advance(ps) + annotation: AST* = _parse_type_ann(ps) + value3: AST* = None + if _accept_op(ps, TokOp.Equal): + # A typedef's right-hand side is a **type** (`const str` / `uint16*`), not a general + # expression: parse it with the annotation rule so descriptors and C-style pointer + # suffixes behave exactly as in annotation position (same as TPC's vast). + if _IsCTypedefAnnotation(annotation) != 0: + value3 = _parse_type_ann(ps) + else: + value3 = _parse_testlist(ps) + if ps.pool is None: + return None + _fix_store_ctx(first) + if ps.pool is None: + return None + node3: AST* = AnnAssign(ps.pool, first, annotation, value3, 1) + _set_pos(node3, lineno, col, 0, 0) + return node3 + # Expression statement + node4: AST* = Expr(ps.pool, first) + _set_pos(node4, lineno, col, 0, 0) + return node4 + + +def _parse_return(ps: Parser*) -> AST*: + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + _advance(ps) + val: AST* = None + if _cur_type(ps) != TokenType.NewLine and _cur_type(ps) != TokenType.EndMarker and _cur_type(ps) != TokenType.Op: + val = _parse_testlist(ps) + elif _cur_type(ps) == TokenType.Op and _cur_op(ps) != TokOp.Semi: + val = _parse_testlist(ps) + node: AST* = Return(ps.pool, val) + _set_pos(node, lineno, col, 0, 0) + return node + + +def _parse_raise(ps: Parser*) -> AST*: + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + _advance(ps) + exc: AST* = None + cause: AST* = None + if _cur_type(ps) != TokenType.NewLine and _cur_type(ps) != TokenType.EndMarker: + exc = _parse_test(ps) + if _cur_kw(ps) == Keyword.From: + _advance(ps) + cause = _parse_test(ps) + node: AST* = Raise(ps.pool, exc, cause) + _set_pos(node, lineno, col, 0, 0) + return node + + +def _parse_global(ps: Parser*) -> AST*: + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + _advance(ps) + names: list[AST*]* = list[AST*](ps.pool, 8) + if _cur_type(ps) == TokenType.Name: + n: AST* = Name(ps.pool, _cur_str(ps), ASTCtx.Store) + _set_pos(n, _cur_lineno(ps), _cur_col(ps), 0, 0) + names.append(n) + _advance(ps) + while _accept_op(ps, TokOp.Comma): + if _cur_type(ps) != TokenType.Name: break + n = Name(ps.pool, _cur_str(ps), ASTCtx.Store) + _set_pos(n, _cur_lineno(ps), _cur_col(ps), 0, 0) + names.append(n) + _advance(ps) + node: AST* = Global(ps.pool, names) + _set_pos(node, lineno, col, 0, 0) + return node + + +def _parse_nonlocal(ps: Parser*) -> AST*: + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + _advance(ps) + names: list[AST*]* = list[AST*](ps.pool, 8) + if _cur_type(ps) == TokenType.Name: + n: AST* = Name(ps.pool, _cur_str(ps), ASTCtx.Store) + _set_pos(n, _cur_lineno(ps), _cur_col(ps), 0, 0) + names.append(n) + _advance(ps) + while _accept_op(ps, TokOp.Comma): + if _cur_type(ps) != TokenType.Name: break + n = Name(ps.pool, _cur_str(ps), ASTCtx.Store) + _set_pos(n, _cur_lineno(ps), _cur_col(ps), 0, 0) + names.append(n) + _advance(ps) + node: AST* = Nonlocal(ps.pool, names) + _set_pos(node, lineno, col, 0, 0) + return node + + +def _parse_import(ps: Parser*) -> AST*: + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + _advance(ps) + names: list[AST*]* = _parse_alias_list(ps) + node: AST* = Import(ps.pool, names) + _set_pos(node, lineno, col, 0, 0) + return node + + +def _parse_from_import(ps: Parser*) -> AST*: + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + _advance(ps) + # Parse module (possibly with dots) + level: int = 0 + while _match_op(ps, TokOp.Dot): + level += 1 + _advance(ps) + while _match_op(ps, TokOp.Ellipsis): + level += 3 + _advance(ps) + module_name: str = None + # The token type for keywords (like import) is also Name, so they must be excluded using kw_subtype, + # otherwise the `import` in `from . import` would be mistakenly treated as a module_name. + # Note: only `import` itself should be excluded. Module names can be keywords (like in `from .match import ...`, + # where match is a Viper match statement keyword), requiring NotKeyword would cause + # `from .match import` to fail parsing and the dependency scan would miss that module. + if _cur_type(ps) == TokenType.Name and _cur_kw(ps) != Keyword.Import: + module_name = _cur_str(ps) + _advance(ps) + while _match_op(ps, TokOp.Dot): + _advance(ps) + if _cur_type(ps) == TokenType.Name and _cur_kw(ps) == Keyword.NotKeyword: + next_name: str = _cur_str(ps) + mlen: int = int(string.strlen(module_name)) # 显式 cast：避免 64→32 隐式窄化 + nlen: int = int(string.strlen(next_name)) # 显式 cast：避免 64→32 隐式窄化 + new_mod: str = ps.pool.alloc(mlen + 1 + nlen + 1) + if new_mod is not None: + string.strcpy(new_mod, module_name) + new_mod[mlen] = '.' + string.strcpy(new_mod + mlen + 1, next_name) + module_name = new_mod + _advance(ps) + if _cur_kw(ps) != Keyword.Import: + ps.error_count += 1 + return None + _advance(ps) + _skip_nl(ps) + # import * + if _match_op(ps, TokOp.Star): + _advance(ps) + star_alias: AST* = Alias(ps.pool, "*", None) + _set_pos(star_alias, lineno, col, 0, 0) + star_names: list[AST*]* = list[AST*](ps.pool, 8) + star_names.append(star_alias) + node: AST* = ImportFrom(ps.pool, module_name, star_names, level) + _set_pos(node, lineno, col, 0, 0) + return node + # ( names ) or names + if _match_op(ps, TokOp.LPar): + _advance(ps) + _skip_nl(ps) + names: list[AST*]* = _parse_alias_list(ps) + _skip_nl(ps) + _expect_op(ps, TokOp.RPar) + else: + names = _parse_alias_list(ps) + node: AST* = ImportFrom(ps.pool, module_name, names, level) + _set_pos(node, lineno, col, 0, 0) + return node + + +def _parse_alias_list(ps: Parser*) -> list[AST*]*: + """Parse alias list.""" + names: list[AST*]* = list[AST*](ps.pool, 8) + while True: + if _cur_type(ps) != TokenType.Name: + break + name_str: str = _cur_str(ps) + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + _advance(ps) + # dotted name + while _match_op(ps, TokOp.Dot): + _advance(ps) + if _cur_type(ps) == TokenType.Name: + next_n: str = _cur_str(ps) + nmlen: int = int(string.strlen(name_str)) # 显式 cast：避免 64→32 隐式窄化 + nxtlen: int = int(string.strlen(next_n)) # 显式 cast：避免 64→32 隐式窄化 + new_name: str = ps.pool.alloc(nmlen + 1 + nxtlen + 1) + if new_name is not None: + string.strcpy(new_name, name_str) + new_name[nmlen] = '.' + string.strcpy(new_name + nmlen + 1, next_n) + name_str = new_name + _advance(ps) + asname: str = None + if _cur_kw(ps) == Keyword.As: + _advance(ps) + if _cur_type(ps) == TokenType.Name: + asname = _cur_str(ps) + _advance(ps) + a: AST* = Alias(ps.pool, name_str, asname) + _set_pos(a, lineno, col, 0, 0) + names.append(a) + if not _accept_op(ps, TokOp.Comma): + break + _skip_nl(ps) + return names + + +def _parse_del(ps: Parser*) -> AST*: + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + _advance(ps) + targets: list[AST*]* = list[AST*](ps.pool, 8) + while True: + t_: AST* = _parse_atom_expr(ps) + if t_ is None: break + _set_ctx(t_, ASTCtx.Del) + targets.append(t_) + if not _accept_op(ps, TokOp.Comma): + break + node: AST* = Delete(ps.pool, targets) + _set_pos(node, lineno, col, 0, 0) + return node + + +def _parse_assert(ps: Parser*) -> AST*: + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + _advance(ps) + test: AST* = _parse_test(ps) + msg: AST* = None + if _accept_op(ps, TokOp.Comma): + msg = _parse_test(ps) + node: AST* = Assert(ps.pool, test, msg) + _set_pos(node, lineno, col, 0, 0) + return node + + +# ============================================================ +# Compound statements +# ============================================================ + +def _parse_compound(ps: Parser*) -> AST*: + kw: int = _cur_kw(ps) + if kw == Keyword.If: + return _parse_if(ps) + if kw == Keyword.While: + return _parse_while(ps) + if kw == Keyword.For: + return _parse_for(ps, 0) + if kw == Keyword.Try: + return _parse_try(ps) + if kw == Keyword.With: + return _parse_with(ps, 0) + if _is_funcdef_kw(ps, kw) != 0: + return _parse_funcdef(ps, 0) + if kw == Keyword.Class: + return _parse_classdef(ps) + if kw == Keyword.Enum: + return _parse_enumdef(ps) + if kw == Keyword.Union: + return _parse_uniondef(ps) + if kw == Keyword.Renum: + return _parse_renumdef(ps) + if kw == Keyword.Struct: + return _parse_structdef(ps) + if kw == Keyword.Async: + _advance(ps) + if _is_funcdef_kw(ps, _cur_kw(ps)) != 0: + return _parse_funcdef(ps, 1) + if _cur_kw(ps) == Keyword.For: + return _parse_for(ps, 1) + if _cur_kw(ps) == Keyword.With: + return _parse_with(ps, 1) + ps.error_count += 1 + return None + if kw == Keyword.Match: + return _parse_match(ps) + if kw == Keyword.Switch: + return _parse_switch(ps) + ps.error_count += 1 + return None + + +# ============================================================ +# cif / celif / celse compile-time conditional statements +# ============================================================ +# `cif EXPR:` is a soft-keyword statement desugared at parse time into +# If(test=c.@CIf(EXPR), body, orelse). `celif`/`celse` extend the same chain and are recognized only +# inside _parse_cif_tail. The marker attribute name '@CIf' is internal-only spelling (a leading '@' +# cannot be produced by the lexer), so a user-written c.CIf(...) never matches it and is rejected. + + +def _MakeCifMarker(ps: Parser*, test: AST*, lineno: int, col: int) -> AST*: + """Build the internal marker call c.@CIf(test) that wraps a cif/celif condition.""" + attr: AST* = Attribute(ps.pool, Name(ps.pool, "c", ASTCtx.Load), "@CIf", ASTCtx.Load) + args: list[AST*]* = list[AST*](ps.pool, 4) + if test is not None: args.append(test) + kw: list[AST*]* = list[AST*](ps.pool, 1) + marker: AST* = Call(ps.pool, attr, args, kw) + _set_pos(marker, lineno, col, 0, 0) + return marker + + +def _RecordCifChainErr(detail: str, lineno: int, col: int): + """Record an invalid cif/celif/celse chain (fail-fast; first only). Message states the fix. + + A single cif chain is cif ... celif ... celse: only `cif` opens a chain, and the chain must be + terminated by `celse`; a plain else/elif cannot close it and celif/celse cannot stand alone. + """ + global _g_parse_err_line, _g_parse_err_col, _g_parse_err_msg + if _g_parse_err_line != 0: + return + _g_parse_err_line = lineno + _g_parse_err_col = col + buf: str = stdlib.malloc(1024) + if buf is not None: + stdio.snprintf(buf, 1024, + "invalid cif chain (%s). cif/celif/celse form a single compile-time conditional chain:" + " only `cif` opens a chain, `celif` extends the current cif chain, and the chain must be" + " terminated by `celse` (a plain else/elif cannot close a cif chain, and celif/celse cannot" + " stand alone). Correct form: cif : ... celif : ... celse: ...", detail) + _g_parse_err_msg = buf + else: + _g_parse_err_msg = detail + + +def _parse_cif(ps: Parser*) -> AST*: + """Parse a `cif EXPR: ...` statement and its celif/celse chain (desugars to nested If nodes).""" + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + _advance(ps) # cif + test: AST* = _parse_test(ps) + _expect_op(ps, TokOp.Colon) + empty: list[AST*]* = list[AST*](ps.pool, 8) + node: If* = If(ps.pool, _MakeCifMarker(ps, test, lineno, col), empty) + _parse_suite_into_children(ps, node) + node.orelse = _parse_cif_tail(ps) + _set_pos(node, lineno, col, 0, 0) + return node + + +def _parse_cif_tail(ps: Parser*) -> list[AST*]*: + """Parse the celif/celse continuation of a cif chain and return the orelse list. + + celif: orelse = [nested If(test=c.@CIf(EXPR), ...)] (same chain, recurses). + celse: orelse = statement list (a plain else; no marker). + A plain elif/else after a cif chain is fail-fast (a cif chain must end with celse). + """ + if _is_named_soft(ps, "celif") != 0: + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + _advance(ps) # celif + test: AST* = _parse_test(ps) + _expect_op(ps, TokOp.Colon) + empty: list[AST*]* = list[AST*](ps.pool, 8) + elif_node: If* = If(ps.pool, _MakeCifMarker(ps, test, lineno, col), empty) + _parse_suite_into_children(ps, elif_node) + elif_node.orelse = _parse_cif_tail(ps) + _set_pos(elif_node, lineno, col, 0, 0) + orelse: list[AST*]* = list[AST*](ps.pool, 8) + orelse.append(elif_node) + return orelse + if _is_named_soft(ps, "celse") != 0: + _advance(ps) # celse + _expect_op(ps, TokOp.Colon) + scratch: Module* = Module(ps.pool) + _parse_suite_into_children(ps, scratch) + return scratch.children + # A plain elif/else cannot close a cif chain; consume and discard its clause to keep parsing. + if _cur_kw(ps) == Keyword.Elif or _cur_kw(ps) == Keyword.Else: + _RecordCifChainErr("a plain elif/else cannot follow a cif/celif branch; the cif chain must be terminated by celse", + _cur_lineno(ps), _cur_col(ps)) + ps.error_count += 1 + is_elif: int = 0 + if _cur_kw(ps) == Keyword.Elif: is_elif = 1 + _advance(ps) # elif / else + if is_elif != 0: _parse_test(ps) + _expect_op(ps, TokOp.Colon) + scratch2: Module* = Module(ps.pool) + _parse_suite_into_children(ps, scratch2) + return list[AST*](ps.pool, 8) + return list[AST*](ps.pool, 8) + + +def _parse_if(ps: Parser*) -> AST*: + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + _advance(ps) # if + test: AST* = _parse_test(ps) + _expect_op(ps, TokOp.Colon) + # First construct with empty orelse, then fill it in later + empty: list[AST*]* = list[AST*](ps.pool, 8) + node: If* = If(ps.pool, test, empty) + _parse_suite_into_children(ps, node) + orelse: list[AST*]* = _parse_if_tail(ps) + node.orelse = orelse + _set_pos(node, lineno, col, 0, 0) + return node + + +def _parse_if_tail(ps: Parser*) -> list[AST*]*: + """Parse elif/else chain and return orelse list. + + elif: orelse = list containing single If node (nested) + else: orelse = statement list + """ + # celif/celse belong to a cif chain only; after a plain if they are fail-fast (and discarded). + if _is_named_soft(ps, "celif") != 0 or _is_named_soft(ps, "celse") != 0: + _RecordCifChainErr("celif/celse follow a plain if statement (not a cif chain); a compile-time chain must start with cif", + _cur_lineno(ps), _cur_col(ps)) + ps.error_count += 1 + is_celif: int = _is_named_soft(ps, "celif") + _advance(ps) # celif / celse + if is_celif != 0: _parse_test(ps) + _expect_op(ps, TokOp.Colon) + scratch_ce: Module* = Module(ps.pool) + _parse_suite_into_children(ps, scratch_ce) + return list[AST*](ps.pool, 8) + if _cur_kw(ps) == Keyword.Elif: + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + _advance(ps) + elif_test: AST* = _parse_test(ps) + _expect_op(ps, TokOp.Colon) + empty: list[AST*]* = list[AST*](ps.pool, 8) + elif_node: If* = If(ps.pool, elif_test, empty) + _parse_suite_into_children(ps, elif_node) + elif_orelse: list[AST*]* = _parse_if_tail(ps) + elif_node.orelse = elif_orelse + _set_pos(elif_node, lineno, col, 0, 0) + orelse: list[AST*]* = list[AST*](ps.pool, 8) + orelse.append(elif_node) + return orelse + if _cur_kw(ps) == Keyword.Else: + _advance(ps) + _expect_op(ps, TokOp.Colon) + scratch: Module* = Module(ps.pool) + _parse_suite_into_children(ps, scratch) + return scratch.children + return list[AST*](ps.pool, 8) # empty list + + +# ============================================================ +# Parsing Compound Statements (Continued) +# ============================================================ + +def _parse_while(ps: Parser*) -> AST*: + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + _advance(ps) # while + test: AST* = _parse_test(ps) + _expect_op(ps, TokOp.Colon) + empty: list[AST*]* = list[AST*](ps.pool, 8) + node: While* = While(ps.pool, test, empty) + _parse_suite_into_children(ps, node) + if _cur_kw(ps) == Keyword.Else: + _advance(ps) + _expect_op(ps, TokOp.Colon) + scratch: Module* = Module(ps.pool) + _parse_suite_into_children(ps, scratch) + node.orelse = scratch.children + _set_pos(node, lineno, col, 0, 0) + return node + + +def _parse_for(ps: Parser*, is_async: int) -> AST*: + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + _advance(ps) # for + target: AST* = _parse_target_list(ps) + if _cur_kw(ps) != Keyword.In: + ps.error_count += 1 + return None + _advance(ps) + iter_: AST* = _parse_or_test(ps) + _expect_op(ps, TokOp.Colon) + empty: list[AST*]* = list[AST*](ps.pool, 8) + node: For* = For(ps.pool, target, iter_, empty, is_async) + _fix_store_ctx(target) + _parse_suite_into_children(ps, node) + if _cur_kw(ps) == Keyword.Else: + _advance(ps) + _expect_op(ps, TokOp.Colon) + scratch: Module* = Module(ps.pool) + _parse_suite_into_children(ps, scratch) + node.orelse = scratch.children + _set_pos(node, lineno, col, 0, 0) + return node + + +def _parse_try(ps: Parser*) -> AST*: + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + _advance(ps) # try + _expect_op(ps, TokOp.Colon) + empty: list[AST*]* = list[AST*](ps.pool, 8) + node: Try* = Try(ps.pool, empty, empty, empty) + _parse_suite_into_children(ps, node) + # handlers (except) + handlers: list[AST*]* = list[AST*](ps.pool, 8) + while _cur_kw(ps) == Keyword.Except: + h: AST* = _parse_except_handler(ps) + if h is not None: + handlers.append(h) + node.handlers = handlers + # else + if _cur_kw(ps) == Keyword.Else: + _advance(ps) + _expect_op(ps, TokOp.Colon) + scratch: Module* = Module(ps.pool) + _parse_suite_into_children(ps, scratch) + node.orelse = scratch.children + # finally + if _cur_kw(ps) == Keyword.Finally: + _advance(ps) + _expect_op(ps, TokOp.Colon) + scratch2: Module* = Module(ps.pool) + _parse_suite_into_children(ps, scratch2) + node.finalbody = scratch2.children + _set_pos(node, lineno, col, 0, 0) + return node + + +def _parse_except_handler(ps: Parser*) -> AST*: + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + _advance(ps) # except + type_: AST* = None + name: str = None + if _cur_op(ps) != TokOp.Colon: + type_ = _parse_test(ps) + if _cur_kw(ps) == Keyword.As: + _advance(ps) + if _cur_type(ps) == TokenType.Name: + name = _cur_str(ps) + _advance(ps) + _expect_op(ps, TokOp.Colon) + node: AST* = ExceptHandler(ps.pool, type_, name) + _parse_suite_into_children(ps, node) + _set_pos(node, lineno, col, 0, 0) + return node + + +def _parse_with(ps: Parser*, is_async: int) -> AST*: + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + _advance(ps) # with + items: list[AST*]* = list[AST*](ps.pool, 8) + # items + while True: + item: AST* = _parse_with_item(ps) + if item is None: break + items.append(item) + if not _accept_op(ps, TokOp.Comma): + break + # Check if we can continue with a newline (with (...): form) + if _cur_type(ps) == TokenType.NewLine or _cur_type(ps) == TokenType.Nl: + break + _expect_op(ps, TokOp.Colon) + node: With* = With(ps.pool, items, is_async) + _parse_suite_into_children(ps, node) + _set_pos(node, lineno, col, 0, 0) + return node + + +def _parse_with_item(ps: Parser*) -> AST*: + """with_item: test ['as' expr]""" + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + context_expr: AST* = _parse_or_test(ps) + if context_expr is None: return None + optional_vars: AST* = None + if _cur_kw(ps) == Keyword.As: + _advance(ps) + optional_vars = _parse_target(ps) + _fix_store_ctx(optional_vars) + node: AST* = WithItem(ps.pool, context_expr, optional_vars) + _set_pos(node, lineno, col, 0, 0) + return node + + +def _parse_funcdef(ps: Parser*, is_async: int) -> AST*: + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + is_extern_fd: int = 0 + is_export_fd: int = 0 + is_define_fd: int = 0 + if _cur_kw(ps) == Keyword.Extern: + # extern declaration: extern [def] A(...) [-> T] [optional ": pass"] + is_extern_fd = 1 + _advance(ps) # extern + if _cur_kw(ps) == Keyword.Def: + _advance(ps) + elif _cur_kw(ps) == Keyword.Export: + # export declaration: export [def] A(...) [-> T] [optional ": pass"] + is_export_fd = 1 + _advance(ps) # export + if _cur_kw(ps) == Keyword.Def: + _advance(ps) + elif _cur_kw(ps) == Keyword.Define: + # define macro function: syntax identical to def; t.CDefine is appended to the return annotation + # (StubGen keeps it as-is -> C #define / alwaysinline) + is_define_fd = 1 + _advance(ps) # define + else: + _advance(ps) # def / fn / ... + if _cur_type(ps) != TokenType.Name: + ps.error_count += 1 + return None + name: str = _cur_str(ps) + _advance(ps) + # PEP 695 generic function type parameters: def add[T](a: T, b: T) -- parsed and stored into + # type_params, consistent with the class parser. type_params maps to int as a fallback during translation. + type_params_fd: list[str]* = None + if _accept_op(ps, TokOp.Lsqb): + type_params_fd = list[str](ps.pool, 8) + while not _match_op(ps, TokOp.Rsqb): + if _cur_type(ps) == TokenType.EndMarker: + break + if _cur_type(ps) == TokenType.Name: + tp_name_fd: str = _cur_str(ps) + _advance(ps) + # Skip the optional bound/constraint: [T: bound] or [T: (A, B)] + if _match_op(ps, TokOp.Colon): + _advance(ps) + _parse_test(ps) + type_params_fd.append(tp_name_fd) + if not _accept_op(ps, TokOp.Comma): + break + _expect_op(ps, TokOp.Rsqb) + args: AST* = None + if _accept_op(ps, TokOp.LPar): + args = _parse_varargslist(ps, 0) + _expect_op(ps, TokOp.RPar) + returns: AST* = None + if _accept_op(ps, TokOp.RArrow): + returns = _parse_type_ann(ps) + if is_extern_fd: + # extern declaration: rewrite the return annotation to | t.State, reusing the existing + # State/external-C-function path (vpi annotation containing "State" -> bare name, no function body) + state_attr_fd: AST* = Attribute(ps.pool, + Name(ps.pool, "t", ASTCtx.Load), + "State", ASTCtx.Load) + if returns != None: + returns = BinOp(ps.pool, returns, OpKind.BitOr, state_attr_fd) + else: + returns = state_attr_fd + elif is_export_fd: + # export declaration: rewrite the return annotation to | t.CExport, reusing the existing + # CExport path (vpi annotation containing "CExport" -> bare name, no module SHA1 header) + export_attr_fd: AST* = Attribute(ps.pool, + Name(ps.pool, "t", ASTCtx.Load), + "CExport", ASTCtx.Load) + if returns != None: + returns = BinOp(ps.pool, returns, OpKind.BitOr, export_attr_fd) + else: + returns = export_attr_fd + elif is_define_fd: + # define macro function: rewrite the return annotation to | t.CDefine, reusing the + # existing CDefine path (StubGen keeps it as-is -> C #define / alwaysinline) + define_attr_fd: AST* = Attribute(ps.pool, + Name(ps.pool, "t", ASTCtx.Load), + "CDefine", ASTCtx.Load) + if returns != None: + returns = BinOp(ps.pool, returns, OpKind.BitOr, define_attr_fd) + else: + returns = define_attr_fd + dec_list: list[AST*]* = ps.pending_decorators + ps.pending_decorators = None + node: FunctionDef* = FunctionDef(ps.pool, name, args, dec_list, returns, is_async, type_params_fd) + if is_extern_fd: + node.flags = node.flags | FLAG_IS_EXTERN + if is_export_fd: + node.flags = node.flags | FLAG_IS_EXPORT + if is_define_fd: + node.flags = node.flags | FLAG_IS_DEFINE + if (is_extern_fd == 0 and is_export_fd == 0 and is_define_fd == 0) or _match_op(ps, TokOp.Colon): + _expect_op(ps, TokOp.Colon) + _parse_suite_into_children(ps, node) + else: + # extern/export without ': pass' (extern A(...) -> int): empty body, declaration only. The NewLine + # must be consumed here: the module/class statement loop only skips Nl, so a leftover NewLine would be + # parsed as an expression next round and fail parse_tokens. + if _cur_type(ps) == TokenType.NewLine: + _advance(ps) + _set_pos(node, lineno, col, 0, 0) + return node + + +def _parse_classdef(ps: Parser*) -> AST*: + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + _advance(ps) # class + if _cur_type(ps) != TokenType.Name: + ps.error_count += 1 + return None + name: str = _cur_str(ps) + _advance(ps) + # Parse type parameters [T] or [T, U] (PEP 695 generic class syntax) + type_params: list[str]* = None + if _accept_op(ps, TokOp.Lsqb): + type_params = list[str](ps.pool, 8) + while not _match_op(ps, TokOp.Rsqb): + if _cur_type(ps) == TokenType.EndMarker: break + if _cur_type(ps) == TokenType.Name: + tp_name: str = _cur_str(ps) + _advance(ps) + # Skip optional bound/constraint: [T: bound] or [T: (A, B)] + if _match_op(ps, TokOp.Colon): + _advance(ps) + _parse_test(ps) + type_params.append(tp_name) + if not _accept_op(ps, TokOp.Comma): + break + _expect_op(ps, TokOp.Rsqb) + bases: list[AST*]* = list[AST*](ps.pool, 8) + keywords: list[AST*]* = list[AST*](ps.pool, 8) + # bases / keywords + if _accept_op(ps, TokOp.LPar): + while not _match_op(ps, TokOp.RPar): + # Cross-line base list inside parens (class X(A,\n B):): skip the newline after a comma, + # otherwise cur=NL is not Name/Comma -> break -> _expect_op(RPar) reports an unexpected token. + _skip_nl(ps) + if _cur_type(ps) == TokenType.EndMarker: break + # base or keyword=value + if _cur_type(ps) == TokenType.Name: + # Parse as expression, if followed by '=', then as keyword=value + e: AST* = _parse_or_test(ps) + if _match_op(ps, TokOp.Equal): + _advance(ps) + v: AST* = _parse_test(ps) + kw_node: AST* = KwNode(ps.pool, e, v) + keywords.append(kw_node) + else: + bases.append(e) + if not _accept_op(ps, TokOp.Comma): + break + _skip_nl(ps) # finish the newline after the last base (class X(A,\n B):) + _expect_op(ps, TokOp.RPar) + _expect_op(ps, TokOp.Colon) + dec_list: list[AST*]* = ps.pending_decorators + ps.pending_decorators = None + node: ClassDef* = ClassDef(ps.pool, name, bases, keywords, dec_list, type_params) + # owner_is_class=1: ps.in_class_body=1 during the class body, so virtual/novirtual modifiers can tell + # whether a method-level modifier is directly in the class body (see _ParseVirtualModified). + _parse_suite_into_children(ps, node, 1) + _set_pos(node, lineno, col, 0, 0) + return node + + +# ============================================================ +# enum sugar (fully equivalent to `class X(t.CEnum):`): `enum Name:` header + member lines +# (`A` / `A: T` / `A = 0`). The synthesized ClassDef is field-identical to _parse_classdef output, so +# downstream (ClassGenEnum._translate_enum_def -> EnumTable) has a single implementation path. +# ============================================================ + +def _enum_member_annassign(ps: Parser*, owner: AST*, + name_node: AST*, + lineno: int, col: int, + end_lineno: int, end_col: int) -> AST*: + """Rewrite a single member-name node into a `Name: t.State` AnnAssign (the uniform enum member form). + + target=Name(id, Store), annotation=Attribute(Name('t'), 'State'), value=None, simple=1 -- + field-identical to a hand-written `class X(t.CEnum): A: t.State`. + """ + ann: AST* = Attribute(ps.pool, + Name(ps.pool, "t", ASTCtx.Load), + "State", ASTCtx.Load) + _set_pos(ann, lineno, col, end_lineno, end_col) + aa: AST* = AnnAssign(ps.pool, name_node, ann, None, 1) + _fix_store_ctx(name_node) + _set_pos(aa, lineno, col, end_lineno, end_col) + aa.parent = owner # maintain the parent pointer like AST.append does + return aa + + +def _convert_enum_bare_members(ps: Parser*, node: ClassDef*): + """Rewrite bare-name / comma-grouped member lines in an enum body into `Name: t.State` AnnAssigns. + + Member collection only recognizes Assign / AnnAssign Name targets, so bare names and comma groups + (`A, B, C`) were silently dropped; now `A` -> `A: t.State` and `A, B, C` -> three AnnAssigns. Valued + or annotated groups are not rewritten here and are caught by the registration-site fail-fast. + """ + children: list[AST*]* = node.children + if children is None: + return + cn: sizet = len(children) + # Rebuild the child list: one comma-group line -> several AnnAssigns. list has no insert, so rebuild + # entirely; the assignment form matches AST.append in base.vp. + node.children = list[AST*](ps.pool, 8) + out: list[AST*]* = node.children + for i in range(int(cn)): + stmt: AST* = children.get(i) + handled: int = 0 + if stmt is not None and stmt.kind() == ASTKind.Expr: + ex: Expr* = (Expr*)(stmt) + if ex.value is not None and ex.value.kind() == ASTKind.Name: + out.append(_enum_member_annassign(ps, node, ex.value, + ex.lineno, ex.col_offset, + ex.end_lineno, ex.end_col_offset)) + handled = 1 + elif ex.value is not None and ex.value.kind() == ASTKind.Tuple: + tup: Tuple* = (Tuple*)(ex.value) + if tup.elts is not None and len(tup.elts) >= 2: + all_names: int = 1 + k: sizet = 0 + while k < len(tup.elts): + el: AST* = tup.elts.get(k) + if el is None or el.kind() != ASTKind.Name: + all_names = 0 + break + k += 1 + if all_names == 1: + for k in range(int(len(tup.elts))): + el2: AST* = tup.elts.get(k) + out.append(_enum_member_annassign(ps, node, el2, + el2.lineno, el2.col_offset, + el2.end_lineno, el2.end_col_offset)) + handled = 1 + if handled == 0 and stmt is not None: + out.append(stmt) + + +def _parse_define_const(ps: Parser*) -> AST*: + """`define NAME[: T] = ` -> an AnnAssign field-identical to `NAME: [T |] t.CDefine = `. + + Same CDefine registration path as hand-written `NAME: t.CDefine = value`. With a type, T goes before + the union (`T | t.CDefine`), matching TPC's expansion. Must end with newline/EOF (fail-fast otherwise). + """ + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + _advance(ps) # define + if _cur_type(ps) != TokenType.Name: + ps.error_count += 1 + _record_parse_err(ps) + return None + name: str = _cur_str(ps) + _advance(ps) + explicit_ann: AST* = None + if _accept_op(ps, TokOp.Colon): + explicit_ann = _parse_type_ann(ps) + if explicit_ann is None: + return None + if _expect_op(ps, TokOp.Equal) == 0: + return None + value: AST* = _parse_test(ps) + if value is None: + ps.error_count += 1 + _record_parse_err(ps) + return None + cdefine_marker: AST* = Attribute(ps.pool, + Name(ps.pool, "t", ASTCtx.Load), + "CDefine", ASTCtx.Load) + annotation: AST* = cdefine_marker + if explicit_ann is not None: + annotation = BinOp(ps.pool, explicit_ann, OpKind.BitOr, cdefine_marker) + target: AST* = Name(ps.pool, name, ASTCtx.Store) + _set_pos(target, lineno, col, 0, 0) + node: AST* = AnnAssign(ps.pool, target, annotation, value, 1) + _set_pos(node, lineno, col, 0, 0) + _skip_nl(ps) + if _cur_type(ps) != TokenType.NewLine and _cur_type(ps) != TokenType.EndMarker: + _record_parse_err(ps) + ps.error_count += 1 + return None + if _cur_type(ps) == TokenType.NewLine: + _advance(ps) + return node + + +def _parse_typedef(ps: Parser*) -> AST*: + """`typedef T1 = T2` -> an AnnAssign field-identical to `T1: t.CTypedef = T2`. + + Same typedef registration/lookup path as the hand-written form (CTypedef branch -> register_ctypedef); + no second implementation. Invalid forms (missing name / '=' / RHS) fail loudly; must end with newline/EOF. + """ + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + _advance(ps) # typedef + if _cur_type(ps) != TokenType.Name: + # Missing alias (e.g. `typedef = 1` / `typedef : x`): fail loudly. + _record_parse_err(ps) + ps.error_count += 1 + return None + name: str = _cur_str(ps) + _advance(ps) + if _expect_op(ps, TokOp.Equal) == 0: + # Missing '=' (e.g. `typedef A`): fail loudly. + return None + # The sugar's RHS is the same type position as `T1: t.CTypedef = T2`: parse it with the + # annotation rule so `typedef X = const str` / `= uint16*` match the hand-written form. + value: AST* = _parse_type_ann(ps) + if value is None: + # Missing RHS (e.g. `typedef A = `): fail loudly. + ps.error_count += 1 + _record_parse_err(ps) + return None + ctypedef_marker: AST* = Attribute(ps.pool, + Name(ps.pool, "t", ASTCtx.Load), + "CTypedef", ASTCtx.Load) + target: AST* = Name(ps.pool, name, ASTCtx.Store) + _set_pos(target, lineno, col, 0, 0) + node: AST* = AnnAssign(ps.pool, target, ctypedef_marker, value, 1) + _set_pos(node, lineno, col, 0, 0) + _skip_nl(ps) + if _cur_type(ps) != TokenType.NewLine and _cur_type(ps) != TokenType.EndMarker: + _record_parse_err(ps) + ps.error_count += 1 + return None + if _cur_type(ps) == TokenType.NewLine: + _advance(ps) + return node + + +def _parse_export_extern_var(ps: Parser*) -> AST*: + """`export NAME: T [= v]` / `extern NAME: T` -> a module-level AnnAssign with storage-class markers. + + export rewrites the annotation to `T | t.CExport` (same marker/path as `export def`: bare name, no + module SHA1 header) and allows an initializer; extern rewrites to `T | t.State` (declaration + bare + name) and forbids an initializer (`extern a: int = 1` fails loudly). Must end with newline/EOF. + """ + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + is_extern: int = 0 + if _cur_kw(ps) == Keyword.Extern: + is_extern = 1 + _advance(ps) # export / extern + if _cur_type(ps) != TokenType.Name: + # Missing variable name (e.g. `export : int`): fail loudly. + _record_parse_err(ps) + ps.error_count += 1 + return None + name: str = _cur_str(ps) + _advance(ps) + if _expect_op(ps, TokOp.Colon) == 0: + # Missing ':' (e.g. `export x`): fail loudly. + return None + ann: AST* = _parse_type_ann(ps) + if ann is None: + ps.error_count += 1 + _record_parse_err(ps) + return None + # Storage-class marker: export -> t.CExport (bare-name export); extern -> t.State (declaration + bare name) + storage_attr: AST* = Attribute(ps.pool, + Name(ps.pool, "t", ASTCtx.Load), + "State" if is_extern != 0 else "CExport", + ASTCtx.Load) + annotation: AST* = BinOp(ps.pool, ann, OpKind.BitOr, storage_attr) + value: AST* = None + if _match_op(ps, TokOp.Equal): + if is_extern != 0: + # An extern variable is externally defined: no initializer allowed in this module. + _RecordDescriptorErr("extern variable declaration cannot have an initializer (the symbol is defined externally)", + lineno, col) + ps.error_count += 1 + return None + _advance(ps) + value = _parse_test(ps) + if value is None: + ps.error_count += 1 + _record_parse_err(ps) + return None + target: AST* = Name(ps.pool, name, ASTCtx.Store) + _set_pos(target, lineno, col, 0, 0) + node: AST* = AnnAssign(ps.pool, target, annotation, value, 1) + _set_pos(node, lineno, col, 0, 0) + _skip_nl(ps) + if _cur_type(ps) != TokenType.NewLine and _cur_type(ps) != TokenType.EndMarker: + _record_parse_err(ps) + ps.error_count += 1 + return None + if _cur_type(ps) == TokenType.NewLine: + _advance(ps) + return node + + +def _parse_enumdef(ps: Parser*) -> AST*: + """enum NAME: sugar -> a ClassDef field-identical to `class NAME(t.CEnum):`. + + Member forms (same as TPC's _ExpandEnumBlocks): bare `A` -> `A: t.State` (auto-numbered, i32 width); + comma group `A, B` -> several `A: t.State` / `B: t.State`; annotated `A: T` and valued `A = 0` as-is. + Base is always Attribute(Name('t'), 'CEnum'), field-identical to `class X(t.CEnum):`. Line/col inherited + from each member statement. + """ + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + _advance(ps) # enum + if _cur_type(ps) != TokenType.Name: + ps.error_count += 1 + return None + name: str = _cur_str(ps) + _advance(ps) + bases: list[AST*]* = list[AST*](ps.pool, 8) + bases.append(Attribute(ps.pool, Name(ps.pool, "t", ASTCtx.Load), + "CEnum", ASTCtx.Load)) + keywords: list[AST*]* = list[AST*](ps.pool, 8) + dec_list: list[AST*]* = ps.pending_decorators + ps.pending_decorators = None + _expect_op(ps, TokOp.Colon) + node: ClassDef* = ClassDef(ps.pool, name, bases, keywords, dec_list, None) + _parse_suite_into_children(ps, node) + _convert_enum_bare_members(ps, node) + _set_pos(node, lineno, col, 0, 0) + return node + + +def _parse_uniondef(ps: Parser*) -> AST*: + """union NAME: sugar -> a ClassDef field-identical to `class NAME(t.CUnion):`. + + Members are parsed as a normal class body (AnnAssign). Base is always Attribute(Name('t'), 'CUnion'), + field-identical to `class X(t.CUnion):`; TPC detects unions via `issubclass(cls, t.CUnion)`, so both + sides share the existing union path (offset 0, size = widest member). No bare-name conversion. + """ + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + _advance(ps) # union + if _cur_type(ps) != TokenType.Name: + ps.error_count += 1 + return None + name: str = _cur_str(ps) + _advance(ps) + bases: list[AST*]* = list[AST*](ps.pool, 8) + bases.append(Attribute(ps.pool, Name(ps.pool, "t", ASTCtx.Load), + "CUnion", ASTCtx.Load)) + keywords: list[AST*]* = list[AST*](ps.pool, 8) + dec_list: list[AST*]* = ps.pending_decorators + ps.pending_decorators = None + _expect_op(ps, TokOp.Colon) + node: ClassDef* = ClassDef(ps.pool, name, bases, keywords, dec_list, None) + _parse_suite_into_children(ps, node) + _set_pos(node, lineno, col, 0, 0) + return node + + +def _parse_renumdef(ps: Parser*) -> AST*: + """renum NAME: sugar -> a ClassDef field-identical to `class NAME(t.REnum):`. + + Members are parsed as a normal class body: a variant is a nested ClassDef (`class Some:` + field + annotations) and a payload-less variant is a plain assignment (`None_ = 0`). Base is always + Attribute(Name('t'), 'REnum'); TPC detects REnum via `issubclass(cls, t.REnum)`. No bare-name conversion. + """ + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + _advance(ps) # renum + if _cur_type(ps) != TokenType.Name: + ps.error_count += 1 + return None + name: str = _cur_str(ps) + _advance(ps) + bases: list[AST*]* = list[AST*](ps.pool, 8) + bases.append(Attribute(ps.pool, Name(ps.pool, "t", ASTCtx.Load), + "REnum", ASTCtx.Load)) + keywords: list[AST*]* = list[AST*](ps.pool, 8) + dec_list: list[AST*]* = ps.pending_decorators + ps.pending_decorators = None + _expect_op(ps, TokOp.Colon) + node: ClassDef* = ClassDef(ps.pool, name, bases, keywords, dec_list, None) + _parse_suite_into_children(ps, node) + _set_pos(node, lineno, col, 0, 0) + return node + + +def _RecordStructErr(detail: str, lineno: int, col: int, extra: str): + """Record an invalid struct form (fail-fast; first only), same reporting path as _RecordVirtualErr. + + Messages always use f-string interpolation: this language's str is i8*, so `str + str` degrades to + raw pointer add and illegal IR. + """ + global _g_parse_err_line, _g_parse_err_col, _g_parse_err_msg + if _g_parse_err_line != 0: + return + _g_parse_err_line = lineno + _g_parse_err_col = col + buf: str = stdlib.malloc(1536) + if buf is not None: + stdio.snprintf(buf, 1536, + "[VPC] invalid struct definition (%s). A struct has the same shape as a class but **can only represent a data type**:" + "(1) the virtual / novirtual modifier is forbidden (a struct has no vtable by nature, @t.NoVTable is built in);" + "(2) defining a def inside a struct is forbidden (use class if methods are needed);" + "(3) it may appear as a variant inside REnum, and a struct may contain another struct." + " Correct example: struct Point: x: int / struct Point: x: int, then a newline with 4-space indented y: int.%s" + " (struct constraints are never silently ignored: illegal forms all abort compilation)", detail, extra) + _g_parse_err_msg = buf + else: + _g_parse_err_msg = detail + + +def _parse_structdef(ps: Parser*) -> AST*: + """struct NAME: / struct NAME(Base): sugar -> a ClassDef field-identical to + `@t.NoVTable class NAME(t.CStruct):`. + + Data-only: forces no vtable via the pseudo decorator @t.NoVTable, appends Attribute(Name('t'), 'CStruct') + after the user bases as a data-class marker, and forbids def in the body (fail-fast via _RecordStructErr). + Nestable in REnum and in another struct. + """ + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + _advance(ps) # struct + if _cur_type(ps) != TokenType.Name: + _RecordStructErr("missing type name after `struct`", lineno, col, + "Correct form: struct Point: (the type name follows struct immediately).") + ps.error_count += 1 + return None + name: str = _cur_str(ps) + _advance(ps) + # Type parameters [T] / [T, U] (same as class, PEP 695 generics) + type_params: list[str]* = None + if _accept_op(ps, TokOp.Lsqb): + type_params = list[str](ps.pool, 8) + while not _match_op(ps, TokOp.Rsqb): + if _cur_type(ps) == TokenType.EndMarker: break + if _cur_type(ps) == TokenType.Name: + tp_name: str = _cur_str(ps) + _advance(ps) + if _match_op(ps, TokOp.Colon): + _advance(ps) + _parse_test(ps) + type_params.append(tp_name) + if not _accept_op(ps, TokOp.Comma): + break + _expect_op(ps, TokOp.Rsqb) + bases: list[AST*]* = list[AST*](ps.pool, 8) + keywords: list[AST*]* = list[AST*](ps.pool, 8) + # bases / keywords (same parse path as _parse_classdef, including cross-line and keyword=value) + if _accept_op(ps, TokOp.LPar): + while not _match_op(ps, TokOp.RPar): + _skip_nl(ps) + if _cur_type(ps) == TokenType.EndMarker: break + if _cur_type(ps) == TokenType.Name: + e: AST* = _parse_or_test(ps) + if _match_op(ps, TokOp.Equal): + _advance(ps) + v: AST* = _parse_test(ps) + kw_node: AST* = KwNode(ps.pool, e, v) + keywords.append(kw_node) + else: + bases.append(e) + if not _accept_op(ps, TokOp.Comma): + break + _skip_nl(ps) + _expect_op(ps, TokOp.RPar) + # Data-class marker base: appended after the user bases (real parent stays at bases[0]) + bases.append(Attribute(ps.pool, Name(ps.pool, "t", ASTCtx.Load), + "CStruct", ASTCtx.Load)) + _expect_op(ps, TokOp.Colon) + # Pseudo decorator @t.NoVTable: appended after the user decorators (same merge as _ParseVirtualModified) + dec_list: list[AST*]* = ps.pending_decorators + ps.pending_decorators = None + marker: AST* = Attribute(ps.pool, Name(ps.pool, "t", ASTCtx.Load), + "NoVTable", ASTCtx.Load) + _set_pos(marker, lineno, col, 0, 0) + merged: list[AST*]* = list[AST*](ps.pool, 8) + if merged is not None: + if dec_list is not None: + dn_s: sizet = len(dec_list) + for di_s in range(int(dn_s)): + merged.append(dec_list.get(di_s)) + merged.append(marker) + node: ClassDef* = ClassDef(ps.pool, name, bases, keywords, merged, type_params) + # owner_is_class=1: same as class (ps.in_class_body=1 during the class body) + _parse_suite_into_children(ps, node, 1) + # struct forbids def: check the direct children + chk: list[AST*]* = node.children + if chk is not None: + cn_s: sizet = len(chk) + for ci_s in range(int(cn_s)): + it_s: AST* = chk.get(ci_s) + if it_s is not None and it_s.kind() == ASTKind.FunctionDef: + fd_s: FunctionDef* = (FunctionDef*)(it_s) + fname_s: str = "(anonymous)" + if fd_s is not None and fd_s.name is not None: + fname_s = fd_s.name + _RecordStructErr(f"def '{fname_s}' defined inside struct '{name}'", + it_s.lineno, it_s.col_offset, + "a struct represents a data type only (same as C's struct);" + " if methods and a vtable are really needed, write the type as a class.") + ps.error_count += 1 + break + _set_pos(node, lineno, col, 0, 0) + return node + + +def _parse_match(ps: Parser*) -> AST*: + """match statement: match subject ':' NewLine Indent case+ Dedent""" + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + _advance(ps) # match + subject: AST* = _parse_test(ps) + _expect_op(ps, TokOp.Colon) + cases: list[AST*]* = list[AST*](ps.pool, 8) + node: AST* = Match(ps.pool, subject, cases, 0) + # Parsing body: Same NewLine/Indent/Dedent pattern as _parse_suite_into_children, + # but parses case clauses into a cases list instead of normal statements into children + _skip_nl(ps) + if _cur_type(ps) == TokenType.NewLine: + _advance(ps) + _skip_nl(ps) + if _cur_type(ps) == TokenType.Indent: + _advance(ps) + while _cur_type(ps) != TokenType.Dedent and _cur_type(ps) != TokenType.EndMarker: + if _cur_kw(ps) == Keyword.Case: + case_node: AST* = _parse_match_case(ps) + if case_node is not None: + cases.append(case_node) + else: + _advance(ps) # Skip non-case token to avoid infinite loop + _expect_type(ps, TokenType.Dedent) + _set_pos(node, lineno, col, 0, 0) + return node + + +# ============================================================ +# switch statement (C switch semantics, desugared to Match(is_switch=1)): `switch :` with +# `case :` / `default:` arms. C switch rules: no fall-through unless +# `fallthrough` is written in the case body. Case labels allow only constants / enum variants / +# multi-value `case 1 | 2:`, not structured patterns. +# ============================================================ + +def _is_switch_default_kw(ps: Parser*) -> int: + """`default` is a context keyword inside a switch suite: matches `default:` or + `default if :` (the latter supports a guard like `case _ if `).""" + if _cur_type(ps) != TokenType.Name: + return 0 + s: str = _cur_str(ps) + if s is None or string.strcmp(s, "default") != 0: + return 0 + nxt: Token* = ps.cur.next + if nxt is None: + return 0 + if nxt.type == TokenType.Op and nxt.op_subtype == TokOp.Colon: + return 1 + if nxt.type == TokenType.Name and nxt.kw_subtype == Keyword.If: + return 1 + return 0 + + +def _RecordSwitchCaseErr(detail: str, lineno: int, col: int): + """Record an invalid switch case label (fail-fast; first only)""" + global _g_parse_err_line, _g_parse_err_col, _g_parse_err_msg + if _g_parse_err_line != 0: + return + _g_parse_err_line = lineno + _g_parse_err_col = col + buf: str = stdlib.malloc(512) + if buf is not None: + stdio.snprintf(buf, 512, + "invalid switch case label (%s). switch supports only constant literals or enum variant labels" + " (multi-value `case 1 | 2:` allowed); it does not accept structural patterns such as sequences/mappings/captures." + " Correct examples: case 1: / case \"a\": / case Color.Red: / case Opt.Some():", detail) + _g_parse_err_msg = buf + else: + _g_parse_err_msg = "invalid switch case label" + + +def _CheckSwitchCasePattern(ps: Parser*, pat: AST*) -> int: + """Validate a switch case label (only constants / enum variants / multi-value patterns allowed)""" + if pat is None: + return 0 + k: int = pat.kind() + if k == ASTKind.MatchValue or k == ASTKind.MatchSingleton or k == ASTKind.MatchClass: + return 1 + if k == ASTKind.MatchAs: + # Only the wildcard (`case _`, equivalent to default) is legal; as-bindings with a subpattern are not + ma_w: MatchAs* = MatchAs*(pat) + if ma_w.pattern is None: + return 1 + return 0 + if k == ASTKind.MatchOr: + mo: MatchOr* = MatchOr*(pat) + pats: list[AST*]* = mo.patterns + if pats is None or len(pats) == 0: + return 0 + i: sizet = 0 + n: sizet = len(pats) + while i < n: + sub: AST* = pats.get(i) + if sub is None: + return 0 + sk: int = sub.kind() + if sk != ASTKind.MatchValue and sk != ASTKind.MatchSingleton \ + and sk != ASTKind.MatchClass: + return 0 + i = i + 1 + return 1 + return 0 + + +def _parse_switch_case(ps: Parser*) -> AST*: + """switch `case [if ] ':' suite` (guard supported). + + A bare name is a capture pattern (MatchAs) in match but a value reference in switch (like C's + `case NAME:`), so `MatchAs(pattern=None, name=X)` is normalized to `MatchValue(Name('X'))`; `_` + stays a wildcard. The guard is parsed like _parse_match_case and passed to MatchCase.guard. + """ + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + _advance(ps) # case + pattern: AST* = _parse_pattern(ps) + if pattern is not None and pattern.kind() == ASTKind.MatchAs: + ma: MatchAs* = MatchAs*(pattern) + if ma.pattern is None and ma.name is not None: + ref: AST* = Name(ps.pool, ma.name, ASTCtx.Load) + _set_pos(ref, pattern.lineno, pattern.col_offset, 0, 0) + mv: AST* = MatchValue(ps.pool, ref) + _set_pos(mv, pattern.lineno, pattern.col_offset, 0, 0) + pattern = mv + if _CheckSwitchCasePattern(ps, pattern) == 0: + _RecordSwitchCaseErr("case label is not a constant/enum variant", lineno, col) + ps.error_count += 1 + # guard: `case if ':'` (same form as _parse_match_case) + guard: AST* = None + if _cur_kw(ps) == Keyword.If: + _advance(ps) + guard = _parse_or_test(ps) + _expect_op(ps, TokOp.Colon) + node: AST* = MatchCase(ps.pool, pattern, guard) + _parse_suite_into_children(ps, node) + _set_pos(node, lineno, col, 0, 0) + return node + + +def _parse_switch_default(ps: Parser*) -> AST*: + """switch `default [if ] ':' suite` -- desugared to `case []` (MatchAs wildcard). + + `default` is switch's spelling of `case _`; it supports a guard like `case _ if `, passed + to MatchCase. Without if the behavior is unchanged (guard=None). + """ + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + _advance(ps) # default + guard: AST* = None + if _cur_kw(ps) == Keyword.If: + _advance(ps) + guard = _parse_or_test(ps) + _expect_op(ps, TokOp.Colon) + wild: AST* = MatchAs(ps.pool, None, None) + _set_pos(wild, lineno, col, 0, 0) + node: AST* = MatchCase(ps.pool, wild, guard) + _parse_suite_into_children(ps, node) + _set_pos(node, lineno, col, 0, 0) + return node + + +def _parse_switch(ps: Parser*) -> AST*: + """switch statement: switch subject ':' NewLine Indent (case|default)+ Dedent""" + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + _advance(ps) # switch + subject: AST* = _parse_test(ps) + _expect_op(ps, TokOp.Colon) + cases: list[AST*]* = list[AST*](ps.pool, 8) + node: AST* = Match(ps.pool, subject, cases, 1) + _skip_nl(ps) + if _cur_type(ps) == TokenType.NewLine: + _advance(ps) + _skip_nl(ps) + if _cur_type(ps) == TokenType.Indent: + _advance(ps) + while _cur_type(ps) != TokenType.Dedent and _cur_type(ps) != TokenType.EndMarker: + if _cur_kw(ps) == Keyword.Case: + case_node: AST* = _parse_switch_case(ps) + if case_node is not None: + cases.append(case_node) + elif _is_switch_default_kw(ps) != 0: + def_node: AST* = _parse_switch_default(ps) + if def_node is not None: + cases.append(def_node) + else: + _advance(ps) # Skip non-case token to avoid infinite loop + _expect_type(ps, TokenType.Dedent) + _set_pos(node, lineno, col, 0, 0) + return node + + +# ============================================================ +# Match pattern parsing +# ============================================================ + +def _make_name_ast(pool: memhub.MemManager*, node: Name*) -> AST*: + # Analysis: "Convert Name* to AST* (Solving the TransPyC Type Inference Problem)" + return node + + +def _parse_pattern(ps: Parser*) -> AST*: + """pattern: or_pattern ['as' NAME] + or_pattern: closed_pattern ('|' closed_pattern)*""" + pat: AST* = _parse_closed_pattern(ps) + if pat is None: return None + # OR pattern + if _match_op(ps, TokOp.VBar): + patterns: list[AST*]* = list[AST*](ps.pool, 8) + patterns.append(pat) + while _accept_op(ps, TokOp.VBar): + p: AST* = _parse_closed_pattern(ps) + if p is not None: + patterns.append(p) + or_node: AST* = MatchOr(ps.pool, patterns) + _set_pos(or_node, pat.lineno, pat.col_offset, 0, 0) + pat = or_node + # AS pattern + if _cur_kw(ps) == Keyword.As: + _advance(ps) + if _cur_type(ps) != TokenType.Name: + ps.error_count += 1 + return pat + name_str: str = _cur_str(ps) + _advance(ps) + as_node: AST* = MatchAs(ps.pool, pat, name_str) + _set_pos(as_node, pat.lineno, pat.col_offset, 0, 0) + return as_node + return pat + + +def _parse_closed_pattern(ps: Parser*) -> AST*: + """closed_pattern: literal | capture | wildcard | value | group | sequence | mapping | class""" + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + ttype: int = _cur_type(ps) + val: AST* + node: AST* + tok: Token* + sval: str + + # Number literal pattern + if ttype == TokenType.Number: + tok = ps.cur + _advance(ps) + if tok.is_float: + val = _new_const_float(ps.pool, tok.float_val, lineno, col) + else: + val = _new_const_int(ps.pool, tok.int_val, lineno, col, tok.int_width, tok.int_unsigned) + node = MatchValue(ps.pool, val) + _set_pos(node, lineno, col, 0, 0) + return node + + # String literal pattern + if ttype == TokenType.String: + sval = _cur_str(ps) + is_dquote_m: int = ps.cur.kw_subtype & 16 + _advance(ps) + val = _new_const_str(ps.pool, sval, lineno, col, is_dquote_m) + node = MatchValue(ps.pool, val) + _set_pos(node, lineno, col, 0, 0) + return node + + # Negative number literal pattern: -1, -1.5 + if _match_op(ps, TokOp.Minus): + _advance(ps) + if _cur_type(ps) == TokenType.Number: + tok = ps.cur + # Combined unary-minus + integer-literal overflow check (same rule as _parse_factor): + # a negative literal with magnitude > 2^63 yields < -2^63 -> fail-fast, no silent wrap. + if tok.is_float == 0 and _lex_int_neg_exceeds_i64(_cur_str(ps)) != 0: + _record_neg_literal_overflow(_cur_str(ps), lineno, col) + ps.error_count += 1 + return None + _advance(ps) + if tok.is_float: + val = _new_const_float(ps.pool, -tok.float_val, lineno, col) + else: + val = _new_const_int(ps.pool, -tok.int_val, lineno, col, tok.int_width, tok.int_unsigned) + node = MatchValue(ps.pool, val) + _set_pos(node, lineno, col, 0, 0) + return node + ps.error_count += 1 + return None + + # Name: None/True/False/_ /capture/value/class + if ttype == TokenType.Name: + kw: int = _cur_kw(ps) + # None/True/False → MatchSingleton + if kw == Keyword.None_ or kw == Keyword.True_ or kw == Keyword.False_: + _advance(ps) + if kw == Keyword.None_: + val = _new_const_none(ps.pool, lineno, col) + elif kw == Keyword.True_: + val = _new_const_bool(ps.pool, 1, lineno, col) + else: + val = _new_const_bool(ps.pool, 0, lineno, col) + node = MatchSingleton(ps.pool, val) + _set_pos(node, lineno, col, 0, 0) + return node + name_str: str = _cur_str(ps) + # Wildcard pattern: _ + if string.strcmp(name_str, "_") == 0: + _advance(ps) + node = MatchAs(ps.pool, None, None) + _set_pos(node, lineno, col, 0, 0) + return node + _advance(ps) + # Check dotted name (value pattern or class pattern) + if _match_op(ps, TokOp.Dot): + name_node: Name* = Name(ps.pool, name_str, ASTCtx.Load) + _set_pos(name_node, lineno, col, 0, 0) + base: AST* = _make_name_ast(ps.pool, name_node) + while _match_op(ps, TokOp.Dot): + _advance(ps) + if _cur_type(ps) != TokenType.Name: + ps.error_count += 1 + break + attr_name: str = _cur_str(ps) + attr_lineno: int = _cur_lineno(ps) + attr_col: int = _cur_col(ps) + _advance(ps) + base = Attribute(ps.pool, base, attr_name, ASTCtx.Load) + _set_pos(base, attr_lineno, attr_col, 0, 0) + # class pattern: dotted.name(...) + if _match_op(ps, TokOp.LPar): + return _parse_class_pattern(ps, base, lineno, col) + # value pattern: dotted.name + node = MatchValue(ps.pool, base) + _set_pos(node, lineno, col, 0, 0) + return node + # class pattern: Name(...) + if _match_op(ps, TokOp.LPar): + cls_node: Name* = Name(ps.pool, name_str, ASTCtx.Load) + _set_pos(cls_node, lineno, col, 0, 0) + cls: AST* = _make_name_ast(ps.pool, cls_node) + return _parse_class_pattern(ps, cls, lineno, col) + # capture pattern: NAME + node = MatchAs(ps.pool, None, name_str) + _set_pos(node, lineno, col, 0, 0) + return node + + # group pattern / sequence pattern (tuple): (...) + if _match_op(ps, TokOp.LPar): + _advance(ps) + inner: AST* = _parse_pattern(ps) + if _accept_op(ps, TokOp.Comma): + # sequence pattern (tuple) + patterns: list[AST*]* = list[AST*](ps.pool, 8) + if inner is not None: + patterns.append(inner) + while not _match_op(ps, TokOp.RPar) and _cur_type(ps) != TokenType.EndMarker: + p: AST* = _parse_pattern(ps) + if p is not None: + patterns.append(p) + if not _accept_op(ps, TokOp.Comma): + break + _expect_op(ps, TokOp.RPar) + node = MatchSequence(ps.pool, patterns) + _set_pos(node, lineno, col, 0, 0) + return node + _expect_op(ps, TokOp.RPar) + return inner # group pattern + + # sequence pattern (list): [...] + if _match_op(ps, TokOp.Lsqb): + return _parse_sequence_pattern(ps, TokOp.Rsqb, lineno, col) + + # mapping pattern: {...} + if _match_op(ps, TokOp.LBrace): + return _parse_mapping_pattern(ps, lineno, col) + + ps.error_count += 1 + return None + + +def _parse_sequence_pattern(ps: Parser*, close_op: int, + lineno: int, col: int) -> AST*: + """sequence_pattern: '[' [pattern_list] ']'""" + patterns: list[AST*]* = list[AST*](ps.pool, 8) + _advance(ps) # [ + while not _match_op(ps, close_op) and _cur_type(ps) != TokenType.EndMarker: + # *name (star pattern) + if _match_op(ps, TokOp.Star): + _advance(ps) + if _cur_type(ps) == TokenType.Name: + star_name: str = _cur_str(ps) + _advance(ps) + star_node: AST* + if string.strcmp(star_name, "_") == 0: + star_node = MatchStar(ps.pool, None) + else: + star_node = MatchStar(ps.pool, star_name) + _set_pos(star_node, lineno, col, 0, 0) + patterns.append(star_node) + else: + p: AST* = _parse_pattern(ps) + if p is not None: + patterns.append(p) + if not _accept_op(ps, TokOp.Comma): + break + _expect_op(ps, close_op) + node: AST* = MatchSequence(ps.pool, patterns) + _set_pos(node, lineno, col, 0, 0) + return node + + +def _parse_mapping_pattern(ps: Parser*, + lineno: int, col: int) -> AST*: + """mapping_pattern: '{' [items] '}' + items: key ':' pattern (',' key ':' pattern)* [',' '**' NAME] | '**' NAME""" + keys: list[AST*]* = list[AST*](ps.pool, 8) + patterns: list[AST*]* = list[AST*](ps.pool, 8) + rest: str = None + node: AST* + key: AST* + val_pat: AST* + _advance(ps) # { + if _match_op(ps, TokOp.RBrace): + _advance(ps) + node = MatchMapping(ps.pool, keys, patterns, rest) + _set_pos(node, lineno, col, 0, 0) + return node + while _cur_type(ps) != TokenType.EndMarker: + if _match_op(ps, TokOp.StarStar): + _advance(ps) + if _cur_type(ps) == TokenType.Name: + rest = _cur_str(ps) + _advance(ps) + break + key = _parse_expr(ps) + _expect_op(ps, TokOp.Colon) + val_pat = _parse_pattern(ps) + if key is not None: + keys.append(key) + if val_pat is not None: + patterns.append(val_pat) + if not _accept_op(ps, TokOp.Comma): + break + _expect_op(ps, TokOp.RBrace) + node = MatchMapping(ps.pool, keys, patterns, rest) + _set_pos(node, lineno, col, 0, 0) + return node + + +def _parse_class_pattern(ps: Parser*, cls: AST*, + lineno: int, col: int) -> AST*: + """class_pattern: cls '(' [pattern_list] ')' + pattern_list: positional_pattern (',' positional_pattern)* + [',' keyword_pattern]* | keyword_pattern (',' keyword_pattern)* + keyword_pattern: NAME '=' pattern""" + patterns: list[AST*]* = list[AST*](ps.pool, 8) + kwd_attrs: list[AST*]* = list[AST*](ps.pool, 8) + kwd_patterns: list[AST*]* = list[AST*](ps.pool, 8) + p: AST* + _advance(ps) # ( + if not _match_op(ps, TokOp.RPar): + while _cur_type(ps) != TokenType.EndMarker: + # keyword pattern: NAME '=' pattern + if _cur_type(ps) == TokenType.Name and ps.cur is not None: + nxt: Token* = ps.cur.next + if nxt is not None and nxt.type == TokenType.Op and nxt.op_subtype == TokOp.Equal: + attr_name: str = _cur_str(ps) + _advance(ps) # NAME + _advance(ps) # = + kwd_pat: AST* = _parse_pattern(ps) + attr_n: Name* = Name(ps.pool, attr_name, ASTCtx.Load) + _set_pos(attr_n, lineno, col, 0, 0) + attr_node: AST* = _make_name_ast(ps.pool, attr_n) + kwd_attrs.append(attr_node) + if kwd_pat is not None: + kwd_patterns.append(kwd_pat) + else: + p = _parse_pattern(ps) + if p is not None: + patterns.append(p) + else: + p = _parse_pattern(ps) + if p is not None: + patterns.append(p) + if not _accept_op(ps, TokOp.Comma): + break + _expect_op(ps, TokOp.RPar) + node: AST* = MatchClass(ps.pool, cls, patterns, kwd_attrs, kwd_patterns) + _set_pos(node, lineno, col, 0, 0) + return node + + +def _parse_match_case(ps: Parser*) -> AST*: + """case pattern [if guard] ':' suite""" + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + _advance(ps) # case + guard: AST* = None + pattern: AST* = _parse_pattern(ps) + if _cur_kw(ps) == Keyword.If: + _advance(ps) + guard = _parse_or_test(ps) + _expect_op(ps, TokOp.Colon) + node: AST* = MatchCase(ps.pool, pattern, guard) + _parse_suite_into_children(ps, node) + _set_pos(node, lineno, col, 0, 0) + return node + + +# ============================================================ +# __attribute__ decorator desugar (C-style attribute syntax): `@__attribute__(section(".text.wo"), +# always_inline)` equals hand-written `@c.Attribute(t.attr.section(...), t.attr.always_inline)`. +# Arguments use unprefixed names, each desugared to `t.attr.` (calls keep their args) with the +# outer call becoming `c.Attribute(...)`, so the result is field-identical and recognized as-is. +# Works for class and def; arguments already in `t.attr.X` / `c.Attribute(...)` form are kept. +# ============================================================ +def _IsMagicAttributeCallee(node: AST*) -> int: + """Whether the node is `__attribute__` (Name form, or the trailing X.__attribute__ form)""" + if node is None: + return 0 + k: int = node.kind() + if k == ASTKind.Name: + nm: Name* = Name*(node) + if nm.id is not None and string.strcmp(nm.id, "__attribute__") == 0: + return 1 + return 0 + if k == ASTKind.Attribute: + at: Attribute* = Attribute*(node) + if at.attr is not None and string.strcmp(at.attr, "__attribute__") == 0: + return 1 + return 0 + + +def _MakeTAttrMarker(ps: Parser*, arg: AST*) -> AST*: + """Desugar one __attribute__ argument into `t.attr.` (calls keep their arguments). + + `always_inline` (Name) -> `t.attr.always_inline`; `section(".text.wo")` (Call) -> `t.attr.section(...)`; + an argument already in `t.attr.X` form is returned as-is. + """ + if arg is None: + return None + k: int = arg.kind() + if k == ASTKind.Attribute: + at0: Attribute* = Attribute*(arg) + if at0.value is not None and at0.value.kind() == ASTKind.Attribute: + return arg + return arg + attr_root: AST* = Attribute(ps.pool, Name(ps.pool, "t", ASTCtx.Load), + "attr", ASTCtx.Load) + if k == ASTKind.Name: + nm2: Name* = Name*(arg) + if nm2.id is None: + return arg + mk: AST* = Attribute(ps.pool, attr_root, nm2.id, ASTCtx.Load) + _set_pos(mk, nm2.lineno, nm2.col_offset, 0, 0) + return mk + if k == ASTKind.Call: + cl2: Call* = Call*(arg) + if cl2.func is None or cl2.func.kind() != ASTKind.Name: + return arg + fnm: Name* = Name*(cl2.func) + if fnm.id is None: + return arg + inner: AST* = Attribute(ps.pool, attr_root, fnm.id, ASTCtx.Load) + _set_pos(inner, fnm.lineno, fnm.col_offset, 0, 0) + mk2: AST* = Call(ps.pool, inner, cl2.args, cl2.keywords) + _set_pos(mk2, cl2.lineno, cl2.col_offset, 0, 0) + return mk2 + return arg + + +def _DesugarMagicAttribute(ps: Parser*, expr: AST*) -> AST*: + """`@__attribute__(...)` -> `@c.Attribute(t.attr.x, ...)`; other decorators are returned as-is""" + if expr is None or expr.kind() != ASTKind.Call: + return expr + cl: Call* = Call*(expr) + if _IsMagicAttributeCallee(cl.func) == 0: + return expr + c_func: AST* = Attribute(ps.pool, Name(ps.pool, "c", ASTCtx.Load), + "Attribute", ASTCtx.Load) + new_args: list[AST*]* = list[AST*](ps.pool, 8) + if new_args is None: + return expr + if cl.args is not None: + an: sizet = len(cl.args) + for ai in range(int(an)): + mk: AST* = _MakeTAttrMarker(ps, cl.args.get(ai)) + if mk is not None: + new_args.append(mk) + new_call: AST* = Call(ps.pool, c_func, new_args, None) + _set_pos(new_call, cl.lineno, cl.col_offset, 0, 0) + return new_call + + +def _parse_decorators(ps: Parser*) -> list[AST*]*: + """Parse a decorator chain, returning a list.""" + decs: list[AST*]* = list[AST*](ps.pool, 8) + while _match_op(ps, TokOp.At): + lineno: int = _cur_lineno(ps) + col: int = _cur_col(ps) + _advance(ps) # @ + expr: AST* = _parse_namedexpr(ps) + # C-style `@__attribute__(...)` desugars into the existing `@c.Attribute(t.attr....)` + expr = _DesugarMagicAttribute(ps, expr) + _skip_nl(ps) + if _cur_type(ps) == TokenType.NewLine: + _advance(ps) + decs.append(expr) + return decs + + +# ============================================================ +# Module entry +# ============================================================ + +def _parse_module_body(ps: Parser*, module_node: AST*): + """Parse the module body, appending to module_node.""" + _skip_nl(ps) + stmt_idx: int = 0 + while _cur_type(ps) != TokenType.EndMarker: + stmt_idx += 1 + # fail-fast anti-hang: over-budget steps (cyclic token list / cursor spinning) report and return + if ps.step_count > ps.step_limit: + _record_parse_err(ps) + return + # Decorator + if _match_op(ps, TokOp.At): + ps.pending_decorators = _parse_decorators(ps) + # Record current token pointer to check for infinite loop + before_cur: Token* = ps.cur + _parse_statement_into(ps, module_node) + # Defense: If parsing doesn't advance the token (like when _parse_atom encounters an unrecognized token and returns None), + # force it to move forward to avoid an infinite loop in the main loop; accumulate error_count for the caller to diagnose + if ps.cur is before_cur: + ps.error_count += 1 + _advance(ps) + _skip_nl(ps) + + +def parse_tokens(pool: memhub.MemManager*, + tokens: Token*) -> AST*: + """Parse an AST Module from a token list.""" + if pool is None or tokens is None: + return None + global _g_parse_err_line, _g_parse_err_col, _g_parse_err_msg + _g_parse_err_line = 0 + _g_parse_err_col = 0 + _g_parse_err_msg = None + # Lexer fail-fast takes priority: overflows recorded during tokenize (e.g. integer literal > 2^64-1) + # are forwarded to the parse error channel here; lexer has no independent fatal exit. + lex_line: int = get_lex_err_line() + if lex_line != 0: + _g_parse_err_line = lex_line + _g_parse_err_col = get_lex_err_col() + _g_parse_err_msg = get_lex_err_msg() + return None + ps: Parser* = new_parser(pool) + if ps is None: return None + _parser_init(ps, tokens, pool) + # fail-fast anti-hang: count the token list first, detecting a cycle. A cyclic list (Token appended + # twice or aliased) would hang the statement loop; exceeding the cap before the tail means a cycle. + tok_count: sizet = 0 + walk_t: Token* = tokens + while walk_t is not None: + walk_t = walk_t.next + tok_count += 1 + if tok_count > PARSER_MAX_TOKENS: + _g_parse_err_line = _cur_lineno(ps) + _g_parse_err_col = _cur_col(ps) + _g_parse_err_msg = "token list is cyclic: the token list emitted by the lexer has no terminator (_emit appended the same Token twice or aliased it); parsing aborted" + return None + # Budget: the watchdog counts "query current token" calls (well below 256 per token in valid parses, + # so it cannot trip falsely) while catching cycles/spins; it also bounds _advance and statement loops. + ps.step_limit = tok_count * 256 + 262144 + module_node: Module* = Module(pool) + _parse_module_body(ps, module_node) + # fail-fast: on a parse error (error_count > 0) return None so the caller aborts, avoiding silently + # compiling broken code (e.g. a function definition left without its def keyword). + if ps.error_count > 0: + # Fallback position/reason: the parser has 40+ `ps.error_count += 1` sites and not all call + # _record_*. Without a record, dump_parse_traceback can only print `File "..."` + + # `SyntaxError` with no line or reason. Fill in a locatable message from the current token so the + # report always has file:line:col + reason. + if _g_parse_err_line == 0: + _g_parse_err_line = _cur_lineno(ps) + _g_parse_err_col = _cur_col(ps) + err_s_e: str = _cur_str(ps) + if err_s_e is None: + err_s_e = "(none)" + buf_e: str = stdlib.malloc(256) + if buf_e is not None: + stdio.snprintf(buf_e, 256, + "syntax error: the parser recorded %d error(s) but no specific reason;" + " token near the error position='%s' (type=%d)", + ps.error_count, err_s_e, _cur_type(ps)); + _g_parse_err_msg = buf_e + else: + _g_parse_err_msg = "syntax error: the parser recorded no specific reason" + return None + return module_node diff --git a/ast/tokens.vp b/ast/tokens.vp index 79c8730..9bc84aa 100644 --- a/ast/tokens.vp +++ b/ast/tokens.vp @@ -200,6 +200,10 @@ class Token: end_lineno: int # Ending line number end_col_offset: int # Ending column offset next: Token* # Next token in linked list + # 显式整型宽度后缀（_int/_uint/_i/_u）：N>0 = 声明宽度，0 = 未指定； + # int_unsigned=1 表示 uint 形式。追加在末尾，保持既有字段偏移不变（Token 经 __sizeof__ 分配并 memset）。 + int_width: int + int_unsigned: int def __new__(self, pool: memhub.MemManager*): """Allocate zero‑initialized Token struct memory from mpool""" @@ -259,7 +263,7 @@ def _kw_index_put(entry: KwEntry*): h: uint64 = _kw_hash_str(entry.name) mask: sizet = sizet(_KW_IDX_CAP - 1) idx: sizet = sizet(h) & mask - for i in range(_KW_IDX_CAP): + for i in range(int(_KW_IDX_CAP)): slot: sizet* = (sizet*)(t.CPtr(uint64(_kw_index) + uint64(idx * 8))) cur_s: sizet = slot[0] if cur_s == 0: @@ -302,7 +306,7 @@ def _kw_lookup(name: str) -> int: h: uint64 = _kw_hash_str(name) mask: sizet = sizet(_KW_IDX_CAP - 1) idx: sizet = sizet(h) & mask - for i in range(_KW_IDX_CAP): + for i in range(int(_KW_IDX_CAP)): slot: sizet* = (sizet*)(t.CPtr(uint64(_kw_index) + uint64(idx * 8))) cur_s: sizet = slot[0] if cur_s == 0: @@ -520,7 +524,7 @@ def token_set_str(pool: memhub.MemManager*, tok: Token*, buf: str = pool.alloc(length + 1) if buf == None: _ast_pool_fatal("token.str_val", length + 1) - for i in range(length): + for i in range(int(length)): buf[i] = src[start + i] buf[length] = '\0' tok.str_val = buf