Viper_VRTE/ast/lexer.vp

import t, c
from stdint import *
import memhub
import string
import stdio
import stdlib
from .tokens import (
    Token, TokenType, TokOp, _init_tables, _kw_lookup, _op_head,
    OpEntry, new_token, token_set_str, token_set_str_literal,
)


# ============================================================
# 整数字面量溢出（词法层 fail-fast）
#
# 整数常量在 token/AST 中是 int64（tokens.vp `int_val` / exprs.vp `int_val`）。
# 旧实现用 `string.atoll`（string.vp:292）在 int64 下 `num = num * 10 + digit`
# 逐位累加 ⇒ 值 > 2^64-1 时**静默回绕**（如 2^64 回绕成 0，2^64-1 回绕成 -1）。
# 本实现在 `_emit_number` 里对**整数**字面量先做纯文本幅值判定
# （`_lex_int_exceeds_u64`，与 AssignGenMixin._int_text_exceeds_64 同口径：按
# 进制统计有效位数 / 与 2^64-1 的十进制文本做长度+字典序比较，**不落进任何定宽
# 容器、不回绕后再比较**），超出 2^64-1 即记录 fail-fast（不再静默回绕）。
# 合法范围 0..2^64-1 仍按 64 位位模式存入 int64（既有合法路径的产物形态不变，
# 如 0xFFFFFFFFFFFFFFFF → int64 -1、0xA5A5A5A5A5A5A5A5 保持其 64 位位模式）。
# 各进制（2/8/10/16）口径与 TPC（Python int(x, 0)，任意精度）一致：接受区间同为
# [-2^63, 2^64-1]（字面量本身无符号，负数侧由一元减表达），超出即拒绝。
#
# 上报通道：lexer 无独立 fatal 出口，故记录到模块级三元组（`_lex_record_overflow`），
# 由 parser 的 parse_tokens 取出后转写 _g_parse_err_*（位置 + 消息），复用既有
# Traceback/SyntaxError 报告路径（"文件:行:列 + 源码行"由该路径补全并终止编译）。
# 这覆盖全部源码字面量（文件级解析必经 parse_tokens）；parse_expression 只用于编译器
# 合成的短类型串，不承载源码字面量，故不另设通道。
# ============================================================
_g_lex_err_line: int = 0
_g_lex_err_col: int = 0
_g_lex_err_msg: str = None


def get_lex_err_line() -> int:
    """Line number of the latest lexer fail-fast (0 = none).

    Consumed by parser.parse_tokens and forwarded to the parse error channel.
    """
    return _g_lex_err_line


def get_lex_err_col() -> int:
    """Column of the latest lexer fail-fast (0-based; -1 = no column info)"""
    return _g_lex_err_col


def get_lex_err_msg() -> str:
    """Message of the latest lexer fail-fast (literal / required width / limit / fix hint)"""
    return _g_lex_err_msg


def _lex_reset_err():
    """Clear the lexer fail-fast triple (called at each tokenize entry to avoid stale errors)"""
    global _g_lex_err_line, _g_lex_err_col, _g_lex_err_msg
    _g_lex_err_line = 0
    _g_lex_err_col = 0
    _g_lex_err_msg = None


def _lex_record_overflow(text: str, lineno: int, col: int):
    """Record an integer literal overflow (fail-fast; first record only); message carries literal
    + required width (>= 65 bits, limit 2^64-1) + fix hint (file:line:col filled later by parse_tokens).
    """
    global _g_lex_err_line, _g_lex_err_col, _g_lex_err_msg
    if _g_lex_err_line != 0:
        return
    _g_lex_err_line = lineno
    _g_lex_err_col = col
    msg_buf: str = stdlib.malloc(512)
    if msg_buf is not None:
        stdio.snprintf(msg_buf, 512,
            "[TPV] 整数字面量溢出: 字面量 '%s' 超出 64 位可表示范围"
            "（按位模式解释的上限为 18446744073709551615 = 2^64-1，"
            "该字面量需要的位宽至少 65 位）。"
            "禁止静默回绕/截断，已终止编译。"
            "修法: 把该字面量改到 0..18446744073709551615 的 64 位位模式范围内；"
            "本语言不存在更宽的整数类型，如需更大数值请拆分为多个 64 位字表达。", text)
        _g_lex_err_msg = msg_buf
    else:
        _g_lex_err_msg = text


def _lex_int_exceeds_u64(text: str) -> int:
    """True if the integer literal magnitude exceeds the 64-bit limit 2^64-1 (1 = yes, 0 = no).
    Pure text compare, no wrap-around: decimal length+lexicographic vs "18446744073709551615",
    0x > 16, 0o > 22 (or 22 leading > '1'), 0b > 64 (literal text carries no sign).
    """
    if text is None:
        return 0
    tl: sizet = string.strlen(text)
    if tl == 0:
        return 0
    # Base detection (prefix letter case-insensitive)
    base: int = 10
    start: sizet = 0
    if tl >= 2 and text[0] == '0':
        pfx: char = text[1]
        if pfx == 'x' or pfx == 'X':
            base = 16
            start = 2
        elif pfx == 'o' or pfx == 'O':
            base = 8
            start = 2
        elif pfx == 'b' or pfx == 'B':
            base = 2
            start = 2
    # Count significant digits (skip '_' and leading zeros) and remember the first one
    sig: int = 0
    first: char = '0'
    j: sizet = start
    while j < tl:
        ch: char = text[j]
        j += 1
        if ch == '_':
            continue
        ok_digit: int = 0
        if base == 16:
            if (ch >= '0' and ch <= '9') or (ch >= 'a' and ch <= 'f') or (ch >= 'A' and ch <= 'F'):
                ok_digit = 1
        elif base == 10:
            if ch >= '0' and ch <= '9':
                ok_digit = 1
        elif base == 8:
            if ch >= '0' and ch <= '7':
                ok_digit = 1
        else:
            if ch == '0' or ch == '1':
                ok_digit = 1
        if ok_digit == 0:
            break
        if sig == 0:
            if ch == '0':
                continue  # leading zeros not counted
            first = ch
        sig += 1
    if sig == 0:
        return 0  # value 0
    if base == 16:
        return 1 if sig > 16 else 0
    if base == 2:
        return 1 if sig > 64 else 0
    if base == 8:
        if sig > 22:
            return 1
        if sig == 22 and first > '1':
            return 1
        return 0
    # Decimal: compare significant-digit count against the 2^64-1 text length
    mlen: sizet = 20
    if sig > mlen:
        return 1
    if sig < mlen:
        return 0
    # Equal length (20 digits): lexicographic compare against '18446744073709551615'
    max_text: str = "18446744073709551615"
    d: sizet = 0
    k: sizet = start
    lead: int = 1
    while k < tl and d < mlen:
        c: char = text[k]
        k += 1
        if c == '_':
            continue
        if c < '0' or c > '9':
            return 0
        if lead != 0 and c == '0':
            continue
        lead = 0
        mc: char = max_text[d]
        if c > mc:
            return 1
        if c < mc:
            return 0
        d += 1
    return 0



def _lex_mag_digit_val(ch: char, base: int) -> int:
    """Digit char -> positional value (hex accepts a-f/A-F; other bases are '0'..base-1)."""
    if base == 16:
        return _hex_val(ch)
    return int(ch) - 48


def _lex_mag_gt_text(text: str, base: int, start: sizet, sig: int, max_text: str) -> int:
    """True if the literal's significant digits exceed the same-length max text (1 = yes).
    Both sides skip '_'/leading zeros, compared by positional value (hex via _hex_val); pure text.
    """
    d: sizet = 0
    k: sizet = start
    tl: sizet = string.strlen(text)
    lead: int = 1
    while k < tl and d < sizet(sig):
        c: char = text[k]
        k += 1
        if c == '_':
            continue
        if base == 16:
            if not ((c >= '0' and c <= '9') or (c >= 'a' and c <= 'f') or (c >= 'A' and c <= 'F')):
                break
        elif base == 10:
            if c < '0' or c > '9':
                break
        elif base == 8:
            if c < '0' or c > '7':
                break
        else:
            if c != '0' and c != '1':
                break
        if lead != 0 and c == '0':
            continue
        lead = 0
        cv: int = _lex_mag_digit_val(c, base)
        mv: int = _lex_mag_digit_val(max_text[d], base)
        if cv > mv:
            return 1
        if cv < mv:
            return 0
        d += 1
    return 0


def _lex_int_neg_exceeds_i64(text: str) -> int:
    """True if the magnitude is strictly greater than 2^63 (1 = yes).
    For unary-minus + literal overflow: a magnitude-valid literal may become < -2^63 after
    negation; exactly 2^63 is legal (INT64_MIN). Pure text compare, no wrap-around.
    """
    if text is None:
        return 0
    tl: sizet = string.strlen(text)
    if tl == 0:
        return 0
    base: int = 10
    start: sizet = 0
    if tl >= 2 and text[0] == '0':
        pfx: char = text[1]
        if pfx == 'x' or pfx == 'X':
            base = 16
            start = 2
        elif pfx == 'o' or pfx == 'O':
            base = 8
            start = 2
        elif pfx == 'b' or pfx == 'B':
            base = 2
            start = 2
    sig: int = 0
    j: sizet = start
    while j < tl:
        ch: char = text[j]
        j += 1
        if ch == '_':
            continue
        ok_digit: int = 0
        if base == 16:
            if (ch >= '0' and ch <= '9') or (ch >= 'a' and ch <= 'f') or (ch >= 'A' and ch <= 'F'):
                ok_digit = 1
        elif base == 10:
            if ch >= '0' and ch <= '9':
                ok_digit = 1
        elif base == 8:
            if ch >= '0' and ch <= '7':
                ok_digit = 1
        else:
            if ch == '0' or ch == '1':
                ok_digit = 1
        if ok_digit == 0:
            break
        if sig == 0 and ch == '0':
            continue  # leading zeros not counted
        sig += 1
    if sig == 0:
        return 0  # value 0
    if base == 16:
        if sig > 16: return 1
        if sig < 16: return 0
        return _lex_mag_gt_text(text, 16, start, sig, "8000000000000000")
    if base == 2:
        if sig > 64: return 1
        if sig < 64: return 0
        return _lex_mag_gt_text(text, 2, start, sig,
            "1000000000000000000000000000000000000000000000000000000000000000")
    if base == 8:
        if sig > 22: return 1
        if sig < 22: return 0
        return _lex_mag_gt_text(text, 8, start, sig, "1000000000000000000000")
    if sig > 19: return 1
    if sig < 19: return 0
    return _lex_mag_gt_text(text, 10, start, sig, "9223372036854775808")


# Maximum Indentation Stack Depth
define LEXER_MAX_INDENT = 256


# ============================================================
# Lexer State Structure
# ============================================================
class Lexer:
    src: str          # Source code (NUL-terminated)
    pos: sizet                  # Current position
    len: sizet                  # Source code length
    lineno: int                 # Current line number (1-based)
    col: int                    # Current column number (0-based)
    pool: rawptr[memhub.MemManager]     # Memory pool
    indent_stack: rawptr[int]  # Indent stack (int array, allocated from pool)
    indent_top: int             # Stack top (-1 for empty stack)
    paren_depth: int            # Paren depth (>0 for no TokenType.NewLine)
    tokens_head: rawptr[Token]    # token list head
    tokens_tail: rawptr[Token]    # token list tail
    at_line_start: int          # 1=line start (need indent)
    error_count: int            # Error count

    def __new__(self, pool: rawptr[memhub.MemManager]):
        ptr: rawptr[Lexer] = pool.alloc(Lexer.__sizeof__())
        if ptr == None:
            # Pool exhausted: fail-fast instead of returning NULL (downstream would treat NULL as a valid Lexer)
            stdio.printf("[AST] Lexer 分配失败：内存池已耗尽（need=%zu 字节）\n",
                         sizet(Lexer.__sizeof__()))
            stdio.fflush(None)
            stdlib.exit(1)
        string.memset(ptr, 0, Lexer.__sizeof__())
        return ptr


def new_lexer(pool: rawptr[memhub.MemManager]) -> rawptr[Lexer]:
    """Allocate from the pool and return a Lexer (cross-module safe wrapper)"""
    return Lexer(pool)


def _lexer_init(lx: rawptr[Lexer], src: str, pool: rawptr[memhub.MemManager]):
    """Initialize Lexer"""
    if lx == None: return
    lx.src = src
    lx.pos = 0
    lx.len = string.strlen(src) if src != None else 0
    lx.lineno = 1
    lx.col = 0
    lx.pool = pool
    # Allocate indentation stack
    lx.indent_stack = pool.alloc(LEXER_MAX_INDENT * 8)
    if lx.indent_stack == None:
        stdio.printf("[AST] Lexer.indent_stack 分配失败：内存池已耗尽（need=%zu 字节）\n",
                     sizet(LEXER_MAX_INDENT * 8))
        stdio.fflush(None)
        stdlib.exit(1)
    string.memset(lx.indent_stack, 0, LEXER_MAX_INDENT * 8)
    lx.indent_top = -1
    # Push 0 (base indent)
    lx.indent_stack[0] = 0
    lx.indent_top = 0
    lx.paren_depth = 0
    lx.tokens_head = None
    lx.tokens_tail = None
    lx.at_line_start = 1
    lx.error_count = 0
    _init_tables(pool)


# ============================================================
# Auxiliary Functions
# ============================================================

def _peek(lx: rawptr[Lexer], offset: sizet) -> char:
    """Peek character at position pos+offset, return '\0' if out of bounds"""
    p: sizet = lx.pos + offset
    if p >= lx.len: return '\0'
    return lx.src[p]


def _peek_cur(lx: rawptr[Lexer]) -> char:
    if lx.pos >= lx.len: return '\0'
    return lx.src[lx.pos]


def _advance(lx: rawptr[Lexer]) -> char:
    """Advance one character, return the character"""
    if lx.pos >= lx.len: return '\0'
    ch: char = lx.src[lx.pos]
    lx.pos += 1
    if ch == '\n':
        lx.lineno += 1
        lx.col = 0
    else:
        lx.col += 1
    return ch


def _is_alpha(ch: char) -> int:
    if ch >= 'a' and ch <= 'z': return 1
    if ch >= 'A' and ch <= 'Z': return 1
    if ch == '_': return 1
    return 0


def _is_digit(ch: char) -> int:
    if ch >= '0' and ch <= '9': return 1
    return 0


def _is_alnum(ch: char) -> int:
    if _is_alpha(ch): return 1
    if _is_digit(ch): return 1
    return 0


def _is_hex(ch: char) -> int:
    if ch >= '0' and ch <= '9': return 1
    if ch >= 'a' and ch <= 'f': return 1
    if ch >= 'A' and ch <= 'F': return 1
    return 0


def _hex_val(ch: char) -> int:
    """Hex char -> value 0-15 (for \xNN escape parsing)"""
    if ch >= '0' and ch <= '9': return int(ch) - 48
    if ch >= 'a' and ch <= 'f': return int(ch) - 87
    if ch >= 'A' and ch <= 'F': return int(ch) - 55
    return 0


def _is_space(ch: char) -> int:
    if ch == ' ':  return 1
    if ch == '\t': return 1
    if ch == '\f': return 1
    if ch == '\v': return 1
    if ch == '\r': return 1
    return 0


def _emit(lx: rawptr[Lexer], tok: rawptr[Token]):
    """Append token to token list"""
    if tok == None: return
    if lx.tokens_head == None:
        lx.tokens_head = tok
        lx.tokens_tail = tok
    else:
        lx.tokens_tail.next = tok
        lx.tokens_tail = tok


def _make_op_token(lx: rawptr[Lexer], op_id: int, length: int,
                   lineno: int, col: int) -> rawptr[Token]:
    tok: rawptr[Token] = new_token(lx.pool, TokenType.Op, lineno, col)
    if tok == None: return None
    tok.op_subtype = op_id
    tok.end_col_offset = col + length
    return tok


# ============================================================
# Indent Handling
# ============================================================

def _handle_indent(lx: rawptr[Lexer]):
    """Handle line start indent, generate TokenType.Indent/TokenType.Dedent.
    Skip empty lines and comment lines."""
    # Skip empty lines and comment lines
    while lx.pos < lx.len:
        # Save line start position for non-empty lines
        line_start_pos: sizet = lx.pos
        line_start_col: int = lx.col
        # Skip whitespace, not across lines
        while lx.pos < lx.len:
            ch: char = lx.src[lx.pos]
            if ch == ' ' or ch == '\t' or ch == '\f' or ch == '\v':
                lx.pos += 1
                lx.col += 1
            elif ch == '\r':
                # CRLF line end \r, skip \n to trigger empty line processing
                lx.pos += 1
            else:
                break
        if lx.pos >= lx.len:
            break
        ch = lx.src[lx.pos]
        if ch == '\n':
            # Empty line, skip
            lx.pos += 1
            lx.lineno += 1
            lx.col = 0
            continue
        if ch == '\r':
            # CRLF line end \r, skip \n to trigger empty line processing
            lx.pos += 1
            if lx.pos < lx.len and lx.src[lx.pos] == '\n':
                lx.pos += 1
            lx.lineno += 1
            lx.col = 0
            continue
        if ch == '#':
            # Comment line, skip to end of line
            while lx.pos < lx.len and lx.src[lx.pos] != '\n':
                lx.pos += 1
            continue
        # Non-empty line: backtrack to line start to allow indent calculation
        lx.pos = line_start_pos
        lx.col = line_start_col
        break

    if lx.pos >= lx.len:
        # End of file, no indent to handle
        lx.at_line_start = 0
        return

    # Calculate current line indent
    indent: int = 0
    save_pos: sizet = lx.pos
    save_col: int = lx.col
    while lx.pos < lx.len:
        ch: char = lx.src[lx.pos]
        if ch == ' ':
            indent += 1
            lx.pos += 1
            lx.col += 1
        elif ch == '\t':
            indent += 8 - (indent % 8)
            lx.pos += 1
            lx.col += 1
        else:
            break

    # If line end (empty line), skip indent calculation
    if lx.pos < lx.len:
        ch = lx.src[lx.pos]
        if ch == '\n' or ch == '\r' or ch == '#':
            # Empty line or comment line, backtrack position to allow loop to handle indent
            lx.pos = save_pos
            lx.col = save_col
            lx.at_line_start = 0
            return

    # Compare indent with top of indent stack
    top_indent: int = lx.indent_stack[lx.indent_top]
    if indent > top_indent:
        # TokenType.Indent
        if lx.indent_top + 1 >= LEXER_MAX_INDENT:
            lx.error_count += 1
            return
        lx.indent_top += 1
        lx.indent_stack[lx.indent_top] = indent
        tok: rawptr[Token] = new_token(lx.pool, TokenType.Indent, lx.lineno, 0)
        _emit(lx, tok)
    else:
        while indent < lx.indent_stack[lx.indent_top] and lx.indent_top > 0:
            lx.indent_top -= 1
            tok: rawptr[Token] = new_token(lx.pool, TokenType.Dedent, lx.lineno, 0)
            _emit(lx, tok)
    lx.at_line_start = 0


# ============================================================
# Identifier/Keyword Handling
# ============================================================

def _read_name(lx: rawptr[Lexer]):
    start: sizet = lx.pos
    start_lineno: int = lx.lineno
    start_col: int = lx.col
    while lx.pos < lx.len:
        ch: char = lx.src[lx.pos]
        if _is_alnum(ch):
            lx.pos += 1
            lx.col += 1
        else:
            break
    length: sizet = lx.pos - start
    tok: rawptr[Token] = new_token(lx.pool, TokenType.Name, start_lineno, start_col)
    if tok == None: return
    token_set_str(lx.pool, tok, lx.src, start, length)
    tok.end_lineno = lx.lineno
    tok.end_col_offset = start_col + int(length)
    # Look up keyword if any
    kw_id: int = _kw_lookup(tok.str_val)
    if kw_id: tok.kw_subtype = kw_id
    _emit(lx, tok)


# ============================================================
# Number Handling
# ============================================================

def _read_number(lx: rawptr[Lexer]):
    start: sizet = lx.pos
    start_lineno: int = lx.lineno
    start_col: int = lx.col
    is_float: int = 0
    is_complex: int = 0

    ch0: char = lx.src[lx.pos]
    # Handle 0x/0o/0b prefix
    if ch0 == '0' and lx.pos + 1 < lx.len:
        ch1: char = lx.src[lx.pos + 1]
        if ch1 == 'x' or ch1 == 'X':
            lx.pos += 2
            lx.col += 2
            while lx.pos < lx.len:
                ch: char = lx.src[lx.pos]
                if _is_hex(ch) or ch == '_':
                    lx.pos += 1
                    lx.col += 1
                else:
                    break
            _emit_number(lx, start, start_lineno, start_col, 0, 0, 0)
            return
        if ch1 == 'o' or ch1 == 'O':
            lx.pos += 2
            lx.col += 2
            while lx.pos < lx.len:
                ch: char = lx.src[lx.pos]
                if ch >= '0' and ch <= '7':
                    lx.pos += 1
                    lx.col += 1
                elif ch == '_':
                    lx.pos += 1
                    lx.col += 1
                else:
                    break
            _emit_number(lx, start, start_lineno, start_col, 0, 0, 0)
            return
        if ch1 == 'b' or ch1 == 'B':
            lx.pos += 2
            lx.col += 2
            while lx.pos < lx.len:
                ch: char = lx.src[lx.pos]
                if ch == '0' or ch == '1' or ch == '_':
                    lx.pos += 1
                    lx.col += 1
                else:
                    break
            _emit_number(lx, start, start_lineno, start_col, 0, 0, 0)
            return

    # Decimal integer/float
    while lx.pos < lx.len:
        ch: char = lx.src[lx.pos]
        if _is_digit(ch) or ch == '_':
            lx.pos += 1
            lx.col += 1
        else:
            break

    # Decimal part
    if lx.pos < lx.len and lx.src[lx.pos] == '.':
        is_float = 1
        lx.pos += 1
        lx.col += 1
        while lx.pos < lx.len:
            ch: char = lx.src[lx.pos]
            if _is_digit(ch) or ch == '_':
                lx.pos += 1
                lx.col += 1
            else:
                break

    # Exponent part
    if lx.pos < lx.len:
        ch: char = lx.src[lx.pos]
        if ch == 'e' or ch == 'E':
            is_float = 1
            lx.pos += 1
            lx.col += 1
            if lx.pos < lx.len:
                ch = lx.src[lx.pos]
                if ch == '+' or ch == '-':
                    lx.pos += 1
                    lx.col += 1
            while lx.pos < lx.len:
                ch: char = lx.src[lx.pos]
                if _is_digit(ch) or ch == '_':
                    lx.pos += 1
                    lx.col += 1
                else:
                    break

    # Complex suffix part
    if lx.pos < lx.len and lx.src[lx.pos] == 'j':
        is_complex = 1
        is_float = 1
        lx.pos += 1
        lx.col += 1

    _emit_number(lx, start, start_lineno, start_col, is_float, is_complex, 0)


def _emit_number(lx: rawptr[Lexer], start: sizet,
                 start_lineno: int, start_col: int,
                 is_float: int, is_complex: int, is_hex: int):
    length: sizet = lx.pos - start
    tok: rawptr[Token] = new_token(lx.pool, TokenType.Number, start_lineno, start_col)
    if tok == None: return
    token_set_str(lx.pool, tok, lx.src, start, length)
    tok.is_float = is_float
    tok.is_complex = is_complex
    tok.end_lineno = lx.lineno
    tok.end_col_offset = start_col + int(length)
    # Parse number
    if is_float:
        tok.float_val = string.atof(tok.str_val)
    else:
        # Integer literal magnitude > 2^64-1: lexer fail-fast (pure text, no silent wrap).
        # Range 0..2^64-1 is still stored into int64 as a 64-bit bit pattern (unchanged shape).
        if _lex_int_exceeds_u64(tok.str_val) != 0:
            _lex_record_overflow(tok.str_val, start_lineno, start_col)
        tok.int_val = string.atoll(tok.str_val)
    _emit(lx, tok)


# ============================================================
# String Handling
# ============================================================

def _read_string(lx: rawptr[Lexer]):
    """Read string literal.
    Supports prefixes: r, b, u, f, rb, br, rf, fr (case insensitive)
    Supports quotes: ', ", ''', 
    Later we need to support string prefixes like i16/i24"""
    start_lineno: int = lx.lineno
    start_col: int = lx.col
    prefix_start: sizet = lx.pos

    # Read prefix (r/b/u/f)
    is_raw: int = 0
    is_bytes: int = 0
    is_fstring: int = 0
    while lx.pos < lx.len:
        ch: char = lx.src[lx.pos]
        if ch == 'r' or ch == 'R':
            is_raw = 1
            lx.pos += 1
            lx.col += 1
        elif ch == 'b' or ch == 'B':
            is_bytes = 1
            lx.pos += 1
            lx.col += 1
        elif ch == 'u' or ch == 'U':
            lx.pos += 1
            lx.col += 1
        elif ch == 'f' or ch == 'F':
            is_fstring = 1
            lx.pos += 1
            lx.col += 1
        else:
            break

    if lx.pos >= lx.len:
        lx.error_count += 1
        return

    # Determine quote type
    quote: char = lx.src[lx.pos]
    if quote != '\'' and quote != '"':
        lx.error_count += 1
        return

    # Check triple quote
    is_triple: int = 0
    if lx.pos + 2 < lx.len:
        if lx.src[lx.pos] == quote and lx.src[lx.pos + 1] == quote and lx.src[lx.pos + 2] == quote:
            is_triple = 1

    content_start: sizet = lx.pos
    if is_triple:
        lx.pos += 3
        lx.col += 3
    else:
        lx.pos += 1
        lx.col += 1

    # Read string content
    content_buf_start: sizet = lx.pos
    # f-string {} expression block nesting depth (for PEP 701 same-quote nested strings)
    fstr_depth: int = 0
    while lx.pos < lx.len:
        ch = lx.src[lx.pos]
        if is_triple:
            if ch == quote:
                if lx.pos + 2 < lx.len:
                    if lx.src[lx.pos + 1] == quote and lx.src[lx.pos + 2] == quote:
                        break
                elif lx.pos + 2 == lx.len:
                    break
            if ch == '\\':
                lx.pos += 1
                lx.col += 1
                if lx.pos < lx.len:
                    if lx.src[lx.pos] == '\n':
                        lx.lineno += 1
                        lx.col = 0
                    lx.pos += 1
                    lx.col += 1
                continue
            if ch == '\n':
                lx.lineno += 1
                lx.col = 0
            lx.pos += 1
            lx.col += 1
        else:
            if is_fstring:
                # {{ and }} are escaped literals, not counted in depth
                if ch == '{' and lx.pos + 1 < lx.len and lx.src[lx.pos + 1] == '{':
                    lx.pos += 2
                    lx.col += 2
                    continue
                if ch == '}' and lx.pos + 1 < lx.len and lx.src[lx.pos + 1] == '}':
                    lx.pos += 2
                    lx.col += 2
                    continue
                if ch == '{':
                    fstr_depth += 1
                    lx.pos += 1
                    lx.col += 1
                    continue
                if ch == '}':
                    if fstr_depth > 0:
                        fstr_depth -= 1
                    lx.pos += 1
                    lx.col += 1
                    continue
                if fstr_depth > 0 and (ch == '\'' or ch == '"'):
                    # Nested string literals allowed inside {} blocks (PEP 701 same-quote nesting):
                    # skip to the matching quote; inner quotes do not terminate the outer f-string.
                    nq: char = ch
                    lx.pos += 1
                    lx.col += 1
                    while lx.pos < lx.len:
                        c2: char = lx.src[lx.pos]
                        if c2 == '\\':
                            lx.pos += 2
                            lx.col += 2
                            continue
                        if c2 == nq:
                            lx.pos += 1
                            lx.col += 1
                            break
                        if c2 == '\n':
                            # single-line string cannot span lines
                            lx.error_count += 1
                            return
                        lx.pos += 1
                        lx.col += 1
                    continue
            if ch == quote:
                break
            if ch == '\n':
                # Single quote string cannot span multiple lines (unless continued)
                lx.error_count += 1
                return
            if ch == '\\' and not is_raw:
                lx.pos += 1
                lx.col += 1
                if lx.pos < lx.len:
                    lx.pos += 1
                    lx.col += 1
                continue
            lx.pos += 1
            lx.col += 1

    content_end: sizet = lx.pos
    # Skip the closing quote
    if is_triple:
        if lx.pos + 2 < lx.len:
            lx.pos += 3
            lx.col += 3
        else:
            lx.pos = lx.len
    else:
        if lx.pos < lx.len:
            lx.pos += 1
            lx.col += 1

    # Create token
    tok: rawptr[Token] = new_token(lx.pool, TokenType.String, start_lineno, start_col)
    if tok == None: return
    content_len: sizet = content_end - content_buf_start
    # esc_len: Actual bytes after escaping (used to check single character)
    esc_len: sizet = content_len

    # Handle escape sequences (non-raw strings)
    if is_raw == 0 and content_len > 0:
        # Allocate conversion buffer (max case: each char is plain, no extension)
        conv_buf: str = lx.pool.alloc(content_len + 1)
        if conv_buf == None:
            token_set_str(lx.pool, tok, lx.src, content_buf_start, content_len)
        else:
            wpos: sizet = 0
            rpos: sizet = 0
            while rpos < content_len:
                ch: char = lx.src[content_buf_start + rpos]
                if ch == '\\' and rpos + 1 < content_len:
                    rpos += 1
                    next_ch: char = lx.src[content_buf_start + rpos]
                    if next_ch == 'n':
                        conv_buf[wpos] = '\n'
                    elif next_ch == 't':
                        conv_buf[wpos] = '\t'
                    elif next_ch == 'r':
                        conv_buf[wpos] = '\r'
                    elif next_ch == 'f':
                        conv_buf[wpos] = '\f'
                    elif next_ch == 'v':
                        conv_buf[wpos] = '\v'
                    elif next_ch == 'a':
                        conv_buf[wpos] = '\a'
                    elif next_ch == 'b':
                        conv_buf[wpos] = '\b'
                    elif next_ch == '\\':
                        conv_buf[wpos] = '\\'
                    elif next_ch == '\'':
                        conv_buf[wpos] = '\''
                    elif next_ch == '"':
                        conv_buf[wpos] = '"'
                    elif next_ch == '0':
                        conv_buf[wpos] = '\0'
                    elif next_ch == 'x' and rpos + 2 < content_len:
                        # \xNN hex escape (matches Python): '\\x1b' -> single byte 0x1B.
                        # Must yield esc_len == 1 so '\xNN' becomes CONST_CHAR, not CONST_STR.
                        h1: char = lx.src[content_buf_start + rpos + 1]
                        h2: char = lx.src[content_buf_start + rpos + 2]
                        if _is_hex(h1) != 0 and _is_hex(h2) != 0:
                            hex_v: int = (_hex_val(h1) << 4) | _hex_val(h2)
                            conv_buf[wpos] = char(hex_v)
                            wpos += 1
                            rpos += 3
                            continue
                        # Invalid hex (\x followed by fewer than 2 hex digits): keep as-is
                        conv_buf[wpos] = '\\'
                        wpos += 1
                        conv_buf[wpos] = next_ch
                    else:
                        # Unknown escape sequence, keep original (includes \xNN)
                        conv_buf[wpos] = '\\'
                        wpos += 1
                        conv_buf[wpos] = next_ch
                    wpos += 1
                    rpos += 1
                else:
                    conv_buf[wpos] = ch
                    wpos += 1
                    rpos += 1
            conv_buf[wpos] = '\0'
            token_set_str_literal(lx.pool, tok, conv_buf)
            esc_len = wpos
    else:
        token_set_str(lx.pool, tok, lx.src, content_buf_start, content_len)
    tok.end_lineno = lx.lineno
    tok.end_col_offset = lx.col
    # flags field: bit0=raw, bit1=bytes, bit2=fstring, bit3=triple, bit4=dquote, bit5=single char candidate
    if is_raw: tok.kw_subtype = tok.kw_subtype | 1
    if is_bytes: tok.kw_subtype = tok.kw_subtype | 2
    if is_fstring: tok.kw_subtype = tok.kw_subtype | 4
    if is_triple: tok.kw_subtype = tok.kw_subtype | 8
    # bit4 (16) = double quote=single quote
    if quote == '"': tok.kw_subtype = tok.kw_subtype | 16
    # bit5 (32) = single-character candidate (not triple quotes, not f-string, exactly 1 byte after escaping)
    # Used for CONST_CHAR checks to prevent '0' and other NUL bytes from being mistaken as empty strings by strlen
    if is_triple == 0 and is_fstring == 0 and esc_len == 1:
        tok.kw_subtype = tok.kw_subtype | 32
    _emit(lx, tok)


# ============================================================
# Operator Matching
# ============================================================

def _read_operator(lx: rawptr[Lexer]):
    """Greedy match operator"""
    start_lineno: int = lx.lineno
    start_col: int = lx.col
    remaining: sizet = lx.len - lx.pos

    # Traverse operator table, find longest match
    best_entry: rawptr[OpEntry] = None
    best_len: int = 0
    cur: rawptr[OpEntry] = _op_head
    while cur != None:
        if cur.length <= int(remaining):
            # Compare cur.name with src[pos:pos+cur.length]
            Matched: int = 1
            for i in range(sizet(cur.length)):
                if lx.src[lx.pos + i] != cur.name[i]:
                    Matched = 0
                    break
            if Matched != 0:
                if cur.length > best_len:
                    best_len = cur.length
                    best_entry = cur
        cur = cur.next

    if best_entry == None:
        # Unknown character, report error and skip
        lx.error_count += 1
        lx.pos += 1
        lx.col += 1
        return

    # Update parenthesis depth
    if best_entry.op_id == TokOp.LPar or best_entry.op_id == TokOp.Lsqb or best_entry.op_id == TokOp.LBrace:
        lx.paren_depth += 1
    elif best_entry.op_id == TokOp.RPar or best_entry.op_id == TokOp.Rsqb or best_entry.op_id == TokOp.RBrace:
        if lx.paren_depth > 0:
            lx.paren_depth -= 1

    tok: rawptr[Token] = _make_op_token(lx, best_entry.op_id, best_entry.length,
                                          start_lineno, start_col)
    if tok == None: return
    token_set_str(lx.pool, tok, lx.src, lx.pos, sizet(best_len))
    lx.pos += sizet(best_len)
    lx.col += best_len
    _emit(lx, tok)


# ============================================================
# Main Lexical Analysis Entry Point
# ============================================================

def tokenize(lx: rawptr[Lexer]) -> rawptr[Token]:
    """Execute tokenization, return token list head"""
    if lx == None: return None
    while lx.pos < lx.len:
        # Handle line start indent
        if lx.at_line_start and lx.paren_depth == 0:
            _handle_indent(lx)
            if lx.pos >= lx.len:
                break

        ch: char = _peek_cur(lx)
        if ch == '\0':
            break

        # New line
        if ch == '\n':
            if lx.paren_depth > 0:
                # # In bi brackets, generates TokenType.Nl
                tok: rawptr[Token] = new_token(lx.pool, TokenType.Nl, lx.lineno, lx.col)
                _emit(lx, tok)
            else:
                # Logical new line
                tok: rawptr[Token] = new_token(lx.pool, TokenType.NewLine, lx.lineno, lx.col)
                _emit(lx, tok)
                lx.at_line_start = 1
            lx.pos += 1
            lx.lineno += 1
            lx.col = 0
            continue

        # Backslash continuation line
        # Support LF and CRLF line endings: under CRLF `\` is followed by '\r' (not '\n'),
        # so checking only '\n' fails and the backslash continuation breaks.
        if ch == '\\' and lx.pos + 1 < lx.len:
            if lx.src[lx.pos + 1] == '\n':
                lx.pos += 2
                lx.lineno += 1
                lx.col = 0
                continue
            if lx.src[lx.pos + 1] == '\r':
                # \ + \r\n: skip \, \r, \n
                lx.pos += 2
                if lx.pos < lx.len and lx.src[lx.pos] == '\n':
                    lx.pos += 1
                lx.lineno += 1
                lx.col = 0
                continue

        # Whitespace
        if _is_space(ch):
            lx.pos += 1
            lx.col += 1
            continue

        # Comment
        if ch == '#':
            while lx.pos < lx.len and lx.src[lx.pos] != '\n':
                lx.pos += 1
            continue

        # String prefixes (like r"...") or strings
        if ch == '\'' or ch == '"':
            _read_string(lx)
            continue
        # A string prefix letter followed by a quote (must be checked before _is_alpha, otherwise f/r/b/u will be treated as TokenType.Name)
        if (ch == 'r' or ch == 'R' or ch == 'b' or ch == 'B' or
            ch == 'u' or ch == 'U' or ch == 'f' or ch == 'F'):
            if lx.pos + 1 < lx.len:
                ch1: char = lx.src[lx.pos + 1]
                if ch1 == '\'' or ch1 == '"':
                    _read_string(lx)
                    continue
                # Double prefixes like rb, br, rf, fr
                if lx.pos + 2 < lx.len:
                    ch2: char = lx.src[lx.pos + 2]
                    if ((ch1 == 'r' or ch1 == 'R' or ch1 == 'b' or ch1 == 'B' or
                         ch1 == 'f' or ch1 == 'F' or ch1 == 'u' or ch1 == 'U') and
                        (ch2 == '\'' or ch2 == '"')):
                        _read_string(lx)
                        continue

        # Identifier/keyword
        if _is_alpha(ch):
            _read_name(lx)
            continue

        # Number
        if _is_digit(ch):
            _read_number(lx)
            continue

        # Operator
        _read_operator(lx)

    # End of file: generate TokenType.NewLine (if there's content)
    if lx.tokens_tail != None:
        if lx.tokens_tail.type != TokenType.NewLine and lx.tokens_tail.type != TokenType.Nl:
            tok: rawptr[Token] = new_token(lx.pool, TokenType.NewLine, lx.lineno, lx.col)
            _emit(lx, tok)

    # Generate remaining remaining TokenType.Dedent
    while lx.indent_top > 0:
        lx.indent_top -= 1
        tok: rawptr[Token] = new_token(lx.pool, TokenType.Dedent, lx.lineno, 0)
        _emit(lx, tok)

    # TokenType.EndMarker
    tok: rawptr[Token] = new_token(lx.pool, TokenType.EndMarker, lx.lineno, 0)
    _emit(lx, tok)

    return lx.tokens_head