774 lines
25 KiB
Plaintext
774 lines
25 KiB
Plaintext
import t, c
|
||
from stdint import *
|
||
import memhub
|
||
import string
|
||
from .tokens import (
|
||
Token, TokenType, TokOp, _init_tables, _kw_lookup, _op_head,
|
||
OpEntry, new_token, token_set_str, token_set_str_literal,
|
||
)
|
||
|
||
|
||
# Maximum Indentation Stack Depth
|
||
LEXER_MAX_INDENT: t.CDefine = 256
|
||
|
||
|
||
# ============================================================
|
||
# Lexer State Structure
|
||
# ============================================================
|
||
class Lexer:
|
||
src: str # Source code (NUL-terminated)
|
||
pos: t.CSizeT # Current position
|
||
len: t.CSizeT # Source code length
|
||
lineno: t.CInt # Current line number (1-based)
|
||
col: t.CInt # Current column number (0-based)
|
||
pool: memhub.MemManager | t.CPtr # Memory pool
|
||
indent_stack: t.CInt | t.CPtr # Indent stack (int array, allocated from pool)
|
||
indent_top: t.CInt # Stack top (-1 for empty stack)
|
||
paren_depth: t.CInt # Paren depth (>0 for no TokenType.NewLine)
|
||
tokens_head: Token | t.CPtr # token list head
|
||
tokens_tail: Token | t.CPtr # token list tail
|
||
at_line_start: t.CInt # 1=line start (need indent)
|
||
error_count: t.CInt # Error count
|
||
|
||
def __new__(self, pool: memhub.MemManager | t.CPtr):
|
||
ptr: Lexer | t.CPtr = pool.alloc(Lexer.__sizeof__())
|
||
if ptr:
|
||
string.memset(ptr, 0, Lexer.__sizeof__())
|
||
return ptr
|
||
|
||
|
||
def new_lexer(pool: memhub.MemManager | t.CPtr) -> Lexer | t.CPtr:
|
||
"""Allocate from the pool and return a Lexer (cross-module safe wrapper)"""
|
||
return Lexer(pool)
|
||
|
||
|
||
def _lexer_init(lx: Lexer | t.CPtr, src: str,
|
||
pool: memhub.MemManager | t.CPtr):
|
||
"""Initialize Lexer"""
|
||
if lx == None: return
|
||
lx.src = src
|
||
lx.pos = 0
|
||
lx.len = string.strlen(src) if src != None else 0
|
||
lx.lineno = 1
|
||
lx.col = 0
|
||
lx.pool = pool
|
||
# Allocate indentation stack
|
||
lx.indent_stack = pool.alloc(LEXER_MAX_INDENT * 8)
|
||
if lx.indent_stack != None:
|
||
string.memset(lx.indent_stack, 0, LEXER_MAX_INDENT * 8)
|
||
lx.indent_top = -1
|
||
# Push 0 (base indent)
|
||
if lx.indent_stack != None:
|
||
lx.indent_stack[0] = 0
|
||
lx.indent_top = 0
|
||
lx.paren_depth = 0
|
||
lx.tokens_head = None
|
||
lx.tokens_tail = None
|
||
lx.at_line_start = 1
|
||
lx.error_count = 0
|
||
_init_tables(pool)
|
||
|
||
|
||
# ============================================================
|
||
# Auxiliary Functions
|
||
# ============================================================
|
||
|
||
def _peek(lx: Lexer | t.CPtr, offset: t.CSizeT) -> t.CChar:
|
||
"""Peek character at position pos+offset, return '\0' if out of bounds"""
|
||
p: t.CSizeT = lx.pos + offset
|
||
if p >= lx.len:
|
||
return '\0'
|
||
return lx.src[p]
|
||
|
||
|
||
def _peek_cur(lx: Lexer | t.CPtr) -> t.CChar:
|
||
if lx.pos >= lx.len:
|
||
return '\0'
|
||
return lx.src[lx.pos]
|
||
|
||
|
||
def _advance(lx: Lexer | t.CPtr) -> t.CChar:
|
||
"""Advance one character, return the character"""
|
||
if lx.pos >= lx.len:
|
||
return '\0'
|
||
ch: t.CChar = lx.src[lx.pos]
|
||
lx.pos += 1
|
||
if ch == '\n':
|
||
lx.lineno += 1
|
||
lx.col = 0
|
||
else:
|
||
lx.col += 1
|
||
return ch
|
||
|
||
|
||
def _is_alpha(ch: t.CChar) -> t.CInt:
|
||
if ch >= 'a' and ch <= 'z': return 1
|
||
if ch >= 'A' and ch <= 'Z': return 1
|
||
if ch == '_': return 1
|
||
return 0
|
||
|
||
|
||
def _is_digit(ch: t.CChar) -> t.CInt:
|
||
if ch >= '0' and ch <= '9': return 1
|
||
return 0
|
||
|
||
|
||
def _is_alnum(ch: t.CChar) -> t.CInt:
|
||
if _is_alpha(ch): return 1
|
||
if _is_digit(ch): return 1
|
||
return 0
|
||
|
||
|
||
def _is_hex(ch: t.CChar) -> t.CInt:
|
||
if ch >= '0' and ch <= '9': return 1
|
||
if ch >= 'a' and ch <= 'f': return 1
|
||
if ch >= 'A' and ch <= 'F': return 1
|
||
return 0
|
||
|
||
|
||
def _is_space(ch: t.CChar) -> t.CInt:
|
||
if ch == ' ': return 1
|
||
if ch == '\t': return 1
|
||
if ch == '\f': return 1
|
||
if ch == '\v': return 1
|
||
if ch == '\r': return 1
|
||
return 0
|
||
|
||
|
||
def _emit(lx: Lexer | t.CPtr, tok: Token | t.CPtr):
|
||
"""Append token to token list"""
|
||
if tok == None: return
|
||
if lx.tokens_head == None:
|
||
lx.tokens_head = tok
|
||
lx.tokens_tail = tok
|
||
else:
|
||
lx.tokens_tail.next = tok
|
||
lx.tokens_tail = tok
|
||
|
||
|
||
def _make_op_token(lx: Lexer | t.CPtr, op_id: t.CInt, length: t.CInt,
|
||
lineno: t.CInt, col: t.CInt) -> Token | t.CPtr:
|
||
tok: Token | t.CPtr = new_token(lx.pool, TokenType.Op, lineno, col)
|
||
if tok == None: return None
|
||
tok.op_subtype = op_id
|
||
tok.end_col_offset = col + length
|
||
return tok
|
||
|
||
|
||
# ============================================================
|
||
# Indent Handling
|
||
# ============================================================
|
||
|
||
def _handle_indent(lx: Lexer | t.CPtr):
|
||
"""Handle line start indent, generate TokenType.Indent/TokenType.Dedent.
|
||
Skip empty lines and comment lines."""
|
||
# Skip empty lines and comment lines
|
||
while lx.pos < lx.len:
|
||
# Save line start position for non-empty lines
|
||
line_start_pos: t.CSizeT = lx.pos
|
||
line_start_col: t.CInt = lx.col
|
||
# Skip whitespace, not across lines
|
||
while lx.pos < lx.len:
|
||
ch: t.CChar = lx.src[lx.pos]
|
||
if ch == ' ' or ch == '\t' or ch == '\f' or ch == '\v':
|
||
lx.pos += 1
|
||
lx.col += 1
|
||
elif ch == '\r':
|
||
# CRLF line end \r, skip \n to trigger empty line processing
|
||
lx.pos += 1
|
||
else:
|
||
break
|
||
if lx.pos >= lx.len:
|
||
break
|
||
ch = lx.src[lx.pos]
|
||
if ch == '\n':
|
||
# Empty line, skip
|
||
lx.pos += 1
|
||
lx.lineno += 1
|
||
lx.col = 0
|
||
continue
|
||
if ch == '\r':
|
||
# CRLF line end \r, skip \n to trigger empty line processing
|
||
lx.pos += 1
|
||
if lx.pos < lx.len and lx.src[lx.pos] == '\n':
|
||
lx.pos += 1
|
||
lx.lineno += 1
|
||
lx.col = 0
|
||
continue
|
||
if ch == '#':
|
||
# Comment line, skip to end of line
|
||
while lx.pos < lx.len and lx.src[lx.pos] != '\n':
|
||
lx.pos += 1
|
||
continue
|
||
# Non-empty line: backtrack to line start to allow indent calculation
|
||
lx.pos = line_start_pos
|
||
lx.col = line_start_col
|
||
break
|
||
|
||
if lx.pos >= lx.len:
|
||
# End of file, no indent to handle
|
||
lx.at_line_start = 0
|
||
return
|
||
|
||
# Calculate current line indent
|
||
indent: t.CInt = 0
|
||
save_pos: t.CSizeT = lx.pos
|
||
save_col: t.CInt = lx.col
|
||
while lx.pos < lx.len:
|
||
ch: t.CChar = lx.src[lx.pos]
|
||
if ch == ' ':
|
||
indent += 1
|
||
lx.pos += 1
|
||
lx.col += 1
|
||
elif ch == '\t':
|
||
indent += 8 - (indent % 8)
|
||
lx.pos += 1
|
||
lx.col += 1
|
||
else:
|
||
break
|
||
|
||
# If line end (empty line), skip indent calculation
|
||
if lx.pos < lx.len:
|
||
ch = lx.src[lx.pos]
|
||
if ch == '\n' or ch == '\r' or ch == '#':
|
||
# Empty line or comment line, backtrack position to allow loop to handle indent
|
||
lx.pos = save_pos
|
||
lx.col = save_col
|
||
lx.at_line_start = 0
|
||
return
|
||
|
||
# Compare indent with top of indent stack
|
||
top_indent: t.CInt = lx.indent_stack[lx.indent_top]
|
||
if indent > top_indent:
|
||
# TokenType.Indent
|
||
if lx.indent_top + 1 >= LEXER_MAX_INDENT:
|
||
lx.error_count += 1
|
||
return
|
||
lx.indent_top += 1
|
||
lx.indent_stack[lx.indent_top] = indent
|
||
tok: Token | t.CPtr = new_token(lx.pool, TokenType.Indent, lx.lineno, 0)
|
||
_emit(lx, tok)
|
||
else:
|
||
while indent < lx.indent_stack[lx.indent_top] and lx.indent_top > 0:
|
||
lx.indent_top -= 1
|
||
tok: Token | t.CPtr = new_token(lx.pool, TokenType.Dedent, lx.lineno, 0)
|
||
_emit(lx, tok)
|
||
lx.at_line_start = 0
|
||
|
||
|
||
# ============================================================
|
||
# Identifier/Keyword Handling
|
||
# ============================================================
|
||
|
||
def _read_name(lx: Lexer | t.CPtr):
|
||
start: t.CSizeT = lx.pos
|
||
start_lineno: t.CInt = lx.lineno
|
||
start_col: t.CInt = lx.col
|
||
while lx.pos < lx.len:
|
||
ch: t.CChar = lx.src[lx.pos]
|
||
if _is_alnum(ch):
|
||
lx.pos += 1
|
||
lx.col += 1
|
||
else:
|
||
break
|
||
length: t.CSizeT = lx.pos - start
|
||
tok: Token | t.CPtr = new_token(lx.pool, TokenType.Name, start_lineno, start_col)
|
||
if tok == None: return
|
||
token_set_str(lx.pool, tok, lx.src, start, length)
|
||
tok.end_lineno = lx.lineno
|
||
tok.end_col_offset = start_col + t.CInt(length)
|
||
# Look up keyword if any
|
||
kw_id: t.CInt = _kw_lookup(tok.str_val)
|
||
if kw_id:
|
||
tok.kw_subtype = kw_id
|
||
_emit(lx, tok)
|
||
|
||
|
||
# ============================================================
|
||
# Number Handling
|
||
# ============================================================
|
||
|
||
def _read_number(lx: Lexer | t.CPtr):
|
||
start: t.CSizeT = lx.pos
|
||
start_lineno: t.CInt = lx.lineno
|
||
start_col: t.CInt = lx.col
|
||
is_float: t.CInt = 0
|
||
is_complex: t.CInt = 0
|
||
|
||
ch0: t.CChar = lx.src[lx.pos]
|
||
# Handle 0x/0o/0b prefix
|
||
if ch0 == '0' and lx.pos + 1 < lx.len:
|
||
ch1: t.CChar = lx.src[lx.pos + 1]
|
||
if ch1 == 'x' or ch1 == 'X':
|
||
lx.pos += 2
|
||
lx.col += 2
|
||
while lx.pos < lx.len:
|
||
ch: t.CChar = lx.src[lx.pos]
|
||
if _is_hex(ch) or ch == '_':
|
||
lx.pos += 1
|
||
lx.col += 1
|
||
else:
|
||
break
|
||
_emit_number(lx, start, start_lineno, start_col, 0, 0, 0)
|
||
return
|
||
if ch1 == 'o' or ch1 == 'O':
|
||
lx.pos += 2
|
||
lx.col += 2
|
||
while lx.pos < lx.len:
|
||
ch: t.CChar = lx.src[lx.pos]
|
||
if ch >= '0' and ch <= '7':
|
||
lx.pos += 1
|
||
lx.col += 1
|
||
elif ch == '_':
|
||
lx.pos += 1
|
||
lx.col += 1
|
||
else:
|
||
break
|
||
_emit_number(lx, start, start_lineno, start_col, 0, 0, 0)
|
||
return
|
||
if ch1 == 'b' or ch1 == 'B':
|
||
lx.pos += 2
|
||
lx.col += 2
|
||
while lx.pos < lx.len:
|
||
ch: t.CChar = lx.src[lx.pos]
|
||
if ch == '0' or ch == '1' or ch == '_':
|
||
lx.pos += 1
|
||
lx.col += 1
|
||
else:
|
||
break
|
||
_emit_number(lx, start, start_lineno, start_col, 0, 0, 0)
|
||
return
|
||
|
||
# Decimal integer/float
|
||
while lx.pos < lx.len:
|
||
ch: t.CChar = lx.src[lx.pos]
|
||
if _is_digit(ch) or ch == '_':
|
||
lx.pos += 1
|
||
lx.col += 1
|
||
else:
|
||
break
|
||
|
||
# Decimal part
|
||
if lx.pos < lx.len and lx.src[lx.pos] == '.':
|
||
is_float = 1
|
||
lx.pos += 1
|
||
lx.col += 1
|
||
while lx.pos < lx.len:
|
||
ch: t.CChar = lx.src[lx.pos]
|
||
if _is_digit(ch) or ch == '_':
|
||
lx.pos += 1
|
||
lx.col += 1
|
||
else:
|
||
break
|
||
|
||
# Exponent part
|
||
if lx.pos < lx.len:
|
||
ch: t.CChar = lx.src[lx.pos]
|
||
if ch == 'e' or ch == 'E':
|
||
is_float = 1
|
||
lx.pos += 1
|
||
lx.col += 1
|
||
if lx.pos < lx.len:
|
||
ch = lx.src[lx.pos]
|
||
if ch == '+' or ch == '-':
|
||
lx.pos += 1
|
||
lx.col += 1
|
||
while lx.pos < lx.len:
|
||
ch: t.CChar = lx.src[lx.pos]
|
||
if _is_digit(ch) or ch == '_':
|
||
lx.pos += 1
|
||
lx.col += 1
|
||
else:
|
||
break
|
||
|
||
# Complex suffix part
|
||
if lx.pos < lx.len and lx.src[lx.pos] == 'j':
|
||
is_complex = 1
|
||
is_float = 1
|
||
lx.pos += 1
|
||
lx.col += 1
|
||
|
||
_emit_number(lx, start, start_lineno, start_col, is_float, is_complex, 0)
|
||
|
||
|
||
def _emit_number(lx: Lexer | t.CPtr, start: t.CSizeT,
|
||
start_lineno: t.CInt, start_col: t.CInt,
|
||
is_float: t.CInt, is_complex: t.CInt, is_hex: t.CInt):
|
||
length: t.CSizeT = lx.pos - start
|
||
tok: Token | t.CPtr = new_token(lx.pool, TokenType.Number, start_lineno, start_col)
|
||
if tok == None: return
|
||
token_set_str(lx.pool, tok, lx.src, start, length)
|
||
tok.is_float = is_float
|
||
tok.is_complex = is_complex
|
||
tok.end_lineno = lx.lineno
|
||
tok.end_col_offset = start_col + t.CInt(length)
|
||
# Parse number
|
||
if is_float:
|
||
tok.float_val = string.atof(tok.str_val)
|
||
else:
|
||
tok.int_val = string.atoll(tok.str_val)
|
||
_emit(lx, tok)
|
||
|
||
|
||
# ============================================================
|
||
# String Handling
|
||
# ============================================================
|
||
|
||
def _read_string(lx: Lexer | t.CPtr):
|
||
"""Read string literal.
|
||
Supports prefixes: r, b, u, f, rb, br, rf, fr (case insensitive)
|
||
Supports quotes: ', ", ''',
|
||
Later we need to support string prefixes like i16/i24"""
|
||
start_lineno: t.CInt = lx.lineno
|
||
start_col: t.CInt = lx.col
|
||
prefix_start: t.CSizeT = lx.pos
|
||
|
||
# Read prefix (r/b/u/f)
|
||
is_raw: t.CInt = 0
|
||
is_bytes: t.CInt = 0
|
||
is_fstring: t.CInt = 0
|
||
while lx.pos < lx.len:
|
||
ch: t.CChar = lx.src[lx.pos]
|
||
if ch == 'r' or ch == 'R':
|
||
is_raw = 1
|
||
lx.pos += 1
|
||
lx.col += 1
|
||
elif ch == 'b' or ch == 'B':
|
||
is_bytes = 1
|
||
lx.pos += 1
|
||
lx.col += 1
|
||
elif ch == 'u' or ch == 'U':
|
||
lx.pos += 1
|
||
lx.col += 1
|
||
elif ch == 'f' or ch == 'F':
|
||
is_fstring = 1
|
||
lx.pos += 1
|
||
lx.col += 1
|
||
else:
|
||
break
|
||
|
||
if lx.pos >= lx.len:
|
||
lx.error_count += 1
|
||
return
|
||
|
||
# Determine quote type
|
||
quote: t.CChar = lx.src[lx.pos]
|
||
if quote != '\'' and quote != '"':
|
||
lx.error_count += 1
|
||
return
|
||
|
||
# Check triple quote
|
||
is_triple: t.CInt = 0
|
||
if lx.pos + 2 < lx.len:
|
||
if lx.src[lx.pos] == quote and lx.src[lx.pos + 1] == quote and lx.src[lx.pos + 2] == quote:
|
||
is_triple = 1
|
||
|
||
content_start: t.CSizeT = lx.pos
|
||
if is_triple:
|
||
lx.pos += 3
|
||
lx.col += 3
|
||
else:
|
||
lx.pos += 1
|
||
lx.col += 1
|
||
|
||
# Read string content
|
||
content_buf_start: t.CSizeT = lx.pos
|
||
while lx.pos < lx.len:
|
||
ch = lx.src[lx.pos]
|
||
if is_triple:
|
||
if ch == quote:
|
||
if lx.pos + 2 < lx.len:
|
||
if lx.src[lx.pos + 1] == quote and lx.src[lx.pos + 2] == quote:
|
||
break
|
||
elif lx.pos + 2 == lx.len:
|
||
break
|
||
if ch == '\\':
|
||
lx.pos += 1
|
||
lx.col += 1
|
||
if lx.pos < lx.len:
|
||
if lx.src[lx.pos] == '\n':
|
||
lx.lineno += 1
|
||
lx.col = 0
|
||
lx.pos += 1
|
||
lx.col += 1
|
||
continue
|
||
if ch == '\n':
|
||
lx.lineno += 1
|
||
lx.col = 0
|
||
lx.pos += 1
|
||
lx.col += 1
|
||
else:
|
||
if ch == quote:
|
||
break
|
||
if ch == '\n':
|
||
# Single quote string cannot span multiple lines (unless continued)
|
||
lx.error_count += 1
|
||
return
|
||
if ch == '\\' and not is_raw:
|
||
lx.pos += 1
|
||
lx.col += 1
|
||
if lx.pos < lx.len:
|
||
lx.pos += 1
|
||
lx.col += 1
|
||
continue
|
||
lx.pos += 1
|
||
lx.col += 1
|
||
|
||
content_end: t.CSizeT = lx.pos
|
||
# Skip the closing quote
|
||
if is_triple:
|
||
if lx.pos + 2 < lx.len:
|
||
lx.pos += 3
|
||
lx.col += 3
|
||
else:
|
||
lx.pos = lx.len
|
||
else:
|
||
if lx.pos < lx.len:
|
||
lx.pos += 1
|
||
lx.col += 1
|
||
|
||
# Create token
|
||
tok: Token | t.CPtr = new_token(lx.pool, TokenType.String, start_lineno, start_col)
|
||
if tok == None: return
|
||
content_len: t.CSizeT = content_end - content_buf_start
|
||
# esc_len: Actual bytes after escaping (used to check single character)
|
||
esc_len: t.CSizeT = content_len
|
||
|
||
# Handle escape sequences (non-raw strings)
|
||
if is_raw == 0 and content_len > 0:
|
||
# Allocate conversion buffer (max case: each char is plain, no extension)
|
||
conv_buf: str = lx.pool.alloc(content_len + 1)
|
||
if conv_buf == None:
|
||
token_set_str(lx.pool, tok, lx.src, content_buf_start, content_len)
|
||
else:
|
||
wpos: t.CSizeT = 0
|
||
rpos: t.CSizeT = 0
|
||
while rpos < content_len:
|
||
ch: t.CChar = lx.src[content_buf_start + rpos]
|
||
if ch == '\\' and rpos + 1 < content_len:
|
||
rpos += 1
|
||
next_ch: t.CChar = lx.src[content_buf_start + rpos]
|
||
if next_ch == 'n':
|
||
conv_buf[wpos] = '\n'
|
||
elif next_ch == 't':
|
||
conv_buf[wpos] = '\t'
|
||
elif next_ch == 'r':
|
||
conv_buf[wpos] = '\r'
|
||
elif next_ch == 'f':
|
||
conv_buf[wpos] = '\f'
|
||
elif next_ch == 'v':
|
||
conv_buf[wpos] = '\v'
|
||
elif next_ch == 'a':
|
||
conv_buf[wpos] = '\a'
|
||
elif next_ch == 'b':
|
||
conv_buf[wpos] = '\b'
|
||
elif next_ch == '\\':
|
||
conv_buf[wpos] = '\\'
|
||
elif next_ch == '\'':
|
||
conv_buf[wpos] = '\''
|
||
elif next_ch == '"':
|
||
conv_buf[wpos] = '"'
|
||
elif next_ch == '0':
|
||
conv_buf[wpos] = '\0'
|
||
else:
|
||
# Unknown escape sequence, keep original (includes \xNN)
|
||
conv_buf[wpos] = '\\'
|
||
wpos += 1
|
||
conv_buf[wpos] = next_ch
|
||
wpos += 1
|
||
rpos += 1
|
||
else:
|
||
conv_buf[wpos] = ch
|
||
wpos += 1
|
||
rpos += 1
|
||
conv_buf[wpos] = '\0'
|
||
token_set_str_literal(lx.pool, tok, conv_buf)
|
||
esc_len = wpos
|
||
else:
|
||
token_set_str(lx.pool, tok, lx.src, content_buf_start, content_len)
|
||
tok.end_lineno = lx.lineno
|
||
tok.end_col_offset = lx.col
|
||
# flags field: bit0=raw, bit1=bytes, bit2=fstring, bit3=triple, bit4=dquote, bit5=single char candidate
|
||
if is_raw: tok.kw_subtype = tok.kw_subtype | 1
|
||
if is_bytes: tok.kw_subtype = tok.kw_subtype | 2
|
||
if is_fstring: tok.kw_subtype = tok.kw_subtype | 4
|
||
if is_triple: tok.kw_subtype = tok.kw_subtype | 8
|
||
# bit4 (16) = double quote=single quote
|
||
if quote == '"': tok.kw_subtype = tok.kw_subtype | 16
|
||
# bit5 (32) = single-character candidate (not triple quotes, not f-string, exactly 1 byte after escaping)
|
||
# Used for CONST_CHAR checks to prevent '0' and other NUL bytes from being mistaken as empty strings by strlen
|
||
if is_triple == 0 and is_fstring == 0 and esc_len == 1:
|
||
tok.kw_subtype = tok.kw_subtype | 32
|
||
_emit(lx, tok)
|
||
|
||
|
||
# ============================================================
|
||
# Operator Matching
|
||
# ============================================================
|
||
|
||
def _read_operator(lx: Lexer | t.CPtr):
|
||
"""Greedy match operator"""
|
||
start_lineno: t.CInt = lx.lineno
|
||
start_col: t.CInt = lx.col
|
||
remaining: t.CSizeT = lx.len - lx.pos
|
||
|
||
# Traverse operator table, find longest match
|
||
best_entry: OpEntry | t.CPtr = None
|
||
best_len: t.CInt = 0
|
||
cur: OpEntry | t.CPtr = _op_head
|
||
while cur != None:
|
||
if cur.length <= t.CInt(remaining):
|
||
# Compare cur.name with src[pos:pos+cur.length]
|
||
match: t.CInt = 1
|
||
i: t.CSizeT = 0
|
||
while i < t.CSizeT(cur.length):
|
||
if lx.src[lx.pos + i] != cur.name[i]:
|
||
match = 0
|
||
break
|
||
i += 1
|
||
if match:
|
||
if cur.length > best_len:
|
||
best_len = cur.length
|
||
best_entry = cur
|
||
cur = cur.next
|
||
|
||
if best_entry == None:
|
||
# Unknown character, report error and skip
|
||
lx.error_count += 1
|
||
lx.pos += 1
|
||
lx.col += 1
|
||
return
|
||
|
||
# Update parenthesis depth
|
||
if best_entry.op_id == TokOp.LPar or best_entry.op_id == TokOp.Lsqb or best_entry.op_id == TokOp.LBrace:
|
||
lx.paren_depth += 1
|
||
elif best_entry.op_id == TokOp.RPar or best_entry.op_id == TokOp.Rsqb or best_entry.op_id == TokOp.RBrace:
|
||
if lx.paren_depth > 0:
|
||
lx.paren_depth -= 1
|
||
|
||
tok: Token | t.CPtr = _make_op_token(lx, best_entry.op_id, best_entry.length,
|
||
start_lineno, start_col)
|
||
if tok == None: return
|
||
token_set_str(lx.pool, tok, lx.src, lx.pos, t.CSizeT(best_len))
|
||
lx.pos += t.CSizeT(best_len)
|
||
lx.col += best_len
|
||
_emit(lx, tok)
|
||
|
||
|
||
# ============================================================
|
||
# Main Lexical Analysis Entry Point
|
||
# ============================================================
|
||
|
||
def tokenize(lx: Lexer | t.CPtr) -> Token | t.CPtr:
|
||
"""Execute tokenization, return token list head"""
|
||
if lx == None: return None
|
||
while lx.pos < lx.len:
|
||
# Handle line start indent
|
||
if lx.at_line_start and lx.paren_depth == 0:
|
||
_handle_indent(lx)
|
||
if lx.pos >= lx.len:
|
||
break
|
||
|
||
ch: t.CChar = _peek_cur(lx)
|
||
if ch == '\0':
|
||
break
|
||
|
||
# New line
|
||
if ch == '\n':
|
||
if lx.paren_depth > 0:
|
||
# # In bi brackets, generates TokenType.Nl
|
||
tok: Token | t.CPtr = new_token(lx.pool, TokenType.Nl, lx.lineno, lx.col)
|
||
_emit(lx, tok)
|
||
else:
|
||
# Logical new line
|
||
tok: Token | t.CPtr = new_token(lx.pool, TokenType.NewLine, lx.lineno, lx.col)
|
||
_emit(lx, tok)
|
||
lx.at_line_start = 1
|
||
lx.pos += 1
|
||
lx.lineno += 1
|
||
lx.col = 0
|
||
continue
|
||
|
||
# Backslash continuation line
|
||
# 支持 LF 与 CRLF（Windows 文件行尾）：CRLF 下 `\` 后是 '\r'（非 '\n'），
|
||
# 只查 '\n' 会失效 → `\` 被当未知字符跳过、'\n' 产生 NEWLINE → 表达式
|
||
# 中断（VpiReader.vp L324 `or \` 行尾续行根因，CRLF 文件）。
|
||
if ch == '\\' and lx.pos + 1 < lx.len:
|
||
if lx.src[lx.pos + 1] == '\n':
|
||
lx.pos += 2
|
||
lx.lineno += 1
|
||
lx.col = 0
|
||
continue
|
||
if lx.src[lx.pos + 1] == '\r':
|
||
# \ + \r\n：跳 \、\r、\n
|
||
lx.pos += 2
|
||
if lx.pos < lx.len and lx.src[lx.pos] == '\n':
|
||
lx.pos += 1
|
||
lx.lineno += 1
|
||
lx.col = 0
|
||
continue
|
||
|
||
# Whitespace
|
||
if _is_space(ch):
|
||
lx.pos += 1
|
||
lx.col += 1
|
||
continue
|
||
|
||
# Comment
|
||
if ch == '#':
|
||
while lx.pos < lx.len and lx.src[lx.pos] != '\n':
|
||
lx.pos += 1
|
||
continue
|
||
|
||
# String prefixes (like r"...") or strings
|
||
if ch == '\'' or ch == '"':
|
||
_read_string(lx)
|
||
continue
|
||
# A string prefix letter followed by a quote (must be checked before _is_alpha, otherwise f/r/b/u will be treated as TokenType.Name)
|
||
if (ch == 'r' or ch == 'R' or ch == 'b' or ch == 'B' or
|
||
ch == 'u' or ch == 'U' or ch == 'f' or ch == 'F'):
|
||
if lx.pos + 1 < lx.len:
|
||
ch1: t.CChar = lx.src[lx.pos + 1]
|
||
if ch1 == '\'' or ch1 == '"':
|
||
_read_string(lx)
|
||
continue
|
||
# Double prefixes like rb, br, rf, fr
|
||
if lx.pos + 2 < lx.len:
|
||
ch2: t.CChar = lx.src[lx.pos + 2]
|
||
if ((ch1 == 'r' or ch1 == 'R' or ch1 == 'b' or ch1 == 'B' or
|
||
ch1 == 'f' or ch1 == 'F' or ch1 == 'u' or ch1 == 'U') and
|
||
(ch2 == '\'' or ch2 == '"')):
|
||
_read_string(lx)
|
||
continue
|
||
|
||
# Identifier/keyword
|
||
if _is_alpha(ch):
|
||
_read_name(lx)
|
||
continue
|
||
|
||
# Number
|
||
if _is_digit(ch):
|
||
_read_number(lx)
|
||
continue
|
||
|
||
# Operator
|
||
_read_operator(lx)
|
||
|
||
# End of file: generate TokenType.NewLine (if there's content)
|
||
if lx.tokens_tail != None:
|
||
if lx.tokens_tail.type != TokenType.NewLine and lx.tokens_tail.type != TokenType.Nl:
|
||
tok: Token | t.CPtr = new_token(lx.pool, TokenType.NewLine, lx.lineno, lx.col)
|
||
_emit(lx, tok)
|
||
|
||
# Generate remaining remaining TokenType.Dedent
|
||
while lx.indent_top > 0:
|
||
lx.indent_top -= 1
|
||
tok: Token | t.CPtr = new_token(lx.pool, TokenType.Dedent, lx.lineno, 0)
|
||
_emit(lx, tok)
|
||
|
||
# TokenType.EndMarker
|
||
tok: Token | t.CPtr = new_token(lx.pool, TokenType.EndMarker, lx.lineno, 0)
|
||
_emit(lx, tok)
|
||
|
||
return lx.tokens_head
|