1192 lines
40 KiB
Plaintext
1192 lines
40 KiB
Plaintext
from stdint import *
|
|
import memhub
|
|
import string
|
|
import stdio
|
|
import stdlib
|
|
from .tokens import (
|
|
Token, TokenType, TokOp, _init_tables, _kw_lookup, _op_head,
|
|
OpEntry, new_token, token_set_str, token_set_str_literal,
|
|
)
|
|
|
|
|
|
# ============================================================
|
|
# Integer literal overflow (lexer-layer fail-fast)
|
|
#
|
|
# Integer constants are int64 in the token/AST (tokens.vp `int_val` / exprs.vp `int_val`).
|
|
# The old implementation used `string.atoll` (string.vp:292) to, under int64, do `num = num * 10 + digit`
|
|
# digit-by-digit accumulation => when the value > 2^64-1 it **silently wraps** (e.g. 2^64 wraps to 0, 2^64-1 wraps to -1).
|
|
# This implementation, in `_emit_number`, first performs a pure-text magnitude check on **integer** literals
|
|
# (`_lex_int_exceeds_u64`, using the same criteria as AssignGenMixin._int_text_exceeds_64: by
|
|
# radix, counting significant digits / comparing length + lexicographic order against the decimal text of 2^64-1, **without landing in any fixed-width
|
|
# container and without wrapping before the comparison**), so exceeding 2^64-1 records a fail-fast (no more silent wrapping).
|
|
# The legal range 0..2^64-1 is still stored into int64 as a 64-bit bit pattern (the shape of the results on the existing legal path is unchanged,
|
|
# e.g. 0xFFFFFFFFFFFFFFFF -> int64 -1, 0xA5A5A5A5A5A5A5A5 keeps its 64-bit bit pattern).
|
|
# The criteria for each radix (2/8/10/16) match TPC's (Python int(x, 0), arbitrary precision): the accepted interval is likewise
|
|
# [-2^63, 2^64-1] (literals themselves are unsigned; the negative side is expressed by unary minus), and anything beyond is rejected.
|
|
#
|
|
# Reporting channel: the lexer has no separate fatal exit, so it records into a module-level triple (`_lex_record_overflow`),
|
|
# which the parser's parse_tokens reads and transcribes into _g_parse_err_* (position + message), reusing the existing
|
|
# Traceback/SyntaxError report path (the "file:line:col + source line" is filled in by that path, which also terminates compilation).
|
|
# This covers all source literals (file-level parsing necessarily goes through parse_tokens); parse_expression is used only for compiler-
|
|
# synthesized short type strings and carries no source literals, so no separate channel is provided.
|
|
# ============================================================
|
|
_g_lex_err_line: int = 0
|
|
_g_lex_err_col: int = 0
|
|
_g_lex_err_msg: str = None
|
|
|
|
|
|
def get_lex_err_line() -> int:
|
|
"""Line number of the latest lexer fail-fast (0 = none).
|
|
|
|
Consumed by parser.parse_tokens and forwarded to the parse error channel.
|
|
"""
|
|
return _g_lex_err_line
|
|
|
|
|
|
def get_lex_err_col() -> int:
|
|
"""Column of the latest lexer fail-fast (0-based; -1 = no column info)"""
|
|
return _g_lex_err_col
|
|
|
|
|
|
def get_lex_err_msg() -> str:
|
|
"""Message of the latest lexer fail-fast (literal / required width / limit / fix hint)"""
|
|
return _g_lex_err_msg
|
|
|
|
|
|
def _lex_reset_err():
|
|
"""Clear the lexer fail-fast triple (called at each tokenize entry to avoid stale errors)"""
|
|
global _g_lex_err_line, _g_lex_err_col, _g_lex_err_msg
|
|
_g_lex_err_line = 0
|
|
_g_lex_err_col = 0
|
|
_g_lex_err_msg = None
|
|
|
|
|
|
def _lex_record_overflow(text: str, lineno: int, col: int):
|
|
"""Record an integer literal overflow (fail-fast; first record only); message carries literal
|
|
+ required width (>= 65 bits, limit 2^64-1) + fix hint (file:line:col filled later by parse_tokens).
|
|
"""
|
|
global _g_lex_err_line, _g_lex_err_col, _g_lex_err_msg
|
|
if _g_lex_err_line != 0:
|
|
return
|
|
_g_lex_err_line = lineno
|
|
_g_lex_err_col = col
|
|
msg_buf: str = stdlib.malloc(512)
|
|
if msg_buf is not None:
|
|
stdio.snprintf(msg_buf, 512,
|
|
"[VPC] integer literal overflow: literal '%s' exceeds the 64-bit representable range"
|
|
" (the bit-pattern interpretation upper bound is 18446744073709551615 = 2^64-1,"
|
|
" the literal needs a width of at least 65 bits)."
|
|
" Silent wrap-around/truncation is forbidden; compilation aborted."
|
|
" Fix: change the literal into the 64-bit bit-pattern range 0..18446744073709551615;"
|
|
" this language has no wider integer type, for larger values split into multiple 64-bit words.", text)
|
|
_g_lex_err_msg = msg_buf
|
|
else:
|
|
_g_lex_err_msg = text
|
|
|
|
|
|
def _lex_int_exceeds_u64(text: str) -> int:
|
|
"""True if the integer literal magnitude exceeds the 64-bit limit 2^64-1 (1 = yes, 0 = no).
|
|
Pure text compare, no wrap-around: decimal length+lexicographic vs "18446744073709551615",
|
|
0x > 16, 0o > 22 (or 22 leading > '1'), 0b > 64 (literal text carries no sign).
|
|
"""
|
|
if text is None:
|
|
return 0
|
|
tl: sizet = string.strlen(text)
|
|
if tl == 0:
|
|
return 0
|
|
# Base detection (prefix letter case-insensitive)
|
|
base: int = 10
|
|
start: sizet = 0
|
|
if tl >= 2 and text[0] == '0':
|
|
pfx: char = text[1]
|
|
if pfx == 'x' or pfx == 'X':
|
|
base = 16
|
|
start = 2
|
|
elif pfx == 'o' or pfx == 'O':
|
|
base = 8
|
|
start = 2
|
|
elif pfx == 'b' or pfx == 'B':
|
|
base = 2
|
|
start = 2
|
|
# Count significant digits (skip '_' and leading zeros) and remember the first one
|
|
sig: int = 0
|
|
first: char = '0'
|
|
j: sizet = start
|
|
while j < tl:
|
|
ch: char = text[j]
|
|
j += 1
|
|
if ch == '_':
|
|
continue
|
|
ok_digit: int = 0
|
|
if base == 16:
|
|
if (ch >= '0' and ch <= '9') or (ch >= 'a' and ch <= 'f') or (ch >= 'A' and ch <= 'F'):
|
|
ok_digit = 1
|
|
elif base == 10:
|
|
if ch >= '0' and ch <= '9':
|
|
ok_digit = 1
|
|
elif base == 8:
|
|
if ch >= '0' and ch <= '7':
|
|
ok_digit = 1
|
|
else:
|
|
if ch == '0' or ch == '1':
|
|
ok_digit = 1
|
|
if ok_digit == 0:
|
|
break
|
|
if sig == 0:
|
|
if ch == '0':
|
|
continue # leading zeros not counted
|
|
first = ch
|
|
sig += 1
|
|
if sig == 0:
|
|
return 0 # value 0
|
|
if base == 16:
|
|
return 1 if sig > 16 else 0
|
|
if base == 2:
|
|
return 1 if sig > 64 else 0
|
|
if base == 8:
|
|
if sig > 22:
|
|
return 1
|
|
if sig == 22 and first > '1':
|
|
return 1
|
|
return 0
|
|
# Decimal: compare significant-digit count against the 2^64-1 text length
|
|
mlen: sizet = 20
|
|
if sig > mlen:
|
|
return 1
|
|
if sig < mlen:
|
|
return 0
|
|
# Equal length (20 digits): lexicographic compare against '18446744073709551615'
|
|
max_text: str = "18446744073709551615"
|
|
d: sizet = 0
|
|
k: sizet = start
|
|
lead: int = 1
|
|
while k < tl and d < mlen:
|
|
c: char = text[k]
|
|
k += 1
|
|
if c == '_':
|
|
continue
|
|
if c < '0' or c > '9':
|
|
return 0
|
|
if lead != 0 and c == '0':
|
|
continue
|
|
lead = 0
|
|
mc: char = max_text[d]
|
|
if c > mc:
|
|
return 1
|
|
if c < mc:
|
|
return 0
|
|
d += 1
|
|
return 0
|
|
|
|
|
|
|
|
def _lex_mag_digit_val(ch: char, base: int) -> int:
|
|
"""Digit char -> positional value (hex accepts a-f/A-F; other bases are '0'..base-1)."""
|
|
if base == 16:
|
|
return _hex_val(ch)
|
|
return int(ch) - 48
|
|
|
|
|
|
def _lex_mag_gt_text(text: str, base: int, start: sizet, sig: int, max_text: str) -> int:
|
|
"""True if the literal's significant digits exceed the same-length max text (1 = yes).
|
|
Both sides skip '_'/leading zeros, compared by positional value (hex via _hex_val); pure text.
|
|
"""
|
|
d: sizet = 0
|
|
k: sizet = start
|
|
tl: sizet = string.strlen(text)
|
|
lead: int = 1
|
|
while k < tl and d < sizet(sig):
|
|
c: char = text[k]
|
|
k += 1
|
|
if c == '_':
|
|
continue
|
|
if base == 16:
|
|
if not ((c >= '0' and c <= '9') or (c >= 'a' and c <= 'f') or (c >= 'A' and c <= 'F')):
|
|
break
|
|
elif base == 10:
|
|
if c < '0' or c > '9':
|
|
break
|
|
elif base == 8:
|
|
if c < '0' or c > '7':
|
|
break
|
|
else:
|
|
if c != '0' and c != '1':
|
|
break
|
|
if lead != 0 and c == '0':
|
|
continue
|
|
lead = 0
|
|
cv: int = _lex_mag_digit_val(c, base)
|
|
mv: int = _lex_mag_digit_val(max_text[d], base)
|
|
if cv > mv:
|
|
return 1
|
|
if cv < mv:
|
|
return 0
|
|
d += 1
|
|
return 0
|
|
|
|
|
|
def _lex_int_neg_exceeds_i64(text: str) -> int:
|
|
"""True if the magnitude is strictly greater than 2^63 (1 = yes).
|
|
For unary-minus + literal overflow: a magnitude-valid literal may become < -2^63 after
|
|
negation; exactly 2^63 is legal (INT64_MIN). Pure text compare, no wrap-around.
|
|
"""
|
|
if text is None:
|
|
return 0
|
|
tl: sizet = string.strlen(text)
|
|
if tl == 0:
|
|
return 0
|
|
base: int = 10
|
|
start: sizet = 0
|
|
if tl >= 2 and text[0] == '0':
|
|
pfx: char = text[1]
|
|
if pfx == 'x' or pfx == 'X':
|
|
base = 16
|
|
start = 2
|
|
elif pfx == 'o' or pfx == 'O':
|
|
base = 8
|
|
start = 2
|
|
elif pfx == 'b' or pfx == 'B':
|
|
base = 2
|
|
start = 2
|
|
sig: int = 0
|
|
j: sizet = start
|
|
while j < tl:
|
|
ch: char = text[j]
|
|
j += 1
|
|
if ch == '_':
|
|
continue
|
|
ok_digit: int = 0
|
|
if base == 16:
|
|
if (ch >= '0' and ch <= '9') or (ch >= 'a' and ch <= 'f') or (ch >= 'A' and ch <= 'F'):
|
|
ok_digit = 1
|
|
elif base == 10:
|
|
if ch >= '0' and ch <= '9':
|
|
ok_digit = 1
|
|
elif base == 8:
|
|
if ch >= '0' and ch <= '7':
|
|
ok_digit = 1
|
|
else:
|
|
if ch == '0' or ch == '1':
|
|
ok_digit = 1
|
|
if ok_digit == 0:
|
|
break
|
|
if sig == 0 and ch == '0':
|
|
continue # leading zeros not counted
|
|
sig += 1
|
|
if sig == 0:
|
|
return 0 # value 0
|
|
if base == 16:
|
|
if sig > 16: return 1
|
|
if sig < 16: return 0
|
|
return _lex_mag_gt_text(text, 16, start, sig, "8000000000000000")
|
|
if base == 2:
|
|
if sig > 64: return 1
|
|
if sig < 64: return 0
|
|
return _lex_mag_gt_text(text, 2, start, sig,
|
|
"1000000000000000000000000000000000000000000000000000000000000000")
|
|
if base == 8:
|
|
if sig > 22: return 1
|
|
if sig < 22: return 0
|
|
return _lex_mag_gt_text(text, 8, start, sig, "1000000000000000000000")
|
|
if sig > 19: return 1
|
|
if sig < 19: return 0
|
|
return _lex_mag_gt_text(text, 10, start, sig, "9223372036854775808")
|
|
|
|
|
|
# Maximum Indentation Stack Depth
|
|
define LEXER_MAX_INDENT = 256
|
|
|
|
|
|
# ============================================================
|
|
# Lexer State Structure
|
|
# ============================================================
|
|
class Lexer:
|
|
src: str # Source code (NUL-terminated)
|
|
pos: sizet # Current position
|
|
len: sizet # Source code length
|
|
lineno: int # Current line number (1-based)
|
|
col: int # Current column number (0-based)
|
|
pool: memhub.MemManager* # Memory pool
|
|
indent_stack: int* # Indent stack (int array, allocated from pool)
|
|
indent_top: int # Stack top (-1 for empty stack)
|
|
paren_depth: int # Paren depth (>0 for no TokenType.NewLine)
|
|
tokens_head: Token* # token list head
|
|
tokens_tail: Token* # token list tail
|
|
at_line_start: int # 1=line start (need indent)
|
|
error_count: int # Error count
|
|
|
|
def __new__(self, pool: memhub.MemManager*):
|
|
ptr: Lexer* = pool.alloc(Lexer.__sizeof__())
|
|
if ptr == None:
|
|
# Pool exhausted: fail-fast instead of returning NULL (downstream would treat NULL as a valid Lexer)
|
|
stdio.printf("[AST] Lexer allocation failed: memory pool exhausted (need=%zu bytes)\n",
|
|
sizet(Lexer.__sizeof__()))
|
|
stdio.fflush(None)
|
|
stdlib.exit(1)
|
|
string.memset(ptr, 0, Lexer.__sizeof__())
|
|
return ptr
|
|
|
|
|
|
def new_lexer(pool: memhub.MemManager*) -> Lexer*:
|
|
"""Allocate from the pool and return a Lexer (cross-module safe wrapper)"""
|
|
return Lexer(pool)
|
|
|
|
|
|
def _lexer_init(lx: Lexer*, src: str, pool: memhub.MemManager*):
|
|
"""Initialize Lexer"""
|
|
if lx == None: return
|
|
lx.src = src
|
|
lx.pos = 0
|
|
lx.len = string.strlen(src) if src != None else 0
|
|
lx.lineno = 1
|
|
lx.col = 0
|
|
lx.pool = pool
|
|
# Allocate indentation stack
|
|
lx.indent_stack = pool.alloc(LEXER_MAX_INDENT * 8)
|
|
if lx.indent_stack == None:
|
|
stdio.printf("[AST] Lexer.indent_stack allocation failed: memory pool exhausted (need=%zu bytes)\n",
|
|
sizet(LEXER_MAX_INDENT * 8))
|
|
stdio.fflush(None)
|
|
stdlib.exit(1)
|
|
string.memset(lx.indent_stack, 0, LEXER_MAX_INDENT * 8)
|
|
lx.indent_top = -1
|
|
# Push 0 (base indent)
|
|
lx.indent_stack[0] = 0
|
|
lx.indent_top = 0
|
|
lx.paren_depth = 0
|
|
lx.tokens_head = None
|
|
lx.tokens_tail = None
|
|
lx.at_line_start = 1
|
|
lx.error_count = 0
|
|
_init_tables(pool)
|
|
|
|
|
|
# ============================================================
|
|
# Auxiliary Functions
|
|
# ============================================================
|
|
|
|
def _peek(lx: Lexer*, offset: sizet) -> char:
|
|
"""Peek character at position pos+offset, return '\0' if out of bounds"""
|
|
p: sizet = lx.pos + offset
|
|
if p >= lx.len: return '\0'
|
|
return lx.src[p]
|
|
|
|
|
|
def _peek_cur(lx: Lexer*) -> char:
|
|
if lx.pos >= lx.len: return '\0'
|
|
return lx.src[lx.pos]
|
|
|
|
|
|
def _advance(lx: Lexer*) -> char:
|
|
"""Advance one character, return the character"""
|
|
if lx.pos >= lx.len: return '\0'
|
|
ch: char = lx.src[lx.pos]
|
|
lx.pos += 1
|
|
if ch == '\n':
|
|
lx.lineno += 1
|
|
lx.col = 0
|
|
else:
|
|
lx.col += 1
|
|
return ch
|
|
|
|
|
|
def _is_alpha(ch: char) -> int:
|
|
if ch >= 'a' and ch <= 'z': return 1
|
|
if ch >= 'A' and ch <= 'Z': return 1
|
|
if ch == '_': return 1
|
|
return 0
|
|
|
|
|
|
def _is_digit(ch: char) -> int:
|
|
if ch >= '0' and ch <= '9': return 1
|
|
return 0
|
|
|
|
|
|
def _is_alnum(ch: char) -> int:
|
|
if _is_alpha(ch): return 1
|
|
if _is_digit(ch): return 1
|
|
return 0
|
|
|
|
|
|
def _is_hex(ch: char) -> int:
|
|
if ch >= '0' and ch <= '9': return 1
|
|
if ch >= 'a' and ch <= 'f': return 1
|
|
if ch >= 'A' and ch <= 'F': return 1
|
|
return 0
|
|
|
|
|
|
def _hex_val(ch: char) -> int:
|
|
"""Hex char -> value 0-15 (for \xNN escape parsing)"""
|
|
if ch >= '0' and ch <= '9': return int(ch) - 48
|
|
if ch >= 'a' and ch <= 'f': return int(ch) - 87
|
|
if ch >= 'A' and ch <= 'F': return int(ch) - 55
|
|
return 0
|
|
|
|
|
|
def _is_space(ch: char) -> int:
|
|
if ch == ' ': return 1
|
|
if ch == '\t': return 1
|
|
if ch == '\f': return 1
|
|
if ch == '\v': return 1
|
|
if ch == '\r': return 1
|
|
return 0
|
|
|
|
|
|
def _emit(lx: Lexer*, tok: Token*):
|
|
"""Append token to token list"""
|
|
if tok == None: return
|
|
if lx.tokens_head == None:
|
|
lx.tokens_head = tok
|
|
lx.tokens_tail = tok
|
|
else:
|
|
lx.tokens_tail.next = tok
|
|
lx.tokens_tail = tok
|
|
|
|
|
|
def _make_op_token(lx: Lexer*, op_id: int, length: int,
|
|
lineno: int, col: int) -> Token*:
|
|
tok: Token* = new_token(lx.pool, TokenType.Op, lineno, col)
|
|
if tok == None: return None
|
|
tok.op_subtype = op_id
|
|
tok.end_col_offset = col + length
|
|
return tok
|
|
|
|
|
|
# ============================================================
|
|
# Indent Handling
|
|
# ============================================================
|
|
|
|
def _handle_indent(lx: Lexer*):
|
|
"""Handle line start indent, generate TokenType.Indent/TokenType.Dedent.
|
|
Skip empty lines and comment lines."""
|
|
# Skip empty lines and comment lines
|
|
while lx.pos < lx.len:
|
|
# Save line start position for non-empty lines
|
|
line_start_pos: sizet = lx.pos
|
|
line_start_col: int = lx.col
|
|
# Skip whitespace, not across lines
|
|
while lx.pos < lx.len:
|
|
ch: char = lx.src[lx.pos]
|
|
if ch == ' ' or ch == '\t' or ch == '\f' or ch == '\v':
|
|
lx.pos += 1
|
|
lx.col += 1
|
|
elif ch == '\r':
|
|
# CRLF line end \r, skip \n to trigger empty line processing
|
|
lx.pos += 1
|
|
else:
|
|
break
|
|
if lx.pos >= lx.len:
|
|
break
|
|
ch = lx.src[lx.pos]
|
|
if ch == '\n':
|
|
# Empty line, skip
|
|
lx.pos += 1
|
|
lx.lineno += 1
|
|
lx.col = 0
|
|
continue
|
|
if ch == '\r':
|
|
# CRLF line end \r, skip \n to trigger empty line processing
|
|
lx.pos += 1
|
|
if lx.pos < lx.len and lx.src[lx.pos] == '\n':
|
|
lx.pos += 1
|
|
lx.lineno += 1
|
|
lx.col = 0
|
|
continue
|
|
if ch == '#':
|
|
# Comment line, skip to end of line
|
|
while lx.pos < lx.len and lx.src[lx.pos] != '\n':
|
|
lx.pos += 1
|
|
continue
|
|
# Non-empty line: backtrack to line start to allow indent calculation
|
|
lx.pos = line_start_pos
|
|
lx.col = line_start_col
|
|
break
|
|
|
|
if lx.pos >= lx.len:
|
|
# End of file, no indent to handle
|
|
lx.at_line_start = 0
|
|
return
|
|
|
|
# Calculate current line indent
|
|
indent: int = 0
|
|
save_pos: sizet = lx.pos
|
|
save_col: int = lx.col
|
|
while lx.pos < lx.len:
|
|
ch: char = lx.src[lx.pos]
|
|
if ch == ' ':
|
|
indent += 1
|
|
lx.pos += 1
|
|
lx.col += 1
|
|
elif ch == '\t':
|
|
indent += 8 - (indent % 8)
|
|
lx.pos += 1
|
|
lx.col += 1
|
|
else:
|
|
break
|
|
|
|
# If line end (empty line), skip indent calculation
|
|
if lx.pos < lx.len:
|
|
ch = lx.src[lx.pos]
|
|
if ch == '\n' or ch == '\r' or ch == '#':
|
|
# Empty line or comment line, backtrack position to allow loop to handle indent
|
|
lx.pos = save_pos
|
|
lx.col = save_col
|
|
lx.at_line_start = 0
|
|
return
|
|
|
|
# Compare indent with top of indent stack
|
|
top_indent: int = lx.indent_stack[lx.indent_top]
|
|
if indent > top_indent:
|
|
# TokenType.Indent
|
|
if lx.indent_top + 1 >= LEXER_MAX_INDENT:
|
|
lx.error_count += 1
|
|
return
|
|
lx.indent_top += 1
|
|
lx.indent_stack[lx.indent_top] = indent
|
|
tok: Token* = new_token(lx.pool, TokenType.Indent, lx.lineno, 0)
|
|
_emit(lx, tok)
|
|
else:
|
|
while indent < lx.indent_stack[lx.indent_top] and lx.indent_top > 0:
|
|
lx.indent_top -= 1
|
|
tok: Token* = new_token(lx.pool, TokenType.Dedent, lx.lineno, 0)
|
|
_emit(lx, tok)
|
|
lx.at_line_start = 0
|
|
|
|
|
|
# ============================================================
|
|
# Identifier/Keyword Handling
|
|
# ============================================================
|
|
|
|
def _read_name(lx: Lexer*):
|
|
start: sizet = lx.pos
|
|
start_lineno: int = lx.lineno
|
|
start_col: int = lx.col
|
|
while lx.pos < lx.len:
|
|
ch: char = lx.src[lx.pos]
|
|
if _is_alnum(ch):
|
|
lx.pos += 1
|
|
lx.col += 1
|
|
else:
|
|
break
|
|
length: sizet = lx.pos - start
|
|
tok: Token* = new_token(lx.pool, TokenType.Name, start_lineno, start_col)
|
|
if tok == None: return
|
|
token_set_str(lx.pool, tok, lx.src, start, length)
|
|
tok.end_lineno = lx.lineno
|
|
tok.end_col_offset = start_col + int(length)
|
|
# Look up keyword if any
|
|
kw_id: int = _kw_lookup(tok.str_val)
|
|
if kw_id: tok.kw_subtype = kw_id
|
|
_emit(lx, tok)
|
|
|
|
|
|
# ============================================================
|
|
# Number Handling
|
|
# ============================================================
|
|
|
|
def _read_number(lx: Lexer*):
|
|
start: sizet = lx.pos
|
|
start_lineno: int = lx.lineno
|
|
start_col: int = lx.col
|
|
is_float: int = 0
|
|
is_complex: int = 0
|
|
|
|
ch0: char = lx.src[lx.pos]
|
|
# Handle 0x/0o/0b prefix
|
|
if ch0 == '0' and lx.pos + 1 < lx.len:
|
|
ch1: char = lx.src[lx.pos + 1]
|
|
if ch1 == 'x' or ch1 == 'X':
|
|
lx.pos += 2
|
|
lx.col += 2
|
|
while lx.pos < lx.len:
|
|
ch: char = lx.src[lx.pos]
|
|
if _is_hex(ch) or ch == '_':
|
|
lx.pos += 1
|
|
lx.col += 1
|
|
else:
|
|
break
|
|
_emit_number(lx, start, start_lineno, start_col, 0, 0, 0)
|
|
return
|
|
if ch1 == 'o' or ch1 == 'O':
|
|
lx.pos += 2
|
|
lx.col += 2
|
|
while lx.pos < lx.len:
|
|
ch: char = lx.src[lx.pos]
|
|
if ch >= '0' and ch <= '7':
|
|
lx.pos += 1
|
|
lx.col += 1
|
|
elif ch == '_':
|
|
lx.pos += 1
|
|
lx.col += 1
|
|
else:
|
|
break
|
|
_emit_number(lx, start, start_lineno, start_col, 0, 0, 0)
|
|
return
|
|
if ch1 == 'b' or ch1 == 'B':
|
|
lx.pos += 2
|
|
lx.col += 2
|
|
while lx.pos < lx.len:
|
|
ch: char = lx.src[lx.pos]
|
|
if ch == '0' or ch == '1' or ch == '_':
|
|
lx.pos += 1
|
|
lx.col += 1
|
|
else:
|
|
break
|
|
_emit_number(lx, start, start_lineno, start_col, 0, 0, 0)
|
|
return
|
|
|
|
# Decimal integer/float
|
|
while lx.pos < lx.len:
|
|
ch: char = lx.src[lx.pos]
|
|
if _is_digit(ch) or ch == '_':
|
|
lx.pos += 1
|
|
lx.col += 1
|
|
else:
|
|
break
|
|
|
|
# Decimal part
|
|
if lx.pos < lx.len and lx.src[lx.pos] == '.':
|
|
is_float = 1
|
|
lx.pos += 1
|
|
lx.col += 1
|
|
while lx.pos < lx.len:
|
|
ch: char = lx.src[lx.pos]
|
|
if _is_digit(ch) or ch == '_':
|
|
lx.pos += 1
|
|
lx.col += 1
|
|
else:
|
|
break
|
|
|
|
# Exponent part
|
|
if lx.pos < lx.len:
|
|
ch: char = lx.src[lx.pos]
|
|
if ch == 'e' or ch == 'E':
|
|
is_float = 1
|
|
lx.pos += 1
|
|
lx.col += 1
|
|
if lx.pos < lx.len:
|
|
ch = lx.src[lx.pos]
|
|
if ch == '+' or ch == '-':
|
|
lx.pos += 1
|
|
lx.col += 1
|
|
while lx.pos < lx.len:
|
|
ch: char = lx.src[lx.pos]
|
|
if _is_digit(ch) or ch == '_':
|
|
lx.pos += 1
|
|
lx.col += 1
|
|
else:
|
|
break
|
|
|
|
# Complex suffix part
|
|
if lx.pos < lx.len and lx.src[lx.pos] == 'j':
|
|
is_complex = 1
|
|
is_float = 1
|
|
lx.pos += 1
|
|
lx.col += 1
|
|
|
|
_emit_number(lx, start, start_lineno, start_col, is_float, is_complex, 0)
|
|
|
|
|
|
# _suffix_is - 源码位置 p 处是否以 s 开头（长度 n，带边界检查）
|
|
def _suffix_is(lx: Lexer*, p: sizet, s: str, n: int) -> int:
|
|
"""Bound-checked prefix compare of lx.src at p against s (n chars); 1=match / 0=no."""
|
|
if p + sizet(n) > lx.len: return 0
|
|
return 1 if string.strncmp(lx.src + p, s, n) == 0 else 0
|
|
|
|
|
|
# _read_width_suffix - 解析整型字面量的显式宽度后缀（_int<N>/_uint<N>/_i<N>/_u<N>）。
|
|
# 命中则推进 lx.pos/col，返回 +N（有符号）/ -N（无符号）；未命中返回 0 且不动位置。
|
|
# N 取 1..64（字面量值本身是 64 位）；后缀之后不得直接跟标识符字符。
|
|
def _read_width_suffix(lx: Lexer*) -> int:
|
|
"""Explicit integer width suffix: +N signed / -N unsigned / 0 none (advances on hit)."""
|
|
if lx.pos >= lx.len: return 0
|
|
if lx.src[lx.pos] != '_': return 0
|
|
p: sizet = lx.pos + 1
|
|
is_u: int = 0
|
|
if _suffix_is(lx, p, "uint", 4) != 0:
|
|
p += 4
|
|
is_u = 1
|
|
elif _suffix_is(lx, p, "int", 3) != 0:
|
|
p += 3
|
|
elif _suffix_is(lx, p, "u", 1) != 0:
|
|
p += 1
|
|
is_u = 1
|
|
elif _suffix_is(lx, p, "i", 1) != 0:
|
|
p += 1
|
|
else:
|
|
return 0
|
|
nbits: int = 0
|
|
ndig: int = 0
|
|
while p < lx.len:
|
|
ch: char = lx.src[p]
|
|
if ch < '0' or ch > '9': break
|
|
nbits = nbits * 10 + (int(ch) - 48)
|
|
ndig += 1
|
|
p += 1
|
|
if ndig == 0 or nbits <= 0 or nbits > 64: return 0
|
|
if p < lx.len:
|
|
ch2: char = lx.src[p]
|
|
if (ch2 >= '0' and ch2 <= '9') or (ch2 >= 'a' and ch2 <= 'z') \
|
|
or (ch2 >= 'A' and ch2 <= 'Z') or ch2 == '_':
|
|
return 0
|
|
adv: sizet = p - lx.pos
|
|
lx.pos = p
|
|
lx.col += int(adv)
|
|
if is_u != 0: return -nbits
|
|
return nbits
|
|
|
|
|
|
def _emit_number(lx: Lexer*, start: sizet,
|
|
start_lineno: int, start_col: int,
|
|
is_float: int, is_complex: int, is_hex: int):
|
|
# 显式宽度后缀（仅整型字面量）：_int<N>/_uint<N>/_i<N>/_u<N>
|
|
int_w: int = 0
|
|
int_u: int = 0
|
|
if is_float == 0 and is_complex == 0:
|
|
sw: int = _read_width_suffix(lx)
|
|
if sw > 0: int_w = sw
|
|
elif sw < 0:
|
|
int_w = -sw
|
|
int_u = 1
|
|
length: sizet = lx.pos - start
|
|
tok: Token* = new_token(lx.pool, TokenType.Number, start_lineno, start_col)
|
|
if tok == None: return
|
|
token_set_str(lx.pool, tok, lx.src, start, length)
|
|
tok.is_float = is_float
|
|
tok.is_complex = is_complex
|
|
tok.int_width = int_w
|
|
tok.int_unsigned = int_u
|
|
tok.end_lineno = lx.lineno
|
|
tok.end_col_offset = start_col + int(length)
|
|
# Parse number
|
|
if is_float:
|
|
tok.float_val = string.atof(tok.str_val)
|
|
else:
|
|
# Integer literal magnitude > 2^64-1: lexer fail-fast (pure text, no silent wrap).
|
|
# Range 0..2^64-1 is still stored into int64 as a 64-bit bit pattern (unchanged shape).
|
|
if _lex_int_exceeds_u64(tok.str_val) != 0:
|
|
_lex_record_overflow(tok.str_val, start_lineno, start_col)
|
|
tok.int_val = string.atoll(tok.str_val)
|
|
_emit(lx, tok)
|
|
|
|
|
|
# ============================================================
|
|
# String Handling
|
|
# ============================================================
|
|
|
|
def _read_string(lx: Lexer*):
|
|
"""Read string literal.
|
|
Supports prefixes: r, b, u, f, rb, br, rf, fr (case insensitive)
|
|
Supports quotes: ', ", ''',
|
|
Later we need to support string prefixes like i16/i24"""
|
|
start_lineno: int = lx.lineno
|
|
start_col: int = lx.col
|
|
prefix_start: sizet = lx.pos
|
|
|
|
# Read prefix (r/b/u/f)
|
|
is_raw: int = 0
|
|
is_bytes: int = 0
|
|
is_fstring: int = 0
|
|
while lx.pos < lx.len:
|
|
ch: char = lx.src[lx.pos]
|
|
if ch == 'r' or ch == 'R':
|
|
is_raw = 1
|
|
lx.pos += 1
|
|
lx.col += 1
|
|
elif ch == 'b' or ch == 'B':
|
|
is_bytes = 1
|
|
lx.pos += 1
|
|
lx.col += 1
|
|
elif ch == 'u' or ch == 'U':
|
|
lx.pos += 1
|
|
lx.col += 1
|
|
elif ch == 'f' or ch == 'F':
|
|
is_fstring = 1
|
|
lx.pos += 1
|
|
lx.col += 1
|
|
else:
|
|
break
|
|
|
|
if lx.pos >= lx.len:
|
|
lx.error_count += 1
|
|
return
|
|
|
|
# Determine quote type
|
|
quote: char = lx.src[lx.pos]
|
|
if quote != '\'' and quote != '"':
|
|
lx.error_count += 1
|
|
return
|
|
|
|
# Check triple quote
|
|
is_triple: int = 0
|
|
if lx.pos + 2 < lx.len:
|
|
if lx.src[lx.pos] == quote and lx.src[lx.pos + 1] == quote and lx.src[lx.pos + 2] == quote:
|
|
is_triple = 1
|
|
|
|
content_start: sizet = lx.pos
|
|
if is_triple:
|
|
lx.pos += 3
|
|
lx.col += 3
|
|
else:
|
|
lx.pos += 1
|
|
lx.col += 1
|
|
|
|
# Read string content
|
|
content_buf_start: sizet = lx.pos
|
|
# f-string {} expression block nesting depth (for PEP 701 same-quote nested strings)
|
|
fstr_depth: int = 0
|
|
while lx.pos < lx.len:
|
|
ch = lx.src[lx.pos]
|
|
if is_triple:
|
|
if ch == quote:
|
|
if lx.pos + 2 < lx.len:
|
|
if lx.src[lx.pos + 1] == quote and lx.src[lx.pos + 2] == quote:
|
|
break
|
|
elif lx.pos + 2 == lx.len:
|
|
break
|
|
if ch == '\\':
|
|
lx.pos += 1
|
|
lx.col += 1
|
|
if lx.pos < lx.len:
|
|
if lx.src[lx.pos] == '\n':
|
|
lx.lineno += 1
|
|
lx.col = 0
|
|
lx.pos += 1
|
|
lx.col += 1
|
|
continue
|
|
if ch == '\n':
|
|
lx.lineno += 1
|
|
lx.col = 0
|
|
lx.pos += 1
|
|
lx.col += 1
|
|
else:
|
|
if is_fstring:
|
|
# {{ and }} are escaped literals, not counted in depth
|
|
if ch == '{' and lx.pos + 1 < lx.len and lx.src[lx.pos + 1] == '{':
|
|
lx.pos += 2
|
|
lx.col += 2
|
|
continue
|
|
if ch == '}' and lx.pos + 1 < lx.len and lx.src[lx.pos + 1] == '}':
|
|
lx.pos += 2
|
|
lx.col += 2
|
|
continue
|
|
if ch == '{':
|
|
fstr_depth += 1
|
|
lx.pos += 1
|
|
lx.col += 1
|
|
continue
|
|
if ch == '}':
|
|
if fstr_depth > 0:
|
|
fstr_depth -= 1
|
|
lx.pos += 1
|
|
lx.col += 1
|
|
continue
|
|
if fstr_depth > 0 and (ch == '\'' or ch == '"'):
|
|
# Nested string literals allowed inside {} blocks (PEP 701 same-quote nesting):
|
|
# skip to the matching quote; inner quotes do not terminate the outer f-string.
|
|
nq: char = ch
|
|
lx.pos += 1
|
|
lx.col += 1
|
|
while lx.pos < lx.len:
|
|
c2: char = lx.src[lx.pos]
|
|
if c2 == '\\':
|
|
lx.pos += 2
|
|
lx.col += 2
|
|
continue
|
|
if c2 == nq:
|
|
lx.pos += 1
|
|
lx.col += 1
|
|
break
|
|
if c2 == '\n':
|
|
# single-line string cannot span lines
|
|
lx.error_count += 1
|
|
return
|
|
lx.pos += 1
|
|
lx.col += 1
|
|
continue
|
|
if ch == quote:
|
|
break
|
|
if ch == '\n':
|
|
# Single quote string cannot span multiple lines (unless continued)
|
|
lx.error_count += 1
|
|
return
|
|
if ch == '\\' and not is_raw:
|
|
lx.pos += 1
|
|
lx.col += 1
|
|
if lx.pos < lx.len:
|
|
lx.pos += 1
|
|
lx.col += 1
|
|
continue
|
|
lx.pos += 1
|
|
lx.col += 1
|
|
|
|
content_end: sizet = lx.pos
|
|
# Skip the closing quote
|
|
if is_triple:
|
|
if lx.pos + 2 < lx.len:
|
|
lx.pos += 3
|
|
lx.col += 3
|
|
else:
|
|
lx.pos = lx.len
|
|
else:
|
|
if lx.pos < lx.len:
|
|
lx.pos += 1
|
|
lx.col += 1
|
|
|
|
# Create token
|
|
tok: Token* = new_token(lx.pool, TokenType.String, start_lineno, start_col)
|
|
if tok == None: return
|
|
content_len: sizet = content_end - content_buf_start
|
|
# esc_len: Actual bytes after escaping (used to check single character)
|
|
esc_len: sizet = content_len
|
|
|
|
# Handle escape sequences (non-raw strings)
|
|
if is_raw == 0 and content_len > 0:
|
|
# Allocate conversion buffer (max case: each char is plain, no extension)
|
|
conv_buf: str = lx.pool.alloc(content_len + 1)
|
|
if conv_buf == None:
|
|
token_set_str(lx.pool, tok, lx.src, content_buf_start, content_len)
|
|
else:
|
|
wpos: sizet = 0
|
|
rpos: sizet = 0
|
|
while rpos < content_len:
|
|
ch: char = lx.src[content_buf_start + rpos]
|
|
if ch == '\\' and rpos + 1 < content_len:
|
|
rpos += 1
|
|
next_ch: char = lx.src[content_buf_start + rpos]
|
|
if next_ch == 'n':
|
|
conv_buf[wpos] = '\n'
|
|
elif next_ch == 't':
|
|
conv_buf[wpos] = '\t'
|
|
elif next_ch == 'r':
|
|
conv_buf[wpos] = '\r'
|
|
elif next_ch == 'f':
|
|
conv_buf[wpos] = '\f'
|
|
elif next_ch == 'v':
|
|
conv_buf[wpos] = '\v'
|
|
elif next_ch == 'a':
|
|
conv_buf[wpos] = '\a'
|
|
elif next_ch == 'b':
|
|
conv_buf[wpos] = '\b'
|
|
elif next_ch == '\\':
|
|
conv_buf[wpos] = '\\'
|
|
elif next_ch == '\'':
|
|
conv_buf[wpos] = '\''
|
|
elif next_ch == '"':
|
|
conv_buf[wpos] = '"'
|
|
elif next_ch == '0':
|
|
conv_buf[wpos] = '\0'
|
|
elif next_ch == 'x' and rpos + 2 < content_len:
|
|
# \xNN hex escape (matches Python): '\\x1b' -> single byte 0x1B.
|
|
# Must yield esc_len == 1 so '\xNN' becomes CONST_CHAR, not CONST_STR.
|
|
h1: char = lx.src[content_buf_start + rpos + 1]
|
|
h2: char = lx.src[content_buf_start + rpos + 2]
|
|
if _is_hex(h1) != 0 and _is_hex(h2) != 0:
|
|
hex_v: int = (_hex_val(h1) << 4) | _hex_val(h2)
|
|
conv_buf[wpos] = char(hex_v)
|
|
wpos += 1
|
|
rpos += 3
|
|
continue
|
|
# Invalid hex (\x followed by fewer than 2 hex digits): keep as-is
|
|
conv_buf[wpos] = '\\'
|
|
wpos += 1
|
|
conv_buf[wpos] = next_ch
|
|
else:
|
|
# Unknown escape sequence, keep original (includes \xNN)
|
|
conv_buf[wpos] = '\\'
|
|
wpos += 1
|
|
conv_buf[wpos] = next_ch
|
|
wpos += 1
|
|
rpos += 1
|
|
else:
|
|
conv_buf[wpos] = ch
|
|
wpos += 1
|
|
rpos += 1
|
|
conv_buf[wpos] = '\0'
|
|
token_set_str_literal(lx.pool, tok, conv_buf)
|
|
esc_len = wpos
|
|
else:
|
|
token_set_str(lx.pool, tok, lx.src, content_buf_start, content_len)
|
|
tok.end_lineno = lx.lineno
|
|
tok.end_col_offset = lx.col
|
|
# flags field: bit0=raw, bit1=bytes, bit2=fstring, bit3=triple, bit4=dquote, bit5=single char candidate
|
|
if is_raw: tok.kw_subtype = tok.kw_subtype | 1
|
|
if is_bytes: tok.kw_subtype = tok.kw_subtype | 2
|
|
if is_fstring: tok.kw_subtype = tok.kw_subtype | 4
|
|
if is_triple: tok.kw_subtype = tok.kw_subtype | 8
|
|
# bit4 (16) = double quote=single quote
|
|
if quote == '"': tok.kw_subtype = tok.kw_subtype | 16
|
|
# bit5 (32) = single-character candidate (not triple quotes, not f-string, exactly 1 byte after escaping)
|
|
# Used for CONST_CHAR checks to prevent '0' and other NUL bytes from being mistaken as empty strings by strlen
|
|
if is_triple == 0 and is_fstring == 0 and esc_len == 1:
|
|
tok.kw_subtype = tok.kw_subtype | 32
|
|
_emit(lx, tok)
|
|
|
|
|
|
# ============================================================
|
|
# Operator Matching
|
|
# ============================================================
|
|
|
|
def _read_operator(lx: Lexer*):
|
|
"""Greedy match operator"""
|
|
start_lineno: int = lx.lineno
|
|
start_col: int = lx.col
|
|
remaining: sizet = lx.len - lx.pos
|
|
|
|
# Traverse operator table, find longest match
|
|
best_entry: OpEntry* = None
|
|
best_len: int = 0
|
|
cur: OpEntry* = _op_head
|
|
while cur != None:
|
|
if cur.length <= int(remaining):
|
|
# Compare cur.name with src[pos:pos+cur.length]
|
|
Matched: int = 1
|
|
for i in range(int(sizet(cur.length))):
|
|
if lx.src[lx.pos + i] != cur.name[i]:
|
|
Matched = 0
|
|
break
|
|
if Matched != 0:
|
|
if cur.length > best_len:
|
|
best_len = cur.length
|
|
best_entry = cur
|
|
cur = cur.next
|
|
|
|
if best_entry == None:
|
|
# Unknown character, report error and skip
|
|
lx.error_count += 1
|
|
lx.pos += 1
|
|
lx.col += 1
|
|
return
|
|
|
|
# Update parenthesis depth
|
|
if best_entry.op_id == TokOp.LPar or best_entry.op_id == TokOp.Lsqb or best_entry.op_id == TokOp.LBrace:
|
|
lx.paren_depth += 1
|
|
elif best_entry.op_id == TokOp.RPar or best_entry.op_id == TokOp.Rsqb or best_entry.op_id == TokOp.RBrace:
|
|
if lx.paren_depth > 0:
|
|
lx.paren_depth -= 1
|
|
|
|
tok: Token* = _make_op_token(lx, best_entry.op_id, best_entry.length,
|
|
start_lineno, start_col)
|
|
if tok == None: return
|
|
token_set_str(lx.pool, tok, lx.src, lx.pos, sizet(best_len))
|
|
lx.pos += sizet(best_len)
|
|
lx.col += best_len
|
|
_emit(lx, tok)
|
|
|
|
|
|
# ============================================================
|
|
# Main Lexical Analysis Entry Point
|
|
# ============================================================
|
|
|
|
def tokenize(lx: Lexer*) -> Token*:
|
|
"""Execute tokenization, return token list head"""
|
|
if lx == None: return None
|
|
while lx.pos < lx.len:
|
|
# Handle line start indent
|
|
if lx.at_line_start and lx.paren_depth == 0:
|
|
_handle_indent(lx)
|
|
if lx.pos >= lx.len:
|
|
break
|
|
|
|
ch: char = _peek_cur(lx)
|
|
if ch == '\0':
|
|
break
|
|
|
|
# New line
|
|
if ch == '\n':
|
|
if lx.paren_depth > 0:
|
|
# # In bi brackets, generates TokenType.Nl
|
|
tok: Token* = new_token(lx.pool, TokenType.Nl, lx.lineno, lx.col)
|
|
_emit(lx, tok)
|
|
else:
|
|
# Logical new line
|
|
tok: Token* = new_token(lx.pool, TokenType.NewLine, lx.lineno, lx.col)
|
|
_emit(lx, tok)
|
|
lx.at_line_start = 1
|
|
lx.pos += 1
|
|
lx.lineno += 1
|
|
lx.col = 0
|
|
continue
|
|
|
|
# Backslash continuation line
|
|
# Support LF and CRLF line endings: under CRLF `\` is followed by '\r' (not '\n'),
|
|
# so checking only '\n' fails and the backslash continuation breaks.
|
|
if ch == '\\' and lx.pos + 1 < lx.len:
|
|
if lx.src[lx.pos + 1] == '\n':
|
|
lx.pos += 2
|
|
lx.lineno += 1
|
|
lx.col = 0
|
|
continue
|
|
if lx.src[lx.pos + 1] == '\r':
|
|
# \ + \r\n: skip \, \r, \n
|
|
lx.pos += 2
|
|
if lx.pos < lx.len and lx.src[lx.pos] == '\n':
|
|
lx.pos += 1
|
|
lx.lineno += 1
|
|
lx.col = 0
|
|
continue
|
|
|
|
# Whitespace
|
|
if _is_space(ch):
|
|
lx.pos += 1
|
|
lx.col += 1
|
|
continue
|
|
|
|
# Comment
|
|
if ch == '#':
|
|
while lx.pos < lx.len and lx.src[lx.pos] != '\n':
|
|
lx.pos += 1
|
|
continue
|
|
|
|
# String prefixes (like r"...") or strings
|
|
if ch == '\'' or ch == '"':
|
|
_read_string(lx)
|
|
continue
|
|
# A string prefix letter followed by a quote (must be checked before _is_alpha, otherwise f/r/b/u will be treated as TokenType.Name)
|
|
if (ch == 'r' or ch == 'R' or ch == 'b' or ch == 'B' or
|
|
ch == 'u' or ch == 'U' or ch == 'f' or ch == 'F'):
|
|
if lx.pos + 1 < lx.len:
|
|
ch1: char = lx.src[lx.pos + 1]
|
|
if ch1 == '\'' or ch1 == '"':
|
|
_read_string(lx)
|
|
continue
|
|
# Double prefixes like rb, br, rf, fr
|
|
if lx.pos + 2 < lx.len:
|
|
ch2: char = lx.src[lx.pos + 2]
|
|
if ((ch1 == 'r' or ch1 == 'R' or ch1 == 'b' or ch1 == 'B' or
|
|
ch1 == 'f' or ch1 == 'F' or ch1 == 'u' or ch1 == 'U') and
|
|
(ch2 == '\'' or ch2 == '"')):
|
|
_read_string(lx)
|
|
continue
|
|
|
|
# Identifier/keyword
|
|
if _is_alpha(ch):
|
|
_read_name(lx)
|
|
continue
|
|
|
|
# Number
|
|
if _is_digit(ch):
|
|
_read_number(lx)
|
|
continue
|
|
|
|
# Operator
|
|
_read_operator(lx)
|
|
|
|
# End of file: generate TokenType.NewLine (if there's content)
|
|
if lx.tokens_tail != None:
|
|
if lx.tokens_tail.type != TokenType.NewLine and lx.tokens_tail.type != TokenType.Nl:
|
|
tok: Token* = new_token(lx.pool, TokenType.NewLine, lx.lineno, lx.col)
|
|
_emit(lx, tok)
|
|
|
|
# Generate remaining remaining TokenType.Dedent
|
|
while lx.indent_top > 0:
|
|
lx.indent_top -= 1
|
|
tok: Token* = new_token(lx.pool, TokenType.Dedent, lx.lineno, 0)
|
|
_emit(lx, tok)
|
|
|
|
# TokenType.EndMarker
|
|
tok: Token* = new_token(lx.pool, TokenType.EndMarker, lx.lineno, 0)
|
|
_emit(lx, tok)
|
|
|
|
return lx.tokens_head
|