405 lines
14 KiB
Plaintext
405 lines
14 KiB
Plaintext
import t, c
|
||
from stdint import *
|
||
import memhub
|
||
import string
|
||
|
||
|
||
# ============================================================
|
||
# Token type enumeration (refer to CPython Lib/token.py)
|
||
# Organized as a type-safe enum using t.CEnum; old names XXX retained as compatible aliases
|
||
# ============================================================
|
||
|
||
class TokenType(t.CEnum):
|
||
EndMarker: t.State
|
||
NewLine: t.State
|
||
Indent: t.State
|
||
Dedent: t.State
|
||
Name: t.State
|
||
Number: t.State
|
||
String: t.State
|
||
FStringStart: t.State
|
||
FStringMiddle: t.State
|
||
FStringEnd: t.State
|
||
Op: t.State
|
||
Nl: t.State
|
||
Comment: t.State
|
||
Encoding: t.State
|
||
|
||
|
||
# ============================================================
|
||
# Keyword enumeration (refer to CPython Lib/keyword.py)
|
||
# Organized as a type-safe enum using t.CEnum; old names KW_XXX are kept as compatible aliases
|
||
# NotKeyword(0) represents a non-keyword (sentinel to avoid conflict between False_=0 and kw_subtype=0)
|
||
# ============================================================
|
||
|
||
class Keyword(t.CEnum):
|
||
NotKeyword: t.State
|
||
False_: t.State
|
||
None_: t.State
|
||
True_: t.State
|
||
And: t.State
|
||
As: t.State
|
||
Assert: t.State
|
||
Async: t.State
|
||
Await: t.State
|
||
Break: t.State
|
||
Class: t.State
|
||
Continue: t.State
|
||
Def: t.State
|
||
Del: t.State
|
||
Elif: t.State
|
||
Else: t.State
|
||
Except: t.State
|
||
Finally: t.State
|
||
For: t.State
|
||
From: t.State
|
||
Global: t.State
|
||
If: t.State
|
||
Import: t.State
|
||
In: t.State
|
||
Is: t.State
|
||
Lambda: t.State
|
||
Nonlocal: t.State
|
||
Not: t.State
|
||
Or: t.State
|
||
Pass: t.State
|
||
Raise: t.State
|
||
Return: t.State
|
||
Try: t.State
|
||
While: t.State
|
||
With: t.State
|
||
Yield: t.State
|
||
Match: t.State
|
||
Case: t.State
|
||
Type: t.State
|
||
# def 的别名关键字（fn/fun/func/function，语义与 def 完全相同）
|
||
Fn: t.State
|
||
Fun: t.State
|
||
Func: t.State
|
||
Function: t.State
|
||
# extern 函数声明关键字（FFI，等价 t.State 标记；软判定见 _is_funcdef_kw）
|
||
Extern: t.State
|
||
|
||
|
||
# ============================================================
|
||
# Operator/Separator Enumeration
|
||
# Organized as a type-safe enum using t.CEnum; old names OP_XXX retained as compatible aliases
|
||
# Stored in the token.op_subtype field
|
||
# ============================================================
|
||
|
||
class TokOp(t.CEnum):
|
||
LPar: t.State # (
|
||
RPar: t.State # )
|
||
Lsqb: t.State # [
|
||
Rsqb: t.State # ]
|
||
LBrace: t.State # {
|
||
RBrace: t.State # }
|
||
Comma: t.State # ,
|
||
Colon: t.State # :
|
||
Dot: t.State # .
|
||
Semi: t.State # ;
|
||
At: t.State # @
|
||
Equal: t.State # =
|
||
RArrow: t.State # ->
|
||
PlusEq: t.State # +=
|
||
MinusEq: t.State # -=
|
||
StarEq: t.State # *=
|
||
SlashEq: t.State # /=
|
||
DSlashEq: t.State # //=
|
||
PercentEq: t.State # %=
|
||
AtEq: t.State # @=
|
||
AmpEq: t.State # &=
|
||
VBarEq: t.State # |=
|
||
CaretEq: t.State # ^=
|
||
GtGtEq: t.State # >>=
|
||
LtLtEq: t.State # <<=
|
||
StarEqEq: t.State # **=
|
||
DSlash: t.State # //
|
||
StarStar: t.State # **
|
||
LtLt: t.State # <<
|
||
GtGt: t.State # >>
|
||
LessEq: t.State # <=
|
||
GreaterEq: t.State # >=
|
||
EqEq: t.State # ==
|
||
ExclaimEq: t.State # !=
|
||
Less: t.State # <
|
||
Greater: t.State # >
|
||
Plus: t.State # +
|
||
Minus: t.State # -
|
||
Star: t.State # *
|
||
Slash: t.State # /
|
||
Percent: t.State # %
|
||
Amp: t.State # &
|
||
VBar: t.State # |
|
||
Caret: t.State # ^
|
||
Tilde: t.State # ~
|
||
ColonEq: t.State # :=
|
||
Ellipsis: t.State # ...
|
||
Bang: t.State # !(Non-standard, used for parsing tolerance)
|
||
|
||
|
||
# ============================================================
|
||
# Token stucture
|
||
# ============================================================
|
||
class Token:
|
||
type: t.CInt # Token type (TokenType.EndMarker / TokenType.NewLine / TokenType.Name / ...)
|
||
op_subtype: t.CInt # Operator subtype (OP_*). Only valid when type == TokenType.Op
|
||
kw_subtype: t.CInt # Keyword subtype (KW_*). Only valid when type == TokenType.Name and it is a keyword
|
||
str_val: str # Token text (NUL‑terminated, allocated from pool)
|
||
int_val: t.CInt64T # Integer value (used by TokenType.Number)
|
||
float_val: t.CDouble # Floating‑point value (used by TokenType.Number)
|
||
is_float: t.CInt # 1 = floating‑point number, 0 = integer
|
||
is_complex: t.CInt # 1 = complex number, 0 = real number
|
||
lineno: t.CInt # Starting line number
|
||
col_offset: t.CInt # Starting column offset
|
||
end_lineno: t.CInt # Ending line number
|
||
end_col_offset: t.CInt # Ending column offset
|
||
next: Token | t.CPtr # Next token in linked list
|
||
|
||
def __new__(self, pool: memhub.MemManager | t.CPtr):
|
||
"""Allocate zero‑initialized Token struct memory from mpool"""
|
||
ptr: Token | t.CPtr = pool.alloc(Token.__sizeof__())
|
||
if ptr:
|
||
string.memset(ptr, 0, Token.__sizeof__())
|
||
return ptr
|
||
|
||
|
||
# ============================================================
|
||
# Keyword lookup table
|
||
# Since TransPyC has no dict literals, implemented with linked lists
|
||
# ============================================================
|
||
class KwEntry:
|
||
name: str
|
||
kw_id: t.CInt
|
||
next: KwEntry | t.CPtr
|
||
|
||
def __new__(self, pool: memhub.MemManager | t.CPtr):
|
||
ptr: KwEntry | t.CPtr = pool.alloc(KwEntry.__sizeof__())
|
||
if ptr:
|
||
string.memset(ptr, 0, KwEntry.__sizeof__())
|
||
return ptr
|
||
|
||
|
||
# Global keyword linked list head (lazy initialization)
|
||
_kw_head: KwEntry | t.CPtr = None
|
||
_kw_pool: memhub.MemManager | t.CPtr = None
|
||
|
||
|
||
def _kw_intern(pool: memhub.MemManager | t.CPtr, name: str, kw_id: t.CInt):
|
||
"""Register keyword name in the linked list"""
|
||
global _kw_head
|
||
entry: KwEntry | t.CPtr = KwEntry(pool)
|
||
if entry == None: return
|
||
slen: t.CSizeT = string.strlen(name)
|
||
buf: str = pool.alloc(slen + 1)
|
||
if buf == None: return
|
||
string.memcpy(buf, name, slen + 1)
|
||
entry.name = buf
|
||
entry.kw_id = kw_id
|
||
entry.next = _kw_head
|
||
_kw_head = entry
|
||
|
||
|
||
def _kw_lookup(name: str) -> t.CInt:
|
||
"""Look up keyword name in the linked list, return the kw_id (0 if not keyword)"""
|
||
cur: KwEntry | t.CPtr = _kw_head
|
||
while cur != None:
|
||
if string.strcmp(cur.name, name) == 0:
|
||
return cur.kw_id
|
||
cur = cur.next
|
||
return 0
|
||
|
||
|
||
def _init_keywords(pool: memhub.MemManager | t.CPtr):
|
||
"""Initialize keyword linked list (only once)"""
|
||
global _kw_head, _kw_pool
|
||
if _kw_head != None: return
|
||
_kw_pool = pool
|
||
_kw_intern(pool, "False", Keyword.False_)
|
||
_kw_intern(pool, "None", Keyword.None_)
|
||
_kw_intern(pool, "True", Keyword.True_)
|
||
_kw_intern(pool, "and", Keyword.And)
|
||
_kw_intern(pool, "as", Keyword.As)
|
||
_kw_intern(pool, "assert", Keyword.Assert)
|
||
_kw_intern(pool, "async", Keyword.Async)
|
||
_kw_intern(pool, "await", Keyword.Await)
|
||
_kw_intern(pool, "break", Keyword.Break)
|
||
_kw_intern(pool, "class", Keyword.Class)
|
||
_kw_intern(pool, "continue", Keyword.Continue)
|
||
_kw_intern(pool, "def", Keyword.Def)
|
||
_kw_intern(pool, "del", Keyword.Del)
|
||
_kw_intern(pool, "elif", Keyword.Elif)
|
||
_kw_intern(pool, "else", Keyword.Else)
|
||
_kw_intern(pool, "except", Keyword.Except)
|
||
_kw_intern(pool, "finally", Keyword.Finally)
|
||
_kw_intern(pool, "for", Keyword.For)
|
||
_kw_intern(pool, "from", Keyword.From)
|
||
_kw_intern(pool, "global", Keyword.Global)
|
||
_kw_intern(pool, "if", Keyword.If)
|
||
_kw_intern(pool, "import", Keyword.Import)
|
||
_kw_intern(pool, "in", Keyword.In)
|
||
_kw_intern(pool, "is", Keyword.Is)
|
||
_kw_intern(pool, "lambda", Keyword.Lambda)
|
||
_kw_intern(pool, "nonlocal", Keyword.Nonlocal)
|
||
_kw_intern(pool, "not", Keyword.Not)
|
||
_kw_intern(pool, "or", Keyword.Or)
|
||
_kw_intern(pool, "pass", Keyword.Pass)
|
||
_kw_intern(pool, "raise", Keyword.Raise)
|
||
_kw_intern(pool, "return", Keyword.Return)
|
||
_kw_intern(pool, "try", Keyword.Try)
|
||
_kw_intern(pool, "while", Keyword.While)
|
||
_kw_intern(pool, "with", Keyword.With)
|
||
_kw_intern(pool, "yield", Keyword.Yield)
|
||
_kw_intern(pool, "match", Keyword.Match)
|
||
_kw_intern(pool, "case", Keyword.Case)
|
||
_kw_intern(pool, "type", Keyword.Type)
|
||
# def 的别名关键字（fn/fun/func/function，语义与 def 完全相同）
|
||
_kw_intern(pool, "fn", Keyword.Fn)
|
||
_kw_intern(pool, "fun", Keyword.Fun)
|
||
_kw_intern(pool, "func", Keyword.Func)
|
||
_kw_intern(pool, "function", Keyword.Function)
|
||
_kw_intern(pool, "extern", Keyword.Extern)
|
||
|
||
|
||
# ============================================================
|
||
# Operator lookup table
|
||
# Since TransPyC doesn't have dict, use an array with linear search
|
||
# Operators are sorted by descending length for greedy matching
|
||
# ============================================================
|
||
class OpEntry:
|
||
name: str # Operator text (e.g. "==", "+=")
|
||
op_id: t.CInt
|
||
length: t.CInt
|
||
next: OpEntry | t.CPtr
|
||
|
||
def __new__(self, pool: memhub.MemManager | t.CPtr):
|
||
ptr: OpEntry | t.CPtr = pool.alloc(OpEntry.__sizeof__())
|
||
if ptr:
|
||
string.memset(ptr, 0, OpEntry.__sizeof__())
|
||
return ptr
|
||
|
||
|
||
_op_head: OpEntry | t.CPtr = None
|
||
|
||
|
||
def _op_intern(pool: memhub.MemManager | t.CPtr, name: str, op_id: t.CInt):
|
||
global _op_head
|
||
slen: t.CSizeT = string.strlen(name)
|
||
entry: OpEntry | t.CPtr = OpEntry(pool)
|
||
if entry == None: return
|
||
buf: str = pool.alloc(slen + 1)
|
||
if buf == None: return
|
||
string.memcpy(buf, name, slen + 1)
|
||
entry.name = buf
|
||
entry.op_id = op_id
|
||
entry.length = t.CInt(slen)
|
||
entry.next = _op_head
|
||
_op_head = entry
|
||
|
||
|
||
def _init_operators(pool: memhub.MemManager | t.CPtr):
|
||
"""Initialize the operator table (register in descending length order for easy greedy matching)"""
|
||
global _op_head
|
||
if _op_head != None: return
|
||
# 3-character operators
|
||
_op_intern(pool, "**=", TokOp.StarEqEq)
|
||
_op_intern(pool, "//=", TokOp.DSlashEq)
|
||
_op_intern(pool, ">>=", TokOp.GtGtEq)
|
||
_op_intern(pool, "<<=", TokOp.LtLtEq)
|
||
_op_intern(pool, "...", TokOp.Ellipsis)
|
||
# 2-character operators
|
||
_op_intern(pool, "->", TokOp.RArrow)
|
||
_op_intern(pool, "+=", TokOp.PlusEq)
|
||
_op_intern(pool, "-=", TokOp.MinusEq)
|
||
_op_intern(pool, "*=", TokOp.StarEq)
|
||
_op_intern(pool, "/=", TokOp.SlashEq)
|
||
_op_intern(pool, "%=", TokOp.PercentEq)
|
||
_op_intern(pool, "@=", TokOp.AtEq)
|
||
_op_intern(pool, "&=", TokOp.AmpEq)
|
||
_op_intern(pool, "|=", TokOp.VBarEq)
|
||
_op_intern(pool, "^=", TokOp.CaretEq)
|
||
_op_intern(pool, "//", TokOp.DSlash)
|
||
_op_intern(pool, "**", TokOp.StarStar)
|
||
_op_intern(pool, "<<", TokOp.LtLt)
|
||
_op_intern(pool, ">>", TokOp.GtGt)
|
||
_op_intern(pool, "<=", TokOp.LessEq)
|
||
_op_intern(pool, ">=", TokOp.GreaterEq)
|
||
_op_intern(pool, "==", TokOp.EqEq)
|
||
_op_intern(pool, "!=", TokOp.ExclaimEq)
|
||
_op_intern(pool, ":=", TokOp.ColonEq)
|
||
# 1-character operators
|
||
_op_intern(pool, "(", TokOp.LPar)
|
||
_op_intern(pool, ")", TokOp.RPar)
|
||
_op_intern(pool, "[", TokOp.Lsqb)
|
||
_op_intern(pool, "]", TokOp.Rsqb)
|
||
_op_intern(pool, "{", TokOp.LBrace)
|
||
_op_intern(pool, "}", TokOp.RBrace)
|
||
_op_intern(pool, ",", TokOp.Comma)
|
||
_op_intern(pool, ":", TokOp.Colon)
|
||
_op_intern(pool, ".", TokOp.Dot)
|
||
_op_intern(pool, ";", TokOp.Semi)
|
||
_op_intern(pool, "@", TokOp.At)
|
||
_op_intern(pool, "=", TokOp.Equal)
|
||
_op_intern(pool, "<", TokOp.Less)
|
||
_op_intern(pool, ">", TokOp.Greater)
|
||
_op_intern(pool, "+", TokOp.Plus)
|
||
_op_intern(pool, "-", TokOp.Minus)
|
||
_op_intern(pool, "*", TokOp.Star)
|
||
_op_intern(pool, "/", TokOp.Slash)
|
||
_op_intern(pool, "%", TokOp.Percent)
|
||
_op_intern(pool, "&", TokOp.Amp)
|
||
_op_intern(pool, "|", TokOp.VBar)
|
||
_op_intern(pool, "^", TokOp.Caret)
|
||
_op_intern(pool, "~", TokOp.Tilde)
|
||
|
||
|
||
def _init_tables(pool: memhub.MemManager | t.CPtr):
|
||
"""Initialize all lookup tables (idempotent)"""
|
||
_init_keywords(pool)
|
||
_init_operators(pool)
|
||
|
||
|
||
# ============================================================
|
||
# Token creation helper functions
|
||
# ============================================================
|
||
|
||
def new_token(pool: memhub.MemManager | t.CPtr, ttype: t.CInt, lineno: t.CInt,
|
||
col_offset: t.CInt) -> Token | t.CPtr:
|
||
"""Create a new empty Token and fill in fields later"""
|
||
tok: Token | t.CPtr = Token(pool)
|
||
if tok == None: return None
|
||
tok.type = ttype
|
||
tok.lineno = lineno
|
||
tok.col_offset = col_offset
|
||
tok.end_lineno = lineno
|
||
tok.end_col_offset = col_offset
|
||
return tok
|
||
|
||
|
||
def token_set_str(pool: memhub.MemManager | t.CPtr, tok: Token | t.CPtr,
|
||
src: str, start: t.CSizeT, length: t.CSizeT):
|
||
"""Copy text from src[start:start + length] to token.str_val
|
||
|
||
Note: When length == 0, still allocate 1 byte for NUL, so that an empty string is distinguished from a NULL pointer.
|
||
"""
|
||
if tok == None: return
|
||
buf: str = pool.alloc(length + 1)
|
||
if buf == None: return
|
||
i: t.CSizeT = 0
|
||
while i < length:
|
||
buf[i] = src[start + i]
|
||
i += 1
|
||
buf[length] = '\0'
|
||
tok.str_val = buf
|
||
|
||
|
||
def token_set_str_literal(pool: memhub.MemManager | t.CPtr, tok: Token | t.CPtr,
|
||
text: str):
|
||
"""Copy a C string to token.str_val"""
|
||
if tok == None or text == None: return
|
||
slen: t.CSizeT = string.strlen(text)
|
||
buf: str = pool.alloc(slen + 1)
|
||
if buf == None: return
|
||
string.memcpy(buf, text, slen + 1)
|
||
tok.str_val = buf
|