TermiNexus
27121f56a7
R12（wiki/新规范 L27-39）：把「i: int = 0 与 while i < N: … i += 1」形式的下标循环改写为「for i in range(N):」。保守判定：仅当条件为裸 NAME < EXPR、循环体无 continue、NAME 在体内无其它赋值、无嵌套重绑定、循环后同层不再引用 NAME 时才改写。共 269 处 / 48 文件。 R13：c.Set 是正在退化的路径——c.Set(*p, v) 期望写作 *p = v，c.Set(a, b) 等于 a = b。本次把 includes 内 45 处 c.Set 全部迁移为赋值，includes 已无 c.Set 调用。 验证：bootstrap 三重重编 SUCCESS（无 vast_c 解析回退）；全量 77/77 顺序回归通过；ZlibTest 内部断言 79 passed / 0 failed。
572 lines
22 KiB
Plaintext
572 lines
22 KiB
Plaintext
import t, c
|
||
from stdint import *
|
||
import memhub
|
||
import string
|
||
import stdio
|
||
import stdlib
|
||
|
||
|
||
def _ast_pool_fatal(what: str, need: sizet):
|
||
"""池分配失败（alloc 返回 NULL）时的统一 fail-fast。
|
||
|
||
此前这些 alloc 失败只是静默返回，留下 str_val=NULL / 哑 token；后续
|
||
`_kw_lookup(tok.str_val)` → `_kw_hash_str` 在 NULL 上取字节，崩成
|
||
ASAN "access-violation on unknown address 0x0"，把真因（内存池耗尽）
|
||
掩盖成空指针崩溃。这里直接报真因并终止。
|
||
"""
|
||
stdio.printf("[AST] %s 分配失败：内存池已耗尽（alloc 返回 NULL，need=%zu 字节）"
|
||
"——上游有块被写穿/丢失，见 [MEM] 报告\n", what, need)
|
||
stdio.fflush(None)
|
||
stdlib.exit(1)
|
||
|
||
|
||
# ============================================================
|
||
# Token type enumeration (refer to CPython Lib/token.py)
|
||
# Organized as a type-safe enum using t.CEnum; old names XXX retained as compatible aliases
|
||
# ============================================================
|
||
|
||
enum TokenType:
|
||
EndMarker
|
||
NewLine
|
||
Indent
|
||
Dedent
|
||
Name
|
||
Number
|
||
String
|
||
FStringStart
|
||
FStringMiddle
|
||
FStringEnd
|
||
Op
|
||
Nl
|
||
Comment
|
||
Encoding
|
||
|
||
|
||
# ============================================================
|
||
# Keyword enumeration (refer to CPython Lib/keyword.py)
|
||
# Organized as a type-safe enum using t.CEnum; old names KW_XXX are kept as compatible aliases
|
||
# NotKeyword(0) represents a non-keyword (sentinel to avoid conflict between False_=0 and kw_subtype=0)
|
||
# ============================================================
|
||
|
||
enum Keyword:
|
||
NotKeyword
|
||
False_
|
||
None_
|
||
True_
|
||
And
|
||
As
|
||
Assert
|
||
Async
|
||
Await
|
||
Break
|
||
Class
|
||
Continue
|
||
Def
|
||
Del
|
||
Elif
|
||
Else
|
||
Except
|
||
Finally
|
||
For
|
||
From
|
||
Global
|
||
If
|
||
Import
|
||
In
|
||
Is
|
||
Lambda
|
||
Nonlocal
|
||
Not
|
||
Or
|
||
Pass
|
||
Raise
|
||
Return
|
||
Try
|
||
While
|
||
With
|
||
Yield
|
||
Match
|
||
Case
|
||
Type
|
||
# def 的别名关键字（fn/fun/func/function，语义与 def 完全相同）
|
||
Fn
|
||
Fun
|
||
Func
|
||
Function
|
||
# extern 函数声明关键字（FFI，等价 t.State 标记；软判定见 _is_funcdef_kw）
|
||
Extern
|
||
# export 函数声明关键字（等价 t.CExport，裸名导出不加 SHA1 头；软判定见 _is_funcdef_kw）
|
||
Export
|
||
# define 宏函数关键字（与 def 语义一致，返回注解自动追加 t.CDefine 标记
|
||
# 为宏函数；软判定见 _is_funcdef_kw）
|
||
Define
|
||
# enum 枚举定义关键字（语法糖：`enum X:` 与 `class X(t.CEnum):` 等价，
|
||
# 见 parser.vp 的 _parse_enumdef）。追加在**表尾**而非插在 Class 之后：
|
||
# 本枚举成员的值按声明序递增，插在中间会平移 Class 之后所有关键字的编号
|
||
# （kw_subtype 数值），追加则既有编号逐字不变。
|
||
Enum
|
||
# typedef 类型别名关键字（语法糖：`typedef T1 = T2` 与
|
||
# `T1: t.CTypedef = T2` 等价，见 parser.vp 的 _parse_typedef）。同样追加在
|
||
# **表尾**：既有关键字编号（kw_subtype）逐字不变。
|
||
Typedef
|
||
# 类型描述符关键字 const / volatile（语法糖：描述符写在类型前，
|
||
# `a: const int` 与 `a: t.CConst | int` 等价，见 parser.vp 的 _parse_type_ann）。
|
||
# 同样追加在**表尾**：既有关键字编号（kw_subtype）逐字不变。
|
||
# 二者是**软关键字**——只在注解位置（`:` 后 / `->` 后）生效；作标识符的写法
|
||
# （t.attr.const()，TransPyV/Test/App/attr_test.vp:142）不受影响：
|
||
# 该处 token 类型仍是 Name，属性名解析只看 TokenType.Name，不看 kw_subtype。
|
||
Const
|
||
Volatile
|
||
# virtual / novirtual 修饰符关键字（语法糖：`virtual class C:` /
|
||
# `novirtual class C:` / `virtual def m(...)` / `novirtual def m(...)`，
|
||
# 与手写类/方法装饰器 `@t.CVTable` / `@t.NoVTable` **同一套表示**，
|
||
# 见 parser.vp 的 _ParseVirtualModified）。同样追加在**表尾**：既有关键字
|
||
# 编号（kw_subtype）逐字不变。
|
||
# 二者是**软关键字**——只在"语句起始 + 紧跟 class/def"时生效；作标识符的写法
|
||
# （`x = d.virtual` / 形参名 / for 目标 / 字典键）不受影响：那些位置按
|
||
# TokenType.Name 解析，不看 kw_subtype。
|
||
Virtual
|
||
Novirtual
|
||
# union 类型定义关键字（语法糖：`union X:` 与 `class X(t.CUnion):` 等价，
|
||
# 见 parser.vp 的 _parse_uniondef）。同样追加在**表尾**：既有关键字编号
|
||
# （kw_subtype）逐字不变；全仓 .vp 源码中 `union` 不作标识符使用。
|
||
Union
|
||
# renum 变体类型定义关键字（语法糖：`renum X:` 与 `class X(t.REnum):` 等价，
|
||
# 见 parser.vp 的 _parse_renumdef）。同样追加在**表尾**：既有关键字编号
|
||
# （kw_subtype）逐字不变；全仓 .vp 源码中 `renum` 不作标识符使用。
|
||
Renum
|
||
# struct 纯数据类型定义关键字（语法糖：`struct X:` 与
|
||
# `class X(t.CStruct):` + `@t.NoVTable` 等价，见 parser.vp 的 _parse_structdef）：
|
||
# 与 class 同形，但**只能表示数据类型**——禁止 virtual/novirtual 修饰、禁止类体内
|
||
# 定义 def。同样追加在**表尾**：既有关键字编号（kw_subtype）逐字不变；
|
||
# 全仓 .vp 源码中 `struct` 不作标识符使用（仅出现在注释/字符串与
|
||
# build_gep_struct 之类复合标识符里）。
|
||
Struct
|
||
# switch 多分支语句关键字（`switch <expr>:` + `case <常量>: ` / `default:`，
|
||
# 见 parser.vp 的 _parse_switch）。C switch 语义：默认**不穿透**，case 体内
|
||
# 写 `fallthrough` 才落到下一个 case。同样追加在**表尾**：既有关键字编号
|
||
# （kw_subtype）逐字不变；全仓 .vp 源码中 `switch` 不作标识符使用。
|
||
Switch
|
||
# fallthrough 语句关键字（仅在 switch 的 case 体内合法；match 内使用报错）。
|
||
# 追加在**表尾**：既有关键字编号（kw_subtype）逐字不变。
|
||
FallThrough
|
||
|
||
|
||
# ============================================================
|
||
# Operator/Separator Enumeration
|
||
# Organized as a type-safe enum using t.CEnum; old names OP_XXX retained as compatible aliases
|
||
# Stored in the token.op_subtype field
|
||
# ============================================================
|
||
|
||
enum TokOp:
|
||
LPar # (
|
||
RPar # )
|
||
Lsqb # [
|
||
Rsqb # ]
|
||
LBrace # {
|
||
RBrace # }
|
||
Comma # ,
|
||
Colon # :
|
||
Dot # .
|
||
Semi # ;
|
||
At # @
|
||
Equal # =
|
||
RArrow # ->
|
||
PlusEq # +=
|
||
MinusEq # -=
|
||
StarEq # *=
|
||
SlashEq # /=
|
||
DSlashEq # //=
|
||
PercentEq # %=
|
||
AtEq # @=
|
||
AmpEq # &=
|
||
VBarEq # |=
|
||
CaretEq # ^=
|
||
GtGtEq # >>=
|
||
LtLtEq # <<=
|
||
StarEqEq # **=
|
||
DSlash # //
|
||
StarStar # **
|
||
LtLt # <<
|
||
GtGt # >>
|
||
LessEq # <=
|
||
GreaterEq # >=
|
||
EqEq # ==
|
||
ExclaimEq # !=
|
||
Less # <
|
||
Greater # >
|
||
Plus # +
|
||
Minus # -
|
||
Star # *
|
||
Slash # /
|
||
Percent # %
|
||
Amp # &
|
||
VBar # |
|
||
Caret # ^
|
||
Tilde # ~
|
||
ColonEq # :=
|
||
Ellipsis # ...
|
||
Bang # !(Non-standard, used for parsing tolerance)
|
||
|
||
|
||
# ============================================================
|
||
# Token stucture
|
||
# ============================================================
|
||
class Token:
|
||
type: int # Token type (TokenType.EndMarker / TokenType.NewLine / TokenType.Name / ...)
|
||
op_subtype: int # Operator subtype (OP_*). Only valid when type == TokenType.Op
|
||
kw_subtype: int # Keyword subtype (KW_*). Only valid when type == TokenType.Name and it is a keyword
|
||
str_val: str # Token text (NUL‑terminated, allocated from pool)
|
||
int_val: int64 # Integer value (used by TokenType.Number)
|
||
float_val: double # Floating‑point value (used by TokenType.Number)
|
||
is_float: int # 1 = floating‑point number, 0 = integer
|
||
is_complex: int # 1 = complex number, 0 = real number
|
||
lineno: int # Starting line number
|
||
col_offset: int # Starting column offset
|
||
end_lineno: int # Ending line number
|
||
end_col_offset: int # Ending column offset
|
||
next: rawptr[Token] # Next token in linked list
|
||
|
||
def __new__(self, pool: rawptr[memhub.MemManager]):
|
||
"""Allocate zero‑initialized Token struct memory from mpool"""
|
||
ptr: rawptr[Token] = pool.alloc(Token.__sizeof__())
|
||
if ptr == None:
|
||
_ast_pool_fatal("Token", sizet(Token.__sizeof__()))
|
||
string.memset(ptr, 0, Token.__sizeof__())
|
||
return ptr
|
||
|
||
|
||
# ============================================================
|
||
# Keyword lookup table
|
||
# Since TransPyC has no dict literals, implemented with linked lists
|
||
# ============================================================
|
||
class KwEntry:
|
||
name: str
|
||
kw_id: int
|
||
next: rawptr[KwEntry]
|
||
|
||
def __new__(self, pool: rawptr[memhub.MemManager]):
|
||
ptr: rawptr[KwEntry] = pool.alloc(KwEntry.__sizeof__())
|
||
if ptr == None:
|
||
_ast_pool_fatal("KwEntry", sizet(KwEntry.__sizeof__()))
|
||
string.memset(ptr, 0, KwEntry.__sizeof__())
|
||
return ptr
|
||
|
||
|
||
# Global keyword linked list head (lazy initialization)
|
||
_kw_head: rawptr[KwEntry] = None
|
||
_kw_pool: rawptr[memhub.MemManager] = None
|
||
|
||
# 关键字哈希索引（开放寻址 | 线性探测）：槽为 u64，0=空槽，否则存 KwEntry 地址。
|
||
# 目的：把 _kw_lookup 的 46 项链表线性扫描降为期望 O(1)（不改链表本身）。索引与
|
||
# 链表同源——在 _init_keywords 里由同一批 _kw_intern 派生，不另抄关键字列表。
|
||
# 关键字为静态集合且一次性注册完毕 ⇒ 索引是"完整"的：查找未命中等价于原扫描
|
||
# 返回 0（无需回退链表扫描）；回退仅用于索引未建成（分配失败）时，保证正确性。
|
||
_kw_index: sizet* = None
|
||
# 索引槽数（2 的幂且 > 2*46，负载 < 0.5，探测必然终止）
|
||
define _KW_IDX_CAP = 128
|
||
# FNV-1a 64 位常量（与 hashtable.vp 同款）
|
||
_KW_FNV_OFFSET: uint64 = 0xcbf29ce484222325
|
||
_KW_FNV_PRIME: uint64 = 0x100000001b3
|
||
|
||
|
||
def _kw_hash_str(key: str) -> uint64:
|
||
"""关键字名 FNV-1a 64 位哈希（同一键必得同一值；仅用于槽定位）"""
|
||
h: uint64 = _KW_FNV_OFFSET
|
||
k: str = key
|
||
while k[0] != 0:
|
||
h = h ^ uint64(k[0])
|
||
h = h * _KW_FNV_PRIME
|
||
k += 1
|
||
return uint64(h)
|
||
|
||
|
||
def _kw_index_put(entry: rawptr[KwEntry]):
|
||
"""把关键字条目写入哈希索引（索引未建成时静默跳过）
|
||
|
||
同键只保留首注册者（与查找"命中即返回"语义一致）；冲突逐键 strcmp 区分，
|
||
不能只比哈希——否则不同关键字撞槽会返回错误 kw_id。
|
||
"""
|
||
if _kw_index == None: return
|
||
h: uint64 = _kw_hash_str(entry.name)
|
||
mask: sizet = sizet(_KW_IDX_CAP - 1)
|
||
idx: sizet = sizet(h) & mask
|
||
for i in range(_KW_IDX_CAP):
|
||
slot: sizet* = (sizet*)(t.CPtr(uint64(_kw_index) + uint64(idx * 8)))
|
||
cur_s: sizet = slot[0]
|
||
if cur_s == 0:
|
||
slot[0] = sizet(entry)
|
||
return
|
||
cur: KwEntry* = (KwEntry*)(t.CPtr(uint64(cur_s)))
|
||
if string.strcmp(cur.name, entry.name) == 0:
|
||
return
|
||
idx = (idx + 1) & mask
|
||
|
||
|
||
def _kw_intern(pool: rawptr[memhub.MemManager], name: str, kw_id: int):
|
||
"""Register keyword name in the linked list"""
|
||
global _kw_head
|
||
entry: rawptr[KwEntry] = KwEntry(pool)
|
||
if entry == None: return
|
||
slen: sizet = string.strlen(name)
|
||
buf: str = pool.alloc(slen + 1)
|
||
if buf == None: return
|
||
string.memcpy(buf, name, slen + 1)
|
||
entry.name = buf
|
||
entry.kw_id = kw_id
|
||
entry.next = _kw_head
|
||
_kw_head = entry
|
||
# 同步写入哈希索引（关键字集合的唯一真源仍是上面的链表）
|
||
_kw_index_put(entry)
|
||
|
||
|
||
def _kw_lookup(name: str) -> int:
|
||
"""Look up keyword name, return the kw_id (0 if not keyword)
|
||
|
||
快速路径：查哈希索引（期望 O(1)，不分配内存）；索引未建成时回退原链表扫描。
|
||
"""
|
||
if name == None:
|
||
# 防御：str_val 为 NULL 时 _kw_hash_str 会在 NULL 上取字节，崩成
|
||
# ASAN "access-violation on unknown address 0x0"，真因（池耗尽）被掩盖。
|
||
stdio.printf("[AST] _kw_lookup 收到 NULL 名称：token.str_val 未分配成功（内存池耗尽 / 未 set）\n")
|
||
stdio.fflush(None)
|
||
stdlib.exit(1)
|
||
if _kw_index != None:
|
||
h: uint64 = _kw_hash_str(name)
|
||
mask: sizet = sizet(_KW_IDX_CAP - 1)
|
||
idx: sizet = sizet(h) & mask
|
||
for i in range(_KW_IDX_CAP):
|
||
slot: sizet* = (sizet*)(t.CPtr(uint64(_kw_index) + uint64(idx * 8)))
|
||
cur_s: sizet = slot[0]
|
||
if cur_s == 0:
|
||
return 0
|
||
cur: KwEntry* = (KwEntry*)(t.CPtr(uint64(cur_s)))
|
||
if string.strcmp(cur.name, name) == 0:
|
||
return cur.kw_id
|
||
idx = (idx + 1) & mask
|
||
return 0
|
||
cur2: rawptr[KwEntry] = _kw_head
|
||
while cur2 != None:
|
||
if string.strcmp(cur2.name, name) == 0:
|
||
return cur2.kw_id
|
||
cur2 = cur2.next
|
||
return 0
|
||
|
||
|
||
def _init_keywords(pool: rawptr[memhub.MemManager]):
|
||
"""Initialize keyword linked list (only once)"""
|
||
global _kw_head, _kw_pool, _kw_index
|
||
if _kw_head != None: return
|
||
_kw_pool = pool
|
||
# 建哈希索引槽数组（一次性分配；分配失败则 _kw_index 保持 None → 查找回退链表扫描）
|
||
idx_buf: sizet* = pool.alloc(_KW_IDX_CAP * 8)
|
||
if idx_buf != None:
|
||
string.memset(idx_buf, 0, _KW_IDX_CAP * 8)
|
||
_kw_index = idx_buf
|
||
_kw_intern(pool, "False", Keyword.False_)
|
||
_kw_intern(pool, "None", Keyword.None_)
|
||
_kw_intern(pool, "True", Keyword.True_)
|
||
_kw_intern(pool, "and", Keyword.And)
|
||
_kw_intern(pool, "as", Keyword.As)
|
||
_kw_intern(pool, "assert", Keyword.Assert)
|
||
_kw_intern(pool, "async", Keyword.Async)
|
||
_kw_intern(pool, "await", Keyword.Await)
|
||
_kw_intern(pool, "break", Keyword.Break)
|
||
_kw_intern(pool, "class", Keyword.Class)
|
||
_kw_intern(pool, "continue", Keyword.Continue)
|
||
_kw_intern(pool, "def", Keyword.Def)
|
||
_kw_intern(pool, "del", Keyword.Del)
|
||
_kw_intern(pool, "elif", Keyword.Elif)
|
||
_kw_intern(pool, "else", Keyword.Else)
|
||
_kw_intern(pool, "except", Keyword.Except)
|
||
_kw_intern(pool, "finally", Keyword.Finally)
|
||
_kw_intern(pool, "for", Keyword.For)
|
||
_kw_intern(pool, "from", Keyword.From)
|
||
_kw_intern(pool, "global", Keyword.Global)
|
||
_kw_intern(pool, "if", Keyword.If)
|
||
_kw_intern(pool, "import", Keyword.Import)
|
||
_kw_intern(pool, "in", Keyword.In)
|
||
_kw_intern(pool, "is", Keyword.Is)
|
||
_kw_intern(pool, "lambda", Keyword.Lambda)
|
||
_kw_intern(pool, "nonlocal", Keyword.Nonlocal)
|
||
_kw_intern(pool, "not", Keyword.Not)
|
||
_kw_intern(pool, "or", Keyword.Or)
|
||
_kw_intern(pool, "pass", Keyword.Pass)
|
||
_kw_intern(pool, "raise", Keyword.Raise)
|
||
_kw_intern(pool, "return", Keyword.Return)
|
||
_kw_intern(pool, "try", Keyword.Try)
|
||
_kw_intern(pool, "while", Keyword.While)
|
||
_kw_intern(pool, "with", Keyword.With)
|
||
_kw_intern(pool, "yield", Keyword.Yield)
|
||
_kw_intern(pool, "match", Keyword.Match)
|
||
_kw_intern(pool, "case", Keyword.Case)
|
||
_kw_intern(pool, "type", Keyword.Type)
|
||
# def 的别名关键字（fn/fun/func/function，语义与 def 完全相同）
|
||
_kw_intern(pool, "fn", Keyword.Fn)
|
||
_kw_intern(pool, "fun", Keyword.Fun)
|
||
_kw_intern(pool, "func", Keyword.Func)
|
||
_kw_intern(pool, "function", Keyword.Function)
|
||
_kw_intern(pool, "extern", Keyword.Extern)
|
||
_kw_intern(pool, "export", Keyword.Export)
|
||
_kw_intern(pool, "define", Keyword.Define)
|
||
_kw_intern(pool, "enum", Keyword.Enum)
|
||
_kw_intern(pool, "typedef", Keyword.Typedef)
|
||
# 类型描述符软关键字（仅注解位置生效，见 parser.vp 的 _parse_type_ann）
|
||
_kw_intern(pool, "const", Keyword.Const)
|
||
_kw_intern(pool, "volatile", Keyword.Volatile)
|
||
# virtual / novirtual 修饰符软关键字（仅"语句起始 + 紧跟 class/def"生效，
|
||
# 见 parser.vp 的 _ParseVirtualModified）
|
||
_kw_intern(pool, "virtual", Keyword.Virtual)
|
||
_kw_intern(pool, "novirtual", Keyword.Novirtual)
|
||
_kw_intern(pool, "union", Keyword.Union)
|
||
# renum 变体类型定义关键字（语法糖，同 enum/union，见 parser.vp 的 _parse_renumdef）
|
||
_kw_intern(pool, "renum", Keyword.Renum)
|
||
# struct 纯数据类型定义关键字（语法糖，同 enum/union/renum，见 parser.vp 的
|
||
# _parse_structdef：基类 t.CStruct + 伪装饰器 @t.NoVTable，禁 virtual/novirtual、禁 def）
|
||
_kw_intern(pool, "struct", Keyword.Struct)
|
||
# switch / fallthrough 关键字（见 parser.vp 的 _parse_switch 与 _parse_small_stmt）
|
||
_kw_intern(pool, "switch", Keyword.Switch)
|
||
_kw_intern(pool, "fallthrough", Keyword.FallThrough)
|
||
|
||
|
||
# ============================================================
|
||
# Operator lookup table
|
||
# Since TransPyC doesn't have dict, use an array with linear search
|
||
# Operators are sorted by descending length for greedy matching
|
||
# ============================================================
|
||
class OpEntry:
|
||
name: str # Operator text (e.g. "==", "+=")
|
||
op_id: int
|
||
length: int
|
||
next: rawptr[OpEntry]
|
||
|
||
def __new__(self, pool: rawptr[memhub.MemManager]):
|
||
ptr: rawptr[OpEntry] = pool.alloc(OpEntry.__sizeof__())
|
||
if ptr: string.memset(ptr, 0, OpEntry.__sizeof__())
|
||
return ptr
|
||
|
||
|
||
_op_head: rawptr[OpEntry] = None
|
||
|
||
|
||
def _op_intern(pool: rawptr[memhub.MemManager], name: str, op_id: int):
|
||
global _op_head
|
||
slen: sizet = string.strlen(name)
|
||
entry: rawptr[OpEntry] = OpEntry(pool)
|
||
if entry == None: return
|
||
buf: str = pool.alloc(slen + 1)
|
||
if buf == None: return
|
||
string.memcpy(buf, name, slen + 1)
|
||
entry.name = buf
|
||
entry.op_id = op_id
|
||
entry.length = int(slen)
|
||
entry.next = _op_head
|
||
_op_head = entry
|
||
|
||
|
||
def _init_operators(pool: rawptr[memhub.MemManager]):
|
||
"""Initialize the operator table (register in descending length order for easy greedy matching)"""
|
||
global _op_head
|
||
if _op_head != None: return
|
||
# 3-character operators
|
||
_op_intern(pool, "**=", TokOp.StarEqEq)
|
||
_op_intern(pool, "//=", TokOp.DSlashEq)
|
||
_op_intern(pool, ">>=", TokOp.GtGtEq)
|
||
_op_intern(pool, "<<=", TokOp.LtLtEq)
|
||
_op_intern(pool, "...", TokOp.Ellipsis)
|
||
# 2-character operators
|
||
_op_intern(pool, "->", TokOp.RArrow)
|
||
_op_intern(pool, "+=", TokOp.PlusEq)
|
||
_op_intern(pool, "-=", TokOp.MinusEq)
|
||
_op_intern(pool, "*=", TokOp.StarEq)
|
||
_op_intern(pool, "/=", TokOp.SlashEq)
|
||
_op_intern(pool, "%=", TokOp.PercentEq)
|
||
_op_intern(pool, "@=", TokOp.AtEq)
|
||
_op_intern(pool, "&=", TokOp.AmpEq)
|
||
_op_intern(pool, "|=", TokOp.VBarEq)
|
||
_op_intern(pool, "^=", TokOp.CaretEq)
|
||
_op_intern(pool, "//", TokOp.DSlash)
|
||
_op_intern(pool, "**", TokOp.StarStar)
|
||
_op_intern(pool, "<<", TokOp.LtLt)
|
||
_op_intern(pool, ">>", TokOp.GtGt)
|
||
_op_intern(pool, "<=", TokOp.LessEq)
|
||
_op_intern(pool, ">=", TokOp.GreaterEq)
|
||
_op_intern(pool, "==", TokOp.EqEq)
|
||
_op_intern(pool, "!=", TokOp.ExclaimEq)
|
||
_op_intern(pool, ":=", TokOp.ColonEq)
|
||
# 1-character operators
|
||
_op_intern(pool, "(", TokOp.LPar)
|
||
_op_intern(pool, ")", TokOp.RPar)
|
||
_op_intern(pool, "[", TokOp.Lsqb)
|
||
_op_intern(pool, "]", TokOp.Rsqb)
|
||
_op_intern(pool, "{", TokOp.LBrace)
|
||
_op_intern(pool, "}", TokOp.RBrace)
|
||
_op_intern(pool, ",", TokOp.Comma)
|
||
_op_intern(pool, ":", TokOp.Colon)
|
||
_op_intern(pool, ".", TokOp.Dot)
|
||
_op_intern(pool, ";", TokOp.Semi)
|
||
_op_intern(pool, "@", TokOp.At)
|
||
_op_intern(pool, "=", TokOp.Equal)
|
||
_op_intern(pool, "<", TokOp.Less)
|
||
_op_intern(pool, ">", TokOp.Greater)
|
||
_op_intern(pool, "+", TokOp.Plus)
|
||
_op_intern(pool, "-", TokOp.Minus)
|
||
_op_intern(pool, "*", TokOp.Star)
|
||
_op_intern(pool, "/", TokOp.Slash)
|
||
_op_intern(pool, "%", TokOp.Percent)
|
||
_op_intern(pool, "&", TokOp.Amp)
|
||
_op_intern(pool, "|", TokOp.VBar)
|
||
_op_intern(pool, "^", TokOp.Caret)
|
||
_op_intern(pool, "~", TokOp.Tilde)
|
||
|
||
|
||
def _init_tables(pool: rawptr[memhub.MemManager]):
|
||
"""Initialize all lookup tables (idempotent)"""
|
||
_init_keywords(pool)
|
||
_init_operators(pool)
|
||
|
||
|
||
# ============================================================
|
||
# Token creation helper functions
|
||
# ============================================================
|
||
|
||
def new_token(pool: rawptr[memhub.MemManager], ttype: int, lineno: int,
|
||
col_offset: int) -> rawptr[Token]:
|
||
"""Create a new empty Token and fill in fields later"""
|
||
tok: rawptr[Token] = Token(pool)
|
||
if tok == None: return None
|
||
tok.type = ttype
|
||
tok.lineno = lineno
|
||
tok.col_offset = col_offset
|
||
tok.end_lineno = lineno
|
||
tok.end_col_offset = col_offset
|
||
return tok
|
||
|
||
|
||
def token_set_str(pool: rawptr[memhub.MemManager], tok: rawptr[Token],
|
||
src: str, start: sizet, length: sizet):
|
||
"""Copy text from src[start:start + length] to token.str_val
|
||
|
||
Note: When length == 0, still allocate 1 byte for NUL, so that an empty string is distinguished from a NULL pointer.
|
||
"""
|
||
if tok == None: return
|
||
buf: str = pool.alloc(length + 1)
|
||
if buf == None:
|
||
_ast_pool_fatal("token.str_val", length + 1)
|
||
for i in range(length):
|
||
buf[i] = src[start + i]
|
||
buf[length] = '\0'
|
||
tok.str_val = buf
|
||
|
||
|
||
def token_set_str_literal(pool: rawptr[memhub.MemManager], tok: rawptr[Token],
|
||
text: str):
|
||
"""Copy a C string to token.str_val"""
|
||
if tok == None or text == None: return
|
||
slen: sizet = string.strlen(text)
|
||
buf: str = pool.alloc(slen + 1)
|
||
if buf == None:
|
||
_ast_pool_fatal("token.str_val(字面量)", slen + 1)
|
||
string.memcpy(buf, text, slen + 1)
|
||
tok.str_val = buf
|