snapshot before regression test
This commit is contained in:
t
105
includes/ast/__init__.py
Normal file
105
includes/ast/__init__.py
Normal file
@@ -0,0 +1,105 @@
|
||||
import t, c
|
||||
from stdint import *
|
||||
import memhub
|
||||
import string
|
||||
|
||||
# 导入子模块
|
||||
from .tokens import (
|
||||
Token, TokenType, Keyword, TokOp, _init_tables,
|
||||
)
|
||||
# 公共部分：基类、枚举、常量、辅助函数、dump
|
||||
from .base import (
|
||||
AST, ASTKind, ASTCtx, OpKind, ASTFlag,
|
||||
# 常量
|
||||
CONST_INT, CONST_FLOAT, CONST_STR, CONST_BOOL, CONST_NONE,
|
||||
FLAG_IS_ASYNC, FLAG_SIMPLE, FLAG_HAS_STAR,
|
||||
# 辅助函数
|
||||
_init_ast, _copy_str, _set_pos, _inherit_pos, _set_parent_list,
|
||||
_binop_from_op, _augop_from_op, _cmpop_from_op,
|
||||
_emit, _emit_str, _emit_int, _dump_list,
|
||||
dump,
|
||||
)
|
||||
# 语句节点
|
||||
from .stmts import (
|
||||
# 模块层
|
||||
Module, Expression, Interactive, FunctionType,
|
||||
# 语句
|
||||
FunctionDef, ClassDef, Return, Delete, Assign, AugAssign, AnnAssign,
|
||||
For, While, If, With, Raise, Try, Assert, Global, Nonlocal,
|
||||
Pass, Break, Continue, Expr, Import, ImportFrom, Match,
|
||||
)
|
||||
# 表达式节点
|
||||
from .exprs import (
|
||||
# 表达式
|
||||
BoolOp, BinOp, UnaryOp, Lambda, IfExp, Dict, Set,
|
||||
ListComp, SetComp, DictComp, GeneratorExp,
|
||||
Await, Yield, YieldFrom, FormattedValue, JoinedStr,
|
||||
Constant, NamedExpr, Attribute, Subscript, Starred, Name,
|
||||
List, Tuple, Slice,
|
||||
Call, Compare, OpNode,
|
||||
)
|
||||
# 辅助节点
|
||||
from .astaux import (
|
||||
ExceptHandler, Arguments, Arg, Keyword, Alias, WithItem,
|
||||
Comprehension,
|
||||
)
|
||||
# 模式匹配
|
||||
from .match import (
|
||||
MatchCase, MatchValue, MatchSingleton, MatchSequence,
|
||||
MatchMapping, MatchClass, MatchStar, MatchAs, MatchOr,
|
||||
)
|
||||
from .visitor import ASTVisitor
|
||||
from .lexer import Lexer, _lexer_init, tokenize, new_lexer
|
||||
from .parser import Parser, _parser_init, parse_tokens, new_parser
|
||||
from .parser import _parse_test
|
||||
|
||||
# ============================================================
|
||||
# 全局内存池指针（遵循项目规则：声明 _mbuddy）
|
||||
# 用户在 main 中初始化：ast._mbuddy = memhub.MemBuddy(arena, size)
|
||||
# ============================================================
|
||||
_mbuddy: memhub.MemBuddy | t.CPtr
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 公共 API: parse
|
||||
# ============================================================
|
||||
|
||||
def parse(src: str, pool: memhub.MemManager | t.CPtr) -> AST | t.CPtr:
|
||||
"""解析 Python 源代码，返回 AST Module 节点。
|
||||
|
||||
参数:
|
||||
src: NUL 结尾的源代码字符串
|
||||
pool: memhub.MemManager 实例，用于分配 AST 节点
|
||||
|
||||
返回:
|
||||
AST Module 节点，失败返回 None
|
||||
"""
|
||||
if src is None or pool is None:
|
||||
return None
|
||||
_init_tables(pool)
|
||||
lx: Lexer | t.CPtr = new_lexer(pool)
|
||||
if lx is None:
|
||||
return None
|
||||
_lexer_init(lx, src, pool)
|
||||
tokens: Token | t.CPtr = tokenize(lx)
|
||||
if tokens is None:
|
||||
return None
|
||||
tree: AST | t.CPtr = parse_tokens(pool, tokens)
|
||||
return tree
|
||||
|
||||
|
||||
def parse_expression(src: str, pool: memhub.MemManager | t.CPtr) -> AST | t.CPtr:
|
||||
"""解析单个表达式，返回 Expression 节点"""
|
||||
if src is None or pool is None:
|
||||
return None
|
||||
_init_tables(pool)
|
||||
lx: Lexer | t.CPtr = new_lexer(pool)
|
||||
if lx is None: return None
|
||||
_lexer_init(lx, src, pool)
|
||||
tokens: Token | t.CPtr = tokenize(lx)
|
||||
if tokens is None: return None
|
||||
ps: Parser | t.CPtr = new_parser(pool)
|
||||
if ps is None: return None
|
||||
_parser_init(ps, tokens, pool)
|
||||
body: AST | t.CPtr = _parse_test(ps)
|
||||
return Expression(pool, body)
|
||||
337
includes/ast/astaux.py
Normal file
337
includes/ast/astaux.py
Normal file
@@ -0,0 +1,337 @@
|
||||
import t, c
|
||||
from stdint import *
|
||||
import memhub
|
||||
import string
|
||||
from .base import (
|
||||
AST, ASTKind,
|
||||
_init_ast, _copy_str, _set_parent_list,
|
||||
_emit, _emit_str, _emit_int, _dump_list,
|
||||
)
|
||||
|
||||
|
||||
# ============================================================
|
||||
# ExceptHandler 节点
|
||||
# ============================================================
|
||||
|
||||
class ExceptHandler(AST):
|
||||
"""ExceptHandler(type: AST, name: str)"""
|
||||
type: AST | t.CPtr
|
||||
name: str
|
||||
|
||||
def __new__(self, pool: memhub.MemManager | t.CPtr,
|
||||
type: AST | t.CPtr, name: str):
|
||||
ptr: ExceptHandler | t.CPtr = pool.alloc(ExceptHandler.__sizeof__())
|
||||
if ptr:
|
||||
string.memset(ptr, 0, ExceptHandler.__sizeof__())
|
||||
return ptr
|
||||
|
||||
def __init__(self, pool: memhub.MemManager | t.CPtr,
|
||||
type: AST | t.CPtr, name: str):
|
||||
_init_ast(self, pool, 0, 0)
|
||||
self.type = type
|
||||
self.name = _copy_str(pool, name)
|
||||
|
||||
def kind(self) -> t.CInt:
|
||||
return ASTKind.ExceptHandler
|
||||
|
||||
def type_name(self) -> str:
|
||||
return "ExceptHandler"
|
||||
|
||||
def dump(self, buf: bytes, size: t.CSizeT,
|
||||
pos: t.CSizeT) -> t.CSizeT:
|
||||
pos = _emit(buf, size, pos, "ExceptHandler(type=")
|
||||
if self.type is not None:
|
||||
pos = self.type.dump(buf, size, pos)
|
||||
else:
|
||||
pos = _emit(buf, size, pos, "None")
|
||||
pos = _emit(buf, size, pos, ", name='")
|
||||
pos = _emit_str(buf, size, pos, self.name)
|
||||
pos = _emit(buf, size, pos, "', body=")
|
||||
pos = _dump_list(self.children, buf, size, pos)
|
||||
pos = _emit(buf, size, pos, ")")
|
||||
return pos
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Arguments 节点
|
||||
# ============================================================
|
||||
|
||||
class Arguments(AST):
|
||||
"""Arguments(args: list[AST | CPtr], vararg: AST, kwarg: AST,
|
||||
defaults: list[AST | CPtr], kw_defaults: list[AST | CPtr])"""
|
||||
args: list[AST | t.CPtr] | t.CPtr
|
||||
vararg: AST | t.CPtr
|
||||
kwarg: AST | t.CPtr
|
||||
defaults: list[AST | t.CPtr] | t.CPtr
|
||||
kw_defaults: list[AST | t.CPtr] | t.CPtr
|
||||
|
||||
def __new__(self, pool: memhub.MemManager | t.CPtr,
|
||||
args: list[AST | t.CPtr] | t.CPtr,
|
||||
vararg: AST | t.CPtr, kwarg: AST | t.CPtr,
|
||||
defaults: list[AST | t.CPtr] | t.CPtr,
|
||||
kw_defaults: list[AST | t.CPtr] | t.CPtr):
|
||||
ptr: Arguments | t.CPtr = pool.alloc(Arguments.__sizeof__())
|
||||
if ptr:
|
||||
string.memset(ptr, 0, Arguments.__sizeof__())
|
||||
return ptr
|
||||
|
||||
def __init__(self, pool: memhub.MemManager | t.CPtr,
|
||||
args: list[AST | t.CPtr] | t.CPtr,
|
||||
vararg: AST | t.CPtr, kwarg: AST | t.CPtr,
|
||||
defaults: list[AST | t.CPtr] | t.CPtr,
|
||||
kw_defaults: list[AST | t.CPtr] | t.CPtr):
|
||||
_init_ast(self, pool, 0, 0)
|
||||
self.args = args
|
||||
self.vararg = vararg
|
||||
self.kwarg = kwarg
|
||||
self.defaults = defaults
|
||||
self.kw_defaults = kw_defaults
|
||||
_set_parent_list(args, self)
|
||||
_set_parent_list(defaults, self)
|
||||
_set_parent_list(kw_defaults, self)
|
||||
|
||||
def kind(self) -> t.CInt:
|
||||
return ASTKind.Arguments
|
||||
|
||||
def type_name(self) -> str:
|
||||
return "Arguments"
|
||||
|
||||
def dump(self, buf: bytes, size: t.CSizeT,
|
||||
pos: t.CSizeT) -> t.CSizeT:
|
||||
pos = _emit(buf, size, pos, "Arguments(args=")
|
||||
pos = _dump_list(self.args, buf, size, pos)
|
||||
pos = _emit(buf, size, pos, ", vararg=")
|
||||
if self.vararg is not None:
|
||||
pos = self.vararg.dump(buf, size, pos)
|
||||
else:
|
||||
pos = _emit(buf, size, pos, "None")
|
||||
pos = _emit(buf, size, pos, ", kwarg=")
|
||||
if self.kwarg is not None:
|
||||
pos = self.kwarg.dump(buf, size, pos)
|
||||
else:
|
||||
pos = _emit(buf, size, pos, "None")
|
||||
pos = _emit(buf, size, pos, ", defaults=")
|
||||
pos = _dump_list(self.defaults, buf, size, pos)
|
||||
pos = _emit(buf, size, pos, ", kw_defaults=")
|
||||
pos = _dump_list(self.kw_defaults, buf, size, pos)
|
||||
pos = _emit(buf, size, pos, ")")
|
||||
return pos
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Arg 节点
|
||||
# ============================================================
|
||||
|
||||
class Arg(AST):
|
||||
"""Arg(arg: str, annotation: AST)"""
|
||||
arg: str
|
||||
annotation: AST | t.CPtr
|
||||
|
||||
def __new__(self, pool: memhub.MemManager | t.CPtr,
|
||||
arg: str, annotation: AST | t.CPtr):
|
||||
ptr: Arg | t.CPtr = pool.alloc(Arg.__sizeof__())
|
||||
if ptr:
|
||||
string.memset(ptr, 0, Arg.__sizeof__())
|
||||
return ptr
|
||||
|
||||
def __init__(self, pool: memhub.MemManager | t.CPtr,
|
||||
arg: str, annotation: AST | t.CPtr):
|
||||
_init_ast(self, pool, 0, 0)
|
||||
self.arg = _copy_str(pool, arg)
|
||||
self.annotation = annotation
|
||||
|
||||
def kind(self) -> t.CInt:
|
||||
return ASTKind.Arg
|
||||
|
||||
def type_name(self) -> str:
|
||||
return "Arg"
|
||||
|
||||
def dump(self, buf: bytes, size: t.CSizeT,
|
||||
pos: t.CSizeT) -> t.CSizeT:
|
||||
pos = _emit(buf, size, pos, "Arg(arg='")
|
||||
pos = _emit_str(buf, size, pos, self.arg)
|
||||
pos = _emit(buf, size, pos, "', annotation=")
|
||||
if self.annotation is not None:
|
||||
pos = self.annotation.dump(buf, size, pos)
|
||||
else:
|
||||
pos = _emit(buf, size, pos, "None")
|
||||
pos = _emit(buf, size, pos, ")")
|
||||
return pos
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Keyword 节点
|
||||
# ============================================================
|
||||
|
||||
class Keyword(AST):
|
||||
"""Keyword(arg: str, value: AST)"""
|
||||
arg: str
|
||||
value: AST | t.CPtr
|
||||
|
||||
def __new__(self, pool: memhub.MemManager | t.CPtr,
|
||||
arg: str, value: AST | t.CPtr):
|
||||
ptr: Keyword | t.CPtr = pool.alloc(Keyword.__sizeof__())
|
||||
if ptr:
|
||||
string.memset(ptr, 0, Keyword.__sizeof__())
|
||||
return ptr
|
||||
|
||||
def __init__(self, pool: memhub.MemManager | t.CPtr,
|
||||
arg: str, value: AST | t.CPtr):
|
||||
_init_ast(self, pool, 0, 0)
|
||||
self.arg = _copy_str(pool, arg)
|
||||
self.value = value
|
||||
|
||||
def kind(self) -> t.CInt:
|
||||
return ASTKind.Keyword
|
||||
|
||||
def type_name(self) -> str:
|
||||
return "Keyword"
|
||||
|
||||
def dump(self, buf: bytes, size: t.CSizeT,
|
||||
pos: t.CSizeT) -> t.CSizeT:
|
||||
pos = _emit(buf, size, pos, "Keyword(arg='")
|
||||
pos = _emit_str(buf, size, pos, self.arg)
|
||||
pos = _emit(buf, size, pos, "', value=")
|
||||
if self.value is not None:
|
||||
pos = self.value.dump(buf, size, pos)
|
||||
else:
|
||||
pos = _emit(buf, size, pos, "None")
|
||||
pos = _emit(buf, size, pos, ")")
|
||||
return pos
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Alias 节点
|
||||
# ============================================================
|
||||
|
||||
class Alias(AST):
|
||||
"""Alias(name: str, asname: str)"""
|
||||
name: str
|
||||
asname: str
|
||||
|
||||
def __new__(self, pool: memhub.MemManager | t.CPtr,
|
||||
name: str, asname: str):
|
||||
ptr: Alias | t.CPtr = pool.alloc(Alias.__sizeof__())
|
||||
if ptr:
|
||||
string.memset(ptr, 0, Alias.__sizeof__())
|
||||
return ptr
|
||||
|
||||
def __init__(self, pool: memhub.MemManager | t.CPtr,
|
||||
name: str, asname: str):
|
||||
_init_ast(self, pool, 0, 0)
|
||||
self.name = _copy_str(pool, name)
|
||||
self.asname = _copy_str(pool, asname)
|
||||
|
||||
def kind(self) -> t.CInt:
|
||||
return ASTKind.Alias
|
||||
|
||||
def type_name(self) -> str:
|
||||
return "Alias"
|
||||
|
||||
def dump(self, buf: bytes, size: t.CSizeT,
|
||||
pos: t.CSizeT) -> t.CSizeT:
|
||||
pos = _emit(buf, size, pos, "Alias(name='")
|
||||
pos = _emit_str(buf, size, pos, self.name)
|
||||
pos = _emit(buf, size, pos, "', asname='")
|
||||
pos = _emit_str(buf, size, pos, self.asname)
|
||||
pos = _emit(buf, size, pos, "')")
|
||||
return pos
|
||||
|
||||
|
||||
# ============================================================
|
||||
# WithItem 节点
|
||||
# ============================================================
|
||||
|
||||
class WithItem(AST):
|
||||
"""WithItem(context_expr: AST, optional_vars: AST)"""
|
||||
context_expr: AST | t.CPtr
|
||||
optional_vars: AST | t.CPtr
|
||||
|
||||
def __new__(self, pool: memhub.MemManager | t.CPtr,
|
||||
context_expr: AST | t.CPtr, optional_vars: AST | t.CPtr):
|
||||
ptr: WithItem | t.CPtr = pool.alloc(WithItem.__sizeof__())
|
||||
if ptr:
|
||||
string.memset(ptr, 0, WithItem.__sizeof__())
|
||||
return ptr
|
||||
|
||||
def __init__(self, pool: memhub.MemManager | t.CPtr,
|
||||
context_expr: AST | t.CPtr, optional_vars: AST | t.CPtr):
|
||||
_init_ast(self, pool, 0, 0)
|
||||
self.context_expr = context_expr
|
||||
self.optional_vars = optional_vars
|
||||
|
||||
def kind(self) -> t.CInt:
|
||||
return ASTKind.WithItem
|
||||
|
||||
def type_name(self) -> str:
|
||||
return "WithItem"
|
||||
|
||||
def dump(self, buf: bytes, size: t.CSizeT,
|
||||
pos: t.CSizeT) -> t.CSizeT:
|
||||
pos = _emit(buf, size, pos, "WithItem(context_expr=")
|
||||
if self.context_expr is not None:
|
||||
pos = self.context_expr.dump(buf, size, pos)
|
||||
else:
|
||||
pos = _emit(buf, size, pos, "None")
|
||||
pos = _emit(buf, size, pos, ", optional_vars=")
|
||||
if self.optional_vars is not None:
|
||||
pos = self.optional_vars.dump(buf, size, pos)
|
||||
else:
|
||||
pos = _emit(buf, size, pos, "None")
|
||||
pos = _emit(buf, size, pos, ")")
|
||||
return pos
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Comprehension 节点
|
||||
# ============================================================
|
||||
|
||||
class Comprehension(AST):
|
||||
"""Comprehension(target: AST, iter: AST, ifs: list[AST | CPtr], is_async: CInt)"""
|
||||
target: AST | t.CPtr
|
||||
iter: AST | t.CPtr
|
||||
ifs: list[AST | t.CPtr] | t.CPtr
|
||||
is_async: t.CInt
|
||||
|
||||
def __new__(self, pool: memhub.MemManager | t.CPtr,
|
||||
target: AST | t.CPtr, iter: AST | t.CPtr,
|
||||
ifs: list[AST | t.CPtr] | t.CPtr, is_async: t.CInt):
|
||||
ptr: Comprehension | t.CPtr = pool.alloc(Comprehension.__sizeof__())
|
||||
if ptr:
|
||||
string.memset(ptr, 0, Comprehension.__sizeof__())
|
||||
return ptr
|
||||
|
||||
def __init__(self, pool: memhub.MemManager | t.CPtr,
|
||||
target: AST | t.CPtr, iter: AST | t.CPtr,
|
||||
ifs: list[AST | t.CPtr] | t.CPtr, is_async: t.CInt):
|
||||
_init_ast(self, pool, 0, 0)
|
||||
self.target = target
|
||||
self.iter = iter
|
||||
self.ifs = ifs
|
||||
self.is_async = is_async
|
||||
_set_parent_list(ifs, self)
|
||||
|
||||
def kind(self) -> t.CInt:
|
||||
return ASTKind.Comprehension
|
||||
|
||||
def type_name(self) -> str:
|
||||
return "Comprehension"
|
||||
|
||||
def dump(self, buf: bytes, size: t.CSizeT,
|
||||
pos: t.CSizeT) -> t.CSizeT:
|
||||
pos = _emit(buf, size, pos, "Comprehension(target=")
|
||||
if self.target is not None:
|
||||
pos = self.target.dump(buf, size, pos)
|
||||
else:
|
||||
pos = _emit(buf, size, pos, "None")
|
||||
pos = _emit(buf, size, pos, ", iter=")
|
||||
if self.iter is not None:
|
||||
pos = self.iter.dump(buf, size, pos)
|
||||
else:
|
||||
pos = _emit(buf, size, pos, "None")
|
||||
pos = _emit(buf, size, pos, ", ifs=")
|
||||
pos = _dump_list(self.ifs, buf, size, pos)
|
||||
pos = _emit(buf, size, pos, ", is_async=")
|
||||
pos = _emit_int(buf, size, pos, t.CInt64T(self.is_async))
|
||||
pos = _emit(buf, size, pos, ")")
|
||||
return pos
|
||||
474
includes/ast/base.py
Normal file
474
includes/ast/base.py
Normal file
@@ -0,0 +1,474 @@
|
||||
import t, c
|
||||
from stdint import *
|
||||
import memhub
|
||||
import string
|
||||
import viperlib
|
||||
from .tokens import TokOp
|
||||
|
||||
|
||||
# ============================================================
|
||||
# ASTKind - 节点类型枚举（kind() 虚函数返回值）
|
||||
# ============================================================
|
||||
|
||||
class ASTKind(t.CEnum):
|
||||
# 核心 15 节点
|
||||
Module: t.State
|
||||
FunctionDef: t.State
|
||||
ClassDef: t.State
|
||||
Assign: t.State
|
||||
If: t.State
|
||||
For: t.State
|
||||
While: t.State
|
||||
Return: t.State
|
||||
Expr: t.State
|
||||
Name: t.State
|
||||
Constant: t.State
|
||||
BinOp: t.State
|
||||
UnaryOp: t.State
|
||||
Call: t.State
|
||||
Compare: t.State
|
||||
# 模块层
|
||||
Expression: t.State
|
||||
Interactive: t.State
|
||||
FunctionType: t.State
|
||||
# 语句
|
||||
Delete: t.State
|
||||
AugAssign: t.State
|
||||
AnnAssign: t.State
|
||||
With: t.State
|
||||
Raise: t.State
|
||||
Try: t.State
|
||||
Assert: t.State
|
||||
Global: t.State
|
||||
Nonlocal: t.State
|
||||
Pass: t.State
|
||||
Break: t.State
|
||||
Continue: t.State
|
||||
Import: t.State
|
||||
ImportFrom: t.State
|
||||
Match: t.State
|
||||
# 表达式
|
||||
BoolOp: t.State
|
||||
Lambda: t.State
|
||||
IfExp: t.State
|
||||
Dict: t.State
|
||||
Set: t.State
|
||||
ListComp: t.State
|
||||
SetComp: t.State
|
||||
DictComp: t.State
|
||||
GeneratorExp: t.State
|
||||
Await: t.State
|
||||
Yield: t.State
|
||||
YieldFrom: t.State
|
||||
FormattedValue: t.State
|
||||
JoinedStr: t.State
|
||||
Attribute: t.State
|
||||
Subscript: t.State
|
||||
Starred: t.State
|
||||
List: t.State
|
||||
Tuple: t.State
|
||||
Slice: t.State
|
||||
NamedExpr: t.State
|
||||
# 辅助节点
|
||||
ExceptHandler: t.State
|
||||
Arguments: t.State
|
||||
Arg: t.State
|
||||
Keyword: t.State
|
||||
Alias: t.State
|
||||
WithItem: t.State
|
||||
Comprehension: t.State
|
||||
OpNode: t.State
|
||||
# 模式匹配
|
||||
MatchCase: t.State
|
||||
MatchValue: t.State
|
||||
MatchSingleton: t.State
|
||||
MatchSequence: t.State
|
||||
MatchMapping: t.State
|
||||
MatchClass: t.State
|
||||
MatchStar: t.State
|
||||
MatchAs: t.State
|
||||
MatchOr: t.State
|
||||
|
||||
|
||||
# ============================================================
|
||||
# ASTCtx - 名称上下文（Load/Store/Del）
|
||||
# ============================================================
|
||||
|
||||
class ASTCtx(t.CEnum):
|
||||
Load: t.State
|
||||
Store: t.State
|
||||
Del: t.State
|
||||
|
||||
|
||||
# ============================================================
|
||||
# OpKind - 运算符枚举（BinOp/UnaryOp/Compare 共用）
|
||||
# ============================================================
|
||||
|
||||
class OpKind(t.CEnum):
|
||||
# 二元算术/位运算
|
||||
Add: t.State
|
||||
Sub: t.State
|
||||
Mult: t.State
|
||||
MatMult: t.State
|
||||
Div: t.State
|
||||
Mod: t.State
|
||||
Pow: t.State
|
||||
LShift: t.State
|
||||
RShift: t.State
|
||||
BitOr: t.State
|
||||
BitXor: t.State
|
||||
BitAnd: t.State
|
||||
FloorDiv: t.State
|
||||
# 布尔运算
|
||||
And: t.State
|
||||
Or: t.State
|
||||
# 比较运算
|
||||
Eq: t.State
|
||||
Ne: t.State
|
||||
Lt: t.State
|
||||
Le: t.State
|
||||
Gt: t.State
|
||||
Ge: t.State
|
||||
Is: t.State
|
||||
IsNot: t.State
|
||||
In: t.State
|
||||
NotIn: t.State
|
||||
# 一元运算
|
||||
Not: t.State
|
||||
UAdd: t.State
|
||||
USub: t.State
|
||||
Invert: t.State
|
||||
# 空
|
||||
NoneOp: t.State
|
||||
|
||||
|
||||
# Constant 子类型标识
|
||||
CONST_INT: t.CDefine = 1
|
||||
CONST_FLOAT: t.CDefine = 2
|
||||
CONST_STR: t.CDefine = 3
|
||||
CONST_BOOL: t.CDefine = 4
|
||||
CONST_NONE: t.CDefine = 5
|
||||
|
||||
# 标志位
|
||||
FLAG_IS_ASYNC: t.CDefine = 1
|
||||
FLAG_SIMPLE: t.CDefine = 2
|
||||
FLAG_HAS_STAR: t.CDefine = 4
|
||||
|
||||
|
||||
# ============================================================
|
||||
# ASTFlag - 节点标志位枚举
|
||||
# ============================================================
|
||||
|
||||
class ASTFlag(t.CEnum):
|
||||
IsAsync: t.State
|
||||
Simple: t.State
|
||||
HasStar: t.State
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 辅助函数
|
||||
# ============================================================
|
||||
|
||||
def _copy_str(pool: memhub.MemManager | t.CPtr, src: str) -> str:
|
||||
"""从 pool 分配并复制 C 字符串"""
|
||||
if src is None: return None
|
||||
slen: t.CSizeT = string.strlen(src)
|
||||
buf: str = pool.alloc(slen + 1)
|
||||
if buf is None: return None
|
||||
string.memcpy(buf, src, slen + 1)
|
||||
return buf
|
||||
|
||||
|
||||
def _set_pos(node: AST | t.CPtr, lineno: t.CInt, col_offset: t.CInt,
|
||||
end_lineno: t.CInt, end_col_offset: t.CInt):
|
||||
if node is None: return
|
||||
node.lineno = lineno
|
||||
node.col_offset = col_offset
|
||||
node.end_lineno = end_lineno
|
||||
node.end_col_offset = end_col_offset
|
||||
|
||||
|
||||
def _inherit_pos(node: AST | t.CPtr, ref: AST | t.CPtr):
|
||||
if node is None or ref is None: return
|
||||
node.lineno = ref.lineno
|
||||
node.col_offset = ref.col_offset
|
||||
node.end_lineno = ref.end_lineno
|
||||
node.end_col_offset = ref.end_col_offset
|
||||
|
||||
|
||||
def _init_ast(node: AST | t.CPtr, pool: memhub.MemManager | t.CPtr,
|
||||
lineno: t.CInt, col: t.CInt):
|
||||
"""初始化 AST 基类字段"""
|
||||
node.parent = None
|
||||
node.pool = pool
|
||||
node.lineno = lineno
|
||||
node.col_offset = col
|
||||
node.end_lineno = lineno
|
||||
node.end_col_offset = col
|
||||
node.children = None
|
||||
|
||||
|
||||
def _set_parent_list(lst: list[AST | t.CPtr] | t.CPtr, parent: AST | t.CPtr):
|
||||
"""为 list 中的每个子节点设置 parent 指针"""
|
||||
if lst is None: return
|
||||
n: t.CSizeT = lst.__len__()
|
||||
i: t.CSizeT = 0
|
||||
while i < n:
|
||||
child: AST | t.CPtr = lst.get(i)
|
||||
if child is not None:
|
||||
child.parent = parent
|
||||
i += 1
|
||||
|
||||
|
||||
def _append_child(lst: list[AST | t.CPtr] | t.CPtr, node: AST | t.CPtr,
|
||||
pool: memhub.MemManager | t.CPtr):
|
||||
"""向 list 追加子节点（模块级函数，避免与 AST.append 方法名冲突）。
|
||||
|
||||
在 AST.append 方法中调用此函数，绕过编译器把 list.append 误解析为
|
||||
AST.append 虚函数调用的类型推断问题。"""
|
||||
if lst is None or node is None:
|
||||
return
|
||||
lst.append(node)
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 运算符映射：TokOp -> OpKind
|
||||
# ============================================================
|
||||
|
||||
def _binop_from_op(op_id: t.CInt) -> t.CInt:
|
||||
if op_id == TokOp.Plus: return OpKind.Add
|
||||
if op_id == TokOp.Minus: return OpKind.Sub
|
||||
if op_id == TokOp.Star: return OpKind.Mult
|
||||
if op_id == TokOp.At: return OpKind.MatMult
|
||||
if op_id == TokOp.Slash: return OpKind.Div
|
||||
if op_id == TokOp.Percent: return OpKind.Mod
|
||||
if op_id == TokOp.StarStar: return OpKind.Pow
|
||||
if op_id == TokOp.LtLt: return OpKind.LShift
|
||||
if op_id == TokOp.GtGt: return OpKind.RShift
|
||||
if op_id == TokOp.VBar: return OpKind.BitOr
|
||||
if op_id == TokOp.Caret: return OpKind.BitXor
|
||||
if op_id == TokOp.Amp: return OpKind.BitAnd
|
||||
if op_id == TokOp.DSlash: return OpKind.FloorDiv
|
||||
return OpKind.NoneOp
|
||||
|
||||
|
||||
def _augop_from_op(op_id: t.CInt) -> t.CInt:
|
||||
if op_id == TokOp.PlusEq: return OpKind.Add
|
||||
if op_id == TokOp.MinusEq: return OpKind.Sub
|
||||
if op_id == TokOp.StarEq: return OpKind.Mult
|
||||
if op_id == TokOp.AtEq: return OpKind.MatMult
|
||||
if op_id == TokOp.SlashEq: return OpKind.Div
|
||||
if op_id == TokOp.PercentEq: return OpKind.Mod
|
||||
if op_id == TokOp.StarEqEq: return OpKind.Pow
|
||||
if op_id == TokOp.LtLtEq: return OpKind.LShift
|
||||
if op_id == TokOp.GtGtEq: return OpKind.RShift
|
||||
if op_id == TokOp.VBarEq: return OpKind.BitOr
|
||||
if op_id == TokOp.CaretEq: return OpKind.BitXor
|
||||
if op_id == TokOp.AmpEq: return OpKind.BitAnd
|
||||
if op_id == TokOp.DSlashEq: return OpKind.FloorDiv
|
||||
return OpKind.NoneOp
|
||||
|
||||
|
||||
def _cmpop_from_op(op_id: t.CInt) -> t.CInt:
|
||||
if op_id == TokOp.EqEq: return OpKind.Eq
|
||||
if op_id == TokOp.ExclaimEq: return OpKind.Ne
|
||||
if op_id == TokOp.Less: return OpKind.Lt
|
||||
if op_id == TokOp.LessEq: return OpKind.Le
|
||||
if op_id == TokOp.Greater: return OpKind.Gt
|
||||
if op_id == TokOp.GreaterEq: return OpKind.Ge
|
||||
return OpKind.NoneOp
|
||||
|
||||
|
||||
# ============================================================
|
||||
# dump 辅助函数
|
||||
# ============================================================
|
||||
|
||||
def _emit(buf: t.CChar | t.CPtr, size: t.CSizeT, pos: t.CSizeT,
|
||||
text: str) -> t.CSizeT:
|
||||
"""追加字面量到 buf+pos，返回新 pos"""
|
||||
if buf is None or pos >= size:
|
||||
return pos
|
||||
cur: t.CChar | t.CPtr = (t.CVoid | t.CPtr)(t.CUInt64T(buf) + pos)
|
||||
rem: t.CSizeT = size - pos
|
||||
n: t.CInt = viperlib.snprintf(cur, rem, "%s", text)
|
||||
if n < 0:
|
||||
return pos
|
||||
return pos + t.CSizeT(n)
|
||||
|
||||
|
||||
def _emit_str(buf: t.CChar | t.CPtr, size: t.CSizeT, pos: t.CSizeT,
|
||||
text: str) -> t.CSizeT:
|
||||
"""追加字符串值到 buf+pos（用 %s 格式）"""
|
||||
if buf is None or pos >= size:
|
||||
return pos
|
||||
cur: t.CChar | t.CPtr = (t.CVoid | t.CPtr)(t.CUInt64T(buf) + pos)
|
||||
rem: t.CSizeT = size - pos
|
||||
n: t.CInt = viperlib.snprintf(cur, rem, "%s", text)
|
||||
if n < 0:
|
||||
return pos
|
||||
return pos + t.CSizeT(n)
|
||||
|
||||
|
||||
def _emit_int(buf: t.CChar | t.CPtr, size: t.CSizeT, pos: t.CSizeT,
|
||||
val: t.CInt64T) -> t.CSizeT:
|
||||
"""追加整数值到 buf+pos"""
|
||||
if buf is None or pos >= size:
|
||||
return pos
|
||||
cur: t.CChar | t.CPtr = (t.CVoid | t.CPtr)(t.CUInt64T(buf) + pos)
|
||||
rem: t.CSizeT = size - pos
|
||||
n: t.CInt = viperlib.snprintf(cur, rem, "%lld", val)
|
||||
if n < 0:
|
||||
return pos
|
||||
return pos + t.CSizeT(n)
|
||||
|
||||
|
||||
def _op_name(op: t.CInt) -> str:
|
||||
"""返回 OpKind 的字符串名称"""
|
||||
if op == OpKind.Add: return "Add"
|
||||
if op == OpKind.Sub: return "Sub"
|
||||
if op == OpKind.Mult: return "Mult"
|
||||
if op == OpKind.MatMult: return "MatMult"
|
||||
if op == OpKind.Div: return "Div"
|
||||
if op == OpKind.Mod: return "Mod"
|
||||
if op == OpKind.Pow: return "Pow"
|
||||
if op == OpKind.LShift: return "LShift"
|
||||
if op == OpKind.RShift: return "RShift"
|
||||
if op == OpKind.BitOr: return "BitOr"
|
||||
if op == OpKind.BitXor: return "BitXor"
|
||||
if op == OpKind.BitAnd: return "BitAnd"
|
||||
if op == OpKind.FloorDiv: return "FloorDiv"
|
||||
if op == OpKind.And: return "And"
|
||||
if op == OpKind.Or: return "Or"
|
||||
if op == OpKind.Eq: return "Eq"
|
||||
if op == OpKind.Ne: return "Ne"
|
||||
if op == OpKind.Lt: return "Lt"
|
||||
if op == OpKind.Le: return "Le"
|
||||
if op == OpKind.Gt: return "Gt"
|
||||
if op == OpKind.Ge: return "Ge"
|
||||
if op == OpKind.Is: return "Is"
|
||||
if op == OpKind.IsNot: return "IsNot"
|
||||
if op == OpKind.In: return "In"
|
||||
if op == OpKind.NotIn: return "NotIn"
|
||||
if op == OpKind.Not: return "Not"
|
||||
if op == OpKind.UAdd: return "UAdd"
|
||||
if op == OpKind.USub: return "USub"
|
||||
if op == OpKind.Invert: return "Invert"
|
||||
return "NoneOp"
|
||||
|
||||
|
||||
def _dump_list(lst: list[AST | t.CPtr] | t.CPtr, buf: t.CChar | t.CPtr,
|
||||
size: t.CSizeT, pos: t.CSizeT) -> t.CSizeT:
|
||||
"""遍历 list[AST | CPtr] 容器，多态 dump 每个元素"""
|
||||
if lst is None:
|
||||
pos = _emit(buf, size, pos, "[]")
|
||||
return pos
|
||||
pos = _emit(buf, size, pos, "[")
|
||||
n: t.CSizeT = lst.__len__()
|
||||
i: t.CSizeT = 0
|
||||
while i < n:
|
||||
if i > 0:
|
||||
pos = _emit(buf, size, pos, ", ")
|
||||
child: AST | t.CPtr = lst.get(i)
|
||||
if child is not None:
|
||||
pos = child.dump(buf, size, pos)
|
||||
i += 1
|
||||
pos = _emit(buf, size, pos, "]")
|
||||
return pos
|
||||
|
||||
|
||||
def _dump_op_list(lst: t.CPtr, buf: t.CChar | t.CPtr,
|
||||
size: t.CSizeT, pos: t.CSizeT) -> t.CSizeT:
|
||||
"""遍历 list[CInt] (OpKind 值)，输出 op 名称列表"""
|
||||
if lst is None:
|
||||
pos = _emit(buf, size, pos, "[]")
|
||||
return pos
|
||||
ops_list: list[t.CInt] | t.CPtr = (list[t.CInt] | t.CPtr)(lst)
|
||||
pos = _emit(buf, size, pos, "[")
|
||||
n: t.CSizeT = ops_list.__len__()
|
||||
i: t.CSizeT = 0
|
||||
while i < n:
|
||||
if i > 0:
|
||||
pos = _emit(buf, size, pos, ", ")
|
||||
op: t.CInt = ops_list.get(i)
|
||||
pos = _emit(buf, size, pos, _op_name(op))
|
||||
i += 1
|
||||
pos = _emit(buf, size, pos, "]")
|
||||
return pos
|
||||
|
||||
|
||||
# ============================================================
|
||||
# AST - 多态基类
|
||||
#
|
||||
# @t.CVTable 启用 vtable，支持 kind()/type_name()/dump()/accept() 虚函数
|
||||
# 子节点用 list[AST | CPtr] 容器存储（O(1) 随机访问，比链表高效）
|
||||
# parent 字段维护父指针（用于向上遍历）
|
||||
# ============================================================
|
||||
|
||||
@t.CVTable
|
||||
class AST:
|
||||
"""AST 节点基类。所有具体节点类继承此类。
|
||||
|
||||
字段:
|
||||
parent: 父节点指针（向上遍历）
|
||||
pool: 分配器（用于子节点/字符串分配）
|
||||
lineno/col_offset: 起始位置
|
||||
end_lineno/end_col_offset: 结束位置
|
||||
children: 子节点列表（body 语句块，通过 append 添加）
|
||||
"""
|
||||
parent: AST | t.CPtr
|
||||
pool: memhub.MemManager | t.CPtr
|
||||
lineno: t.CInt
|
||||
col_offset: t.CInt
|
||||
end_lineno: t.CInt
|
||||
end_col_offset: t.CInt
|
||||
children: list[AST | t.CPtr] | t.CPtr
|
||||
|
||||
def kind(self) -> t.CInt:
|
||||
"""返回节点类型（ASTKind 值），子类覆盖"""
|
||||
return 0
|
||||
|
||||
def type_name(self) -> str:
|
||||
"""返回节点类型名，子类覆盖"""
|
||||
return "AST"
|
||||
|
||||
def dump(self, buf: t.CChar | t.CPtr, size: t.CSizeT,
|
||||
pos: t.CSizeT) -> t.CSizeT:
|
||||
"""多态 dump：子类覆盖以自定义格式。返回写入后的 pos"""
|
||||
return pos
|
||||
|
||||
def accept(self, visitor: t.CPtr):
|
||||
"""访问者模式入口（待实现）：ASTVisitor 在 __visitor.py 定义后，
|
||||
子类覆盖此方法分派到 visitor.visit_X(self)。当前为空实现。"""
|
||||
pass
|
||||
|
||||
def append(self, node: AST | t.CPtr):
|
||||
"""将 node 追加为子节点（添加到 children 列表），设置 parent 指针。
|
||||
|
||||
用于 parser 构建语句块：module.append(stmt), if_node.append(stmt) 等。
|
||||
首次调用时懒初始化 children 列表。
|
||||
|
||||
注意：list[AST|CPtr] 当前按值复制存储（编译器限制），所以必须先设置
|
||||
node.parent 再 append，这样副本中的 parent 才是正确的。"""
|
||||
if self is None or node is None:
|
||||
return
|
||||
if self.children is None:
|
||||
self.children = list[AST | t.CPtr](self.pool, 8)
|
||||
node.parent = self
|
||||
_append_child(self.children, node, self.pool)
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 模块级 dump 函数（供 ast.dump(node, buf, size) 调用）
|
||||
# ============================================================
|
||||
|
||||
def dump(node: AST | t.CPtr, buf: t.CChar | t.CPtr, size: t.CSizeT):
|
||||
"""将 AST 节点序列化到 buf（容量 size）"""
|
||||
if node is None or buf is None or size == 0:
|
||||
return
|
||||
buf[0] = '\0'
|
||||
final_pos: t.CSizeT = node.dump(buf, size, 0)
|
||||
# NUL 终止，防止 printf 越界
|
||||
if final_pos < size:
|
||||
buf[final_pos] = '\0'
|
||||
else:
|
||||
buf[size - 1] = '\0'
|
||||
1234
includes/ast/exprs.py
Normal file
1234
includes/ast/exprs.py
Normal file
File diff suppressed because it is too large
Load Diff
743
includes/ast/lexer.py
Normal file
743
includes/ast/lexer.py
Normal file
@@ -0,0 +1,743 @@
|
||||
import t, c
|
||||
from stdint import *
|
||||
import memhub
|
||||
import string
|
||||
from .tokens import (
|
||||
Token, TokenType, TokOp, _init_tables, _kw_lookup, _op_head,
|
||||
OpEntry, new_token, token_set_str, token_set_str_literal,
|
||||
)
|
||||
|
||||
|
||||
# 缩进栈最大深度
|
||||
LEXER_MAX_INDENT: t.CDefine = 256
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Lexer 状态结构体
|
||||
# ============================================================
|
||||
class Lexer:
|
||||
src: str # 源代码（NUL 结尾）
|
||||
pos: t.CSizeT # 当前位置
|
||||
len: t.CSizeT # 源代码长度
|
||||
lineno: t.CInt # 当前行号（从 1 开始）
|
||||
col: t.CInt # 当前列号（从 0 开始）
|
||||
pool: memhub.MemManager | t.CPtr # 内存池
|
||||
indent_stack: t.CInt | t.CPtr # 缩进栈（int 数组，从 pool 分配）
|
||||
indent_top: t.CInt # 栈顶索引（-1 表示空栈）
|
||||
paren_depth: t.CInt # 括号深度（>0 时不产生 TokenType.NewLine）
|
||||
tokens_head: Token | t.CPtr # token 链表头
|
||||
tokens_tail: Token | t.CPtr # token 链表尾
|
||||
at_line_start: t.CInt # 1=行首（需处理缩进）
|
||||
error_count: t.CInt # 错误计数
|
||||
|
||||
def __new__(self, pool: memhub.MemManager | t.CPtr):
|
||||
ptr: Lexer | t.CPtr = pool.alloc(Lexer.__sizeof__())
|
||||
if ptr:
|
||||
string.memset(ptr, 0, Lexer.__sizeof__())
|
||||
return ptr
|
||||
|
||||
|
||||
def new_lexer(pool: memhub.MemManager | t.CPtr) -> Lexer | t.CPtr:
|
||||
"""从 pool 分配并返回 Lexer（跨模块安全包装）"""
|
||||
return Lexer(pool)
|
||||
|
||||
|
||||
def _lexer_init(lx: Lexer | t.CPtr, src: str,
|
||||
pool: memhub.MemManager | t.CPtr):
|
||||
"""初始化 Lexer"""
|
||||
if lx == None: return
|
||||
lx.src = src
|
||||
lx.pos = 0
|
||||
lx.len = string.strlen(src) if src != None else 0
|
||||
lx.lineno = 1
|
||||
lx.col = 0
|
||||
lx.pool = pool
|
||||
# 分配缩进栈
|
||||
lx.indent_stack = pool.alloc(LEXER_MAX_INDENT * 8)
|
||||
if lx.indent_stack != None:
|
||||
string.memset(lx.indent_stack, 0, LEXER_MAX_INDENT * 8)
|
||||
lx.indent_top = -1
|
||||
# 栈底压入 0（基础缩进）
|
||||
if lx.indent_stack != None:
|
||||
lx.indent_stack[0] = 0
|
||||
lx.indent_top = 0
|
||||
lx.paren_depth = 0
|
||||
lx.tokens_head = None
|
||||
lx.tokens_tail = None
|
||||
lx.at_line_start = 1
|
||||
lx.error_count = 0
|
||||
_init_tables(pool)
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 辅助函数
|
||||
# ============================================================
|
||||
|
||||
def _peek(lx: Lexer | t.CPtr, offset: t.CSizeT) -> t.CChar:
|
||||
"""查看 pos+offset 处字符，越界返回 '\0'"""
|
||||
p: t.CSizeT = lx.pos + offset
|
||||
if p >= lx.len:
|
||||
return '\0'
|
||||
return lx.src[p]
|
||||
|
||||
|
||||
def _peek_cur(lx: Lexer | t.CPtr) -> t.CChar:
|
||||
if lx.pos >= lx.len:
|
||||
return '\0'
|
||||
return lx.src[lx.pos]
|
||||
|
||||
|
||||
def _advance(lx: Lexer | t.CPtr) -> t.CChar:
|
||||
"""前进一个字符，返回该字符"""
|
||||
if lx.pos >= lx.len:
|
||||
return '\0'
|
||||
ch: t.CChar = lx.src[lx.pos]
|
||||
lx.pos += 1
|
||||
if ch == '\n':
|
||||
lx.lineno += 1
|
||||
lx.col = 0
|
||||
else:
|
||||
lx.col += 1
|
||||
return ch
|
||||
|
||||
|
||||
def _is_alpha(ch: t.CChar) -> t.CInt:
|
||||
if ch >= 'a' and ch <= 'z': return 1
|
||||
if ch >= 'A' and ch <= 'Z': return 1
|
||||
if ch == '_': return 1
|
||||
return 0
|
||||
|
||||
|
||||
def _is_digit(ch: t.CChar) -> t.CInt:
|
||||
if ch >= '0' and ch <= '9': return 1
|
||||
return 0
|
||||
|
||||
|
||||
def _is_alnum(ch: t.CChar) -> t.CInt:
|
||||
if _is_alpha(ch): return 1
|
||||
if _is_digit(ch): return 1
|
||||
return 0
|
||||
|
||||
|
||||
def _is_hex(ch: t.CChar) -> t.CInt:
|
||||
if ch >= '0' and ch <= '9': return 1
|
||||
if ch >= 'a' and ch <= 'f': return 1
|
||||
if ch >= 'A' and ch <= 'F': return 1
|
||||
return 0
|
||||
|
||||
|
||||
def _is_space(ch: t.CChar) -> t.CInt:
|
||||
if ch == ' ': return 1
|
||||
if ch == '\t': return 1
|
||||
if ch == '\f': return 1
|
||||
if ch == '\v': return 1
|
||||
if ch == '\r': return 1
|
||||
return 0
|
||||
|
||||
|
||||
def _emit(lx: Lexer | t.CPtr, tok: Token | t.CPtr):
|
||||
"""将 token 追加到链表"""
|
||||
if tok == None: return
|
||||
if lx.tokens_head == None:
|
||||
lx.tokens_head = tok
|
||||
lx.tokens_tail = tok
|
||||
else:
|
||||
lx.tokens_tail.next = tok
|
||||
lx.tokens_tail = tok
|
||||
|
||||
|
||||
def _make_op_token(lx: Lexer | t.CPtr, op_id: t.CInt, length: t.CInt,
|
||||
lineno: t.CInt, col: t.CInt) -> Token | t.CPtr:
|
||||
tok: Token | t.CPtr = new_token(lx.pool, TokenType.Op, lineno, col)
|
||||
if tok == None: return None
|
||||
tok.op_subtype = op_id
|
||||
tok.end_col_offset = col + length
|
||||
return tok
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 缩进处理
|
||||
# ============================================================
|
||||
|
||||
def _handle_indent(lx: Lexer | t.CPtr):
|
||||
"""处理行首缩进，生成 TokenType.Indent/TokenType.Dedent。
|
||||
跳过空行和注释行（不产生缩进变化）。"""
|
||||
# 先跳过空行和注释行
|
||||
while lx.pos < lx.len:
|
||||
# 记住行首位置，用于非空行回退
|
||||
line_start_pos: t.CSizeT = lx.pos
|
||||
line_start_col: t.CInt = lx.col
|
||||
# 跳过空白（不跨行）
|
||||
while lx.pos < lx.len:
|
||||
ch: t.CChar = lx.src[lx.pos]
|
||||
if ch == ' ' or ch == '\t' or ch == '\f' or ch == '\v':
|
||||
lx.pos += 1
|
||||
lx.col += 1
|
||||
elif ch == '\r':
|
||||
# CRLF 行尾符的 \r，跳过（让 \n 触发空行处理）
|
||||
lx.pos += 1
|
||||
else:
|
||||
break
|
||||
if lx.pos >= lx.len:
|
||||
break
|
||||
ch = lx.src[lx.pos]
|
||||
if ch == '\n':
|
||||
# 空行，跳过
|
||||
lx.pos += 1
|
||||
lx.lineno += 1
|
||||
lx.col = 0
|
||||
continue
|
||||
if ch == '\r':
|
||||
# CRLF 空行（\r 后跟 \n），跳过 \r
|
||||
lx.pos += 1
|
||||
if lx.pos < lx.len and lx.src[lx.pos] == '\n':
|
||||
lx.pos += 1
|
||||
lx.lineno += 1
|
||||
lx.col = 0
|
||||
continue
|
||||
if ch == '#':
|
||||
# 注释行，跳过到行尾
|
||||
while lx.pos < lx.len and lx.src[lx.pos] != '\n':
|
||||
lx.pos += 1
|
||||
continue
|
||||
# 非空行：回退到行首，让后续缩进计算能读到前导空格
|
||||
lx.pos = line_start_pos
|
||||
lx.col = line_start_col
|
||||
break
|
||||
|
||||
if lx.pos >= lx.len:
|
||||
# 文件末尾，不再处理缩进
|
||||
lx.at_line_start = 0
|
||||
return
|
||||
|
||||
# 计算当前行缩进
|
||||
indent: t.CInt = 0
|
||||
save_pos: t.CSizeT = lx.pos
|
||||
save_col: t.CInt = lx.col
|
||||
while lx.pos < lx.len:
|
||||
ch: t.CChar = lx.src[lx.pos]
|
||||
if ch == ' ':
|
||||
indent += 1
|
||||
lx.pos += 1
|
||||
lx.col += 1
|
||||
elif ch == '\t':
|
||||
indent += 8 - (indent % 8)
|
||||
lx.pos += 1
|
||||
lx.col += 1
|
||||
else:
|
||||
break
|
||||
|
||||
# 如果行尾（空行），不处理
|
||||
if lx.pos < lx.len:
|
||||
ch = lx.src[lx.pos]
|
||||
if ch == '\n' or ch == '\r' or ch == '#':
|
||||
# 空行或注释行，回退位置让主循环处理
|
||||
lx.pos = save_pos
|
||||
lx.col = save_col
|
||||
lx.at_line_start = 0
|
||||
return
|
||||
|
||||
# 与缩进栈比较
|
||||
top_indent: t.CInt = lx.indent_stack[lx.indent_top]
|
||||
if indent > top_indent:
|
||||
# TokenType.Indent
|
||||
if lx.indent_top + 1 >= LEXER_MAX_INDENT:
|
||||
lx.error_count += 1
|
||||
return
|
||||
lx.indent_top += 1
|
||||
lx.indent_stack[lx.indent_top] = indent
|
||||
tok: Token | t.CPtr = new_token(lx.pool, TokenType.Indent, lx.lineno, 0)
|
||||
_emit(lx, tok)
|
||||
else:
|
||||
while indent < lx.indent_stack[lx.indent_top] and lx.indent_top > 0:
|
||||
lx.indent_top -= 1
|
||||
tok: Token | t.CPtr = new_token(lx.pool, TokenType.Dedent, lx.lineno, 0)
|
||||
_emit(lx, tok)
|
||||
lx.at_line_start = 0
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 标识符/关键字
|
||||
# ============================================================
|
||||
|
||||
def _read_name(lx: Lexer | t.CPtr):
|
||||
start: t.CSizeT = lx.pos
|
||||
start_lineno: t.CInt = lx.lineno
|
||||
start_col: t.CInt = lx.col
|
||||
while lx.pos < lx.len:
|
||||
ch: t.CChar = lx.src[lx.pos]
|
||||
if _is_alnum(ch):
|
||||
lx.pos += 1
|
||||
lx.col += 1
|
||||
else:
|
||||
break
|
||||
length: t.CSizeT = lx.pos - start
|
||||
tok: Token | t.CPtr = new_token(lx.pool, TokenType.Name, start_lineno, start_col)
|
||||
if tok == None: return
|
||||
token_set_str(lx.pool, tok, lx.src, start, length)
|
||||
tok.end_lineno = lx.lineno
|
||||
tok.end_col_offset = start_col + t.CInt(length)
|
||||
# 查找关键字
|
||||
kw_id: t.CInt = _kw_lookup(tok.str_val)
|
||||
if kw_id:
|
||||
tok.kw_subtype = kw_id
|
||||
_emit(lx, tok)
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 数字
|
||||
# ============================================================
|
||||
|
||||
def _read_number(lx: Lexer | t.CPtr):
|
||||
start: t.CSizeT = lx.pos
|
||||
start_lineno: t.CInt = lx.lineno
|
||||
start_col: t.CInt = lx.col
|
||||
is_float: t.CInt = 0
|
||||
is_complex: t.CInt = 0
|
||||
|
||||
ch0: t.CChar = lx.src[lx.pos]
|
||||
# 处理 0x/0o/0b 前缀
|
||||
if ch0 == '0' and lx.pos + 1 < lx.len:
|
||||
ch1: t.CChar = lx.src[lx.pos + 1]
|
||||
if ch1 == 'x' or ch1 == 'X':
|
||||
lx.pos += 2
|
||||
lx.col += 2
|
||||
while lx.pos < lx.len:
|
||||
ch: t.CChar = lx.src[lx.pos]
|
||||
if _is_hex(ch) or ch == '_':
|
||||
lx.pos += 1
|
||||
lx.col += 1
|
||||
else:
|
||||
break
|
||||
_emit_number(lx, start, start_lineno, start_col, 0, 0, 0)
|
||||
return
|
||||
if ch1 == 'o' or ch1 == 'O':
|
||||
lx.pos += 2
|
||||
lx.col += 2
|
||||
while lx.pos < lx.len:
|
||||
ch: t.CChar = lx.src[lx.pos]
|
||||
if ch >= '0' and ch <= '7':
|
||||
lx.pos += 1
|
||||
lx.col += 1
|
||||
elif ch == '_':
|
||||
lx.pos += 1
|
||||
lx.col += 1
|
||||
else:
|
||||
break
|
||||
_emit_number(lx, start, start_lineno, start_col, 0, 0, 0)
|
||||
return
|
||||
if ch1 == 'b' or ch1 == 'B':
|
||||
lx.pos += 2
|
||||
lx.col += 2
|
||||
while lx.pos < lx.len:
|
||||
ch: t.CChar = lx.src[lx.pos]
|
||||
if ch == '0' or ch == '1' or ch == '_':
|
||||
lx.pos += 1
|
||||
lx.col += 1
|
||||
else:
|
||||
break
|
||||
_emit_number(lx, start, start_lineno, start_col, 0, 0, 0)
|
||||
return
|
||||
|
||||
# 十进制整数/浮点
|
||||
while lx.pos < lx.len:
|
||||
ch: t.CChar = lx.src[lx.pos]
|
||||
if _is_digit(ch) or ch == '_':
|
||||
lx.pos += 1
|
||||
lx.col += 1
|
||||
else:
|
||||
break
|
||||
|
||||
# 小数部分
|
||||
if lx.pos < lx.len and lx.src[lx.pos] == '.':
|
||||
is_float = 1
|
||||
lx.pos += 1
|
||||
lx.col += 1
|
||||
while lx.pos < lx.len:
|
||||
ch: t.CChar = lx.src[lx.pos]
|
||||
if _is_digit(ch) or ch == '_':
|
||||
lx.pos += 1
|
||||
lx.col += 1
|
||||
else:
|
||||
break
|
||||
|
||||
# 指数部分
|
||||
if lx.pos < lx.len:
|
||||
ch: t.CChar = lx.src[lx.pos]
|
||||
if ch == 'e' or ch == 'E':
|
||||
is_float = 1
|
||||
lx.pos += 1
|
||||
lx.col += 1
|
||||
if lx.pos < lx.len:
|
||||
ch = lx.src[lx.pos]
|
||||
if ch == '+' or ch == '-':
|
||||
lx.pos += 1
|
||||
lx.col += 1
|
||||
while lx.pos < lx.len:
|
||||
ch: t.CChar = lx.src[lx.pos]
|
||||
if _is_digit(ch) or ch == '_':
|
||||
lx.pos += 1
|
||||
lx.col += 1
|
||||
else:
|
||||
break
|
||||
|
||||
# 虚数后缀
|
||||
if lx.pos < lx.len and lx.src[lx.pos] == 'j':
|
||||
is_complex = 1
|
||||
is_float = 1
|
||||
lx.pos += 1
|
||||
lx.col += 1
|
||||
|
||||
_emit_number(lx, start, start_lineno, start_col, is_float, is_complex, 0)
|
||||
|
||||
|
||||
def _emit_number(lx: Lexer | t.CPtr, start: t.CSizeT,
|
||||
start_lineno: t.CInt, start_col: t.CInt,
|
||||
is_float: t.CInt, is_complex: t.CInt, is_hex: t.CInt):
|
||||
length: t.CSizeT = lx.pos - start
|
||||
tok: Token | t.CPtr = new_token(lx.pool, TokenType.Number, start_lineno, start_col)
|
||||
if tok == None: return
|
||||
token_set_str(lx.pool, tok, lx.src, start, length)
|
||||
tok.is_float = is_float
|
||||
tok.is_complex = is_complex
|
||||
tok.end_lineno = lx.lineno
|
||||
tok.end_col_offset = start_col + t.CInt(length)
|
||||
# 解析数值
|
||||
if is_float:
|
||||
tok.float_val = string.atof(tok.str_val)
|
||||
else:
|
||||
tok.int_val = string.atoll(tok.str_val)
|
||||
_emit(lx, tok)
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 字符串
|
||||
# ============================================================
|
||||
|
||||
def _read_string(lx: Lexer | t.CPtr):
|
||||
"""读取字符串字面量。
|
||||
支持前缀: r, b, u, f, rb, br, rf, fr（大小写不敏感）
|
||||
支持引号: ', ", ''', """
|
||||
start_lineno: t.CInt = lx.lineno
|
||||
start_col: t.CInt = lx.col
|
||||
prefix_start: t.CSizeT = lx.pos
|
||||
|
||||
# 读取前缀（r/b/u/f）
|
||||
is_raw: t.CInt = 0
|
||||
is_bytes: t.CInt = 0
|
||||
is_fstring: t.CInt = 0
|
||||
while lx.pos < lx.len:
|
||||
ch: t.CChar = lx.src[lx.pos]
|
||||
if ch == 'r' or ch == 'R':
|
||||
is_raw = 1
|
||||
lx.pos += 1
|
||||
lx.col += 1
|
||||
elif ch == 'b' or ch == 'B':
|
||||
is_bytes = 1
|
||||
lx.pos += 1
|
||||
lx.col += 1
|
||||
elif ch == 'u' or ch == 'U':
|
||||
lx.pos += 1
|
||||
lx.col += 1
|
||||
elif ch == 'f' or ch == 'F':
|
||||
is_fstring = 1
|
||||
lx.pos += 1
|
||||
lx.col += 1
|
||||
else:
|
||||
break
|
||||
|
||||
if lx.pos >= lx.len:
|
||||
lx.error_count += 1
|
||||
return
|
||||
|
||||
# 确定引号
|
||||
quote: t.CChar = lx.src[lx.pos]
|
||||
if quote != '\'' and quote != '"':
|
||||
lx.error_count += 1
|
||||
return
|
||||
|
||||
# 检查三引号
|
||||
is_triple: t.CInt = 0
|
||||
if lx.pos + 2 < lx.len:
|
||||
if lx.src[lx.pos] == quote and lx.src[lx.pos + 1] == quote and lx.src[lx.pos + 2] == quote:
|
||||
is_triple = 1
|
||||
|
||||
content_start: t.CSizeT = lx.pos
|
||||
if is_triple:
|
||||
lx.pos += 3
|
||||
lx.col += 3
|
||||
else:
|
||||
lx.pos += 1
|
||||
lx.col += 1
|
||||
|
||||
# 读取字符串内容
|
||||
content_buf_start: t.CSizeT = lx.pos
|
||||
while lx.pos < lx.len:
|
||||
ch = lx.src[lx.pos]
|
||||
if is_triple:
|
||||
if ch == quote:
|
||||
if lx.pos + 2 < lx.len:
|
||||
if lx.src[lx.pos + 1] == quote and lx.src[lx.pos + 2] == quote:
|
||||
break
|
||||
elif lx.pos + 2 == lx.len:
|
||||
break
|
||||
if ch == '\\':
|
||||
lx.pos += 1
|
||||
lx.col += 1
|
||||
if lx.pos < lx.len:
|
||||
if lx.src[lx.pos] == '\n':
|
||||
lx.lineno += 1
|
||||
lx.col = 0
|
||||
lx.pos += 1
|
||||
lx.col += 1
|
||||
continue
|
||||
if ch == '\n':
|
||||
lx.lineno += 1
|
||||
lx.col = 0
|
||||
lx.pos += 1
|
||||
lx.col += 1
|
||||
else:
|
||||
if ch == quote:
|
||||
break
|
||||
if ch == '\n':
|
||||
# 单引号字符串不能跨行（除非续行）
|
||||
lx.error_count += 1
|
||||
return
|
||||
if ch == '\\' and not is_raw:
|
||||
lx.pos += 1
|
||||
lx.col += 1
|
||||
if lx.pos < lx.len:
|
||||
lx.pos += 1
|
||||
lx.col += 1
|
||||
continue
|
||||
lx.pos += 1
|
||||
lx.col += 1
|
||||
|
||||
content_end: t.CSizeT = lx.pos
|
||||
# 跳过结束引号
|
||||
if is_triple:
|
||||
if lx.pos + 2 < lx.len:
|
||||
lx.pos += 3
|
||||
lx.col += 3
|
||||
else:
|
||||
lx.pos = lx.len
|
||||
else:
|
||||
if lx.pos < lx.len:
|
||||
lx.pos += 1
|
||||
lx.col += 1
|
||||
|
||||
# 创建 token
|
||||
tok: Token | t.CPtr = new_token(lx.pool, TokenType.String, start_lineno, start_col)
|
||||
if tok == None: return
|
||||
content_len: t.CSizeT = content_end - content_buf_start
|
||||
|
||||
# 处理转义序列（非 raw 字符串）
|
||||
if is_raw == 0 and content_len > 0:
|
||||
# 分配转换缓冲区（最大情况：每个字符都是普通字符，无需扩展）
|
||||
conv_buf: str = lx.pool.alloc(content_len + 1)
|
||||
if conv_buf == None:
|
||||
token_set_str(lx.pool, tok, lx.src, content_buf_start, content_len)
|
||||
else:
|
||||
wpos: t.CSizeT = 0
|
||||
rpos: t.CSizeT = 0
|
||||
while rpos < content_len:
|
||||
ch: t.CChar = lx.src[content_buf_start + rpos]
|
||||
if ch == '\\' and rpos + 1 < content_len:
|
||||
rpos += 1
|
||||
next_ch: t.CChar = lx.src[content_buf_start + rpos]
|
||||
if next_ch == 'n':
|
||||
conv_buf[wpos] = '\n'
|
||||
elif next_ch == 't':
|
||||
conv_buf[wpos] = '\t'
|
||||
elif next_ch == 'r':
|
||||
conv_buf[wpos] = '\r'
|
||||
elif next_ch == '\\':
|
||||
conv_buf[wpos] = '\\'
|
||||
elif next_ch == '\'':
|
||||
conv_buf[wpos] = '\''
|
||||
elif next_ch == '"':
|
||||
conv_buf[wpos] = '"'
|
||||
elif next_ch == '0':
|
||||
conv_buf[wpos] = '\0'
|
||||
else:
|
||||
# 未知转义，保留原样（包括 \xNN 等以后再处理）
|
||||
conv_buf[wpos] = '\\'
|
||||
wpos += 1
|
||||
conv_buf[wpos] = next_ch
|
||||
wpos += 1
|
||||
rpos += 1
|
||||
else:
|
||||
conv_buf[wpos] = ch
|
||||
wpos += 1
|
||||
rpos += 1
|
||||
conv_buf[wpos] = '\0'
|
||||
token_set_str_literal(lx.pool, tok, conv_buf)
|
||||
else:
|
||||
token_set_str(lx.pool, tok, lx.src, content_buf_start, content_len)
|
||||
tok.end_lineno = lx.lineno
|
||||
tok.end_col_offset = lx.col
|
||||
# flags 字段复用：bit0=raw, bit1=bytes, bit2=fstring, bit3=triple
|
||||
if is_raw: tok.kw_subtype = tok.kw_subtype | 1
|
||||
if is_bytes: tok.kw_subtype = tok.kw_subtype | 2
|
||||
if is_fstring: tok.kw_subtype = tok.kw_subtype | 4
|
||||
if is_triple: tok.kw_subtype = tok.kw_subtype | 8
|
||||
_emit(lx, tok)
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 运算符
|
||||
# ============================================================
|
||||
|
||||
def _read_operator(lx: Lexer | t.CPtr):
|
||||
"""贪婪匹配运算符"""
|
||||
start_lineno: t.CInt = lx.lineno
|
||||
start_col: t.CInt = lx.col
|
||||
remaining: t.CSizeT = lx.len - lx.pos
|
||||
|
||||
# 遍历运算符表，找最长匹配
|
||||
best_entry: OpEntry | t.CPtr = None
|
||||
best_len: t.CInt = 0
|
||||
cur: OpEntry | t.CPtr = _op_head
|
||||
while cur != None:
|
||||
if cur.length <= t.CInt(remaining):
|
||||
# 比较 cur.name 与 src[pos:pos+cur.length]
|
||||
match: t.CInt = 1
|
||||
i: t.CSizeT = 0
|
||||
while i < t.CSizeT(cur.length):
|
||||
if lx.src[lx.pos + i] != cur.name[i]:
|
||||
match = 0
|
||||
break
|
||||
i += 1
|
||||
if match:
|
||||
if cur.length > best_len:
|
||||
best_len = cur.length
|
||||
best_entry = cur
|
||||
cur = cur.next
|
||||
|
||||
if best_entry == None:
|
||||
# 未知字符，报错并跳过
|
||||
lx.error_count += 1
|
||||
lx.pos += 1
|
||||
lx.col += 1
|
||||
return
|
||||
|
||||
# 更新括号深度
|
||||
if best_entry.op_id == TokOp.LPar or best_entry.op_id == TokOp.Lsqb or best_entry.op_id == TokOp.LBrace:
|
||||
lx.paren_depth += 1
|
||||
elif best_entry.op_id == TokOp.RPar or best_entry.op_id == TokOp.Rsqb or best_entry.op_id == TokOp.RBrace:
|
||||
if lx.paren_depth > 0:
|
||||
lx.paren_depth -= 1
|
||||
|
||||
tok: Token | t.CPtr = _make_op_token(lx, best_entry.op_id, best_entry.length,
|
||||
start_lineno, start_col)
|
||||
if tok == None: return
|
||||
token_set_str(lx.pool, tok, lx.src, lx.pos, t.CSizeT(best_len))
|
||||
lx.pos += t.CSizeT(best_len)
|
||||
lx.col += best_len
|
||||
_emit(lx, tok)
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 主词法分析入口
|
||||
# ============================================================
|
||||
|
||||
def tokenize(lx: Lexer | t.CPtr) -> Token | t.CPtr:
|
||||
"""执行词法分析，返回 token 链表头"""
|
||||
if lx == None: return None
|
||||
while lx.pos < lx.len:
|
||||
# 行首缩进处理
|
||||
if lx.at_line_start and lx.paren_depth == 0:
|
||||
_handle_indent(lx)
|
||||
if lx.pos >= lx.len:
|
||||
break
|
||||
|
||||
ch: t.CChar = _peek_cur(lx)
|
||||
if ch == '\0':
|
||||
break
|
||||
|
||||
# 换行
|
||||
if ch == '\n':
|
||||
if lx.paren_depth > 0:
|
||||
# 括号内，产生 TokenType.Nl
|
||||
tok: Token | t.CPtr = new_token(lx.pool, TokenType.Nl, lx.lineno, lx.col)
|
||||
_emit(lx, tok)
|
||||
else:
|
||||
# 逻辑换行
|
||||
tok: Token | t.CPtr = new_token(lx.pool, TokenType.NewLine, lx.lineno, lx.col)
|
||||
_emit(lx, tok)
|
||||
lx.at_line_start = 1
|
||||
lx.pos += 1
|
||||
lx.lineno += 1
|
||||
lx.col = 0
|
||||
continue
|
||||
|
||||
# 反斜线续行
|
||||
if ch == '\\' and lx.pos + 1 < lx.len and lx.src[lx.pos + 1] == '\n':
|
||||
lx.pos += 2
|
||||
lx.lineno += 1
|
||||
lx.col = 0
|
||||
continue
|
||||
|
||||
# 空白
|
||||
if _is_space(ch):
|
||||
lx.pos += 1
|
||||
lx.col += 1
|
||||
continue
|
||||
|
||||
# 注释
|
||||
if ch == '#':
|
||||
while lx.pos < lx.len and lx.src[lx.pos] != '\n':
|
||||
lx.pos += 1
|
||||
continue
|
||||
|
||||
# 字符串前缀（如 r"..."）或字符串
|
||||
if ch == '\'' or ch == '"':
|
||||
_read_string(lx)
|
||||
continue
|
||||
# 字符串前缀字母后跟引号（必须在 _is_alpha 之前检测，否则 f/r/b/u 会被当作 TokenType.Name）
|
||||
if (ch == 'r' or ch == 'R' or ch == 'b' or ch == 'B' or
|
||||
ch == 'u' or ch == 'U' or ch == 'f' or ch == 'F'):
|
||||
if lx.pos + 1 < lx.len:
|
||||
ch1: t.CChar = lx.src[lx.pos + 1]
|
||||
if ch1 == '\'' or ch1 == '"':
|
||||
_read_string(lx)
|
||||
continue
|
||||
# 双前缀如 rb, br, rf, fr
|
||||
if lx.pos + 2 < lx.len:
|
||||
ch2: t.CChar = lx.src[lx.pos + 2]
|
||||
if ((ch1 == 'r' or ch1 == 'R' or ch1 == 'b' or ch1 == 'B' or
|
||||
ch1 == 'f' or ch1 == 'F' or ch1 == 'u' or ch1 == 'U') and
|
||||
(ch2 == '\'' or ch2 == '"')):
|
||||
_read_string(lx)
|
||||
continue
|
||||
|
||||
# 标识符/关键字
|
||||
if _is_alpha(ch):
|
||||
_read_name(lx)
|
||||
continue
|
||||
|
||||
# 数字
|
||||
if _is_digit(ch):
|
||||
_read_number(lx)
|
||||
continue
|
||||
|
||||
# 运算符
|
||||
_read_operator(lx)
|
||||
|
||||
# 文件结尾：产生 TokenType.NewLine（如果有内容）
|
||||
if lx.tokens_tail != None:
|
||||
if lx.tokens_tail.type != TokenType.NewLine and lx.tokens_tail.type != TokenType.Nl:
|
||||
tok: Token | t.CPtr = new_token(lx.pool, TokenType.NewLine, lx.lineno, lx.col)
|
||||
_emit(lx, tok)
|
||||
|
||||
# 产生剩余 TokenType.Dedent
|
||||
while lx.indent_top > 0:
|
||||
lx.indent_top -= 1
|
||||
tok: Token | t.CPtr = new_token(lx.pool, TokenType.Dedent, lx.lineno, 0)
|
||||
_emit(lx, tok)
|
||||
|
||||
# TokenType.EndMarker
|
||||
tok: Token | t.CPtr = new_token(lx.pool, TokenType.EndMarker, lx.lineno, 0)
|
||||
_emit(lx, tok)
|
||||
|
||||
return lx.tokens_head
|
||||
363
includes/ast/match.py
Normal file
363
includes/ast/match.py
Normal file
@@ -0,0 +1,363 @@
|
||||
import t, c
|
||||
from stdint import *
|
||||
import memhub
|
||||
import string
|
||||
from .base import (
|
||||
AST, ASTKind,
|
||||
_init_ast, _copy_str, _set_parent_list,
|
||||
_emit, _emit_str, _emit_int, _dump_list,
|
||||
)
|
||||
|
||||
|
||||
# ============================================================
|
||||
# MatchCase 节点
|
||||
# ============================================================
|
||||
|
||||
class MatchCase(AST):
|
||||
"""MatchCase(pattern: AST, guard: AST)"""
|
||||
pattern: AST | t.CPtr
|
||||
guard: AST | t.CPtr
|
||||
|
||||
def __new__(self, pool: memhub.MemManager | t.CPtr,
|
||||
pattern: AST | t.CPtr, guard: AST | t.CPtr):
|
||||
ptr: MatchCase | t.CPtr = pool.alloc(MatchCase.__sizeof__())
|
||||
if ptr:
|
||||
string.memset(ptr, 0, MatchCase.__sizeof__())
|
||||
return ptr
|
||||
|
||||
def __init__(self, pool: memhub.MemManager | t.CPtr,
|
||||
pattern: AST | t.CPtr, guard: AST | t.CPtr):
|
||||
_init_ast(self, pool, 0, 0)
|
||||
self.pattern = pattern
|
||||
self.guard = guard
|
||||
|
||||
def kind(self) -> t.CInt:
|
||||
return ASTKind.MatchCase
|
||||
|
||||
def type_name(self) -> str:
|
||||
return "MatchCase"
|
||||
|
||||
def dump(self, buf: t.CChar | t.CPtr, size: t.CSizeT,
|
||||
pos: t.CSizeT) -> t.CSizeT:
|
||||
pos = _emit(buf, size, pos, "MatchCase(pattern=")
|
||||
if self.pattern is not None:
|
||||
pos = self.pattern.dump(buf, size, pos)
|
||||
pos = _emit(buf, size, pos, ", guard=")
|
||||
if self.guard is not None:
|
||||
pos = self.guard.dump(buf, size, pos)
|
||||
pos = _emit(buf, size, pos, ", body=")
|
||||
pos = _dump_list(self.children, buf, size, pos)
|
||||
pos = _emit(buf, size, pos, ")")
|
||||
return pos
|
||||
|
||||
|
||||
# ============================================================
|
||||
# MatchValue 节点
|
||||
# ============================================================
|
||||
|
||||
class MatchValue(AST):
|
||||
"""MatchValue(value: AST)"""
|
||||
value: AST | t.CPtr
|
||||
|
||||
def __new__(self, pool: memhub.MemManager | t.CPtr, value: AST | t.CPtr):
|
||||
ptr: MatchValue | t.CPtr = pool.alloc(MatchValue.__sizeof__())
|
||||
if ptr:
|
||||
string.memset(ptr, 0, MatchValue.__sizeof__())
|
||||
return ptr
|
||||
|
||||
def __init__(self, pool: memhub.MemManager | t.CPtr, value: AST | t.CPtr):
|
||||
_init_ast(self, pool, 0, 0)
|
||||
self.value = value
|
||||
|
||||
def kind(self) -> t.CInt:
|
||||
return ASTKind.MatchValue
|
||||
|
||||
def type_name(self) -> str:
|
||||
return "MatchValue"
|
||||
|
||||
def dump(self, buf: t.CChar | t.CPtr, size: t.CSizeT,
|
||||
pos: t.CSizeT) -> t.CSizeT:
|
||||
pos = _emit(buf, size, pos, "MatchValue(value=")
|
||||
if self.value is not None:
|
||||
pos = self.value.dump(buf, size, pos)
|
||||
pos = _emit(buf, size, pos, ")")
|
||||
return pos
|
||||
|
||||
|
||||
# ============================================================
|
||||
# MatchSingleton 节点
|
||||
# ============================================================
|
||||
|
||||
class MatchSingleton(AST):
|
||||
"""MatchSingleton(value: AST)"""
|
||||
value: AST | t.CPtr
|
||||
|
||||
def __new__(self, pool: memhub.MemManager | t.CPtr, value: AST | t.CPtr):
|
||||
ptr: MatchSingleton | t.CPtr = pool.alloc(MatchSingleton.__sizeof__())
|
||||
if ptr:
|
||||
string.memset(ptr, 0, MatchSingleton.__sizeof__())
|
||||
return ptr
|
||||
|
||||
def __init__(self, pool: memhub.MemManager | t.CPtr, value: AST | t.CPtr):
|
||||
_init_ast(self, pool, 0, 0)
|
||||
self.value = value
|
||||
|
||||
def kind(self) -> t.CInt:
|
||||
return ASTKind.MatchSingleton
|
||||
|
||||
def type_name(self) -> str:
|
||||
return "MatchSingleton"
|
||||
|
||||
def dump(self, buf: t.CChar | t.CPtr, size: t.CSizeT,
|
||||
pos: t.CSizeT) -> t.CSizeT:
|
||||
pos = _emit(buf, size, pos, "MatchSingleton(value=")
|
||||
if self.value is not None:
|
||||
pos = self.value.dump(buf, size, pos)
|
||||
pos = _emit(buf, size, pos, ")")
|
||||
return pos
|
||||
|
||||
|
||||
# ============================================================
|
||||
# MatchSequence 节点
|
||||
# ============================================================
|
||||
|
||||
class MatchSequence(AST):
|
||||
"""MatchSequence(patterns: list[AST | t.CPtr])"""
|
||||
patterns: list[AST | t.CPtr] | t.CPtr
|
||||
|
||||
def __new__(self, pool: memhub.MemManager | t.CPtr,
|
||||
patterns: list[AST | t.CPtr] | t.CPtr):
|
||||
ptr: MatchSequence | t.CPtr = pool.alloc(MatchSequence.__sizeof__())
|
||||
if ptr:
|
||||
string.memset(ptr, 0, MatchSequence.__sizeof__())
|
||||
return ptr
|
||||
|
||||
def __init__(self, pool: memhub.MemManager | t.CPtr,
|
||||
patterns: list[AST | t.CPtr] | t.CPtr):
|
||||
_init_ast(self, pool, 0, 0)
|
||||
self.patterns = patterns
|
||||
_set_parent_list(patterns, self)
|
||||
|
||||
def kind(self) -> t.CInt:
|
||||
return ASTKind.MatchSequence
|
||||
|
||||
def type_name(self) -> str:
|
||||
return "MatchSequence"
|
||||
|
||||
def dump(self, buf: t.CChar | t.CPtr, size: t.CSizeT,
|
||||
pos: t.CSizeT) -> t.CSizeT:
|
||||
pos = _emit(buf, size, pos, "MatchSequence(patterns=")
|
||||
pos = _dump_list(self.patterns, buf, size, pos)
|
||||
pos = _emit(buf, size, pos, ")")
|
||||
return pos
|
||||
|
||||
|
||||
# ============================================================
|
||||
# MatchMapping 节点
|
||||
# ============================================================
|
||||
|
||||
class MatchMapping(AST):
|
||||
"""MatchMapping(keys: list[AST | t.CPtr], patterns: list[AST | t.CPtr], rest: str)"""
|
||||
keys: list[AST | t.CPtr] | t.CPtr
|
||||
patterns: list[AST | t.CPtr] | t.CPtr
|
||||
rest: str
|
||||
|
||||
def __new__(self, pool: memhub.MemManager | t.CPtr,
|
||||
keys: list[AST | t.CPtr] | t.CPtr,
|
||||
patterns: list[AST | t.CPtr] | t.CPtr,
|
||||
rest: str):
|
||||
ptr: MatchMapping | t.CPtr = pool.alloc(MatchMapping.__sizeof__())
|
||||
if ptr:
|
||||
string.memset(ptr, 0, MatchMapping.__sizeof__())
|
||||
return ptr
|
||||
|
||||
def __init__(self, pool: memhub.MemManager | t.CPtr,
|
||||
keys: list[AST | t.CPtr] | t.CPtr,
|
||||
patterns: list[AST | t.CPtr] | t.CPtr,
|
||||
rest: str):
|
||||
_init_ast(self, pool, 0, 0)
|
||||
self.keys = keys
|
||||
self.patterns = patterns
|
||||
self.rest = _copy_str(pool, rest)
|
||||
_set_parent_list(keys, self)
|
||||
_set_parent_list(patterns, self)
|
||||
|
||||
def kind(self) -> t.CInt:
|
||||
return ASTKind.MatchMapping
|
||||
|
||||
def type_name(self) -> str:
|
||||
return "MatchMapping"
|
||||
|
||||
def dump(self, buf: t.CChar | t.CPtr, size: t.CSizeT,
|
||||
pos: t.CSizeT) -> t.CSizeT:
|
||||
pos = _emit(buf, size, pos, "MatchMapping(keys=")
|
||||
pos = _dump_list(self.keys, buf, size, pos)
|
||||
pos = _emit(buf, size, pos, ", patterns=")
|
||||
pos = _dump_list(self.patterns, buf, size, pos)
|
||||
pos = _emit(buf, size, pos, ", rest='")
|
||||
pos = _emit_str(buf, size, pos, self.rest)
|
||||
pos = _emit(buf, size, pos, "')")
|
||||
return pos
|
||||
|
||||
|
||||
# ============================================================
|
||||
# MatchClass 节点
|
||||
# ============================================================
|
||||
|
||||
class MatchClass(AST):
|
||||
"""MatchClass(cls: AST, patterns: list[AST | t.CPtr],
|
||||
kwd_attrs: list[AST | t.CPtr], kwd_patterns: list[AST | t.CPtr])"""
|
||||
cls: AST | t.CPtr
|
||||
patterns: list[AST | t.CPtr] | t.CPtr
|
||||
kwd_attrs: list[AST | t.CPtr] | t.CPtr
|
||||
kwd_patterns: list[AST | t.CPtr] | t.CPtr
|
||||
|
||||
def __new__(self, pool: memhub.MemManager | t.CPtr,
|
||||
cls: AST | t.CPtr,
|
||||
patterns: list[AST | t.CPtr] | t.CPtr,
|
||||
kwd_attrs: list[AST | t.CPtr] | t.CPtr,
|
||||
kwd_patterns: list[AST | t.CPtr] | t.CPtr):
|
||||
ptr: MatchClass | t.CPtr = pool.alloc(MatchClass.__sizeof__())
|
||||
if ptr:
|
||||
string.memset(ptr, 0, MatchClass.__sizeof__())
|
||||
return ptr
|
||||
|
||||
def __init__(self, pool: memhub.MemManager | t.CPtr,
|
||||
cls: AST | t.CPtr,
|
||||
patterns: list[AST | t.CPtr] | t.CPtr,
|
||||
kwd_attrs: list[AST | t.CPtr] | t.CPtr,
|
||||
kwd_patterns: list[AST | t.CPtr] | t.CPtr):
|
||||
_init_ast(self, pool, 0, 0)
|
||||
self.cls = cls
|
||||
self.patterns = patterns
|
||||
self.kwd_attrs = kwd_attrs
|
||||
self.kwd_patterns = kwd_patterns
|
||||
_set_parent_list(patterns, self)
|
||||
_set_parent_list(kwd_attrs, self)
|
||||
_set_parent_list(kwd_patterns, self)
|
||||
|
||||
def kind(self) -> t.CInt:
|
||||
return ASTKind.MatchClass
|
||||
|
||||
def type_name(self) -> str:
|
||||
return "MatchClass"
|
||||
|
||||
def dump(self, buf: t.CChar | t.CPtr, size: t.CSizeT,
|
||||
pos: t.CSizeT) -> t.CSizeT:
|
||||
pos = _emit(buf, size, pos, "MatchClass(cls=")
|
||||
if self.cls is not None:
|
||||
pos = self.cls.dump(buf, size, pos)
|
||||
pos = _emit(buf, size, pos, ", patterns=")
|
||||
pos = _dump_list(self.patterns, buf, size, pos)
|
||||
pos = _emit(buf, size, pos, ", kwd_attrs=")
|
||||
pos = _dump_list(self.kwd_attrs, buf, size, pos)
|
||||
pos = _emit(buf, size, pos, ", kwd_patterns=")
|
||||
pos = _dump_list(self.kwd_patterns, buf, size, pos)
|
||||
pos = _emit(buf, size, pos, ")")
|
||||
return pos
|
||||
|
||||
|
||||
# ============================================================
|
||||
# MatchStar 节点
|
||||
# ============================================================
|
||||
|
||||
class MatchStar(AST):
|
||||
"""MatchStar(name: str)"""
|
||||
name: str
|
||||
|
||||
def __new__(self, pool: memhub.MemManager | t.CPtr, name: str):
|
||||
ptr: MatchStar | t.CPtr = pool.alloc(MatchStar.__sizeof__())
|
||||
if ptr:
|
||||
string.memset(ptr, 0, MatchStar.__sizeof__())
|
||||
return ptr
|
||||
|
||||
def __init__(self, pool: memhub.MemManager | t.CPtr, name: str):
|
||||
_init_ast(self, pool, 0, 0)
|
||||
self.name = _copy_str(pool, name)
|
||||
|
||||
def kind(self) -> t.CInt:
|
||||
return ASTKind.MatchStar
|
||||
|
||||
def type_name(self) -> str:
|
||||
return "MatchStar"
|
||||
|
||||
def dump(self, buf: t.CChar | t.CPtr, size: t.CSizeT,
|
||||
pos: t.CSizeT) -> t.CSizeT:
|
||||
pos = _emit(buf, size, pos, "MatchStar(name='")
|
||||
pos = _emit_str(buf, size, pos, self.name)
|
||||
pos = _emit(buf, size, pos, "')")
|
||||
return pos
|
||||
|
||||
|
||||
# ============================================================
|
||||
# MatchAs 节点
|
||||
# ============================================================
|
||||
|
||||
class MatchAs(AST):
|
||||
"""MatchAs(pattern: AST, name: str)"""
|
||||
pattern: AST | t.CPtr
|
||||
name: str
|
||||
|
||||
def __new__(self, pool: memhub.MemManager | t.CPtr,
|
||||
pattern: AST | t.CPtr, name: str):
|
||||
ptr: MatchAs | t.CPtr = pool.alloc(MatchAs.__sizeof__())
|
||||
if ptr:
|
||||
string.memset(ptr, 0, MatchAs.__sizeof__())
|
||||
return ptr
|
||||
|
||||
def __init__(self, pool: memhub.MemManager | t.CPtr,
|
||||
pattern: AST | t.CPtr, name: str):
|
||||
_init_ast(self, pool, 0, 0)
|
||||
self.pattern = pattern
|
||||
self.name = _copy_str(pool, name)
|
||||
|
||||
def kind(self) -> t.CInt:
|
||||
return ASTKind.MatchAs
|
||||
|
||||
def type_name(self) -> str:
|
||||
return "MatchAs"
|
||||
|
||||
def dump(self, buf: t.CChar | t.CPtr, size: t.CSizeT,
|
||||
pos: t.CSizeT) -> t.CSizeT:
|
||||
pos = _emit(buf, size, pos, "MatchAs(pattern=")
|
||||
if self.pattern is not None:
|
||||
pos = self.pattern.dump(buf, size, pos)
|
||||
pos = _emit(buf, size, pos, ", name='")
|
||||
pos = _emit_str(buf, size, pos, self.name)
|
||||
pos = _emit(buf, size, pos, "')")
|
||||
return pos
|
||||
|
||||
|
||||
# ============================================================
|
||||
# MatchOr 节点
|
||||
# ============================================================
|
||||
|
||||
class MatchOr(AST):
|
||||
"""MatchOr(patterns: list[AST | t.CPtr])"""
|
||||
patterns: list[AST | t.CPtr] | t.CPtr
|
||||
|
||||
def __new__(self, pool: memhub.MemManager | t.CPtr,
|
||||
patterns: list[AST | t.CPtr] | t.CPtr):
|
||||
ptr: MatchOr | t.CPtr = pool.alloc(MatchOr.__sizeof__())
|
||||
if ptr:
|
||||
string.memset(ptr, 0, MatchOr.__sizeof__())
|
||||
return ptr
|
||||
|
||||
def __init__(self, pool: memhub.MemManager | t.CPtr,
|
||||
patterns: list[AST | t.CPtr] | t.CPtr):
|
||||
_init_ast(self, pool, 0, 0)
|
||||
self.patterns = patterns
|
||||
_set_parent_list(patterns, self)
|
||||
|
||||
def kind(self) -> t.CInt:
|
||||
return ASTKind.MatchOr
|
||||
|
||||
def type_name(self) -> str:
|
||||
return "MatchOr"
|
||||
|
||||
def dump(self, buf: t.CChar | t.CPtr, size: t.CSizeT,
|
||||
pos: t.CSizeT) -> t.CSizeT:
|
||||
pos = _emit(buf, size, pos, "MatchOr(patterns=")
|
||||
pos = _dump_list(self.patterns, buf, size, pos)
|
||||
pos = _emit(buf, size, pos, ")")
|
||||
return pos
|
||||
2345
includes/ast/parser.py
Normal file
2345
includes/ast/parser.py
Normal file
File diff suppressed because it is too large
Load Diff
1150
includes/ast/stmts.py
Normal file
1150
includes/ast/stmts.py
Normal file
File diff suppressed because it is too large
Load Diff
391
includes/ast/tokens.py
Normal file
391
includes/ast/tokens.py
Normal file
@@ -0,0 +1,391 @@
|
||||
import t, c
|
||||
from stdint import *
|
||||
import memhub
|
||||
import string
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Token 类型枚举（参考 CPython Lib/token.py）
|
||||
# 用 t.CEnum 组织为类型安全枚举；旧名 XXX 保留为兼容别名
|
||||
# ============================================================
|
||||
|
||||
class TokenType(t.CEnum):
|
||||
EndMarker: t.State
|
||||
NewLine: t.State
|
||||
Indent: t.State
|
||||
Dedent: t.State
|
||||
Name: t.State
|
||||
Number: t.State
|
||||
String: t.State
|
||||
FStringStart: t.State
|
||||
FStringMiddle: t.State
|
||||
FStringEnd: t.State
|
||||
Op: t.State
|
||||
Nl: t.State
|
||||
Comment: t.State
|
||||
Encoding: t.State
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 关键字枚举（参考 CPython Lib/keyword.py）
|
||||
# 用 t.CEnum 组织为类型安全枚举；旧名 KW_XXX 保留为兼容别名
|
||||
# NotKeyword(0) 表示非关键字（哨兵，避免 False_=0 与 kw_subtype=0 冲突）
|
||||
# ============================================================
|
||||
|
||||
class Keyword(t.CEnum):
|
||||
NotKeyword: t.State
|
||||
False_: t.State
|
||||
None_: t.State
|
||||
True_: t.State
|
||||
And: t.State
|
||||
As: t.State
|
||||
Assert: t.State
|
||||
Async: t.State
|
||||
Await: t.State
|
||||
Break: t.State
|
||||
Class: t.State
|
||||
Continue: t.State
|
||||
Def: t.State
|
||||
Del: t.State
|
||||
Elif: t.State
|
||||
Else: t.State
|
||||
Except: t.State
|
||||
Finally: t.State
|
||||
For: t.State
|
||||
From: t.State
|
||||
Global: t.State
|
||||
If: t.State
|
||||
Import: t.State
|
||||
In: t.State
|
||||
Is: t.State
|
||||
Lambda: t.State
|
||||
Nonlocal: t.State
|
||||
Not: t.State
|
||||
Or: t.State
|
||||
Pass: t.State
|
||||
Raise: t.State
|
||||
Return: t.State
|
||||
Try: t.State
|
||||
While: t.State
|
||||
With: t.State
|
||||
Yield: t.State
|
||||
Match: t.State
|
||||
Case: t.State
|
||||
Type: t.State
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 运算符/分隔符枚举
|
||||
# 用 t.CEnum 组织为类型安全枚举；旧名 OP_XXX 保留为兼容别名
|
||||
# 存储在 token.op_subtype 字段
|
||||
# ============================================================
|
||||
|
||||
class TokOp(t.CEnum):
|
||||
LPar: t.State # (
|
||||
RPar: t.State # )
|
||||
Lsqb: t.State # [
|
||||
Rsqb: t.State # ]
|
||||
LBrace: t.State # {
|
||||
RBrace: t.State # }
|
||||
Comma: t.State # ,
|
||||
Colon: t.State # :
|
||||
Dot: t.State # .
|
||||
Semi: t.State # ;
|
||||
At: t.State # @
|
||||
Equal: t.State # =
|
||||
RArrow: t.State # ->
|
||||
PlusEq: t.State # +=
|
||||
MinusEq: t.State # -=
|
||||
StarEq: t.State # *=
|
||||
SlashEq: t.State # /=
|
||||
DSlashEq: t.State # //=
|
||||
PercentEq: t.State # %=
|
||||
AtEq: t.State # @=
|
||||
AmpEq: t.State # &=
|
||||
VBarEq: t.State # |=
|
||||
CaretEq: t.State # ^=
|
||||
GtGtEq: t.State # >>=
|
||||
LtLtEq: t.State # <<=
|
||||
StarEqEq: t.State # **=
|
||||
DSlash: t.State # //
|
||||
StarStar: t.State # **
|
||||
LtLt: t.State # <<
|
||||
GtGt: t.State # >>
|
||||
LessEq: t.State # <=
|
||||
GreaterEq: t.State # >=
|
||||
EqEq: t.State # ==
|
||||
ExclaimEq: t.State # !=
|
||||
Less: t.State # <
|
||||
Greater: t.State # >
|
||||
Plus: t.State # +
|
||||
Minus: t.State # -
|
||||
Star: t.State # *
|
||||
Slash: t.State # /
|
||||
Percent: t.State # %
|
||||
Amp: t.State # &
|
||||
VBar: t.State # |
|
||||
Caret: t.State # ^
|
||||
Tilde: t.State # ~
|
||||
ColonEq: t.State # :=
|
||||
Ellipsis: t.State # ...
|
||||
Bang: t.State # ! (非标准，用于解析容错)
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Token 结构体
|
||||
# ============================================================
|
||||
class Token:
|
||||
type: t.CInt # Token 类型（TokenType.EndMarker/TokenType.NewLine/TokenType.Name/...）
|
||||
op_subtype: t.CInt # 运算符子类型（OP_*），仅 type==TokenType.Op 时有效
|
||||
kw_subtype: t.CInt # 关键字子类型（KW_*），仅 type==TokenType.Name 且是关键字时有效
|
||||
str_val: str # token 文本（NUL 结尾，从 pool 分配）
|
||||
int_val: t.CInt64T # 整数值（TokenType.Number 用）
|
||||
float_val: t.CDouble # 浮点值（TokenType.Number 用）
|
||||
is_float: t.CInt # 1=浮点数, 0=整数
|
||||
is_complex: t.CInt # 1=复数, 0=实数
|
||||
lineno: t.CInt # 起始行号
|
||||
col_offset: t.CInt # 起始列偏移
|
||||
end_lineno: t.CInt # 结束行号
|
||||
end_col_offset: t.CInt # 结束列偏移
|
||||
next: Token | t.CPtr # 链表下一个 token
|
||||
|
||||
def __new__(self, pool: memhub.MemManager | t.CPtr):
|
||||
"""从 mpool 分配 Token 结构体内存并零初始化"""
|
||||
ptr: Token | t.CPtr = pool.alloc(Token.__sizeof__())
|
||||
if ptr:
|
||||
string.memset(ptr, 0, Token.__sizeof__())
|
||||
return ptr
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 关键字查找表
|
||||
# 由于 TransPyC 无 dict 字面量，用链表实现
|
||||
# ============================================================
|
||||
class KwEntry:
|
||||
name: str
|
||||
kw_id: t.CInt
|
||||
next: KwEntry | t.CPtr
|
||||
|
||||
def __new__(self, pool: memhub.MemManager | t.CPtr):
|
||||
ptr: KwEntry | t.CPtr = pool.alloc(KwEntry.__sizeof__())
|
||||
if ptr:
|
||||
string.memset(ptr, 0, KwEntry.__sizeof__())
|
||||
return ptr
|
||||
|
||||
|
||||
# 全局关键字链表头（懒初始化）
|
||||
_kw_head: KwEntry | t.CPtr = None
|
||||
_kw_pool: memhub.MemManager | t.CPtr = None
|
||||
|
||||
|
||||
def _kw_intern(pool: memhub.MemManager | t.CPtr, name: str, kw_id: t.CInt):
|
||||
"""将关键字名注册到链表"""
|
||||
global _kw_head
|
||||
entry: KwEntry | t.CPtr = KwEntry(pool)
|
||||
if entry == None: return
|
||||
slen: t.CSizeT = string.strlen(name)
|
||||
buf: str = pool.alloc(slen + 1)
|
||||
if buf == None: return
|
||||
string.memcpy(buf, name, slen + 1)
|
||||
entry.name = buf
|
||||
entry.kw_id = kw_id
|
||||
entry.next = _kw_head
|
||||
_kw_head = entry
|
||||
|
||||
|
||||
def _kw_lookup(name: str) -> t.CInt:
|
||||
"""查找关键字，返回 kw_id（0 表示非关键字）"""
|
||||
cur: KwEntry | t.CPtr = _kw_head
|
||||
while cur != None:
|
||||
if string.strcmp(cur.name, name) == 0:
|
||||
return cur.kw_id
|
||||
cur = cur.next
|
||||
return 0
|
||||
|
||||
|
||||
def _init_keywords(pool: memhub.MemManager | t.CPtr):
|
||||
"""初始化关键字表（仅一次）"""
|
||||
global _kw_head, _kw_pool
|
||||
if _kw_head != None: return
|
||||
_kw_pool = pool
|
||||
_kw_intern(pool, "False", Keyword.False_)
|
||||
_kw_intern(pool, "None", Keyword.None_)
|
||||
_kw_intern(pool, "True", Keyword.True_)
|
||||
_kw_intern(pool, "and", Keyword.And)
|
||||
_kw_intern(pool, "as", Keyword.As)
|
||||
_kw_intern(pool, "assert", Keyword.Assert)
|
||||
_kw_intern(pool, "async", Keyword.Async)
|
||||
_kw_intern(pool, "await", Keyword.Await)
|
||||
_kw_intern(pool, "break", Keyword.Break)
|
||||
_kw_intern(pool, "class", Keyword.Class)
|
||||
_kw_intern(pool, "continue", Keyword.Continue)
|
||||
_kw_intern(pool, "def", Keyword.Def)
|
||||
_kw_intern(pool, "del", Keyword.Del)
|
||||
_kw_intern(pool, "elif", Keyword.Elif)
|
||||
_kw_intern(pool, "else", Keyword.Else)
|
||||
_kw_intern(pool, "except", Keyword.Except)
|
||||
_kw_intern(pool, "finally", Keyword.Finally)
|
||||
_kw_intern(pool, "for", Keyword.For)
|
||||
_kw_intern(pool, "from", Keyword.From)
|
||||
_kw_intern(pool, "global", Keyword.Global)
|
||||
_kw_intern(pool, "if", Keyword.If)
|
||||
_kw_intern(pool, "import", Keyword.Import)
|
||||
_kw_intern(pool, "in", Keyword.In)
|
||||
_kw_intern(pool, "is", Keyword.Is)
|
||||
_kw_intern(pool, "lambda", Keyword.Lambda)
|
||||
_kw_intern(pool, "nonlocal", Keyword.Nonlocal)
|
||||
_kw_intern(pool, "not", Keyword.Not)
|
||||
_kw_intern(pool, "or", Keyword.Or)
|
||||
_kw_intern(pool, "pass", Keyword.Pass)
|
||||
_kw_intern(pool, "raise", Keyword.Raise)
|
||||
_kw_intern(pool, "return", Keyword.Return)
|
||||
_kw_intern(pool, "try", Keyword.Try)
|
||||
_kw_intern(pool, "while", Keyword.While)
|
||||
_kw_intern(pool, "with", Keyword.With)
|
||||
_kw_intern(pool, "yield", Keyword.Yield)
|
||||
_kw_intern(pool, "match", Keyword.Match)
|
||||
_kw_intern(pool, "case", Keyword.Case)
|
||||
_kw_intern(pool, "type", Keyword.Type)
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 运算符查找表
|
||||
# 由于 TransPyC 无 dict，用数组+线性查找
|
||||
# 运算符按长度降序排列，以便贪婪匹配
|
||||
# ============================================================
|
||||
class OpEntry:
|
||||
name: str # 运算符文本（如 "==", "+="）
|
||||
op_id: t.CInt
|
||||
length: t.CInt
|
||||
next: OpEntry | t.CPtr
|
||||
|
||||
def __new__(self, pool: memhub.MemManager | t.CPtr):
|
||||
ptr: OpEntry | t.CPtr = pool.alloc(OpEntry.__sizeof__())
|
||||
if ptr:
|
||||
string.memset(ptr, 0, OpEntry.__sizeof__())
|
||||
return ptr
|
||||
|
||||
|
||||
_op_head: OpEntry | t.CPtr = None
|
||||
|
||||
|
||||
def _op_intern(pool: memhub.MemManager | t.CPtr, name: str, op_id: t.CInt):
|
||||
global _op_head
|
||||
slen: t.CSizeT = string.strlen(name)
|
||||
entry: OpEntry | t.CPtr = OpEntry(pool)
|
||||
if entry == None: return
|
||||
buf: str = pool.alloc(slen + 1)
|
||||
if buf == None: return
|
||||
string.memcpy(buf, name, slen + 1)
|
||||
entry.name = buf
|
||||
entry.op_id = op_id
|
||||
entry.length = t.CInt(slen)
|
||||
entry.next = _op_head
|
||||
_op_head = entry
|
||||
|
||||
|
||||
def _init_operators(pool: memhub.MemManager | t.CPtr):
|
||||
"""初始化运算符表（按长度降序注册，便于贪婪匹配）"""
|
||||
global _op_head
|
||||
if _op_head != None: return
|
||||
# 3 字符运算符
|
||||
_op_intern(pool, "**=", TokOp.StarEqEq)
|
||||
_op_intern(pool, "//=", TokOp.DSlashEq)
|
||||
_op_intern(pool, ">>=", TokOp.GtGtEq)
|
||||
_op_intern(pool, "<<=", TokOp.LtLtEq)
|
||||
_op_intern(pool, "...", TokOp.Ellipsis)
|
||||
# 2 字符运算符
|
||||
_op_intern(pool, "->", TokOp.RArrow)
|
||||
_op_intern(pool, "+=", TokOp.PlusEq)
|
||||
_op_intern(pool, "-=", TokOp.MinusEq)
|
||||
_op_intern(pool, "*=", TokOp.StarEq)
|
||||
_op_intern(pool, "/=", TokOp.SlashEq)
|
||||
_op_intern(pool, "%=", TokOp.PercentEq)
|
||||
_op_intern(pool, "@=", TokOp.AtEq)
|
||||
_op_intern(pool, "&=", TokOp.AmpEq)
|
||||
_op_intern(pool, "|=", TokOp.VBarEq)
|
||||
_op_intern(pool, "^=", TokOp.CaretEq)
|
||||
_op_intern(pool, "//", TokOp.DSlash)
|
||||
_op_intern(pool, "**", TokOp.StarStar)
|
||||
_op_intern(pool, "<<", TokOp.LtLt)
|
||||
_op_intern(pool, ">>", TokOp.GtGt)
|
||||
_op_intern(pool, "<=", TokOp.LessEq)
|
||||
_op_intern(pool, ">=", TokOp.GreaterEq)
|
||||
_op_intern(pool, "==", TokOp.EqEq)
|
||||
_op_intern(pool, "!=", TokOp.ExclaimEq)
|
||||
_op_intern(pool, ":=", TokOp.ColonEq)
|
||||
# 1 字符运算符
|
||||
_op_intern(pool, "(", TokOp.LPar)
|
||||
_op_intern(pool, ")", TokOp.RPar)
|
||||
_op_intern(pool, "[", TokOp.Lsqb)
|
||||
_op_intern(pool, "]", TokOp.Rsqb)
|
||||
_op_intern(pool, "{", TokOp.LBrace)
|
||||
_op_intern(pool, "}", TokOp.RBrace)
|
||||
_op_intern(pool, ",", TokOp.Comma)
|
||||
_op_intern(pool, ":", TokOp.Colon)
|
||||
_op_intern(pool, ".", TokOp.Dot)
|
||||
_op_intern(pool, ";", TokOp.Semi)
|
||||
_op_intern(pool, "@", TokOp.At)
|
||||
_op_intern(pool, "=", TokOp.Equal)
|
||||
_op_intern(pool, "<", TokOp.Less)
|
||||
_op_intern(pool, ">", TokOp.Greater)
|
||||
_op_intern(pool, "+", TokOp.Plus)
|
||||
_op_intern(pool, "-", TokOp.Minus)
|
||||
_op_intern(pool, "*", TokOp.Star)
|
||||
_op_intern(pool, "/", TokOp.Slash)
|
||||
_op_intern(pool, "%", TokOp.Percent)
|
||||
_op_intern(pool, "&", TokOp.Amp)
|
||||
_op_intern(pool, "|", TokOp.VBar)
|
||||
_op_intern(pool, "^", TokOp.Caret)
|
||||
_op_intern(pool, "~", TokOp.Tilde)
|
||||
|
||||
|
||||
def _init_tables(pool: memhub.MemManager | t.CPtr):
|
||||
"""初始化所有查找表（幂等）"""
|
||||
_init_keywords(pool)
|
||||
_init_operators(pool)
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Token 创建辅助函数
|
||||
# ============================================================
|
||||
|
||||
def new_token(pool: memhub.MemManager | t.CPtr, ttype: t.CInt, lineno: t.CInt,
|
||||
col_offset: t.CInt) -> Token | t.CPtr:
|
||||
"""创建一个空 Token，调用方填充字段"""
|
||||
tok: Token | t.CPtr = Token(pool)
|
||||
if tok == None: return None
|
||||
tok.type = ttype
|
||||
tok.lineno = lineno
|
||||
tok.col_offset = col_offset
|
||||
tok.end_lineno = lineno
|
||||
tok.end_col_offset = col_offset
|
||||
return tok
|
||||
|
||||
|
||||
def token_set_str(pool: memhub.MemManager | t.CPtr, tok: Token | t.CPtr,
|
||||
src: str, start: t.CSizeT, length: t.CSizeT):
|
||||
"""从 src[start:start+length] 复制文本到 token.str_val
|
||||
|
||||
注意：length == 0 时也分配 1 字节存 NUL，让空字符串与 NULL 指针区分开。
|
||||
"""
|
||||
if tok == None: return
|
||||
buf: str = pool.alloc(length + 1)
|
||||
if buf == None: return
|
||||
i: t.CSizeT = 0
|
||||
while i < length:
|
||||
buf[i] = src[start + i]
|
||||
i += 1
|
||||
buf[length] = '\0'
|
||||
tok.str_val = buf
|
||||
|
||||
|
||||
def token_set_str_literal(pool: memhub.MemManager | t.CPtr, tok: Token | t.CPtr,
|
||||
text: str):
|
||||
"""直接复制一个 C 字符串到 token.str_val"""
|
||||
if tok == None or text == None: return
|
||||
slen: t.CSizeT = string.strlen(text)
|
||||
buf: str = pool.alloc(slen + 1)
|
||||
if buf == None: return
|
||||
string.memcpy(buf, text, slen + 1)
|
||||
tok.str_val = buf
|
||||
Reference in New Issue
Block a user