TermiNexus
2dbf348329
新 cProfile（232s）显示 _PosInfo tottime 1.74s→2.84s：增量 while 顺延的 Python 循环开销大于 _bisect（C 级 O(log n)），且 _Make 先 start 后 end 两次调用跨行时退化。回滚为 bisect + 保留 isascii 快速路径（避免 encode 分配）。
523 lines
21 KiB
Python
523 lines
21 KiB
Python
"""vast.tokenizer - Viper/Python 词法分析器
|
||
|
||
将源码字符串切分为 token 流，与 CPython 3.12 ``tokenize`` 语义对齐：
|
||
|
||
- 支持缩进 INDENT/DEDENT、逻辑行 NEWLINE、括号内续行
|
||
- 支持字符串字面量（单引号/双引号/三引号、r/b/u/f 前缀与组合前缀）
|
||
- f-string 作为整体 FSTRING token 输出，内部表达式由 Parser 递归解析
|
||
- 数字字面量（十进制/十六进制/八进制/二进制/浮点/复数 j/下划线分隔）
|
||
- 运算符最长匹配
|
||
|
||
位置信息与 CPython 一致：``col_offset`` 为 UTF-8 字节偏移（非字符偏移），
|
||
行号为 1-based。行列信息通过行起始索引表 + bisect 统一换算，保证跨行
|
||
token（字符串/三引号）的行列准确。
|
||
|
||
软关键字 fn/fun/func/function 不作为独立 token 类型处理（并入解析器），
|
||
tokenizer 产出普通 NAME，由 Parser 在语句首识别为函数定义关键字。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import bisect
|
||
|
||
# =============================================================================
|
||
# Token 类型常量
|
||
# =============================================================================
|
||
ENDMARKER = 'ENDMARKER'
|
||
NAME = 'NAME'
|
||
NUMBER = 'NUMBER'
|
||
STRING = 'STRING'
|
||
FSTRING = 'FSTRING'
|
||
OP = 'OP'
|
||
NEWLINE = 'NEWLINE'
|
||
INDENT = 'INDENT'
|
||
DEDENT = 'DEDENT'
|
||
COMMENT = 'COMMENT'
|
||
NL = 'NL'
|
||
|
||
# 多字符运算符（最长优先，含 Python 3.12 全部运算符）
|
||
_OPERATORS: tuple[str, ...] = (
|
||
'**=', '//=', '<<=', '>>=', '**', '//', '<<', '>>', '<=', '>=', '==', '!=',
|
||
':=', '->', '+=', '-=', '*=', '/=', '%=', '@=', '&=', '|=', '^=', '...',
|
||
'(', ')', '[', ']', '{', '}', ',', ':', '.', ';', '+', '-', '*', '/', '%',
|
||
'@', '&', '|', '^', '~', '<', '>', '=', '!',
|
||
)
|
||
|
||
# 按首字符分组的运算符表（组内保持最长优先顺序）。_EmitOp 每 token 只遍历
|
||
# 同首字符的少数候选（平均 2-3 个），替代原 42 项全表 startswith 扫描
|
||
# （cProfile 热点：58 万次调用 tottime 3.6s，首字符分组后显著下降）。
|
||
_OP_BY_FIRST: dict[str, tuple[str, ...]] = {}
|
||
for _op in _OPERATORS:
|
||
_first: str = _op[0]
|
||
if _first in _OP_BY_FIRST:
|
||
_OP_BY_FIRST[_first] = _OP_BY_FIRST[_first] + (_op,)
|
||
else:
|
||
_OP_BY_FIRST[_first] = (_op,)
|
||
|
||
# 行首空白 + 关键字 + 空白 + 标识符 → 软关键字函数定义（Parser 层处理）
|
||
SOFT_KEYWORDS: frozenset[str] = frozenset({'fn', 'fun', 'func', 'function', 'extern'})
|
||
|
||
# 字符串前缀字母集合（允许任意顺序/大小写组合，如 fr、Rb、BF）
|
||
_STRING_PREFIX_CHARS: frozenset[str] = frozenset('rRbBuUfF')
|
||
_F_PREFIX = frozenset('fF')
|
||
_R_PREFIX = frozenset('rR')
|
||
_B_PREFIX = frozenset('bB')
|
||
|
||
|
||
class Token:
|
||
"""词法 token。
|
||
|
||
Attributes:
|
||
Type: token 类型（NAME/NUMBER/STRING/FSTRING/OP/NEWLINE/...）
|
||
Value: token 原始文本
|
||
Line: 1-based 行号
|
||
Col: UTF-8 字节偏移（0-based）
|
||
EndLine: 结束行号
|
||
EndCol: 结束列（UTF-8 字节偏移）
|
||
Start: 源码字符偏移（含 UTF-8 字节偏移换算依赖，供 f-string 内部定位）
|
||
End: 源码字符偏移（exclusive）
|
||
"""
|
||
__slots__ = ('Type', 'Value', 'Line', 'Col', 'EndLine', 'EndCol', 'Start', 'End')
|
||
|
||
def __init__(self, tok_type: str, value: str, line: int, col: int,
|
||
end_line: int, end_col: int) -> None:
|
||
self.Type: str = tok_type
|
||
self.Value: str = value
|
||
self.Line: int = line
|
||
self.Col: int = col
|
||
self.EndLine: int = end_line
|
||
self.EndCol: int = end_col
|
||
self.Start: int = -1
|
||
self.End: int = -1
|
||
|
||
def __repr__(self) -> str:
|
||
return (f'Token({self.Type}, {self.Value!r}, '
|
||
f'{self.Line}:{self.Col}-{self.EndLine}:{self.EndCol})')
|
||
|
||
|
||
def _IsIdentStart(ch: str) -> bool:
|
||
"""标识符起始字符：字母或下划线（兼容非 ASCII）。"""
|
||
return ch == '_' or ch.isalpha() or ord(ch) > 127
|
||
|
||
|
||
def _IsIdentPart(ch: str) -> bool:
|
||
"""标识符组成部分：字母/数字/下划线。"""
|
||
return ch == '_' or ch.isalnum() or ord(ch) > 127
|
||
|
||
|
||
class Tokenizer:
|
||
"""Viper/Python 源码 → token 流。
|
||
|
||
Usage::
|
||
|
||
toks: list[Token] = Tokenizer(source).Tokenize()
|
||
"""
|
||
|
||
def __init__(self, source: str) -> None:
|
||
self.Source: str = source
|
||
self.Length: int = len(source)
|
||
self.Pos: int = 0 # 当前字符索引
|
||
self.Tokens: list[Token] = []
|
||
# 括号深度：>0 表示在括号内（逻辑行不结束）
|
||
self.BracketDepth: int = 0
|
||
# 缩进栈：0 为根
|
||
self.Indents: list[int] = [0]
|
||
# 行首标志（决定是否进行缩进分析）
|
||
self.AtLineStart: bool = True
|
||
# 反斜杠续行延续标志：续行后的物理行行首空白不产生 INDENT
|
||
self.ContinuationLine: bool = False
|
||
# 空行/注释行标志：本物理行无实质内容，行尾换行产 NL 而非 NEWLINE
|
||
self.BlankLine: bool = False
|
||
# 注释 token 收集（Parser 忽略，但保留供诊断）
|
||
self.Comments: list[Token] = []
|
||
# 行起始字符索引表（索引 0 对应第 1 行），用于行列换算
|
||
self._LineStarts: list[int] = [0]
|
||
|
||
# ------------------------------------------------------------------
|
||
# 位置换算
|
||
# ------------------------------------------------------------------
|
||
def _PosInfo(self, pos: int) -> tuple[int, int]:
|
||
"""将字符索引换算为 (1-based 行号, UTF-8 字节列偏移)。
|
||
|
||
优化（cProfile 热点：~300 万次调用）：列偏移对 ASCII 行用 len()
|
||
直接量取，避免每次 encode 分配字节串。行查找保持 bisect（C 级
|
||
_bisect，O(log n)）——增量 while 顺延实测反而更慢（Python 循环
|
||
开销 > C bisect，且 _Make 先 start 后 end 两次调用跨行时退化）。
|
||
"""
|
||
idx: int = bisect.bisect_right(self._LineStarts, pos) - 1
|
||
line: int = idx + 1
|
||
seg: str = self.Source[self._LineStarts[idx]:pos]
|
||
if seg.isascii():
|
||
col_bytes: int = len(seg)
|
||
else:
|
||
col_bytes = len(seg.encode('utf-8'))
|
||
return line, col_bytes
|
||
|
||
def _Make(self, tok_type: str, start: int, end: int) -> Token:
|
||
"""以 [start, end) 源码区间构造 token。"""
|
||
start_line, start_col = self._PosInfo(start)
|
||
end_line, end_col = self._PosInfo(end)
|
||
tok: Token = Token(tok_type, self.Source[start:end], start_line, start_col,
|
||
end_line, end_col)
|
||
tok.Start = start
|
||
tok.End = end
|
||
return tok
|
||
|
||
def _RegisterNewline(self, pos_after_nl: int) -> None:
|
||
"""登记换行：pos_after_nl 是换行序列之后的字符索引（即新行行首）。"""
|
||
self._LineStarts.append(pos_after_nl)
|
||
|
||
# ------------------------------------------------------------------
|
||
# 主入口
|
||
# ------------------------------------------------------------------
|
||
def Tokenize(self) -> list[Token]:
|
||
"""执行词法分析，返回完整 token 流（以 ENDMARKER 结尾）。"""
|
||
while self.Pos < self.Length:
|
||
ch: str = self.Source[self.Pos]
|
||
if (self.AtLineStart and self.BracketDepth == 0
|
||
and not self.ContinuationLine and ch not in '\r\n'):
|
||
self._HandleIndent()
|
||
if self.Pos >= self.Length:
|
||
break
|
||
continue
|
||
if ch in ' \t\f':
|
||
self.Pos += 1
|
||
continue
|
||
if ch == '\r':
|
||
self.Pos += 1
|
||
if self.Pos < self.Length and self.Source[self.Pos] == '\n':
|
||
self.Pos += 1
|
||
self._HandleNewline(self.Pos)
|
||
continue
|
||
if ch == '\n':
|
||
self.Pos += 1
|
||
self._HandleNewline(self.Pos)
|
||
continue
|
||
if ch == '#':
|
||
self._ScanComment()
|
||
continue
|
||
if ch == '\\':
|
||
self._ScanContinuation()
|
||
continue
|
||
if ch in '([{':
|
||
self.BracketDepth += 1
|
||
self._EmitOp()
|
||
continue
|
||
if ch in ')]}':
|
||
if self.BracketDepth > 0:
|
||
self.BracketDepth -= 1
|
||
self._EmitOp()
|
||
continue
|
||
if ch == "'" or ch == '"' or _IsIdentStart(ch):
|
||
if self._TryScanString():
|
||
continue
|
||
if ch.isdigit():
|
||
self._ScanNumber()
|
||
continue
|
||
if _IsIdentStart(ch):
|
||
self._ScanName()
|
||
continue
|
||
if ch.isdigit() or (ch == '.' and self.Pos + 1 < self.Length
|
||
and self.Source[self.Pos + 1].isdigit()):
|
||
self._ScanNumber()
|
||
continue
|
||
self._EmitOp()
|
||
self._FinishFile()
|
||
return self.Tokens
|
||
|
||
# ------------------------------------------------------------------
|
||
# 换行 / 缩进 / 续行 / 注释
|
||
# ------------------------------------------------------------------
|
||
def _HandleNewline(self, pos_after_nl: int) -> None:
|
||
"""处理物理换行：括号外为 NEWLINE，括号内为 NL。
|
||
|
||
Args:
|
||
pos_after_nl: 换行序列之后的字符索引（新行行首）。
|
||
"""
|
||
# 找换行序列起点（\n 或 \r\n）
|
||
nl_start: int = self.Pos - 1
|
||
if nl_start > 0 and self.Source[nl_start] == '\n' and self.Source[nl_start - 1] == '\r':
|
||
nl_start -= 1
|
||
if self.BracketDepth > 0 or self.BlankLine or (self.AtLineStart and not self.ContinuationLine):
|
||
self.Tokens.append(self._Make(NL, nl_start, self.Pos))
|
||
else:
|
||
self.Tokens.append(self._Make(NEWLINE, nl_start, self.Pos))
|
||
self._RegisterNewline(pos_after_nl)
|
||
self.AtLineStart = True
|
||
self.ContinuationLine = False
|
||
self.BlankLine = False
|
||
|
||
def _HandleIndent(self) -> None:
|
||
"""行首缩进分析：产出 INDENT/DEDENT 或忽略。"""
|
||
start: int = self.Pos
|
||
col: int = 0
|
||
while self.Pos < self.Length:
|
||
ch: str = self.Source[self.Pos]
|
||
if ch == ' ':
|
||
col += 1
|
||
self.Pos += 1
|
||
elif ch == '\t':
|
||
col = (col // 8 + 1) * 8
|
||
self.Pos += 1
|
||
elif ch == '\f':
|
||
self.Pos += 1
|
||
else:
|
||
break
|
||
self.AtLineStart = False
|
||
# 空行/注释行不产生缩进 token，且行尾换行应产 NL（逻辑行不结束）
|
||
if self.Pos >= self.Length or self.Source[self.Pos] in '\r\n#':
|
||
self.BlankLine = True
|
||
return
|
||
cur: int = self.Indents[-1]
|
||
if col > cur:
|
||
self.Indents.append(col)
|
||
self.Tokens.append(self._Make(INDENT, start, self.Pos))
|
||
elif col < cur:
|
||
while self.Indents and self.Indents[-1] > col:
|
||
self.Indents.pop()
|
||
self.Tokens.append(self._Make(DEDENT, self.Pos, self.Pos))
|
||
|
||
def _ScanContinuation(self) -> None:
|
||
"""反斜杠续行：跳过 \\ + 换行（逻辑行内，不产生 NEWLINE）。"""
|
||
if self.Pos + 1 < self.Length and self.Source[self.Pos + 1] in '\r\n':
|
||
self.Pos += 1
|
||
if self.Pos < self.Length and self.Source[self.Pos] == '\r':
|
||
self.Pos += 1
|
||
if self.Pos < self.Length and self.Source[self.Pos] == '\n':
|
||
self.Pos += 1
|
||
self._RegisterNewline(self.Pos)
|
||
self.AtLineStart = True
|
||
self.ContinuationLine = True
|
||
else:
|
||
self.Pos += 1
|
||
|
||
def _ScanComment(self) -> None:
|
||
"""扫描注释到行尾（不产出 COMMENT token，仅保留）。"""
|
||
start: int = self.Pos
|
||
while self.Pos < self.Length and self.Source[self.Pos] not in '\r\n':
|
||
self.Pos += 1
|
||
tok: Token = self._Make(COMMENT, start, self.Pos)
|
||
self.Comments.append(tok)
|
||
|
||
# ------------------------------------------------------------------
|
||
# 基础 token
|
||
# ------------------------------------------------------------------
|
||
def _EmitOp(self) -> None:
|
||
"""运算符最长匹配（含 ... 与增强赋值运算符）。"""
|
||
start: int = self.Pos
|
||
ops: tuple[str, ...] | None = _OP_BY_FIRST.get(self.Source[self.Pos])
|
||
if ops is not None:
|
||
for op in ops:
|
||
if self.Source.startswith(op, self.Pos):
|
||
self.Pos += len(op)
|
||
self.Tokens.append(self._Make(OP, start, self.Pos))
|
||
self.AtLineStart = False
|
||
return
|
||
# 未知字符：单字符 OP 兜底
|
||
self.Pos += 1
|
||
self.Tokens.append(self._Make(OP, start, self.Pos))
|
||
self.AtLineStart = False
|
||
|
||
def _ScanName(self) -> None:
|
||
"""扫描标识符（内联 _IsIdentPart 避免 70 万次函数调用开销）。"""
|
||
start: int = self.Pos
|
||
src: str = self.Source
|
||
n: int = self.Length
|
||
pos: int = self.Pos
|
||
while pos < n:
|
||
ch: str = src[pos]
|
||
if ch == '_' or ch.isalnum() or ord(ch) > 127:
|
||
pos += 1
|
||
else:
|
||
break
|
||
self.Pos = pos
|
||
self.Tokens.append(self._Make(NAME, start, self.Pos))
|
||
self.AtLineStart = False
|
||
|
||
def _ScanNumber(self) -> None:
|
||
"""扫描数字字面量：进制前缀/十进制/浮点/指数/复数 j/下划线。"""
|
||
start: int = self.Pos
|
||
# 进制前缀 0x/0o/0b
|
||
if self.Source[self.Pos] == '0' and self.Pos + 1 < self.Length:
|
||
nxt: str = self.Source[self.Pos + 1]
|
||
if nxt in 'xXoObB':
|
||
self.Pos += 2
|
||
while self.Pos < self.Length and (self.Source[self.Pos].isalnum()
|
||
or self.Source[self.Pos] == '_'):
|
||
self.Pos += 1
|
||
self.Tokens.append(self._Make(NUMBER, start, self.Pos))
|
||
self.AtLineStart = False
|
||
return
|
||
# 整数部分
|
||
while self.Pos < self.Length and (self.Source[self.Pos].isdigit()
|
||
or self.Source[self.Pos] == '_'):
|
||
self.Pos += 1
|
||
# 浮点小数点
|
||
if self.Pos < self.Length and self.Source[self.Pos] == '.':
|
||
self.Pos += 1
|
||
while self.Pos < self.Length and (self.Source[self.Pos].isdigit()
|
||
or self.Source[self.Pos] == '_'):
|
||
self.Pos += 1
|
||
# 指数
|
||
if self.Pos < self.Length and self.Source[self.Pos] in 'eE':
|
||
save: int = self.Pos
|
||
self.Pos += 1
|
||
if self.Pos < self.Length and self.Source[self.Pos] in '+-':
|
||
self.Pos += 1
|
||
if self.Pos < self.Length and (self.Source[self.Pos].isdigit()
|
||
or self.Source[self.Pos] == '_'):
|
||
while self.Pos < self.Length and (self.Source[self.Pos].isdigit()
|
||
or self.Source[self.Pos] == '_'):
|
||
self.Pos += 1
|
||
else:
|
||
self.Pos = save # 不是指数，回退
|
||
# 复数 j
|
||
if self.Pos < self.Length and self.Source[self.Pos] in 'jJ':
|
||
self.Pos += 1
|
||
self.Tokens.append(self._Make(NUMBER, start, self.Pos))
|
||
self.AtLineStart = False
|
||
|
||
# ------------------------------------------------------------------
|
||
# 字符串
|
||
# ------------------------------------------------------------------
|
||
def _TryScanString(self) -> bool:
|
||
"""尝试从当前位置扫描字符串字面量（含 r/b/u/f 前缀）。
|
||
|
||
- 前缀字母序列后紧跟引号 → 按字符串扫描，返回 True
|
||
- 否则回退位置，返回 False（由调用方走标识符/数字路径）
|
||
"""
|
||
start: int = self.Pos
|
||
# 收集前缀字母
|
||
while self.Pos < self.Length and self.Source[self.Pos] in _STRING_PREFIX_CHARS:
|
||
self.Pos += 1
|
||
# 前缀后必须是引号，否则回退
|
||
if self.Pos >= self.Length or self.Source[self.Pos] not in '\'"':
|
||
self.Pos = start
|
||
return False
|
||
prefix: str = self.Source[start:self.Pos]
|
||
is_fstring: bool = any(c in _F_PREFIX for c in prefix)
|
||
quote: str = self.Source[self.Pos]
|
||
triple: bool = self.Source.startswith(quote * 3, self.Pos)
|
||
if triple:
|
||
self.Pos += 3
|
||
else:
|
||
self.Pos += 1
|
||
self._ScanStringBody(quote, triple, is_fstring)
|
||
end: int = self.Pos
|
||
tok_type: str = FSTRING if is_fstring else STRING
|
||
self.Tokens.append(self._Make(tok_type, start, end))
|
||
self.AtLineStart = False
|
||
return True
|
||
|
||
def _ScanStringBody(self, quote: str, triple: bool, is_fstring: bool) -> None:
|
||
"""扫描字符串主体直到结束引号（处理转义、f-string 花括号、跨行登记）。"""
|
||
while self.Pos < self.Length:
|
||
ch: str = self.Source[self.Pos]
|
||
if ch == '\\':
|
||
# 反斜杠转义：\{{ 中反斜杠只转义自身，后续 {{ 仍需按花括号转义处理
|
||
if self.Pos + 1 < self.Length and self.Source[self.Pos + 1] in '{}':
|
||
self.Pos += 1
|
||
else:
|
||
self.Pos += 2
|
||
continue
|
||
if ch == '\n':
|
||
# 三引号字符串跨行：登记行起始，保证后续行列换算准确
|
||
self.Pos += 1
|
||
self._RegisterNewline(self.Pos)
|
||
continue
|
||
if is_fstring and ch == '{':
|
||
if self.Source.startswith('{{', self.Pos):
|
||
self.Pos += 2
|
||
continue
|
||
self._ScanFStringExpr()
|
||
continue
|
||
if is_fstring and ch == '}':
|
||
if self.Source.startswith('}}', self.Pos):
|
||
self.Pos += 2
|
||
continue
|
||
self.Pos += 1
|
||
continue
|
||
if ch == quote:
|
||
if triple:
|
||
if self.Source.startswith(quote * 3, self.Pos):
|
||
self.Pos += 3
|
||
return
|
||
self.Pos += 1
|
||
continue
|
||
else:
|
||
self.Pos += 1
|
||
return
|
||
self.Pos += 1
|
||
# 未闭合字符串：直接结束（Parser 层报语法错误）
|
||
|
||
def _ScanFStringExpr(self) -> None:
|
||
"""跳过 f-string 的 {expr[!conv][:spec]} 表达式段。
|
||
|
||
表达式内可含字符串字面量（含嵌套 f-string）与括号嵌套，需整体跳过；
|
||
跨行时登记行起始（三引号 f-string）。
|
||
"""
|
||
self.Pos += 1 # 跳过 {
|
||
depth: int = 1
|
||
while self.Pos < self.Length:
|
||
ch: str = self.Source[self.Pos]
|
||
if ch == '\\':
|
||
self.Pos += 2
|
||
continue
|
||
if ch == '\n':
|
||
self.Pos += 1
|
||
self._RegisterNewline(self.Pos)
|
||
continue
|
||
if ch in '\'"':
|
||
self._SkipInnerString()
|
||
continue
|
||
if ch == '(' or ch == '[' or ch == '{':
|
||
depth += 1
|
||
self.Pos += 1
|
||
continue
|
||
if ch == ')' or ch == ']':
|
||
depth -= 1
|
||
self.Pos += 1
|
||
continue
|
||
if ch == '}':
|
||
if depth == 1:
|
||
self.Pos += 1
|
||
return
|
||
depth -= 1
|
||
self.Pos += 1
|
||
continue
|
||
self.Pos += 1
|
||
|
||
def _SkipInnerString(self) -> None:
|
||
"""跳过 f-string 表达式内部的字符串字面量（含前缀与三引号）。"""
|
||
save: int = self.Pos
|
||
while self.Pos < self.Length and self.Source[self.Pos] in _STRING_PREFIX_CHARS:
|
||
self.Pos += 1
|
||
if self.Pos >= self.Length or self.Source[self.Pos] not in '\'"':
|
||
self.Pos = save
|
||
self.Pos += 1
|
||
return
|
||
inner_quote: str = self.Source[self.Pos]
|
||
triple: bool = self.Source.startswith(inner_quote * 3, self.Pos)
|
||
if triple:
|
||
self.Pos += 3
|
||
self._ScanStringBody(inner_quote, True, False)
|
||
else:
|
||
self.Pos += 1
|
||
self._ScanStringBody(inner_quote, False, False)
|
||
|
||
# ------------------------------------------------------------------
|
||
# 文件尾
|
||
# ------------------------------------------------------------------
|
||
def _FinishFile(self) -> None:
|
||
"""文件末尾：补 NEWLINE、DEDENT、ENDMARKER。"""
|
||
if self.Tokens and self.Tokens[-1].Type not in (NEWLINE, DEDENT, INDENT):
|
||
self.Tokens.append(self._Make(NEWLINE, self.Pos, self.Pos))
|
||
while len(self.Indents) > 1:
|
||
self.Indents.pop()
|
||
self.Tokens.append(self._Make(DEDENT, self.Pos, self.Pos))
|
||
self.Tokens.append(self._Make(ENDMARKER, self.Pos, self.Pos))
|
||
|
||
|
||
def TokenizeSource(source: str) -> list[Token]:
|
||
"""便捷入口：源码 → token 流。"""
|
||
return Tokenizer(source).Tokenize()
|