"""vast.tokenizer - Viper/Python 词法分析器将源码字符串切分为 token 流，与 CPython 3.12 ``tokenize`` 语义对齐： - 支持缩进 INDENT/DEDENT、逻辑行 NEWLINE、括号内续行 - 支持字符串字面量（单引号/双引号/三引号、r/b/u/f 前缀与组合前缀） - f-string 作为整体 FSTRING token 输出，内部表达式由 Parser 递归解析 - 数字字面量（十进制/十六进制/八进制/二进制/浮点/复数 j/下划线分隔） - 运算符最长匹配位置信息与 CPython 一致：``col_offset`` 为 UTF-8 字节偏移（非字符偏移），行号为 1-based。行列信息通过行起始索引表 + bisect 统一换算，保证跨行 token（字符串/三引号）的行列准确。软关键字 fn/fun/func/function 不作为独立 token 类型处理（并入解析器）， tokenizer 产出普通 NAME，由 Parser 在语句首识别为函数定义关键字。 """ from __future__ import annotations import bisect # ============================================================================= # Token 类型常量 # ============================================================================= ENDMARKER = 'ENDMARKER' NAME = 'NAME' NUMBER = 'NUMBER' STRING = 'STRING' FSTRING = 'FSTRING' OP = 'OP' NEWLINE = 'NEWLINE' INDENT = 'INDENT' DEDENT = 'DEDENT' COMMENT = 'COMMENT' NL = 'NL' # 多字符运算符（最长优先，含 Python 3.12 全部运算符） _OPERATORS: tuple[str, ...] = ( '**=', '//=', '<<=', '>>=', '**', '//', '<<', '>>', '<=', '>=', '==', '!=', ':=', '->', '+=', '-=', '*=', '/=', '%=', '@=', '&=', '|=', '^=', '...', '(', ')', '[', ']', '{', '}', ',', ':', '.', ';', '+', '-', '*', '/', '%', '@', '&', '|', '^', '~', '<', '>', '=', '!', ) # 按首字符分组的运算符表（组内保持最长优先顺序）。_EmitOp 每 token 只遍历 # 同首字符的少数候选（平均 2-3 个），替代原 42 项全表 startswith 扫描 # （cProfile 热点：58 万次调用 tottime 3.6s，首字符分组后显著下降）。 _OP_BY_FIRST: dict[str, tuple[str, ...]] = {} for _op in _OPERATORS: _first: str = _op[0] if _first in _OP_BY_FIRST: _OP_BY_FIRST[_first] = _OP_BY_FIRST[_first] + (_op,) else: _OP_BY_FIRST[_first] = (_op,) # 行首空白 + 关键字 + 空白 + 标识符 → 软关键字函数定义（Parser 层处理） SOFT_KEYWORDS: frozenset[str] = frozenset({'fn', 'fun', 'func', 'function', 'extern'}) # 字符串前缀字母集合（允许任意顺序/大小写组合，如 fr、Rb、BF） _STRING_PREFIX_CHARS: frozenset[str] = frozenset('rRbBuUfF') _F_PREFIX = frozenset('fF') _R_PREFIX = frozenset('rR') _B_PREFIX = frozenset('bB') class Token: """词法 token。 Attributes: Type: token 类型（NAME/NUMBER/STRING/FSTRING/OP/NEWLINE/...） Value: token 原始文本 Line: 1-based 行号 Col: UTF-8 字节偏移（0-based） EndLine: 结束行号 EndCol: 结束列（UTF-8 字节偏移） Start: 源码字符偏移（含 UTF-8 字节偏移换算依赖，供 f-string 内部定位） End: 源码字符偏移（exclusive） """ __slots__ = ('Type', 'Value', 'Line', 'Col', 'EndLine', 'EndCol', 'Start', 'End', 'IntWidth', 'IntUnsigned') def __init__(self, tok_type: str, value: str, line: int, col: int, end_line: int, end_col: int, int_width: int = 0, int_unsigned: int = 0) -> None: self.Type: str = tok_type self.Value: str = value self.Line: int = line self.Col: int = col self.EndLine: int = end_line self.EndCol: int = end_col self.Start: int = -1 self.End: int = -1 # 整型字面量显式宽度后缀（_int/_uint/_i/_u）：0 = 未指定 self.IntWidth: int = int_width self.IntUnsigned: int = int_unsigned def __repr__(self) -> str: return (f'Token({self.Type}, {self.Value!r}, ' f'{self.Line}:{self.Col}-{self.EndLine}:{self.EndCol})') def _IsIdentStart(ch: str) -> bool: """标识符起始字符：字母或下划线（兼容非 ASCII）。""" return ch == '_' or ch.isalpha() or ord(ch) > 127 def _IsIdentPart(ch: str) -> bool: """标识符组成部分：字母/数字/下划线。""" return ch == '_' or ch.isalnum() or ord(ch) > 127 class Tokenizer: """Viper/Python 源码 → token 流。 Usage:: toks: list[Token] = Tokenizer(source).Tokenize() """ def __init__(self, source: str) -> None: self.Source: str = source self.Length: int = len(source) self.Pos: int = 0 # 当前字符索引 self.Tokens: list[Token] = [] # 括号深度：>0 表示在括号内（逻辑行不结束） self.BracketDepth: int = 0 # 缩进栈：0 为根 self.Indents: list[int] = [0] # 行首标志（决定是否进行缩进分析） self.AtLineStart: bool = True # 反斜杠续行延续标志：续行后的物理行行首空白不产生 INDENT self.ContinuationLine: bool = False # 空行/注释行标志：本物理行无实质内容，行尾换行产 NL 而非 NEWLINE self.BlankLine: bool = False # 注释 token 收集（Parser 忽略，但保留供诊断） self.Comments: list[Token] = [] # 行起始字符索引表（索引 0 对应第 1 行），用于行列换算 self._LineStarts: list[int] = [0] # ------------------------------------------------------------------ # 位置换算 # ------------------------------------------------------------------ def _PosInfo(self, pos: int) -> tuple[int, int]: """将字符索引换算为 (1-based 行号, UTF-8 字节列偏移)。优化（cProfile 热点：~300 万次调用）：列偏移对 ASCII 行用 len() 直接量取，避免每次 encode 分配字节串。行查找保持 bisect（C 级 _bisect，O(log n)）——增量 while 顺延实测反而更慢（Python 循环开销 > C bisect，且 _Make 先 start 后 end 两次调用跨行时退化）。 """ idx: int = bisect.bisect_right(self._LineStarts, pos) - 1 line: int = idx + 1 seg: str = self.Source[self._LineStarts[idx]:pos] if seg.isascii(): col_bytes: int = len(seg) else: col_bytes = len(seg.encode('utf-8')) return line, col_bytes def _Make(self, tok_type: str, start: int, end: int, int_width: int = 0, int_unsigned: int = 0) -> Token: """以 [start, end) 源码区间构造 token。""" start_line, start_col = self._PosInfo(start) end_line, end_col = self._PosInfo(end) tok: Token = Token(tok_type, self.Source[start:end], start_line, start_col, end_line, end_col, int_width, int_unsigned) tok.Start = start tok.End = end return tok def _RegisterNewline(self, pos_after_nl: int) -> None: """登记换行：pos_after_nl 是换行序列之后的字符索引（即新行行首）。""" self._LineStarts.append(pos_after_nl) # ------------------------------------------------------------------ # 主入口 # ------------------------------------------------------------------ def Tokenize(self) -> list[Token]: """执行词法分析，返回完整 token 流（以 ENDMARKER 结尾）。""" while self.Pos < self.Length: ch: str = self.Source[self.Pos] if (self.AtLineStart and self.BracketDepth == 0 and not self.ContinuationLine and ch not in '\r\n'): self._HandleIndent() if self.Pos >= self.Length: break continue if ch in ' \t\f\v': self.Pos += 1 continue if ch == '\r': self.Pos += 1 if self.Pos < self.Length and self.Source[self.Pos] == '\n': self.Pos += 1 self._HandleNewline(self.Pos) continue if ch == '\n': self.Pos += 1 self._HandleNewline(self.Pos) continue if ch == '#': self._ScanComment() continue if ch == '\\': self._ScanContinuation() continue if ch in '([{': self.BracketDepth += 1 self._EmitOp() continue if ch in ')]}': if self.BracketDepth > 0: self.BracketDepth -= 1 self._EmitOp() continue if ch == "'" or ch == '"' or _IsIdentStart(ch): if self._TryScanString(): continue if ch.isdigit(): self._ScanNumber() continue if _IsIdentStart(ch): self._ScanName() continue if ch.isdigit() or (ch == '.' and self.Pos + 1 < self.Length and self.Source[self.Pos + 1].isdigit()): self._ScanNumber() continue self._EmitOp() self._FinishFile() return self.Tokens # ------------------------------------------------------------------ # 换行 / 缩进 / 续行 / 注释 # ------------------------------------------------------------------ def _HandleNewline(self, pos_after_nl: int) -> None: """处理物理换行：括号外为 NEWLINE，括号内为 NL。 Args: pos_after_nl: 换行序列之后的字符索引（新行行首）。 """ # 找换行序列起点（\n 或 \r\n） nl_start: int = self.Pos - 1 if nl_start > 0 and self.Source[nl_start] == '\n' and self.Source[nl_start - 1] == '\r': nl_start -= 1 if self.BracketDepth > 0 or self.BlankLine or (self.AtLineStart and not self.ContinuationLine): self.Tokens.append(self._Make(NL, nl_start, self.Pos)) else: self.Tokens.append(self._Make(NEWLINE, nl_start, self.Pos)) self._RegisterNewline(pos_after_nl) self.AtLineStart = True self.ContinuationLine = False self.BlankLine = False def _HandleIndent(self) -> None: """行首缩进分析：产出 INDENT/DEDENT 或忽略。""" start: int = self.Pos col: int = 0 while self.Pos < self.Length: ch: str = self.Source[self.Pos] if ch == ' ': col += 1 self.Pos += 1 elif ch == '\t': col = (col // 8 + 1) * 8 self.Pos += 1 elif ch == '\f' or ch == '\v': # \f（换页）与 \v（垂直制表）不计入缩进列，直接跳过；\v 与 Viper # 词法层 includes/ast/lexer.vp `_is_space` 一致（对齐语言定义）。 self.Pos += 1 else: break self.AtLineStart = False # 空行/注释行不产生缩进 token，且行尾换行应产 NL（逻辑行不结束） if self.Pos >= self.Length or self.Source[self.Pos] in '\r\n#': self.BlankLine = True return cur: int = self.Indents[-1] if col > cur: self.Indents.append(col) self.Tokens.append(self._Make(INDENT, start, self.Pos)) elif col < cur: while self.Indents and self.Indents[-1] > col: self.Indents.pop() self.Tokens.append(self._Make(DEDENT, self.Pos, self.Pos)) def _ScanContinuation(self) -> None: """反斜杠续行：跳过 \\ + 换行（逻辑行内，不产生 NEWLINE）。""" if self.Pos + 1 < self.Length and self.Source[self.Pos + 1] in '\r\n': self.Pos += 1 if self.Pos < self.Length and self.Source[self.Pos] == '\r': self.Pos += 1 if self.Pos < self.Length and self.Source[self.Pos] == '\n': self.Pos += 1 self._RegisterNewline(self.Pos) self.AtLineStart = True self.ContinuationLine = True else: self.Pos += 1 def _ScanComment(self) -> None: """扫描注释到行尾（不产出 COMMENT token，仅保留）。""" start: int = self.Pos while self.Pos < self.Length and self.Source[self.Pos] not in '\r\n': self.Pos += 1 tok: Token = self._Make(COMMENT, start, self.Pos) self.Comments.append(tok) # ------------------------------------------------------------------ # 基础 token # ------------------------------------------------------------------ def _EmitOp(self) -> None: """运算符最长匹配（含 ... 与增强赋值运算符）。""" start: int = self.Pos ops: tuple[str, ...] | None = _OP_BY_FIRST.get(self.Source[self.Pos]) if ops is not None: for op in ops: if self.Source.startswith(op, self.Pos): self.Pos += len(op) self.Tokens.append(self._Make(OP, start, self.Pos)) self.AtLineStart = False return # 未知字符：单字符 OP 兜底 self.Pos += 1 self.Tokens.append(self._Make(OP, start, self.Pos)) self.AtLineStart = False def _ScanName(self) -> None: """扫描标识符（内联 _IsIdentPart 避免 70 万次函数调用开销）。""" start: int = self.Pos src: str = self.Source n: int = self.Length pos: int = self.Pos while pos < n: ch: str = src[pos] if ch == '_' or ch.isalnum() or ord(ch) > 127: pos += 1 else: break self.Pos = pos self.Tokens.append(self._Make(NAME, start, self.Pos)) self.AtLineStart = False def _TryScanWidthSuffix(self) -> tuple[int, int]: """整型字面量显式宽度后缀 `_int`/`_uint`/`_i`/`_u`。命中则推进 Pos 并返回 `(N, 无符?)`；否则返回 `(0, 0)` 且不动位置。 N 取 1..64（字面量值本身是 64 位）；后缀之后不得直接跟标识符字符。 """ src: str = self.Source pos: int = self.Pos if pos >= self.Length or src[pos] != '_': return 0, 0 p: int = pos + 1 unsigned: int = 0 if src.startswith('uint', p): p += 4 unsigned = 1 elif src.startswith('int', p): p += 3 elif src.startswith('u', p): p += 1 unsigned = 1 elif src.startswith('i', p): p += 1 else: return 0, 0 digits_start: int = p while p < self.Length and src[p].isdigit(): p += 1 if p == digits_start: return 0, 0 width: int = int(src[digits_start:p]) if width <= 0 or width > 64: return 0, 0 if p < self.Length and (src[p].isalnum() or src[p] == '_'): return 0, 0 self.Pos = p return width, unsigned def _ScanNumber(self) -> None: """扫描数字字面量：进制前缀/十进制/浮点/指数/复数 j/下划线/显式宽度后缀。""" start: int = self.Pos # 进制前缀 0x/0o/0b if self.Source[self.Pos] == '0' and self.Pos + 1 < self.Length: nxt: str = self.Source[self.Pos + 1] if nxt in 'xXoObB': self.Pos += 2 while self.Pos < self.Length and (self.Source[self.Pos].isalnum() or self.Source[self.Pos] == '_'): self.Pos += 1 hw, hu = self._TryScanWidthSuffix() self.Tokens.append(self._Make(NUMBER, start, self.Pos, hw, hu)) self.AtLineStart = False return # 整数部分 while self.Pos < self.Length and (self.Source[self.Pos].isdigit() or self.Source[self.Pos] == '_'): self.Pos += 1 # 浮点小数点 if self.Pos < self.Length and self.Source[self.Pos] == '.': self.Pos += 1 while self.Pos < self.Length and (self.Source[self.Pos].isdigit() or self.Source[self.Pos] == '_'): self.Pos += 1 # 指数 if self.Pos < self.Length and self.Source[self.Pos] in 'eE': save: int = self.Pos self.Pos += 1 if self.Pos < self.Length and self.Source[self.Pos] in '+-': self.Pos += 1 if self.Pos < self.Length and (self.Source[self.Pos].isdigit() or self.Source[self.Pos] == '_'): while self.Pos < self.Length and (self.Source[self.Pos].isdigit() or self.Source[self.Pos] == '_'): self.Pos += 1 else: self.Pos = save # 不是指数，回退 # 复数 j if self.Pos < self.Length and self.Source[self.Pos] in 'jJ': self.Pos += 1 # 显式宽度后缀（仅整数字面量；浮点/复数不接受） iw: int = 0 iu: int = 0 body: str = self.Source[start:self.Pos] if not any(c in body for c in '.eEjJ'): iw, iu = self._TryScanWidthSuffix() self.Tokens.append(self._Make(NUMBER, start, self.Pos, iw, iu)) self.AtLineStart = False # ------------------------------------------------------------------ # 字符串 # ------------------------------------------------------------------ def _TryScanString(self) -> bool: """尝试从当前位置扫描字符串字面量（含 r/b/u/f 前缀）。 - 前缀字母序列后紧跟引号 → 按字符串扫描，返回 True - 否则回退位置，返回 False（由调用方走标识符/数字路径） """ start: int = self.Pos # 收集前缀字母 while self.Pos < self.Length and self.Source[self.Pos] in _STRING_PREFIX_CHARS: self.Pos += 1 # 前缀后必须是引号，否则回退 if self.Pos >= self.Length or self.Source[self.Pos] not in '\'"': self.Pos = start return False prefix: str = self.Source[start:self.Pos] is_fstring: bool = any(c in _F_PREFIX for c in prefix) quote: str = self.Source[self.Pos] triple: bool = self.Source.startswith(quote * 3, self.Pos) if triple: self.Pos += 3 else: self.Pos += 1 self._ScanStringBody(quote, triple, is_fstring) end: int = self.Pos tok_type: str = FSTRING if is_fstring else STRING self.Tokens.append(self._Make(tok_type, start, end)) self.AtLineStart = False return True def _ScanStringBody(self, quote: str, triple: bool, is_fstring: bool) -> None: """扫描字符串主体直到结束引号（处理转义、f-string 花括号、跨行登记）。""" while self.Pos < self.Length: ch: str = self.Source[self.Pos] if ch == '\\': # 反斜杠转义：\{{ 中反斜杠只转义自身，后续 {{ 仍需按花括号转义处理 if self.Pos + 1 < self.Length and self.Source[self.Pos + 1] in '{}': self.Pos += 1 else: self.Pos += 2 continue if ch == '\n': # 三引号字符串跨行：登记行起始，保证后续行列换算准确 self.Pos += 1 self._RegisterNewline(self.Pos) continue if is_fstring and ch == '{': if self.Source.startswith('{{', self.Pos): self.Pos += 2 continue self._ScanFStringExpr() continue if is_fstring and ch == '}': if self.Source.startswith('}}', self.Pos): self.Pos += 2 continue self.Pos += 1 continue if ch == quote: if triple: if self.Source.startswith(quote * 3, self.Pos): self.Pos += 3 return self.Pos += 1 continue else: self.Pos += 1 return self.Pos += 1 # 未闭合字符串：直接结束（Parser 层报语法错误） def _ScanFStringExpr(self) -> None: """跳过 f-string 的 {expr[!conv][:spec]} 表达式段。表达式内可含字符串字面量（含嵌套 f-string）与括号嵌套，需整体跳过；跨行时登记行起始（三引号 f-string）。 """ self.Pos += 1 # 跳过 { depth: int = 1 while self.Pos < self.Length: ch: str = self.Source[self.Pos] if ch == '\\': self.Pos += 2 continue if ch == '\n': self.Pos += 1 self._RegisterNewline(self.Pos) continue if ch in '\'"': self._SkipInnerString() continue if ch == '(' or ch == '[' or ch == '{': depth += 1 self.Pos += 1 continue if ch == ')' or ch == ']': depth -= 1 self.Pos += 1 continue if ch == '}': if depth == 1: self.Pos += 1 return depth -= 1 self.Pos += 1 continue self.Pos += 1 def _SkipInnerString(self) -> None: """跳过 f-string 表达式内部的字符串字面量（含前缀与三引号）。""" save: int = self.Pos while self.Pos < self.Length and self.Source[self.Pos] in _STRING_PREFIX_CHARS: self.Pos += 1 if self.Pos >= self.Length or self.Source[self.Pos] not in '\'"': self.Pos = save self.Pos += 1 return inner_quote: str = self.Source[self.Pos] triple: bool = self.Source.startswith(inner_quote * 3, self.Pos) if triple: self.Pos += 3 self._ScanStringBody(inner_quote, True, False) else: self.Pos += 1 self._ScanStringBody(inner_quote, False, False) # ------------------------------------------------------------------ # 文件尾 # ------------------------------------------------------------------ def _FinishFile(self) -> None: """文件末尾：补 NEWLINE、DEDENT、ENDMARKER。""" if self.Tokens and self.Tokens[-1].Type not in (NEWLINE, DEDENT, INDENT): self.Tokens.append(self._Make(NEWLINE, self.Pos, self.Pos)) while len(self.Indents) > 1: self.Indents.pop() self.Tokens.append(self._Make(DEDENT, self.Pos, self.Pos)) self.Tokens.append(self._Make(ENDMARKER, self.Pos, self.Pos)) def TokenizeSource(source: str) -> list[Token]: """便捷入口：源码 → token 流。""" return Tokenizer(source).Tokenize()