GVSDS/TransPyC
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
6fc9e19e2462ac9d12bbb5ee4393d92db6236e7d
TransPyC/lib/vast/Tokenizer.py
T
TermiNexus 6fc9e19e24 feat(vast): 新增自研 Viper/Python 语法解析器取代标准库 ast 
TPC 侧取缔对 Python 标准库 ast 模块的依赖，改用 lib/vast 包：
- Nodes: 与 CPython 3.12 ast 完全同构的 131 个节点类(含抽象基类继承)
- Tokenizer: 词法分析(缩进/字符串/f-string/数字/运算符/软关键字)
- ExprParser: 表达式递归下降解析(优先级/lambda/推导式/f-string/切片)
- Parser: 语句解析(def/class/if/for/try/with/match/import/注解)
- Utils/Unparser: walk/dump/unparse/copy_location/literal_eval 等辅助
- 软关键字 fn/fun/func/function 并入解析器, 废除 ExpandSoftKeywords 预处理
- lib/_bootstrap 将 lib 目录加入 sys.path, import vast as ast 零修改替换
- 52 个文件 import ast 替换为 import vast as ast; 94 文件差分测试 0 失败
2026-08-17 09:36:38 +08:00

491 lines
20 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""vast.tokenizer - Viper/Python 词法分析器
将源码字符串切分为 token 流，与 CPython 3.12 ``tokenize`` 语义对齐：
- 支持缩进 INDENT/DEDENT、逻辑行 NEWLINE、括号内续行
- 支持字符串字面量（单引号/双引号/三引号、r/b/u/f 前缀与组合前缀）
- f-string 作为整体 FSTRING token 输出，内部表达式由 Parser 递归解析
- 数字字面量（十进制/十六进制/八进制/二进制/浮点/复数 j/下划线分隔）
- 运算符最长匹配
位置信息与 CPython 一致：``col_offset`` 为 UTF-8 字节偏移（非字符偏移），
行号为 1-based。行列信息通过行起始索引表 + bisect 统一换算，保证跨行
token（字符串/三引号）的行列准确。
软关键字 fn/fun/func/function 不作为独立 token 类型处理（并入解析器），
tokenizer 产出普通 NAME，由 Parser 在语句首识别为函数定义关键字。
"""
from __future__ import annotations
import bisect
# =============================================================================
# Token 类型常量
# =============================================================================
ENDMARKER = 'ENDMARKER'
NAME = 'NAME'
NUMBER = 'NUMBER'
STRING = 'STRING'
FSTRING = 'FSTRING'
OP = 'OP'
NEWLINE = 'NEWLINE'
INDENT = 'INDENT'
DEDENT = 'DEDENT'
COMMENT = 'COMMENT'
NL = 'NL'
# 多字符运算符（最长优先，含 Python 3.12 全部运算符）
_OPERATORS: tuple[str, ...] = (
'**=', '//=', '<<=', '>>=', '**', '//', '<<', '>>', '<=', '>=', '==', '!=',
':=', '->', '+=', '-=', '*=', '/=', '%=', '@=', '&=', '|=', '^=', '...',
'(', ')', '[', ']', '{', '}', ',', ':', '.', ';', '+', '-', '*', '/', '%',
'@', '&', '|', '^', '~', '<', '>', '=', '!',
)
# 行首空白 + 关键字 + 空白 + 标识符 → 软关键字函数定义（Parser 层处理）
SOFT_KEYWORDS: frozenset[str] = frozenset({'fn', 'fun', 'func', 'function'})
# 字符串前缀字母集合（允许任意顺序/大小写组合，如 fr、Rb、BF）
_STRING_PREFIX_CHARS: frozenset[str] = frozenset('rRbBuUfF')
_F_PREFIX = frozenset('fF')
_R_PREFIX = frozenset('rR')
_B_PREFIX = frozenset('bB')
class Token:
"""词法 token。
Attributes:
Type: token 类型（NAME/NUMBER/STRING/FSTRING/OP/NEWLINE/...）
Value: token 原始文本
Line: 1-based 行号
Col: UTF-8 字节偏移（0-based）
EndLine: 结束行号
EndCol: 结束列（UTF-8 字节偏移）
Start: 源码字符偏移（含 UTF-8 字节偏移换算依赖，供 f-string 内部定位）
End: 源码字符偏移（exclusive）
"""
__slots__ = ('Type', 'Value', 'Line', 'Col', 'EndLine', 'EndCol', 'Start', 'End')
def __init__(self, tok_type: str, value: str, line: int, col: int,
end_line: int, end_col: int) -> None:
self.Type: str = tok_type
self.Value: str = value
self.Line: int = line
self.Col: int = col
self.EndLine: int = end_line
self.EndCol: int = end_col
self.Start: int = -1
self.End: int = -1
def __repr__(self) -> str:
return (f'Token({self.Type}, {self.Value!r}, '
f'{self.Line}:{self.Col}-{self.EndLine}:{self.EndCol})')
def _IsIdentStart(ch: str) -> bool:
"""标识符起始字符：字母或下划线（兼容非 ASCII）。"""
return ch == '_' or ch.isalpha() or ord(ch) > 127
def _IsIdentPart(ch: str) -> bool:
"""标识符组成部分：字母/数字/下划线。"""
return ch == '_' or ch.isalnum() or ord(ch) > 127
class Tokenizer:
"""Viper/Python 源码 → token 流。
Usage::
toks: list[Token] = Tokenizer(source).Tokenize()
"""
def __init__(self, source: str) -> None:
self.Source: str = source
self.Length: int = len(source)
self.Pos: int = 0 # 当前字符索引
self.Tokens: list[Token] = []
# 括号深度：>0 表示在括号内（逻辑行不结束）
self.BracketDepth: int = 0
# 缩进栈：0 为根
self.Indents: list[int] = [0]
# 行首标志（决定是否进行缩进分析）
self.AtLineStart: bool = True
# 反斜杠续行延续标志：续行后的物理行行首空白不产生 INDENT
self.ContinuationLine: bool = False
# 空行/注释行标志：本物理行无实质内容，行尾换行产 NL 而非 NEWLINE
self.BlankLine: bool = False
# 注释 token 收集（Parser 忽略，但保留供诊断）
self.Comments: list[Token] = []
# 行起始字符索引表（索引 0 对应第 1 行），用于行列换算
self._LineStarts: list[int] = [0]
# ------------------------------------------------------------------
# 位置换算
# ------------------------------------------------------------------
def _PosInfo(self, pos: int) -> tuple[int, int]:
"""将字符索引换算为 (1-based 行号, UTF-8 字节列偏移)。"""
idx: int = bisect.bisect_right(self._LineStarts, pos) - 1
line: int = idx + 1
col_bytes: int = len(self.Source[self._LineStarts[idx]:pos].encode('utf-8'))
return line, col_bytes
def _Make(self, tok_type: str, start: int, end: int) -> Token:
"""以 [start, end) 源码区间构造 token。"""
start_line, start_col = self._PosInfo(start)
end_line, end_col = self._PosInfo(end)
tok: Token = Token(tok_type, self.Source[start:end], start_line, start_col,
end_line, end_col)
tok.Start = start
tok.End = end
return tok
def _RegisterNewline(self, pos_after_nl: int) -> None:
"""登记换行：pos_after_nl 是换行序列之后的字符索引（即新行行首）。"""
self._LineStarts.append(pos_after_nl)
# ------------------------------------------------------------------
# 主入口
# ------------------------------------------------------------------
def Tokenize(self) -> list[Token]:
"""执行词法分析，返回完整 token 流（以 ENDMARKER 结尾）。"""
while self.Pos < self.Length:
ch: str = self.Source[self.Pos]
if (self.AtLineStart and self.BracketDepth == 0
and not self.ContinuationLine and ch not in '\r\n'):
self._HandleIndent()
if self.Pos >= self.Length:
break
continue
if ch in ' \t\f':
self.Pos += 1
continue
if ch == '\r':
self.Pos += 1
if self.Pos < self.Length and self.Source[self.Pos] == '\n':
self.Pos += 1
self._HandleNewline(self.Pos)
continue
if ch == '\n':
self.Pos += 1
self._HandleNewline(self.Pos)
continue
if ch == '#':
self._ScanComment()
continue
if ch == '\\':
self._ScanContinuation()
continue
if ch in '([{':
self.BracketDepth += 1
self._EmitOp()
continue
if ch in ')]}':
if self.BracketDepth > 0:
self.BracketDepth -= 1
self._EmitOp()
continue
if ch == "'" or ch == '"' or _IsIdentStart(ch):
if self._TryScanString():
continue
if ch.isdigit():
self._ScanNumber()
continue
if _IsIdentStart(ch):
self._ScanName()
continue
if ch.isdigit() or (ch == '.' and self.Pos + 1 < self.Length
and self.Source[self.Pos + 1].isdigit()):
self._ScanNumber()
continue
self._EmitOp()
self._FinishFile()
return self.Tokens
# ------------------------------------------------------------------
# 换行 / 缩进 / 续行 / 注释
# ------------------------------------------------------------------
def _HandleNewline(self, pos_after_nl: int) -> None:
"""处理物理换行：括号外为 NEWLINE，括号内为 NL。
Args:
pos_after_nl: 换行序列之后的字符索引（新行行首）。
"""
# 找换行序列起点（\n 或 \r\n）
nl_start: int = self.Pos - 1
if nl_start > 0 and self.Source[nl_start] == '\n' and self.Source[nl_start - 1] == '\r':
nl_start -= 1
if self.BracketDepth > 0 or self.BlankLine or (self.AtLineStart and not self.ContinuationLine):
self.Tokens.append(self._Make(NL, nl_start, self.Pos))
else:
self.Tokens.append(self._Make(NEWLINE, nl_start, self.Pos))
self._RegisterNewline(pos_after_nl)
self.AtLineStart = True
self.ContinuationLine = False
self.BlankLine = False
def _HandleIndent(self) -> None:
"""行首缩进分析：产出 INDENT/DEDENT 或忽略。"""
start: int = self.Pos
col: int = 0
while self.Pos < self.Length:
ch: str = self.Source[self.Pos]
if ch == ' ':
col += 1
self.Pos += 1
elif ch == '\t':
col = (col // 8 + 1) * 8
self.Pos += 1
elif ch == '\f':
self.Pos += 1
else:
break
self.AtLineStart = False
# 空行/注释行不产生缩进 token，且行尾换行应产 NL（逻辑行不结束）
if self.Pos >= self.Length or self.Source[self.Pos] in '\r\n#':
self.BlankLine = True
return
cur: int = self.Indents[-1]
if col > cur:
self.Indents.append(col)
self.Tokens.append(self._Make(INDENT, start, self.Pos))
elif col < cur:
while self.Indents and self.Indents[-1] > col:
self.Indents.pop()
self.Tokens.append(self._Make(DEDENT, self.Pos, self.Pos))
def _ScanContinuation(self) -> None:
"""反斜杠续行：跳过 \\ + 换行（逻辑行内，不产生 NEWLINE）。"""
if self.Pos + 1 < self.Length and self.Source[self.Pos + 1] in '\r\n':
self.Pos += 1
if self.Pos < self.Length and self.Source[self.Pos] == '\r':
self.Pos += 1
if self.Pos < self.Length and self.Source[self.Pos] == '\n':
self.Pos += 1
self._RegisterNewline(self.Pos)
self.AtLineStart = True
self.ContinuationLine = True
else:
self.Pos += 1
def _ScanComment(self) -> None:
"""扫描注释到行尾（不产出 COMMENT token，仅保留）。"""
start: int = self.Pos
while self.Pos < self.Length and self.Source[self.Pos] not in '\r\n':
self.Pos += 1
tok: Token = self._Make(COMMENT, start, self.Pos)
self.Comments.append(tok)
# ------------------------------------------------------------------
# 基础 token
# ------------------------------------------------------------------
def _EmitOp(self) -> None:
"""运算符最长匹配（含 ... 与增强赋值运算符）。"""
start: int = self.Pos
for op in _OPERATORS:
if self.Source.startswith(op, self.Pos):
self.Pos += len(op)
self.Tokens.append(self._Make(OP, start, self.Pos))
self.AtLineStart = False
return
# 未知字符：单字符 OP 兜底
self.Pos += 1
self.Tokens.append(self._Make(OP, start, self.Pos))
self.AtLineStart = False
def _ScanName(self) -> None:
"""扫描标识符。"""
start: int = self.Pos
while self.Pos < self.Length and _IsIdentPart(self.Source[self.Pos]):
self.Pos += 1
self.Tokens.append(self._Make(NAME, start, self.Pos))
self.AtLineStart = False
def _ScanNumber(self) -> None:
"""扫描数字字面量：进制前缀/十进制/浮点/指数/复数 j/下划线。"""
start: int = self.Pos
# 进制前缀 0x/0o/0b
if self.Source[self.Pos] == '0' and self.Pos + 1 < self.Length:
nxt: str = self.Source[self.Pos + 1]
if nxt in 'xXoObB':
self.Pos += 2
while self.Pos < self.Length and (self.Source[self.Pos].isalnum()
or self.Source[self.Pos] == '_'):
self.Pos += 1
self.Tokens.append(self._Make(NUMBER, start, self.Pos))
self.AtLineStart = False
return
# 整数部分
while self.Pos < self.Length and (self.Source[self.Pos].isdigit()
or self.Source[self.Pos] == '_'):
self.Pos += 1
# 浮点小数点
if self.Pos < self.Length and self.Source[self.Pos] == '.':
self.Pos += 1
while self.Pos < self.Length and (self.Source[self.Pos].isdigit()
or self.Source[self.Pos] == '_'):
self.Pos += 1
# 指数
if self.Pos < self.Length and self.Source[self.Pos] in 'eE':
save: int = self.Pos
self.Pos += 1
if self.Pos < self.Length and self.Source[self.Pos] in '+-':
self.Pos += 1
if self.Pos < self.Length and (self.Source[self.Pos].isdigit()
or self.Source[self.Pos] == '_'):
while self.Pos < self.Length and (self.Source[self.Pos].isdigit()
or self.Source[self.Pos] == '_'):
self.Pos += 1
else:
self.Pos = save # 不是指数，回退
# 复数 j
if self.Pos < self.Length and self.Source[self.Pos] in 'jJ':
self.Pos += 1
self.Tokens.append(self._Make(NUMBER, start, self.Pos))
self.AtLineStart = False
# ------------------------------------------------------------------
# 字符串
# ------------------------------------------------------------------
def _TryScanString(self) -> bool:
"""尝试从当前位置扫描字符串字面量（含 r/b/u/f 前缀）。
- 前缀字母序列后紧跟引号 → 按字符串扫描，返回 True
- 否则回退位置，返回 False（由调用方走标识符/数字路径）
"""
start: int = self.Pos
# 收集前缀字母
while self.Pos < self.Length and self.Source[self.Pos] in _STRING_PREFIX_CHARS:
self.Pos += 1
# 前缀后必须是引号，否则回退
if self.Pos >= self.Length or self.Source[self.Pos] not in '\'"':
self.Pos = start
return False
prefix: str = self.Source[start:self.Pos]
is_fstring: bool = any(c in _F_PREFIX for c in prefix)
quote: str = self.Source[self.Pos]
triple: bool = self.Source.startswith(quote * 3, self.Pos)
if triple:
self.Pos += 3
else:
self.Pos += 1
self._ScanStringBody(quote, triple, is_fstring)
end: int = self.Pos
tok_type: str = FSTRING if is_fstring else STRING
self.Tokens.append(self._Make(tok_type, start, end))
self.AtLineStart = False
return True
def _ScanStringBody(self, quote: str, triple: bool, is_fstring: bool) -> None:
"""扫描字符串主体直到结束引号（处理转义、f-string 花括号、跨行登记）。"""
while self.Pos < self.Length:
ch: str = self.Source[self.Pos]
if ch == '\\':
# 反斜杠转义：\{{ 中反斜杠只转义自身，后续 {{ 仍需按花括号转义处理
if self.Pos + 1 < self.Length and self.Source[self.Pos + 1] in '{}':
self.Pos += 1
else:
self.Pos += 2
continue
if ch == '\n':
# 三引号字符串跨行：登记行起始，保证后续行列换算准确
self.Pos += 1
self._RegisterNewline(self.Pos)
continue
if is_fstring and ch == '{':
if self.Source.startswith('{{', self.Pos):
self.Pos += 2
continue
self._ScanFStringExpr()
continue
if is_fstring and ch == '}':
if self.Source.startswith('}}', self.Pos):
self.Pos += 2
continue
self.Pos += 1
continue
if ch == quote:
if triple:
if self.Source.startswith(quote * 3, self.Pos):
self.Pos += 3
return
self.Pos += 1
else:
self.Pos += 1
return
self.Pos += 1
# 未闭合字符串：直接结束（Parser 层报语法错误）
def _ScanFStringExpr(self) -> None:
"""跳过 f-string 的 {expr[!conv][:spec]} 表达式段。
表达式内可含字符串字面量（含嵌套 f-string）与括号嵌套，需整体跳过；
跨行时登记行起始（三引号 f-string）。
"""
self.Pos += 1 # 跳过 {
depth: int = 1
while self.Pos < self.Length:
ch: str = self.Source[self.Pos]
if ch == '\\':
self.Pos += 2
continue
if ch == '\n':
self.Pos += 1
self._RegisterNewline(self.Pos)
continue
if ch in '\'"':
self._SkipInnerString()
continue
if ch == '(' or ch == '[' or ch == '{':
depth += 1
self.Pos += 1
continue
if ch == ')' or ch == ']':
depth -= 1
self.Pos += 1
continue
if ch == '}':
if depth == 1:
self.Pos += 1
return
depth -= 1
self.Pos += 1
continue
self.Pos += 1
def _SkipInnerString(self) -> None:
"""跳过 f-string 表达式内部的字符串字面量（含前缀与三引号）。"""
save: int = self.Pos
while self.Pos < self.Length and self.Source[self.Pos] in _STRING_PREFIX_CHARS:
self.Pos += 1
if self.Pos >= self.Length or self.Source[self.Pos] not in '\'"':
self.Pos = save
self.Pos += 1
return
inner_quote: str = self.Source[self.Pos]
triple: bool = self.Source.startswith(inner_quote * 3, self.Pos)
if triple:
self.Pos += 3
self._ScanStringBody(inner_quote, True, False)
else:
self.Pos += 1
self._ScanStringBody(inner_quote, False, False)
# ------------------------------------------------------------------
# 文件尾
# ------------------------------------------------------------------
def _FinishFile(self) -> None:
"""文件末尾：补 NEWLINE、DEDENT、ENDMARKER。"""
if self.Tokens and self.Tokens[-1].Type not in (NEWLINE, DEDENT, INDENT):
self.Tokens.append(self._Make(NEWLINE, self.Pos, self.Pos))
while len(self.Indents) > 1:
self.Indents.pop()
self.Tokens.append(self._Make(DEDENT, self.Pos, self.Pos))
self.Tokens.append(self._Make(ENDMARKER, self.Pos, self.Pos))
def TokenizeSource(source: str) -> list[Token]:
"""便捷入口：源码 → token 流。"""
return Tokenizer(source).Tokenize()
Reference in new issue View Git Blame Copy Permalink