69 lines
1.9 KiB
Python
69 lines
1.9 KiB
Python
"""vast tokenizer 与 CPython tokenize 差分测试（临时脚本）"""
|
|
import sys
|
|
import io
|
|
import tokenize
|
|
|
|
sys.path.insert(0, r'd:\Users\TermiNexus\Desktop\TransPyC')
|
|
from lib.vast.Tokenizer import Tokenizer
|
|
|
|
|
|
def cpy_tokens(src):
|
|
out = []
|
|
try:
|
|
for t in tokenize.generate_tokens(io.StringIO(src).readline):
|
|
out.append((tokenize.tok_name[t.type], t.string, t.start[0], t.start[1], t.end[0], t.end[1]))
|
|
except tokenize.TokenError:
|
|
pass
|
|
return out
|
|
|
|
|
|
def vast_tokens(src):
|
|
out = []
|
|
for t in Tokenizer(src).Tokenize():
|
|
out.append((t.Type, t.Value, t.Line, t.Col, t.EndLine, t.EndCol))
|
|
return out
|
|
|
|
|
|
TESTS = [
|
|
'x = 1\n',
|
|
'def foo(a, b=2):\n return a + b\n',
|
|
'if x > 0:\n y = 1\nelse:\n y = 2\n',
|
|
'a = [1, 2, 3]\nb = (x for x in a)\n',
|
|
's = "hello"\nt = chr(0x41)\n',
|
|
'f"{name} is {age} years old"\n',
|
|
'x = 0x1F + 0b101 + 1_000_000\n',
|
|
'class Foo(Bar):\n @staticmethod\n def m(self):\n pass\n',
|
|
'a\\\n + b\n',
|
|
'x = (1 +\n 2)\n',
|
|
'd = {"a": 1, "b": [1,2]}\n',
|
|
'def f(x: int = 3) -> int:\n return x\n',
|
|
'try:\n pass\nexcept Exception as e:\n raise\n',
|
|
'x = a.b.c[0].d()\n',
|
|
'fn foo():\n return 1\n',
|
|
]
|
|
|
|
|
|
def main():
|
|
failed = 0
|
|
for src in TESTS:
|
|
c = cpy_tokens(src)
|
|
v = vast_tokens(src)
|
|
# 类型序列对比（排除 COMMENT/NL/ENCODING 差异）
|
|
ct = [(x[0], x[1]) for x in c if x[0] not in ('COMMENT', 'NL', 'ENCODING')]
|
|
vt = [(x[0], x[1]) for x in v if x[0] not in ('COMMENT', 'NL')]
|
|
if ct == vt:
|
|
print('OK ', repr(src[:44]))
|
|
else:
|
|
failed += 1
|
|
print('DIFF', repr(src[:44]))
|
|
print(' cpy :', ct)
|
|
print(' vast:', vt)
|
|
print(' cpy detail :', c)
|
|
print(' vast detail:', v)
|
|
print('---')
|
|
print('FAILED:', failed)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|