fix(vast): 修复 Tokenizer 行号登记与 import/f-string 位置属性与 CPython 不一致

- _ScanStringBody 三引号内引号对 Pos+=1 后未 continue,双重自增
  跳过一个字符,若恰为换行则漏登 _LineStarts 行首,导致其后所有
  行号偏移 -1、字面量 col_offset 超行尾,TPC 引号判断失败将
  单引号单字符误推断为 i8*(1.LOG 编译错误根因)
- alias 节点位置改为名称起始(asname 结束),与 CPython 一致
- f-string 内部表达式绝对偏移补上 FSTRING token 源码起始
This commit is contained in:
TermiNexus committed 2026-08-17 10:08:43 +08:00
1 parent 920c0ba677
commit d7213d2197
3 files changed
+31 -18

No files matched your search

+2 -1
View File
@@ -991,7 +991,8 @@ class ExprParserMixin(TokenStream):
k += 1
self._FStringExprEnd = k
# 解析表达式:子 token 流
expr_node: N.expr = self._ParseSubExpression(expr_text, outer_tok, expr_start + rel_base)
# abs_base = FSTRING token 源码起始 + body 内偏移,映射回源码绝对偏移
expr_node: N.expr = self._ParseSubExpression(expr_text, outer_tok, outer_tok.Start + expr_start + rel_base)
fv: N.FormattedValue = N.FormattedValue(value=expr_node, conversion=conversion, format_spec=format_spec)
fv.lineno = outer_tok.Line
fv.col_offset = outer_tok.Col
+28 -17
View File
@@ -315,19 +315,23 @@ class Parser(ExprParserMixin):
start: Token = self._Next() # 'import'
names: list[N.alias] = []
while True:
name_parts: list[str] = []
asname: str | None = None
# 点分模块名
name_parts.append(self._ExpectName().Value)
# 点分模块名:alias 位置从首个名称起、到末个名称(或 asname)止
name_tok: Token = self._ExpectName()
end_tok: Token = name_tok
name_parts: list[str] = [name_tok.Value]
while self._AtOp('.'):
self._Next()
name_parts.append(self._ExpectName().Value)
end_tok = self._ExpectName()
name_parts.append(end_tok.Value)
full: str = '.'.join(name_parts)
asname: str | None = None
if self._AtName('as'):
self._Next()
asname = self._ExpectName().Value
as_tok: Token = self._ExpectName()
asname = as_tok.Value
end_tok = as_tok
alias: N.alias = N.alias(name=full, asname=asname)
self._FixLoc(alias, start)
self._FixLoc(alias, name_tok, end_tok)
names.append(alias)
if self._AtOp(','):
self._Next()
@@ -359,7 +363,7 @@ class Parser(ExprParserMixin):
self._SkipNL()
if not self._AtOp(')'):
while True:
names.append(self._ParseImportAlias(start))
names.append(self._ParseImportAlias())
self._SkipNL()
if self._AtOp(','):
self._Next()
@@ -371,13 +375,13 @@ class Parser(ExprParserMixin):
self._ExpectOp(')')
else:
if self._AtOp('*'):
self._Next()
star_tok: Token = self._Next()
alias = N.alias(name='*', asname=None)
self._FixLoc(alias, start)
self._FixLoc(alias, star_tok)
names.append(alias)
else:
while True:
names.append(self._ParseImportAlias(start))
names.append(self._ParseImportAlias())
if self._AtOp(','):
self._Next()
continue
@@ -386,15 +390,22 @@ class Parser(ExprParserMixin):
self._FixLoc(node, start)
return node
def _ParseImportAlias(self, start: Token) -> N.alias:
"""解析单个 import 别名:name [as alias]。"""
name: str = self._ExpectName().Value
def _ParseImportAlias(self) -> N.alias:
"""解析单个 import 别名:name [as alias]。
alias 节点位置与 CPython 一致:起点为首个名称 token,
终点为 asname(存在时)或名称结束 token。
"""
name_tok: Token = self._ExpectName()
asname: str | None = None
end_tok: Token = name_tok
if self._AtName('as'):
self._Next()
asname = self._ExpectName().Value
alias: N.alias = N.alias(name=name, asname=asname)
self._FixLoc(alias, start)
as_tok: Token = self._ExpectName()
asname = as_tok.Value
end_tok = as_tok
alias: N.alias = N.alias(name=name_tok.Value, asname=asname)
self._FixLoc(alias, name_tok, end_tok)
return alias
def _ParseGlobal(self, kw: str) -> N.AST:
+1
View File
@@ -411,6 +411,7 @@ class Tokenizer:
self.Pos += 3
return
self.Pos += 1
continue
else:
self.Pos += 1
return