fix(vast): 修复 Tokenizer 行号登记与 import/f-string 位置属性与 CPython 不一致
- _ScanStringBody 三引号内引号对 Pos+=1 后未 continue,双重自增 跳过一个字符,若恰为换行则漏登 _LineStarts 行首,导致其后所有 行号偏移 -1、字面量 col_offset 超行尾,TPC 引号判断失败将 单引号单字符误推断为 i8*(1.LOG 编译错误根因) - alias 节点位置改为名称起始(asname 结束),与 CPython 一致 - f-string 内部表达式绝对偏移补上 FSTRING token 源码起始
This commit is contained in:
1 parent
920c0ba677
commit
d7213d2197
3 files changed
+31
-18
No files matched your search
@@ -991,7 +991,8 @@ class ExprParserMixin(TokenStream):
|
||||
k += 1
|
||||
self._FStringExprEnd = k
|
||||
# 解析表达式:子 token 流
|
||||
expr_node: N.expr = self._ParseSubExpression(expr_text, outer_tok, expr_start + rel_base)
|
||||
# abs_base = FSTRING token 源码起始 + body 内偏移,映射回源码绝对偏移
|
||||
expr_node: N.expr = self._ParseSubExpression(expr_text, outer_tok, outer_tok.Start + expr_start + rel_base)
|
||||
fv: N.FormattedValue = N.FormattedValue(value=expr_node, conversion=conversion, format_spec=format_spec)
|
||||
fv.lineno = outer_tok.Line
|
||||
fv.col_offset = outer_tok.Col
|
||||
|
||||
+28
-17
@@ -315,19 +315,23 @@ class Parser(ExprParserMixin):
|
||||
start: Token = self._Next() # 'import'
|
||||
names: list[N.alias] = []
|
||||
while True:
|
||||
name_parts: list[str] = []
|
||||
asname: str | None = None
|
||||
# 点分模块名
|
||||
name_parts.append(self._ExpectName().Value)
|
||||
# 点分模块名:alias 位置从首个名称起、到末个名称(或 asname)止
|
||||
name_tok: Token = self._ExpectName()
|
||||
end_tok: Token = name_tok
|
||||
name_parts: list[str] = [name_tok.Value]
|
||||
while self._AtOp('.'):
|
||||
self._Next()
|
||||
name_parts.append(self._ExpectName().Value)
|
||||
end_tok = self._ExpectName()
|
||||
name_parts.append(end_tok.Value)
|
||||
full: str = '.'.join(name_parts)
|
||||
asname: str | None = None
|
||||
if self._AtName('as'):
|
||||
self._Next()
|
||||
asname = self._ExpectName().Value
|
||||
as_tok: Token = self._ExpectName()
|
||||
asname = as_tok.Value
|
||||
end_tok = as_tok
|
||||
alias: N.alias = N.alias(name=full, asname=asname)
|
||||
self._FixLoc(alias, start)
|
||||
self._FixLoc(alias, name_tok, end_tok)
|
||||
names.append(alias)
|
||||
if self._AtOp(','):
|
||||
self._Next()
|
||||
@@ -359,7 +363,7 @@ class Parser(ExprParserMixin):
|
||||
self._SkipNL()
|
||||
if not self._AtOp(')'):
|
||||
while True:
|
||||
names.append(self._ParseImportAlias(start))
|
||||
names.append(self._ParseImportAlias())
|
||||
self._SkipNL()
|
||||
if self._AtOp(','):
|
||||
self._Next()
|
||||
@@ -371,13 +375,13 @@ class Parser(ExprParserMixin):
|
||||
self._ExpectOp(')')
|
||||
else:
|
||||
if self._AtOp('*'):
|
||||
self._Next()
|
||||
star_tok: Token = self._Next()
|
||||
alias = N.alias(name='*', asname=None)
|
||||
self._FixLoc(alias, start)
|
||||
self._FixLoc(alias, star_tok)
|
||||
names.append(alias)
|
||||
else:
|
||||
while True:
|
||||
names.append(self._ParseImportAlias(start))
|
||||
names.append(self._ParseImportAlias())
|
||||
if self._AtOp(','):
|
||||
self._Next()
|
||||
continue
|
||||
@@ -386,15 +390,22 @@ class Parser(ExprParserMixin):
|
||||
self._FixLoc(node, start)
|
||||
return node
|
||||
|
||||
def _ParseImportAlias(self, start: Token) -> N.alias:
|
||||
"""解析单个 import 别名:name [as alias]。"""
|
||||
name: str = self._ExpectName().Value
|
||||
def _ParseImportAlias(self) -> N.alias:
|
||||
"""解析单个 import 别名:name [as alias]。
|
||||
|
||||
alias 节点位置与 CPython 一致:起点为首个名称 token,
|
||||
终点为 asname(存在时)或名称结束 token。
|
||||
"""
|
||||
name_tok: Token = self._ExpectName()
|
||||
asname: str | None = None
|
||||
end_tok: Token = name_tok
|
||||
if self._AtName('as'):
|
||||
self._Next()
|
||||
asname = self._ExpectName().Value
|
||||
alias: N.alias = N.alias(name=name, asname=asname)
|
||||
self._FixLoc(alias, start)
|
||||
as_tok: Token = self._ExpectName()
|
||||
asname = as_tok.Value
|
||||
end_tok = as_tok
|
||||
alias: N.alias = N.alias(name=name_tok.Value, asname=asname)
|
||||
self._FixLoc(alias, name_tok, end_tok)
|
||||
return alias
|
||||
|
||||
def _ParseGlobal(self, kw: str) -> N.AST:
|
||||
|
||||
@@ -411,6 +411,7 @@ class Tokenizer:
|
||||
self.Pos += 3
|
||||
return
|
||||
self.Pos += 1
|
||||
continue
|
||||
else:
|
||||
self.Pos += 1
|
||||
return
|
||||
|
||||
Reference in new issue
Block a user