TermiNexus
4d5a0d203b
- __attribute__ 脱糖出的 t / c 模块根改为「隐式模块根」（零长度合成名 +
isViperImplicitModule 标记），作用域链取不到时回退 builtins 直接查
symbolTable，故不再要求源码 import t / import c；手写裸 t / c 仍报未定义。
补上 builtins.vpi 缺失的 import c（原先只有 import t，装饰器路径一直报
「未定义 c」）。
- 新增左下角状态栏分析进度：逐文件回调 → 后台线程节流 50ms 回传
analysisProgress 消息 → 前台 sendNotification('viper/analysisStatus')
→ 客户端 StatusBarItem 显示 `C:\xxx\1.vp (4/90) 4.44% loading...`，
分析结束自动隐藏，让用户能区分「插件在忙」与「插件卡死」。
- viperAttribute.test.ts 补回被误删的 hover 用例并新增 2 个回归用例；
jest 794 → 796 通过。
6639 lines
267 KiB
TypeScript
6639 lines
267 KiB
TypeScript
/*
|
||
* parser.ts
|
||
* Copyright (c) Microsoft Corporation.
|
||
* Licensed under the MIT license.
|
||
* Author: Eric Traut
|
||
*
|
||
* Based on code from python-language-server repository:
|
||
* https://github.com/Microsoft/python-language-server
|
||
*
|
||
* Parser for the Python language. Converts a stream of tokens
|
||
* into an abstract syntax tree (AST).
|
||
*/
|
||
|
||
import { Char } from '../common/charCodes';
|
||
import { appendArray } from '../common/collectionUtils';
|
||
import { assert } from '../common/debug';
|
||
import { Diagnostic, DiagnosticAddendum } from '../common/diagnostic';
|
||
import { DiagnosticSink } from '../common/diagnosticSink';
|
||
import { LanguageDialect } from '../common/dialect';
|
||
import { convertOffsetsToRange } from '../common/positionUtils';
|
||
import {
|
||
PythonVersion,
|
||
latestStablePythonVersion,
|
||
pythonVersion3_10,
|
||
pythonVersion3_11,
|
||
pythonVersion3_12,
|
||
pythonVersion3_13,
|
||
pythonVersion3_14,
|
||
pythonVersion3_15,
|
||
pythonVersion3_3,
|
||
pythonVersion3_5,
|
||
pythonVersion3_6,
|
||
pythonVersion3_8,
|
||
pythonVersion3_9,
|
||
} from '../common/pythonVersion';
|
||
import { hashString } from '../common/stringUtils';
|
||
import { TextRange } from '../common/textRange';
|
||
import { TextRangeCollection } from '../common/textRangeCollection';
|
||
import { timingStats } from '../common/timing';
|
||
import { LocAddendum, LocMessage } from '../localization/localize';
|
||
import {
|
||
ArgCategory,
|
||
ArgumentNode,
|
||
AssertNode,
|
||
AssignmentExpressionNode,
|
||
AssignmentNode,
|
||
AugmentedAssignmentNode,
|
||
AwaitNode,
|
||
BinaryOperationNode,
|
||
BreakNode,
|
||
CallNode,
|
||
CaseNode,
|
||
ClassNode,
|
||
ComprehensionForIfNode,
|
||
ComprehensionForNode,
|
||
ComprehensionIfNode,
|
||
ComprehensionNode,
|
||
ConstantNode,
|
||
ContinueNode,
|
||
DecoratorNode,
|
||
DelNode,
|
||
DictionaryEntryNode,
|
||
DictionaryExpandEntryNode,
|
||
DictionaryKeyEntryNode,
|
||
DictionaryNode,
|
||
EllipsisNode,
|
||
ErrorExpressionCategory,
|
||
ErrorNode,
|
||
ExceptNode,
|
||
ExpressionNode,
|
||
ForNode,
|
||
FormatStringNode,
|
||
FunctionAnnotationNode,
|
||
FunctionNode,
|
||
GlobalNode,
|
||
IfNode,
|
||
ImportAsNode,
|
||
ImportFromAsNode,
|
||
ImportFromNode,
|
||
ImportNode,
|
||
IndexNode,
|
||
LambdaNode,
|
||
ListNode,
|
||
MatchNode,
|
||
MemberAccessNode,
|
||
ModuleNameNode,
|
||
ModuleNode,
|
||
NameNode,
|
||
NonlocalNode,
|
||
NumberNode,
|
||
ParamCategory,
|
||
ParameterNode,
|
||
ParseNode,
|
||
ParseNodeType,
|
||
PassNode,
|
||
PatternAsNode,
|
||
PatternAtomNode,
|
||
PatternCaptureNode,
|
||
PatternClassArgumentNode,
|
||
PatternClassNode,
|
||
PatternLiteralNode,
|
||
PatternMappingEntryNode,
|
||
PatternMappingExpandEntryNode,
|
||
PatternMappingKeyEntryNode,
|
||
PatternMappingNode,
|
||
PatternSequenceNode,
|
||
PatternValueNode,
|
||
RaiseNode,
|
||
ReturnNode,
|
||
SetNode,
|
||
SliceNode,
|
||
StatementListNode,
|
||
StatementNode,
|
||
StringListNode,
|
||
StringNode,
|
||
SuiteNode,
|
||
TernaryNode,
|
||
TryNode,
|
||
TupleNode,
|
||
TypeAliasNode,
|
||
TypeAnnotationNode,
|
||
TypeParamKind,
|
||
TypeParameterListNode,
|
||
TypeParameterNode,
|
||
UnaryOperationNode,
|
||
UnpackNode,
|
||
WhileNode,
|
||
WithItemNode,
|
||
WithNode,
|
||
YieldFromNode,
|
||
YieldNode,
|
||
extendRange,
|
||
getNextNodeId,
|
||
} from './parseNodes';
|
||
import * as StringTokenUtils from './stringTokenUtils';
|
||
import { Tokenizer, TokenizerOutput } from './tokenizer';
|
||
import {
|
||
DedentToken,
|
||
FStringEndToken,
|
||
FStringMiddleToken,
|
||
FStringStartToken,
|
||
IdentifierToken,
|
||
IndentToken,
|
||
KeywordToken,
|
||
KeywordType,
|
||
NumberToken,
|
||
OperatorToken,
|
||
OperatorType,
|
||
StringToken,
|
||
StringTokenFlags,
|
||
Token,
|
||
TokenType,
|
||
} from './tokenizerTypes';
|
||
|
||
interface ListResult<T> {
|
||
list: T[];
|
||
trailingComma: boolean;
|
||
parseError?: ErrorNode | undefined;
|
||
}
|
||
|
||
interface SubscriptListResult {
|
||
list: ArgumentNode[];
|
||
trailingComma: boolean;
|
||
}
|
||
|
||
const commentRegEx = /^(\s*#\s*type:\s*)([^\r\n]*)/;
|
||
const ignoreCommentRegEx = /^ignore(\s|\[|$)/;
|
||
|
||
export class ParseOptions {
|
||
isStubFile: boolean;
|
||
pythonVersion: PythonVersion;
|
||
reportInvalidStringEscapeSequence: boolean;
|
||
skipFunctionAndClassBody: boolean;
|
||
useNotebookMode: boolean;
|
||
reportErrorsForParsedStringContents: boolean;
|
||
// Viper 方言开关；undefined 表示使用默认 Python 语义。
|
||
dialect: LanguageDialect | undefined;
|
||
|
||
constructor() {
|
||
this.isStubFile = false;
|
||
this.pythonVersion = latestStablePythonVersion;
|
||
this.reportInvalidStringEscapeSequence = false;
|
||
this.skipFunctionAndClassBody = false;
|
||
this.useNotebookMode = false;
|
||
this.reportErrorsForParsedStringContents = false;
|
||
this.dialect = undefined;
|
||
}
|
||
}
|
||
|
||
export interface ParserOutput {
|
||
parseTree: ModuleNode;
|
||
importedModules: ModuleImport[];
|
||
futureImports: Set<string>;
|
||
containsWildcardImport: boolean;
|
||
typingSymbolAliases: Map<string, string>;
|
||
hasTypeAnnotations: boolean;
|
||
lines: TextRangeCollection<TextRange>;
|
||
}
|
||
|
||
export interface ParseFileResults {
|
||
text: string;
|
||
contentHash: number;
|
||
parserOutput: ParserOutput;
|
||
tokenizerOutput: TokenizerOutput;
|
||
}
|
||
|
||
export interface ParseExpressionTextResults<T extends ParseNode> {
|
||
parseTree?: T | undefined;
|
||
lines: TextRangeCollection<TextRange>;
|
||
diagnostics: Diagnostic[];
|
||
}
|
||
|
||
export interface ModuleImport {
|
||
nameNode: ModuleNameNode;
|
||
leadingDots: number;
|
||
nameParts: string[];
|
||
|
||
// Used for "from X import Y" pattern. An empty
|
||
// array implies "from X import *".
|
||
importedSymbols: Set<string> | undefined;
|
||
}
|
||
|
||
export interface ArgListResult {
|
||
args: ArgumentNode[];
|
||
trailingComma: boolean;
|
||
}
|
||
|
||
export const enum ParseTextMode {
|
||
Expression,
|
||
VariableAnnotation,
|
||
FunctionAnnotation,
|
||
}
|
||
|
||
// Limit the max child node depth to prevent stack overflows.
|
||
const maxChildNodeDepth = 256;
|
||
|
||
export class Parser {
|
||
private _fileContents?: string;
|
||
private _tokenizerOutput?: TokenizerOutput;
|
||
private _tokens?: TextRangeCollection<Token>;
|
||
private _tokenCount = 0;
|
||
private _tokenIndex = 0;
|
||
private _areErrorsSuppressed = false;
|
||
private _parseOptions: ParseOptions = new ParseOptions();
|
||
private _diagSink: DiagnosticSink = new DiagnosticSink();
|
||
private _isInLoop = false;
|
||
private _isInFunction = false;
|
||
private _isInExceptionGroup = false;
|
||
private _isParsingTypeAnnotation = false;
|
||
private _isParsingIndexTrailer = false;
|
||
// Viper 扩展：正在解析 `[SIZE]T` 的元素类型（此时即使身处下标实参内，
|
||
// 也允许再出现 `[SIZE]T`，使 `list[[2][3]int]` 这类嵌套合法）。
|
||
private _isParsingViperBracketArrayElement = false;
|
||
// Viper 扩展：正在解析类型描述符（const / volatile）修饰的主体类型。
|
||
// 此时禁止在表达式层就地消费 C 指针后缀 `*`，使其统一冒泡到最外层
|
||
// _parseTypeAnnotation 应用 —— 保证 `const T*` 恒为 `(CConst | T)*`，
|
||
// 不因 `*` 之后是行尾还是下一行内容而在两条解析路径间摇摆
|
||
// （见 _isAtViperPointerTypeSuffix 对换行的处理）。
|
||
private _isParsingViperTypeDescriptorInner = false;
|
||
// Viper 扩展：当前所处的类声明形态，用于校验 vtable 修饰方法的出现位置：
|
||
// `virtual def` 仅允许位于 class / virtual class 内；
|
||
// `novirtual def` 允许位于 class / virtual class / novirtual class 内。
|
||
// 进入函数体（suite）时置为 undefined：方法体内的 def 不是类成员。
|
||
private _viperEnclosingClassKind: 'plain' | 'virtual' | 'novirtual' | undefined = undefined;
|
||
private _isParsingQuotedText = false;
|
||
private _isInFinallyBlock = false;
|
||
private _isInFinallyLoop = false;
|
||
private _futureImports = new Set<string>();
|
||
private _importedModules: ModuleImport[] = [];
|
||
private _containsWildcardImport = false;
|
||
private _assignmentExpressionsAllowed = true;
|
||
private _typingImportAliases: string[] = [];
|
||
private _typingSymbolAliases: Map<string, string> = new Map<string, string>();
|
||
private _maxChildDepthMap = new Map<number, number>();
|
||
private _hasTypeAnnotations = false;
|
||
|
||
parseSourceFile(fileContents: string, parseOptions: ParseOptions, diagSink: DiagnosticSink): ParseFileResults {
|
||
this._hasTypeAnnotations = false;
|
||
timingStats.tokenizeFileTime.timeOperation(() => {
|
||
this._startNewParse(fileContents, 0, fileContents.length, parseOptions, diagSink);
|
||
});
|
||
|
||
const moduleNode = ModuleNode.create({ start: 0, length: fileContents.length });
|
||
|
||
timingStats.parseFileTime.timeOperation(() => {
|
||
while (!this._atEof()) {
|
||
if (!this._consumeTokenIfType(TokenType.NewLine)) {
|
||
// Handle a common error case and try to recover.
|
||
const nextToken = this._peekToken();
|
||
if (nextToken.type === TokenType.Indent) {
|
||
this._getNextToken();
|
||
const indentToken = nextToken as IndentToken;
|
||
if (indentToken.isIndentAmbiguous) {
|
||
this._addSyntaxError(LocMessage.inconsistentTabs(), indentToken);
|
||
} else {
|
||
this._addSyntaxError(LocMessage.unexpectedIndent(), nextToken);
|
||
}
|
||
}
|
||
|
||
const statement = this._parseStatement();
|
||
if (!statement) {
|
||
// Perform basic error recovery to get to the next line.
|
||
this._consumeTokensUntilType([TokenType.NewLine]);
|
||
} else {
|
||
statement.parent = moduleNode;
|
||
moduleNode.d.statements.push(statement);
|
||
}
|
||
}
|
||
}
|
||
});
|
||
|
||
assert(this._tokenizerOutput !== undefined);
|
||
return {
|
||
text: fileContents,
|
||
contentHash: hashString(fileContents),
|
||
parserOutput: {
|
||
parseTree: moduleNode,
|
||
importedModules: this._importedModules,
|
||
futureImports: this._futureImports,
|
||
containsWildcardImport: this._containsWildcardImport,
|
||
typingSymbolAliases: this._typingSymbolAliases,
|
||
hasTypeAnnotations: this._hasTypeAnnotations,
|
||
lines: this._tokenizerOutput!.lines,
|
||
},
|
||
tokenizerOutput: this._tokenizerOutput!,
|
||
};
|
||
}
|
||
|
||
parseTextExpression(
|
||
fileContents: string,
|
||
textOffset: number,
|
||
textLength: number,
|
||
parseOptions: ParseOptions,
|
||
parseTextMode: ParseTextMode.Expression,
|
||
initialParenDepth?: number,
|
||
typingSymbolAliases?: Map<string, string>
|
||
): ParseExpressionTextResults<ExpressionNode>;
|
||
parseTextExpression(
|
||
fileContents: string,
|
||
textOffset: number,
|
||
textLength: number,
|
||
parseOptions: ParseOptions,
|
||
parseTextMode: ParseTextMode.VariableAnnotation,
|
||
initialParenDepth?: number,
|
||
typingSymbolAliases?: Map<string, string>
|
||
): ParseExpressionTextResults<ExpressionNode>;
|
||
parseTextExpression(
|
||
fileContents: string,
|
||
textOffset: number,
|
||
textLength: number,
|
||
parseOptions: ParseOptions,
|
||
parseTextMode: ParseTextMode.FunctionAnnotation,
|
||
initialParenDepth?: number,
|
||
typingSymbolAliases?: Map<string, string>
|
||
): ParseExpressionTextResults<FunctionAnnotationNode>;
|
||
parseTextExpression(
|
||
fileContents: string,
|
||
textOffset: number,
|
||
textLength: number,
|
||
parseOptions: ParseOptions,
|
||
parseTextMode = ParseTextMode.Expression,
|
||
initialParenDepth = 0,
|
||
typingSymbolAliases?: Map<string, string>
|
||
): ParseExpressionTextResults<ExpressionNode | FunctionAnnotationNode> {
|
||
const diagSink = new DiagnosticSink();
|
||
this._startNewParse(fileContents, textOffset, textLength, parseOptions, diagSink, initialParenDepth);
|
||
|
||
if (typingSymbolAliases) {
|
||
this._typingSymbolAliases = new Map<string, string>(typingSymbolAliases);
|
||
}
|
||
|
||
let parseTree: ExpressionNode | FunctionAnnotationNode | undefined;
|
||
if (parseTextMode === ParseTextMode.VariableAnnotation) {
|
||
this._isParsingQuotedText = true;
|
||
parseTree = this._parseTypeAnnotation();
|
||
} else if (parseTextMode === ParseTextMode.FunctionAnnotation) {
|
||
this._isParsingQuotedText = true;
|
||
parseTree = this._parseFunctionTypeAnnotation();
|
||
} else {
|
||
const exprListResult = this._parseTestOrStarExpressionList(
|
||
/* allowAssignmentExpression */ false,
|
||
/* allowMultipleUnpack */ true
|
||
);
|
||
if (exprListResult.parseError) {
|
||
parseTree = exprListResult.parseError;
|
||
} else {
|
||
if (exprListResult.list.length === 0) {
|
||
this._addSyntaxError(LocMessage.expectedExpr(), this._peekToken());
|
||
}
|
||
parseTree = this._makeExpressionOrTuple(exprListResult, /* enclosedInParens */ false);
|
||
}
|
||
}
|
||
|
||
if (this._peekTokenType() === TokenType.NewLine) {
|
||
this._getNextToken();
|
||
}
|
||
|
||
if (!this._atEof()) {
|
||
this._addSyntaxError(LocMessage.unexpectedExprToken(), this._peekToken());
|
||
}
|
||
|
||
return {
|
||
parseTree,
|
||
lines: this._tokenizerOutput!.lines,
|
||
diagnostics: diagSink.fetchAndClear(),
|
||
};
|
||
}
|
||
|
||
private _startNewParse(
|
||
fileContents: string,
|
||
textOffset: number,
|
||
textLength: number,
|
||
parseOptions: ParseOptions,
|
||
diagSink: DiagnosticSink,
|
||
initialParenDepth = 0
|
||
) {
|
||
this._fileContents = fileContents;
|
||
this._parseOptions = parseOptions;
|
||
this._diagSink = diagSink;
|
||
|
||
// Tokenize the file contents.
|
||
const tokenizer = new Tokenizer();
|
||
this._tokenizerOutput = tokenizer.tokenize(
|
||
fileContents,
|
||
textOffset,
|
||
textLength,
|
||
initialParenDepth,
|
||
this._parseOptions.useNotebookMode
|
||
);
|
||
this._tokens = this._tokenizerOutput.tokens;
|
||
this._tokenCount = this._tokens.count;
|
||
this._tokenIndex = 0;
|
||
}
|
||
|
||
// stmt: simple_stmt | compound_stmt
|
||
// compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt
|
||
// | funcdef | classdef | decorated | async_stmt
|
||
private _parseStatement(): StatementNode | ErrorNode | undefined {
|
||
// Handle the errant condition of a dedent token here to provide
|
||
// better recovery.
|
||
if (this._consumeTokenIfType(TokenType.Dedent)) {
|
||
this._addSyntaxError(LocMessage.unexpectedUnindent(), this._peekToken());
|
||
}
|
||
|
||
switch (this._peekKeywordType()) {
|
||
case KeywordType.ViperModifier: {
|
||
// Viper 扩展：inline/export/extern 修饰关键字。
|
||
// 与 def（或其别名）搭配时跳过修饰关键字并正常解析函数；
|
||
// extern/export 还支持直接声明形式：`extern A(...)`（等价 `extern def A(...)`），
|
||
// 此时以下一个标识符作为函数名解析函数定义；
|
||
// 其余单独使用时报错。
|
||
const modifierToken = this._peekToken() as KeywordToken;
|
||
const modifierText = this._fileContents!.slice(
|
||
modifierToken.start,
|
||
modifierToken.start + modifierToken.length
|
||
);
|
||
// Viper 扩展：`virtual class X` / `novirtual class X`（类级 vtable 标记），
|
||
// `virtual def f(...)` / `novirtual def f(...)`（方法级 vtable 标记）。
|
||
// 语义等价 `@t.CVTable class X` / `@t.NoVTable class X`（见 t.vpi），
|
||
// 仅记录在节点上供编译器使用；类型层无额外语义。
|
||
// 必须在下方通用「修饰关键字 + def」分支之前，否则方法级修饰的
|
||
// 位置校验（virtual def 不得在 novirtual class 内 / 类外）会被跳过。
|
||
if (modifierText === 'virtual' || modifierText === 'novirtual') {
|
||
const nextKeywordType =
|
||
this._peekToken(1).type === TokenType.Keyword
|
||
? (this._peekToken(1) as KeywordToken).keywordType
|
||
: undefined;
|
||
if (nextKeywordType === KeywordType.Class) {
|
||
this._getNextToken();
|
||
return this._parseClassDef(/* decorators */ undefined, modifierText);
|
||
}
|
||
if (nextKeywordType === KeywordType.Def) {
|
||
this._validateViperVirtualMethodPlacement(modifierText, modifierToken);
|
||
this._getNextToken();
|
||
// 不传 defTokenOverride：`def` 关键字仍须由 _parseFunctionDef 消费。
|
||
return this._parseFunctionDef(
|
||
/* asyncToken */ undefined,
|
||
/* decorators */ undefined,
|
||
modifierText
|
||
);
|
||
}
|
||
this._addSyntaxError(
|
||
LocMessage.viperVirtualModifierRequiresClassOrDef().format({ name: modifierText }),
|
||
modifierToken
|
||
);
|
||
this._getNextToken();
|
||
return this._parseSimpleStatement();
|
||
}
|
||
if (
|
||
this._peekToken(1).type === TokenType.Keyword &&
|
||
(this._peekToken(1) as KeywordToken).keywordType === KeywordType.Def
|
||
) {
|
||
this._getNextToken();
|
||
return this._parseFunctionDef(/* asyncToken */ undefined, /* decorators */ undefined, modifierText);
|
||
}
|
||
if (
|
||
(modifierText === 'extern' || modifierText === 'export') &&
|
||
this._peekToken(1).type === TokenType.Identifier
|
||
) {
|
||
// extern/export 直接声明形式：`extern A(...)`。
|
||
// 消费修饰关键字后，下一个标识符即函数名；defToken 用修饰关键字
|
||
// 作为范围起点（diagnostic 定位）。
|
||
this._getNextToken();
|
||
return this._parseFunctionDef(
|
||
/* asyncToken */ undefined,
|
||
/* decorators */ undefined,
|
||
modifierText,
|
||
modifierToken
|
||
);
|
||
}
|
||
// Viper 扩展：`inline while cond:` / `inline for x in y:`（内联循环标记）。
|
||
// 内联是代码生成期的提示，类型层无额外语义，此处仅消费修饰关键字并
|
||
// 按普通 while / for 语句解析（高亮由 TextMate grammar 负责）。
|
||
if (modifierText === 'inline' && this._peekToken(1).type === TokenType.Keyword) {
|
||
const nextKeywordType = (this._peekToken(1) as KeywordToken).keywordType;
|
||
if (nextKeywordType === KeywordType.While) {
|
||
this._getNextToken();
|
||
return this._parseWhileStatement();
|
||
}
|
||
if (nextKeywordType === KeywordType.For) {
|
||
this._getNextToken();
|
||
return this._parseForStatement();
|
||
}
|
||
}
|
||
this._addSyntaxError(
|
||
LocMessage.viperModifierRequiresDef().format({ name: modifierText }),
|
||
modifierToken
|
||
);
|
||
this._getNextToken();
|
||
return this._parseSimpleStatement();
|
||
}
|
||
|
||
case KeywordType.If:
|
||
return this._parseIfStatement();
|
||
|
||
case KeywordType.While:
|
||
return this._parseWhileStatement();
|
||
|
||
case KeywordType.For:
|
||
return this._parseForStatement();
|
||
|
||
case KeywordType.Try:
|
||
return this._parseTryStatement();
|
||
|
||
case KeywordType.With:
|
||
return this._parseWithStatement();
|
||
|
||
case KeywordType.Def:
|
||
return this._parseFunctionDef();
|
||
|
||
case KeywordType.Define:
|
||
// Viper 扩展：`define NAME: 类型 = 值` / `define NAME = 值` 为编译期常量声明
|
||
// （值必须能存入声明的类型位宽，见 checker 的整数字面量位宽校验）。
|
||
// 与 `define f(...)`（宏函数，等价 def）区分：常量声明的名字后跟 ':' 或 '='。
|
||
if (this._isViperDefineConstDeclaration()) {
|
||
this._getNextToken();
|
||
return this._parseSimpleStatement();
|
||
}
|
||
|
||
// Viper 扩展：define 宏函数（与 def 语法一致，语义为编译期宏展开）。
|
||
return this._parseFunctionDef();
|
||
|
||
case KeywordType.Class:
|
||
return this._parseClassDef();
|
||
|
||
case KeywordType.Async:
|
||
return this._parseAsyncStatement();
|
||
|
||
case KeywordType.Match: {
|
||
// Match is considered a "soft" keyword, so we will treat
|
||
// it as an identifier if it is followed by an unexpected
|
||
// token.
|
||
const peekToken = this._peekToken(1);
|
||
let isInvalidMatchToken = false;
|
||
|
||
if (
|
||
peekToken.type === TokenType.Colon ||
|
||
peekToken.type === TokenType.Semicolon ||
|
||
peekToken.type === TokenType.Comma ||
|
||
peekToken.type === TokenType.Dot ||
|
||
peekToken.type === TokenType.NewLine ||
|
||
peekToken.type === TokenType.EndOfStream
|
||
) {
|
||
isInvalidMatchToken = true;
|
||
} else if (peekToken.type === TokenType.Operator) {
|
||
const operatorToken = peekToken as OperatorToken;
|
||
if (
|
||
operatorToken.operatorType !== OperatorType.Multiply &&
|
||
operatorToken.operatorType !== OperatorType.Add &&
|
||
operatorToken.operatorType !== OperatorType.BitwiseInvert &&
|
||
operatorToken.operatorType !== OperatorType.Subtract
|
||
) {
|
||
isInvalidMatchToken = true;
|
||
}
|
||
}
|
||
|
||
if (!isInvalidMatchToken) {
|
||
// Try to parse the match statement. If it doesn't appear to
|
||
// be a match statement, treat as a non-keyword and reparse.
|
||
const matchStatement = this._parseMatchStatement();
|
||
if (matchStatement) {
|
||
return matchStatement;
|
||
}
|
||
}
|
||
break;
|
||
}
|
||
}
|
||
|
||
if (this._peekOperatorType() === OperatorType.MatrixMultiply) {
|
||
return this._parseDecorated();
|
||
}
|
||
|
||
return this._parseSimpleStatement();
|
||
}
|
||
|
||
// async_stmt: 'async' (funcdef | with_stmt | for_stmt)
|
||
private _parseAsyncStatement(): StatementNode | undefined {
|
||
const asyncToken = this._getKeywordToken(KeywordType.Async);
|
||
|
||
switch (this._peekKeywordType()) {
|
||
case KeywordType.Def:
|
||
return this._parseFunctionDef(asyncToken);
|
||
|
||
case KeywordType.With:
|
||
return this._parseWithStatement(asyncToken);
|
||
|
||
case KeywordType.For:
|
||
return this._parseForStatement(asyncToken);
|
||
}
|
||
|
||
this._addSyntaxError(LocMessage.unexpectedAsyncToken(), asyncToken);
|
||
|
||
return undefined;
|
||
}
|
||
|
||
// lazy_import_stmt: "lazy" import_stmt | "lazy" from_import_stmt
|
||
private _parseLazyImportStatement(): ImportNode | ImportFromNode {
|
||
const lazyToken = this._getKeywordToken(KeywordType.Lazy);
|
||
|
||
if (!this._parseOptions.isStubFile && PythonVersion.isLessThan(this._getLanguageVersion(), pythonVersion3_15)) {
|
||
this._addSyntaxError(LocMessage.lazyImportIllegal(), lazyToken);
|
||
}
|
||
|
||
const nextToken = this._peekToken();
|
||
let node: ImportNode | ImportFromNode;
|
||
|
||
if (nextToken.type === TokenType.Keyword && (nextToken as KeywordToken).keywordType === KeywordType.From) {
|
||
node = this._parseFromStatement();
|
||
} else {
|
||
node = this._parseImportStatement();
|
||
}
|
||
|
||
node.d.isLazy = true;
|
||
node.d.lazyToken = lazyToken as KeywordToken;
|
||
extendRange(node, lazyToken);
|
||
|
||
// PEP 810 disallows wildcard imports with 'lazy'.
|
||
if (node.nodeType === ParseNodeType.ImportFrom && node.d.isWildcardImport) {
|
||
this._addSyntaxError(LocMessage.lazyImportWildcardIllegal(), node.d.wildcardToken ?? lazyToken);
|
||
}
|
||
|
||
return node;
|
||
}
|
||
|
||
// type_alias_stmt: "type" name [type_param_seq] = expr
|
||
private _parseTypeAliasStatement(): TypeAliasNode {
|
||
const typeToken = this._getKeywordToken(KeywordType.Type);
|
||
|
||
const nameToken = this._getTokenIfIdentifier();
|
||
assert(nameToken !== undefined);
|
||
const name = NameNode.create(nameToken);
|
||
|
||
let typeParameters: TypeParameterListNode | undefined;
|
||
if (this._peekToken().type === TokenType.OpenBracket) {
|
||
typeParameters = this._parseTypeParameterList();
|
||
}
|
||
|
||
const assignToken = this._peekToken();
|
||
if (
|
||
assignToken.type !== TokenType.Operator ||
|
||
(assignToken as OperatorToken).operatorType !== OperatorType.Assign
|
||
) {
|
||
this._addSyntaxError(LocMessage.expectedEquals(), assignToken);
|
||
} else {
|
||
this._getNextToken();
|
||
}
|
||
|
||
const wasParsingTypeAnnotation = this._isParsingTypeAnnotation;
|
||
this._isParsingTypeAnnotation = true;
|
||
let expression = this._parseTestExpression(/* allowAssignmentExpression */ false);
|
||
// Viper 扩展：类型别名同样支持 C 指针类型后缀（type P = T*）。
|
||
expression = this._parseViperPointerTypeSuffix(expression);
|
||
this._isParsingTypeAnnotation = wasParsingTypeAnnotation;
|
||
|
||
return TypeAliasNode.create(typeToken, name, expression, typeParameters);
|
||
}
|
||
|
||
// type_param_seq: '[' (type_param ',')+ ']'
|
||
private _parseTypeParameterList(): TypeParameterListNode {
|
||
const typeVariableNodes: TypeParameterNode[] = [];
|
||
|
||
const openBracketToken = this._getNextToken();
|
||
assert(openBracketToken.type === TokenType.OpenBracket);
|
||
|
||
while (true) {
|
||
const firstToken = this._peekToken();
|
||
|
||
if (firstToken.type === TokenType.CloseBracket) {
|
||
if (typeVariableNodes.length === 0) {
|
||
this._addSyntaxError(LocMessage.typeParametersMissing(), this._peekToken());
|
||
}
|
||
break;
|
||
}
|
||
|
||
const typeVarNode = this._parseTypeParameter();
|
||
if (!typeVarNode) {
|
||
break;
|
||
}
|
||
|
||
typeVariableNodes.push(typeVarNode);
|
||
|
||
if (!this._consumeTokenIfType(TokenType.Comma)) {
|
||
break;
|
||
}
|
||
}
|
||
|
||
const closingToken = this._peekToken();
|
||
if (closingToken.type !== TokenType.CloseBracket) {
|
||
this._addSyntaxError(LocMessage.expectedCloseBracket(), this._peekToken());
|
||
this._consumeTokensUntilType([TokenType.NewLine, TokenType.CloseBracket, TokenType.Colon]);
|
||
} else {
|
||
this._getNextToken();
|
||
}
|
||
|
||
return TypeParameterListNode.create(openBracketToken, closingToken, typeVariableNodes);
|
||
}
|
||
|
||
// type_param: ['*' | '**'] NAME [':' bound_expr] ['=' default_expr]
|
||
private _parseTypeParameter(): TypeParameterNode | undefined {
|
||
let typeParamCategory = TypeParamKind.TypeVar;
|
||
if (this._consumeTokenIfOperator(OperatorType.Multiply)) {
|
||
typeParamCategory = TypeParamKind.TypeVarTuple;
|
||
} else if (this._consumeTokenIfOperator(OperatorType.Power)) {
|
||
typeParamCategory = TypeParamKind.ParamSpec;
|
||
}
|
||
|
||
const nameToken = this._getTokenIfIdentifier();
|
||
if (!nameToken) {
|
||
this._addSyntaxError(LocMessage.expectedTypeParameterName(), this._peekToken());
|
||
return undefined;
|
||
}
|
||
|
||
const name = NameNode.create(nameToken);
|
||
|
||
let boundExpression: ExpressionNode | undefined;
|
||
if (this._consumeTokenIfType(TokenType.Colon)) {
|
||
boundExpression = this._parseExpression(/* allowUnpack */ false);
|
||
|
||
if (typeParamCategory !== TypeParamKind.TypeVar) {
|
||
this._addSyntaxError(LocMessage.typeParameterBoundNotAllowed(), boundExpression);
|
||
}
|
||
}
|
||
|
||
let defaultExpression: ExpressionNode | undefined;
|
||
if (this._consumeTokenIfOperator(OperatorType.Assign)) {
|
||
defaultExpression = this._parseExpression(
|
||
/* allowUnpack */ typeParamCategory === TypeParamKind.TypeVarTuple
|
||
);
|
||
|
||
if (
|
||
!this._parseOptions.isStubFile &&
|
||
PythonVersion.isLessThan(this._getLanguageVersion(), pythonVersion3_13)
|
||
) {
|
||
this._addSyntaxError(LocMessage.typeVarDefaultIllegal(), defaultExpression);
|
||
}
|
||
}
|
||
|
||
return TypeParameterNode.create(name, typeParamCategory, boundExpression, defaultExpression);
|
||
}
|
||
|
||
// match_stmt: "match" subject_expr ':' NEWLINE INDENT case_block+ DEDENT
|
||
// subject_expr:
|
||
// | star_named_expression ',' star_named_expressions?
|
||
// | named_expression
|
||
private _parseMatchStatement(): MatchNode | undefined {
|
||
// Parse the subject expression with errors suppressed. If it's not
|
||
// followed by a colon, we'll assume this is not a match statement.
|
||
// We need to do this because "match" is considered a soft keyword,
|
||
// and we need to distinguish between "match(2)" and "match (2):"
|
||
// and between "match[2]" and "match [2]:"
|
||
let smellsLikeMatchStatement = false;
|
||
this._suppressErrors(() => {
|
||
const curTokenIndex = this._tokenIndex;
|
||
|
||
this._getKeywordToken(KeywordType.Match);
|
||
const expression = this._parseTestOrStarListAsExpression(
|
||
/* allowAssignmentExpression */ true,
|
||
/* allowMultipleUnpack */ true,
|
||
ErrorExpressionCategory.MissingPatternSubject,
|
||
() => LocMessage.expectedReturnExpr()
|
||
);
|
||
smellsLikeMatchStatement =
|
||
expression.nodeType !== ParseNodeType.Error && this._peekToken().type === TokenType.Colon;
|
||
|
||
// Set the token index back to the start.
|
||
this._tokenIndex = curTokenIndex;
|
||
});
|
||
|
||
if (!smellsLikeMatchStatement) {
|
||
return undefined;
|
||
}
|
||
|
||
const matchToken = this._getKeywordToken(KeywordType.Match);
|
||
|
||
const subjectExpression = this._parseTestOrStarListAsExpression(
|
||
/* allowAssignmentExpression */ true,
|
||
/* allowMultipleUnpack */ true,
|
||
ErrorExpressionCategory.MissingPatternSubject,
|
||
() => LocMessage.expectedReturnExpr()
|
||
);
|
||
const matchNode = MatchNode.create(matchToken, subjectExpression);
|
||
|
||
const nextToken = this._peekToken();
|
||
|
||
if (!this._consumeTokenIfType(TokenType.Colon)) {
|
||
this._addSyntaxError(LocMessage.expectedColon(), nextToken);
|
||
|
||
// Try to perform parse recovery by consuming tokens until
|
||
// we find the end of the line.
|
||
if (this._consumeTokensUntilType([TokenType.NewLine, TokenType.Colon])) {
|
||
this._getNextToken();
|
||
}
|
||
} else {
|
||
extendRange(matchNode, nextToken);
|
||
|
||
if (!this._consumeTokenIfType(TokenType.NewLine)) {
|
||
this._addSyntaxError(LocMessage.expectedNewline(), nextToken);
|
||
} else {
|
||
const possibleIndent = this._peekToken();
|
||
if (!this._consumeTokenIfType(TokenType.Indent)) {
|
||
this._addSyntaxError(LocMessage.expectedIndentedBlock(), this._peekToken());
|
||
} else {
|
||
const indentToken = possibleIndent as IndentToken;
|
||
if (indentToken.isIndentAmbiguous) {
|
||
this._addSyntaxError(LocMessage.inconsistentTabs(), indentToken);
|
||
}
|
||
}
|
||
|
||
while (true) {
|
||
// Handle a common error here and see if we can recover.
|
||
const possibleUnexpectedIndent = this._peekToken();
|
||
if (possibleUnexpectedIndent.type === TokenType.Indent) {
|
||
this._getNextToken();
|
||
const indentToken = possibleUnexpectedIndent as IndentToken;
|
||
if (indentToken.isIndentAmbiguous) {
|
||
this._addSyntaxError(LocMessage.inconsistentTabs(), indentToken);
|
||
} else {
|
||
this._addSyntaxError(LocMessage.unexpectedIndent(), possibleUnexpectedIndent);
|
||
}
|
||
}
|
||
|
||
const caseStatement = this._parseCaseStatement();
|
||
if (!caseStatement) {
|
||
// Perform basic error recovery to get to the next line.
|
||
if (this._consumeTokensUntilType([TokenType.NewLine, TokenType.Colon])) {
|
||
this._getNextToken();
|
||
}
|
||
} else {
|
||
caseStatement.parent = matchNode;
|
||
matchNode.d.cases.push(caseStatement);
|
||
}
|
||
|
||
const dedentToken = this._peekToken() as DedentToken;
|
||
if (this._consumeTokenIfType(TokenType.Dedent)) {
|
||
if (!dedentToken.matchesIndent) {
|
||
this._addSyntaxError(LocMessage.inconsistentIndent(), dedentToken);
|
||
}
|
||
if (dedentToken.isDedentAmbiguous) {
|
||
this._addSyntaxError(LocMessage.inconsistentTabs(), dedentToken);
|
||
}
|
||
break;
|
||
}
|
||
|
||
if (this._peekTokenType() === TokenType.EndOfStream) {
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
|
||
if (matchNode.d.cases.length > 0) {
|
||
extendRange(matchNode, matchNode.d.cases[matchNode.d.cases.length - 1]);
|
||
} else {
|
||
this._addSyntaxError(LocMessage.zeroCaseStatementsFound(), matchToken);
|
||
}
|
||
}
|
||
|
||
// This feature requires Python 3.10.
|
||
if (PythonVersion.isLessThan(this._getLanguageVersion(), pythonVersion3_10)) {
|
||
this._addSyntaxError(LocMessage.matchIncompatible(), matchToken);
|
||
}
|
||
|
||
// Validate that only the last entry uses an irrefutable pattern.
|
||
for (let i = 0; i < matchNode.d.cases.length - 1; i++) {
|
||
const caseNode = matchNode.d.cases[i];
|
||
if (!caseNode.d.guardExpr && caseNode.d.isIrrefutable) {
|
||
this._addSyntaxError(LocMessage.casePatternIsIrrefutable(), caseNode.d.pattern);
|
||
}
|
||
}
|
||
|
||
return matchNode;
|
||
}
|
||
|
||
// case_block: "case" patterns [guard] ':' block
|
||
// patterns: sequence_pattern | as_pattern
|
||
// guard: 'if' named_expression
|
||
private _parseCaseStatement(): CaseNode | undefined {
|
||
const caseToken = this._peekToken();
|
||
|
||
if (!this._consumeTokenIfKeyword(KeywordType.Case)) {
|
||
this._addSyntaxError(LocMessage.expectedCase(), caseToken);
|
||
return undefined;
|
||
}
|
||
|
||
const patternList = this._parsePatternSequence();
|
||
let casePattern: PatternAtomNode;
|
||
|
||
if (patternList.parseError) {
|
||
casePattern = patternList.parseError;
|
||
} else if (patternList.list.length === 0) {
|
||
this._addSyntaxError(LocMessage.expectedPatternExpr(), this._peekToken());
|
||
casePattern = ErrorNode.create(caseToken, ErrorExpressionCategory.MissingPattern);
|
||
} else if (patternList.list.length === 1 && !patternList.trailingComma) {
|
||
const pattern = patternList.list[0].d.orPatterns[0];
|
||
|
||
if (pattern.nodeType === ParseNodeType.PatternCapture && pattern.d.isStar) {
|
||
casePattern = PatternSequenceNode.create(patternList.list[0], patternList.list);
|
||
} else {
|
||
casePattern = patternList.list[0];
|
||
}
|
||
} else {
|
||
casePattern = PatternSequenceNode.create(patternList.list[0], patternList.list);
|
||
}
|
||
|
||
if (casePattern.nodeType !== ParseNodeType.Error) {
|
||
const globalNameMap = new Map<string, NameNode>();
|
||
const localNameMap = new Map<string, NameNode>();
|
||
this._reportDuplicatePatternCaptureTargets(casePattern, globalNameMap, localNameMap);
|
||
}
|
||
|
||
let guardExpression: ExpressionNode | undefined;
|
||
if (this._consumeTokenIfKeyword(KeywordType.If)) {
|
||
guardExpression = this._parseTestExpression(/* allowAssignmentExpression */ true);
|
||
}
|
||
|
||
const suite = this._parseSuite(this._isInFunction);
|
||
return CaseNode.create(caseToken, casePattern, this._isPatternIrrefutable(casePattern), guardExpression, suite);
|
||
}
|
||
|
||
// PEP 634 defines the concept of an "irrefutable" pattern - a pattern that
|
||
// will always be matched.
|
||
private _isPatternIrrefutable(node: PatternAtomNode): boolean {
|
||
if (node.nodeType === ParseNodeType.PatternCapture) {
|
||
return true;
|
||
}
|
||
|
||
if (node.nodeType === ParseNodeType.PatternAs) {
|
||
return node.d.orPatterns.some((pattern) => this._isPatternIrrefutable(pattern));
|
||
}
|
||
|
||
return false;
|
||
}
|
||
|
||
// Reports any situations where a capture target (a variable that receives part of a pattern)
|
||
// appears twice within the same pattern. This is complicated by the fact that duplicate targets
|
||
// are allowed in separate "or" clauses, so we need to track the targets we've seen globally
|
||
// as well as the targets we've seen locally within the current "or" clause.
|
||
private _reportDuplicatePatternCaptureTargets(
|
||
node: PatternAtomNode,
|
||
globalNameMap: Map<string, NameNode>,
|
||
localNameMap: Map<string, NameNode>
|
||
) {
|
||
const reportTargetIfDuplicate = (nameNode: NameNode) => {
|
||
if (globalNameMap.has(nameNode.d.value) || localNameMap.has(nameNode.d.value)) {
|
||
this._addSyntaxError(
|
||
LocMessage.duplicateCapturePatternTarget().format({
|
||
name: nameNode.d.value,
|
||
}),
|
||
nameNode
|
||
);
|
||
} else {
|
||
localNameMap.set(nameNode.d.value, nameNode);
|
||
}
|
||
};
|
||
|
||
switch (node.nodeType) {
|
||
case ParseNodeType.PatternSequence: {
|
||
node.d.entries.forEach((subpattern) => {
|
||
this._reportDuplicatePatternCaptureTargets(subpattern, globalNameMap, localNameMap);
|
||
});
|
||
break;
|
||
}
|
||
|
||
case ParseNodeType.PatternClass: {
|
||
node.d.args.forEach((arg) => {
|
||
this._reportDuplicatePatternCaptureTargets(arg.d.pattern, globalNameMap, localNameMap);
|
||
});
|
||
break;
|
||
}
|
||
|
||
case ParseNodeType.PatternAs: {
|
||
if (node.d.target) {
|
||
reportTargetIfDuplicate(node.d.target);
|
||
}
|
||
|
||
const orLocalNameMaps = node.d.orPatterns.map((subpattern) => {
|
||
const orLocalNameMap = new Map<string, NameNode>();
|
||
this._reportDuplicatePatternCaptureTargets(subpattern, localNameMap, orLocalNameMap);
|
||
return orLocalNameMap;
|
||
});
|
||
|
||
const combinedLocalOrNameMap = new Map<string, NameNode>();
|
||
orLocalNameMaps.forEach((orLocalNameMap) => {
|
||
orLocalNameMap.forEach((node) => {
|
||
if (!combinedLocalOrNameMap.has(node.d.value)) {
|
||
combinedLocalOrNameMap.set(node.d.value, node);
|
||
reportTargetIfDuplicate(node);
|
||
}
|
||
});
|
||
});
|
||
break;
|
||
}
|
||
|
||
case ParseNodeType.PatternCapture: {
|
||
if (!node.d.isWildcard) {
|
||
reportTargetIfDuplicate(node.d.target);
|
||
}
|
||
break;
|
||
}
|
||
|
||
case ParseNodeType.PatternMapping: {
|
||
node.d.entries.forEach((mapEntry) => {
|
||
if (mapEntry.nodeType === ParseNodeType.PatternMappingExpandEntry) {
|
||
reportTargetIfDuplicate(mapEntry.d.target);
|
||
} else {
|
||
this._reportDuplicatePatternCaptureTargets(mapEntry.d.keyPattern, globalNameMap, localNameMap);
|
||
this._reportDuplicatePatternCaptureTargets(
|
||
mapEntry.d.valuePattern,
|
||
globalNameMap,
|
||
localNameMap
|
||
);
|
||
}
|
||
});
|
||
break;
|
||
}
|
||
|
||
case ParseNodeType.PatternLiteral:
|
||
case ParseNodeType.PatternValue:
|
||
case ParseNodeType.Error: {
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
|
||
private _getPatternTargetNames(node: PatternAtomNode, nameSet: Set<string>): void {
|
||
switch (node.nodeType) {
|
||
case ParseNodeType.PatternSequence: {
|
||
node.d.entries.forEach((subpattern) => {
|
||
this._getPatternTargetNames(subpattern, nameSet);
|
||
});
|
||
break;
|
||
}
|
||
|
||
case ParseNodeType.PatternClass: {
|
||
node.d.args.forEach((arg) => {
|
||
this._getPatternTargetNames(arg.d.pattern, nameSet);
|
||
});
|
||
break;
|
||
}
|
||
|
||
case ParseNodeType.PatternAs: {
|
||
if (node.d.target) {
|
||
nameSet.add(node.d.target.d.value);
|
||
}
|
||
node.d.orPatterns.forEach((subpattern) => {
|
||
this._getPatternTargetNames(subpattern, nameSet);
|
||
});
|
||
break;
|
||
}
|
||
|
||
case ParseNodeType.PatternCapture: {
|
||
if (!node.d.isWildcard) {
|
||
nameSet.add(node.d.target.d.value);
|
||
}
|
||
break;
|
||
}
|
||
|
||
case ParseNodeType.PatternMapping: {
|
||
node.d.entries.forEach((mapEntry) => {
|
||
if (mapEntry.nodeType === ParseNodeType.PatternMappingExpandEntry) {
|
||
nameSet.add(mapEntry.d.target.d.value);
|
||
} else {
|
||
this._getPatternTargetNames(mapEntry.d.keyPattern, nameSet);
|
||
this._getPatternTargetNames(mapEntry.d.valuePattern, nameSet);
|
||
}
|
||
});
|
||
break;
|
||
}
|
||
|
||
case ParseNodeType.PatternLiteral:
|
||
case ParseNodeType.PatternValue:
|
||
case ParseNodeType.Error: {
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
|
||
private _parsePatternSequence() {
|
||
const patternList = this._parseExpressionListGeneric(() => this._parsePatternAs());
|
||
|
||
// Check for more than one star entry.
|
||
const starEntries = patternList.list.filter(
|
||
(entry) =>
|
||
entry.d.orPatterns.length === 1 &&
|
||
entry.d.orPatterns[0].nodeType === ParseNodeType.PatternCapture &&
|
||
entry.d.orPatterns[0].d.isStar
|
||
);
|
||
|
||
if (starEntries.length > 1) {
|
||
this._addSyntaxError(LocMessage.duplicateStarPattern(), starEntries[1].d.orPatterns[0]);
|
||
}
|
||
|
||
return patternList;
|
||
}
|
||
|
||
// as_pattern: or_pattern ['as' NAME]
|
||
// or_pattern: '|'.pattern_atom+
|
||
private _parsePatternAs(): PatternAsNode {
|
||
const orPatterns: PatternAtomNode[] = [];
|
||
|
||
while (true) {
|
||
const patternAtom = this._parsePatternAtom();
|
||
orPatterns.push(patternAtom);
|
||
|
||
if (!this._consumeTokenIfOperator(OperatorType.BitwiseOr)) {
|
||
break;
|
||
}
|
||
}
|
||
|
||
if (orPatterns.length > 1) {
|
||
// Star patterns cannot be ORed with other patterns.
|
||
orPatterns.forEach((patternAtom) => {
|
||
if (patternAtom.nodeType === ParseNodeType.PatternCapture && patternAtom.d.isStar) {
|
||
this._addSyntaxError(LocMessage.starPatternInOrPattern(), patternAtom);
|
||
}
|
||
});
|
||
}
|
||
|
||
let target: NameNode | undefined;
|
||
if (this._consumeTokenIfKeyword(KeywordType.As)) {
|
||
const nameToken = this._getTokenIfIdentifier();
|
||
if (nameToken) {
|
||
target = NameNode.create(nameToken);
|
||
} else {
|
||
this._addSyntaxError(LocMessage.expectedNameAfterAs(), this._peekToken());
|
||
}
|
||
}
|
||
|
||
// Star patterns cannot be used with AS pattern.
|
||
if (
|
||
target &&
|
||
orPatterns.length === 1 &&
|
||
orPatterns[0].nodeType === ParseNodeType.PatternCapture &&
|
||
orPatterns[0].d.isStar
|
||
) {
|
||
this._addSyntaxError(LocMessage.starPatternInAsPattern(), orPatterns[0]);
|
||
}
|
||
|
||
// Validate that irrefutable patterns are not in any entries other than the last.
|
||
orPatterns.forEach((orPattern, index) => {
|
||
if (index < orPatterns.length - 1 && this._isPatternIrrefutable(orPattern)) {
|
||
this._addSyntaxError(LocMessage.orPatternIrrefutable(), orPattern);
|
||
}
|
||
});
|
||
|
||
// Validate that all bound variables are the same within all or patterns.
|
||
const fullNameSet = new Set<string>();
|
||
orPatterns.forEach((orPattern) => {
|
||
this._getPatternTargetNames(orPattern, fullNameSet);
|
||
});
|
||
|
||
orPatterns.forEach((orPattern) => {
|
||
const localNameSet = new Set<string>();
|
||
this._getPatternTargetNames(orPattern, localNameSet);
|
||
|
||
if (localNameSet.size < fullNameSet.size) {
|
||
const missingNames = Array.from(fullNameSet.keys()).filter((name) => !localNameSet.has(name));
|
||
const diag = new DiagnosticAddendum();
|
||
diag.addMessage(
|
||
LocAddendum.orPatternMissingName().format({
|
||
name: missingNames.map((name) => `"${name}"`).join(', '),
|
||
})
|
||
);
|
||
this._addSyntaxError(LocMessage.orPatternMissingName() + diag.getString(), orPattern);
|
||
}
|
||
});
|
||
|
||
return PatternAsNode.create(orPatterns, target);
|
||
}
|
||
|
||
// pattern_atom:
|
||
// | literal_pattern
|
||
// | name_or_attr
|
||
// | '(' as_pattern ')'
|
||
// | '[' [sequence_pattern] ']'
|
||
// | '(' [sequence_pattern] ')'
|
||
// | '{' [items_pattern] '}'
|
||
// | name_or_attr '(' [pattern_arguments ','?] ')'
|
||
// name_or_attr: attr | NAME
|
||
// attr: name_or_attr '.' NAME
|
||
// sequence_pattern: ','.maybe_star_pattern+ ','?
|
||
// maybe_star_pattern: '*' NAME | pattern
|
||
// items_pattern: ','.key_value_pattern+ ','?
|
||
private _parsePatternAtom(): PatternAtomNode {
|
||
const patternLiteral = this._parsePatternLiteral();
|
||
if (patternLiteral) {
|
||
return patternLiteral;
|
||
}
|
||
|
||
const patternCaptureOrValue = this._parsePatternCaptureOrValue();
|
||
if (patternCaptureOrValue) {
|
||
const openParenToken = this._peekToken();
|
||
if (
|
||
patternCaptureOrValue.nodeType === ParseNodeType.Error ||
|
||
!this._consumeTokenIfType(TokenType.OpenParenthesis)
|
||
) {
|
||
return patternCaptureOrValue;
|
||
}
|
||
|
||
const args = this._parseClassPatternArgList();
|
||
|
||
const classNameExpr =
|
||
patternCaptureOrValue.nodeType === ParseNodeType.PatternCapture
|
||
? patternCaptureOrValue.d.target
|
||
: patternCaptureOrValue.d.expr;
|
||
const classPattern = PatternClassNode.create(classNameExpr, args);
|
||
|
||
if (!this._consumeTokenIfType(TokenType.CloseParenthesis)) {
|
||
this._addSyntaxError(LocMessage.expectedCloseParen(), openParenToken);
|
||
|
||
// Consume the remainder of tokens on the line for error
|
||
// recovery.
|
||
this._consumeTokensUntilType([TokenType.NewLine]);
|
||
|
||
// Extend the node's range to include the rest of the line.
|
||
// This helps the signatureHelpProvider.
|
||
extendRange(classPattern, this._peekToken());
|
||
}
|
||
|
||
return classPattern;
|
||
}
|
||
|
||
const nextToken = this._peekToken();
|
||
const nextOperator = this._peekOperatorType();
|
||
|
||
if (nextOperator === OperatorType.Multiply) {
|
||
const starToken = this._getNextToken();
|
||
const identifierToken = this._getTokenIfIdentifier();
|
||
if (!identifierToken) {
|
||
this._addSyntaxError(LocMessage.expectedIdentifier(), this._peekToken());
|
||
return ErrorNode.create(starToken, ErrorExpressionCategory.MissingExpression);
|
||
} else {
|
||
return PatternCaptureNode.create(NameNode.create(identifierToken), starToken);
|
||
}
|
||
}
|
||
|
||
if (nextToken.type === TokenType.OpenParenthesis || nextToken.type === TokenType.OpenBracket) {
|
||
const startToken = this._getNextToken();
|
||
const patternList = this._parsePatternSequence();
|
||
let casePattern: PatternAtomNode;
|
||
|
||
if (patternList.parseError) {
|
||
casePattern = patternList.parseError;
|
||
} else if (
|
||
patternList.list.length === 1 &&
|
||
!patternList.trailingComma &&
|
||
startToken.type === TokenType.OpenParenthesis
|
||
) {
|
||
const pattern = patternList.list[0].d.orPatterns[0];
|
||
|
||
if (pattern.nodeType === ParseNodeType.PatternCapture && pattern.d.isStar) {
|
||
casePattern = PatternSequenceNode.create(startToken, patternList.list);
|
||
} else {
|
||
casePattern = patternList.list[0];
|
||
}
|
||
|
||
extendRange(casePattern, nextToken);
|
||
} else {
|
||
casePattern = PatternSequenceNode.create(startToken, patternList.list);
|
||
}
|
||
|
||
const endToken = this._peekToken();
|
||
if (
|
||
this._consumeTokenIfType(
|
||
nextToken.type === TokenType.OpenParenthesis ? TokenType.CloseParenthesis : TokenType.CloseBracket
|
||
)
|
||
) {
|
||
extendRange(casePattern, endToken);
|
||
} else {
|
||
this._addSyntaxError(
|
||
nextToken.type === TokenType.OpenParenthesis
|
||
? LocMessage.expectedCloseParen()
|
||
: LocMessage.expectedCloseBracket(),
|
||
nextToken
|
||
);
|
||
this._consumeTokensUntilType([
|
||
TokenType.Colon,
|
||
nextToken.type === TokenType.OpenParenthesis ? TokenType.CloseParenthesis : TokenType.CloseBracket,
|
||
]);
|
||
}
|
||
|
||
return casePattern;
|
||
} else if (nextToken.type === TokenType.OpenCurlyBrace) {
|
||
const firstToken = this._getNextToken();
|
||
const mappingPattern = this._parsePatternMapping(firstToken);
|
||
const lastToken = this._peekToken();
|
||
|
||
if (this._consumeTokenIfType(TokenType.CloseCurlyBrace)) {
|
||
extendRange(mappingPattern, lastToken);
|
||
} else {
|
||
this._addSyntaxError(LocMessage.expectedCloseBrace(), nextToken);
|
||
this._consumeTokensUntilType([TokenType.Colon, TokenType.CloseCurlyBrace]);
|
||
}
|
||
|
||
return mappingPattern;
|
||
}
|
||
|
||
return this._handleExpressionParseError(
|
||
ErrorExpressionCategory.MissingPattern,
|
||
LocMessage.expectedPatternExpr()
|
||
);
|
||
}
|
||
|
||
// pattern_arguments:
|
||
// | positional_patterns [',' keyword_patterns]
|
||
// | keyword_patterns
|
||
// positional_patterns: ','.as_pattern+
|
||
// keyword_patterns: ','.keyword_pattern+
|
||
private _parseClassPatternArgList(): PatternClassArgumentNode[] {
|
||
const argList: PatternClassArgumentNode[] = [];
|
||
let sawKeywordArg = false;
|
||
|
||
while (true) {
|
||
const nextTokenType = this._peekTokenType();
|
||
if (
|
||
nextTokenType === TokenType.CloseParenthesis ||
|
||
nextTokenType === TokenType.NewLine ||
|
||
nextTokenType === TokenType.EndOfStream
|
||
) {
|
||
break;
|
||
}
|
||
|
||
const arg = this._parseClassPatternArgument();
|
||
if (arg.d.name) {
|
||
sawKeywordArg = true;
|
||
} else if (sawKeywordArg && !arg.d.name) {
|
||
this._addSyntaxError(LocMessage.positionArgAfterNamedArg(), arg);
|
||
}
|
||
argList.push(arg);
|
||
|
||
if (!this._consumeTokenIfType(TokenType.Comma)) {
|
||
break;
|
||
}
|
||
}
|
||
|
||
return argList;
|
||
}
|
||
|
||
// keyword_pattern: NAME '=' as_pattern
|
||
private _parseClassPatternArgument(): PatternClassArgumentNode {
|
||
const firstToken = this._peekToken();
|
||
const secondToken = this._peekToken(1);
|
||
|
||
let keywordName: NameNode | undefined;
|
||
|
||
if (
|
||
(firstToken.type === TokenType.Identifier || firstToken.type === TokenType.Keyword) &&
|
||
secondToken.type === TokenType.Operator &&
|
||
(secondToken as OperatorToken).operatorType === OperatorType.Assign
|
||
) {
|
||
const classNameToken = this._getTokenIfIdentifier();
|
||
if (classNameToken !== undefined) {
|
||
keywordName = NameNode.create(classNameToken);
|
||
this._getNextToken();
|
||
}
|
||
}
|
||
|
||
const pattern = this._parsePatternAs();
|
||
|
||
return PatternClassArgumentNode.create(pattern, keywordName);
|
||
}
|
||
|
||
// literal_pattern:
|
||
// | signed_number
|
||
// | signed_number '+' NUMBER
|
||
// | signed_number '-' NUMBER
|
||
// | strings
|
||
// | 'None'
|
||
// | 'True'
|
||
// | 'False'
|
||
private _parsePatternLiteral(): PatternLiteralNode | undefined {
|
||
const nextToken = this._peekToken();
|
||
const nextOperator = this._peekOperatorType();
|
||
|
||
if (nextToken.type === TokenType.Number || nextOperator === OperatorType.Subtract) {
|
||
return this._parsePatternLiteralNumber();
|
||
}
|
||
|
||
if (nextToken.type === TokenType.String) {
|
||
const stringList = this._parseAtom() as StringListNode;
|
||
assert(stringList.nodeType === ParseNodeType.StringList);
|
||
|
||
// Check for f-strings, which are not allowed.
|
||
stringList.d.strings.forEach((stringAtom) => {
|
||
if (stringAtom.nodeType === ParseNodeType.FormatString) {
|
||
this._addSyntaxError(LocMessage.formatStringInPattern(), stringAtom);
|
||
}
|
||
});
|
||
|
||
return PatternLiteralNode.create(stringList);
|
||
}
|
||
|
||
if (nextToken.type === TokenType.Keyword) {
|
||
const keywordToken = nextToken as KeywordToken;
|
||
if (
|
||
keywordToken.keywordType === KeywordType.False ||
|
||
keywordToken.keywordType === KeywordType.True ||
|
||
keywordToken.keywordType === KeywordType.None
|
||
) {
|
||
return PatternLiteralNode.create(this._parseAtom());
|
||
}
|
||
}
|
||
|
||
return undefined;
|
||
}
|
||
|
||
// signed_number: NUMBER | '-' NUMBER
|
||
private _parsePatternLiteralNumber(): PatternLiteralNode {
|
||
const expression = this._parseArithmeticExpression();
|
||
let realValue: ExpressionNode | undefined;
|
||
let imagValue: ExpressionNode | undefined;
|
||
|
||
if (expression.nodeType === ParseNodeType.BinaryOperation) {
|
||
if (expression.d.operator === OperatorType.Subtract || expression.d.operator === OperatorType.Add) {
|
||
realValue = expression.d.leftExpr;
|
||
imagValue = expression.d.rightExpr;
|
||
}
|
||
} else {
|
||
realValue = expression;
|
||
}
|
||
|
||
if (realValue) {
|
||
if (realValue.nodeType === ParseNodeType.UnaryOperation && realValue.d.operator === OperatorType.Subtract) {
|
||
realValue = realValue.d.expr;
|
||
}
|
||
|
||
if (realValue.nodeType !== ParseNodeType.Number || (imagValue !== undefined && realValue.d.isImaginary)) {
|
||
this._addSyntaxError(LocMessage.expectedComplexNumberLiteral(), expression);
|
||
imagValue = undefined;
|
||
}
|
||
}
|
||
|
||
if (imagValue) {
|
||
if (imagValue.nodeType === ParseNodeType.UnaryOperation && imagValue.d.operator === OperatorType.Subtract) {
|
||
imagValue = imagValue.d.expr;
|
||
}
|
||
|
||
if (imagValue.nodeType !== ParseNodeType.Number || !imagValue.d.isImaginary) {
|
||
this._addSyntaxError(LocMessage.expectedComplexNumberLiteral(), expression);
|
||
}
|
||
}
|
||
|
||
return PatternLiteralNode.create(expression);
|
||
}
|
||
|
||
private _parsePatternMapping(firstToken: Token): PatternMappingNode | ErrorNode {
|
||
const itemList = this._parseExpressionListGeneric(() => this._parsePatternMappingItem());
|
||
|
||
if (itemList.list.length > 0) {
|
||
// Verify there's at most one ** entry.
|
||
const starStarEntries = itemList.list.filter(
|
||
(entry) => entry.nodeType === ParseNodeType.PatternMappingExpandEntry
|
||
);
|
||
if (starStarEntries.length > 1) {
|
||
this._addSyntaxError(LocMessage.duplicateStarStarPattern(), starStarEntries[1]);
|
||
}
|
||
|
||
return PatternMappingNode.create(firstToken, itemList.list);
|
||
}
|
||
|
||
return itemList.parseError || ErrorNode.create(this._peekToken(), ErrorExpressionCategory.MissingPattern);
|
||
}
|
||
|
||
// key_value_pattern:
|
||
// | (literal_pattern | attr) ':' as_pattern
|
||
// | '**' NAME
|
||
private _parsePatternMappingItem(): PatternMappingEntryNode | ErrorNode {
|
||
let keyExpression: PatternLiteralNode | PatternValueNode | ErrorNode | undefined;
|
||
const doubleStar = this._peekToken();
|
||
|
||
if (this._consumeTokenIfOperator(OperatorType.Power)) {
|
||
const identifierToken = this._getTokenIfIdentifier();
|
||
if (!identifierToken) {
|
||
this._addSyntaxError(LocMessage.expectedIdentifier(), this._peekToken());
|
||
return ErrorNode.create(this._peekToken(), ErrorExpressionCategory.MissingPattern);
|
||
}
|
||
|
||
const nameNode = NameNode.create(identifierToken);
|
||
if (identifierToken.value === '_') {
|
||
this._addSyntaxError(LocMessage.starStarWildcardNotAllowed(), nameNode);
|
||
}
|
||
|
||
return PatternMappingExpandEntryNode.create(doubleStar, nameNode);
|
||
}
|
||
|
||
const patternLiteral = this._parsePatternLiteral();
|
||
if (patternLiteral) {
|
||
keyExpression = patternLiteral;
|
||
} else {
|
||
const patternCaptureOrValue = this._parsePatternCaptureOrValue();
|
||
if (patternCaptureOrValue) {
|
||
if (patternCaptureOrValue.nodeType === ParseNodeType.PatternValue) {
|
||
keyExpression = patternCaptureOrValue;
|
||
} else {
|
||
this._addSyntaxError(LocMessage.expectedPatternValue(), patternCaptureOrValue);
|
||
keyExpression = ErrorNode.create(this._peekToken(), ErrorExpressionCategory.MissingPattern);
|
||
}
|
||
}
|
||
}
|
||
|
||
if (!keyExpression) {
|
||
this._addSyntaxError(LocMessage.expectedPatternExpr(), this._peekToken());
|
||
keyExpression = ErrorNode.create(this._peekToken(), ErrorExpressionCategory.MissingPattern);
|
||
}
|
||
|
||
let valuePattern: PatternAtomNode | undefined;
|
||
if (!this._consumeTokenIfType(TokenType.Colon)) {
|
||
this._addSyntaxError(LocMessage.expectedColon(), this._peekToken());
|
||
valuePattern = ErrorNode.create(this._peekToken(), ErrorExpressionCategory.MissingPattern);
|
||
} else {
|
||
valuePattern = this._parsePatternAs();
|
||
}
|
||
|
||
return PatternMappingKeyEntryNode.create(keyExpression, valuePattern);
|
||
}
|
||
|
||
private _parsePatternCaptureOrValue(): PatternCaptureNode | PatternValueNode | ErrorNode | undefined {
|
||
const nextToken = this._peekToken();
|
||
|
||
if (nextToken.type === TokenType.Identifier || nextToken.type === TokenType.Keyword) {
|
||
let nameOrMember: NameNode | MemberAccessNode | undefined;
|
||
|
||
while (true) {
|
||
const identifierToken = this._getTokenIfIdentifier();
|
||
if (identifierToken) {
|
||
const nameNode = NameNode.create(identifierToken);
|
||
nameOrMember = nameOrMember ? MemberAccessNode.create(nameOrMember, nameNode) : nameNode;
|
||
} else {
|
||
this._addSyntaxError(LocMessage.expectedIdentifier(), this._peekToken());
|
||
break;
|
||
}
|
||
|
||
if (!this._consumeTokenIfType(TokenType.Dot)) {
|
||
break;
|
||
}
|
||
}
|
||
|
||
if (!nameOrMember) {
|
||
this._addSyntaxError(LocMessage.expectedIdentifier(), this._peekToken());
|
||
return ErrorNode.create(this._peekToken(), ErrorExpressionCategory.MissingPattern);
|
||
}
|
||
|
||
if (nameOrMember.nodeType === ParseNodeType.MemberAccess) {
|
||
return PatternValueNode.create(nameOrMember);
|
||
}
|
||
|
||
return PatternCaptureNode.create(nameOrMember);
|
||
}
|
||
|
||
return undefined;
|
||
}
|
||
|
||
// if_stmt: 'if' test_suite ('elif' test_suite)* ['else' suite]
|
||
// test_suite: test suite
|
||
// test: or_test ['if' or_test 'else' test] | lambdef
|
||
private _parseIfStatement(keywordType: KeywordType.If | KeywordType.Elif = KeywordType.If): IfNode {
|
||
const ifOrElifToken = this._getKeywordToken(keywordType);
|
||
|
||
const test = this._parseTestExpression(/* allowAssignmentExpression */ true);
|
||
const suite = this._parseSuite(this._isInFunction);
|
||
const ifNode = IfNode.create(ifOrElifToken, test, suite);
|
||
|
||
if (this._consumeTokenIfKeyword(KeywordType.Else)) {
|
||
ifNode.d.elseSuite = this._parseSuite(this._isInFunction);
|
||
ifNode.d.elseSuite.parent = ifNode;
|
||
extendRange(ifNode, ifNode.d.elseSuite);
|
||
} else if (this._peekKeywordType() === KeywordType.Elif) {
|
||
// Recursively handle an "elif" statement.
|
||
ifNode.d.elseSuite = this._parseIfStatement(KeywordType.Elif);
|
||
ifNode.d.elseSuite.parent = ifNode;
|
||
extendRange(ifNode, ifNode.d.elseSuite);
|
||
}
|
||
|
||
return ifNode;
|
||
}
|
||
|
||
private _parseExceptSuite<T>(isExceptionGroup: boolean, callback: () => T): T {
|
||
const wasInExceptionGroup = this._isInExceptionGroup;
|
||
|
||
if (isExceptionGroup) {
|
||
this._isInExceptionGroup = true;
|
||
}
|
||
const result = callback();
|
||
|
||
this._isInExceptionGroup = wasInExceptionGroup;
|
||
|
||
return result;
|
||
}
|
||
|
||
private _parseLoopSuite(): SuiteNode {
|
||
const wasInLoop = this._isInLoop;
|
||
const wasInExceptionGroup = this._isInExceptionGroup;
|
||
this._isInExceptionGroup = false;
|
||
this._isInLoop = true;
|
||
|
||
// Record the fact that we are no longer in a finally block
|
||
// that is contained within a loop. A loop within the finally
|
||
// block resets this. See PEP 765 for details.
|
||
const wasInFinallyLoop = this._isInFinallyLoop;
|
||
this._isInFinallyLoop = false;
|
||
|
||
let typeComment: StringToken | undefined;
|
||
const suite = this._parseSuite(this._isInFunction, /* skipBody */ false, () => {
|
||
const comment = this._getTypeAnnotationCommentText();
|
||
if (comment) {
|
||
typeComment = comment;
|
||
}
|
||
});
|
||
|
||
this._isInLoop = wasInLoop;
|
||
this._isInFinallyLoop = wasInFinallyLoop;
|
||
this._isInExceptionGroup = wasInExceptionGroup;
|
||
|
||
if (typeComment) {
|
||
suite.d.typeComment = typeComment;
|
||
}
|
||
|
||
return suite;
|
||
}
|
||
|
||
// suite: ':' (simple_stmt | NEWLINE INDENT stmt+ DEDENT)
|
||
// allowTrailingIndent（Viper 扩展）：允许 simple_stmt 之后再接 NEWLINE INDENT stmt+ DEDENT，
|
||
// 用于 `enum X: a, b, c` 后换行 + TAB 续写枚举成员。
|
||
private _parseSuite(
|
||
isFunction = false,
|
||
skipBody = false,
|
||
postColonCallback?: () => void,
|
||
allowTrailingIndent = false
|
||
): SuiteNode {
|
||
const nextToken = this._peekToken();
|
||
const suite = SuiteNode.create(nextToken);
|
||
|
||
if (!this._consumeTokenIfType(TokenType.Colon)) {
|
||
this._addSyntaxError(LocMessage.expectedColon(), nextToken);
|
||
|
||
// Try to perform parse recovery by consuming tokens.
|
||
if (this._consumeTokensUntilType([TokenType.NewLine, TokenType.Colon])) {
|
||
if (this._peekTokenType() === TokenType.Colon) {
|
||
this._getNextToken();
|
||
} else if (this._peekToken(1).type !== TokenType.Indent) {
|
||
// Bail so we resume the at the next statement.
|
||
// We can't parse as a simple statement as we've skipped all but the newline.
|
||
this._getNextToken();
|
||
return suite;
|
||
}
|
||
}
|
||
}
|
||
|
||
if (skipBody) {
|
||
if (this._consumeTokenIfType(TokenType.NewLine)) {
|
||
let indent = 0;
|
||
while (true) {
|
||
const nextToken = this._getNextToken();
|
||
if (nextToken.type === TokenType.Indent) {
|
||
indent++;
|
||
}
|
||
|
||
if (nextToken.type === TokenType.Dedent) {
|
||
if ((nextToken as DedentToken).isDedentAmbiguous) {
|
||
this._addSyntaxError(LocMessage.inconsistentTabs(), nextToken);
|
||
}
|
||
|
||
indent--;
|
||
|
||
if (indent === 0) {
|
||
break;
|
||
}
|
||
}
|
||
|
||
if (nextToken.type === TokenType.EndOfStream) {
|
||
break;
|
||
}
|
||
}
|
||
} else {
|
||
// consume tokens
|
||
this._parseSimpleStatement();
|
||
}
|
||
|
||
if (this._tokenIndex > 0) {
|
||
extendRange(suite, this._tokenizerOutput!.tokens.getItemAt(this._tokenIndex - 1));
|
||
}
|
||
|
||
return suite;
|
||
}
|
||
|
||
if (postColonCallback) {
|
||
postColonCallback();
|
||
}
|
||
|
||
const wasFunction = this._isInFunction;
|
||
this._isInFunction = isFunction;
|
||
|
||
if (this._consumeTokenIfType(TokenType.NewLine)) {
|
||
if (postColonCallback) {
|
||
postColonCallback();
|
||
}
|
||
|
||
const possibleIndent = this._peekToken();
|
||
if (!this._consumeTokenIfType(TokenType.Indent)) {
|
||
this._addSyntaxError(LocMessage.expectedIndentedBlock(), this._peekToken());
|
||
return suite;
|
||
}
|
||
|
||
const bodyIndentToken = possibleIndent as IndentToken;
|
||
if (bodyIndentToken.isIndentAmbiguous) {
|
||
this._addSyntaxError(LocMessage.inconsistentTabs(), bodyIndentToken);
|
||
}
|
||
|
||
while (true) {
|
||
// Handle a common error here and see if we can recover.
|
||
const nextToken = this._peekToken();
|
||
if (nextToken.type === TokenType.Indent) {
|
||
this._getNextToken();
|
||
const indentToken = nextToken as IndentToken;
|
||
if (indentToken.isIndentAmbiguous) {
|
||
this._addSyntaxError(LocMessage.inconsistentTabs(), indentToken);
|
||
} else {
|
||
this._addSyntaxError(LocMessage.unexpectedIndent(), nextToken);
|
||
}
|
||
} else if (nextToken.type === TokenType.Dedent) {
|
||
// When we see a dedent, stop before parsing the dedented statement.
|
||
const dedentToken = nextToken as DedentToken;
|
||
if (!dedentToken.matchesIndent) {
|
||
this._addSyntaxError(LocMessage.inconsistentIndent(), dedentToken);
|
||
}
|
||
if (dedentToken.isDedentAmbiguous) {
|
||
this._addSyntaxError(LocMessage.inconsistentTabs(), dedentToken);
|
||
}
|
||
|
||
// When the suite is incomplete (no statements), leave the dedent token for
|
||
// recovery. This allows a single dedent token to cause us to break out of
|
||
// multiple levels of nested suites. Also extend the suite's range in this
|
||
// case so it is multi-line as this works better with indentationUtils.
|
||
if (suite.d.statements.length > 0) {
|
||
this._consumeTokenIfType(TokenType.Dedent);
|
||
} else {
|
||
extendRange(suite, dedentToken);
|
||
}
|
||
|
||
// Did this dedent take us to an indent amount that is less than the
|
||
// initial indent of the suite body?
|
||
if (!bodyIndentToken || dedentToken.indentAmount < bodyIndentToken.indentAmount) {
|
||
break;
|
||
} else if (dedentToken.indentAmount === bodyIndentToken.indentAmount) {
|
||
// If the next token is also a dedent that reduces the indent
|
||
// level to a less than the initial indent of the suite body, swallow
|
||
// the extra dedent to help recover the parse.
|
||
const nextToken = this._peekToken();
|
||
if (this._consumeTokenIfType(TokenType.Dedent)) {
|
||
extendRange(suite, nextToken);
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
|
||
const statement = this._parseStatement();
|
||
if (!statement) {
|
||
// Perform basic error recovery to get to the next line.
|
||
this._consumeTokensUntilType([TokenType.NewLine]);
|
||
} else {
|
||
statement.parent = suite;
|
||
suite.d.statements.push(statement);
|
||
}
|
||
|
||
if (this._peekTokenType() === TokenType.EndOfStream) {
|
||
break;
|
||
}
|
||
}
|
||
} else {
|
||
const simpleStatement = this._parseSimpleStatement();
|
||
suite.d.statements.push(simpleStatement);
|
||
simpleStatement.parent = suite;
|
||
|
||
// Viper 扩展：`enum X: a, b, c` 允许换行 + TAB 继续写成员（等效缩进块）。
|
||
// 注：_parseSimpleStatement 已吃掉行尾的 NewLine，这里只需看 Indent。
|
||
if (allowTrailingIndent && this._peekTokenType() === TokenType.Indent) {
|
||
const bodyIndentToken = this._getNextToken() as IndentToken;
|
||
if (bodyIndentToken.isIndentAmbiguous) {
|
||
this._addSyntaxError(LocMessage.inconsistentTabs(), bodyIndentToken);
|
||
}
|
||
|
||
while (true) {
|
||
const nextToken = this._peekToken();
|
||
if (nextToken.type === TokenType.Dedent || nextToken.type === TokenType.EndOfStream) {
|
||
if (nextToken.type === TokenType.Dedent) {
|
||
this._getNextToken();
|
||
}
|
||
break;
|
||
}
|
||
|
||
const statement = this._parseStatement();
|
||
if (!statement) {
|
||
// Perform basic error recovery to get to the next line.
|
||
this._consumeTokensUntilType([TokenType.NewLine]);
|
||
} else {
|
||
statement.parent = suite;
|
||
suite.d.statements.push(statement);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
if (suite.d.statements.length > 0) {
|
||
extendRange(suite, suite.d.statements[suite.d.statements.length - 1]);
|
||
}
|
||
|
||
this._isInFunction = wasFunction;
|
||
|
||
return suite;
|
||
}
|
||
|
||
// for_stmt: [async] 'for' exprlist 'in' testlist suite ['else' suite]
|
||
private _parseForStatement(asyncToken?: KeywordToken): ForNode {
|
||
const forToken = this._getKeywordToken(KeywordType.For);
|
||
|
||
const targetExpr = this._parseExpressionListAsPossibleTuple(
|
||
ErrorExpressionCategory.MissingExpression,
|
||
() => LocMessage.expectedExpr(),
|
||
forToken
|
||
);
|
||
|
||
let seqExpr: ExpressionNode;
|
||
let forSuite: SuiteNode;
|
||
let elseSuite: SuiteNode | undefined;
|
||
|
||
if (!this._consumeTokenIfKeyword(KeywordType.In)) {
|
||
seqExpr = this._handleExpressionParseError(ErrorExpressionCategory.MissingIn, LocMessage.expectedIn());
|
||
forSuite = SuiteNode.create(this._peekToken());
|
||
} else {
|
||
seqExpr = this._parseTestOrStarListAsExpression(
|
||
/* allowAssignmentExpression */ false,
|
||
/* allowMultipleUnpack */ true,
|
||
ErrorExpressionCategory.MissingExpression,
|
||
() => LocMessage.expectedInExpr()
|
||
);
|
||
|
||
forSuite = this._parseLoopSuite();
|
||
|
||
// Versions of Python earlier than 3.9 didn't allow unpack operators if the
|
||
// tuple wasn't enclosed in parentheses.
|
||
if (
|
||
PythonVersion.isLessThan(this._getLanguageVersion(), pythonVersion3_9) &&
|
||
!this._parseOptions.isStubFile
|
||
) {
|
||
if (seqExpr.nodeType === ParseNodeType.Tuple && !seqExpr.d.hasParens) {
|
||
let sawStar = false;
|
||
seqExpr.d.items.forEach((expr) => {
|
||
if (expr.nodeType === ParseNodeType.Unpack && !sawStar) {
|
||
this._addSyntaxError(LocMessage.unpackOperatorNotAllowed(), expr);
|
||
sawStar = true;
|
||
}
|
||
});
|
||
}
|
||
}
|
||
|
||
if (this._consumeTokenIfKeyword(KeywordType.Else)) {
|
||
elseSuite = this._parseSuite(this._isInFunction);
|
||
}
|
||
}
|
||
|
||
const forNode = ForNode.create(forToken, targetExpr, seqExpr, forSuite);
|
||
forNode.d.elseSuite = elseSuite;
|
||
if (elseSuite) {
|
||
extendRange(forNode, elseSuite);
|
||
elseSuite.parent = forNode;
|
||
}
|
||
|
||
if (asyncToken) {
|
||
forNode.d.isAsync = true;
|
||
forNode.d.asyncToken = asyncToken;
|
||
extendRange(forNode, asyncToken);
|
||
}
|
||
|
||
if (forSuite.d.typeComment) {
|
||
forNode.d.typeComment = forSuite.d.typeComment;
|
||
}
|
||
|
||
return forNode;
|
||
}
|
||
|
||
// comp_iter: comp_for | comp_if
|
||
private _tryParseComprehension(target: ParseNode, isGenerator: boolean): ComprehensionNode | undefined {
|
||
const compFor = this._tryParseCompForStatement();
|
||
|
||
if (!compFor) {
|
||
return undefined;
|
||
}
|
||
|
||
if (target.nodeType === ParseNodeType.Unpack) {
|
||
this._addSyntaxError(LocMessage.unpackIllegalInComprehension(), target);
|
||
} else if (target.nodeType === ParseNodeType.DictionaryExpandEntry) {
|
||
this._addSyntaxError(LocMessage.dictExpandIllegalInComprehension(), target);
|
||
}
|
||
|
||
const compNode = ComprehensionNode.create(target, isGenerator);
|
||
|
||
const forIfList: ComprehensionForIfNode[] = [compFor];
|
||
while (true) {
|
||
const compIter = this._tryParseCompForStatement() || this._tryParseCompIfStatement();
|
||
if (!compIter) {
|
||
break;
|
||
}
|
||
compIter.parent = compNode;
|
||
forIfList.push(compIter);
|
||
}
|
||
|
||
compNode.d.forIfNodes = forIfList;
|
||
if (forIfList.length > 0) {
|
||
forIfList.forEach((comp) => {
|
||
comp.parent = compNode;
|
||
});
|
||
extendRange(compNode, forIfList[forIfList.length - 1]);
|
||
}
|
||
return compNode;
|
||
}
|
||
|
||
// comp_for: ['async'] 'for' exprlist 'in' or_test [comp_iter]
|
||
private _tryParseCompForStatement(): ComprehensionForNode | undefined {
|
||
const startTokenKeywordType = this._peekKeywordType();
|
||
|
||
if (startTokenKeywordType === KeywordType.Async) {
|
||
const nextToken = this._peekToken(1) as KeywordToken;
|
||
if (nextToken.type !== TokenType.Keyword || nextToken.keywordType !== KeywordType.For) {
|
||
return undefined;
|
||
}
|
||
} else if (startTokenKeywordType !== KeywordType.For) {
|
||
return undefined;
|
||
}
|
||
|
||
let asyncToken: KeywordToken | undefined;
|
||
if (this._peekKeywordType() === KeywordType.Async) {
|
||
asyncToken = this._getKeywordToken(KeywordType.Async);
|
||
}
|
||
|
||
const forToken = this._getKeywordToken(KeywordType.For);
|
||
|
||
const targetExpr = this._parseExpressionListAsPossibleTuple(
|
||
ErrorExpressionCategory.MissingExpression,
|
||
() => LocMessage.expectedExpr(),
|
||
forToken
|
||
);
|
||
let seqExpr: ExpressionNode | undefined;
|
||
|
||
if (!this._consumeTokenIfKeyword(KeywordType.In)) {
|
||
seqExpr = this._handleExpressionParseError(ErrorExpressionCategory.MissingIn, LocMessage.expectedIn());
|
||
} else {
|
||
this._disallowAssignmentExpression(() => {
|
||
seqExpr = this._parseOrTest();
|
||
});
|
||
}
|
||
|
||
const compForNode = ComprehensionForNode.create(asyncToken || forToken, targetExpr, seqExpr!);
|
||
|
||
if (asyncToken) {
|
||
compForNode.d.isAsync = true;
|
||
compForNode.d.asyncToken = asyncToken;
|
||
}
|
||
|
||
return compForNode;
|
||
}
|
||
|
||
// comp_if: 'if' test_nocond [comp_iter]
|
||
// comp_iter: comp_for | comp_if
|
||
private _tryParseCompIfStatement(): ComprehensionIfNode | undefined {
|
||
if (this._peekKeywordType() !== KeywordType.If) {
|
||
return undefined;
|
||
}
|
||
|
||
const ifToken = this._getKeywordToken(KeywordType.If);
|
||
const ifExpr =
|
||
this._tryParseLambdaExpression() ||
|
||
this._parseAssignmentExpression(/* disallowAssignmentExpression */ true);
|
||
|
||
const compIfNode = ComprehensionIfNode.create(ifToken, ifExpr);
|
||
|
||
return compIfNode;
|
||
}
|
||
|
||
// while_stmt: 'while' test suite ['else' suite]
|
||
private _parseWhileStatement(): WhileNode {
|
||
const whileToken = this._getKeywordToken(KeywordType.While);
|
||
|
||
const whileNode = WhileNode.create(
|
||
whileToken,
|
||
this._parseTestExpression(/* allowAssignmentExpression */ true),
|
||
this._parseLoopSuite()
|
||
);
|
||
|
||
if (this._consumeTokenIfKeyword(KeywordType.Else)) {
|
||
whileNode.d.elseSuite = this._parseSuite(this._isInFunction);
|
||
whileNode.d.elseSuite.parent = whileNode;
|
||
extendRange(whileNode, whileNode.d.elseSuite);
|
||
}
|
||
|
||
return whileNode;
|
||
}
|
||
|
||
// try_stmt: ('try' suite
|
||
// ((except_clause suite)+
|
||
// ['else' suite]
|
||
// ['finally' suite] |
|
||
// 'finally' suite))
|
||
// except_clause: 'except' [test ['as' NAME]]
|
||
private _parseTryStatement(): TryNode {
|
||
const tryToken = this._getKeywordToken(KeywordType.Try);
|
||
const trySuite = this._parseSuite(this._isInFunction);
|
||
const tryNode = TryNode.create(tryToken, trySuite);
|
||
let sawCatchAllExcept = false;
|
||
let reportedExceptGroupMismatch = false;
|
||
|
||
while (true) {
|
||
const exceptToken = this._peekToken();
|
||
if (!this._consumeTokenIfKeyword(KeywordType.Except)) {
|
||
break;
|
||
}
|
||
|
||
// See if this is a Python 3.11 exception group.
|
||
const possibleStarToken = this._peekToken();
|
||
let isExceptGroup = false;
|
||
if (this._consumeTokenIfOperator(OperatorType.Multiply)) {
|
||
if (
|
||
PythonVersion.isLessThan(this._getLanguageVersion(), pythonVersion3_11) &&
|
||
!this._parseOptions.isStubFile
|
||
) {
|
||
this._addSyntaxError(LocMessage.exceptionGroupIncompatible(), possibleStarToken);
|
||
}
|
||
|
||
isExceptGroup = true;
|
||
|
||
if (!reportedExceptGroupMismatch && tryNode.d.exceptClauses.some((clause) => !clause.d.isExceptGroup)) {
|
||
this._addSyntaxError(LocMessage.exceptGroupMismatch(), possibleStarToken);
|
||
reportedExceptGroupMismatch = true;
|
||
}
|
||
} else {
|
||
if (!reportedExceptGroupMismatch && tryNode.d.exceptClauses.some((clause) => clause.d.isExceptGroup)) {
|
||
this._addSyntaxError(LocMessage.exceptGroupMismatch(), possibleStarToken);
|
||
reportedExceptGroupMismatch = true;
|
||
}
|
||
}
|
||
|
||
let typeExpr: ExpressionNode | undefined;
|
||
let symbolName: IdentifierToken | undefined;
|
||
let isAsKeywordAllowed = true;
|
||
|
||
if (this._peekTokenType() !== TokenType.Colon) {
|
||
const listResult = this._parseExpressionListGeneric(() =>
|
||
this._parseTestExpression(/* allowAssignmentExpression */ true)
|
||
);
|
||
if (listResult.parseError) {
|
||
typeExpr = listResult.parseError;
|
||
} else {
|
||
typeExpr = this._makeExpressionOrTuple(listResult, /* enclosedInParens */ false);
|
||
|
||
// Python 3.14 allows more than one exception type to be provided in
|
||
// an except clause.
|
||
if (listResult.list.length > 1) {
|
||
if (PythonVersion.isLessThan(this._getLanguageVersion(), pythonVersion3_14)) {
|
||
this._addSyntaxError(LocMessage.exceptRequiresParens(), typeExpr);
|
||
}
|
||
|
||
isAsKeywordAllowed = false;
|
||
}
|
||
}
|
||
|
||
if (this._consumeTokenIfKeyword(KeywordType.As)) {
|
||
if (!isAsKeywordAllowed) {
|
||
this._addSyntaxError(LocMessage.exceptWithAsRequiresParens(), typeExpr);
|
||
}
|
||
|
||
symbolName = this._getTokenIfIdentifier();
|
||
if (!symbolName) {
|
||
this._addSyntaxError(LocMessage.expectedNameAfterAs(), this._peekToken());
|
||
}
|
||
}
|
||
} else if (isExceptGroup) {
|
||
this._addSyntaxError(LocMessage.exceptGroupRequiresType(), this._peekToken());
|
||
}
|
||
|
||
if (!typeExpr) {
|
||
if (sawCatchAllExcept) {
|
||
this._addSyntaxError(LocMessage.duplicateCatchAll(), exceptToken);
|
||
}
|
||
sawCatchAllExcept = true;
|
||
} else {
|
||
if (sawCatchAllExcept) {
|
||
this._addSyntaxError(LocMessage.namedExceptAfterCatchAll(), typeExpr);
|
||
}
|
||
}
|
||
|
||
const exceptSuite = this._parseExceptSuite(isExceptGroup, () => this._parseSuite(this._isInFunction));
|
||
const exceptNode = ExceptNode.create(exceptToken, exceptSuite, isExceptGroup);
|
||
if (typeExpr) {
|
||
exceptNode.d.typeExpr = typeExpr;
|
||
exceptNode.d.typeExpr.parent = exceptNode;
|
||
}
|
||
|
||
if (symbolName) {
|
||
exceptNode.d.name = NameNode.create(symbolName);
|
||
exceptNode.d.name.parent = exceptNode;
|
||
}
|
||
|
||
tryNode.d.exceptClauses.push(exceptNode);
|
||
exceptNode.parent = tryNode;
|
||
}
|
||
|
||
if (tryNode.d.exceptClauses.length > 0) {
|
||
extendRange(tryNode, tryNode.d.exceptClauses[tryNode.d.exceptClauses.length - 1]);
|
||
|
||
if (this._consumeTokenIfKeyword(KeywordType.Else)) {
|
||
tryNode.d.elseSuite = this._parseSuite(this._isInFunction);
|
||
tryNode.d.elseSuite.parent = tryNode;
|
||
extendRange(tryNode, tryNode.d.elseSuite);
|
||
}
|
||
}
|
||
|
||
if (this._consumeTokenIfKeyword(KeywordType.Finally)) {
|
||
const wasInFinallyBlock = this._isInFinallyBlock;
|
||
const wasInFinallyLoop = this._isInFinallyLoop;
|
||
this._isInFinallyBlock = true;
|
||
this._isInFinallyLoop = this._isInLoop;
|
||
|
||
tryNode.d.finallySuite = this._parseSuite(this._isInFunction);
|
||
|
||
this._isInFinallyBlock = wasInFinallyBlock;
|
||
this._isInFinallyLoop = wasInFinallyLoop;
|
||
|
||
tryNode.d.finallySuite.parent = tryNode;
|
||
extendRange(tryNode, tryNode.d.finallySuite);
|
||
}
|
||
|
||
if (!tryNode.d.finallySuite && tryNode.d.exceptClauses.length === 0) {
|
||
this._addSyntaxError(LocMessage.tryWithoutExcept(), tryToken);
|
||
}
|
||
|
||
return tryNode;
|
||
}
|
||
|
||
// funcdef: 'def' NAME parameters ['->' test] ':' suite
|
||
// parameters: '(' [typedargslist] ')'
|
||
private _parseFunctionDef(
|
||
asyncToken?: KeywordToken,
|
||
decorators?: DecoratorNode[],
|
||
viperModifier?: string,
|
||
defTokenOverride?: Token
|
||
): FunctionNode | ErrorNode {
|
||
// Viper 扩展：extern/export 直接声明形式（`extern A(...)`）没有 def 关键字，
|
||
// 用修饰关键字 token 作为范围起点；否则正常消费 Def 关键字 token。
|
||
// define 宏函数关键字与 def 语义一致，语法完全相同，仅标记 isDefine。
|
||
let defToken: Token;
|
||
let isDefine = false;
|
||
if (defTokenOverride) {
|
||
defToken = defTokenOverride;
|
||
} else {
|
||
const curToken = this._peekToken();
|
||
if (
|
||
curToken.type === TokenType.Keyword &&
|
||
(curToken as KeywordToken).keywordType === KeywordType.Define
|
||
) {
|
||
isDefine = true;
|
||
defToken = this._getNextToken() as KeywordToken;
|
||
} else {
|
||
defToken = this._getKeywordToken(KeywordType.Def);
|
||
}
|
||
}
|
||
|
||
const nameToken = this._getTokenIfIdentifier();
|
||
if (!nameToken) {
|
||
this._addSyntaxError(LocMessage.expectedFunctionName(), defToken);
|
||
return ErrorNode.create(
|
||
defToken,
|
||
ErrorExpressionCategory.MissingFunctionParameterList,
|
||
undefined,
|
||
decorators
|
||
);
|
||
}
|
||
|
||
let typeParameters: TypeParameterListNode | undefined;
|
||
const possibleOpenBracket = this._peekToken();
|
||
if (possibleOpenBracket.type === TokenType.OpenBracket) {
|
||
typeParameters = this._parseTypeParameterList();
|
||
|
||
if (
|
||
!this._parseOptions.isStubFile &&
|
||
PythonVersion.isLessThan(this._getLanguageVersion(), pythonVersion3_12)
|
||
) {
|
||
this._addSyntaxError(LocMessage.functionTypeParametersIllegal(), typeParameters);
|
||
}
|
||
}
|
||
const openParenToken = this._peekToken();
|
||
if (!this._consumeTokenIfType(TokenType.OpenParenthesis)) {
|
||
this._addSyntaxError(LocMessage.expectedOpenParen(), this._peekToken());
|
||
return ErrorNode.create(
|
||
nameToken,
|
||
ErrorExpressionCategory.MissingFunctionParameterList,
|
||
NameNode.create(nameToken),
|
||
decorators
|
||
);
|
||
}
|
||
|
||
const paramList = this._parseVarArgsList(TokenType.CloseParenthesis, /* allowAnnotations */ true);
|
||
|
||
if (!this._consumeTokenIfType(TokenType.CloseParenthesis)) {
|
||
this._addSyntaxError(LocMessage.expectedCloseParen(), openParenToken);
|
||
this._consumeTokensUntilType([TokenType.Colon]);
|
||
}
|
||
|
||
let returnType: ExpressionNode | undefined;
|
||
if (this._consumeTokenIfType(TokenType.Arrow)) {
|
||
returnType = this._parseTypeAnnotation();
|
||
}
|
||
|
||
let functionTypeAnnotationToken: StringToken | undefined;
|
||
const wasInExceptionGroup = this._isInExceptionGroup;
|
||
this._isInExceptionGroup = false;
|
||
|
||
const wasInFinallyBlock = this._isInFinallyBlock;
|
||
const wasInFinallyLoop = this._isInFinallyLoop;
|
||
this._isInFinallyBlock = false;
|
||
this._isInFinallyLoop = false;
|
||
|
||
let suite: SuiteNode;
|
||
const possibleSuiteStart = this._peekToken();
|
||
if (
|
||
(viperModifier === 'extern' || viperModifier === 'export') &&
|
||
possibleSuiteStart.type !== TokenType.Colon
|
||
) {
|
||
// Viper extern/export 声明允许省略函数体（`extern def f(...) -> T` 无冒号、无 pass）。
|
||
// 创建空 suite 并消费行尾换行；`: pass` 方式仍走 _parseSuite 正常解析。
|
||
suite = SuiteNode.create(possibleSuiteStart);
|
||
if (this._consumeTokenIfType(TokenType.NewLine)) {
|
||
extendRange(suite, this._tokenizerOutput!.tokens.getItemAt(this._tokenIndex - 1));
|
||
}
|
||
} else {
|
||
// Viper 扩展：函数体内 `virtual def` / `novirtual def` 不是类成员，
|
||
// 解析函数体期间清空所处的类形态，使嵌套 def 的位置校验按"不在类内"处理。
|
||
const wasEnclosingClassKind = this._viperEnclosingClassKind;
|
||
this._viperEnclosingClassKind = undefined;
|
||
|
||
suite = this._parseSuite(/* isFunction */ true, this._parseOptions.skipFunctionAndClassBody, () => {
|
||
if (!functionTypeAnnotationToken) {
|
||
functionTypeAnnotationToken = this._getTypeAnnotationCommentText();
|
||
}
|
||
});
|
||
|
||
this._viperEnclosingClassKind = wasEnclosingClassKind;
|
||
}
|
||
|
||
this._isInExceptionGroup = wasInExceptionGroup;
|
||
this._isInFinallyBlock = wasInFinallyBlock;
|
||
this._isInFinallyLoop = wasInFinallyLoop;
|
||
|
||
const functionNode = FunctionNode.create(defToken, NameNode.create(nameToken), suite, typeParameters, viperModifier, isDefine);
|
||
functionNode.d.viperModifier = viperModifier;
|
||
if (asyncToken) {
|
||
functionNode.d.isAsync = true;
|
||
extendRange(functionNode, asyncToken);
|
||
}
|
||
|
||
functionNode.d.params = paramList;
|
||
paramList.forEach((param) => {
|
||
param.parent = functionNode;
|
||
});
|
||
|
||
if (decorators) {
|
||
functionNode.d.decorators = decorators;
|
||
decorators.forEach((decorator) => {
|
||
decorator.parent = functionNode;
|
||
});
|
||
|
||
if (decorators.length > 0) {
|
||
extendRange(functionNode, decorators[0]);
|
||
}
|
||
}
|
||
|
||
if (returnType) {
|
||
functionNode.d.returnAnnotation = returnType;
|
||
functionNode.d.returnAnnotation.parent = functionNode;
|
||
extendRange(functionNode, returnType);
|
||
}
|
||
|
||
// If there was a type annotation comment for the function,
|
||
// parse it now.
|
||
if (functionTypeAnnotationToken) {
|
||
this._parseFunctionTypeAnnotationComment(functionTypeAnnotationToken, functionNode);
|
||
}
|
||
|
||
return functionNode;
|
||
}
|
||
|
||
// typedargslist: (
|
||
// tfpdef ['=' test] (',' tfpdef ['=' test])*
|
||
// [ ','
|
||
// [
|
||
// '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]]
|
||
// | '**' tfpdef [',']
|
||
// ]
|
||
// ]
|
||
// | '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]]
|
||
// | '**' tfpdef [','])
|
||
// tfpdef: NAME [':' test]
|
||
// vfpdef: NAME;
|
||
private _parseVarArgsList(terminator: TokenType, allowAnnotations: boolean): ParameterNode[] {
|
||
const paramMap = new Map<string, string>();
|
||
const paramList: ParameterNode[] = [];
|
||
let sawDefaultParam = false;
|
||
let reportedNonDefaultParamErr = false;
|
||
let sawKeywordOnlySeparator = false;
|
||
let sawPositionOnlySeparator = false;
|
||
let sawKeywordOnlyParamAfterSeparator = false;
|
||
let sawArgs = false;
|
||
let sawKwArgs = false;
|
||
|
||
while (true) {
|
||
if (this._peekTokenType() === terminator) {
|
||
break;
|
||
}
|
||
|
||
const param = this._parseParameter(allowAnnotations);
|
||
if (!param) {
|
||
this._consumeTokensUntilType([terminator]);
|
||
break;
|
||
}
|
||
|
||
if (param.d.name) {
|
||
const name = param.d.name.d.value;
|
||
if (paramMap.has(name)) {
|
||
this._addSyntaxError(LocMessage.duplicateParam().format({ name }), param.d.name);
|
||
} else {
|
||
paramMap.set(name, name);
|
||
}
|
||
} else if (param.d.category === ParamCategory.Simple) {
|
||
if (paramList.length === 0) {
|
||
this._addSyntaxError(LocMessage.positionOnlyFirstParam(), param);
|
||
}
|
||
}
|
||
|
||
if (param.d.category === ParamCategory.Simple) {
|
||
if (!param.d.name) {
|
||
if (sawPositionOnlySeparator) {
|
||
this._addSyntaxError(LocMessage.duplicatePositionOnly(), param);
|
||
} else if (sawKeywordOnlySeparator) {
|
||
this._addSyntaxError(LocMessage.positionOnlyAfterKeywordOnly(), param);
|
||
} else if (sawArgs) {
|
||
this._addSyntaxError(LocMessage.positionOnlyAfterArgs(), param);
|
||
}
|
||
sawPositionOnlySeparator = true;
|
||
} else {
|
||
if (sawKeywordOnlySeparator) {
|
||
sawKeywordOnlyParamAfterSeparator = true;
|
||
}
|
||
|
||
if (param.d.defaultValue) {
|
||
sawDefaultParam = true;
|
||
} else if (sawDefaultParam && !sawKeywordOnlySeparator && !sawArgs) {
|
||
// Report this error only once.
|
||
if (!reportedNonDefaultParamErr) {
|
||
this._addSyntaxError(LocMessage.nonDefaultAfterDefault(), param);
|
||
reportedNonDefaultParamErr = true;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
paramList.push(param);
|
||
|
||
if (param.d.category === ParamCategory.ArgsList) {
|
||
if (!param.d.name) {
|
||
if (sawKeywordOnlySeparator) {
|
||
this._addSyntaxError(LocMessage.duplicateKeywordOnly(), param);
|
||
} else if (sawArgs) {
|
||
this._addSyntaxError(LocMessage.keywordOnlyAfterArgs(), param);
|
||
}
|
||
sawKeywordOnlySeparator = true;
|
||
} else {
|
||
if (sawKeywordOnlySeparator || sawArgs) {
|
||
this._addSyntaxError(LocMessage.duplicateArgsParam(), param);
|
||
}
|
||
sawArgs = true;
|
||
}
|
||
}
|
||
|
||
if (param.d.category === ParamCategory.KwargsDict) {
|
||
if (sawKwArgs) {
|
||
this._addSyntaxError(LocMessage.duplicateKwargsParam(), param);
|
||
}
|
||
sawKwArgs = true;
|
||
|
||
// A **kwargs cannot immediately follow a keyword-only separator ("*").
|
||
if (sawKeywordOnlySeparator && !sawKeywordOnlyParamAfterSeparator) {
|
||
this._addSyntaxError(LocMessage.keywordParameterMissing(), param);
|
||
}
|
||
} else if (sawKwArgs) {
|
||
this._addSyntaxError(LocMessage.paramAfterKwargsParam(), param);
|
||
}
|
||
|
||
const foundComma = this._consumeTokenIfType(TokenType.Comma);
|
||
|
||
if (allowAnnotations && !param.d.annotation) {
|
||
// Look for a type annotation comment at the end of the line.
|
||
const typeAnnotationComment = this._parseVariableTypeAnnotationComment();
|
||
if (typeAnnotationComment) {
|
||
param.d.annotationComment = typeAnnotationComment;
|
||
param.d.annotationComment.parent = param;
|
||
extendRange(param, param.d.annotationComment);
|
||
}
|
||
}
|
||
|
||
if (!foundComma) {
|
||
break;
|
||
}
|
||
}
|
||
|
||
if (paramList.length > 0) {
|
||
const lastParam = paramList[paramList.length - 1];
|
||
if (lastParam.d.category === ParamCategory.ArgsList && !lastParam.d.name) {
|
||
this._addSyntaxError(LocMessage.expectedNamedParameter(), lastParam);
|
||
}
|
||
}
|
||
|
||
return paramList;
|
||
}
|
||
|
||
private _parseParameter(allowAnnotations: boolean): ParameterNode {
|
||
let starCount = 0;
|
||
let slashCount = 0;
|
||
const firstToken = this._peekToken();
|
||
|
||
if (this._consumeTokenIfOperator(OperatorType.Multiply)) {
|
||
starCount = 1;
|
||
} else if (this._consumeTokenIfOperator(OperatorType.Power)) {
|
||
starCount = 2;
|
||
} else if (this._consumeTokenIfOperator(OperatorType.Divide)) {
|
||
if (
|
||
PythonVersion.isLessThan(this._getLanguageVersion(), pythonVersion3_8) &&
|
||
!this._parseOptions.isStubFile
|
||
) {
|
||
this._addSyntaxError(LocMessage.positionOnlyIncompatible(), firstToken);
|
||
}
|
||
slashCount = 1;
|
||
}
|
||
|
||
const paramName = this._getTokenIfIdentifier();
|
||
if (!paramName) {
|
||
if (starCount === 1) {
|
||
const paramNode = ParameterNode.create(firstToken, ParamCategory.ArgsList);
|
||
return paramNode;
|
||
} else if (slashCount === 1) {
|
||
const paramNode = ParameterNode.create(firstToken, ParamCategory.Simple);
|
||
return paramNode;
|
||
}
|
||
|
||
// Check for the Python 2.x parameter sublist syntax and handle it gracefully.
|
||
if (this._peekTokenType() === TokenType.OpenParenthesis) {
|
||
const sublistStart = this._getNextToken();
|
||
if (this._consumeTokensUntilType([TokenType.CloseParenthesis])) {
|
||
this._getNextToken();
|
||
}
|
||
this._addSyntaxError(LocMessage.sublistParamsIncompatible(), sublistStart);
|
||
} else {
|
||
this._addSyntaxError(LocMessage.expectedParamName(), this._peekToken());
|
||
}
|
||
}
|
||
|
||
let paramType = ParamCategory.Simple;
|
||
if (starCount === 1) {
|
||
paramType = ParamCategory.ArgsList;
|
||
} else if (starCount === 2) {
|
||
paramType = ParamCategory.KwargsDict;
|
||
}
|
||
const paramNode = ParameterNode.create(firstToken, paramType);
|
||
if (paramName) {
|
||
paramNode.d.name = NameNode.create(paramName);
|
||
paramNode.d.name.parent = paramNode;
|
||
extendRange(paramNode, paramName);
|
||
}
|
||
|
||
if (allowAnnotations && this._consumeTokenIfType(TokenType.Colon)) {
|
||
paramNode.d.annotation = this._parseTypeAnnotation(paramType === ParamCategory.ArgsList);
|
||
paramNode.d.annotation.parent = paramNode;
|
||
extendRange(paramNode, paramNode.d.annotation);
|
||
}
|
||
|
||
if (this._consumeTokenIfOperator(OperatorType.Assign)) {
|
||
paramNode.d.defaultValue = this._parseTestExpression(/* allowAssignmentExpression */ false);
|
||
paramNode.d.defaultValue.parent = paramNode;
|
||
extendRange(paramNode, paramNode.d.defaultValue);
|
||
|
||
if (starCount > 0) {
|
||
this._addSyntaxError(LocMessage.defaultValueNotAllowed(), paramNode.d.defaultValue);
|
||
}
|
||
}
|
||
|
||
return paramNode;
|
||
}
|
||
|
||
// with_stmt: 'with' with_item (',' with_item)* ':' suite
|
||
// Python 3.10 adds support for optional parentheses around
|
||
// with_item list.
|
||
private _parseWithStatement(asyncToken?: KeywordToken): WithNode {
|
||
const withToken = this._getKeywordToken(KeywordType.With);
|
||
let withItemList: WithItemNode[] = [];
|
||
|
||
const possibleParen = this._peekToken();
|
||
|
||
// If the expression starts with a paren, parse it as though the
|
||
// paren is enclosing the list of "with items". This is done as a
|
||
// "dry run" to determine whether the entire list of "with items"
|
||
// is enclosed in parentheses.
|
||
let isParenthesizedWithItemList = false;
|
||
let isParenthesizedDisallowed = false;
|
||
|
||
if (possibleParen.type === TokenType.OpenParenthesis) {
|
||
const openParenTokenIndex = this._tokenIndex;
|
||
|
||
this._suppressErrors(() => {
|
||
this._getNextToken();
|
||
while (true) {
|
||
withItemList.push(this._parseWithItem());
|
||
if (!this._consumeTokenIfType(TokenType.Comma)) {
|
||
break;
|
||
}
|
||
|
||
if (this._peekToken().type === TokenType.CloseParenthesis) {
|
||
break;
|
||
}
|
||
}
|
||
|
||
if (
|
||
this._peekToken().type === TokenType.CloseParenthesis &&
|
||
this._peekToken(1).type === TokenType.Colon
|
||
) {
|
||
isParenthesizedWithItemList = true;
|
||
|
||
// Some forms of parenthesized context with statements were not
|
||
// allowed prior to Python 3.9. Is this such a form?
|
||
isParenthesizedDisallowed = withItemList.length !== 1 || withItemList[0].d.target !== undefined;
|
||
}
|
||
|
||
this._tokenIndex = openParenTokenIndex;
|
||
withItemList = [];
|
||
});
|
||
}
|
||
|
||
if (isParenthesizedWithItemList) {
|
||
this._consumeTokenIfType(TokenType.OpenParenthesis);
|
||
if (isParenthesizedDisallowed && PythonVersion.isLessThan(this._getLanguageVersion(), pythonVersion3_9)) {
|
||
this._addSyntaxError(LocMessage.parenthesizedContextManagerIllegal(), possibleParen);
|
||
}
|
||
}
|
||
|
||
while (true) {
|
||
withItemList.push(this._parseWithItem());
|
||
|
||
if (!this._consumeTokenIfType(TokenType.Comma)) {
|
||
break;
|
||
}
|
||
|
||
if (this._peekToken().type === TokenType.CloseParenthesis) {
|
||
break;
|
||
}
|
||
}
|
||
|
||
if (isParenthesizedWithItemList) {
|
||
if (!this._consumeTokenIfType(TokenType.CloseParenthesis)) {
|
||
this._addSyntaxError(LocMessage.expectedCloseParen(), possibleParen);
|
||
}
|
||
}
|
||
|
||
let typeComment: StringToken | undefined;
|
||
const withSuite = this._parseSuite(this._isInFunction, /* skipBody */ false, () => {
|
||
const comment = this._getTypeAnnotationCommentText();
|
||
if (comment) {
|
||
typeComment = comment;
|
||
}
|
||
});
|
||
const withNode = WithNode.create(withToken, withSuite);
|
||
if (asyncToken) {
|
||
withNode.d.isAsync = true;
|
||
withNode.d.asyncToken = asyncToken;
|
||
extendRange(withNode, asyncToken);
|
||
}
|
||
|
||
if (typeComment) {
|
||
withNode.d.typeComment = typeComment;
|
||
}
|
||
|
||
withNode.d.withItems = withItemList;
|
||
withItemList.forEach((withItem) => {
|
||
withItem.parent = withNode;
|
||
});
|
||
|
||
return withNode;
|
||
}
|
||
|
||
// with_item: test ['as' expr]
|
||
private _parseWithItem(): WithItemNode {
|
||
const expr = this._parseTestExpression(/* allowAssignmentExpression */ true);
|
||
const itemNode = WithItemNode.create(expr);
|
||
|
||
if (this._consumeTokenIfKeyword(KeywordType.As)) {
|
||
itemNode.d.target = this._parseExpression(/* allowUnpack */ false);
|
||
itemNode.d.target.parent = itemNode;
|
||
extendRange(itemNode, itemNode.d.target);
|
||
}
|
||
|
||
return itemNode;
|
||
}
|
||
|
||
// decorators: decorator+
|
||
// decorated: decorators (classdef | funcdef | async_funcdef)
|
||
private _parseDecorated(): StatementNode | undefined {
|
||
const decoratorList: DecoratorNode[] = [];
|
||
|
||
while (true) {
|
||
if (this._peekOperatorType() === OperatorType.MatrixMultiply) {
|
||
decoratorList.push(this._parseDecorator());
|
||
} else {
|
||
break;
|
||
}
|
||
}
|
||
|
||
const nextToken = this._peekToken() as KeywordToken;
|
||
if (nextToken.type === TokenType.Keyword) {
|
||
if (nextToken.keywordType === KeywordType.Async) {
|
||
this._getNextToken();
|
||
|
||
if (this._peekKeywordType() !== KeywordType.Def) {
|
||
this._addSyntaxError(LocMessage.expectedFunctionAfterAsync(), this._peekToken());
|
||
} else {
|
||
return this._parseFunctionDef(nextToken, decoratorList);
|
||
}
|
||
} else if (nextToken.keywordType === KeywordType.Def) {
|
||
return this._parseFunctionDef(undefined, decoratorList);
|
||
} else if (nextToken.keywordType === KeywordType.Define) {
|
||
// Viper 扩展：装饰的 define 宏函数。
|
||
return this._parseFunctionDef(undefined, decoratorList);
|
||
} else if (nextToken.keywordType === KeywordType.Class) {
|
||
return this._parseClassDef(decoratorList);
|
||
} else if (nextToken.keywordType === KeywordType.ViperModifier) {
|
||
// Viper 扩展：装饰器之后的 vtable 修饰（如 `@Foo` + `virtual class X`
|
||
// / `virtual def f(...)`）。与 _parseStatement 的同名分支保持一致。
|
||
const modifierToken = this._peekToken();
|
||
const modifierText = this._fileContents!.slice(
|
||
modifierToken.start,
|
||
modifierToken.start + modifierToken.length
|
||
);
|
||
if (modifierText === 'virtual' || modifierText === 'novirtual') {
|
||
const afterModifierKeywordType =
|
||
this._peekToken(1).type === TokenType.Keyword
|
||
? (this._peekToken(1) as KeywordToken).keywordType
|
||
: undefined;
|
||
if (afterModifierKeywordType === KeywordType.Class) {
|
||
this._getNextToken();
|
||
return this._parseClassDef(decoratorList, modifierText);
|
||
}
|
||
if (afterModifierKeywordType === KeywordType.Def) {
|
||
this._validateViperVirtualMethodPlacement(modifierText, modifierToken);
|
||
this._getNextToken();
|
||
return this._parseFunctionDef(undefined, decoratorList, modifierText);
|
||
}
|
||
this._addSyntaxError(
|
||
LocMessage.viperVirtualModifierRequiresClassOrDef().format({ name: modifierText }),
|
||
modifierToken
|
||
);
|
||
}
|
||
// 其他修饰关键字（inline/export/extern）不在装饰器后支持，
|
||
// 保持原行为：落到下方 expectedAfterDecorator 报错。
|
||
}
|
||
}
|
||
|
||
this._addSyntaxError(LocMessage.expectedAfterDecorator(), this._peekToken());
|
||
|
||
// Return a dummy class declaration so the completion provider has
|
||
// some parse nodes to work with.
|
||
return ClassNode.createDummyForDecorators(decoratorList);
|
||
}
|
||
|
||
// decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
|
||
private _parseDecorator(): DecoratorNode {
|
||
const atOperator = this._getNextToken() as OperatorToken;
|
||
assert(atOperator.operatorType === OperatorType.MatrixMultiply);
|
||
|
||
const expression = this._parseTestExpression(/* allowAssignmentExpression */ true);
|
||
|
||
// Versions of Python prior to 3.9 support a limited set of
|
||
// expression forms.
|
||
if (PythonVersion.isLessThan(this._getLanguageVersion(), pythonVersion3_9)) {
|
||
let isSupportedExpressionForm = false;
|
||
if (this._isNameOrMemberAccessExpression(expression)) {
|
||
isSupportedExpressionForm = true;
|
||
} else if (
|
||
expression.nodeType === ParseNodeType.Call &&
|
||
this._isNameOrMemberAccessExpression(expression.d.leftExpr)
|
||
) {
|
||
isSupportedExpressionForm = true;
|
||
}
|
||
|
||
if (!isSupportedExpressionForm) {
|
||
this._addSyntaxError(LocMessage.expectedDecoratorExpr(), expression);
|
||
}
|
||
}
|
||
|
||
// Viper 扩展：`@__attribute__(section(".x"), always_inline)` desugar 为
|
||
// `@c.Attribute(t.attr.section(".x"), t.attr.always_inline)`
|
||
// ——与 TPC/TPV 的 includes/ast/parser.vp `_DesugarMagicAttribute` 同形，
|
||
// class 与 def 共用（装饰器链统一走本函数）；其它装饰器原样返回。
|
||
const decoratorNode = DecoratorNode.create(
|
||
atOperator,
|
||
this._desugarViperMagicAttribute(expression)
|
||
);
|
||
|
||
if (!this._consumeTokenIfType(TokenType.NewLine)) {
|
||
this._addSyntaxError(LocMessage.expectedDecoratorNewline(), this._peekToken());
|
||
this._consumeTokensUntilType([TokenType.NewLine]);
|
||
}
|
||
|
||
return decoratorNode;
|
||
}
|
||
|
||
private _isNameOrMemberAccessExpression(expression: ExpressionNode): boolean {
|
||
if (expression.nodeType === ParseNodeType.Name) {
|
||
return true;
|
||
} else if (expression.nodeType === ParseNodeType.MemberAccess) {
|
||
return this._isNameOrMemberAccessExpression(expression.d.leftExpr);
|
||
}
|
||
|
||
return false;
|
||
}
|
||
|
||
// Viper 扩展：判断 `define` 之后是否为编译期常量声明（`NAME: 类型 = 值` / `NAME = 值`）。
|
||
// 位置基于 define 关键字：`_peekToken(1)` 是名字，`_peekToken(2)` 是 ':' 或 '='。
|
||
private _isViperDefineConstDeclaration(): boolean {
|
||
if (this._peekToken(1).type !== TokenType.Identifier) {
|
||
return false;
|
||
}
|
||
|
||
const afterName = this._peekToken(2);
|
||
if (afterName.type === TokenType.Colon) {
|
||
return true;
|
||
}
|
||
|
||
return afterName.type === TokenType.Operator && (afterName as OperatorToken).operatorType === OperatorType.Assign;
|
||
}
|
||
|
||
// Viper 扩展：校验 vtable 修饰方法（`virtual def` / `novirtual def`）的出现位置。
|
||
// - 必须位于类体内（模块级、函数体内均非法）；
|
||
// - `virtual def` 不能位于 `novirtual class` 内（该形态的类不生成 vtable）。
|
||
private _validateViperVirtualMethodPlacement(modifierText: string, modifierToken: Token): void {
|
||
const enclosingClassKind = this._viperEnclosingClassKind;
|
||
if (!enclosingClassKind) {
|
||
this._addSyntaxError(
|
||
modifierText === 'virtual'
|
||
? LocMessage.viperVirtualDefPlacement()
|
||
: LocMessage.viperNoVirtualDefPlacement(),
|
||
modifierToken
|
||
);
|
||
return;
|
||
}
|
||
|
||
if (modifierText === 'virtual' && enclosingClassKind === 'novirtual') {
|
||
this._addSyntaxError(LocMessage.viperVirtualDefInNoVirtualClass(), modifierToken);
|
||
}
|
||
}
|
||
|
||
// classdef: 'class' NAME ['(' [arglist] ')'] suite
|
||
private _parseClassDef(decorators?: DecoratorNode[], viperClassModifier?: string): ClassNode {
|
||
const classToken = this._getKeywordToken(KeywordType.Class);
|
||
|
||
let nameToken = this._getTokenIfIdentifier();
|
||
if (!nameToken) {
|
||
this._addSyntaxError(LocMessage.expectedClassName(), this._peekToken());
|
||
nameToken = IdentifierToken.create(0, 0, '', /* comments */ undefined);
|
||
}
|
||
|
||
let typeParameters: TypeParameterListNode | undefined;
|
||
const possibleOpenBracket = this._peekToken();
|
||
if (possibleOpenBracket.type === TokenType.OpenBracket) {
|
||
typeParameters = this._parseTypeParameterList();
|
||
|
||
if (
|
||
!this._parseOptions.isStubFile &&
|
||
PythonVersion.isLessThan(this._getLanguageVersion(), pythonVersion3_12)
|
||
) {
|
||
this._addSyntaxError(LocMessage.classTypeParametersIllegal(), typeParameters);
|
||
}
|
||
}
|
||
|
||
let argList: ArgumentNode[] = [];
|
||
const openParenToken = this._peekToken();
|
||
if (this._consumeTokenIfType(TokenType.OpenParenthesis)) {
|
||
argList = this._parseArgList().args;
|
||
|
||
if (!this._consumeTokenIfType(TokenType.CloseParenthesis)) {
|
||
this._addSyntaxError(LocMessage.expectedCloseParen(), openParenToken);
|
||
}
|
||
}
|
||
|
||
// Viper 扩展：类体内的 vtable 修饰形态（供 `virtual def` / `novirtual def` 位置校验）。
|
||
const enclosingClassKind: 'plain' | 'virtual' | 'novirtual' =
|
||
viperClassModifier === 'virtual' ? 'virtual' : viperClassModifier === 'novirtual' ? 'novirtual' : 'plain';
|
||
const wasEnclosingClassKind = this._viperEnclosingClassKind;
|
||
this._viperEnclosingClassKind = enclosingClassKind;
|
||
|
||
const suite = this._parseSuite(
|
||
/* isFunction */ false,
|
||
this._parseOptions.skipFunctionAndClassBody,
|
||
/* postColonCallback */ undefined,
|
||
/* allowTrailingIndent */ classToken.isViperEnum === true
|
||
);
|
||
|
||
this._viperEnclosingClassKind = wasEnclosingClassKind;
|
||
|
||
const classNode = ClassNode.create(classToken, NameNode.create(nameToken), suite, typeParameters);
|
||
classNode.d.arguments = argList;
|
||
argList.forEach((arg) => {
|
||
arg.parent = classNode;
|
||
});
|
||
|
||
// Viper 扩展：`enum X:` 语法声明的枚举（tokenizer 在 class 关键字 token 上打标记）。
|
||
if (classToken.isViperEnum) {
|
||
classNode.d.isViperEnum = true;
|
||
}
|
||
|
||
// Viper 扩展：`virtual class X` / `novirtual class X` 的修饰关键字。
|
||
if (viperClassModifier) {
|
||
classNode.d.viperClassModifier = viperClassModifier;
|
||
}
|
||
|
||
if (decorators) {
|
||
classNode.d.decorators = decorators;
|
||
if (decorators.length > 0) {
|
||
decorators.forEach((decorator) => {
|
||
decorator.parent = classNode;
|
||
});
|
||
extendRange(classNode, decorators[0]);
|
||
}
|
||
}
|
||
|
||
return classNode;
|
||
}
|
||
|
||
private _parsePassStatement(): PassNode {
|
||
return PassNode.create(this._getKeywordToken(KeywordType.Pass));
|
||
}
|
||
|
||
private _parseBreakStatement(): BreakNode {
|
||
const breakToken = this._getKeywordToken(KeywordType.Break);
|
||
|
||
if (!this._isInLoop) {
|
||
this._addSyntaxError(LocMessage.breakOutsideLoop(), breakToken);
|
||
} else if (this._isInExceptionGroup) {
|
||
this._addSyntaxError(LocMessage.breakInExceptionGroup(), breakToken);
|
||
}
|
||
|
||
if (this._isInFinallyLoop && PythonVersion.isGreaterOrEqualTo(this._getLanguageVersion(), pythonVersion3_14)) {
|
||
this._addSyntaxError(LocMessage.finallyBreak(), breakToken);
|
||
}
|
||
|
||
return BreakNode.create(breakToken);
|
||
}
|
||
|
||
private _parseContinueStatement(): ContinueNode {
|
||
const continueToken = this._getKeywordToken(KeywordType.Continue);
|
||
|
||
if (!this._isInLoop) {
|
||
this._addSyntaxError(LocMessage.continueOutsideLoop(), continueToken);
|
||
} else if (this._isInExceptionGroup) {
|
||
this._addSyntaxError(LocMessage.continueInExceptionGroup(), continueToken);
|
||
}
|
||
|
||
if (this._isInFinallyLoop && PythonVersion.isGreaterOrEqualTo(this._getLanguageVersion(), pythonVersion3_14)) {
|
||
this._addSyntaxError(LocMessage.finallyContinue(), continueToken);
|
||
}
|
||
|
||
return ContinueNode.create(continueToken);
|
||
}
|
||
|
||
// return_stmt: 'return' [testlist]
|
||
private _parseReturnStatement(): ReturnNode {
|
||
const returnToken = this._getKeywordToken(KeywordType.Return);
|
||
|
||
const returnNode = ReturnNode.create(returnToken);
|
||
|
||
if (!this._isInFunction) {
|
||
this._addSyntaxError(LocMessage.returnOutsideFunction(), returnToken);
|
||
} else if (this._isInExceptionGroup) {
|
||
this._addSyntaxError(LocMessage.returnInExceptionGroup(), returnToken);
|
||
}
|
||
|
||
if (this._isInFinallyBlock && PythonVersion.isGreaterOrEqualTo(this._getLanguageVersion(), pythonVersion3_14)) {
|
||
this._addSyntaxError(LocMessage.finallyReturn(), returnToken);
|
||
}
|
||
|
||
if (!this._isNextTokenNeverExpression()) {
|
||
const returnExpr = this._parseTestOrStarListAsExpression(
|
||
/* allowAssignmentExpression */ true,
|
||
/* allowMultipleUnpack */ true,
|
||
ErrorExpressionCategory.MissingExpression,
|
||
() => LocMessage.expectedReturnExpr()
|
||
);
|
||
this._reportConditionalErrorForStarTupleElement(returnExpr);
|
||
returnNode.d.expr = returnExpr;
|
||
returnNode.d.expr.parent = returnNode;
|
||
extendRange(returnNode, returnExpr);
|
||
}
|
||
|
||
return returnNode;
|
||
}
|
||
|
||
// import_from: ('from' (('.' | '...')* dotted_name | ('.' | '...')+)
|
||
// 'import' ('*' | '(' import_as_names ')' | import_as_names))
|
||
// import_as_names: import_as_name (',' import_as_name)* [',']
|
||
// import_as_name: NAME ['as' NAME]
|
||
private _parseFromStatement(): ImportFromNode {
|
||
const fromToken = this._getKeywordToken(KeywordType.From);
|
||
|
||
const modName = this._parseDottedModuleName(/* allowJustDots */ true);
|
||
const importFromNode = ImportFromNode.create(fromToken, modName);
|
||
|
||
// Handle imports from __future__ specially because they can
|
||
// change the way we interpret the rest of the file.
|
||
const isFutureImport =
|
||
modName.d.leadingDots === 0 &&
|
||
modName.d.nameParts.length === 1 &&
|
||
modName.d.nameParts[0].d.value === '__future__';
|
||
|
||
const possibleInputToken = this._peekToken();
|
||
if (!this._consumeTokenIfKeyword(KeywordType.Import)) {
|
||
this._addSyntaxError(LocMessage.expectedImport(), this._peekToken());
|
||
if (!modName.d.hasTrailingDot) {
|
||
importFromNode.d.missingImport = true;
|
||
}
|
||
} else {
|
||
extendRange(importFromNode, possibleInputToken);
|
||
|
||
// Look for "*" token.
|
||
const possibleStarToken = this._peekToken();
|
||
if (this._consumeTokenIfOperator(OperatorType.Multiply)) {
|
||
extendRange(importFromNode, possibleStarToken);
|
||
importFromNode.d.isWildcardImport = true;
|
||
importFromNode.d.wildcardToken = possibleStarToken;
|
||
this._containsWildcardImport = true;
|
||
} else {
|
||
const openParenToken = this._peekToken();
|
||
const inParen = this._consumeTokenIfType(TokenType.OpenParenthesis);
|
||
let trailingCommaToken: Token | undefined;
|
||
|
||
while (true) {
|
||
const importName = this._getTokenIfIdentifier();
|
||
if (!importName) {
|
||
break;
|
||
}
|
||
|
||
trailingCommaToken = undefined;
|
||
|
||
const importFromAsNode = ImportFromAsNode.create(NameNode.create(importName));
|
||
|
||
if (this._consumeTokenIfKeyword(KeywordType.As)) {
|
||
const aliasName = this._getTokenIfIdentifier();
|
||
if (!aliasName) {
|
||
this._addSyntaxError(LocMessage.expectedImportAlias(), this._peekToken());
|
||
} else {
|
||
importFromAsNode.d.alias = NameNode.create(aliasName);
|
||
importFromAsNode.d.alias.parent = importFromAsNode;
|
||
extendRange(importFromAsNode, aliasName);
|
||
}
|
||
}
|
||
|
||
importFromNode.d.imports.push(importFromAsNode);
|
||
importFromAsNode.parent = importFromNode;
|
||
extendRange(importFromNode, importFromAsNode);
|
||
|
||
if (isFutureImport) {
|
||
// Add the future import by name.
|
||
this._futureImports.add(importName.value);
|
||
}
|
||
|
||
const nextToken = this._peekToken();
|
||
if (!this._consumeTokenIfType(TokenType.Comma)) {
|
||
break;
|
||
}
|
||
trailingCommaToken = nextToken;
|
||
}
|
||
|
||
if (importFromNode.d.imports.length === 0) {
|
||
this._addSyntaxError(LocMessage.expectedImportSymbols(), this._peekToken());
|
||
}
|
||
|
||
if (inParen) {
|
||
importFromNode.d.usesParens = true;
|
||
|
||
const nextToken = this._peekToken();
|
||
if (!this._consumeTokenIfType(TokenType.CloseParenthesis)) {
|
||
this._addSyntaxError(LocMessage.expectedCloseParen(), openParenToken);
|
||
} else {
|
||
extendRange(importFromNode, nextToken);
|
||
}
|
||
} else if (trailingCommaToken) {
|
||
this._addSyntaxError(LocMessage.trailingCommaInFromImport(), trailingCommaToken);
|
||
}
|
||
}
|
||
}
|
||
|
||
this._importedModules.push({
|
||
nameNode: importFromNode.d.module,
|
||
leadingDots: importFromNode.d.module.d.leadingDots,
|
||
nameParts: importFromNode.d.module.d.nameParts.map((p) => p.d.value),
|
||
importedSymbols: new Set<string>(importFromNode.d.imports.map((imp) => imp.d.name.d.value)),
|
||
});
|
||
|
||
let isTypingImport = false;
|
||
if (importFromNode.d.module.d.nameParts.length === 1) {
|
||
const firstNamePartValue = importFromNode.d.module.d.nameParts[0].d.value;
|
||
if (firstNamePartValue === 'typing' || firstNamePartValue === 'typing_extensions') {
|
||
isTypingImport = true;
|
||
}
|
||
}
|
||
|
||
if (isTypingImport) {
|
||
const typingSymbolsOfInterest = ['Literal', 'TypeAlias', 'Annotated'];
|
||
|
||
if (importFromNode.d.isWildcardImport) {
|
||
typingSymbolsOfInterest.forEach((s) => {
|
||
this._typingSymbolAliases.set(s, s);
|
||
});
|
||
} else {
|
||
importFromNode.d.imports.forEach((imp) => {
|
||
if (typingSymbolsOfInterest.some((s) => s === imp.d.name.d.value)) {
|
||
this._typingSymbolAliases.set(imp.d.alias?.d.value || imp.d.name.d.value, imp.d.name.d.value);
|
||
}
|
||
});
|
||
}
|
||
}
|
||
|
||
return importFromNode;
|
||
}
|
||
|
||
// import_name: 'import' dotted_as_names
|
||
// dotted_as_names: dotted_as_name (',' dotted_as_name)*
|
||
// dotted_as_name: dotted_name ['as' NAME]
|
||
private _parseImportStatement(): ImportNode {
|
||
const importToken = this._getKeywordToken(KeywordType.Import);
|
||
|
||
const importNode = ImportNode.create(importToken);
|
||
|
||
while (true) {
|
||
const modName = this._parseDottedModuleName();
|
||
|
||
const importAsNode = ImportAsNode.create(modName);
|
||
|
||
if (this._consumeTokenIfKeyword(KeywordType.As)) {
|
||
const aliasToken = this._getTokenIfIdentifier();
|
||
if (aliasToken) {
|
||
importAsNode.d.alias = NameNode.create(aliasToken);
|
||
importAsNode.d.alias.parent = importAsNode;
|
||
extendRange(importAsNode, importAsNode.d.alias);
|
||
} else {
|
||
this._addSyntaxError(LocMessage.expectedImportAlias(), this._peekToken());
|
||
}
|
||
}
|
||
|
||
if (importAsNode.d.module.d.leadingDots > 0) {
|
||
this._addSyntaxError(LocMessage.relativeImportNotAllowed(), importAsNode.d.module);
|
||
}
|
||
|
||
importNode.d.list.push(importAsNode);
|
||
importAsNode.parent = importNode;
|
||
|
||
const nameParts = importAsNode.d.module.d.nameParts.map((p) => p.d.value);
|
||
|
||
if (
|
||
importAsNode.d.alias ||
|
||
importAsNode.d.module.d.leadingDots > 0 ||
|
||
importAsNode.d.module.d.nameParts.length === 0
|
||
) {
|
||
this._importedModules.push({
|
||
nameNode: importAsNode.d.module,
|
||
leadingDots: importAsNode.d.module.d.leadingDots,
|
||
nameParts,
|
||
importedSymbols: undefined,
|
||
});
|
||
} else {
|
||
// Implicitly import all modules in the multi-part name if we
|
||
// are not assigning the final module to an alias.
|
||
importAsNode.d.module.d.nameParts.forEach((_, index) => {
|
||
this._importedModules.push({
|
||
nameNode: importAsNode.d.module,
|
||
leadingDots: importAsNode.d.module.d.leadingDots,
|
||
nameParts: nameParts.slice(0, index + 1),
|
||
importedSymbols: undefined,
|
||
});
|
||
});
|
||
}
|
||
|
||
if (modName.d.nameParts.length === 1) {
|
||
const firstNamePartValue = modName.d.nameParts[0].d.value;
|
||
if (firstNamePartValue === 'typing' || firstNamePartValue === 'typing_extensions') {
|
||
this._typingImportAliases.push(importAsNode.d.alias?.d.value || firstNamePartValue);
|
||
}
|
||
}
|
||
|
||
if (!this._consumeTokenIfType(TokenType.Comma)) {
|
||
break;
|
||
}
|
||
}
|
||
|
||
if (importNode.d.list.length > 0) {
|
||
extendRange(importNode, importNode.d.list[importNode.d.list.length - 1]);
|
||
}
|
||
|
||
return importNode;
|
||
}
|
||
|
||
// ('.' | '...')* dotted_name | ('.' | '...')+
|
||
// dotted_name: NAME ('.' NAME)*
|
||
private _parseDottedModuleName(allowJustDots = false): ModuleNameNode {
|
||
const moduleNameNode = ModuleNameNode.create(this._peekToken());
|
||
|
||
while (true) {
|
||
const token = this._getTokenIfType(TokenType.Ellipsis) ?? this._getTokenIfType(TokenType.Dot);
|
||
if (token) {
|
||
if (token.type === TokenType.Ellipsis) {
|
||
moduleNameNode.d.leadingDots += 3;
|
||
} else {
|
||
moduleNameNode.d.leadingDots++;
|
||
}
|
||
|
||
extendRange(moduleNameNode, token);
|
||
} else {
|
||
break;
|
||
}
|
||
}
|
||
|
||
while (true) {
|
||
const identifier = this._getTokenIfIdentifier();
|
||
if (!identifier) {
|
||
if (!allowJustDots || moduleNameNode.d.leadingDots === 0 || moduleNameNode.d.nameParts.length > 0) {
|
||
this._addSyntaxError(LocMessage.expectedModuleName(), this._peekToken());
|
||
moduleNameNode.d.hasTrailingDot = true;
|
||
}
|
||
break;
|
||
}
|
||
|
||
const namePart = NameNode.create(identifier);
|
||
moduleNameNode.d.nameParts.push(namePart);
|
||
namePart.parent = moduleNameNode;
|
||
extendRange(moduleNameNode, namePart);
|
||
|
||
const nextToken = this._peekToken();
|
||
if (!this._consumeTokenIfType(TokenType.Dot)) {
|
||
break;
|
||
}
|
||
|
||
// Extend the module name to include the dot.
|
||
extendRange(moduleNameNode, nextToken);
|
||
}
|
||
|
||
return moduleNameNode;
|
||
}
|
||
|
||
private _parseGlobalStatement(): GlobalNode {
|
||
const globalToken = this._getKeywordToken(KeywordType.Global);
|
||
|
||
const globalNode = GlobalNode.create(globalToken);
|
||
globalNode.d.targets = this._parseNameList();
|
||
if (globalNode.d.targets.length > 0) {
|
||
globalNode.d.targets.forEach((name) => {
|
||
name.parent = globalNode;
|
||
});
|
||
extendRange(globalNode, globalNode.d.targets[globalNode.d.targets.length - 1]);
|
||
}
|
||
return globalNode;
|
||
}
|
||
|
||
private _parseNonlocalStatement(): NonlocalNode {
|
||
const nonlocalToken = this._getKeywordToken(KeywordType.Nonlocal);
|
||
|
||
const nonlocalNode = NonlocalNode.create(nonlocalToken);
|
||
nonlocalNode.d.targets = this._parseNameList();
|
||
if (nonlocalNode.d.targets.length > 0) {
|
||
nonlocalNode.d.targets.forEach((name) => {
|
||
name.parent = nonlocalNode;
|
||
});
|
||
extendRange(nonlocalNode, nonlocalNode.d.targets[nonlocalNode.d.targets.length - 1]);
|
||
}
|
||
return nonlocalNode;
|
||
}
|
||
|
||
private _parseNameList(): NameNode[] {
|
||
const nameList: NameNode[] = [];
|
||
|
||
while (true) {
|
||
const name = this._getTokenIfIdentifier();
|
||
if (!name) {
|
||
this._addSyntaxError(LocMessage.expectedIdentifier(), this._peekToken());
|
||
break;
|
||
}
|
||
|
||
nameList.push(NameNode.create(name));
|
||
|
||
if (!this._consumeTokenIfType(TokenType.Comma)) {
|
||
break;
|
||
}
|
||
}
|
||
|
||
return nameList;
|
||
}
|
||
|
||
// raise_stmt: 'raise' [test ['from' test]]
|
||
// (old) raise_stmt: 'raise' [test [',' test [',' test]]]
|
||
private _parseRaiseStatement(): RaiseNode {
|
||
const raiseToken = this._getKeywordToken(KeywordType.Raise);
|
||
|
||
const raiseNode = RaiseNode.create(raiseToken);
|
||
if (!this._isNextTokenNeverExpression()) {
|
||
raiseNode.d.expr = this._parseTestExpression(/* allowAssignmentExpression */ true);
|
||
raiseNode.d.expr.parent = raiseNode;
|
||
extendRange(raiseNode, raiseNode.d.expr);
|
||
|
||
if (this._consumeTokenIfKeyword(KeywordType.From)) {
|
||
raiseNode.d.fromExpr = this._parseTestExpression(/* allowAssignmentExpression */ true);
|
||
raiseNode.d.fromExpr.parent = raiseNode;
|
||
extendRange(raiseNode, raiseNode.d.fromExpr);
|
||
}
|
||
}
|
||
|
||
return raiseNode;
|
||
}
|
||
|
||
// assert_stmt: 'assert' test [',' test]
|
||
private _parseAssertStatement(): AssertNode {
|
||
const assertToken = this._getKeywordToken(KeywordType.Assert);
|
||
|
||
const expr = this._parseTestExpression(/* allowAssignmentExpression */ false);
|
||
const assertNode = AssertNode.create(assertToken, expr);
|
||
|
||
if (this._consumeTokenIfType(TokenType.Comma)) {
|
||
const exceptionExpr = this._parseTestExpression(/* allowAssignmentExpression */ false);
|
||
assertNode.d.exceptionExpr = exceptionExpr;
|
||
assertNode.d.exceptionExpr.parent = assertNode;
|
||
extendRange(assertNode, exceptionExpr);
|
||
}
|
||
|
||
return assertNode;
|
||
}
|
||
|
||
// del_stmt: 'del' exprlist
|
||
private _parseDelStatement(): DelNode {
|
||
const delToken = this._getKeywordToken(KeywordType.Del);
|
||
|
||
const exprListResult = this._parseExpressionList(/* allowStar */ true);
|
||
if (!exprListResult.parseError && exprListResult.list.length === 0) {
|
||
this._addSyntaxError(LocMessage.expectedDelExpr(), this._peekToken());
|
||
}
|
||
const delNode = DelNode.create(delToken);
|
||
delNode.d.targets = exprListResult.list;
|
||
if (delNode.d.targets.length > 0) {
|
||
delNode.d.targets.forEach((expr) => {
|
||
expr.parent = delNode;
|
||
});
|
||
extendRange(delNode, delNode.d.targets[delNode.d.targets.length - 1]);
|
||
}
|
||
return delNode;
|
||
}
|
||
|
||
// yield_expr: 'yield' [yield_arg]
|
||
// yield_arg: 'from' test | testlist
|
||
private _parseYieldExpression(): YieldNode | YieldFromNode {
|
||
const yieldToken = this._getKeywordToken(KeywordType.Yield);
|
||
|
||
const nextToken = this._peekToken();
|
||
if (this._consumeTokenIfKeyword(KeywordType.From)) {
|
||
if (PythonVersion.isLessThan(this._getLanguageVersion(), pythonVersion3_3)) {
|
||
this._addSyntaxError(LocMessage.yieldFromIllegal(), nextToken);
|
||
}
|
||
return YieldFromNode.create(yieldToken, this._parseTestExpression(/* allowAssignmentExpression */ false));
|
||
}
|
||
|
||
let exprList: ExpressionNode | undefined;
|
||
if (!this._isNextTokenNeverExpression()) {
|
||
exprList = this._parseTestOrStarListAsExpression(
|
||
/* allowAssignmentExpression */ false,
|
||
/* allowMultipleUnpack */ true,
|
||
ErrorExpressionCategory.MissingExpression,
|
||
() => LocMessage.expectedYieldExpr()
|
||
);
|
||
this._reportConditionalErrorForStarTupleElement(exprList);
|
||
}
|
||
|
||
return YieldNode.create(yieldToken, exprList);
|
||
}
|
||
|
||
private _tryParseYieldExpression(): YieldNode | YieldFromNode | undefined {
|
||
if (this._peekKeywordType() !== KeywordType.Yield) {
|
||
return undefined;
|
||
}
|
||
|
||
return this._parseYieldExpression();
|
||
}
|
||
|
||
// simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
|
||
private _parseSimpleStatement(): StatementListNode {
|
||
const statement = StatementListNode.create(this._peekToken());
|
||
|
||
while (true) {
|
||
// Swallow invalid tokens to make sure we make forward progress.
|
||
if (this._peekTokenType() === TokenType.Invalid) {
|
||
const invalidToken = this._getNextToken();
|
||
const text = this._fileContents!.substr(invalidToken.start, invalidToken.length);
|
||
|
||
const firstCharCode = text.charCodeAt(0);
|
||
|
||
// If the invalid token is a line-continuation backslash at the end of the file,
|
||
// report a clearer error message consistent with Python: "Unexpected EOF".
|
||
const nextTok = this._peekToken();
|
||
const nextNextTok = this._peekToken(1);
|
||
const isBackslash = firstCharCode === Char.Backslash;
|
||
const isAtEofLineContinuation =
|
||
isBackslash && nextTok.type === TokenType.NewLine && nextNextTok.type === TokenType.EndOfStream;
|
||
|
||
if (isAtEofLineContinuation) {
|
||
this._addSyntaxError(LocMessage.unexpectedEof(), invalidToken);
|
||
} else {
|
||
// Remove any non-printable characters.
|
||
this._addSyntaxError(
|
||
LocMessage.invalidTokenChars().format({ text: `\\u${firstCharCode.toString(16)}` }),
|
||
invalidToken
|
||
);
|
||
}
|
||
|
||
this._consumeTokensUntilType([TokenType.NewLine]);
|
||
break;
|
||
}
|
||
|
||
const smallStatement = this._parseSmallStatement();
|
||
statement.d.statements.push(smallStatement);
|
||
smallStatement.parent = statement;
|
||
extendRange(statement, smallStatement);
|
||
|
||
if (smallStatement.nodeType === ParseNodeType.Error) {
|
||
// No need to log an error here. We assume that
|
||
// it was already logged by _parseSmallStatement.
|
||
break;
|
||
}
|
||
|
||
// Consume the semicolon if present.
|
||
if (!this._consumeTokenIfType(TokenType.Semicolon)) {
|
||
break;
|
||
}
|
||
|
||
const nextTokenType = this._peekTokenType();
|
||
if (nextTokenType === TokenType.NewLine || nextTokenType === TokenType.EndOfStream) {
|
||
break;
|
||
}
|
||
}
|
||
|
||
if (!this._consumeTokenIfType(TokenType.NewLine)) {
|
||
this._addSyntaxError(LocMessage.expectedNewlineOrSemicolon(), this._peekToken());
|
||
}
|
||
|
||
return statement;
|
||
}
|
||
|
||
// small_stmt: (expr_stmt | del_stmt | pass_stmt | flow_stmt |
|
||
// import_stmt | global_stmt | nonlocal_stmt | assert_stmt)
|
||
// flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt
|
||
// import_stmt: import_name | import_from
|
||
private _parseSmallStatement(): ParseNode {
|
||
// Viper 扩展：`typedef T1 = T2` 软关键字语句。
|
||
// 仅当语句起始处形如 `typedef <Name> =` 时按 typedef 解析；
|
||
// 否则 `typedef` 仍是普通标识符（`typedef = 1`、`typedef(x)` 不受影响）。
|
||
if (this._isAtViperTypedefStatement()) {
|
||
return this._parseViperTypedefStatement();
|
||
}
|
||
|
||
switch (this._peekKeywordType()) {
|
||
case KeywordType.Pass:
|
||
return this._parsePassStatement();
|
||
|
||
case KeywordType.Break:
|
||
return this._parseBreakStatement();
|
||
|
||
case KeywordType.Continue:
|
||
return this._parseContinueStatement();
|
||
|
||
case KeywordType.Return:
|
||
return this._parseReturnStatement();
|
||
|
||
case KeywordType.From:
|
||
return this._parseFromStatement();
|
||
|
||
case KeywordType.Import:
|
||
return this._parseImportStatement();
|
||
|
||
case KeywordType.Global:
|
||
return this._parseGlobalStatement();
|
||
|
||
case KeywordType.Nonlocal:
|
||
return this._parseNonlocalStatement();
|
||
|
||
case KeywordType.Raise:
|
||
return this._parseRaiseStatement();
|
||
|
||
case KeywordType.Assert:
|
||
return this._parseAssertStatement();
|
||
|
||
case KeywordType.Del:
|
||
return this._parseDelStatement();
|
||
|
||
case KeywordType.Yield:
|
||
return this._parseYieldExpression();
|
||
|
||
case KeywordType.Lazy: {
|
||
// Lazy is considered a "soft" keyword, so we will treat it
|
||
// as an identifier if it is not followed by "import" or "from".
|
||
const peekToken1 = this._peekToken(1);
|
||
if (
|
||
peekToken1.type === TokenType.Keyword &&
|
||
((peekToken1 as KeywordToken).keywordType === KeywordType.Import ||
|
||
(peekToken1 as KeywordToken).keywordType === KeywordType.From)
|
||
) {
|
||
return this._parseLazyImportStatement();
|
||
}
|
||
break;
|
||
}
|
||
|
||
case KeywordType.Type: {
|
||
// Type is considered a "soft" keyword, so we will treat it
|
||
// as an identifier if it is followed by an unexpected token.
|
||
|
||
const peekToken1 = this._peekToken(1);
|
||
const peekToken2 = this._peekToken(2);
|
||
let isInvalidTypeToken = true;
|
||
|
||
if (
|
||
peekToken1.type === TokenType.Identifier ||
|
||
(peekToken1.type === TokenType.Keyword && KeywordToken.isSoftKeyword(peekToken1 as KeywordToken))
|
||
) {
|
||
if (peekToken2.type === TokenType.OpenBracket) {
|
||
isInvalidTypeToken = false;
|
||
} else if (
|
||
peekToken2.type === TokenType.Operator &&
|
||
(peekToken2 as OperatorToken).operatorType === OperatorType.Assign
|
||
) {
|
||
isInvalidTypeToken = false;
|
||
}
|
||
}
|
||
|
||
if (!isInvalidTypeToken) {
|
||
return this._parseTypeAliasStatement();
|
||
}
|
||
break;
|
||
}
|
||
}
|
||
|
||
return this._parseExpressionStatement();
|
||
}
|
||
|
||
// Viper 扩展：判定语句起始处是否为 `typedef <Name> =` 形状。
|
||
// `typedef` 是软关键字：仅当后跟「标识符」再跟 `=` 时才是 typedef 语句，
|
||
// 否则保持普通标识符解析（`typedef = 1` 是赋值、`typedef(x)` 是调用）。
|
||
private _isAtViperTypedefStatement(): boolean {
|
||
if (this._parseOptions.dialect !== 'viper') {
|
||
return false;
|
||
}
|
||
const keywordToken = this._peekToken();
|
||
if (keywordToken.type !== TokenType.Identifier || (keywordToken as IdentifierToken).value !== 'typedef') {
|
||
return false;
|
||
}
|
||
if (this._peekToken(1).type !== TokenType.Identifier) {
|
||
return false;
|
||
}
|
||
const assignToken = this._peekToken(2);
|
||
return (
|
||
assignToken.type === TokenType.Operator &&
|
||
(assignToken as OperatorToken).operatorType === OperatorType.Assign
|
||
);
|
||
}
|
||
|
||
// Viper 扩展：`typedef T1 = T2` 语句（与 `T1: t.CTypedef = T2` 完全等价）。
|
||
// 解析层脱糖为「带注解的赋值」节点：左值 `T1`，注解为合成的零长 `t.CTypedef`，
|
||
// 右值为 T2 —— 与手写 `T1: t.CTypedef = T2` 的 AST 同形，从而复用既有的
|
||
// TypeAlias 语义（绑定、类型求值、hover、语义高亮均一致）。
|
||
//
|
||
// 取舍：不依赖源码文件 `import t`。合成 `t.CTypedef` 并不真去解析 `t` 符号：
|
||
// 类型求值层识别 t.CTypedef 只看成员名文本（见 typeEvaluator 的
|
||
// isExplicitTypeAliasDeclaration / hoverProvider 的 CTypedef 快速路径），
|
||
// 合成 `t` 标记 isViperTypedef 后，未 import t 时静默退化为 Unknown
|
||
// （与 `[SIZE]T` 数组脱糖合成 `t.CArray` 的取舍相同）。
|
||
private _parseViperTypedefStatement(): ExpressionNode {
|
||
const typedefToken = this._getNextToken();
|
||
assert(typedefToken.type === TokenType.Identifier);
|
||
|
||
const nameToken = this._getNextToken() as IdentifierToken;
|
||
const nameNode = NameNode.create(nameToken);
|
||
|
||
// 消费 '='（_isAtViperTypedefStatement 已确认存在）。
|
||
this._getNextToken();
|
||
|
||
// 右值与 `T1: t.CTypedef = T2` 完全同路解析。
|
||
const rightExpr =
|
||
this._tryParseYieldExpression() ??
|
||
this._parseTestOrStarListAsExpression(
|
||
/* allowAssignmentExpression */ false,
|
||
/* allowMultipleUnpack */ true,
|
||
ErrorExpressionCategory.MissingExpression,
|
||
() => LocMessage.expectedAssignRightHandExpr()
|
||
);
|
||
|
||
// 合成 `t.CTypedef` 注解：零长度、位置紧贴别名名之后（满足
|
||
// "子节点按位置递增且不重叠" 的 AST 不变量），故不产生语义高亮
|
||
// （semanticTokenProvider 跳过 length <= 0 的名字），也不参与位置命中。
|
||
const syntheticStart = nameToken.start + nameToken.length;
|
||
const syntheticTNameNode = NameNode.create(
|
||
IdentifierToken.create(syntheticStart, 0, 't', /* comments */ undefined)
|
||
);
|
||
syntheticTNameNode.d.isViperTypedef = true;
|
||
const cTypedefNameNode = NameNode.create(
|
||
IdentifierToken.create(syntheticStart, 0, 'CTypedef', /* comments */ undefined)
|
||
);
|
||
const annotationNode = MemberAccessNode.create(syntheticTNameNode, cTypedefNameNode);
|
||
|
||
const leftExpr = TypeAnnotationNode.create(nameNode, annotationNode);
|
||
return AssignmentNode.create(leftExpr, rightExpr);
|
||
}
|
||
|
||
private _makeExpressionOrTuple(
|
||
exprListResult: ListResult<ExpressionNode>,
|
||
enclosedInParens: boolean
|
||
): ExpressionNode {
|
||
// A single-element tuple with no trailing comma is simply an expression
|
||
// that's surrounded by parens.
|
||
if (exprListResult.list.length === 1 && !exprListResult.trailingComma) {
|
||
if (exprListResult.list[0].nodeType === ParseNodeType.Unpack) {
|
||
this._addSyntaxError(LocMessage.unpackOperatorNotAllowed(), exprListResult.list[0]);
|
||
}
|
||
return exprListResult.list[0];
|
||
}
|
||
|
||
// To accommodate empty tuples ("()"), we will reach back to get
|
||
// the opening parenthesis as the opening token.
|
||
|
||
const tupleStartRange: TextRange =
|
||
exprListResult.list.length > 0 ? exprListResult.list[0] : this._peekToken(-1);
|
||
|
||
const tupleNode = TupleNode.create(tupleStartRange, enclosedInParens);
|
||
tupleNode.d.items = exprListResult.list;
|
||
if (exprListResult.list.length > 0) {
|
||
exprListResult.list.forEach((expr) => {
|
||
expr.parent = tupleNode;
|
||
});
|
||
extendRange(tupleNode, exprListResult.list[exprListResult.list.length - 1]);
|
||
}
|
||
|
||
return tupleNode;
|
||
}
|
||
|
||
private _parseExpressionListAsPossibleTuple(
|
||
errorCategory: ErrorExpressionCategory,
|
||
getErrorString: () => string,
|
||
errorToken: Token
|
||
): ExpressionNode {
|
||
if (this._isNextTokenNeverExpression()) {
|
||
this._addSyntaxError(getErrorString(), errorToken);
|
||
return ErrorNode.create(errorToken, errorCategory);
|
||
}
|
||
|
||
const exprListResult = this._parseExpressionList(/* allowStar */ true);
|
||
if (exprListResult.parseError) {
|
||
return exprListResult.parseError;
|
||
}
|
||
return this._makeExpressionOrTuple(exprListResult, /* enclosedInParens */ false);
|
||
}
|
||
|
||
private _parseTestListAsExpression(
|
||
errorCategory: ErrorExpressionCategory,
|
||
getErrorString: () => string
|
||
): ExpressionNode {
|
||
if (this._isNextTokenNeverExpression()) {
|
||
return this._handleExpressionParseError(errorCategory, getErrorString());
|
||
}
|
||
|
||
const exprListResult = this._parseTestExpressionList();
|
||
if (exprListResult.parseError) {
|
||
return exprListResult.parseError;
|
||
}
|
||
return this._makeExpressionOrTuple(exprListResult, /* enclosedInParens */ false);
|
||
}
|
||
|
||
private _parseTestOrStarListAsExpression(
|
||
allowAssignmentExpression: boolean,
|
||
allowMultipleUnpack: boolean,
|
||
errorCategory: ErrorExpressionCategory,
|
||
getErrorString: () => string
|
||
): ExpressionNode {
|
||
if (this._isNextTokenNeverExpression()) {
|
||
return this._handleExpressionParseError(errorCategory, getErrorString());
|
||
}
|
||
|
||
const exprListResult = this._parseTestOrStarExpressionList(allowAssignmentExpression, allowMultipleUnpack);
|
||
if (exprListResult.parseError) {
|
||
return exprListResult.parseError;
|
||
}
|
||
return this._makeExpressionOrTuple(exprListResult, /* enclosedInParens */ false);
|
||
}
|
||
|
||
private _parseExpressionList(allowStar: boolean): ListResult<ExpressionNode> {
|
||
return this._parseExpressionListGeneric(() => this._parseExpression(allowStar));
|
||
}
|
||
|
||
// testlist: test (',' test)* [',']
|
||
private _parseTestExpressionList(): ListResult<ExpressionNode> {
|
||
return this._parseExpressionListGeneric(() => this._parseTestExpression(/* allowAssignmentExpression */ false));
|
||
}
|
||
|
||
private _parseTestOrStarExpressionList(
|
||
allowAssignmentExpression: boolean,
|
||
allowMultipleUnpack: boolean
|
||
): ListResult<ExpressionNode> {
|
||
const exprListResult = this._parseExpressionListGeneric(() =>
|
||
this._parseTestOrStarExpression(allowAssignmentExpression)
|
||
);
|
||
|
||
if (!allowMultipleUnpack && !exprListResult.parseError) {
|
||
let sawStar = false;
|
||
for (const expr of exprListResult.list) {
|
||
if (expr.nodeType === ParseNodeType.Unpack) {
|
||
if (sawStar) {
|
||
this._addSyntaxError(LocMessage.duplicateUnpack(), expr);
|
||
break;
|
||
}
|
||
sawStar = true;
|
||
}
|
||
}
|
||
}
|
||
|
||
return exprListResult;
|
||
}
|
||
|
||
// exp_or_star: expr | star_expr
|
||
// expr: xor_expr ('|' xor_expr)*
|
||
// star_expr: '*' expr
|
||
private _parseExpression(allowUnpack: boolean): ExpressionNode {
|
||
const startToken = this._peekToken();
|
||
|
||
if (allowUnpack && this._consumeTokenIfOperator(OperatorType.Multiply)) {
|
||
return UnpackNode.create(startToken, this._parseExpression(/* allowUnpack */ false));
|
||
}
|
||
|
||
return this._parseBitwiseOrExpression();
|
||
}
|
||
|
||
// test_or_star: test | star_expr
|
||
private _parseTestOrStarExpression(allowAssignmentExpression: boolean): ExpressionNode {
|
||
if (this._peekOperatorType() === OperatorType.Multiply) {
|
||
// Viper 扩展：前缀 '*' 在表达式位置可能是 C 风格一元解引用（`*p`），
|
||
// 也可能是解包元素（`[*a]`、`(*p, q)` 中的 `*a`/`*p`）。
|
||
// 先按一元解引用解析，若后随逗号/闭括号等分隔符（解包元素上下文）则改判为 Unpack。
|
||
if (this._parseOptions.dialect === 'viper') {
|
||
return this._parseViperStarExprOrDeref();
|
||
}
|
||
return this._parseExpression(/* allowUnpack */ true);
|
||
}
|
||
|
||
return this._parseTestExpression(allowAssignmentExpression);
|
||
}
|
||
|
||
// Viper 扩展：解析前缀 '*'。先消费 '*' 并以非解包模式解析其操作数，
|
||
// 若操作数后紧跟逗号/闭括号等（表示它是解包元素），改判为 Unpack 节点；
|
||
// 否则构造一元 '*'（解引用）节点，与 C 语义一致。
|
||
// 注意：`)` 不触发改判——`(*p)` 是单元素括号（解引用），而 `(*p,)`/`(*p, q)`
|
||
// 的 `*p` 后是逗号（解包元素）。
|
||
private _parseViperStarExprOrDeref(): ExpressionNode {
|
||
const startToken = this._peekToken();
|
||
this._getNextToken(); // 消费 '*'
|
||
const expression = this._parseExpression(/* allowUnpack */ false);
|
||
|
||
const nextType = this._peekToken().type;
|
||
if (
|
||
nextType === TokenType.Comma ||
|
||
nextType === TokenType.CloseBracket ||
|
||
nextType === TokenType.CloseCurlyBrace
|
||
) {
|
||
return UnpackNode.create(startToken, expression);
|
||
}
|
||
|
||
return this._createUnaryOperationNode(startToken, expression, OperatorType.Multiply);
|
||
}
|
||
|
||
// test: or_test ['if' or_test 'else' test] | lambdef
|
||
private _parseTestExpression(allowAssignmentExpression: boolean): ExpressionNode {
|
||
if (this._peekKeywordType() === KeywordType.Lambda) {
|
||
return this._parseLambdaExpression();
|
||
}
|
||
|
||
const ifExpr = this._parseAssignmentExpression(!allowAssignmentExpression);
|
||
if (ifExpr.nodeType === ParseNodeType.Error) {
|
||
return ifExpr;
|
||
}
|
||
|
||
if (!this._consumeTokenIfKeyword(KeywordType.If)) {
|
||
return ifExpr;
|
||
}
|
||
|
||
const testExpr = this._parseOrTest();
|
||
if (testExpr.nodeType === ParseNodeType.Error) {
|
||
return testExpr;
|
||
}
|
||
|
||
if (!this._consumeTokenIfKeyword(KeywordType.Else)) {
|
||
return TernaryNode.create(
|
||
ifExpr,
|
||
testExpr,
|
||
this._handleExpressionParseError(ErrorExpressionCategory.MissingElse, LocMessage.expectedElse())
|
||
);
|
||
}
|
||
|
||
const elseExpr = this._parseTestExpression(/* allowAssignmentExpression */ true);
|
||
|
||
return TernaryNode.create(ifExpr, testExpr, elseExpr);
|
||
}
|
||
|
||
// assign_expr: NAME := test
|
||
private _parseAssignmentExpression(disallowAssignmentExpression = false) {
|
||
const leftExpr = this._parseOrTest();
|
||
if (leftExpr.nodeType === ParseNodeType.Error) {
|
||
return leftExpr;
|
||
}
|
||
|
||
if (leftExpr.nodeType !== ParseNodeType.Name) {
|
||
return leftExpr;
|
||
}
|
||
|
||
const walrusToken = this._peekToken();
|
||
if (!this._consumeTokenIfOperator(OperatorType.Walrus)) {
|
||
return leftExpr;
|
||
}
|
||
|
||
if (!this._assignmentExpressionsAllowed || disallowAssignmentExpression) {
|
||
this._addSyntaxError(LocMessage.walrusNotAllowed(), walrusToken);
|
||
}
|
||
|
||
if (PythonVersion.isLessThan(this._getLanguageVersion(), pythonVersion3_8)) {
|
||
this._addSyntaxError(LocMessage.walrusIllegal(), walrusToken);
|
||
}
|
||
|
||
const rightExpr = this._parseTestExpression(/* allowAssignmentExpression */ false);
|
||
|
||
return AssignmentExpressionNode.create(leftExpr, walrusToken, rightExpr);
|
||
}
|
||
|
||
// or_test: and_test ('or' and_test)*
|
||
private _parseOrTest(): ExpressionNode {
|
||
let leftExpr = this._parseAndTest();
|
||
if (leftExpr.nodeType === ParseNodeType.Error) {
|
||
return leftExpr;
|
||
}
|
||
|
||
while (true) {
|
||
const peekToken = this._peekToken();
|
||
if (!this._consumeTokenIfKeyword(KeywordType.Or)) {
|
||
break;
|
||
}
|
||
const rightExpr = this._parseAndTest();
|
||
leftExpr = this._createBinaryOperationNode(leftExpr, rightExpr, peekToken, OperatorType.Or);
|
||
}
|
||
|
||
return leftExpr;
|
||
}
|
||
|
||
// and_test: not_test ('and' not_test)*
|
||
private _parseAndTest(): ExpressionNode {
|
||
let leftExpr = this._parseNotTest();
|
||
if (leftExpr.nodeType === ParseNodeType.Error) {
|
||
return leftExpr;
|
||
}
|
||
|
||
while (true) {
|
||
const peekToken = this._peekToken();
|
||
if (!this._consumeTokenIfKeyword(KeywordType.And)) {
|
||
break;
|
||
}
|
||
const rightExpr = this._parseNotTest();
|
||
leftExpr = this._createBinaryOperationNode(leftExpr, rightExpr, peekToken, OperatorType.And);
|
||
}
|
||
|
||
return leftExpr;
|
||
}
|
||
|
||
// not_test: 'not' not_test | comparison
|
||
private _parseNotTest(): ExpressionNode {
|
||
const notToken = this._peekToken();
|
||
if (this._consumeTokenIfKeyword(KeywordType.Not)) {
|
||
const notExpr = this._parseNotTest();
|
||
return this._createUnaryOperationNode(notToken, notExpr, OperatorType.Not);
|
||
}
|
||
|
||
return this._parseComparison();
|
||
}
|
||
|
||
// comparison: expr (comp_op expr)*
|
||
// comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'
|
||
private _parseComparison(): ExpressionNode {
|
||
let leftExpr = this._parseBitwiseOrExpression();
|
||
if (leftExpr.nodeType === ParseNodeType.Error) {
|
||
return leftExpr;
|
||
}
|
||
|
||
while (true) {
|
||
let comparisonOperator: OperatorType | undefined;
|
||
const peekToken = this._peekToken();
|
||
|
||
if (Tokenizer.isOperatorComparison(this._peekOperatorType())) {
|
||
comparisonOperator = this._peekOperatorType();
|
||
if (comparisonOperator === OperatorType.LessOrGreaterThan) {
|
||
this._addSyntaxError(LocMessage.operatorLessOrGreaterDeprecated(), peekToken);
|
||
comparisonOperator = OperatorType.NotEquals;
|
||
}
|
||
this._getNextToken();
|
||
} else if (this._consumeTokenIfKeyword(KeywordType.In)) {
|
||
comparisonOperator = OperatorType.In;
|
||
} else if (this._consumeTokenIfKeyword(KeywordType.Is)) {
|
||
if (this._consumeTokenIfKeyword(KeywordType.Not)) {
|
||
comparisonOperator = OperatorType.IsNot;
|
||
} else {
|
||
comparisonOperator = OperatorType.Is;
|
||
}
|
||
} else if (this._peekKeywordType() === KeywordType.Not) {
|
||
const tokenAfterNot = this._peekToken(1);
|
||
if (
|
||
tokenAfterNot.type === TokenType.Keyword &&
|
||
(tokenAfterNot as KeywordToken).keywordType === KeywordType.In
|
||
) {
|
||
this._getNextToken();
|
||
this._getNextToken();
|
||
comparisonOperator = OperatorType.NotIn;
|
||
}
|
||
}
|
||
|
||
if (comparisonOperator === undefined) {
|
||
break;
|
||
}
|
||
|
||
const rightExpr = this._parseComparison();
|
||
leftExpr = this._createBinaryOperationNode(leftExpr, rightExpr, peekToken, comparisonOperator);
|
||
}
|
||
|
||
return leftExpr;
|
||
}
|
||
|
||
// expr: xor_expr ('|' xor_expr)*
|
||
private _parseBitwiseOrExpression(): ExpressionNode {
|
||
let leftExpr = this._parseBitwiseXorExpression();
|
||
if (leftExpr.nodeType === ParseNodeType.Error) {
|
||
return leftExpr;
|
||
}
|
||
|
||
while (true) {
|
||
const peekToken = this._peekToken();
|
||
if (!this._consumeTokenIfOperator(OperatorType.BitwiseOr)) {
|
||
break;
|
||
}
|
||
const rightExpr = this._parseBitwiseXorExpression();
|
||
leftExpr = this._createBinaryOperationNode(leftExpr, rightExpr, peekToken, OperatorType.BitwiseOr);
|
||
}
|
||
|
||
return leftExpr;
|
||
}
|
||
|
||
// xor_expr: and_expr ('^' and_expr)*
|
||
private _parseBitwiseXorExpression(): ExpressionNode {
|
||
let leftExpr = this._parseBitwiseAndExpression();
|
||
if (leftExpr.nodeType === ParseNodeType.Error) {
|
||
return leftExpr;
|
||
}
|
||
|
||
while (true) {
|
||
const peekToken = this._peekToken();
|
||
if (!this._consumeTokenIfOperator(OperatorType.BitwiseXor)) {
|
||
break;
|
||
}
|
||
const rightExpr = this._parseBitwiseAndExpression();
|
||
leftExpr = this._createBinaryOperationNode(leftExpr, rightExpr, peekToken, OperatorType.BitwiseXor);
|
||
}
|
||
|
||
return leftExpr;
|
||
}
|
||
|
||
// and_expr: shift_expr ('&' shift_expr)*
|
||
private _parseBitwiseAndExpression(): ExpressionNode {
|
||
let leftExpr = this._parseShiftExpression();
|
||
if (leftExpr.nodeType === ParseNodeType.Error) {
|
||
return leftExpr;
|
||
}
|
||
|
||
while (true) {
|
||
const peekToken = this._peekToken();
|
||
if (!this._consumeTokenIfOperator(OperatorType.BitwiseAnd)) {
|
||
break;
|
||
}
|
||
const rightExpr = this._parseShiftExpression();
|
||
leftExpr = this._createBinaryOperationNode(leftExpr, rightExpr, peekToken, OperatorType.BitwiseAnd);
|
||
}
|
||
|
||
return leftExpr;
|
||
}
|
||
|
||
// shift_expr: arith_expr (('<<'|'>>') arith_expr)*
|
||
private _parseShiftExpression(): ExpressionNode {
|
||
let leftExpr = this._parseArithmeticExpression();
|
||
if (leftExpr.nodeType === ParseNodeType.Error) {
|
||
return leftExpr;
|
||
}
|
||
|
||
let peekToken = this._peekToken();
|
||
let nextOperator = this._peekOperatorType();
|
||
while (nextOperator === OperatorType.LeftShift || nextOperator === OperatorType.RightShift) {
|
||
this._getNextToken();
|
||
const rightExpr = this._parseArithmeticExpression();
|
||
leftExpr = this._createBinaryOperationNode(leftExpr, rightExpr, peekToken, nextOperator);
|
||
peekToken = this._peekToken();
|
||
nextOperator = this._peekOperatorType();
|
||
}
|
||
|
||
return leftExpr;
|
||
}
|
||
|
||
// arith_expr: term (('+'|'-') term)*
|
||
private _parseArithmeticExpression(): ExpressionNode {
|
||
let leftExpr = this._parseArithmeticTerm();
|
||
if (leftExpr.nodeType === ParseNodeType.Error) {
|
||
return leftExpr;
|
||
}
|
||
|
||
let peekToken = this._peekToken();
|
||
let nextOperator = this._peekOperatorType();
|
||
while (nextOperator === OperatorType.Add || nextOperator === OperatorType.Subtract) {
|
||
this._getNextToken();
|
||
const rightExpr = this._parseArithmeticTerm();
|
||
if (rightExpr.nodeType === ParseNodeType.Error) {
|
||
return rightExpr;
|
||
}
|
||
|
||
leftExpr = this._createBinaryOperationNode(leftExpr, rightExpr, peekToken, nextOperator);
|
||
peekToken = this._peekToken();
|
||
nextOperator = this._peekOperatorType();
|
||
}
|
||
|
||
return leftExpr;
|
||
}
|
||
|
||
// term: factor (('*'|'@'|'/'|'%'|'//') factor)*
|
||
private _parseArithmeticTerm(): ExpressionNode {
|
||
let leftExpr = this._parseArithmeticFactor();
|
||
if (leftExpr.nodeType === ParseNodeType.Error) {
|
||
return leftExpr;
|
||
}
|
||
|
||
let peekToken = this._peekToken();
|
||
let nextOperator = this._peekOperatorType();
|
||
while (
|
||
nextOperator === OperatorType.Multiply ||
|
||
nextOperator === OperatorType.MatrixMultiply ||
|
||
nextOperator === OperatorType.Divide ||
|
||
nextOperator === OperatorType.Mod ||
|
||
nextOperator === OperatorType.FloorDivide
|
||
) {
|
||
// Viper 扩展：后缀 '*' 是 C 指针类型（T* = CPtr[T]），不是二元乘法。
|
||
// 若该星号串后紧跟"无右操作数"位置（']' / ',' / ')' / ':' /
|
||
// '=' / 换行 / ';' / 文件结束），就地按指针后缀消费。此判定不限于类型注解
|
||
// 上下文：调用实参中的类型名（如 `va_arg(ap, int64*)`）同样适用。
|
||
// 若不在后缀位置：类型注解上下文停止解析，交由
|
||
// _parseTypeAnnotation/_parseTypeAliasStatement 的后缀处理消费；
|
||
// 其他上下文则按真实二元乘法继续解析（`a * b`、`N * 2` 行为不变）。
|
||
if (
|
||
this._parseOptions.dialect === 'viper' &&
|
||
nextOperator === OperatorType.Multiply
|
||
) {
|
||
if (!this._isParsingViperTypeDescriptorInner && this._isAtViperPointerTypeSuffix()) {
|
||
leftExpr = this._parseViperPointerTypeSuffix(leftExpr);
|
||
peekToken = this._peekToken();
|
||
nextOperator = this._peekOperatorType();
|
||
continue;
|
||
}
|
||
if (this._isParsingTypeAnnotation) {
|
||
break;
|
||
}
|
||
}
|
||
|
||
this._getNextToken();
|
||
const rightExpr = this._parseArithmeticFactor();
|
||
leftExpr = this._createBinaryOperationNode(leftExpr, rightExpr, peekToken, nextOperator);
|
||
peekToken = this._peekToken();
|
||
nextOperator = this._peekOperatorType();
|
||
}
|
||
|
||
return leftExpr;
|
||
}
|
||
|
||
// factor: ('+'|'-'|'~'|'*'|'&') factor | power
|
||
// power: atom_expr ['**' factor]
|
||
// Viper 扩展：一元 '*'（解引用）与 '&'（取地址）与 C 语义一致，
|
||
// 仅在 Viper 方言下将前缀 '*'/'&' 解析为一元运算。
|
||
private _parseArithmeticFactor(): ExpressionNode {
|
||
const nextToken = this._peekToken();
|
||
const nextOperator = this._peekOperatorType();
|
||
const isViperUnaryOp =
|
||
this._parseOptions.dialect === 'viper' &&
|
||
(nextOperator === OperatorType.Multiply || nextOperator === OperatorType.BitwiseAnd);
|
||
if (
|
||
nextOperator === OperatorType.Add ||
|
||
nextOperator === OperatorType.Subtract ||
|
||
nextOperator === OperatorType.BitwiseInvert ||
|
||
isViperUnaryOp
|
||
) {
|
||
this._getNextToken();
|
||
const expression = this._parseArithmeticFactor();
|
||
return this._createUnaryOperationNode(nextToken, expression, nextOperator);
|
||
}
|
||
|
||
const leftExpr = this._parseAtomExpression();
|
||
if (leftExpr.nodeType === ParseNodeType.Error) {
|
||
return leftExpr;
|
||
}
|
||
|
||
const peekToken = this._peekToken();
|
||
const nextPowerOperator = this._peekOperatorType();
|
||
// Viper 扩展：'**' 是二级指针类型后缀（T** = CPtr[CPtr[T]]），
|
||
// 词法化为 Power 算子（'**'），不是幂运算。若其后紧跟"无右操作数"位置
|
||
// （如 `list[ast.AST**]`、`x: int**`、调用实参 `va_arg(ap, short**)`），
|
||
// 就地按指针后缀消费。此判定不限于类型注解上下文；
|
||
// 若不在后缀位置：类型注解上下文跳过幂运算分支，交由
|
||
// _parseTypeAnnotation/_parseTypeAliasStatement 的后缀处理消费；
|
||
// 其他上下文则按真实幂运算解析（`a ** b` 行为不变）。
|
||
if (nextPowerOperator === OperatorType.Power && this._parseOptions.dialect === 'viper') {
|
||
if (this._isAtViperPointerTypeSuffix()) {
|
||
return this._parseViperPointerTypeSuffix(leftExpr);
|
||
}
|
||
if (this._isParsingTypeAnnotation) {
|
||
return leftExpr;
|
||
}
|
||
}
|
||
if (nextPowerOperator === OperatorType.Power) {
|
||
this._getNextToken();
|
||
const rightExpr = this._parseArithmeticFactor();
|
||
return this._createBinaryOperationNode(leftExpr, rightExpr, peekToken, OperatorType.Power);
|
||
}
|
||
|
||
return leftExpr;
|
||
}
|
||
|
||
// Determines whether the expression refers to a type exported by the typing
|
||
// or typing_extensions modules. We can directly evaluate the types at binding
|
||
// time. We assume here that the code isn't making use of some custom type alias
|
||
// to refer to the typing types.
|
||
private _isTypingAnnotation(typeAnnotation: ExpressionNode, name: string): boolean {
|
||
if (typeAnnotation.nodeType === ParseNodeType.Name) {
|
||
const alias = this._typingSymbolAliases.get(typeAnnotation.d.value);
|
||
if (alias === name) {
|
||
return true;
|
||
}
|
||
} else if (typeAnnotation.nodeType === ParseNodeType.MemberAccess) {
|
||
if (typeAnnotation.d.leftExpr.nodeType === ParseNodeType.Name && typeAnnotation.d.member.d.value === name) {
|
||
const baseName = typeAnnotation.d.leftExpr.d.value;
|
||
return this._typingImportAliases.some((alias) => alias === baseName);
|
||
}
|
||
}
|
||
|
||
return false;
|
||
}
|
||
|
||
// atom_expr: ['await'] atom trailer*
|
||
// trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
|
||
private _parseAtomExpression(): ExpressionNode {
|
||
let awaitToken: KeywordToken | undefined;
|
||
if (this._peekKeywordType() === KeywordType.Await) {
|
||
awaitToken = this._getKeywordToken(KeywordType.Await);
|
||
if (PythonVersion.isLessThan(this._getLanguageVersion(), pythonVersion3_5)) {
|
||
this._addSyntaxError(LocMessage.awaitIllegal(), awaitToken);
|
||
}
|
||
}
|
||
|
||
// Viper 扩展：值表达式中的 `volatile` 前缀 —— `a: int = volatile f()` /
|
||
// `a: int = volatile 结构体()`。语义：该调用（函数调用或结构体构造的
|
||
// __new__ / __before_init__ / __init__）被标记为 volatile（阻止编译器优化，
|
||
// 加 LLVM volatile）。仅 LSP 侧解析（TPC/TPV 不处理该形态）：
|
||
// 消费 `volatile` 后必须紧跟「调用」形态，否则报语法错误；
|
||
// `volatile = 2` 这类裸用因下一 token 非表达式起点而不进入本分支，仍照旧失败。
|
||
if (this._isAtViperVolatileValuePrefix()) {
|
||
return this._parseViperVolatileValue();
|
||
}
|
||
|
||
let atomExpression = this._parseAtom();
|
||
if (atomExpression.nodeType === ParseNodeType.Error) {
|
||
return atomExpression;
|
||
}
|
||
|
||
// Consume trailers.
|
||
while (true) {
|
||
// Viper 扩展：`T*(x)` / `T**(x)` —— 星号串与基类型、'(' 均**紧贴**时按 C 风格
|
||
// 指针强转调用解析（判据与 TPC ExprParser._AtPtrSuffixCall 同源）。
|
||
// 先把星串消费为「指针类型后缀」，紧接的 '(' 由下一轮 trailer 按调用处理，
|
||
// 产物与既有 `(T*)x` 逐字同形：Call(func = 标记的指针后缀, args = 实参)。
|
||
// 因此 `T*(x)[0] = v` 这类后续下标（乃至赋值目标）自然成立——TPC 同样把
|
||
// `T*(x)` 解析成调用，故 `uint32*(info.vendor_id)[0] = res.ebx` 是合法赋值目标。
|
||
if (this._isAtViperPointerSuffixCastCall(atomExpression)) {
|
||
atomExpression = this._parseViperPointerTypeSuffix(atomExpression);
|
||
continue;
|
||
}
|
||
|
||
// Is it a function call?
|
||
const startOfTrailerToken = this._peekToken();
|
||
if (this._consumeTokenIfType(TokenType.OpenParenthesis)) {
|
||
// Generally, function calls are not allowed within type annotations,
|
||
// but they are permitted in "Annotated" annotations.
|
||
const wasParsingTypeAnnotation = this._isParsingTypeAnnotation;
|
||
this._isParsingTypeAnnotation = false;
|
||
|
||
const argListResult = this._parseArgList();
|
||
const callNode = CallNode.create(atomExpression, argListResult.args, argListResult.trailingComma);
|
||
|
||
if (argListResult.args.length > 1 || argListResult.trailingComma) {
|
||
argListResult.args.forEach((arg) => {
|
||
if (arg.d.valueExpr.nodeType === ParseNodeType.Comprehension) {
|
||
if (!arg.d.valueExpr.d.hasParens) {
|
||
this._addSyntaxError(LocMessage.generatorNotParenthesized(), arg.d.valueExpr);
|
||
}
|
||
}
|
||
});
|
||
}
|
||
|
||
const nextToken = this._peekToken();
|
||
let isArgListTerminated = false;
|
||
if (!this._consumeTokenIfType(TokenType.CloseParenthesis)) {
|
||
this._addSyntaxError(LocMessage.expectedCloseParen(), startOfTrailerToken);
|
||
|
||
// Consume the remainder of tokens on the line for error
|
||
// recovery.
|
||
this._consumeTokensUntilType([TokenType.NewLine]);
|
||
|
||
// Extend the node's range to include the rest of the line.
|
||
// This helps the signatureHelpProvider.
|
||
extendRange(callNode, this._peekToken());
|
||
} else {
|
||
extendRange(callNode, nextToken);
|
||
isArgListTerminated = true;
|
||
}
|
||
|
||
this._isParsingTypeAnnotation = wasParsingTypeAnnotation;
|
||
|
||
const maxDepth = this._maxChildDepthMap.get(atomExpression.id) ?? 0;
|
||
if (maxDepth >= maxChildNodeDepth) {
|
||
atomExpression = ErrorNode.create(callNode, ErrorExpressionCategory.MaxDepthExceeded);
|
||
this._addSyntaxError(LocMessage.maxParseDepthExceeded(), atomExpression);
|
||
} else {
|
||
atomExpression = callNode;
|
||
this._maxChildDepthMap.set(callNode.id, maxDepth + 1);
|
||
}
|
||
|
||
// If the argument list wasn't terminated, break out of the loop
|
||
if (!isArgListTerminated) {
|
||
break;
|
||
}
|
||
} else if (this._consumeTokenIfType(TokenType.OpenBracket)) {
|
||
// Is it an index operator?
|
||
|
||
// This is an unfortunate hack that's necessary to accommodate 'Literal'
|
||
// and 'Annotated' type annotations properly. We need to suspend treating
|
||
// strings as type annotations within a Literal or Annotated subscript.
|
||
const wasParsingIndexTrailer = this._isParsingIndexTrailer;
|
||
const wasParsingTypeAnnotation = this._isParsingTypeAnnotation;
|
||
|
||
if (
|
||
this._isTypingAnnotation(atomExpression, 'Literal') ||
|
||
this._isTypingAnnotation(atomExpression, 'Annotated')
|
||
) {
|
||
this._isParsingTypeAnnotation = false;
|
||
}
|
||
|
||
this._isParsingIndexTrailer = true;
|
||
const subscriptList = this._parseSubscriptList();
|
||
this._isParsingTypeAnnotation = wasParsingTypeAnnotation;
|
||
this._isParsingIndexTrailer = wasParsingIndexTrailer;
|
||
|
||
const closingToken = this._peekToken();
|
||
|
||
const indexNode = IndexNode.create(
|
||
atomExpression,
|
||
subscriptList.list,
|
||
subscriptList.trailingComma,
|
||
closingToken
|
||
);
|
||
extendRange(indexNode, indexNode);
|
||
|
||
if (!this._consumeTokenIfType(TokenType.CloseBracket)) {
|
||
// Handle the error case, but don't use the error node in this
|
||
// case because it creates problems for the completion provider.
|
||
this._handleExpressionParseError(
|
||
ErrorExpressionCategory.MissingIndexCloseBracket,
|
||
LocMessage.expectedCloseBracket(),
|
||
startOfTrailerToken,
|
||
indexNode
|
||
);
|
||
}
|
||
|
||
const maxDepth = this._maxChildDepthMap.get(atomExpression.id) ?? 0;
|
||
if (maxDepth >= maxChildNodeDepth) {
|
||
atomExpression = ErrorNode.create(indexNode, ErrorExpressionCategory.MaxDepthExceeded);
|
||
this._addSyntaxError(LocMessage.maxParseDepthExceeded(), atomExpression);
|
||
} else {
|
||
atomExpression = indexNode;
|
||
this._maxChildDepthMap.set(indexNode.id, maxDepth + 1);
|
||
}
|
||
} else if (this._consumeTokenIfType(TokenType.Dot)) {
|
||
// Is it a member access?
|
||
const memberName = this._getTokenIfIdentifier();
|
||
if (!memberName) {
|
||
return this._handleExpressionParseError(
|
||
ErrorExpressionCategory.MissingMemberAccessName,
|
||
LocMessage.expectedMemberName(),
|
||
startOfTrailerToken,
|
||
atomExpression,
|
||
[TokenType.Keyword]
|
||
);
|
||
}
|
||
|
||
const memberAccessNode = MemberAccessNode.create(atomExpression, NameNode.create(memberName));
|
||
|
||
const maxDepth = this._maxChildDepthMap.get(atomExpression.id) ?? 0;
|
||
if (maxDepth >= maxChildNodeDepth) {
|
||
atomExpression = ErrorNode.create(memberAccessNode, ErrorExpressionCategory.MaxDepthExceeded);
|
||
this._addSyntaxError(LocMessage.maxParseDepthExceeded(), atomExpression);
|
||
} else {
|
||
atomExpression = memberAccessNode;
|
||
this._maxChildDepthMap.set(memberAccessNode.id, maxDepth + 1);
|
||
}
|
||
} else {
|
||
break;
|
||
}
|
||
}
|
||
|
||
if (awaitToken) {
|
||
return AwaitNode.create(awaitToken, atomExpression);
|
||
}
|
||
|
||
return atomExpression;
|
||
}
|
||
|
||
// subscriptlist: subscript (',' subscript)* [',']
|
||
private _parseSubscriptList(): SubscriptListResult {
|
||
const argList: ArgumentNode[] = [];
|
||
let sawKeywordArg = false;
|
||
let trailingComma = false;
|
||
|
||
while (true) {
|
||
const firstToken = this._peekToken();
|
||
|
||
if (firstToken.type !== TokenType.Colon && this._isNextTokenNeverExpression()) {
|
||
break;
|
||
}
|
||
|
||
let argType = ArgCategory.Simple;
|
||
if (this._consumeTokenIfOperator(OperatorType.Multiply)) {
|
||
argType = ArgCategory.UnpackedList;
|
||
} else if (this._consumeTokenIfOperator(OperatorType.Power)) {
|
||
argType = ArgCategory.UnpackedDictionary;
|
||
}
|
||
|
||
const startOfSubscriptIndex = this._tokenIndex;
|
||
// Viper 扩展：下标实参位置的 `[SIZE]T`（如 list[[4]int]）先按数组类型解析。
|
||
let valueExpr =
|
||
argType === ArgCategory.Simple && this._isAtViperSubscriptBracketArrayType()
|
||
? this._parseViperBracketArrayType()
|
||
: this._parsePossibleSlice();
|
||
let nameIdentifier: IdentifierToken | undefined;
|
||
|
||
// Is this a keyword argument?
|
||
if (argType === ArgCategory.Simple) {
|
||
if (this._consumeTokenIfOperator(OperatorType.Assign)) {
|
||
const nameExpr = valueExpr;
|
||
valueExpr = this._parsePossibleSlice();
|
||
|
||
if (nameExpr.nodeType === ParseNodeType.Name) {
|
||
nameIdentifier = nameExpr.d.token;
|
||
} else {
|
||
this._addSyntaxError(LocMessage.expectedParamName(), nameExpr);
|
||
}
|
||
} else if (
|
||
valueExpr.nodeType === ParseNodeType.Name &&
|
||
this._peekOperatorType() === OperatorType.Walrus
|
||
) {
|
||
this._tokenIndex = startOfSubscriptIndex;
|
||
valueExpr = this._parseTestExpression(/* allowAssignmentExpression */ true);
|
||
|
||
// Python 3.10 and newer allow assignment expressions to be used inside of a subscript.
|
||
if (
|
||
!this._parseOptions.isStubFile &&
|
||
PythonVersion.isLessThan(this._getLanguageVersion(), pythonVersion3_10)
|
||
) {
|
||
this._addSyntaxError(LocMessage.assignmentExprInSubscript(), valueExpr);
|
||
}
|
||
}
|
||
}
|
||
|
||
const argNode = ArgumentNode.create(firstToken, valueExpr, argType);
|
||
if (nameIdentifier) {
|
||
argNode.d.name = NameNode.create(nameIdentifier);
|
||
argNode.d.name.parent = argNode;
|
||
}
|
||
|
||
if (argNode.d.name) {
|
||
sawKeywordArg = true;
|
||
} else if (sawKeywordArg && argNode.d.argCategory === ArgCategory.Simple) {
|
||
this._addSyntaxError(LocMessage.positionArgAfterNamedArg(), argNode);
|
||
}
|
||
argList.push(argNode);
|
||
|
||
if (argNode.d.name) {
|
||
this._addSyntaxError(LocMessage.keywordSubscriptIllegal(), argNode.d.name);
|
||
}
|
||
|
||
if (argType !== ArgCategory.Simple) {
|
||
const unpackListAllowed =
|
||
this._parseOptions.isStubFile ||
|
||
this._isParsingQuotedText ||
|
||
PythonVersion.isGreaterOrEqualTo(this._getLanguageVersion(), pythonVersion3_11);
|
||
|
||
if (argType === ArgCategory.UnpackedList && !unpackListAllowed) {
|
||
this._addSyntaxError(LocMessage.unpackedSubscriptIllegal(), argNode);
|
||
}
|
||
|
||
if (argType === ArgCategory.UnpackedDictionary) {
|
||
this._addSyntaxError(LocMessage.unpackedDictSubscriptIllegal(), argNode);
|
||
}
|
||
}
|
||
|
||
if (!this._consumeTokenIfType(TokenType.Comma)) {
|
||
trailingComma = false;
|
||
break;
|
||
}
|
||
|
||
trailingComma = true;
|
||
}
|
||
|
||
// An empty subscript list is illegal.
|
||
if (argList.length === 0) {
|
||
const errorNode = this._handleExpressionParseError(
|
||
ErrorExpressionCategory.MissingIndexOrSlice,
|
||
LocMessage.expectedSliceIndex(),
|
||
/* targetToken */ undefined,
|
||
/* childNode */ undefined,
|
||
[TokenType.CloseBracket]
|
||
);
|
||
argList.push(ArgumentNode.create(this._peekToken(), errorNode, ArgCategory.Simple));
|
||
}
|
||
|
||
return {
|
||
list: argList,
|
||
trailingComma,
|
||
};
|
||
}
|
||
|
||
// subscript: test | [test] ':' [test] [sliceop]
|
||
// sliceop: ':' [test]
|
||
private _parsePossibleSlice(): ExpressionNode {
|
||
const firstToken = this._peekToken();
|
||
const sliceExpressions: (ExpressionNode | undefined)[] = [undefined, undefined, undefined];
|
||
let sliceIndex = 0;
|
||
let sawColon = false;
|
||
|
||
while (true) {
|
||
const nextTokenType = this._peekTokenType();
|
||
if (nextTokenType === TokenType.CloseBracket || nextTokenType === TokenType.Comma) {
|
||
break;
|
||
}
|
||
|
||
if (nextTokenType !== TokenType.Colon) {
|
||
// Python 3.10 and newer allow assignment expressions to be used inside of a subscript.
|
||
const allowAssignmentExpression =
|
||
this._parseOptions.isStubFile ||
|
||
PythonVersion.isGreaterOrEqualTo(this._getLanguageVersion(), pythonVersion3_10);
|
||
sliceExpressions[sliceIndex] = this._parseTestExpression(allowAssignmentExpression);
|
||
}
|
||
sliceIndex++;
|
||
|
||
if (sliceIndex >= 3 || !this._consumeTokenIfType(TokenType.Colon)) {
|
||
break;
|
||
}
|
||
sawColon = true;
|
||
}
|
||
|
||
// If this was a simple expression with no colons return it.
|
||
if (!sawColon) {
|
||
if (sliceExpressions[0]) {
|
||
return sliceExpressions[0];
|
||
}
|
||
|
||
return ErrorNode.create(this._peekToken(), ErrorExpressionCategory.MissingIndexOrSlice);
|
||
}
|
||
|
||
const sliceNode = SliceNode.create(firstToken);
|
||
sliceNode.d.startValue = sliceExpressions[0];
|
||
if (sliceNode.d.startValue) {
|
||
sliceNode.d.startValue.parent = sliceNode;
|
||
}
|
||
sliceNode.d.endValue = sliceExpressions[1];
|
||
if (sliceNode.d.endValue) {
|
||
sliceNode.d.endValue.parent = sliceNode;
|
||
}
|
||
sliceNode.d.stepValue = sliceExpressions[2];
|
||
if (sliceNode.d.stepValue) {
|
||
sliceNode.d.stepValue.parent = sliceNode;
|
||
}
|
||
const extension = sliceExpressions[2] || sliceExpressions[1] || sliceExpressions[0];
|
||
if (extension) {
|
||
extendRange(sliceNode, extension);
|
||
}
|
||
|
||
return sliceNode;
|
||
}
|
||
|
||
// arglist: argument (',' argument)* [',']
|
||
private _parseArgList(): ArgListResult {
|
||
const argList: ArgumentNode[] = [];
|
||
let sawKeywordArg = false;
|
||
let sawUnpackedKeywordArg = false;
|
||
let trailingComma = false;
|
||
|
||
while (true) {
|
||
const nextTokenType = this._peekTokenType();
|
||
if (
|
||
nextTokenType === TokenType.CloseParenthesis ||
|
||
nextTokenType === TokenType.NewLine ||
|
||
nextTokenType === TokenType.EndOfStream
|
||
) {
|
||
break;
|
||
}
|
||
|
||
trailingComma = false;
|
||
const arg = this._parseArgument();
|
||
if (arg.d.name) {
|
||
sawKeywordArg = true;
|
||
} else {
|
||
if (sawKeywordArg && arg.d.argCategory === ArgCategory.Simple) {
|
||
this._addSyntaxError(LocMessage.positionArgAfterNamedArg(), arg);
|
||
}
|
||
|
||
if (sawUnpackedKeywordArg && arg.d.argCategory !== ArgCategory.UnpackedDictionary) {
|
||
this._addSyntaxError(LocMessage.positionArgAfterUnpackedDictArg(), arg);
|
||
}
|
||
}
|
||
if (arg.d.argCategory === ArgCategory.UnpackedDictionary) {
|
||
sawUnpackedKeywordArg = true;
|
||
}
|
||
argList.push(arg);
|
||
|
||
if (!this._consumeTokenIfType(TokenType.Comma)) {
|
||
break;
|
||
}
|
||
|
||
trailingComma = true;
|
||
}
|
||
|
||
return { args: argList, trailingComma };
|
||
}
|
||
|
||
// argument: ( test [comp_for] |
|
||
// test '=' test |
|
||
// '**' test |
|
||
// '*' test )
|
||
private _parseArgument(): ArgumentNode {
|
||
const firstToken = this._peekToken();
|
||
|
||
let argType = ArgCategory.Simple;
|
||
if (this._consumeTokenIfOperator(OperatorType.Multiply)) {
|
||
argType = ArgCategory.UnpackedList;
|
||
} else if (this._consumeTokenIfOperator(OperatorType.Power)) {
|
||
argType = ArgCategory.UnpackedDictionary;
|
||
}
|
||
|
||
let valueExpr = this._parseTestExpression(/* allowAssignmentExpression */ true);
|
||
let nameIdentifier: IdentifierToken | undefined;
|
||
|
||
if (argType === ArgCategory.Simple) {
|
||
if (this._consumeTokenIfOperator(OperatorType.Assign)) {
|
||
const nameExpr = valueExpr;
|
||
valueExpr = this._parseTestExpression(/* allowAssignmentExpression */ false);
|
||
|
||
if (nameExpr.nodeType === ParseNodeType.Name) {
|
||
nameIdentifier = nameExpr.d.token;
|
||
} else {
|
||
this._addSyntaxError(LocMessage.expectedParamName(), nameExpr);
|
||
}
|
||
} else {
|
||
const comprehension = this._tryParseComprehension(valueExpr, /* isGenerator */ true);
|
||
if (comprehension) {
|
||
valueExpr = comprehension;
|
||
}
|
||
}
|
||
}
|
||
|
||
const argNode = ArgumentNode.create(firstToken, valueExpr, argType);
|
||
if (nameIdentifier) {
|
||
argNode.d.name = NameNode.create(nameIdentifier);
|
||
argNode.d.name.parent = argNode;
|
||
}
|
||
|
||
return argNode;
|
||
}
|
||
|
||
// atom: ('(' [yield_expr | testlist_comp] ')' |
|
||
// '[' [testlist_comp] ']' |
|
||
// '{' [dictorsetmaker] '}' |
|
||
// NAME | NUMBER | STRING+ | '...' | 'None' | 'True' | 'False' | '__debug__')
|
||
private _parseAtom(): ExpressionNode {
|
||
const nextToken = this._peekToken();
|
||
|
||
if (nextToken.type === TokenType.Ellipsis) {
|
||
return EllipsisNode.create(this._getNextToken());
|
||
}
|
||
|
||
if (nextToken.type === TokenType.Number) {
|
||
return NumberNode.create(this._getNextToken() as NumberToken);
|
||
}
|
||
|
||
if (nextToken.type === TokenType.Identifier) {
|
||
const identifierToken = this._getNextToken() as IdentifierToken;
|
||
|
||
// Viper 扩展：注解位置的裸 `callable` 等价 `Callable`（t.Callable /
|
||
// typing.Callable），支持 `callable[[A, B], R]`、`callable[..., R]`、
|
||
// `callable[[], R]`。**仅类型注解位置**脱糖：值位置的 `callable(x)`
|
||
// 仍是内置函数（isinstance 风格的 TypeIs 判定）。
|
||
// `Callable` 声明在 builtins stub 中（t.CTypedef = t.Callable），
|
||
// 故不要求源码显式 `import t`。
|
||
if (
|
||
this._parseOptions.dialect === 'viper' &&
|
||
this._isParsingTypeAnnotation &&
|
||
identifierToken.value === 'callable'
|
||
) {
|
||
return NameNode.create(
|
||
IdentifierToken.create(
|
||
identifierToken.start,
|
||
identifierToken.length,
|
||
'Callable',
|
||
identifierToken.comments
|
||
)
|
||
);
|
||
}
|
||
|
||
return NameNode.create(identifierToken);
|
||
}
|
||
|
||
if (nextToken.type === TokenType.String || nextToken.type === TokenType.FStringStart) {
|
||
return this._parseStringList();
|
||
}
|
||
|
||
if (nextToken.type === TokenType.Backtick) {
|
||
this._getNextToken();
|
||
|
||
// Atoms with backticks are no longer allowed in Python 3.x, but they
|
||
// were a thing in Python 2.x. We'll parse them to improve parse recovery
|
||
// and emit an error.
|
||
this._addSyntaxError(LocMessage.backticksIllegal(), nextToken);
|
||
|
||
const expressionNode = this._parseTestListAsExpression(ErrorExpressionCategory.MissingExpression, () =>
|
||
LocMessage.expectedExpr()
|
||
);
|
||
|
||
this._consumeTokenIfType(TokenType.Backtick);
|
||
return expressionNode;
|
||
}
|
||
|
||
if (nextToken.type === TokenType.OpenParenthesis) {
|
||
const possibleTupleNode = this._parseTupleAtom();
|
||
|
||
if (
|
||
possibleTupleNode.nodeType === ParseNodeType.UnaryOperation ||
|
||
possibleTupleNode.nodeType === ParseNodeType.Await ||
|
||
possibleTupleNode.nodeType === ParseNodeType.BinaryOperation
|
||
) {
|
||
// Mark binary expressions as parenthesized so we don't attempt
|
||
// to use comparison chaining, which isn't appropriate when the
|
||
// expression is parenthesized. Unary and await expressions
|
||
// are also marked to be able to display them unambiguously.
|
||
possibleTupleNode.d.hasParens = true;
|
||
}
|
||
|
||
if (
|
||
possibleTupleNode.nodeType === ParseNodeType.StringList ||
|
||
possibleTupleNode.nodeType === ParseNodeType.Comprehension ||
|
||
possibleTupleNode.nodeType === ParseNodeType.AssignmentExpression
|
||
) {
|
||
possibleTupleNode.d.hasParens = true;
|
||
}
|
||
|
||
// Viper 扩展：`(T)x` / `(T*)x` —— 括号内是类型原子（Name / MemberAccess /
|
||
// Subscript / 指针类型后缀 T*），其后紧跟操作数时按 C 强转解析为调用，
|
||
// 语义等价于已支持的 `T(x)` / `(T)(x)` 通路。
|
||
// 仅当括号内为类型原子形态才生效，故 `(a)*b`、`(a) * (b)` 等乘法、
|
||
// `(*p)x`（前缀解引用）完全不受影响；`(T*)[i]` 因 '[' 非操作数起点仍为下标。
|
||
const viperCastNode = this._tryParseViperPointerCast(possibleTupleNode);
|
||
if (viperCastNode) {
|
||
return viperCastNode;
|
||
}
|
||
|
||
return possibleTupleNode;
|
||
} else if (nextToken.type === TokenType.OpenBracket) {
|
||
return this._parseListAtom();
|
||
} else if (nextToken.type === TokenType.OpenCurlyBrace) {
|
||
return this._parseDictionaryOrSetAtom();
|
||
}
|
||
|
||
if (nextToken.type === TokenType.Keyword) {
|
||
const keywordToken = nextToken as KeywordToken;
|
||
if (
|
||
keywordToken.keywordType === KeywordType.False ||
|
||
keywordToken.keywordType === KeywordType.True ||
|
||
keywordToken.keywordType === KeywordType.Debug ||
|
||
keywordToken.keywordType === KeywordType.None
|
||
) {
|
||
return ConstantNode.create(this._getNextToken() as KeywordToken);
|
||
}
|
||
|
||
// Make an identifier out of the keyword.
|
||
const keywordAsIdentifier = this._getTokenIfIdentifier();
|
||
if (keywordAsIdentifier) {
|
||
return NameNode.create(keywordAsIdentifier);
|
||
}
|
||
}
|
||
|
||
return this._handleExpressionParseError(ErrorExpressionCategory.MissingExpression, LocMessage.expectedExpr());
|
||
}
|
||
|
||
// Allocates a dummy "error expression" and consumes the remainder
|
||
// of the tokens on the line for error recovery. A partially-completed
|
||
// child node can be passed to help the completion provider determine
|
||
// what to do.
|
||
private _handleExpressionParseError(
|
||
category: ErrorExpressionCategory,
|
||
errorMsg: string,
|
||
targetToken?: Token,
|
||
childNode?: ExpressionNode,
|
||
additionalStopTokens?: TokenType[]
|
||
): ErrorNode {
|
||
this._addSyntaxError(errorMsg, targetToken ?? this._peekToken());
|
||
|
||
const stopTokens = [TokenType.NewLine];
|
||
if (additionalStopTokens) {
|
||
appendArray(stopTokens, additionalStopTokens);
|
||
}
|
||
|
||
// Using a token that is not included in the error node creates problems.
|
||
// Sibling nodes in parse tree shouldn't overlap each other.
|
||
const nextToken = this._peekToken();
|
||
const initialRange: TextRange = stopTokens.some((k) => nextToken.type === k)
|
||
? targetToken ?? childNode ?? TextRange.create(nextToken.start, /* length */ 0)
|
||
: nextToken;
|
||
const expr = ErrorNode.create(initialRange, category, childNode);
|
||
this._consumeTokensUntilType(stopTokens);
|
||
|
||
return expr;
|
||
}
|
||
|
||
// lambdef: 'lambda' [varargslist] ':' test
|
||
private _parseLambdaExpression(allowConditional = true): LambdaNode {
|
||
const lambdaToken = this._getKeywordToken(KeywordType.Lambda);
|
||
|
||
const argList = this._parseVarArgsList(TokenType.Colon, /* allowAnnotations */ false);
|
||
|
||
if (!this._consumeTokenIfType(TokenType.Colon)) {
|
||
this._addSyntaxError(LocMessage.expectedColon(), this._peekToken());
|
||
}
|
||
|
||
let testExpr: ExpressionNode;
|
||
if (allowConditional) {
|
||
testExpr = this._parseTestExpression(/* allowAssignmentExpression */ false);
|
||
} else {
|
||
testExpr = this._tryParseLambdaExpression(/* allowConditional */ false) || this._parseOrTest();
|
||
}
|
||
|
||
const lambdaNode = LambdaNode.create(lambdaToken, testExpr);
|
||
lambdaNode.d.params = argList;
|
||
argList.forEach((arg) => {
|
||
arg.parent = lambdaNode;
|
||
});
|
||
return lambdaNode;
|
||
}
|
||
|
||
private _tryParseLambdaExpression(allowConditional = true): LambdaNode | undefined {
|
||
if (this._peekKeywordType() !== KeywordType.Lambda) {
|
||
return undefined;
|
||
}
|
||
|
||
return this._parseLambdaExpression(allowConditional);
|
||
}
|
||
|
||
// ('(' [yield_expr | testlist_comp] ')'
|
||
// testlist_comp: (test | star_expr) (comp_for | (',' (test | star_expr))* [','])
|
||
private _parseTupleAtom(): ExpressionNode {
|
||
const startParen = this._getNextToken();
|
||
assert(startParen.type === TokenType.OpenParenthesis);
|
||
|
||
const yieldExpr = this._tryParseYieldExpression();
|
||
if (yieldExpr) {
|
||
if (this._peekTokenType() !== TokenType.CloseParenthesis) {
|
||
return this._handleExpressionParseError(
|
||
ErrorExpressionCategory.MissingTupleCloseParen,
|
||
LocMessage.expectedCloseParen(),
|
||
startParen,
|
||
yieldExpr
|
||
);
|
||
} else {
|
||
extendRange(yieldExpr, this._getNextToken());
|
||
}
|
||
|
||
return yieldExpr;
|
||
}
|
||
|
||
const exprListResult = this._parseTestListWithComprehension(/* isGenerator */ true);
|
||
const tupleOrExpression = this._makeExpressionOrTuple(exprListResult, /* enclosedInParens */ true);
|
||
|
||
extendRange(tupleOrExpression, startParen);
|
||
|
||
if (this._peekTokenType() !== TokenType.CloseParenthesis) {
|
||
return this._handleExpressionParseError(
|
||
ErrorExpressionCategory.MissingTupleCloseParen,
|
||
LocMessage.expectedCloseParen(),
|
||
startParen,
|
||
exprListResult.parseError ?? tupleOrExpression
|
||
);
|
||
} else {
|
||
extendRange(tupleOrExpression, this._getNextToken());
|
||
}
|
||
|
||
return tupleOrExpression;
|
||
}
|
||
|
||
// '[' [testlist_comp] ']'
|
||
// testlist_comp: (test | star_expr) (comp_for | (',' (test | star_expr))* [','])
|
||
private _parseListAtom(): ListNode | ErrorNode {
|
||
const startBracket = this._getNextToken();
|
||
assert(startBracket.type === TokenType.OpenBracket);
|
||
|
||
const exprListResult = this._parseTestListWithComprehension(/* isGenerator */ false);
|
||
const closeBracket: Token | undefined = this._peekToken();
|
||
if (!this._consumeTokenIfType(TokenType.CloseBracket)) {
|
||
return this._handleExpressionParseError(
|
||
ErrorExpressionCategory.MissingListCloseBracket,
|
||
LocMessage.expectedCloseBracket(),
|
||
startBracket,
|
||
exprListResult.parseError ?? _createList()
|
||
);
|
||
}
|
||
|
||
return _createList();
|
||
|
||
function _createList() {
|
||
const listAtom = ListNode.create(startBracket);
|
||
|
||
if (closeBracket) {
|
||
extendRange(listAtom, closeBracket);
|
||
}
|
||
|
||
if (exprListResult.list.length > 0) {
|
||
exprListResult.list.forEach((expr) => {
|
||
expr.parent = listAtom;
|
||
});
|
||
extendRange(listAtom, exprListResult.list[exprListResult.list.length - 1]);
|
||
}
|
||
|
||
listAtom.d.items = exprListResult.list;
|
||
return listAtom;
|
||
}
|
||
}
|
||
|
||
private _parseTestListWithComprehension(isGenerator: boolean): ListResult<ExpressionNode> {
|
||
let sawComprehension = false;
|
||
|
||
return this._parseExpressionListGeneric(
|
||
() => {
|
||
let expr = this._parseTestOrStarExpression(/* allowAssignmentExpression */ true);
|
||
const comprehension = this._tryParseComprehension(expr, isGenerator);
|
||
if (comprehension) {
|
||
expr = comprehension;
|
||
sawComprehension = true;
|
||
}
|
||
return expr;
|
||
},
|
||
() => this._isNextTokenNeverExpression(),
|
||
() => sawComprehension
|
||
);
|
||
}
|
||
|
||
// '{' [dictorsetmaker] '}'
|
||
// dictorsetmaker: (
|
||
// (dictentry (comp_for | (',' dictentry)* [',']))
|
||
// | (setentry (comp_for | (',' setentry)* [',']))
|
||
// )
|
||
// dictentry: (test ':' test | '**' expr)
|
||
// setentry: test | star_expr
|
||
private _parseDictionaryOrSetAtom(): DictionaryNode | SetNode {
|
||
const startBrace = this._getNextToken();
|
||
assert(startBrace.type === TokenType.OpenCurlyBrace);
|
||
|
||
const dictionaryEntries: DictionaryEntryNode[] = [];
|
||
const setEntries: ExpressionNode[] = [];
|
||
let isDictionary = false;
|
||
let isSet = false;
|
||
let sawComprehension = false;
|
||
let isFirstEntry = true;
|
||
let trailingCommaToken: Token | undefined;
|
||
|
||
while (true) {
|
||
if (this._peekTokenType() === TokenType.CloseCurlyBrace) {
|
||
break;
|
||
}
|
||
|
||
trailingCommaToken = undefined;
|
||
|
||
let doubleStarExpression: ExpressionNode | undefined;
|
||
let keyExpression: ExpressionNode | undefined;
|
||
let valueExpression: ExpressionNode | undefined;
|
||
const doubleStar = this._peekToken();
|
||
|
||
if (this._consumeTokenIfOperator(OperatorType.Power)) {
|
||
doubleStarExpression = this._parseExpression(/* allowUnpack */ false);
|
||
} else {
|
||
// A dictionary key is never a forward-reference type annotation, even
|
||
// when the dictionary appears within a type annotation (e.g. the field
|
||
// names of an inline TypedDict such as `TypedDict[{'x': int}]`). Suspend
|
||
// type-annotation parsing for the key so its string isn't parsed into a
|
||
// forward-reference expression. The value remains a type annotation and
|
||
// must continue to allow forward references.
|
||
const wasParsingTypeAnnotation = this._isParsingTypeAnnotation;
|
||
this._isParsingTypeAnnotation = false;
|
||
keyExpression = this._parseTestOrStarExpression(/* allowAssignmentExpression */ true);
|
||
this._isParsingTypeAnnotation = wasParsingTypeAnnotation;
|
||
|
||
// Allow walrus operators in this context only for Python 3.10 and newer.
|
||
// Older versions of Python generated a syntax error in this context.
|
||
let isWalrusAllowed = PythonVersion.isGreaterOrEqualTo(this._getLanguageVersion(), pythonVersion3_10);
|
||
|
||
if (this._consumeTokenIfType(TokenType.Colon)) {
|
||
valueExpression = this._parseTestExpression(/* allowAssignmentExpression */ false);
|
||
isWalrusAllowed = false;
|
||
}
|
||
|
||
if (
|
||
!isWalrusAllowed &&
|
||
keyExpression.nodeType === ParseNodeType.AssignmentExpression &&
|
||
!keyExpression.d.hasParens
|
||
) {
|
||
this._addSyntaxError(LocMessage.walrusNotAllowed(), keyExpression.d.walrusToken);
|
||
}
|
||
}
|
||
|
||
if (keyExpression && valueExpression) {
|
||
if (keyExpression.nodeType === ParseNodeType.Unpack) {
|
||
this._addSyntaxError(LocMessage.unpackInDict(), keyExpression);
|
||
}
|
||
|
||
if (isSet) {
|
||
this._addSyntaxError(LocMessage.keyValueInSet(), valueExpression);
|
||
} else {
|
||
const keyEntryNode = DictionaryKeyEntryNode.create(keyExpression, valueExpression);
|
||
let dictEntry: DictionaryEntryNode = keyEntryNode;
|
||
const comprehension = this._tryParseComprehension(keyEntryNode, /* isGenerator */ false);
|
||
if (comprehension) {
|
||
dictEntry = comprehension;
|
||
sawComprehension = true;
|
||
|
||
if (!isFirstEntry) {
|
||
this._addSyntaxError(LocMessage.comprehensionInDict(), dictEntry);
|
||
}
|
||
}
|
||
dictionaryEntries.push(dictEntry);
|
||
isDictionary = true;
|
||
}
|
||
} else if (doubleStarExpression) {
|
||
if (isSet) {
|
||
this._addSyntaxError(LocMessage.unpackInSet(), doubleStarExpression);
|
||
} else {
|
||
const listEntryNode = DictionaryExpandEntryNode.create(doubleStarExpression);
|
||
extendRange(listEntryNode, doubleStar);
|
||
let expandEntryNode: DictionaryEntryNode = listEntryNode;
|
||
const comprehension = this._tryParseComprehension(listEntryNode, /* isGenerator */ false);
|
||
if (comprehension) {
|
||
expandEntryNode = comprehension;
|
||
sawComprehension = true;
|
||
|
||
if (!isFirstEntry) {
|
||
this._addSyntaxError(LocMessage.comprehensionInDict(), doubleStarExpression);
|
||
}
|
||
}
|
||
dictionaryEntries.push(expandEntryNode);
|
||
isDictionary = true;
|
||
}
|
||
} else {
|
||
assert(keyExpression !== undefined);
|
||
if (keyExpression) {
|
||
if (isDictionary) {
|
||
const missingValueErrorNode = ErrorNode.create(
|
||
this._peekToken(),
|
||
ErrorExpressionCategory.MissingDictValue
|
||
);
|
||
const keyEntryNode = DictionaryKeyEntryNode.create(keyExpression, missingValueErrorNode);
|
||
dictionaryEntries.push(keyEntryNode);
|
||
this._addSyntaxError(LocMessage.dictKeyValuePairs(), keyExpression);
|
||
} else {
|
||
const comprehension = this._tryParseComprehension(keyExpression, /* isGenerator */ false);
|
||
if (comprehension) {
|
||
keyExpression = comprehension;
|
||
sawComprehension = true;
|
||
|
||
if (!isFirstEntry) {
|
||
this._addSyntaxError(LocMessage.comprehensionInSet(), keyExpression);
|
||
}
|
||
}
|
||
setEntries.push(keyExpression);
|
||
isSet = true;
|
||
}
|
||
}
|
||
}
|
||
|
||
// List comprehension statements always end the list.
|
||
if (sawComprehension) {
|
||
break;
|
||
}
|
||
|
||
if (this._peekTokenType() !== TokenType.Comma) {
|
||
break;
|
||
}
|
||
|
||
trailingCommaToken = this._getNextToken();
|
||
|
||
isFirstEntry = false;
|
||
}
|
||
|
||
let closeCurlyBrace: Token | undefined = this._peekToken();
|
||
if (!this._consumeTokenIfType(TokenType.CloseCurlyBrace)) {
|
||
this._addSyntaxError(LocMessage.expectedCloseBrace(), startBrace);
|
||
closeCurlyBrace = undefined;
|
||
}
|
||
|
||
if (isSet) {
|
||
const setAtom = SetNode.create(startBrace);
|
||
if (closeCurlyBrace) {
|
||
extendRange(setAtom, closeCurlyBrace);
|
||
}
|
||
|
||
if (setEntries.length > 0) {
|
||
extendRange(setAtom, setEntries[setEntries.length - 1]);
|
||
}
|
||
|
||
setEntries.forEach((entry) => {
|
||
entry.parent = setAtom;
|
||
});
|
||
|
||
setAtom.d.items = setEntries;
|
||
return setAtom;
|
||
}
|
||
|
||
const dictionaryAtom = DictionaryNode.create(startBrace);
|
||
|
||
if (trailingCommaToken) {
|
||
dictionaryAtom.d.trailingCommaToken = trailingCommaToken;
|
||
extendRange(dictionaryAtom, trailingCommaToken);
|
||
}
|
||
|
||
if (closeCurlyBrace) {
|
||
extendRange(dictionaryAtom, closeCurlyBrace);
|
||
}
|
||
|
||
if (dictionaryEntries.length > 0) {
|
||
dictionaryEntries.forEach((entry) => {
|
||
entry.parent = dictionaryAtom;
|
||
});
|
||
extendRange(dictionaryAtom, dictionaryEntries[dictionaryEntries.length - 1]);
|
||
}
|
||
dictionaryAtom.d.items = dictionaryEntries;
|
||
return dictionaryAtom;
|
||
}
|
||
|
||
private _parseExpressionListGeneric<T extends ParseNode = ExpressionNode>(
|
||
parser: () => T | ErrorNode,
|
||
terminalCheck: () => boolean = () => this._isNextTokenNeverExpression(),
|
||
finalEntryCheck: () => boolean = () => false
|
||
): ListResult<T> {
|
||
let trailingComma = false;
|
||
const list: T[] = [];
|
||
let parseError: ErrorNode | undefined;
|
||
|
||
while (true) {
|
||
if (terminalCheck()) {
|
||
break;
|
||
}
|
||
|
||
const expr = parser();
|
||
if (expr.nodeType === ParseNodeType.Error) {
|
||
parseError = expr as ErrorNode;
|
||
break;
|
||
}
|
||
list.push(expr);
|
||
|
||
// Should we stop without checking for a trailing comma?
|
||
if (finalEntryCheck()) {
|
||
break;
|
||
}
|
||
|
||
if (!this._consumeTokenIfType(TokenType.Comma)) {
|
||
trailingComma = false;
|
||
break;
|
||
}
|
||
|
||
trailingComma = true;
|
||
}
|
||
|
||
return { trailingComma, list, parseError };
|
||
}
|
||
|
||
// expr_stmt: testlist_star_expr (annassign | augassign (yield_expr | testlist) |
|
||
// ('=' (yield_expr | testlist_star_expr))*)
|
||
// testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [',']
|
||
// annassign: ':' test ['=' (yield_expr | testlist_star_expr)]
|
||
// augassign: ('+=' | '-=' | '*=' | '@=' | '/=' | '%=' | '&=' | '|=' | '^=' |
|
||
// '<<=' | '>>=' | '**=' | '//=')
|
||
private _parseExpressionStatement(): ExpressionNode {
|
||
let leftExpr = this._parseTestOrStarListAsExpression(
|
||
/* allowAssignmentExpression */ false,
|
||
/* allowMultipleUnpack */ false,
|
||
ErrorExpressionCategory.MissingExpression,
|
||
() => LocMessage.expectedExpr()
|
||
);
|
||
let annotationExpr: ExpressionNode | undefined;
|
||
|
||
if (leftExpr.nodeType === ParseNodeType.Error) {
|
||
return leftExpr;
|
||
}
|
||
|
||
// Is this a type annotation assignment?
|
||
if (this._consumeTokenIfType(TokenType.Colon)) {
|
||
// 保留原始赋值目标（NameNode），后续 `*=` 拆分重包装时避免嵌套 TypeAnnotationNode。
|
||
const targetExpr = leftExpr;
|
||
annotationExpr = this._parseTypeAnnotation();
|
||
leftExpr = TypeAnnotationNode.create(targetExpr, annotationExpr);
|
||
|
||
if (
|
||
!this._parseOptions.isStubFile &&
|
||
PythonVersion.isLessThan(this._getLanguageVersion(), pythonVersion3_6)
|
||
) {
|
||
this._addSyntaxError(LocMessage.varAnnotationIllegal(), annotationExpr);
|
||
}
|
||
|
||
// Viper 扩展：`px: T* = &x` 无空格时，词法将 `*` 与 `=` 合并为 `*=`（同理 `**=`）。
|
||
// 此时 `*`/`**` 是指针类型后缀、`=` 是赋值号，需拆开处理：
|
||
// 消费 `*=` 作为一层指针（`**=` 两层），并视为已消费赋值号 `=`。
|
||
let isViperStarEqualAssign = false;
|
||
if (this._parseOptions.dialect === 'viper') {
|
||
const starAssignOperator = this._peekOperatorType();
|
||
if (
|
||
starAssignOperator === OperatorType.MultiplyEqual ||
|
||
starAssignOperator === OperatorType.PowerEqual
|
||
) {
|
||
const starToken = this._getNextToken();
|
||
annotationExpr = this._createUnaryOperationNode(starToken, annotationExpr, OperatorType.Multiply);
|
||
if (starAssignOperator === OperatorType.PowerEqual) {
|
||
annotationExpr = this._createUnaryOperationNode(
|
||
starToken,
|
||
annotationExpr,
|
||
OperatorType.Multiply
|
||
);
|
||
}
|
||
leftExpr = TypeAnnotationNode.create(targetExpr, annotationExpr);
|
||
isViperStarEqualAssign = true;
|
||
}
|
||
}
|
||
|
||
if (!isViperStarEqualAssign && !this._consumeTokenIfOperator(OperatorType.Assign)) {
|
||
return leftExpr;
|
||
}
|
||
|
||
// This is an unfortunate hack that's necessary to accommodate 'TypeAlias'
|
||
// declarations properly. We need to treat this assignment differently than
|
||
// most because the expression on the right side is treated like a type
|
||
// annotation and therefore allows string-literal forward declarations.
|
||
const isTypeAliasDeclaration = this._isTypingAnnotation(annotationExpr, 'TypeAlias');
|
||
|
||
const wasParsingTypeAnnotation = this._isParsingTypeAnnotation;
|
||
if (isTypeAliasDeclaration) {
|
||
this._isParsingTypeAnnotation = true;
|
||
}
|
||
|
||
const rightExpr =
|
||
this._tryParseYieldExpression() ??
|
||
this._parseTestOrStarListAsExpression(
|
||
/* allowAssignmentExpression */ false,
|
||
/* allowMultipleUnpack */ true,
|
||
ErrorExpressionCategory.MissingExpression,
|
||
() => LocMessage.expectedAssignRightHandExpr()
|
||
);
|
||
|
||
this._isParsingTypeAnnotation = wasParsingTypeAnnotation;
|
||
|
||
return AssignmentNode.create(leftExpr, rightExpr);
|
||
}
|
||
|
||
// Is this a simple assignment?
|
||
if (this._consumeTokenIfOperator(OperatorType.Assign)) {
|
||
return this._parseChainAssignments(leftExpr);
|
||
}
|
||
|
||
if (Tokenizer.isOperatorAssignment(this._peekOperatorType())) {
|
||
const operatorToken = this._getNextToken() as OperatorToken;
|
||
|
||
const rightExpr =
|
||
this._tryParseYieldExpression() ??
|
||
this._parseTestOrStarListAsExpression(
|
||
/* allowAssignmentExpression */ false,
|
||
/* allowMultipleUnpack */ true,
|
||
ErrorExpressionCategory.MissingExpression,
|
||
() => LocMessage.expectedBinaryRightHandExpr()
|
||
);
|
||
this._reportConditionalErrorForStarTupleElement(rightExpr, pythonVersion3_9);
|
||
|
||
// Make a shallow copy of the dest expression but give it a new ID.
|
||
const destExpr = Object.assign({}, leftExpr);
|
||
destExpr.id = getNextNodeId();
|
||
|
||
return AugmentedAssignmentNode.create(leftExpr, rightExpr, operatorToken.operatorType, destExpr);
|
||
}
|
||
|
||
return leftExpr;
|
||
}
|
||
|
||
private _parseChainAssignments(leftExpr: ExpressionNode): ExpressionNode {
|
||
// Make a list of assignment targets.
|
||
const assignmentTargets = [leftExpr];
|
||
let rightExpr: ExpressionNode;
|
||
|
||
while (true) {
|
||
rightExpr =
|
||
this._tryParseYieldExpression() ??
|
||
this._parseTestOrStarListAsExpression(
|
||
/* allowAssignmentExpression */ false,
|
||
/* allowMultipleUnpack */ true,
|
||
ErrorExpressionCategory.MissingExpression,
|
||
() => LocMessage.expectedAssignRightHandExpr()
|
||
);
|
||
|
||
if (rightExpr.nodeType === ParseNodeType.Error) {
|
||
break;
|
||
}
|
||
|
||
// Continue until we've consumed the entire chain.
|
||
if (!this._consumeTokenIfOperator(OperatorType.Assign)) {
|
||
break;
|
||
}
|
||
|
||
assignmentTargets.push(rightExpr);
|
||
}
|
||
|
||
// Create a tree of assignment expressions starting with the first one.
|
||
// The final RHS value is assigned to the targets left to right in Python.
|
||
let assignmentNode = AssignmentNode.create(assignmentTargets[0], rightExpr);
|
||
|
||
// Look for a type annotation comment at the end of the line.
|
||
const typeAnnotationComment = this._parseVariableTypeAnnotationComment();
|
||
if (typeAnnotationComment) {
|
||
if (assignmentTargets.length > 1) {
|
||
// Type comments are not allowed for chained assignments for the
|
||
// same reason that variable type annotations don't support
|
||
// chained assignments. Note that a type comment was used here
|
||
// so it can be later reported as an error by the binder.
|
||
assignmentNode.d.chainedAnnotationComment = typeAnnotationComment;
|
||
} else {
|
||
assignmentNode.d.annotationComment = typeAnnotationComment;
|
||
assignmentNode.d.annotationComment.parent = assignmentNode;
|
||
extendRange(assignmentNode, assignmentNode.d.annotationComment);
|
||
}
|
||
}
|
||
|
||
assignmentTargets.forEach((target, index) => {
|
||
if (index > 0) {
|
||
assignmentNode = AssignmentNode.create(target, assignmentNode);
|
||
}
|
||
});
|
||
|
||
return assignmentNode;
|
||
}
|
||
|
||
private _parseFunctionTypeAnnotation(): FunctionAnnotationNode | undefined {
|
||
const openParenToken = this._peekToken();
|
||
if (!this._consumeTokenIfType(TokenType.OpenParenthesis)) {
|
||
this._addSyntaxError(LocMessage.expectedOpenParen(), this._peekToken());
|
||
return undefined;
|
||
}
|
||
|
||
let paramAnnotations: ExpressionNode[] = [];
|
||
|
||
while (true) {
|
||
const nextTokenType = this._peekTokenType();
|
||
if (
|
||
nextTokenType === TokenType.CloseParenthesis ||
|
||
nextTokenType === TokenType.NewLine ||
|
||
nextTokenType === TokenType.EndOfStream
|
||
) {
|
||
break;
|
||
}
|
||
|
||
// Consume "*" or "**" indicators but don't do anything with them.
|
||
// (We don't enforce that these are present, absent, or match
|
||
// the corresponding parameter types.)
|
||
this._consumeTokenIfOperator(OperatorType.Multiply) || this._consumeTokenIfOperator(OperatorType.Power);
|
||
|
||
const paramAnnotation = this._parseTypeAnnotation();
|
||
paramAnnotations.push(paramAnnotation);
|
||
|
||
if (!this._consumeTokenIfType(TokenType.Comma)) {
|
||
break;
|
||
}
|
||
}
|
||
|
||
if (!this._consumeTokenIfType(TokenType.CloseParenthesis)) {
|
||
this._addSyntaxError(LocMessage.expectedCloseParen(), openParenToken);
|
||
this._consumeTokensUntilType([TokenType.Colon]);
|
||
}
|
||
|
||
if (!this._consumeTokenIfType(TokenType.Arrow)) {
|
||
this._addSyntaxError(LocMessage.expectedArrow(), this._peekToken());
|
||
return undefined;
|
||
}
|
||
|
||
const returnType = this._parseTypeAnnotation();
|
||
|
||
let isParamListEllipsis = false;
|
||
if (paramAnnotations.length === 1 && paramAnnotations[0].nodeType === ParseNodeType.Ellipsis) {
|
||
paramAnnotations = [];
|
||
isParamListEllipsis = true;
|
||
}
|
||
|
||
return FunctionAnnotationNode.create(openParenToken, isParamListEllipsis, paramAnnotations, returnType);
|
||
}
|
||
|
||
private _parseTypeAnnotation(allowUnpack = false): ExpressionNode {
|
||
// Temporary set a flag that indicates we're parsing a type annotation.
|
||
const wasParsingTypeAnnotation = this._isParsingTypeAnnotation;
|
||
this._isParsingTypeAnnotation = true;
|
||
|
||
// Allow unpack operators.
|
||
const startToken = this._peekToken();
|
||
const isUnpack = this._consumeTokenIfOperator(OperatorType.Multiply);
|
||
|
||
if (
|
||
isUnpack &&
|
||
allowUnpack &&
|
||
!this._parseOptions.isStubFile &&
|
||
!this._isParsingQuotedText &&
|
||
PythonVersion.isLessThan(this._getLanguageVersion(), pythonVersion3_11)
|
||
) {
|
||
this._addSyntaxError(LocMessage.unpackedSubscriptIllegal(), startToken);
|
||
}
|
||
|
||
// Viper 扩展：注解主体（类型描述符前缀 / 定长数组 / 普通类型表达式）。
|
||
let result = this._parseViperTypeAnnotationCore();
|
||
|
||
// Viper 扩展：C 指针类型后缀 T* / T** / T***（须在还原 _isParsingTypeAnnotation 之前，
|
||
// 使 _parseArithmeticTerm/_parseArithmeticFactor 的星号守卫仍生效）。
|
||
// 后缀统一在最外层应用，故 `const int*` 得到 `(CConst | int)*`
|
||
// （指针指向被描述符修饰的类型），与 `int*` 中 `*` 绑定到类型的位置一致。
|
||
result = this._parseViperPointerTypeSuffix(result);
|
||
|
||
if (isUnpack) {
|
||
result = UnpackNode.create(startToken, result);
|
||
}
|
||
|
||
this._isParsingTypeAnnotation = wasParsingTypeAnnotation;
|
||
this._hasTypeAnnotations = true;
|
||
|
||
return result;
|
||
}
|
||
|
||
// Viper 扩展：注解主体 = 类型描述符前缀（const / volatile）+ 类型表达式。
|
||
// 与 _parseTypeAnnotation 分离，使 C 指针后缀 `*` 由调用方在最外层统一应用：
|
||
// `const int*` 的脱糖结果 `CConst | int` 先成形，再整体加一层指针后缀。
|
||
private _parseViperTypeAnnotationCore(): ExpressionNode {
|
||
const attributeNode = this._parseViperAttributePrefix();
|
||
if (attributeNode) {
|
||
return attributeNode;
|
||
}
|
||
const descriptorNode = this._parseViperTypeDescriptorPrefix();
|
||
if (descriptorNode) {
|
||
return descriptorNode;
|
||
}
|
||
return this._parseViperBracketArrayTypeIfNeeded();
|
||
}
|
||
|
||
// Viper 扩展：注解位置的 `__attribute__(...)` 修饰器前缀。
|
||
// 解析层脱糖为等价的联合类型注解，与手写 `T | t.attr.section(".x")` 的 AST 同形
|
||
// （编译器按 AST 扫描属性：TPC HandlesAssign._ExtractGlobalAttrs /
|
||
// TPV TranslatorGenMixin._scan_global_tattr），从而完全复用既有属性标记语义：
|
||
// __attribute__(section(".x")) const int → (t.attr.section(".x") | (CConst | int))
|
||
// __attribute__(a, b) int → ((t.attr.a | t.attr.b) | int)
|
||
// 与 const / volatile 同族：只作修饰，不改变数据类型。
|
||
// 语法与装饰器形态一致（实参用无前缀属性名，逐项脱糖为 `t.attr.<name>`；
|
||
// 已是 `t.attr.X` / `c.Attribute(...)` 形态的实参原样保留，允许混写）。
|
||
// 注意：与 GCC 的 `__attribute__((a, b))` 不同，Viper 只用**单层**括号。
|
||
private _parseViperAttributePrefix(): ExpressionNode | undefined {
|
||
if (this._parseOptions.dialect !== 'viper' || !this._isAtViperAttributePrefix(0)) {
|
||
return undefined;
|
||
}
|
||
|
||
const attributeToken = this._peekToken();
|
||
this._getNextToken(); // 消费 `__attribute__`
|
||
this._getNextToken(); // 消费 '('
|
||
|
||
const argListResult = this._parseArgList();
|
||
if (!this._consumeTokenIfType(TokenType.CloseParenthesis)) {
|
||
this._addSyntaxError(LocMessage.expectedCloseParen(), this._peekToken());
|
||
this._consumeTokensUntilType([TokenType.NewLine]);
|
||
}
|
||
|
||
const markers: ExpressionNode[] = [];
|
||
argListResult.args.forEach((arg) => {
|
||
if (arg.d.argCategory === ArgCategory.Simple) {
|
||
markers.push(this._makeViperAttrMarker(arg.d.valueExpr));
|
||
}
|
||
});
|
||
|
||
// 修饰器后必须跟类型（同 const / volatile 的硬关键字约束）。
|
||
const nextToken = this._peekToken();
|
||
if (!this._isViperTypeAtomStartToken(nextToken) && !this._isAtViperAttributePrefix(0)) {
|
||
this._addSyntaxError(
|
||
LocMessage.viperTypeDescriptorRequiresType().format({ name: '__attribute__' }),
|
||
attributeToken
|
||
);
|
||
return ErrorNode.create(attributeToken, ErrorExpressionCategory.MissingExpression);
|
||
}
|
||
|
||
// 递归解析被修饰的主体（支持 `__attribute__(...) const int`、
|
||
// `const __attribute__(...) int`、多个属性前缀等任意组合）。
|
||
const innerType = this._parseViperTypeAnnotationCore();
|
||
|
||
// 合成 '|' 链（'|' token 零长度，源码中不存在）：marker1 | marker2 | ... | inner
|
||
let modifierNode: ExpressionNode | undefined;
|
||
for (const marker of markers) {
|
||
if (!modifierNode) {
|
||
modifierNode = marker;
|
||
continue;
|
||
}
|
||
modifierNode = this._createBinaryOperationNode(
|
||
modifierNode,
|
||
marker,
|
||
this._createViperSyntheticBarToken(attributeToken),
|
||
OperatorType.BitwiseOr
|
||
);
|
||
}
|
||
|
||
if (!modifierNode) {
|
||
return innerType;
|
||
}
|
||
|
||
return this._createBinaryOperationNode(
|
||
modifierNode,
|
||
innerType,
|
||
this._createViperSyntheticBarToken(attributeToken),
|
||
OperatorType.BitwiseOr
|
||
);
|
||
}
|
||
|
||
// Viper 扩展：合成零长度的 '|' 算子 token（源码中不存在，仅承载位置信息）。
|
||
private _createViperSyntheticBarToken(anchorToken: Token): OperatorToken {
|
||
return OperatorToken.create(
|
||
anchorToken.start + anchorToken.length,
|
||
0,
|
||
OperatorType.BitwiseOr,
|
||
/* comments */ undefined
|
||
);
|
||
}
|
||
|
||
// Viper 扩展：当前（可带 offset）是否为注解位置的 `__attribute__(` 前缀。
|
||
// `__attribute__` 是普通标识符（非关键字），后必须紧邻 '('。
|
||
private _isAtViperAttributePrefix(offset: number): boolean {
|
||
const nameToken = this._peekToken(offset);
|
||
if (nameToken.type !== TokenType.Identifier) {
|
||
return false;
|
||
}
|
||
if (this._fileContents!.slice(nameToken.start, nameToken.start + nameToken.length) !== '__attribute__') {
|
||
return false;
|
||
}
|
||
return this._peekToken(offset + 1).type === TokenType.OpenParenthesis;
|
||
}
|
||
|
||
// Viper 扩展：把 `__attribute__(...)` 的实参脱糖为 `t.attr.<name>` 标记。
|
||
// - `always_inline`（Name） → `t.attr.always_inline`
|
||
// - `section(".x")`（Call，callee 是 Name）→ `t.attr.section(".x")`
|
||
// - 已是 `t.attr.X` / `c.Attribute(...)` 形态 → 原样保留（允许混写）
|
||
// 与 TPC/TPV 的 includes/ast/parser.vp `_MakeTAttrMarker` 逐条对应；
|
||
// 成员名复用实参自身的 token（真实位置），故属性名上的跳转/悬停仍指向 t.vpi。
|
||
// 根 `t` 是合成的「隐式模块根」（见 _createViperImplicitModuleName），
|
||
// 不要求源码 `import t`。
|
||
private _makeViperAttrMarker(argExpr: ExpressionNode): ExpressionNode {
|
||
const attrRoot = MemberAccessNode.create(
|
||
this._createViperImplicitModuleName(argExpr.start, 't'),
|
||
NameNode.create(IdentifierToken.create(argExpr.start, 0, 'attr', /* comments */ undefined))
|
||
);
|
||
|
||
if (argExpr.nodeType === ParseNodeType.Name) {
|
||
return MemberAccessNode.create(attrRoot, NameNode.create(argExpr.d.token));
|
||
}
|
||
|
||
if (argExpr.nodeType === ParseNodeType.Call && argExpr.d.leftExpr.nodeType === ParseNodeType.Name) {
|
||
const inner = MemberAccessNode.create(attrRoot, NameNode.create(argExpr.d.leftExpr.d.token));
|
||
return CallNode.create(inner, argExpr.d.args, argExpr.d.trailingComma);
|
||
}
|
||
|
||
// 已是属性标记形态（t.attr.X / c.Attribute(...)）：原样保留。
|
||
return argExpr;
|
||
}
|
||
|
||
// Viper 扩展：合成 zero-length 的「隐式模块根」名（`t` / `c`）。
|
||
// `__attribute__` 的脱糖节点里引用 `t` / `c`，但**不要求源码 import t / import c**
|
||
// —— 与编译器一致（TPC/TPV 的属性识别是 AST 级的，不看符号是否可见）。
|
||
// 该名在词法作用域里解析不到时，typeEvaluator 会回退到 builtins 作用域的同名
|
||
// 模块绑定（见 NameNode.d.isViperImplicitModule）。
|
||
private _createViperImplicitModuleName(start: number, name: string): NameNode {
|
||
const nameNode = NameNode.create(IdentifierToken.create(start, 0, name, /* comments */ undefined));
|
||
nameNode.d.isViperImplicitModule = true;
|
||
return nameNode;
|
||
}
|
||
|
||
// Viper 扩展：把 `@__attribute__(...)` 装饰器脱糖为
|
||
// `@c.Attribute(t.attr.a, t.attr.b)`——与 TPC/TPV 的
|
||
// includes/ast/parser.vp `_DesugarMagicAttribute` 逐字段同形，因此 class 与 def
|
||
// 共用同一形态；其它装饰器原样返回。
|
||
private _desugarViperMagicAttribute(expr: ExpressionNode): ExpressionNode {
|
||
if (this._parseOptions.dialect !== 'viper' || expr.nodeType !== ParseNodeType.Call) {
|
||
return expr;
|
||
}
|
||
|
||
if (!this._isViperMagicAttributeCallee(expr.d.leftExpr)) {
|
||
return expr;
|
||
}
|
||
|
||
const attributeCallee = MemberAccessNode.create(
|
||
this._createViperImplicitModuleName(expr.start, 'c'),
|
||
NameNode.create(IdentifierToken.create(expr.start, 0, 'Attribute', /* comments */ undefined))
|
||
);
|
||
|
||
const newArgs = expr.d.args.map((arg) => {
|
||
if (arg.d.argCategory !== ArgCategory.Simple) {
|
||
return arg;
|
||
}
|
||
return ArgumentNode.create(undefined, this._makeViperAttrMarker(arg.d.valueExpr), ArgCategory.Simple);
|
||
});
|
||
|
||
return CallNode.create(attributeCallee, newArgs, expr.d.trailingComma);
|
||
}
|
||
|
||
// Viper 扩展：装饰器被调用者是否为 `__attribute__`（Name 形态，或 `X.__attribute__` 尾段形态）。
|
||
private _isViperMagicAttributeCallee(node: ExpressionNode): boolean {
|
||
if (node.nodeType === ParseNodeType.Name) {
|
||
return node.d.value === '__attribute__';
|
||
}
|
||
if (node.nodeType === ParseNodeType.MemberAccess) {
|
||
return node.d.member.d.value === '__attribute__';
|
||
}
|
||
return false;
|
||
}
|
||
|
||
// Viper 扩展：判定注解位置 `const` / `volatile` / `__attribute__(` 之后是否跟类型。
|
||
private _isViperTypeAtomStartToken(token: Token): boolean {
|
||
return (
|
||
token.type === TokenType.Identifier ||
|
||
token.type === TokenType.OpenBracket ||
|
||
(token.type === TokenType.Keyword &&
|
||
(token as KeywordToken).keywordType === KeywordType.ViperTypeDescriptor)
|
||
);
|
||
}
|
||
|
||
// Viper 扩展：当前是否为值表达式中的 `volatile <调用>` 前缀。
|
||
// 仅 `volatile` 有值位置用法（`const` 没有）；且下一 token 必须是表达式起点
|
||
// （标识符 / '('），否则不算（`volatile = 2`、`volatile: int` 等仍走原错误路径）。
|
||
private _isAtViperVolatileValuePrefix(): boolean {
|
||
if (this._parseOptions.dialect !== 'viper' || this._isParsingTypeAnnotation) {
|
||
return false;
|
||
}
|
||
const descriptorToken = this._peekToken();
|
||
if (
|
||
descriptorToken.type !== TokenType.Keyword ||
|
||
(descriptorToken as KeywordToken).keywordType !== KeywordType.ViperTypeDescriptor
|
||
) {
|
||
return false;
|
||
}
|
||
if (
|
||
this._fileContents!.slice(descriptorToken.start, descriptorToken.start + descriptorToken.length) !==
|
||
'volatile'
|
||
) {
|
||
return false;
|
||
}
|
||
const nextTokenType = this._peekToken(1).type;
|
||
return nextTokenType === TokenType.Identifier || nextTokenType === TokenType.OpenParenthesis;
|
||
}
|
||
|
||
// Viper 扩展：消费值位置的 `volatile` 前缀，返回被修饰的调用节点。
|
||
// 语义上等价于该调用本身（volatile 仅影响代码生成，不改变类型），故直接返回
|
||
// CallNode：`volatile f()` 的类型即 f 的返回类型、`volatile 结构体()` 即结构体实例。
|
||
// 非调用形态（如 `volatile x`）报语法错误但仍返回已解析的表达式以维持错误恢复。
|
||
private _parseViperVolatileValue(): ExpressionNode {
|
||
const volatileToken = this._getNextToken();
|
||
const innerExpression = this._parseAtomExpression();
|
||
if (innerExpression.nodeType !== ParseNodeType.Call) {
|
||
this._addSyntaxError(LocMessage.viperVolatileValueRequiresCall(), volatileToken);
|
||
}
|
||
return innerExpression;
|
||
}
|
||
|
||
// Viper 扩展：类型描述符前缀 `const T` / `volatile T`。
|
||
// 解析层脱糖为等价的联合类型注解 `CConst | T` / `CVolatile | T`，
|
||
// 与手写 `a: t.CConst | CInt` 的 AST 同形，从而完全复用既有 TypeSet
|
||
// 修饰符语义（类型求值、赋值兼容、运算跳过修饰符、hover、语义高亮均一致），
|
||
// 不引入第二套修饰机制。
|
||
//
|
||
// 硬关键字：const / volatile 是 Viper 保留字（tokenizer 生成
|
||
// KeywordType.ViperTypeDescriptor），语法上只允许「类型注解前缀」这一种用法，
|
||
// 其余位置（`const = 1`、`x.volatile`、注解中不跟类型等）由 parser 报语法错误。
|
||
//
|
||
// 脱糖目标 `CConst` / `CVolatile` 是 builtins stub 中的别名
|
||
// （见 typeshed-fallback/stdlib/builtins.vpi），故不要求源码 import t。
|
||
private _parseViperTypeDescriptorPrefix(): ExpressionNode | undefined {
|
||
if (this._parseOptions.dialect !== 'viper') {
|
||
return undefined;
|
||
}
|
||
|
||
const descriptorToken = this._peekToken();
|
||
if (
|
||
descriptorToken.type !== TokenType.Keyword ||
|
||
(descriptorToken as KeywordToken).keywordType !== KeywordType.ViperTypeDescriptor
|
||
) {
|
||
return undefined;
|
||
}
|
||
|
||
const descriptorName = this._fileContents!.slice(
|
||
descriptorToken.start,
|
||
descriptorToken.start + descriptorToken.length
|
||
);
|
||
const memberName = descriptorName === 'const' ? 'CConst' : 'CVolatile';
|
||
|
||
// 硬关键字：紧随其后必须是类型原子的起点（标识符 / '[' / 另一个描述符 /
|
||
// `__attribute__(` 属性前缀），否则报语法错误（如 `const = 1`、`a: const`）。
|
||
const nextToken = this._peekToken(1);
|
||
const isTypeAtomStart =
|
||
this._isViperTypeAtomStartToken(nextToken) || this._isAtViperAttributePrefix(1);
|
||
if (!isTypeAtomStart) {
|
||
this._addSyntaxError(
|
||
LocMessage.viperTypeDescriptorRequiresType().format({ name: descriptorName }),
|
||
descriptorToken
|
||
);
|
||
this._getNextToken();
|
||
return ErrorNode.create(descriptorToken, ErrorExpressionCategory.MissingExpression);
|
||
}
|
||
|
||
// 消费描述符关键字。
|
||
this._getNextToken();
|
||
|
||
// 递归解析描述符修饰的主体：支持 `const volatile int`、`const [4]int`。
|
||
// 期间禁止在表达式层就地消费指针后缀 `*`（置 _isParsingViperTypeDescriptorInner），
|
||
// 使 `const int*` 恒为 `(CConst | int)*` —— 指针后缀由 _parseTypeAnnotation
|
||
// 统一在最外层应用（指针指向被描述符修饰的类型）。
|
||
const wasDescriptorInner = this._isParsingViperTypeDescriptorInner;
|
||
this._isParsingViperTypeDescriptorInner = true;
|
||
const innerType = this._parseViperTypeAnnotationCore();
|
||
this._isParsingViperTypeDescriptorInner = wasDescriptorInner;
|
||
|
||
// 合成描述符名：零长度、位置落在描述符关键字起点，不产生语义高亮
|
||
// （semanticTokenProvider 跳过 length <= 0 的名字），也不参与位置命中。
|
||
const modifierNameNode = NameNode.create(
|
||
IdentifierToken.create(descriptorToken.start, 0, memberName, /* comments */ undefined)
|
||
);
|
||
modifierNameNode.d.isViperTypeDescriptor = true;
|
||
|
||
// 合成 '|' 算子 token：源码中不存在，位置紧随描述符关键字，零长度。
|
||
const operatorToken = OperatorToken.create(
|
||
descriptorToken.start + descriptorToken.length,
|
||
0,
|
||
OperatorType.BitwiseOr,
|
||
/* comments */ undefined
|
||
);
|
||
return BinaryOperationNode.create(modifierNameNode, innerType, operatorToken, OperatorType.BitwiseOr);
|
||
}
|
||
|
||
// Viper 扩展：判定当前 '*'/'**' 串是否为类型注解内的 C 指针后缀。
|
||
// 判据：星号串（'*' 一层 / '**' 两层，可含换行）之后紧跟"无右操作数"位置，
|
||
// 即 ']' / ',' / ')' / ':' / '=' / 换行 / ';' / 文件结束。
|
||
// 因此 `t.CArray[X, N * 2]`、`a ** b` 这类'*'后有操作数的真实运算不会被误判。
|
||
private _isAtViperPointerTypeSuffix(): boolean {
|
||
let offset = 0;
|
||
let starCount = 0;
|
||
while (true) {
|
||
const token = this._peekToken(offset);
|
||
if (token.type === TokenType.Operator) {
|
||
const operatorType = (token as OperatorToken).operatorType;
|
||
if (operatorType === OperatorType.Multiply) {
|
||
starCount += 1;
|
||
offset += 1;
|
||
continue;
|
||
}
|
||
if (operatorType === OperatorType.Power) {
|
||
starCount += 2;
|
||
offset += 1;
|
||
continue;
|
||
}
|
||
}
|
||
if (token.type === TokenType.NewLine) {
|
||
offset += 1;
|
||
continue;
|
||
}
|
||
break;
|
||
}
|
||
|
||
if (starCount === 0) {
|
||
return false;
|
||
}
|
||
|
||
const nextToken = this._peekToken(offset);
|
||
switch (nextToken.type) {
|
||
case TokenType.CloseBracket:
|
||
case TokenType.Comma:
|
||
case TokenType.CloseParenthesis:
|
||
case TokenType.Colon:
|
||
case TokenType.Semicolon:
|
||
case TokenType.NewLine:
|
||
case TokenType.EndOfStream:
|
||
return true;
|
||
case TokenType.Operator:
|
||
return (nextToken as OperatorToken).operatorType === OperatorType.Assign;
|
||
}
|
||
return false;
|
||
}
|
||
|
||
// Viper 扩展：判定当前 '*'/'**' 串是否为 `T*(x)` 形式的 C 风格指针强转调用。
|
||
// 判据（三条同时成立，与 TPC ExprParser._AtPtrSuffixCall 同源）：
|
||
// 1. base 是「类型原子」形态（Name / MemberAccess / Index）；
|
||
// 2. base 与星串**紧贴**（源码字符相邻，无空白/换行）；
|
||
// 3. 星串内部逐个紧贴，且其**紧贴**一个 '('。
|
||
// 为什么 '(' 不并入 _isAtViperPointerTypeSuffix 的「无右操作数位置」：`a * (b)` 是
|
||
// 合法乘法（'(' 可作乘法右操作数的起点）。故叠加「前驱是类型原子 + 两侧紧贴」两条，
|
||
// 使 `a * (b + c)`、`f(x) * (y)`、`(i + 1) * (j)` 仍按乘法解析。
|
||
private _isAtViperPointerSuffixCastCall(baseExpr: ExpressionNode): boolean {
|
||
if (this._parseOptions.dialect !== 'viper') {
|
||
return false;
|
||
}
|
||
switch (baseExpr.nodeType) {
|
||
case ParseNodeType.Name:
|
||
case ParseNodeType.MemberAccess:
|
||
case ParseNodeType.Index:
|
||
break;
|
||
default:
|
||
return false;
|
||
}
|
||
|
||
// base 与星串是否紧贴：用已消费的最后一个 token 的结束位置做判据。
|
||
const prevToken = this._peekToken(-1);
|
||
let starEndOffset = prevToken.start + prevToken.length;
|
||
let offset = 0;
|
||
let starCount = 0;
|
||
while (true) {
|
||
const token = this._peekToken(offset);
|
||
const operatorType =
|
||
token.type === TokenType.Operator ? (token as OperatorToken).operatorType : undefined;
|
||
if (operatorType !== OperatorType.Multiply && operatorType !== OperatorType.Power) {
|
||
break;
|
||
}
|
||
if (token.start !== starEndOffset) {
|
||
// base 与星串之间、或星串内部有空白：非 C 风格强转写法。
|
||
return false;
|
||
}
|
||
starEndOffset = token.start + token.length;
|
||
starCount += 1;
|
||
offset += 1;
|
||
}
|
||
|
||
if (starCount === 0) {
|
||
return false;
|
||
}
|
||
|
||
const nextToken = this._peekToken(offset);
|
||
return nextToken.type === TokenType.OpenParenthesis && nextToken.start === starEndOffset;
|
||
}
|
||
|
||
// Viper 扩展：定长数组类型语法 `[SIZE]T`（等价 t.CArray[T, SIZE]）与
|
||
// 不定长写法 `[]T`（等价 t.CArray[T]，与 `t.CArray[T]` 单参数一致）。
|
||
// 生效位置：
|
||
// - 类型注解起点（变量/参数/返回值/类成员/type 别名）；
|
||
// - 下标实参内，但仅当形如 `[SIZE]T` 时（见 _isAtViperSubscriptBracketArrayType），
|
||
// 使 `list[[4]int]` 可用，同时不影响 `t.Callable[[int, str], None]`；
|
||
// - `[SIZE]T` 的元素类型内（支持 `[2][3]T`、`list[[2][3]int]`）。
|
||
// 值表达式里的 `[SIZE]` 仍是 list 字面量。
|
||
private _parseViperBracketArrayTypeIfNeeded(): ExpressionNode {
|
||
if (
|
||
this._parseOptions.dialect === 'viper' &&
|
||
(!this._isParsingIndexTrailer || this._isParsingViperBracketArrayElement) &&
|
||
this._peekTokenType() === TokenType.OpenBracket
|
||
) {
|
||
return this._parseViperBracketArrayType();
|
||
}
|
||
return this._parseTestExpression(/* allowAssignmentExpression */ false);
|
||
}
|
||
|
||
// Viper 扩展：判断下标实参位置（如 `list[[4]int]`）是否是定长数组类型 `[SIZE]T`。
|
||
// 与 `t.Callable[[int, str], None]` / `t.Callable[[], None]` 的实参列表区别在于
|
||
// 匹配的 `]` 之后是否紧跟元素类型：`[4]int` 是数组，`[int, str]`、`[]` 不是。
|
||
// 仅做 token 前瞻，不解析、不产生诊断。
|
||
private _isAtViperSubscriptBracketArrayType(): boolean {
|
||
if (this._parseOptions.dialect !== 'viper' || this._peekToken(0).type !== TokenType.OpenBracket) {
|
||
return false;
|
||
}
|
||
|
||
// 跳过 [SIZE]：SIZE 内允许嵌套下标，按括号配对找匹配的 ']'。
|
||
let offset = 1;
|
||
let depth = 0;
|
||
while (true) {
|
||
const token = this._peekToken(offset);
|
||
if (token.type === TokenType.OpenBracket) {
|
||
depth += 1;
|
||
} else if (token.type === TokenType.CloseBracket) {
|
||
if (depth === 0) {
|
||
break;
|
||
}
|
||
depth -= 1;
|
||
} else if (token.type === TokenType.NewLine || token.type === TokenType.EndOfStream) {
|
||
return false;
|
||
}
|
||
offset += 1;
|
||
}
|
||
|
||
// ']' 之后必须是元素类型的起始（标识符，或嵌套数组的 '['）。
|
||
const elementStartToken = this._peekToken(offset + 1);
|
||
return elementStartToken.type === TokenType.Identifier || elementStartToken.type === TokenType.OpenBracket;
|
||
}
|
||
|
||
// Viper 扩展：解析 `[SIZE]T` / `[]T` 并脱糖为等价的 `t.CArray[T, SIZE]` /
|
||
// `t.CArray[T]` 下标表达式，使类型求值、语义高亮、类型显示完全复用既有
|
||
// t.CArray 机制（元素类型、索引返回 T、数组名退化为指针、字符数组降级
|
||
// str/bytes、单参数不定长等），不引入第二套语义。
|
||
// 元素类型递归解析，故支持 `[2][3]T`（等价 t.CArray[t.CArray[T, 3], 2]）、
|
||
// `[4]T*`（指针后缀）、`[4]T | t.CPtr`（联合）等。
|
||
private _parseViperBracketArrayType(): ExpressionNode {
|
||
const openBracketToken = this._getNextToken();
|
||
assert(openBracketToken.type === TokenType.OpenBracket);
|
||
|
||
let sizeExpr: ExpressionNode | undefined;
|
||
if (this._peekTokenType() !== TokenType.CloseBracket) {
|
||
sizeExpr = this._parseTestExpression(/* allowAssignmentExpression */ false);
|
||
}
|
||
|
||
const closeBracketToken = this._peekToken();
|
||
if (!this._consumeTokenIfType(TokenType.CloseBracket)) {
|
||
this._addSyntaxError(LocMessage.expectedCloseBracket(), this._peekToken());
|
||
}
|
||
|
||
// 元素类型：递归走注解解析，天然支持嵌套 `[2][3]T` 与指针后缀 `[4]T*`。
|
||
// 期间允许在下标实参内继续出现 `[SIZE]T`（如 list[[2][3]int]）。
|
||
const wasParsingBracketArrayElement = this._isParsingViperBracketArrayElement;
|
||
this._isParsingViperBracketArrayElement = true;
|
||
const elementType = this._parseTypeAnnotation();
|
||
this._isParsingViperBracketArrayElement = wasParsingBracketArrayElement;
|
||
|
||
// 合成 `t.CArray`：零长度、位置落在 '[' 上，因此不产生任何语义高亮
|
||
// （semanticTokenProvider 跳过 length <= 0 的名字）也不参与位置命中。
|
||
const syntheticTNameNode = NameNode.create(
|
||
IdentifierToken.create(openBracketToken.start, 0, 't', /* comments */ undefined)
|
||
);
|
||
syntheticTNameNode.d.isViperBracketArraySynthetic = true;
|
||
const cArrayNameNode = NameNode.create(
|
||
IdentifierToken.create(openBracketToken.start, 0, 'CArray', /* comments */ undefined)
|
||
);
|
||
const memberAccessNode = MemberAccessNode.create(syntheticTNameNode, cArrayNameNode);
|
||
|
||
// items 按**源码顺序**排列（[长度] 在 '[' 内、元素类型在 ']' 之后），
|
||
// 以满足"子节点必须按位置递增且不重叠"的 AST 不变量；
|
||
// 类型求值在 getTypeArgs 中按语义顺序（元素类型在前）重排实参。
|
||
const items: ArgumentNode[] = [];
|
||
if (sizeExpr) {
|
||
items.push(ArgumentNode.create(/* startToken */ undefined, sizeExpr, ArgCategory.Simple));
|
||
}
|
||
items.push(ArgumentNode.create(/* startToken */ undefined, elementType, ArgCategory.Simple));
|
||
|
||
const indexNode = IndexNode.create(memberAccessNode, items, /* trailingComma */ false, closeBracketToken);
|
||
indexNode.d.isViperBracketArray = true;
|
||
|
||
// 让整个下标节点的范围覆盖 `[SIZE]T` 全文（含 ']' 之后的元素类型），
|
||
// 使 hover/定位落在元素类型内时仍能沿父链找到该数组类型节点。
|
||
extendRange(indexNode, elementType);
|
||
|
||
return indexNode;
|
||
}
|
||
|
||
// Viper 扩展：解析 C 指针类型后缀 `T*` / `T**` / `T***`。
|
||
// 每个 '*' 对应一层指针；'**' 词法化为单个 Power 算子、等价两层指针。
|
||
// 结果用 UnaryOperationNode(operator=Multiply) 嵌套表示：
|
||
// T* → *(T) → CPtr[T]
|
||
// T** → *(*T) → CPtr[CPtr[T]]
|
||
// 仅 Viper 方言生效（Python 方言中 '*'/'**' 是二元乘法/幂运算）。
|
||
private _parseViperPointerTypeSuffix(baseExpr: ExpressionNode): ExpressionNode {
|
||
if (this._parseOptions.dialect !== 'viper') {
|
||
return baseExpr;
|
||
}
|
||
|
||
let result = baseExpr;
|
||
while (true) {
|
||
const nextOperator = this._peekOperatorType();
|
||
if (nextOperator === OperatorType.Multiply) {
|
||
const starToken = this._getNextToken();
|
||
result = this._markViperPointerTypeSuffix(
|
||
this._createUnaryOperationNode(starToken, result, OperatorType.Multiply)
|
||
);
|
||
} else if (nextOperator === OperatorType.Power) {
|
||
const starToken = this._getNextToken();
|
||
result = this._markViperPointerTypeSuffix(
|
||
this._createUnaryOperationNode(starToken, result, OperatorType.Multiply)
|
||
);
|
||
result = this._markViperPointerTypeSuffix(
|
||
this._createUnaryOperationNode(starToken, result, OperatorType.Multiply)
|
||
);
|
||
} else {
|
||
break;
|
||
}
|
||
}
|
||
return result;
|
||
}
|
||
|
||
// Viper 扩展：标记由 C 指针类型后缀 `T*` 生成的一元 '*' 节点（区别于前缀解引用 `*x`）。
|
||
// 供 `(T*)x` 强转解析（_parseAtom）区分「括号内的指针类型」与「括号内的解引用」。
|
||
private _markViperPointerTypeSuffix(node: ExpressionNode): ExpressionNode {
|
||
if (node.nodeType === ParseNodeType.UnaryOperation) {
|
||
(node as UnaryOperationNode).d.isViperPointerTypeSuffix = true;
|
||
}
|
||
return node;
|
||
}
|
||
|
||
// Viper 扩展：`(T)x` / `(T*)x` 强转解析。
|
||
// 当括号内是「类型原子」形态且紧随其后是操作数起点时，将其后的操作数作为
|
||
// 该强转的唯一实参，构造 CallNode 返回。
|
||
// 类型原子：
|
||
// - Name：`(Buf)v`；
|
||
// - MemberAccess：`(t.CInt)x`；
|
||
// - Subscript：`(list[X])v`；
|
||
// - 指针后缀一元 '*'：`(T*)x`（由 _parseViperPointerTypeSuffix 标记）。
|
||
// 语义等价于已支持的 `T(x)` / `(T)(x)` 调用式强转通路：
|
||
// CallNode(func = 括号内类型表达式, args = [操作数])。
|
||
// 边界：`(a)*b`、`(a) * (b)` 仍是乘法（'*' 不是操作数起点）；
|
||
// `(T*)[i]` 仍是下标（'[' 不是操作数起点）；
|
||
// `(a)b` 由语法错误变为「调用 a(b)」——可接受的超集，
|
||
// 值为类型即强转，否则由语义层报「不可调用」。
|
||
private _tryParseViperPointerCast(possibleTupleNode: ExpressionNode): ExpressionNode | undefined {
|
||
if (this._parseOptions.dialect !== 'viper') {
|
||
return undefined;
|
||
}
|
||
if (!this._isViperCastTypeAtom(possibleTupleNode)) {
|
||
return undefined;
|
||
}
|
||
if (!this._isAtViperCastOperandStart()) {
|
||
return undefined;
|
||
}
|
||
|
||
// 按 C 的强转优先级解析操作数（一元表达式级），故 `(T*)a + b` = `((T*)a) + b`。
|
||
const operand = this._parseArithmeticFactor();
|
||
const argNode = ArgumentNode.create(/* startToken */ undefined, operand, ArgCategory.Simple);
|
||
return CallNode.create(possibleTupleNode, [argNode], /* trailingComma */ false);
|
||
}
|
||
|
||
// Viper 扩展：判定括号内的表达式是否为「类型原子」（可用作 C 强转的类型部分）。
|
||
// 仅接受 Name / MemberAccess / Subscript / 指针后缀一元 '*'，
|
||
// 故 `(1+2)a`、`(*p)a`（前缀解引用）等不会被当作强转。
|
||
private _isViperCastTypeAtom(node: ExpressionNode): boolean {
|
||
switch (node.nodeType) {
|
||
case ParseNodeType.Name:
|
||
case ParseNodeType.MemberAccess:
|
||
case ParseNodeType.Index:
|
||
return true;
|
||
case ParseNodeType.UnaryOperation:
|
||
return !!(node as UnaryOperationNode).d.isViperPointerTypeSuffix;
|
||
default:
|
||
return false;
|
||
}
|
||
}
|
||
|
||
// Viper 扩展：判定当前位置是否为 `(T*)` 之后可作为强转操作数的表达式起点。
|
||
// 仅接受操作数起点（标识符 / 数字 / 字符串 / f-string / 字面量关键字 / 字典集合 / 反引号），
|
||
// 排除运算符与 '('、'['：后两者由 _parseAtomExpression 的 trailer 分别按函数调用、
|
||
// 下标处理（即 `(T*)(x)` 走既有调用通路，`(T*)[i]` 保持下标语义），不受本扩展影响。
|
||
private _isAtViperCastOperandStart(): boolean {
|
||
const nextToken = this._peekToken();
|
||
switch (nextToken.type) {
|
||
case TokenType.Identifier:
|
||
case TokenType.Number:
|
||
case TokenType.String:
|
||
case TokenType.FStringStart:
|
||
case TokenType.OpenCurlyBrace:
|
||
case TokenType.Backtick:
|
||
return true;
|
||
case TokenType.Keyword: {
|
||
const keywordToken = nextToken as KeywordToken;
|
||
return (
|
||
keywordToken.keywordType === KeywordType.False ||
|
||
keywordToken.keywordType === KeywordType.True ||
|
||
keywordToken.keywordType === KeywordType.None
|
||
);
|
||
}
|
||
default:
|
||
return false;
|
||
}
|
||
}
|
||
|
||
private _reportStringTokenErrors(
|
||
stringToken: StringToken | FStringStartToken,
|
||
unescapedResult?: StringTokenUtils.UnescapedString
|
||
) {
|
||
if (stringToken.flags & StringTokenFlags.Unterminated) {
|
||
this._addSyntaxError(LocMessage.stringUnterminated(), stringToken);
|
||
}
|
||
|
||
if (unescapedResult?.nonAsciiInBytes) {
|
||
this._addSyntaxError(LocMessage.stringNonAsciiBytes(), stringToken);
|
||
}
|
||
|
||
if (stringToken.flags & StringTokenFlags.Format) {
|
||
if (PythonVersion.isLessThan(this._getLanguageVersion(), pythonVersion3_6)) {
|
||
this._addSyntaxError(LocMessage.formatStringIllegal(), stringToken);
|
||
}
|
||
|
||
if (stringToken.flags & StringTokenFlags.Bytes) {
|
||
this._addSyntaxError(LocMessage.formatStringBytes(), stringToken);
|
||
}
|
||
|
||
if (stringToken.flags & StringTokenFlags.Unicode) {
|
||
this._addSyntaxError(LocMessage.formatStringUnicode(), stringToken);
|
||
}
|
||
|
||
if (stringToken.flags & StringTokenFlags.Template) {
|
||
this._addSyntaxError(LocMessage.formatStringTemplate(), stringToken);
|
||
}
|
||
}
|
||
|
||
if (stringToken.flags & StringTokenFlags.Template) {
|
||
if (PythonVersion.isLessThan(this._getLanguageVersion(), pythonVersion3_14)) {
|
||
this._addSyntaxError(LocMessage.templateStringIllegal(), stringToken);
|
||
}
|
||
|
||
if (stringToken.flags & StringTokenFlags.Bytes) {
|
||
this._addSyntaxError(LocMessage.templateStringBytes(), stringToken);
|
||
}
|
||
|
||
if (stringToken.flags & StringTokenFlags.Unicode) {
|
||
this._addSyntaxError(LocMessage.templateStringUnicode(), stringToken);
|
||
}
|
||
}
|
||
}
|
||
|
||
private _makeStringNode(stringToken: StringToken): StringNode {
|
||
const unescapedResult = StringTokenUtils.getUnescapedString(stringToken);
|
||
this._reportStringTokenErrors(stringToken, unescapedResult);
|
||
return StringNode.create(stringToken, unescapedResult.value);
|
||
}
|
||
|
||
private _getTypeAnnotationCommentText(): StringToken | undefined {
|
||
if (this._tokenIndex === 0) {
|
||
return undefined;
|
||
}
|
||
|
||
const curToken = this._tokenizerOutput!.tokens.getItemAt(this._tokenIndex - 1);
|
||
const nextToken = this._tokenizerOutput!.tokens.getItemAt(this._tokenIndex);
|
||
|
||
if (curToken.start + curToken.length === nextToken.start) {
|
||
return undefined;
|
||
}
|
||
|
||
const interTokenContents = this._fileContents!.slice(curToken.start + curToken.length, nextToken.start);
|
||
const match = interTokenContents.match(commentRegEx);
|
||
if (!match) {
|
||
return undefined;
|
||
}
|
||
|
||
// Synthesize a string token and StringNode.
|
||
const typeString = match[2];
|
||
|
||
// Ignore all "ignore" comments. Include "[" in the regular
|
||
// expression because mypy supports ignore comments of the
|
||
// form ignore[errorCode, ...]. We'll treat these as regular
|
||
// ignore statements (as though no errorCodes were included).
|
||
if (typeString.trim().match(ignoreCommentRegEx)) {
|
||
return undefined;
|
||
}
|
||
|
||
const tokenOffset = curToken.start + curToken.length + match[1].length;
|
||
return StringToken.create(
|
||
tokenOffset,
|
||
typeString.length,
|
||
StringTokenFlags.None,
|
||
typeString,
|
||
0,
|
||
/* comments */ undefined
|
||
);
|
||
}
|
||
|
||
private _parseVariableTypeAnnotationComment(): ExpressionNode | undefined {
|
||
const stringToken = this._getTypeAnnotationCommentText();
|
||
if (!stringToken) {
|
||
return undefined;
|
||
}
|
||
|
||
const stringNode = this._makeStringNode(stringToken);
|
||
const stringListNode = StringListNode.create([stringNode]);
|
||
const parser = new Parser();
|
||
const parseResults = parser.parseTextExpression(
|
||
this._fileContents!,
|
||
stringToken.start,
|
||
stringToken.length,
|
||
this._parseOptions,
|
||
ParseTextMode.VariableAnnotation,
|
||
/* initialParenDepth */ undefined,
|
||
this._typingSymbolAliases
|
||
);
|
||
|
||
parseResults.diagnostics.forEach((diag) => {
|
||
this._addSyntaxError(diag.message, stringListNode);
|
||
});
|
||
|
||
if (!parseResults.parseTree) {
|
||
return undefined;
|
||
}
|
||
|
||
return parseResults.parseTree;
|
||
}
|
||
|
||
private _parseFunctionTypeAnnotationComment(stringToken: StringToken, functionNode: FunctionNode): void {
|
||
const stringNode = this._makeStringNode(stringToken);
|
||
const stringListNode = StringListNode.create([stringNode]);
|
||
const parser = new Parser();
|
||
const parseResults = parser.parseTextExpression(
|
||
this._fileContents!,
|
||
stringToken.start,
|
||
stringToken.length,
|
||
this._parseOptions,
|
||
ParseTextMode.FunctionAnnotation,
|
||
/* initialParenDepth */ undefined,
|
||
this._typingSymbolAliases
|
||
);
|
||
|
||
parseResults.diagnostics.forEach((diag) => {
|
||
this._addSyntaxError(diag.message, stringListNode);
|
||
});
|
||
|
||
if (!parseResults.parseTree) {
|
||
return;
|
||
}
|
||
|
||
const functionAnnotation = parseResults.parseTree;
|
||
|
||
functionNode.d.funcAnnotationComment = functionAnnotation;
|
||
functionAnnotation.parent = functionNode;
|
||
extendRange(functionNode, functionAnnotation);
|
||
}
|
||
|
||
private _parseFStringReplacementField(
|
||
fieldExpressions: ExpressionNode[],
|
||
middleTokens: FStringMiddleToken[],
|
||
formatExpressions: ExpressionNode[],
|
||
nestingDepth = 0
|
||
): boolean {
|
||
let nextToken = this._getNextToken();
|
||
|
||
// The caller should have already confirmed that the next token is an open brace.
|
||
assert(nextToken.type === TokenType.OpenCurlyBrace);
|
||
|
||
// Consume the expression.
|
||
const expr =
|
||
this._tryParseYieldExpression() ??
|
||
this._parseTestOrStarListAsExpression(
|
||
/* allowAssignmentExpression */ true,
|
||
/* allowMultipleUnpack */ true,
|
||
ErrorExpressionCategory.MissingExpression,
|
||
() => LocMessage.expectedExpr()
|
||
);
|
||
|
||
fieldExpressions.push(expr);
|
||
|
||
if (expr.nodeType === ParseNodeType.Error) {
|
||
return false;
|
||
}
|
||
|
||
// Consume an optional "=" token after the expression.
|
||
nextToken = this._peekToken();
|
||
if (
|
||
nextToken.type === TokenType.Operator &&
|
||
(nextToken as OperatorToken).operatorType === OperatorType.Assign
|
||
) {
|
||
// This feature requires Python 3.8 or newer.
|
||
if (PythonVersion.isLessThan(this._parseOptions.pythonVersion, pythonVersion3_8)) {
|
||
this._addSyntaxError(LocMessage.formatStringDebuggingIllegal(), nextToken);
|
||
}
|
||
|
||
this._getNextToken();
|
||
nextToken = this._peekToken();
|
||
}
|
||
|
||
// Consume an optional !r, !s, or !a token.
|
||
if (nextToken.type === TokenType.ExclamationMark) {
|
||
this._getNextToken();
|
||
nextToken = this._peekToken();
|
||
|
||
if (nextToken.type !== TokenType.Identifier) {
|
||
this._addSyntaxError(LocMessage.formatStringExpectedConversion(), nextToken);
|
||
} else {
|
||
this._getNextToken();
|
||
nextToken = this._peekToken();
|
||
}
|
||
}
|
||
|
||
if (nextToken.type === TokenType.Colon) {
|
||
this._getNextToken();
|
||
this._parseFStringFormatString(fieldExpressions, middleTokens, formatExpressions, nestingDepth);
|
||
nextToken = this._peekToken();
|
||
}
|
||
|
||
if (nextToken.type !== TokenType.CloseCurlyBrace) {
|
||
this._addSyntaxError(LocMessage.formatStringUnterminated(), nextToken);
|
||
return false;
|
||
} else {
|
||
this._getNextToken();
|
||
}
|
||
|
||
// Indicate success.
|
||
return true;
|
||
}
|
||
|
||
private _parseFStringFormatString(
|
||
fieldExpressions: ExpressionNode[],
|
||
middleTokens: FStringMiddleToken[],
|
||
formatExpressions: ExpressionNode[],
|
||
nestingDepth: number
|
||
) {
|
||
while (true) {
|
||
const nextToken = this._peekToken();
|
||
|
||
if (nextToken.type === TokenType.CloseCurlyBrace || nextToken.type === TokenType.FStringEnd) {
|
||
break;
|
||
}
|
||
|
||
if (nextToken.type === TokenType.FStringMiddle) {
|
||
this._getNextToken();
|
||
continue;
|
||
}
|
||
|
||
if (nextToken.type === TokenType.OpenCurlyBrace) {
|
||
// The Python interpreter reports an error at the point where the
|
||
// nesting level exceeds 1. Don't report the error again for deeper nestings.
|
||
if (nestingDepth === 2) {
|
||
this._addSyntaxError(LocMessage.formatStringNestedFormatSpecifier(), nextToken);
|
||
}
|
||
|
||
this._parseFStringReplacementField(fieldExpressions, middleTokens, formatExpressions, nestingDepth + 1);
|
||
continue;
|
||
}
|
||
|
||
break;
|
||
}
|
||
}
|
||
|
||
private _parseFormatString(startToken: FStringStartToken): FormatStringNode {
|
||
const middleTokens: FStringMiddleToken[] = [];
|
||
const fieldExpressions: ExpressionNode[] = [];
|
||
const formatExpressions: ExpressionNode[] = [];
|
||
let endToken: FStringEndToken | undefined = undefined;
|
||
|
||
// Consume middle tokens and expressions until we hit a "{" or "}" token.
|
||
while (true) {
|
||
const nextToken = this._peekToken();
|
||
|
||
if (nextToken.type === TokenType.FStringEnd) {
|
||
endToken = nextToken as FStringEndToken;
|
||
|
||
if ((endToken.flags & StringTokenFlags.Unterminated) !== 0) {
|
||
this._addSyntaxError(LocMessage.stringUnterminated(), startToken);
|
||
}
|
||
this._getNextToken();
|
||
break;
|
||
}
|
||
|
||
if (nextToken.type === TokenType.FStringMiddle) {
|
||
middleTokens.push(nextToken as FStringMiddleToken);
|
||
this._getNextToken();
|
||
continue;
|
||
}
|
||
|
||
if (nextToken.type === TokenType.OpenCurlyBrace) {
|
||
if (!this._parseFStringReplacementField(fieldExpressions, middleTokens, formatExpressions)) {
|
||
// An error was reported. Try to recover the parse.
|
||
if (this._consumeTokensUntilType([TokenType.FStringEnd, TokenType.NewLine])) {
|
||
if (this._peekToken().type === TokenType.FStringEnd) {
|
||
this._getNextToken();
|
||
}
|
||
}
|
||
break;
|
||
}
|
||
continue;
|
||
}
|
||
|
||
// We've hit an error. Try to recover as gracefully as possible.
|
||
if (nextToken.type !== TokenType.NewLine) {
|
||
// Consume tokens until we find the end.
|
||
if (this._consumeTokensUntilType([TokenType.FStringEnd])) {
|
||
this._getNextToken();
|
||
}
|
||
}
|
||
|
||
this._addSyntaxError(
|
||
nextToken.type === TokenType.CloseCurlyBrace
|
||
? LocMessage.formatStringBrace()
|
||
: LocMessage.stringUnterminated(),
|
||
nextToken
|
||
);
|
||
break;
|
||
}
|
||
|
||
this._reportStringTokenErrors(startToken);
|
||
|
||
return FormatStringNode.create(startToken, endToken, middleTokens, fieldExpressions, formatExpressions);
|
||
}
|
||
|
||
private _createBinaryOperationNode(
|
||
leftExpression: ExpressionNode,
|
||
rightExpression: ExpressionNode,
|
||
operatorToken: Token,
|
||
operator: OperatorType
|
||
) {
|
||
const binaryNode = BinaryOperationNode.create(leftExpression, rightExpression, operatorToken, operator);
|
||
|
||
// Determine if we're exceeding the max parse depth. If so, replace
|
||
// the subnode with an error node. Otherwise we risk crashing in the binder
|
||
// or type evaluator.
|
||
const leftMaxDepth = this._maxChildDepthMap.get(leftExpression.id) ?? 0;
|
||
const rightMaxDepth = this._maxChildDepthMap.get(rightExpression.id) ?? 0;
|
||
|
||
if (leftMaxDepth >= maxChildNodeDepth || rightMaxDepth >= maxChildNodeDepth) {
|
||
this._addSyntaxError(LocMessage.maxParseDepthExceeded(), binaryNode);
|
||
return ErrorNode.create(binaryNode, ErrorExpressionCategory.MaxDepthExceeded);
|
||
}
|
||
|
||
this._maxChildDepthMap.set(binaryNode.id, Math.max(leftMaxDepth, rightMaxDepth) + 1);
|
||
return binaryNode;
|
||
}
|
||
|
||
private _createUnaryOperationNode(operatorToken: Token, expression: ExpressionNode, operator: OperatorType) {
|
||
const unaryNode = UnaryOperationNode.create(operatorToken, expression, operator);
|
||
|
||
// Determine if we're exceeding the max parse depth. If so, replace
|
||
// the subnode with an error node. Otherwise we risk crashing in the binder
|
||
// or type evaluator.
|
||
|
||
const maxDepth = this._maxChildDepthMap.get(expression.id) ?? 0;
|
||
if (maxDepth >= maxChildNodeDepth) {
|
||
this._addSyntaxError(LocMessage.maxParseDepthExceeded(), unaryNode);
|
||
return ErrorNode.create(unaryNode, ErrorExpressionCategory.MaxDepthExceeded);
|
||
}
|
||
|
||
this._maxChildDepthMap.set(unaryNode.id, maxDepth + 1);
|
||
return unaryNode;
|
||
}
|
||
|
||
private _parseStringList(): StringListNode {
|
||
const stringList: (StringNode | FormatStringNode)[] = [];
|
||
|
||
while (true) {
|
||
const nextToken = this._peekToken();
|
||
if (nextToken.type === TokenType.String) {
|
||
stringList.push(this._makeStringNode(this._getNextToken() as StringToken));
|
||
} else if (nextToken.type === TokenType.FStringStart) {
|
||
stringList.push(this._parseFormatString(this._getNextToken() as FStringStartToken));
|
||
} else {
|
||
break;
|
||
}
|
||
}
|
||
|
||
const stringNode = StringListNode.create(stringList);
|
||
|
||
// If we're parsing a type annotation, parse the contents of the string.
|
||
if (this._isParsingTypeAnnotation) {
|
||
// Don't allow multiple strings because we have no way of reporting
|
||
// parse errors that span strings.
|
||
if (stringNode.d.strings.length > 1) {
|
||
if (this._isParsingQuotedText) {
|
||
this._addSyntaxError(LocMessage.annotationSpansStrings(), stringNode);
|
||
}
|
||
} else if (stringNode.d.strings[0].nodeType === ParseNodeType.FormatString) {
|
||
if (this._isParsingQuotedText) {
|
||
this._addSyntaxError(LocMessage.annotationFormatString(), stringNode);
|
||
}
|
||
} else {
|
||
const stringToken = stringNode.d.strings[0].d.token;
|
||
const stringValue = StringTokenUtils.getUnescapedString(
|
||
stringNode.d.strings[0].d.token,
|
||
/* elideCrlf */ false
|
||
);
|
||
const unescapedString = stringValue.value;
|
||
const tokenOffset = stringToken.start;
|
||
const prefixLength = stringToken.prefixLength + stringToken.quoteMarkLength;
|
||
|
||
// Don't allow escape characters because we have no way of mapping
|
||
// error ranges back to the escaped text.
|
||
if (unescapedString.length !== stringToken.length - prefixLength - stringToken.quoteMarkLength) {
|
||
if (this._isParsingQuotedText) {
|
||
this._addSyntaxError(LocMessage.annotationStringEscape(), stringNode);
|
||
}
|
||
} else if (
|
||
(stringToken.flags &
|
||
(StringTokenFlags.Raw |
|
||
StringTokenFlags.Bytes |
|
||
StringTokenFlags.Format |
|
||
StringTokenFlags.Template)) ===
|
||
0
|
||
) {
|
||
const parser = new Parser();
|
||
const parseResults = parser.parseTextExpression(
|
||
this._fileContents!,
|
||
tokenOffset + prefixLength,
|
||
unescapedString.length,
|
||
this._parseOptions,
|
||
ParseTextMode.VariableAnnotation,
|
||
(stringNode.d.strings[0].d.token.flags & StringTokenFlags.Triplicate) !== 0 ? 1 : 0,
|
||
this._typingSymbolAliases
|
||
);
|
||
|
||
if (
|
||
parseResults.diagnostics.length === 0 ||
|
||
this._parseOptions.reportErrorsForParsedStringContents
|
||
) {
|
||
parseResults.diagnostics.forEach((diag) => {
|
||
this._addSyntaxError(diag.message, stringNode);
|
||
});
|
||
|
||
if (parseResults.parseTree) {
|
||
stringNode.d.annotation = parseResults.parseTree;
|
||
stringNode.d.annotation.parent = stringNode;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
return stringNode;
|
||
}
|
||
|
||
// Python 3.8 added support for star (unpack) expressions in tuples
|
||
// following a return or yield statement in cases where the tuple
|
||
// wasn't surrounded in parentheses.
|
||
private _reportConditionalErrorForStarTupleElement(
|
||
possibleTupleExpr: ExpressionNode,
|
||
pythonVersion = pythonVersion3_8
|
||
) {
|
||
if (possibleTupleExpr.nodeType !== ParseNodeType.Tuple) {
|
||
return;
|
||
}
|
||
|
||
if (possibleTupleExpr.d.hasParens) {
|
||
return;
|
||
}
|
||
|
||
if (PythonVersion.isGreaterOrEqualTo(this._parseOptions.pythonVersion, pythonVersion)) {
|
||
return;
|
||
}
|
||
|
||
for (const expr of possibleTupleExpr.d.items) {
|
||
if (expr.nodeType === ParseNodeType.Unpack) {
|
||
this._addSyntaxError(LocMessage.unpackTuplesIllegal(), expr);
|
||
return;
|
||
}
|
||
}
|
||
}
|
||
|
||
// Peeks at the next token and returns true if it can never
|
||
// represent the start of an expression.
|
||
private _isNextTokenNeverExpression(): boolean {
|
||
const nextToken = this._peekToken();
|
||
switch (nextToken.type) {
|
||
case TokenType.Keyword: {
|
||
switch (this._peekKeywordType()) {
|
||
case KeywordType.For:
|
||
case KeywordType.In:
|
||
case KeywordType.If:
|
||
return true;
|
||
}
|
||
break;
|
||
}
|
||
|
||
case TokenType.Operator: {
|
||
switch (this._peekOperatorType()) {
|
||
case OperatorType.AddEqual:
|
||
case OperatorType.SubtractEqual:
|
||
case OperatorType.MultiplyEqual:
|
||
case OperatorType.DivideEqual:
|
||
case OperatorType.ModEqual:
|
||
case OperatorType.BitwiseAndEqual:
|
||
case OperatorType.BitwiseOrEqual:
|
||
case OperatorType.BitwiseXorEqual:
|
||
case OperatorType.LeftShiftEqual:
|
||
case OperatorType.RightShiftEqual:
|
||
case OperatorType.PowerEqual:
|
||
case OperatorType.FloorDivideEqual:
|
||
case OperatorType.Assign:
|
||
return true;
|
||
}
|
||
break;
|
||
}
|
||
|
||
case TokenType.Indent:
|
||
case TokenType.Dedent:
|
||
case TokenType.NewLine:
|
||
case TokenType.EndOfStream:
|
||
case TokenType.Semicolon:
|
||
case TokenType.CloseParenthesis:
|
||
case TokenType.CloseBracket:
|
||
case TokenType.CloseCurlyBrace:
|
||
case TokenType.Comma:
|
||
case TokenType.Colon:
|
||
case TokenType.ExclamationMark:
|
||
case TokenType.FStringMiddle:
|
||
case TokenType.FStringEnd:
|
||
return true;
|
||
}
|
||
|
||
return false;
|
||
}
|
||
|
||
private _disallowAssignmentExpression(callback: () => void) {
|
||
const wasAllowed = this._assignmentExpressionsAllowed;
|
||
this._assignmentExpressionsAllowed = false;
|
||
|
||
callback();
|
||
|
||
this._assignmentExpressionsAllowed = wasAllowed;
|
||
}
|
||
|
||
private _getNextToken(): Token {
|
||
const token = this._tokens!.getItemAt(this._tokenIndex);
|
||
if (!this._atEof()) {
|
||
this._tokenIndex++;
|
||
}
|
||
|
||
return token;
|
||
}
|
||
|
||
private _atEof(): boolean {
|
||
// Are we pointing at the last token in the stream (which is
|
||
// assumed to be an end-of-stream token)?
|
||
return this._tokenIndex >= this._tokenCount - 1;
|
||
}
|
||
|
||
private _peekToken(count = 0): Token {
|
||
const targetIndex = this._tokenIndex + count;
|
||
if (targetIndex < 0) {
|
||
return this._tokens!.getItemAt(0);
|
||
}
|
||
|
||
if (targetIndex >= this._tokenCount) {
|
||
return this._tokens!.getItemAt(this._tokenCount - 1);
|
||
}
|
||
|
||
return this._tokens!.getItemAt(targetIndex);
|
||
}
|
||
|
||
private _peekTokenType(): TokenType {
|
||
return this._peekToken().type;
|
||
}
|
||
|
||
private _peekKeywordType(): KeywordType | undefined {
|
||
const nextToken = this._peekToken();
|
||
if (nextToken.type !== TokenType.Keyword) {
|
||
return undefined;
|
||
}
|
||
|
||
return (nextToken as KeywordToken).keywordType;
|
||
}
|
||
|
||
private _peekOperatorType(): OperatorType | undefined {
|
||
const nextToken = this._peekToken();
|
||
if (nextToken.type !== TokenType.Operator) {
|
||
return undefined;
|
||
}
|
||
|
||
return (nextToken as OperatorToken).operatorType;
|
||
}
|
||
|
||
private _getTokenIfIdentifier(): IdentifierToken | undefined {
|
||
const nextToken = this._peekToken();
|
||
if (nextToken.type === TokenType.Identifier) {
|
||
return this._getNextToken() as IdentifierToken;
|
||
}
|
||
|
||
// If the next token is invalid, treat it as an identifier.
|
||
if (nextToken.type === TokenType.Invalid) {
|
||
this._getNextToken();
|
||
this._addSyntaxError(LocMessage.invalidIdentifierChar(), nextToken);
|
||
return IdentifierToken.create(nextToken.start, nextToken.length, '', nextToken.comments);
|
||
}
|
||
|
||
// If this is a "soft keyword", it can be converted into an identifier.
|
||
if (nextToken.type === TokenType.Keyword) {
|
||
const keywordToken = nextToken as KeywordToken;
|
||
if (KeywordToken.isSoftKeyword(keywordToken)) {
|
||
const keywordText = this._fileContents!.substr(nextToken.start, nextToken.length);
|
||
this._getNextToken();
|
||
return IdentifierToken.create(nextToken.start, nextToken.length, keywordText, nextToken.comments);
|
||
}
|
||
}
|
||
|
||
return undefined;
|
||
}
|
||
|
||
// Consumes tokens until the next one in the stream is
|
||
// either a specified terminator or the end-of-stream
|
||
// token.
|
||
private _consumeTokensUntilType(terminators: TokenType[]): boolean {
|
||
while (true) {
|
||
const token = this._peekToken();
|
||
if (terminators.some((term) => term === token.type)) {
|
||
return true;
|
||
}
|
||
|
||
if (token.type === TokenType.EndOfStream) {
|
||
return false;
|
||
}
|
||
|
||
this._getNextToken();
|
||
}
|
||
}
|
||
|
||
private _getTokenIfType(tokenType: TokenType): Token | undefined {
|
||
if (this._peekTokenType() === tokenType) {
|
||
return this._getNextToken();
|
||
}
|
||
|
||
return undefined;
|
||
}
|
||
|
||
private _consumeTokenIfType(tokenType: TokenType): boolean {
|
||
return !!this._getTokenIfType(tokenType);
|
||
}
|
||
|
||
private _consumeTokenIfKeyword(keywordType: KeywordType): boolean {
|
||
if (this._peekKeywordType() === keywordType) {
|
||
this._getNextToken();
|
||
return true;
|
||
}
|
||
|
||
return false;
|
||
}
|
||
|
||
private _consumeTokenIfOperator(operatorType: OperatorType): boolean {
|
||
if (this._peekOperatorType() === operatorType) {
|
||
this._getNextToken();
|
||
return true;
|
||
}
|
||
|
||
return false;
|
||
}
|
||
|
||
private _getKeywordToken(keywordType: KeywordType): KeywordToken {
|
||
const keywordToken = this._getNextToken() as KeywordToken;
|
||
assert(keywordToken.type === TokenType.Keyword);
|
||
assert(keywordToken.keywordType === keywordType);
|
||
return keywordToken;
|
||
}
|
||
|
||
private _getLanguageVersion() {
|
||
return this._parseOptions.pythonVersion;
|
||
}
|
||
|
||
private _suppressErrors(callback: () => void) {
|
||
const errorsWereSuppressed = this._areErrorsSuppressed;
|
||
try {
|
||
this._areErrorsSuppressed = true;
|
||
callback();
|
||
} finally {
|
||
this._areErrorsSuppressed = errorsWereSuppressed;
|
||
}
|
||
}
|
||
|
||
private _addSyntaxError(message: string, range: TextRange) {
|
||
assert(range !== undefined);
|
||
|
||
if (!this._areErrorsSuppressed) {
|
||
this._diagSink.addError(
|
||
message,
|
||
convertOffsetsToRange(range.start, range.start + range.length, this._tokenizerOutput!.lines)
|
||
);
|
||
}
|
||
}
|
||
}
|