TermiNexus
aef18c2891
Validation / Typecheck (push) Has been cancelled
Validation / Style (push) Has been cancelled
Validation / Test macos-latest (push) Has been cancelled
Validation / Test ubuntu-latest (push) Has been cancelled
Validation / Test windows-latest (push) Has been cancelled
Validation / Build (push) Has been cancelled
Validation / Required (push) Has been cancelled
1609 lines
66 KiB
TypeScript
1609 lines
66 KiB
TypeScript
/*
|
|
* sourceFile.ts
|
|
* Copyright (c) Microsoft Corporation.
|
|
* Licensed under the MIT license.
|
|
* Author: Eric Traut
|
|
*
|
|
* Class that represents a single Python source or stub file.
|
|
*/
|
|
|
|
import { isMainThread } from 'worker_threads';
|
|
|
|
import { OperationCanceledException } from '../common/cancellationUtils';
|
|
import { appendArray } from '../common/collectionUtils';
|
|
import {
|
|
ConfigOptions,
|
|
DiagnosticRuleSet,
|
|
ExecutionEnvironment,
|
|
getBasicDiagnosticRuleSet,
|
|
} from '../common/configOptions';
|
|
import { ConsoleInterface, StandardConsole } from '../common/console';
|
|
import { assert } from '../common/debug';
|
|
import { Diagnostic, DiagnosticCategory, TaskListToken, convertLevelToCategory } from '../common/diagnostic';
|
|
import { DiagnosticRule } from '../common/diagnosticRules';
|
|
import { DiagnosticSink, TextRangeDiagnosticSink } from '../common/diagnosticSink';
|
|
import { FileSystem } from '../common/fileSystem';
|
|
import { LogTracker, getPathForLogging } from '../common/logTracker';
|
|
import { stripFileExtension } from '../common/pathUtils';
|
|
import { convertOffsetsToRange, convertTextRangeToRange } from '../common/positionUtils';
|
|
import { ServiceKeys } from '../common/serviceKeys';
|
|
import { ServiceProvider } from '../common/serviceProvider';
|
|
import '../common/serviceProviderExtensions';
|
|
import * as StringUtils from '../common/stringUtils';
|
|
import { Range, TextRange, getEmptyRange } from '../common/textRange';
|
|
import { TextRangeCollection } from '../common/textRangeCollection';
|
|
import { Duration, timingStats } from '../common/timing';
|
|
import { Uri } from '../common/uri/uri';
|
|
import { LocMessage } from '../localization/localize';
|
|
import { ModuleNode } from '../parser/parseNodes';
|
|
import { ModuleImport, ParseFileResults, ParseOptions, Parser, ParserOutput } from '../parser/parser';
|
|
import { IgnoreComment, Tokenizer, TokenizerOutput } from '../parser/tokenizer';
|
|
import { Token } from '../parser/tokenizerTypes';
|
|
import { AnalyzerFileInfo, ImportLookup } from './analyzerFileInfo';
|
|
import * as AnalyzerNodeInfo from './analyzerNodeInfo';
|
|
import { Binder } from './binder';
|
|
import { CellChainIndexProvider } from './cellChainIndex';
|
|
import { Checker } from './checker';
|
|
import { CircularDependency } from './circularDependency';
|
|
import * as CommentUtils from './commentUtils';
|
|
import { ImportResolver } from './importResolver';
|
|
import { ImportResult } from './importResult';
|
|
import { ParseTreeCleanerWalker } from './parseTreeCleaner';
|
|
import { Scope } from './scope';
|
|
import { SymbolTable } from './symbol';
|
|
import { TestWalker } from './testWalker';
|
|
import { TypeEvaluator } from './typeEvaluatorTypes';
|
|
|
|
// Limit the number of import cycles tracked per source file.
|
|
const _maxImportCyclesPerFile = 4;
|
|
|
|
// Allow files up to 50MB in length, same as VS Code.
|
|
// https://github.com/microsoft/vscode/blob/1e750a7514f365585d8dab1a7a82e0938481ea2f/src/vs/editor/common/model/textModel.ts#L194
|
|
export const maxSourceFileSize = 50 * 1024 * 1024;
|
|
|
|
interface ResolveImportResult {
|
|
imports: ImportResult[];
|
|
builtinsImportResult?: ImportResult | undefined;
|
|
}
|
|
|
|
// Indicates whether IPython syntax is supported and if so, what
|
|
// type of notebook support is in use.
|
|
export enum IPythonMode {
|
|
// Not a notebook. This is the only falsy enum value, so you
|
|
// can test if IPython is supported via "if (ipythonMode)"
|
|
None = 0,
|
|
// Each cell is its own document.
|
|
CellDocs,
|
|
}
|
|
|
|
// A monotonically increasing number used to create unique file IDs.
|
|
let nextUniqueFileId = 1;
|
|
|
|
class WriteableData {
|
|
// Number that is incremented every time the diagnostics
|
|
// are updated.
|
|
diagnosticVersion = 0;
|
|
|
|
// Generation count of the file contents. When the contents
|
|
// change, this is incremented.
|
|
fileContentsVersion = 0;
|
|
|
|
// Number that is incremented every time semantic of the file
|
|
// might have changed.
|
|
semanticVersion = 0;
|
|
|
|
// Length and hash of the file the last time it was read from disk.
|
|
lastFileContentLength: number | undefined = undefined;
|
|
lastFileContentHash: number | undefined = undefined;
|
|
|
|
// Client's version of the file. Undefined implies that contents
|
|
// need to be read from disk.
|
|
clientDocumentContents: string | undefined;
|
|
clientDocumentVersion: number | undefined;
|
|
|
|
// Version of file contents that have been analyzed.
|
|
analyzedFileContentsVersion = -1;
|
|
|
|
// Do we need to walk the parse tree and clean
|
|
// the binder information hanging from it?
|
|
parseTreeNeedsCleaning = false;
|
|
|
|
parsedFileContents: string | undefined;
|
|
tokenizerLines: TextRangeCollection<TextRange> | undefined;
|
|
tokenizerOutput: TokenizerOutput | undefined;
|
|
lineCount: number | undefined;
|
|
|
|
moduleSymbolTable: SymbolTable | undefined;
|
|
|
|
// Reentrancy check for binding and checking.
|
|
isBindingInProgress = false;
|
|
isCheckingInProgress = false;
|
|
|
|
// Diagnostics generated during different phases of analysis.
|
|
parseDiagnostics: Diagnostic[] = [];
|
|
commentDiagnostics: Diagnostic[] = [];
|
|
bindDiagnostics: Diagnostic[] = [];
|
|
checkerDiagnostics: Diagnostic[] = [];
|
|
taskListDiagnostics: Diagnostic[] = [];
|
|
typeIgnoreLines = new Map<number, IgnoreComment>();
|
|
typeIgnoreAll: IgnoreComment | undefined;
|
|
viperIgnoreLines = new Map<number, IgnoreComment>();
|
|
|
|
// Accumulated and filtered diagnostics that combines all of the
|
|
// above information. This needs to be recomputed any time the
|
|
// above change.
|
|
accumulatedDiagnostics: Diagnostic[] = [];
|
|
diagnosticsWithoutFileIgnore: Diagnostic[] = [];
|
|
|
|
// Circular dependencies that have been reported in this file.
|
|
circularDependencies: CircularDependency[] = [];
|
|
noCircularDependencyConfirmed = false;
|
|
|
|
// Did we hit the maximum import depth?
|
|
hitMaxImportDepth: number | undefined;
|
|
|
|
// Do we need to perform a binding step?
|
|
isBindingNeeded = true;
|
|
|
|
// Do we have valid diagnostic results from a checking pass?
|
|
isCheckingNeeded = true;
|
|
|
|
// Time (in ms) that the last check() call required for this file.
|
|
checkTime: number | undefined;
|
|
|
|
// Information about implicit and explicit imports from this file.
|
|
imports: ImportResult[] | undefined;
|
|
builtinsImport: ImportResult | undefined;
|
|
// True if the file appears to have been deleted.
|
|
isFileDeleted = false;
|
|
|
|
parserOutput: ParserOutput | undefined;
|
|
|
|
constructor() {
|
|
// Empty
|
|
}
|
|
|
|
debugPrint() {
|
|
return `WritableData:
|
|
diagnosticVersion=${this.diagnosticVersion},
|
|
noCircularDependencyConfirmed=${this.noCircularDependencyConfirmed},
|
|
isBindingNeeded=${this.isBindingNeeded},
|
|
isBindingInProgress=${this.isBindingInProgress},
|
|
isCheckingInProgress=${this.isCheckingInProgress},
|
|
isCheckingNeeded=${this.isCheckingNeeded},
|
|
isFileDeleted=${this.isFileDeleted},
|
|
hitMaxImportDepth=${this.hitMaxImportDepth},
|
|
parseTreeNeedsCleaning=${this.parseTreeNeedsCleaning},
|
|
fileContentsVersion=${this.fileContentsVersion},
|
|
analyzedFileContentsVersion=${this.analyzedFileContentsVersion},
|
|
clientDocumentVersion=${this.clientDocumentVersion},
|
|
lastFileContentLength=${this.lastFileContentLength},
|
|
lastFileContentHash=${this.lastFileContentHash},
|
|
typeIgnoreAll=${this.typeIgnoreAll},
|
|
imports=${this.imports?.length},
|
|
builtinsImport=${this.builtinsImport?.importName},
|
|
circularDependencies=${this.circularDependencies?.length},
|
|
parseDiagnostics=${this.parseDiagnostics?.length},
|
|
commentDiagnostics=${this.commentDiagnostics?.length},
|
|
bindDiagnostics=${this.bindDiagnostics?.length},
|
|
checkerDiagnostics=${this.checkerDiagnostics?.length},
|
|
taskListDiagnostics=${this.taskListDiagnostics?.length},
|
|
accumulatedDiagnostics=${this.accumulatedDiagnostics?.length},
|
|
typeIgnoreLines=${this.typeIgnoreLines?.size},
|
|
viperIgnoreLines=${this.viperIgnoreLines?.size},
|
|
checkTime=${this.checkTime},
|
|
clientDocumentContents=${this.clientDocumentContents?.length},
|
|
parseResults=${this.parserOutput?.parseTree.length},
|
|
semanticVersion=${this.semanticVersion}, `;
|
|
}
|
|
}
|
|
|
|
export interface SourceFileEditMode {
|
|
readonly isEditMode: boolean;
|
|
}
|
|
|
|
export class SourceFile {
|
|
// Console interface to use for debugging.
|
|
private _console: ConsoleInterface;
|
|
|
|
// Uri unique to this file within the workspace. May not represent
|
|
// a real file on disk.
|
|
private readonly _uri: Uri;
|
|
|
|
// A short string that is guaranteed to uniquely
|
|
// identify this file.
|
|
private readonly _fileId: string;
|
|
|
|
// Getter to lazily compute the module name from the file URI.
|
|
private _moduleNameGetter: (file: Uri) => string;
|
|
|
|
// Period-delimited import path for the module.
|
|
private _cachedModuleName: string | undefined;
|
|
|
|
// True if file is a type-hint (.pyi) file versus a python
|
|
// (.py) file.
|
|
private readonly _isStubFile: boolean;
|
|
|
|
// True if the file was imported as a third-party import.
|
|
private readonly _isThirdPartyImport: boolean;
|
|
|
|
// True if the file is the "typing.pyi" file, which needs
|
|
// special-case handling.
|
|
private readonly _isTypingStubFile: boolean;
|
|
|
|
// True if the file is the "typing_extensions.pyi" file, which needs
|
|
// special-case handling.
|
|
private readonly _isTypingExtensionsStubFile: boolean;
|
|
|
|
// True if the file is the "_typeshed.pyi" file, which needs special-
|
|
// case handling.
|
|
private readonly _isTypeshedStubFile: boolean;
|
|
|
|
// True if the file one of the other built-in stub files
|
|
// that require special-case handling: "collections.pyi",
|
|
// "dataclasses.pyi", "abc.pyi", "asyncio/coroutines.pyi".
|
|
private readonly _isBuiltInStubFile: boolean;
|
|
|
|
// True if the file is part of a package that contains a
|
|
// "py.typed" file.
|
|
private readonly _isThirdPartyPyTypedPresent: boolean;
|
|
|
|
private readonly _editMode: SourceFileEditMode;
|
|
|
|
// Settings that control which diagnostics should be output. The rules
|
|
// are initialized to the basic set. They should be updated after the
|
|
// the file is parsed.
|
|
private _diagnosticRuleSet = getBasicDiagnosticRuleSet();
|
|
|
|
// Indicate whether this file is for ipython or not.
|
|
private _ipythonMode = IPythonMode.None;
|
|
private _logTracker: LogTracker;
|
|
private _preEditData: WriteableData | undefined;
|
|
|
|
// Data that changes when the source file changes.
|
|
private _writableData: WriteableData;
|
|
|
|
readonly fileSystem: FileSystem;
|
|
|
|
constructor(
|
|
readonly serviceProvider: ServiceProvider,
|
|
uri: Uri,
|
|
moduleNameGetter: (file: Uri) => string,
|
|
isThirdPartyImport: boolean,
|
|
isThirdPartyPyTypedPresent: boolean,
|
|
editMode: SourceFileEditMode,
|
|
console?: ConsoleInterface,
|
|
logTracker?: LogTracker,
|
|
ipythonMode?: IPythonMode
|
|
) {
|
|
this.fileSystem = serviceProvider.get(ServiceKeys.fs);
|
|
this._console = console || new StandardConsole();
|
|
this._writableData = new WriteableData();
|
|
|
|
this._editMode = editMode;
|
|
this._uri = uri;
|
|
this._fileId = this._makeFileId(uri);
|
|
this._moduleNameGetter = moduleNameGetter;
|
|
this._isStubFile = uri.hasExtension('.vpi') || uri.hasExtension('.vpnci');
|
|
this._isThirdPartyImport = isThirdPartyImport;
|
|
this._isThirdPartyPyTypedPresent = isThirdPartyPyTypedPresent;
|
|
const fileName = uri.fileName;
|
|
this._isTypingStubFile =
|
|
this._isStubFile && (this._uri.pathEndsWith('stdlib/typing.vpi') || fileName === 'typing_extensions.vpi');
|
|
this._isTypingExtensionsStubFile = this._isStubFile && fileName === 'typing_extensions.vpi';
|
|
this._isTypeshedStubFile =
|
|
this._isStubFile &&
|
|
(this._uri.pathEndsWith('stdlib/_typeshed/__init__.vpi') ||
|
|
this._uri.pathEndsWith('stdlib/_typeshed/_type_checker_internals.vpi'));
|
|
|
|
this._isBuiltInStubFile = false;
|
|
if (this._isStubFile) {
|
|
if (
|
|
this._uri.pathEndsWith('stdlib/collections/__init__.vpi') ||
|
|
this._uri.pathEndsWith('stdlib/asyncio/futures.vpi') ||
|
|
this._uri.pathEndsWith('stdlib/asyncio/tasks.vpi') ||
|
|
this._uri.pathEndsWith('stdlib/builtins.vpi') ||
|
|
this._uri.pathEndsWith('stdlib/_importlib_modulespec.vpi') ||
|
|
this._uri.pathEndsWith('stdlib/dataclasses.vpi') ||
|
|
this._uri.pathEndsWith('stdlib/abc.vpi') ||
|
|
this._uri.pathEndsWith('stdlib/enum.vpi') ||
|
|
this._uri.pathEndsWith('stdlib/queue.vpi') ||
|
|
this._uri.pathEndsWith('stdlib/string/templatelib.vpi') ||
|
|
this._uri.pathEndsWith('stdlib/types.vpi') ||
|
|
this._uri.pathEndsWith('stdlib/warnings.vpi')
|
|
) {
|
|
this._isBuiltInStubFile = true;
|
|
}
|
|
}
|
|
|
|
// 'FG' or 'BG' based on current thread.
|
|
this._logTracker = logTracker ?? new LogTracker(console, isMainThread ? 'FG' : 'BG');
|
|
this._ipythonMode = ipythonMode ?? IPythonMode.None;
|
|
}
|
|
|
|
// Sets the initial diagnostic rule set from the execution environment's
|
|
// config-level overrides. This should be called immediately after
|
|
// construction so the file has the correct rules before parse/bind.
|
|
setInitialDiagnosticRuleSet(ruleSet: DiagnosticRuleSet) {
|
|
this._diagnosticRuleSet = { ...ruleSet };
|
|
}
|
|
|
|
getIPythonMode(): IPythonMode {
|
|
return this._ipythonMode;
|
|
}
|
|
|
|
getUri(): Uri {
|
|
return this._uri;
|
|
}
|
|
|
|
getModuleName(): string {
|
|
if (!this._cachedModuleName) {
|
|
// Call the module name getter. If it returns '' (which can happen if the file is not part
|
|
// of the project), fall back to the file name.)
|
|
return this._moduleNameGetter(this._uri) || stripFileExtension(this._uri.fileName);
|
|
}
|
|
|
|
return this._cachedModuleName;
|
|
}
|
|
|
|
clearCachedModuleName() {
|
|
this._cachedModuleName = undefined;
|
|
}
|
|
|
|
getDiagnosticVersion(): number {
|
|
return this._writableData.diagnosticVersion;
|
|
}
|
|
|
|
getParseDiagnostics(): Diagnostic[] {
|
|
return this._writableData.parseDiagnostics;
|
|
}
|
|
|
|
isStubFile() {
|
|
return this._isStubFile;
|
|
}
|
|
|
|
isTypingStubFile() {
|
|
return this._isTypingStubFile;
|
|
}
|
|
|
|
isTypeshedStubFile() {
|
|
return this._isTypeshedStubFile;
|
|
}
|
|
|
|
isBuiltInStubFile() {
|
|
return this._isBuiltInStubFile;
|
|
}
|
|
|
|
isThirdPartyPyTypedPresent() {
|
|
return this._isThirdPartyPyTypedPresent;
|
|
}
|
|
|
|
// Returns a list of cached diagnostics from the latest analysis job.
|
|
// If the prevVersion is specified, the method returns undefined if
|
|
// the diagnostics haven't changed.
|
|
getDiagnostics(options: ConfigOptions, prevDiagnosticVersion?: number): Diagnostic[] | undefined {
|
|
if (this._writableData.diagnosticVersion === prevDiagnosticVersion) {
|
|
return undefined;
|
|
}
|
|
|
|
return this._writableData.accumulatedDiagnostics;
|
|
}
|
|
|
|
getDiagnosticsWithoutFileIgnore(): Diagnostic[] {
|
|
return this._writableData.diagnosticsWithoutFileIgnore;
|
|
}
|
|
|
|
getImports(): ImportResult[] {
|
|
return this._writableData.imports || [];
|
|
}
|
|
|
|
getBuiltinsImport(): ImportResult | undefined {
|
|
return this._writableData.builtinsImport;
|
|
}
|
|
|
|
getModuleSymbolTable(): SymbolTable | undefined {
|
|
return this._writableData.moduleSymbolTable;
|
|
}
|
|
|
|
getCheckTime() {
|
|
return this._writableData.checkTime;
|
|
}
|
|
|
|
restore(): string | undefined {
|
|
// If we had an edit, return our text.
|
|
if (this._preEditData) {
|
|
const text = this._writableData.clientDocumentContents!;
|
|
this._writableData = this._preEditData;
|
|
this._preEditData = undefined;
|
|
|
|
return text;
|
|
}
|
|
|
|
return undefined;
|
|
}
|
|
|
|
// Indicates whether the contents of the file have changed since
|
|
// the last analysis was performed.
|
|
didContentsChangeOnDisk(): boolean {
|
|
// If this is an open file any content changes will be
|
|
// provided through the editor. We can assume contents
|
|
// didn't change without us knowing about them.
|
|
if (this._writableData.clientDocumentContents) {
|
|
return false;
|
|
}
|
|
|
|
// If the file was never read previously we can't tell if the file has changed or not so
|
|
// we'll assume that it has. Otherwise, we may fail to analyze a file that was changed.
|
|
if (this._writableData.lastFileContentLength === undefined) {
|
|
return true;
|
|
}
|
|
|
|
// Read in the latest file contents and see if the hash matches
|
|
// that of the previous contents.
|
|
try {
|
|
// Read the file's contents.
|
|
if (this.fileSystem.existsSync(this._uri)) {
|
|
const fileContents = this.fileSystem.readFileSync(this._uri, 'utf8');
|
|
|
|
if (fileContents.length !== this._writableData.lastFileContentLength) {
|
|
return true;
|
|
}
|
|
|
|
if (StringUtils.hashString(fileContents) !== this._writableData.lastFileContentHash) {
|
|
return true;
|
|
}
|
|
} else {
|
|
// No longer exists, so yes it has changed.
|
|
return true;
|
|
}
|
|
} catch (error) {
|
|
return true;
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
// Drop parse and binding info to save memory. It is used
|
|
// in cases where memory is low. When info is needed, the file
|
|
// will be re-parsed and rebound.
|
|
dropParseAndBindInfo(): void {
|
|
// If we are actively binding or checking this file, we can't
|
|
// safely drop parse and binding info.
|
|
if (this._writableData.isBindingInProgress || this._writableData.isCheckingInProgress) {
|
|
return;
|
|
}
|
|
|
|
this._fireFileDirtyEvent();
|
|
|
|
this._writableData.parserOutput = undefined;
|
|
this._writableData.tokenizerLines = undefined;
|
|
this._writableData.tokenizerOutput = undefined;
|
|
this._writableData.parsedFileContents = undefined;
|
|
this._writableData.moduleSymbolTable = undefined;
|
|
this._writableData.isBindingNeeded = true;
|
|
this._writableData.imports = [];
|
|
}
|
|
|
|
markDirty(): void {
|
|
this._writableData.fileContentsVersion++;
|
|
this._writableData.semanticVersion++;
|
|
this._writableData.noCircularDependencyConfirmed = false;
|
|
this._writableData.isCheckingNeeded = true;
|
|
this._writableData.isBindingNeeded = true;
|
|
this._writableData.moduleSymbolTable = undefined;
|
|
this._writableData.lineCount = undefined;
|
|
|
|
this._fireFileDirtyEvent();
|
|
}
|
|
|
|
markReanalysisRequired(forceRebinding: boolean): void {
|
|
// Keep the parse info, but reset the analysis to the beginning.
|
|
this._writableData.semanticVersion++;
|
|
this._writableData.isCheckingNeeded = true;
|
|
this._writableData.noCircularDependencyConfirmed = false;
|
|
|
|
// If the file contains a wildcard import or __all__ symbols,
|
|
// we need to rebind because a dependent import may have changed.
|
|
if (this._writableData.parserOutput) {
|
|
if (
|
|
this._writableData.parserOutput.containsWildcardImport ||
|
|
AnalyzerNodeInfo.getDunderAllInfo(this._writableData.parserOutput.parseTree) !== undefined ||
|
|
forceRebinding
|
|
) {
|
|
// We don't need to rebuild index data since wildcard
|
|
// won't affect user file indices. User file indices
|
|
// don't contain import alias info.
|
|
this._writableData.parseTreeNeedsCleaning = true;
|
|
this._writableData.isBindingNeeded = true;
|
|
this._writableData.moduleSymbolTable = undefined;
|
|
}
|
|
}
|
|
}
|
|
|
|
getFileContentsVersion() {
|
|
return this._writableData.fileContentsVersion;
|
|
}
|
|
|
|
getClientVersion() {
|
|
return this._writableData.clientDocumentVersion;
|
|
}
|
|
|
|
getSemanticVersion() {
|
|
return this._writableData.semanticVersion;
|
|
}
|
|
|
|
getRange() {
|
|
return { start: { line: 0, character: 0 }, end: { line: this._writableData.lineCount ?? 0, character: 0 } };
|
|
}
|
|
|
|
getOpenFileContents() {
|
|
return this._writableData.clientDocumentContents;
|
|
}
|
|
|
|
getFileContent(): string | undefined {
|
|
// Get current buffer content if the file is opened.
|
|
const openFileContent = this.getOpenFileContents();
|
|
if (openFileContent !== undefined) {
|
|
return openFileContent;
|
|
}
|
|
|
|
// Ensure that the content used here is identical to the content obtained from the parse results.
|
|
if (!this.isParseRequired() && this._writableData.parsedFileContents !== undefined) {
|
|
return this._writableData.parsedFileContents;
|
|
}
|
|
|
|
// Otherwise, get content from file system.
|
|
try {
|
|
// Check the file's length before attempting to read its full contents.
|
|
const fileStat = this.fileSystem.statSync(this._uri);
|
|
if (fileStat.size > maxSourceFileSize) {
|
|
this._console.error(
|
|
`File length of "${this._uri}" is ${fileStat.size} ` +
|
|
`which exceeds the maximum supported file size of ${maxSourceFileSize}`
|
|
);
|
|
throw new Error('File larger than max');
|
|
}
|
|
|
|
return this.fileSystem.readFileSync(this._uri, 'utf8');
|
|
} catch (error) {
|
|
return undefined;
|
|
}
|
|
}
|
|
|
|
setClientVersion(version: number | null, contents: string): void {
|
|
// Save pre edit state if in edit mode.
|
|
this._cachePreEditState();
|
|
|
|
if (version === null) {
|
|
this._writableData.clientDocumentVersion = undefined;
|
|
this._writableData.clientDocumentContents = undefined;
|
|
|
|
// Since the file is no longer open, dump the tokenizer output
|
|
// so it doesn't consume memory.
|
|
this._writableData.tokenizerOutput = undefined;
|
|
} else {
|
|
this._writableData.clientDocumentVersion = version;
|
|
this._writableData.clientDocumentContents = contents;
|
|
|
|
const contentsHash = StringUtils.hashString(contents);
|
|
|
|
// Have the contents of the file changed?
|
|
if (
|
|
contents.length !== this._writableData.lastFileContentLength ||
|
|
contentsHash !== this._writableData.lastFileContentHash
|
|
) {
|
|
this.markDirty();
|
|
}
|
|
|
|
this._writableData.lastFileContentLength = contents.length;
|
|
this._writableData.lastFileContentHash = contentsHash;
|
|
this._writableData.isFileDeleted = false;
|
|
}
|
|
}
|
|
|
|
prepareForClose() {
|
|
this._fireFileDirtyEvent();
|
|
}
|
|
|
|
isFileDeleted() {
|
|
return this._writableData.isFileDeleted;
|
|
}
|
|
|
|
isParseRequired() {
|
|
return (
|
|
!this._writableData.parserOutput ||
|
|
this._writableData.analyzedFileContentsVersion !== this._writableData.fileContentsVersion
|
|
);
|
|
}
|
|
|
|
isBindingRequired() {
|
|
if (this._writableData.isBindingInProgress) {
|
|
return false;
|
|
}
|
|
|
|
if (this.isParseRequired()) {
|
|
return true;
|
|
}
|
|
|
|
return this._writableData.isBindingNeeded;
|
|
}
|
|
|
|
isCheckingRequired() {
|
|
return this._writableData.isCheckingNeeded;
|
|
}
|
|
|
|
getParseResults(): ParseFileResults | undefined {
|
|
if (this.isParseRequired()) {
|
|
return undefined;
|
|
}
|
|
|
|
assert(this._writableData.parserOutput !== undefined && this._writableData.parsedFileContents !== undefined);
|
|
|
|
// If we've cached the tokenizer output, use the cached version.
|
|
// Otherwise re-tokenize the contents on demand.
|
|
const tokenizeContents = this._tokenizeContents.bind(this);
|
|
const parsedFileContents = this._writableData.parsedFileContents;
|
|
const contentHash =
|
|
this._writableData.lastFileContentHash || StringUtils.hashString(this._writableData.parsedFileContents);
|
|
let tokenizerOutput: TokenizerOutput | undefined = this._writableData.tokenizerOutput;
|
|
|
|
return {
|
|
contentHash,
|
|
parserOutput: this._writableData.parserOutput,
|
|
get tokenizerOutput(): TokenizerOutput {
|
|
// Lazily tokenize the file contents only when accessed for the first time.
|
|
if (!tokenizerOutput) {
|
|
tokenizerOutput = tokenizeContents(parsedFileContents, contentHash);
|
|
}
|
|
return tokenizerOutput!;
|
|
},
|
|
text: this._writableData.parsedFileContents,
|
|
};
|
|
}
|
|
|
|
getParserOutput(): ParserOutput | undefined {
|
|
if (this.isParseRequired()) {
|
|
return undefined;
|
|
}
|
|
|
|
assert(this._writableData.parserOutput !== undefined);
|
|
|
|
return this._writableData.parserOutput;
|
|
}
|
|
|
|
// Adds a new circular dependency for this file but only if
|
|
// it hasn't already been added.
|
|
addCircularDependency(configOptions: ConfigOptions, circDependency: CircularDependency) {
|
|
let updatedDependencyList = false;
|
|
|
|
// Some topologies can result in a massive number of cycles. We'll cut it off.
|
|
if (this._writableData.circularDependencies.length < _maxImportCyclesPerFile) {
|
|
if (!this._writableData.circularDependencies.some((dep) => dep.isEqual(circDependency))) {
|
|
this._writableData.circularDependencies.push(circDependency);
|
|
updatedDependencyList = true;
|
|
}
|
|
}
|
|
|
|
if (updatedDependencyList) {
|
|
this._recomputeDiagnostics(configOptions);
|
|
}
|
|
}
|
|
|
|
setNoCircularDependencyConfirmed() {
|
|
this._writableData.noCircularDependencyConfirmed = true;
|
|
}
|
|
|
|
isNoCircularDependencyConfirmed() {
|
|
return !this.isParseRequired() && this._writableData.noCircularDependencyConfirmed;
|
|
}
|
|
|
|
setHitMaxImportDepth(maxImportDepth: number) {
|
|
this._writableData.hitMaxImportDepth = maxImportDepth;
|
|
}
|
|
|
|
// Parse the file and update the state. Callers should wait for completion
|
|
// (or at least cancel) prior to calling again. It returns true if a parse
|
|
// was required and false if the parse information was up to date already.
|
|
parse(configOptions: ConfigOptions, importResolver: ImportResolver, content?: string): boolean {
|
|
return this._logTracker.log(`parsing: ${this._getPathForLogging(this._uri)}`, (logState) => {
|
|
// If the file is already parsed, we can skip.
|
|
if (!this.isParseRequired()) {
|
|
logState.suppress();
|
|
return false;
|
|
}
|
|
|
|
const diagSink = this.createDiagnosticSink();
|
|
let fileContents = this.getOpenFileContents();
|
|
if (fileContents === undefined) {
|
|
try {
|
|
const startTime = timingStats.readFileTime.totalTime;
|
|
timingStats.readFileTime.timeOperation(() => {
|
|
// Read the file's contents.
|
|
fileContents = content ?? this.getFileContent();
|
|
if (fileContents === undefined) {
|
|
throw new Error("Can't get file content");
|
|
}
|
|
|
|
// Remember the length and hash for comparison purposes.
|
|
this._writableData.lastFileContentLength = fileContents.length;
|
|
this._writableData.lastFileContentHash = StringUtils.hashString(fileContents);
|
|
});
|
|
logState.add(`fs read ${timingStats.readFileTime.totalTime - startTime}ms`);
|
|
} catch (error) {
|
|
diagSink.addError(`Source file could not be read`, getEmptyRange());
|
|
fileContents = '';
|
|
|
|
if (!this.fileSystem.existsSync(this._uri)) {
|
|
this._writableData.isFileDeleted = true;
|
|
}
|
|
}
|
|
}
|
|
|
|
try {
|
|
// Parse the token stream, building the abstract syntax tree.
|
|
const parseFileResults = this._parseFile(
|
|
configOptions,
|
|
this._uri,
|
|
fileContents!,
|
|
this._ipythonMode !== IPythonMode.None,
|
|
diagSink
|
|
);
|
|
|
|
assert(parseFileResults !== undefined && parseFileResults.tokenizerOutput !== undefined);
|
|
this._writableData.parserOutput = parseFileResults.parserOutput;
|
|
this._writableData.tokenizerLines = parseFileResults.tokenizerOutput.lines;
|
|
this._writableData.parsedFileContents = fileContents;
|
|
this._writableData.typeIgnoreLines = parseFileResults.tokenizerOutput.typeIgnoreLines;
|
|
this._writableData.typeIgnoreAll = parseFileResults.tokenizerOutput.typeIgnoreAll;
|
|
this._writableData.viperIgnoreLines = parseFileResults.tokenizerOutput.viperIgnoreLines;
|
|
this._writableData.lineCount = parseFileResults.tokenizerOutput.lines.length;
|
|
|
|
// Cache the tokenizer output only if this file is open.
|
|
if (this._writableData.clientDocumentContents !== undefined) {
|
|
this._writableData.tokenizerOutput = parseFileResults.tokenizerOutput;
|
|
}
|
|
|
|
// Resolve imports.
|
|
const execEnvironment = configOptions.findExecEnvironment(this._uri);
|
|
timingStats.resolveImportsTime.timeOperation(() => {
|
|
const importResult = this._resolveImports(
|
|
importResolver,
|
|
parseFileResults.parserOutput.importedModules,
|
|
execEnvironment
|
|
);
|
|
|
|
this._writableData.imports = importResult.imports;
|
|
this._writableData.builtinsImport = importResult.builtinsImportResult;
|
|
|
|
this._writableData.parseDiagnostics = diagSink.fetchAndClear();
|
|
|
|
this._writableData.taskListDiagnostics = [];
|
|
this._addTaskListDiagnostics(
|
|
configOptions.taskListTokens,
|
|
parseFileResults,
|
|
this._writableData.taskListDiagnostics
|
|
);
|
|
});
|
|
|
|
// Is this file in a "strict" path?
|
|
const useStrict =
|
|
configOptions.strict.find((strictFileSpec) => this._uri.matchesRegex(strictFileSpec.regExp)) !==
|
|
undefined;
|
|
|
|
const commentDiags: CommentUtils.CommentDiagnostic[] = [];
|
|
this._diagnosticRuleSet = CommentUtils.getFileLevelDirectives(
|
|
parseFileResults.tokenizerOutput.tokens,
|
|
parseFileResults.tokenizerOutput.lines,
|
|
execEnvironment.diagnosticRuleSet,
|
|
useStrict,
|
|
commentDiags
|
|
);
|
|
|
|
this._writableData.commentDiagnostics = [];
|
|
|
|
commentDiags.forEach((commentDiag) => {
|
|
this._writableData.commentDiagnostics.push(
|
|
new Diagnostic(
|
|
DiagnosticCategory.Error,
|
|
commentDiag.message,
|
|
convertTextRangeToRange(commentDiag.range, parseFileResults.tokenizerOutput.lines)
|
|
)
|
|
);
|
|
});
|
|
} catch (e: any) {
|
|
const message: string =
|
|
(e.stack ? e.stack.toString() : undefined) ||
|
|
(typeof e.message === 'string' ? e.message : undefined) ||
|
|
JSON.stringify(e);
|
|
this._console.error(
|
|
LocMessage.internalParseError().format({
|
|
file: this.getUri().toUserVisibleString(),
|
|
message,
|
|
})
|
|
);
|
|
|
|
// Create dummy parse results.
|
|
this._writableData.parsedFileContents = '';
|
|
this._writableData.tokenizerLines = new TextRangeCollection<TextRange>([]);
|
|
|
|
this._writableData.parserOutput = {
|
|
parseTree: ModuleNode.create({ start: 0, length: 0 }),
|
|
importedModules: [],
|
|
futureImports: new Set<string>(),
|
|
containsWildcardImport: false,
|
|
typingSymbolAliases: new Map<string, string>(),
|
|
hasTypeAnnotations: false,
|
|
lines: this._writableData.tokenizerLines,
|
|
};
|
|
|
|
this._writableData.tokenizerOutput = {
|
|
tokens: new TextRangeCollection<Token>([]),
|
|
lines: this._writableData.tokenizerLines,
|
|
typeIgnoreAll: undefined,
|
|
typeIgnoreLines: new Map<number, IgnoreComment>(),
|
|
viperIgnoreLines: new Map<number, IgnoreComment>(),
|
|
predominantEndOfLineSequence: '\n',
|
|
hasPredominantTabSequence: false,
|
|
predominantTabSequence: ' ',
|
|
predominantSingleQuoteCharacter: "'",
|
|
};
|
|
|
|
this._writableData.imports = undefined;
|
|
this._writableData.builtinsImport = undefined;
|
|
|
|
const diagSink = this.createDiagnosticSink();
|
|
diagSink.addError(
|
|
LocMessage.internalParseError().format({
|
|
file: this.getUri().toUserVisibleString(),
|
|
message,
|
|
}),
|
|
getEmptyRange()
|
|
);
|
|
this._writableData.parseDiagnostics = diagSink.fetchAndClear();
|
|
this._writableData.taskListDiagnostics = diagSink.fetchAndClear();
|
|
|
|
// Do not rethrow the exception, swallow it here. Callers are not
|
|
// prepared to handle an exception.
|
|
}
|
|
|
|
this._writableData.analyzedFileContentsVersion = this._writableData.fileContentsVersion;
|
|
this._writableData.isBindingNeeded = true;
|
|
this._writableData.isCheckingNeeded = true;
|
|
this._writableData.parseTreeNeedsCleaning = false;
|
|
this._writableData.hitMaxImportDepth = undefined;
|
|
|
|
this._recomputeDiagnostics(configOptions);
|
|
|
|
return true;
|
|
});
|
|
}
|
|
|
|
bind(
|
|
configOptions: ConfigOptions,
|
|
importLookup: ImportLookup,
|
|
builtinsScope: Scope | undefined,
|
|
futureImports: Set<string>,
|
|
cellChainIndex: CellChainIndexProvider | undefined
|
|
) {
|
|
assert(!this.isParseRequired(), 'Bind called before parsing');
|
|
assert(this.isBindingRequired(), 'Bind called unnecessarily');
|
|
assert(!this._writableData.isBindingInProgress, 'Bind called while binding in progress');
|
|
assert(this._writableData.parserOutput !== undefined, 'Parse results not available');
|
|
|
|
return this._logTracker.log(`binding: ${this._getPathForLogging(this._uri)}`, () => {
|
|
try {
|
|
// Perform name binding.
|
|
timingStats.bindTime.timeOperation(() => {
|
|
this._cleanParseTreeIfRequired();
|
|
|
|
const fileInfo = this._buildFileInfo(configOptions, importLookup, builtinsScope, futureImports);
|
|
AnalyzerNodeInfo.setFileInfo(this._writableData.parserOutput!.parseTree, fileInfo);
|
|
|
|
const binder = new Binder(fileInfo, configOptions.indexGenerationMode, cellChainIndex);
|
|
this._writableData.isBindingInProgress = true;
|
|
binder.bindModule(this._writableData.parserOutput!.parseTree);
|
|
|
|
// If we're in "test mode" (used for unit testing), run an additional
|
|
// "test walker" over the parse tree to validate its internal consistency.
|
|
if (configOptions.internalTestMode) {
|
|
const testWalker = new TestWalker();
|
|
testWalker.walk(this._writableData.parserOutput!.parseTree);
|
|
}
|
|
|
|
this._writableData.bindDiagnostics = fileInfo.diagnosticSink.fetchAndClear();
|
|
const moduleScope = AnalyzerNodeInfo.getScope(this._writableData.parserOutput!.parseTree);
|
|
assert(moduleScope !== undefined, 'Module scope not returned by binder');
|
|
this._writableData.moduleSymbolTable = moduleScope!.symbolTable;
|
|
});
|
|
} catch (e: any) {
|
|
const message: string =
|
|
(e.stack ? e.stack.toString() : undefined) ||
|
|
(typeof e.message === 'string' ? e.message : undefined) ||
|
|
JSON.stringify(e);
|
|
this._console.error(
|
|
LocMessage.internalBindError().format({
|
|
file: this.getUri().toUserVisibleString(),
|
|
message,
|
|
})
|
|
);
|
|
|
|
const diagSink = this.createDiagnosticSink();
|
|
diagSink.addError(
|
|
LocMessage.internalBindError().format({
|
|
file: this.getUri().toUserVisibleString(),
|
|
message,
|
|
}),
|
|
getEmptyRange()
|
|
);
|
|
this._writableData.bindDiagnostics = diagSink.fetchAndClear();
|
|
|
|
// Do not rethrow the exception, swallow it here. Callers are not
|
|
// prepared to handle an exception.
|
|
} finally {
|
|
this._writableData.isBindingInProgress = false;
|
|
}
|
|
|
|
// Prepare for the next stage of the analysis.
|
|
this._writableData.isCheckingNeeded = true;
|
|
this._writableData.isBindingNeeded = false;
|
|
|
|
this._recomputeDiagnostics(configOptions);
|
|
});
|
|
}
|
|
|
|
check(
|
|
configOptions: ConfigOptions,
|
|
importLookup: ImportLookup,
|
|
importResolver: ImportResolver,
|
|
evaluator: TypeEvaluator,
|
|
dependentFiles?: ParserOutput[]
|
|
) {
|
|
assert(!this.isParseRequired(), `Check called before parsing: state=${this._writableData.debugPrint()}`);
|
|
assert(!this.isBindingRequired(), `Check called before binding: state=${this._writableData.debugPrint()}`);
|
|
assert(!this._writableData.isBindingInProgress, 'Check called while binding in progress');
|
|
assert(!this._writableData.isCheckingInProgress, 'Check called while checking in progress');
|
|
assert(this.isCheckingRequired(), 'Check called unnecessarily');
|
|
assert(this._writableData.parserOutput !== undefined, 'Parse results not available');
|
|
|
|
return this._logTracker.log(`checking: ${this._getPathForLogging(this._uri)}`, () => {
|
|
try {
|
|
timingStats.typeCheckerTime.timeOperation(() => {
|
|
const checkDuration = new Duration();
|
|
const checker = new Checker(
|
|
importResolver,
|
|
evaluator,
|
|
this._writableData.parserOutput!,
|
|
dependentFiles
|
|
);
|
|
this._writableData.isCheckingInProgress = true;
|
|
checker.check();
|
|
this._writableData.isCheckingNeeded = false;
|
|
|
|
const fileInfo = AnalyzerNodeInfo.getFileInfo(this._writableData.parserOutput!.parseTree)!;
|
|
this._writableData.checkerDiagnostics = fileInfo.diagnosticSink.fetchAndClear();
|
|
this._writableData.checkTime = checkDuration.getDurationInMilliseconds();
|
|
});
|
|
} catch (e: any) {
|
|
const isCancellation = OperationCanceledException.is(e);
|
|
if (!isCancellation) {
|
|
const message: string =
|
|
(e.stack ? e.stack.toString() : undefined) ||
|
|
(typeof e.message === 'string' ? e.message : undefined) ||
|
|
JSON.stringify(e);
|
|
this._console.error(
|
|
LocMessage.internalTypeCheckingError().format({
|
|
file: this.getUri().toUserVisibleString(),
|
|
message,
|
|
})
|
|
);
|
|
const diagSink = this.createDiagnosticSink();
|
|
diagSink.addError(
|
|
LocMessage.internalTypeCheckingError().format({
|
|
file: this.getUri().toUserVisibleString(),
|
|
message,
|
|
}),
|
|
getEmptyRange()
|
|
);
|
|
|
|
this._writableData.checkerDiagnostics = diagSink.fetchAndClear();
|
|
|
|
// Mark the file as complete so we don't get into an infinite loop.
|
|
this._writableData.isCheckingNeeded = false;
|
|
}
|
|
|
|
throw e;
|
|
} finally {
|
|
this._writableData.isCheckingInProgress = false;
|
|
|
|
// Clear any circular dependencies associated with this file.
|
|
// These will be detected by the program module and associated
|
|
// with the source file right before it is finalized.
|
|
this._writableData.circularDependencies = [];
|
|
|
|
this._recomputeDiagnostics(configOptions);
|
|
}
|
|
});
|
|
}
|
|
|
|
test_enableIPythonMode(enable: boolean) {
|
|
this._ipythonMode = enable ? IPythonMode.CellDocs : IPythonMode.None;
|
|
}
|
|
|
|
protected createDiagnosticSink(): DiagnosticSink {
|
|
return new DiagnosticSink();
|
|
}
|
|
|
|
protected createTextRangeDiagnosticSink(lines: TextRangeCollection<TextRange>): TextRangeDiagnosticSink {
|
|
return new TextRangeDiagnosticSink(lines);
|
|
}
|
|
|
|
// Creates a short string that can be used to uniquely identify
|
|
// this file from all other files. It is used in the type evaluator
|
|
// to distinguish between types that are defined in different files
|
|
// or scopes.
|
|
private _makeFileId(uri: Uri) {
|
|
const maxNameLength = 8;
|
|
|
|
// Use a small portion of the file name to help with debugging.
|
|
let fileName = uri.fileNameWithoutExtensions;
|
|
if (fileName.length > maxNameLength) {
|
|
fileName = fileName.substring(fileName.length - maxNameLength);
|
|
}
|
|
|
|
// Append a number to guarantee uniqueness.
|
|
const uniqueNumber = nextUniqueFileId++;
|
|
|
|
// Use a "/" to separate the two components, since this
|
|
// character will never appear in a file name.
|
|
return `${fileName}/${uniqueNumber.toString()}`;
|
|
}
|
|
|
|
// Computes an updated set of accumulated diagnostics for the file
|
|
// based on the partial diagnostics from various analysis stages.
|
|
private _recomputeDiagnostics(configOptions: ConfigOptions) {
|
|
this._writableData.diagnosticVersion++;
|
|
|
|
let includeWarningsAndErrors = true;
|
|
|
|
// If a file was imported as a third-party file, don't report
|
|
// any errors for it. The user can't fix them anyway.
|
|
if (this._isThirdPartyImport) {
|
|
includeWarningsAndErrors = false;
|
|
}
|
|
|
|
let diagList: Diagnostic[] = [];
|
|
appendArray(diagList, this._writableData.parseDiagnostics);
|
|
appendArray(diagList, this._writableData.commentDiagnostics);
|
|
appendArray(diagList, this._writableData.bindDiagnostics);
|
|
appendArray(diagList, this._writableData.checkerDiagnostics);
|
|
appendArray(diagList, this._writableData.taskListDiagnostics);
|
|
|
|
const prefilteredDiagList = diagList;
|
|
const typeIgnoreLinesClone = new Map(this._writableData.typeIgnoreLines);
|
|
const viperIgnoreLinesClone = new Map(this._writableData.viperIgnoreLines);
|
|
|
|
// Filter the diagnostics based on "type: ignore" lines.
|
|
if (this._diagnosticRuleSet.enableTypeIgnoreComments) {
|
|
if (this._writableData.typeIgnoreLines.size > 0) {
|
|
diagList = diagList.filter((d) => {
|
|
if (
|
|
d.category !== DiagnosticCategory.UnusedCode &&
|
|
d.category !== DiagnosticCategory.UnreachableCode &&
|
|
d.category !== DiagnosticCategory.Deprecated
|
|
) {
|
|
for (let line = d.range.start.line; line <= d.range.end.line; line++) {
|
|
if (this._writableData.typeIgnoreLines.has(line)) {
|
|
typeIgnoreLinesClone.delete(line);
|
|
return false;
|
|
}
|
|
}
|
|
}
|
|
|
|
return true;
|
|
});
|
|
}
|
|
}
|
|
|
|
// Filter the diagnostics based on "viper: ignore" lines.
|
|
if (this._writableData.viperIgnoreLines.size > 0) {
|
|
diagList = diagList.filter((d) => {
|
|
if (
|
|
d.category !== DiagnosticCategory.UnusedCode &&
|
|
d.category !== DiagnosticCategory.UnreachableCode &&
|
|
d.category !== DiagnosticCategory.Deprecated
|
|
) {
|
|
for (let line = d.range.start.line; line <= d.range.end.line; line++) {
|
|
const viperIgnoreComment = this._writableData.viperIgnoreLines.get(line);
|
|
if (viperIgnoreComment) {
|
|
if (!viperIgnoreComment.rulesList) {
|
|
viperIgnoreLinesClone.delete(line);
|
|
return false;
|
|
}
|
|
|
|
const diagRule = d.getRule();
|
|
if (!diagRule) {
|
|
// If there's no diagnostic rule, it won't match
|
|
// against a rules list.
|
|
return true;
|
|
}
|
|
|
|
// Did we find this rule in the list?
|
|
if (viperIgnoreComment.rulesList.find((rule) => rule.text === diagRule)) {
|
|
// Update the viperIgnoreLinesClone to remove this rule.
|
|
const oldClone = viperIgnoreLinesClone.get(line);
|
|
if (oldClone?.rulesList) {
|
|
const filteredRulesList = oldClone.rulesList.filter(
|
|
(rule) => rule.text !== diagRule
|
|
);
|
|
if (filteredRulesList.length === 0) {
|
|
viperIgnoreLinesClone.delete(line);
|
|
} else {
|
|
viperIgnoreLinesClone.set(line, {
|
|
range: oldClone.range,
|
|
rulesList: filteredRulesList,
|
|
});
|
|
}
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
return true;
|
|
}
|
|
}
|
|
}
|
|
|
|
return true;
|
|
});
|
|
}
|
|
|
|
const unnecessaryTypeIgnoreDiags: Diagnostic[] = [];
|
|
|
|
// Skip this step if type checking is needed. Otherwise we'll likely produce
|
|
// incorrect (false positive) reportUnnecessaryTypeIgnoreComment diagnostics
|
|
// until checking is performed on this file.
|
|
if (
|
|
this._diagnosticRuleSet.reportUnnecessaryTypeIgnoreComment !== 'none' &&
|
|
!this._writableData.isCheckingNeeded
|
|
) {
|
|
const diagCategory = convertLevelToCategory(this._diagnosticRuleSet.reportUnnecessaryTypeIgnoreComment);
|
|
|
|
const prefilteredErrorList = prefilteredDiagList.filter(
|
|
(diag) =>
|
|
diag.category === DiagnosticCategory.Error ||
|
|
diag.category === DiagnosticCategory.Warning ||
|
|
diag.category === DiagnosticCategory.Information
|
|
);
|
|
|
|
const isUnreachableCodeRange = (range: Range) => {
|
|
return prefilteredDiagList.find(
|
|
(diag) =>
|
|
diag.category === DiagnosticCategory.UnreachableCode &&
|
|
diag.range.start.line <= range.start.line &&
|
|
diag.range.end.line >= range.end.line
|
|
);
|
|
};
|
|
|
|
if (prefilteredErrorList.length === 0 && this._writableData.typeIgnoreAll !== undefined) {
|
|
const rangeStart = this._writableData.typeIgnoreAll.range.start;
|
|
const rangeEnd = rangeStart + this._writableData.typeIgnoreAll.range.length;
|
|
const range = convertOffsetsToRange(rangeStart, rangeEnd, this._writableData.tokenizerLines!);
|
|
|
|
if (!isUnreachableCodeRange(range) && this._diagnosticRuleSet.enableTypeIgnoreComments) {
|
|
const diag = new Diagnostic(diagCategory, LocMessage.unnecessaryTypeIgnore(), range);
|
|
diag.setRule(DiagnosticRule.reportUnnecessaryTypeIgnoreComment);
|
|
unnecessaryTypeIgnoreDiags.push(diag);
|
|
}
|
|
}
|
|
|
|
typeIgnoreLinesClone.forEach((ignoreComment) => {
|
|
if (this._writableData.tokenizerLines!) {
|
|
const rangeStart = ignoreComment.range.start;
|
|
const rangeEnd = rangeStart + ignoreComment.range.length;
|
|
const range = convertOffsetsToRange(rangeStart, rangeEnd, this._writableData.tokenizerLines!);
|
|
|
|
if (!isUnreachableCodeRange(range) && this._diagnosticRuleSet.enableTypeIgnoreComments) {
|
|
const diag = new Diagnostic(diagCategory, LocMessage.unnecessaryTypeIgnore(), range);
|
|
diag.setRule(DiagnosticRule.reportUnnecessaryTypeIgnoreComment);
|
|
unnecessaryTypeIgnoreDiags.push(diag);
|
|
}
|
|
}
|
|
});
|
|
|
|
viperIgnoreLinesClone.forEach((ignoreComment) => {
|
|
if (this._writableData.tokenizerLines!) {
|
|
if (!ignoreComment.rulesList) {
|
|
const rangeStart = ignoreComment.range.start;
|
|
const rangeEnd = rangeStart + ignoreComment.range.length;
|
|
const range = convertOffsetsToRange(rangeStart, rangeEnd, this._writableData.tokenizerLines!);
|
|
|
|
if (!isUnreachableCodeRange(range)) {
|
|
const diag = new Diagnostic(diagCategory, LocMessage.unnecessaryTypeIgnore(), range);
|
|
diag.setRule(DiagnosticRule.reportUnnecessaryTypeIgnoreComment);
|
|
unnecessaryTypeIgnoreDiags.push(diag);
|
|
}
|
|
} else {
|
|
ignoreComment.rulesList.forEach((unusedRule) => {
|
|
const rangeStart = unusedRule.range.start;
|
|
const rangeEnd = rangeStart + unusedRule.range.length;
|
|
const range = convertOffsetsToRange(
|
|
rangeStart,
|
|
rangeEnd,
|
|
this._writableData.tokenizerLines!
|
|
);
|
|
|
|
if (!isUnreachableCodeRange(range)) {
|
|
const diag = new Diagnostic(
|
|
diagCategory,
|
|
LocMessage.unnecessaryViperIgnoreRule().format({
|
|
name: unusedRule.text,
|
|
}),
|
|
range
|
|
);
|
|
diag.setRule(DiagnosticRule.reportUnnecessaryTypeIgnoreComment);
|
|
unnecessaryTypeIgnoreDiags.push(diag);
|
|
}
|
|
});
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
if (
|
|
this._diagnosticRuleSet.reportImportCycles !== 'none' &&
|
|
this._writableData.circularDependencies.length > 0
|
|
) {
|
|
const category = convertLevelToCategory(this._diagnosticRuleSet.reportImportCycles);
|
|
|
|
this._writableData.circularDependencies.forEach((cirDep) => {
|
|
const diag = new Diagnostic(
|
|
category,
|
|
LocMessage.importCycleDetected() +
|
|
'\n' +
|
|
cirDep
|
|
.getPaths()
|
|
.map((path) => ' ' + path.toUserVisibleString())
|
|
.join('\n'),
|
|
getEmptyRange()
|
|
);
|
|
diag.setRule(DiagnosticRule.reportImportCycles);
|
|
diagList.push(diag);
|
|
});
|
|
}
|
|
|
|
if (this._writableData.hitMaxImportDepth !== undefined) {
|
|
diagList.push(
|
|
new Diagnostic(
|
|
DiagnosticCategory.Error,
|
|
LocMessage.importDepthExceeded().format({ depth: this._writableData.hitMaxImportDepth }),
|
|
getEmptyRange()
|
|
)
|
|
);
|
|
}
|
|
|
|
// If there is a "type: ignore" comment at the top of the file, clear
|
|
// the diagnostic list of all error, warning, and information diagnostics.
|
|
if (this._diagnosticRuleSet.enableTypeIgnoreComments) {
|
|
if (this._writableData.typeIgnoreAll !== undefined) {
|
|
diagList = diagList.filter(
|
|
(diag) =>
|
|
diag.category !== DiagnosticCategory.Error &&
|
|
diag.category !== DiagnosticCategory.Warning &&
|
|
diag.category !== DiagnosticCategory.Information
|
|
);
|
|
}
|
|
}
|
|
|
|
// Now add in the "unnecessary type ignore" diagnostics.
|
|
diagList = diagList.concat(unnecessaryTypeIgnoreDiags);
|
|
|
|
// If we're not returning any diagnostics, filter out all of
|
|
// the errors and warnings, leaving only the unreachable code
|
|
// and deprecated diagnostics.
|
|
if (!includeWarningsAndErrors) {
|
|
diagList = diagList.filter(
|
|
(diag) =>
|
|
diag.category === DiagnosticCategory.UnusedCode ||
|
|
diag.category === DiagnosticCategory.UnreachableCode ||
|
|
diag.category === DiagnosticCategory.Deprecated
|
|
);
|
|
}
|
|
|
|
// Capture the fully-filtered diagnostics before any file-level ignore clears them, so
|
|
// consumers that need the unsuppressed list (e.g. ignored-file quick fixes) can access it.
|
|
// For non-ignored files this is the same array reference as accumulatedDiagnostics below.
|
|
this._writableData.diagnosticsWithoutFileIgnore = diagList;
|
|
|
|
// If the file is in the ignore list, clear the diagnostic list.
|
|
if (configOptions.ignore.find((ignoreFileSpec) => this._uri.matchesRegex(ignoreFileSpec.regExp))) {
|
|
diagList = [];
|
|
}
|
|
|
|
this._writableData.accumulatedDiagnostics = diagList;
|
|
}
|
|
|
|
private _cachePreEditState() {
|
|
// If this is our first write, then make a copy of the writable data.
|
|
if (!this._editMode.isEditMode || this._preEditData) {
|
|
return;
|
|
}
|
|
|
|
// Copy over the writable data.
|
|
this._preEditData = this._writableData;
|
|
|
|
// Recreate all the writable data from scratch.
|
|
this._writableData = new WriteableData();
|
|
}
|
|
|
|
// Get all task list diagnostics for the current file and add them
|
|
// to the specified diagnostic list.
|
|
private _addTaskListDiagnostics(
|
|
taskListTokens: TaskListToken[] | undefined,
|
|
parseFileResults: ParseFileResults,
|
|
diagList: Diagnostic[]
|
|
) {
|
|
if (!taskListTokens || taskListTokens.length === 0 || !diagList) {
|
|
return;
|
|
}
|
|
|
|
const tokenizerOutput = parseFileResults.tokenizerOutput;
|
|
const fileContents = parseFileResults.text;
|
|
|
|
for (let i = 0; i < tokenizerOutput.tokens.count; i++) {
|
|
const token = tokenizerOutput.tokens.getItemAt(i);
|
|
|
|
// If there are no comments, skip this token.
|
|
if (!token.comments || token.comments.length === 0) {
|
|
continue;
|
|
}
|
|
|
|
for (const comment of token.comments) {
|
|
for (const taskToken of taskListTokens) {
|
|
// Match: optional leading whitespace, then taskToken.text (case-insensitive),
|
|
// then either (whitespace to end) or (non-alphanumeric char).
|
|
const commentStart = comment.start;
|
|
const commentEnd = commentStart + comment.length;
|
|
const taskText = taskToken.text;
|
|
const taskLen = taskText.length;
|
|
|
|
// Skip leading whitespace within the source text range.
|
|
let pos = commentStart;
|
|
while (pos < commentEnd) {
|
|
const ch = fileContents.charCodeAt(pos);
|
|
if (ch === 0x20 || ch === 0x09 || ch === 0x0a || ch === 0x0d || ch === 0x0c || ch === 0x0b) {
|
|
pos++;
|
|
} else {
|
|
break;
|
|
}
|
|
}
|
|
|
|
// Check if the task token text matches (case-insensitive).
|
|
if (pos + taskLen > commentEnd) {
|
|
continue;
|
|
}
|
|
|
|
let matched = true;
|
|
for (let k = 0; k < taskLen; k++) {
|
|
const a = fileContents.charCodeAt(pos + k);
|
|
const b = taskText.charCodeAt(k);
|
|
if (a !== b && (a | 0x20) !== (b | 0x20)) {
|
|
matched = false;
|
|
break;
|
|
}
|
|
}
|
|
if (!matched) {
|
|
continue;
|
|
}
|
|
|
|
// After the token, require whitespace-to-end or a non-word character.
|
|
const afterPos = pos + taskLen;
|
|
if (afterPos < commentEnd) {
|
|
const ch = fileContents.charCodeAt(afterPos);
|
|
// Check if ch is a word character [a-zA-Z0-9_]
|
|
const isWord =
|
|
(ch >= 0x61 && ch <= 0x7a) ||
|
|
(ch >= 0x41 && ch <= 0x5a) ||
|
|
(ch >= 0x30 && ch <= 0x39) ||
|
|
ch === 0x5f;
|
|
if (isWord) {
|
|
continue;
|
|
}
|
|
}
|
|
|
|
// Match succeeded. pos is the offset of the task token in the source text.
|
|
const rangeEnd = TextRange.getEnd(comment);
|
|
const range = convertOffsetsToRange(pos, rangeEnd, tokenizerOutput.lines!);
|
|
|
|
const commentValue = comment.value;
|
|
diagList.push(
|
|
new Diagnostic(DiagnosticCategory.TaskItem, commentValue.trim(), range, taskToken.priority)
|
|
);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
private _buildFileInfo(
|
|
configOptions: ConfigOptions,
|
|
importLookup: ImportLookup,
|
|
builtinsScope: Scope | undefined,
|
|
futureImports: Set<string>
|
|
) {
|
|
assert(this._writableData.parserOutput !== undefined, 'Parse results not available');
|
|
const analysisDiagnostics = this.createTextRangeDiagnosticSink(this._writableData.tokenizerLines!);
|
|
|
|
const fileInfo: AnalyzerFileInfo = {
|
|
importLookup,
|
|
futureImports,
|
|
builtinsScope,
|
|
diagnosticSink: analysisDiagnostics,
|
|
executionEnvironment: configOptions.findExecEnvironment(this._uri),
|
|
diagnosticRuleSet: this._diagnosticRuleSet,
|
|
lines: this._writableData.tokenizerLines!,
|
|
typingSymbolAliases: this._writableData.parserOutput!.typingSymbolAliases,
|
|
definedConstants: configOptions.defineConstant,
|
|
fileId: this._fileId,
|
|
fileUri: this._uri,
|
|
moduleName: this.getModuleName(),
|
|
isStubFile: this._isStubFile,
|
|
isTypingStubFile: this._isTypingStubFile,
|
|
isTypingExtensionsStubFile: this._isTypingExtensionsStubFile,
|
|
isTypeshedStubFile: this._isTypeshedStubFile,
|
|
isBuiltInStubFile: this._isBuiltInStubFile,
|
|
isInPyTypedPackage: this._isThirdPartyPyTypedPresent,
|
|
ipythonMode: this._ipythonMode,
|
|
accessedSymbolSet: new Set<number>(),
|
|
};
|
|
return fileInfo;
|
|
}
|
|
|
|
private _cleanParseTreeIfRequired() {
|
|
if (this._writableData.parserOutput) {
|
|
if (this._writableData.parseTreeNeedsCleaning) {
|
|
const cleanerWalker = new ParseTreeCleanerWalker(this._writableData.parserOutput.parseTree);
|
|
cleanerWalker.clean();
|
|
this._writableData.parseTreeNeedsCleaning = false;
|
|
}
|
|
}
|
|
}
|
|
|
|
private _resolveImports(
|
|
importResolver: ImportResolver,
|
|
moduleImports: ModuleImport[],
|
|
execEnv: ExecutionEnvironment
|
|
): ResolveImportResult {
|
|
const imports: ImportResult[] = [];
|
|
|
|
const resolveAndAddIfNotSelf = (nameParts: string[], skipMissingImport = false) => {
|
|
const importResult = importResolver.resolveImport(this._uri, execEnv, {
|
|
leadingDots: 0,
|
|
nameParts,
|
|
importedSymbols: undefined,
|
|
});
|
|
|
|
if (skipMissingImport && !importResult.isImportFound) {
|
|
return undefined;
|
|
}
|
|
|
|
// Avoid importing module from the module file itself.
|
|
if (importResult.resolvedUris.length === 0 || importResult.resolvedUris[0] !== this._uri) {
|
|
imports.push(importResult);
|
|
return importResult;
|
|
}
|
|
|
|
return undefined;
|
|
};
|
|
|
|
// Always include an implicit import of the builtins module.
|
|
let builtinsImportResult: ImportResult | undefined;
|
|
|
|
// If this is a project source file (not a stub), try to resolve
|
|
// the __builtins__ stub first.
|
|
if (!this._isThirdPartyImport && !this._isStubFile) {
|
|
builtinsImportResult = resolveAndAddIfNotSelf(['__builtins__'], /* skipMissingImport */ true);
|
|
}
|
|
|
|
if (!builtinsImportResult) {
|
|
builtinsImportResult = resolveAndAddIfNotSelf(['builtins']);
|
|
}
|
|
|
|
resolveAndAddIfNotSelf(['_typeshed', '_type_checker_internals'], /* skipMissingImport */ true);
|
|
|
|
for (const moduleImport of moduleImports) {
|
|
const importResult = importResolver.resolveImport(this._uri, execEnv, {
|
|
leadingDots: moduleImport.leadingDots,
|
|
nameParts: moduleImport.nameParts,
|
|
importedSymbols: moduleImport.importedSymbols,
|
|
});
|
|
|
|
imports.push(importResult);
|
|
|
|
// Associate the import results with the module import
|
|
// name node in the parse tree so we can access it later
|
|
// (for hover and definition support).
|
|
if (moduleImport.nameParts.length === moduleImport.nameNode.d.nameParts.length) {
|
|
AnalyzerNodeInfo.setImportInfo(moduleImport.nameNode, importResult);
|
|
} else {
|
|
// For implicit imports of higher-level modules within a multi-part
|
|
// module name, the moduleImport.nameParts will refer to the subset
|
|
// of the multi-part name rather than the full multi-part name. In this
|
|
// case, store the import info on the name part node.
|
|
assert(moduleImport.nameParts.length > 0);
|
|
assert(moduleImport.nameParts.length - 1 < moduleImport.nameNode.d.nameParts.length);
|
|
AnalyzerNodeInfo.setImportInfo(
|
|
moduleImport.nameNode.d.nameParts[moduleImport.nameParts.length - 1],
|
|
importResult
|
|
);
|
|
}
|
|
}
|
|
|
|
return {
|
|
imports,
|
|
builtinsImportResult,
|
|
};
|
|
}
|
|
|
|
private _getPathForLogging(fileUri: Uri) {
|
|
return getPathForLogging(this.fileSystem, fileUri);
|
|
}
|
|
|
|
private _parseFile(
|
|
configOptions: ConfigOptions,
|
|
fileUri: Uri,
|
|
fileContents: string,
|
|
useNotebookMode: boolean,
|
|
diagSink: DiagnosticSink
|
|
): ParseFileResults {
|
|
// Use the configuration options to determine the environment zin which
|
|
// this source file will be executed.
|
|
const execEnvironment = configOptions.findExecEnvironment(fileUri);
|
|
|
|
const parseOptions = new ParseOptions();
|
|
parseOptions.useNotebookMode = useNotebookMode;
|
|
if (fileUri.pathEndsWith('vpi')) {
|
|
parseOptions.isStubFile = true;
|
|
}
|
|
parseOptions.pythonVersion = execEnvironment.pythonVersion;
|
|
parseOptions.dialect = execEnvironment.dialect;
|
|
parseOptions.skipFunctionAndClassBody = configOptions.indexGenerationMode ?? false;
|
|
|
|
// Parse the token stream, building the abstract syntax tree.
|
|
const parser = new Parser();
|
|
return parser.parseSourceFile(fileContents, parseOptions, diagSink);
|
|
}
|
|
|
|
private _tokenizeContents(fileContents: string, contentHash: number): TokenizerOutput {
|
|
const tokenizer = new Tokenizer();
|
|
const output = tokenizer.tokenize(fileContents);
|
|
|
|
// When the file is open, cache the tokenizer results.
|
|
// Because the tokenizer is lazy, ensure that the state remains unchanged before caching its output.
|
|
if (
|
|
this._writableData.clientDocumentContents !== undefined &&
|
|
this._writableData.lastFileContentHash === contentHash
|
|
) {
|
|
this._writableData.tokenizerOutput = output;
|
|
|
|
// Replace the existing tokenizerLines with the newly-returned
|
|
// version. They should have the same contents, but we want to use
|
|
// the same object so the older object can be deallocated.
|
|
this._writableData.tokenizerLines = output.lines;
|
|
}
|
|
|
|
return output;
|
|
}
|
|
|
|
private _fireFileDirtyEvent() {
|
|
this.serviceProvider.tryGet(ServiceKeys.stateMutationListeners)?.forEach((l) => {
|
|
try {
|
|
l.onFileDirty?.(this._uri);
|
|
} catch (ex: any) {
|
|
const console = this.serviceProvider.tryGet(ServiceKeys.console);
|
|
if (console) {
|
|
console.error(`State mutation listener exception: ${ex.message}`);
|
|
}
|
|
}
|
|
});
|
|
}
|
|
}
|