Viper_VSCode/packages/vipeye-internal/src/analyzer/sourceEnumerator.ts

/*
 * sourceEnumerator.ts
 * Copyright (c) Microsoft Corporation.
 * Licensed under the MIT license.
 *
 * Logic for enumerating all of the Python source files in
 * a project.
 */

import { ConsoleInterface } from '../common/console';
import { FileSystem } from '../common/fileSystem';
import { Uri } from '../common/uri/uri';
import { FileSpec, getFileSystemEntriesWithSymlinkedDirectories, tryRealpath, tryStat } from '../common/uri/uriUtils';

export interface SourceEnumerateResult {
    matches: Map<string, Uri>;
    autoExcludedDirs: Uri[];
    isComplete: boolean;
}

const envMarkers = [['bin', 'activate'], ['Scripts', 'activate'], ['pyvenv.cfg'], ['conda-meta']];

interface DirToExplore {
    uri: Uri;
    includeRegExp: RegExp;
    hasDirectoryWildcard: boolean;
}

export class SourceEnumerator {
    private _elapsedTimeInMs = 0;
    private _includesToExplore: FileSpec[];
    private _dirsToExplore: DirToExplore[] = [];
    private _matches = new Map<string, Uri>();
    private _autoExcludeDirs: Uri[] = [];
    private _isComplete = false;
    private _numFilesVisited = 0;
    private _loggedLongOperationError = false;
    private _seenDirs = new Set<string>();
    // This tracks symlinked directory roots across the entire enumeration cycle,
    // potentially spanning multiple include roots, so Pylance can later filter
    // workspace indexing against the full discovered set.
    private readonly _symlinkedDirectoryRoots = new Map<string, Uri>();

    constructor(
        include: FileSpec[],
        private _excludes: FileSpec[],
        private _autoExcludeVenv: boolean,
        private _fs: FileSystem,
        private _console: ConsoleInterface
    ) {
        this._includesToExplore = include.slice(0).reverse();

        this._console.log(`Searching for source files`);
    }

    getSymlinkedDirectoryRoots(): Uri[] {
        return Array.from(this._symlinkedDirectoryRoots.values());
    }

    // Enumerates as many files as possible within the specified
    // time limit and returns all matching files.
    enumerate(timeLimitInMs: number): SourceEnumerateResult {
        const startTime = Date.now();

        while (!this._isComplete) {
            if (this._doNext()) {
                if (!this._isComplete) {
                    this._finish();
                }
            }

            const elapsedTime = Date.now() - startTime;
            if (timeLimitInMs > 0 && elapsedTime > timeLimitInMs) {
                break;
            }
        }

        this._elapsedTimeInMs += Date.now() - startTime;

        if (!this._loggedLongOperationError) {
            const longOperationLimitInMs = 10000;
            const nFilesToSuggestSubfolder = 50;

            // If this is taking a long time, log an error to help the user
            // diagnose and mitigate the problem.
            if (this._elapsedTimeInMs >= longOperationLimitInMs && this._numFilesVisited >= nFilesToSuggestSubfolder) {
                this._console.error(
                    `Enumeration of workspace source files is taking longer than ${
                        longOperationLimitInMs * 0.001
                    } seconds.\n` +
                        'This may be because:\n' +
                        '* You have opened your home directory or entire hard drive as a workspace\n' +
                        '* Your workspace contains a very large number of directories and files\n' +
                        '* Your workspace contains a symlink to a directory with many files\n' +
                        '* Your workspace is remote, and file enumeration is slow\n' +
                        'To reduce this time, open a workspace directory with fewer files ' +
                        'or add a viperconfig.json configuration file with an "exclude" section to exclude ' +
                        'subdirectories from your workspace. For more details, refer to ' +
                        'https://github.com/TermiNexus/TransPyC/wiki/viper-configuration.'
                );

                this._loggedLongOperationError = true;
            }
        }

        return {
            matches: this._matches,
            autoExcludedDirs: this._autoExcludeDirs,
            isComplete: this._isComplete,
        };
    }

    private _recordSymlinkedDirectoryRoot(root: Uri): void {
        for (const existingRoot of this._symlinkedDirectoryRoots.values()) {
            if (root.isChild(existingRoot)) {
                return;
            }
        }

        for (const [key, existingRoot] of this._symlinkedDirectoryRoots.entries()) {
            if (existingRoot.isChild(root)) {
                this._symlinkedDirectoryRoots.delete(key);
            }
        }

        this._symlinkedDirectoryRoots.set(root.key, root);
    }

    // Performs the next enumeration action. Returns true if complete.
    private _doNext(): boolean {
        const dirToExplore = this._dirsToExplore.pop();
        if (dirToExplore) {
            this._exploreDir(dirToExplore);
            return false;
        }

        const includeToExplore = this._includesToExplore.pop();
        if (includeToExplore) {
            this._exploreInclude(includeToExplore);
            return false;
        }

        return true;
    }

    private _exploreDir(dir: DirToExplore) {
        const realDirPath = tryRealpath(this._fs, dir.uri);
        if (!realDirPath) {
            this._console.warn(`Skipping broken link "${dir.uri}"`);
            return;
        }

        if (realDirPath.key !== dir.uri.key) {
            this._recordSymlinkedDirectoryRoot(dir.uri);
        }

        if (this._seenDirs.has(realDirPath.key)) {
            this._console.info(`Skipping recursive symlink "${dir.uri}" -> "${realDirPath}"`);
            return;
        }
        this._seenDirs.add(realDirPath.key);

        if (this._autoExcludeVenv) {
            if (envMarkers.some((f) => this._fs.existsSync(dir.uri.resolvePaths(...f)))) {
                this._autoExcludeDirs.push(dir.uri);
                this._console.info(`Auto-excluding ${dir.uri.toUserVisibleString()}`);
                return;
            }
        }

        const { files, directories, symlinkedDirectories } = getFileSystemEntriesWithSymlinkedDirectories(
            this._fs,
            dir.uri
        );

        for (const symlinkedDir of symlinkedDirectories) {
            this._recordSymlinkedDirectoryRoot(symlinkedDir);
        }

        for (const file of files) {
            if (FileSpec.matchIncludeFileSpec(dir.includeRegExp, this._excludes, file)) {
                this._numFilesVisited++;
                this._matches.set(file.key, file);
            }
        }

        for (const subDir of directories.slice().reverse()) {
            if (subDir.matchesRegex(dir.includeRegExp) || dir.hasDirectoryWildcard) {
                if (!FileSpec.isInPath(subDir, this._excludes)) {
                    this._dirsToExplore.push({
                        uri: subDir,
                        includeRegExp: dir.includeRegExp,
                        hasDirectoryWildcard: dir.hasDirectoryWildcard,
                    });
                }
            }
        }
    }

    private _exploreInclude(includeSpec: FileSpec) {
        if (FileSpec.isInPath(includeSpec.wildcardRoot, this._excludes)) {
            return;
        }

        this._seenDirs.clear();

        // Skip enumeration for non-file URI schemes (e.g., memfs:, zowe-uss:).
        // These require async file system access that isn't available here.
        if (includeSpec.wildcardRoot.scheme !== 'file' && includeSpec.wildcardRoot.scheme !== '') {
            this._console.info(
                `Skipping file enumeration for non-file URI scheme "${includeSpec.wildcardRoot.scheme}".`
            );
            return;
        }

        const stat = tryStat(this._fs, includeSpec.wildcardRoot);
        if (stat?.isFile()) {
            this._matches.set(includeSpec.wildcardRoot.key, includeSpec.wildcardRoot);
        } else if (stat?.isDirectory()) {
            this._dirsToExplore.push({
                uri: includeSpec.wildcardRoot,
                includeRegExp: includeSpec.regExp,
                hasDirectoryWildcard: includeSpec.hasDirectoryWildcard,
            });
        } else {
            this._console.error(
                `File or directory "${includeSpec.wildcardRoot.toUserVisibleString()}" does not exist.`
            );
        }
    }

    private _finish() {
        this._isComplete = true;

        const fileCount = this._matches.size;
        if (fileCount === 0) {
            this._console.info(`No source files found.`);
        } else {
            this._console.info(`Found ${fileCount} ` + `source ${fileCount === 1 ? 'file' : 'files'}`);
        }
    }
}