vg-coder-cli 2.0.1 → 2.0.4

package/vg/packages/core/src/lib/scanner/file-scanner.ts

@@ -1,675 +0,0 @@
-import * as fs from 'fs-extra';
-import * as path from 'path';
-import { IgnoreManager } from '../ignore/ignore-manager';
-
-export interface ScannerOptions {
-    maxFileSize?: number;
-    extensions?: string[];
-    includeHidden?: boolean;
-    maxDepth?: number;
-    [key: string]: any;
-}
-
-export interface FileNode {
-    path: string;
-    name: string;
-    size: number;
-    type: 'directory' | 'file';
-    extension?: string;
-    children?: FileNode[];
-}
-
-export interface FileContent {
-    path: string;
-    relativePath: string;
-    name: string;
-    extension?: string;
-    size: number;
-    content: string;
-    lines: number;
-    encoding: string;
-}
-
-export interface ScanResult {
-    tree: FileNode;
-    files: FileContent[];
-    stats: {
-        totalFiles: number;
-        filteredFiles: number;
-        processedFiles: number;
-        scanTime: number;
-        projectPath: string;
-    };
-}
-
-/**
- * Scanner to traverse and concatenate source code files
- */
-export class FileScanner {
-    private options: ScannerOptions;
-    private ignoreManager: IgnoreManager;
-
-    constructor(private projectPath: string, options: ScannerOptions = {}) {
-        this.options = {
-            maxFileSize: options.maxFileSize || 1024 * 1024, // 1MB default
-            extensions: options.extensions || this.getDefaultExtensions(),
-            includeHidden: options.includeHidden || false,
-            maxDepth: options.maxDepth || 10,
-            ...options
-        };
-        this.ignoreManager = new IgnoreManager(projectPath);
-    }
-
-    /**
-     * Get list of default extensions
-     */
-    getDefaultExtensions(): string[] {
-        return [
-            // Web Frontend
-            '.js', '.jsx', '.ts', '.tsx', '.vue', '.svelte',
-            '.html', '.htm', '.css', '.scss', '.sass', '.less',
-
-            // Backend
-            '.java', '.kt', '.scala', '.groovy',
-            '.py', '.rb', '.php', '.go', '.rs',
-            '.cs', '.vb', '.fs',
-            '.cpp', '.c', '.h', '.hpp',
-
-            // Config files
-            '.json', '.yaml', '.yml', '.toml', '.ini', '.conf',
-            '.xml', '.properties',
-
-            // Build files
-            '.gradle', '.maven', '.sbt',
-
-            // Scripts
-            '.sh', '.bat', '.ps1', '.cmd',
-
-            // Documentation
-            '.md', '.txt', '.rst',
-
-            // SQL
-            '.sql', '.ddl', '.dml'
-        ];
-    }
-
-    /**
-     * Scan entire project and return file structure
-     */
-    async scanProject(): Promise<ScanResult> {
-        const startTime = Date.now();
-
-        try {
-            // Get directory structure
-            const tree = await this.buildFileTree();
-
-            // Extract files
-            const files = await this.extractFiles(tree);
-
-            // Filter files by ignore rules
-            const filteredFiles = await this.filterFiles(files);
-
-            // Read file contents
-            const filesWithContent = await this.readFilesContent(filteredFiles);
-
-            const endTime = Date.now();
-
-            return {
-                tree,
-                files: filesWithContent,
-                stats: {
-                    totalFiles: files.length,
-                    filteredFiles: filteredFiles.length,
-                    processedFiles: filesWithContent.length,
-                    scanTime: endTime - startTime,
-                    projectPath: this.projectPath
-                }
-            };
-        } catch (error: any) {
-            throw new Error(`Failed to scan project: ${error.message}`);
-        }
-    }
-
-    /**
-     * Build file tree
-     */
-    async buildFileTree(): Promise<FileNode> {
-        // Use manual scanning instead of directory-tree to ensure all Java files are found
-        return await this.scanDirectoryManually(this.projectPath);
-    }
-
-    /**
-     * Manually scan directory to ensure all files are found
-     */
-    async scanDirectoryManually(dirPath: string, relativePath = ''): Promise<FileNode> {
-        const stats = await fs.stat(dirPath);
-        const name = path.basename(dirPath);
-
-        const node: FileNode = {
-            path: dirPath,
-            name: name,
-            size: stats.size,
-            type: stats.isDirectory() ? 'directory' : 'file',
-            extension: stats.isFile() ? path.extname(name) : undefined
-        };
-
-        if (stats.isDirectory()) {
-            try {
-                const entries = await fs.readdir(dirPath);
-                const children: FileNode[] = [];
-
-                for (const entry of entries) {
-                    const entryPath = path.join(dirPath, entry);
-                    const entryRelativePath = path.join(relativePath, entry);
-
-                    // Skip ignored directories
-                    if (this.shouldIgnoreDirectoryName(entry)) {
-                        continue;
-                    }
-
-                    try {
-                        const childNode = await this.scanDirectoryManually(entryPath, entryRelativePath);
-                        if (childNode) {
-                            children.push(childNode);
-                        }
-                    } catch (error) {
-                        // Skip files/directories that can't be accessed
-                        continue;
-                    }
-                }
-
-                if (children.length > 0) {
-                    node.children = children;
-                }
-            } catch (error) {
-                // Skip directories that can't be read
-            }
-        }
-
-        return node;
-    }
-
-    /**
-     * Get regex for build directories and hidden files to ignore
-     */
-    getBuildIgnoreRegex(): RegExp {
-        // Create regex pattern for directories to ignore
-        const ignorePatterns = [
-            // Hidden directories (starting with .)
-            '^\\..*',
-
-            // Build directories
-            '^build$',
-            '^target$',
-            '^dist$',
-            '^out$',
-            '^bin$',
-
-            // Dependencies
-            '^node_modules$',
-            '^vendor$',
-
-            // Temporary files
-            '^tmp$',
-            '^temp$',
-
-            // Logs
-            '^logs$',
-            '^log$',
-
-            // Coverage reports
-            '^coverage$'
-        ];
-
-        // Combine all patterns into one regex
-        const combinedPattern = ignorePatterns.join('|');
-        return new RegExp(`(${combinedPattern})`);
-    }
-
-    /**
-     * Extract list of files from tree
-     */
-    async extractFiles(tree: FileNode): Promise<any[]> {
-        const files: any[] = [];
-
-        const traverse = (node: FileNode, currentPath = '') => {
-            // Skip ignored directories
-            if (this.shouldIgnorePath(node.path || node.name)) {
-                return;
-            }
-
-            // If node has children, it's a directory
-            if (node.children) {
-                node.children.forEach(child => traverse(child, currentPath));
-            } else {
-                // If no children, it's a file
-                const relativePath = path.relative(this.projectPath, node.path);
-
-                // Skip files in ignored directories
-                if (!this.shouldIgnorePath(relativePath)) {
-                    // Filter by extension
-                    const extensions = this.options.extensions || this.getDefaultExtensions();
-                    const hasValidExtension = extensions.some(ext =>
-                        node.name.toLowerCase().endsWith(ext.toLowerCase())
-                    );
-
-                    if (hasValidExtension) {
-                        files.push({
-                            path: node.path,
-                            relativePath: relativePath,
-                            name: node.name,
-                            extension: node.extension,
-                            size: node.size
-                        });
-                    }
-                }
-            }
-        };
-
-        traverse(tree);
-        return files;
-    }
-
-    /**
-     * Check if path should be ignored
-     */
-    shouldIgnorePath(filePath: string): boolean {
-        const pathParts = filePath.split(path.sep);
-
-        // Check each part of the path
-        for (const part of pathParts) {
-            if (this.shouldIgnoreDirectoryName(part)) {
-                return true;
-            }
-        }
-
-        return false;
-    }
-
-    /**
-     * Check if directory/file should be ignored
-     */
-    shouldIgnoreDirectoryName(name: string): boolean {
-        // List of directories/files to ignore
-        const ignoreList = [
-            // Hidden directories (starting with .)
-            '.git', '.svn', '.hg',
-            '.gradle', '.maven', '.cache', '.npm',
-            '.idea', '.vscode', '.eclipse',
-            '.DS_Store', '.tmp',
-
-            // Build directories
-            'build', 'target', 'dist', 'out', 'bin',
-
-            // Dependencies
-            'node_modules', 'vendor',
-
-            // Temporary files
-            'tmp', 'temp',
-
-            // Logs
-            'logs', 'log',
-
-            // Coverage reports
-            'coverage', '.nyc_output'
-        ];
-
-        // Check if name starts with . (hidden files/dirs)
-        if (name.startsWith('.')) {
-            return true;
-        }
-
-        // Check if name is in ignore list
-        return ignoreList.includes(name);
-    }
-
-    /**
-     * Filter files by ignore rules
-     */
-    async filterFiles(files: any[]): Promise<any[]> {
-        const filtered: any[] = [];
-
-        for (const file of files) {
-            // Check size
-            if (this.options.maxFileSize && file.size > this.options.maxFileSize) {
-                continue;
-            }
-
-            // Check ignore rules
-            const shouldIgnore = await this.ignoreManager.shouldIgnore(file.relativePath);
-            if (!shouldIgnore) {
-                filtered.push(file);
-            }
-        }
-
-        return filtered;
-    }
-
-    /**
-     * Read file contents
-     */
-    async readFilesContent(files: any[]): Promise<FileContent[]> {
-        const filesWithContent: FileContent[] = [];
-
-        for (const file of files) {
-            try {
-                const content = await this.readFileContent(file.path);
-                if (content !== null) {
-                    filesWithContent.push({
-                        ...file,
-                        content,
-                        lines: content.split('\n').length,
-                        encoding: 'utf8'
-                    });
-                }
-            } catch (error: any) {
-                // Log error but continue with other files
-                console.warn(`Warning: Could not read file ${file.relativePath}: ${error.message}`);
-            }
-        }
-
-        return filesWithContent;
-    }
-
-    /**
-     * Read content of a single file
-     */
-    async readFileContent(filePath: string): Promise<string | null> {
-        try {
-            // Check if file is binary
-            if (await this.isBinaryFile(filePath)) {
-                return null;
-            }
-
-            const content = await fs.readFile(filePath, 'utf8');
-            return content;
-        } catch (error: any) {
-            if (error.code === 'ENOENT') {
-                return null;
-            }
-            throw error;
-        }
-    }
-
-    /**
-     * Check if file is binary
-     */
-    async isBinaryFile(filePath: string): Promise<boolean> {
-        try {
-            const buffer = await fs.readFile(filePath);
-
-            // Check first 1024 bytes
-            const chunk = buffer.slice(0, 1024);
-
-            // If null bytes found, likely binary
-            for (let i = 0; i < chunk.length; i++) {
-                if (chunk[i] === 0) {
-                    return true;
-                }
-            }
-
-            return false;
-        } catch (error) {
-            return true; // Assume binary if can't read
-        }
-    }
-
-    /**
-     * Create combined content from all files
-     */
-    async createCombinedContent(files: FileContent[], options: any = {}): Promise<string> {
-        const {
-            includeStats = true,
-            includeTree = true,
-            headerTemplate = this.getDefaultHeaderTemplate(),
-            separatorTemplate = this.getDefaultSeparatorTemplate()
-        } = options;
-
-        let content = '';
-
-        // Header with project info
-        if (includeStats) {
-            content += this.generateProjectHeader(files);
-            content += '\n\n';
-        }
-
-        // Directory structure
-        if (includeTree) {
-            content += this.generateTreeStructure(files);
-            content += '\n\n';
-        }
-
-        // File contents
-        for (let i = 0; i < files.length; i++) {
-            const file = files[i];
-
-            // File header
-            content += headerTemplate
-                .replace('{path}', file.relativePath)
-                .replace('{name}', file.name)
-                .replace('{extension}', file.extension || '')
-                .replace('{size}', file.size)
-                .replace('{lines}', file.lines);
-
-            content += '\n';
-            content += file.content;
-            content += '\n';
-
-            // Separator between files
-            if (i < files.length - 1) {
-                content += separatorTemplate;
-                content += '\n';
-            }
-        }
-
-        return content;
-    }
-
-    /**
-     * Default file header template
-     */
-    getDefaultHeaderTemplate(): string {
-        return `
-================================================================================
-File: {path}
-Size: {size} bytes | Lines: {lines}
-================================================================================`;
-    }
-
-    /**
-     * Default separator template
-     */
-    getDefaultSeparatorTemplate(): string {
-        return '\n\n';
-    }
-
-    /**
-     * Create combined content for AI tools with correct formatting
-     */
-    async createCombinedContentForAI(files: FileContent[], options: any = {}): Promise<string> {
-        const {
-            includeStats = false,
-            includeTree = false,
-            preserveLineNumbers = true
-        } = options;
-
-        let content = '';
-
-        // Header with project info (optional)
-        if (includeStats) {
-            content += this.generateProjectHeader(files);
-            content += '\n\n';
-        }
-
-        // Directory structure (optional)
-        if (includeTree) {
-            content += this.generateTreeStructure(files);
-            content += '\n\n';
-        }
-
-        // File contents with correct formatting
-        for (let i = 0; i < files.length; i++) {
-            const file = files[i];
-
-            // File boundary marker - does not affect line numbering
-            content += `// ===== FILE: ${file.relativePath} =====\n`;
-
-            // Original file content
-            content += file.content;
-
-            // Ensure file ends with newline
-            if (!file.content.endsWith('\n')) {
-                content += '\n';
-            }
-
-            // Separator between files
-            if (i < files.length - 1) {
-                content += '\n';
-            }
-        }
-
-        return content;
-    }
-
-    /**
-     * Generate project info header
-     */
-    generateProjectHeader(files: FileContent[]): string {
-        const totalSize = files.reduce((sum, file) => sum + file.size, 0);
-        const totalLines = files.reduce((sum, file) => sum + file.lines, 0);
-
-        return `
-# Project Analysis Report
-Generated: ${new Date().toISOString()}
-Project Path: ${this.projectPath}
-
-## Statistics
-- Total Files: ${files.length}
-- Total Size: ${this.formatBytes(totalSize)}
-- Total Lines: ${totalLines.toLocaleString()}
-- Extensions: ${[...new Set(files.map(f => f.extension))].filter(Boolean).join(', ')}
-`;
-    }
-
-    /**
-     * Generate tree structure
-     */
-    generateTreeStructure(files: FileContent[]): string {
-        const tree: any = {};
-
-        // Build tree structure
-        files.forEach(file => {
-            const parts = file.relativePath.split(path.sep);
-            let current = tree;
-
-            parts.forEach((part, index) => {
-                if (index === parts.length - 1) {
-                    // File
-                    current[part] = file;
-                } else {
-                    // Directory
-                    if (!current[part]) {
-                        current[part] = {};
-                    }
-                    current = current[part];
-                }
-            });
-        });
-
-        // Generate tree string
-        let treeStr = '\n## Project Structure\n```\n';
-        treeStr += this.renderTree(tree, '', true);
-        treeStr += '```\n';
-
-        return treeStr;
-    }
-
-    /**
-     * Render tree structure
-     */
-    renderTree(node: any, prefix = '', isLast = true): string {
-        let result = '';
-        const entries = Object.entries(node);
-
-        entries.forEach(([name, value]: [string, any], index) => {
-            const isLastEntry = index === entries.length - 1;
-            const connector = isLastEntry ? '└── ' : '├── ';
-
-            result += prefix + connector + name + '\n';
-
-            if (typeof value === 'object' && !value.relativePath) {
-                // Directory
-                const newPrefix = prefix + (isLastEntry ? '    ' : '│   ');
-                result += this.renderTree(value, newPrefix, isLastEntry);
-            }
-        });
-
-        return result;
-    }
-
-    /**
-     * Render project tree structure
-     */
-    renderProjectTree(tree: FileNode, maxDepth = 50): string {
-        if (!tree) return 'No tree structure available';
-
-        const renderNode = (node: FileNode, prefix = '', depth = 0): string => {
-            if (depth > maxDepth) return '';
-
-            let result = '';
-
-            if (node.children && node.children.length > 0) {
-                // Filter out ignored directories
-                const filteredChildren = node.children.filter(child =>
-                    !this.shouldIgnoreDirectoryName(child.name)
-                );
-
-                // Sort children: directories first, then files
-                filteredChildren.sort((a, b) => {
-                    const aIsDir = a.children && a.children.length > 0;
-                    const bIsDir = b.children && b.children.length > 0;
-                    if (aIsDir && !bIsDir) return -1;
-                    if (!aIsDir && bIsDir) return 1;
-                    return a.name.localeCompare(b.name);
-                });
-
-                filteredChildren.forEach((child, index) => {
-                    const isLast = index === filteredChildren.length - 1;
-                    const connector = isLast ? '└── ' : '├── ';
-                    const nextPrefix = prefix + (isLast ? '    ' : '│   ');
-                    const isDirectory = child.children !== undefined;
-
-                    result += `${prefix}${connector}${child.name}`;
-                    if (!isDirectory) {
-                        // It's a file, show extension
-                        result += ` (${child.extension || 'file'})`;
-                    }
-                    result += '\n';
-
-                    if (isDirectory) {
-                        // Always render all children, no depth limit for full structure
-                        result += renderNode(child, nextPrefix, depth + 1);
-                    }
-                });
-            }
-
-            return result;
-        };
-
-        let result = `${tree.name}/\n`;
-        result += renderNode(tree);
-
-        return result;
-    }
-
-    /**
-     * Format bytes to human readable
-     */
-    formatBytes(bytes: number): string {
-        if (bytes === 0) return '0 Bytes';
-        const k = 1024;
-        const sizes = ['Bytes', 'KB', 'MB', 'GB'];
-        const i = Math.floor(Math.log(bytes) / Math.log(k));
-        return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
-    }
-}