vg-coder-cli 2.0.1 → 2.0.4

package/vg/packages/core/src/lib/tokenizer/token-manager.ts

@@ -1,435 +0,0 @@
-import { encoding_for_model, get_encoding, Tiktoken, TiktokenModel } from 'tiktoken';
-
-export interface TokenManagerOptions {
-    model?: string;
-    maxTokens?: number;
-    chunkOverlap?: number;
-    preserveStructure?: boolean;
-}
-
-export interface TokenAnalysis {
-    file: string;
-    headerTokens: number;
-    contentTokens: number;
-    totalTokens: number;
-    exceedsLimit: boolean;
-    chunksNeeded: number;
-}
-
-export interface TokenAnalysisSummary {
-    files: TokenAnalysis[];
-    summary: {
-        totalFiles: number;
-        totalTokens: number;
-        averageTokensPerFile: number;
-        filesExceedingLimit: number;
-        totalChunks: number;
-        estimatedChunks: number;
-    };
-}
-
-export interface ChunkMetadata {
-    type?: string;
-    filePath?: string;
-    partNumber?: number;
-    [key: string]: any;
-}
-
-export interface ContentChunk {
-    content: string;
-    tokens: number;
-    chunkIndex: number;
-    totalChunks: number;
-    metadata: ChunkMetadata;
-}
-
-/**
- * Manage token counting and chunking
- */
-export class TokenManager {
-    private options: TokenManagerOptions;
-    private encoding: Tiktoken | null = null;
-
-    constructor(options: TokenManagerOptions = {}) {
-        this.options = {
-            model: options.model || 'gpt-4',
-            maxTokens: options.maxTokens || 8000,
-            chunkOverlap: options.chunkOverlap || 200,
-            preserveStructure: options.preserveStructure !== false,
-            ...options
-        };
-
-        this.initializeEncoding();
-    }
-
-    /**
-     * Initialize encoding
-     */
-    initializeEncoding(): void {
-        try {
-            // Try using encoding for specific model
-            this.encoding = encoding_for_model(this.options.model as TiktokenModel);
-        } catch (error) {
-            try {
-                // Fallback to cl100k_base (GPT-4, GPT-3.5-turbo)
-                this.encoding = get_encoding('cl100k_base');
-            } catch (fallbackError) {
-                // Fallback to p50k_base (GPT-3)
-                this.encoding = get_encoding('p50k_base');
-            }
-        }
-    }
-
-    /**
-     * Count tokens in text
-     */
-    countTokens(text: string): number {
-        if (!text || typeof text !== 'string') {
-            return 0;
-        }
-
-        try {
-            if (this.encoding) {
-                const tokens = this.encoding.encode(text);
-                return tokens.length;
-            }
-            return Math.ceil(text.length / 4);
-        } catch (error) {
-            // Fallback: estimate based on characters (rough approximation)
-            return Math.ceil(text.length / 4);
-        }
-    }
-
-    /**
-     * Analyze token usage for a file
-     */
-    analyzeFile(file: { relativePath: string; content: string; size: number; lines: number }): TokenAnalysis {
-        const headerTokens = this.countTokens(this.generateFileHeader(file));
-        const contentTokens = this.countTokens(file.content);
-        const totalTokens = headerTokens + contentTokens;
-
-        return {
-            file: file.relativePath,
-            headerTokens,
-            contentTokens,
-            totalTokens,
-            exceedsLimit: totalTokens > (this.options.maxTokens || 8000),
-            chunksNeeded: Math.ceil(totalTokens / (this.options.maxTokens || 8000))
-        };
-    }
-
-    /**
-     * Analyze token usage for all files
-     */
-    analyzeFiles(files: any[]): TokenAnalysisSummary {
-        const analyses = files.map(file => this.analyzeFile(file));
-
-        const totalTokens = analyses.reduce((sum, analysis) => sum + analysis.totalTokens, 0);
-        const filesExceedingLimit = analyses.filter(analysis => analysis.exceedsLimit);
-        const totalChunks = analyses.reduce((sum, analysis) => sum + analysis.chunksNeeded, 0);
-
-        return {
-            files: analyses,
-            summary: {
-                totalFiles: files.length,
-                totalTokens,
-                averageTokensPerFile: Math.round(totalTokens / files.length),
-                filesExceedingLimit: filesExceedingLimit.length,
-                totalChunks,
-                estimatedChunks: Math.ceil(totalTokens / (this.options.maxTokens || 8000))
-            }
-        };
-    }
-
-    /**
-     * Split content into chunks
-     */
-    async chunkContent(content: string, metadata: ChunkMetadata = {}): Promise<ContentChunk[]> {
-        const totalTokens = this.countTokens(content);
-        const maxTokens = this.options.maxTokens || 8000;
-
-        if (totalTokens <= maxTokens) {
-            return [{
-                content,
-                tokens: totalTokens,
-                chunkIndex: 0,
-                totalChunks: 1,
-                metadata
-            }];
-        }
-
-        return this.options.preserveStructure
-            ? await this.chunkByStructure(content, metadata)
-            : await this.chunkByTokens(content, metadata);
-    }
-
-    /**
-     * Split by structure (prioritize keeping files intact)
-     */
-    async chunkByStructure(content: string, metadata: ChunkMetadata = {}): Promise<ContentChunk[]> {
-        const chunks: ContentChunk[] = [];
-        const lines = content.split('\n');
-        let currentChunk = '';
-        let currentTokens = 0;
-        let chunkIndex = 0;
-        const maxTokens = this.options.maxTokens || 8000;
-
-        // Find file boundaries
-        const fileBoundaries = this.findFileBoundaries(lines);
-
-        for (let i = 0; i < fileBoundaries.length; i++) {
-            const boundary = fileBoundaries[i];
-            const fileContent = lines.slice(boundary.start, boundary.end).join('\n');
-            const fileTokens = this.countTokens(fileContent);
-
-            // If adding this file exceeds limit
-            if (currentTokens + fileTokens > maxTokens && currentChunk) {
-                // Save current chunk
-                chunks.push({
-                    content: currentChunk.trim(),
-                    tokens: currentTokens,
-                    chunkIndex: chunkIndex++,
-                    totalChunks: 0, // Will update later
-                    metadata: { ...metadata, type: 'structure' }
-                });
-
-                currentChunk = '';
-                currentTokens = 0;
-            }
-
-            // If file itself is too large, split it
-            if (fileTokens > maxTokens) {
-                const fileChunks = await this.chunkLargeFile(fileContent, boundary.filePath);
-                chunks.push(...fileChunks.map(chunk => ({
-                    ...chunk,
-                    chunkIndex: chunkIndex++,
-                    metadata: { ...metadata, ...chunk.metadata, type: 'large-file' }
-                })));
-            } else {
-                currentChunk += fileContent + '\n';
-                currentTokens += fileTokens;
-            }
-        }
-
-        // Add last chunk
-        if (currentChunk.trim()) {
-            chunks.push({
-                content: currentChunk.trim(),
-                tokens: currentTokens,
-                chunkIndex: chunkIndex++,
-                totalChunks: 0,
-                metadata: { ...metadata, type: 'structure' }
-            });
-        }
-
-        // Update totalChunks
-        chunks.forEach(chunk => {
-            chunk.totalChunks = chunks.length;
-        });
-
-        return chunks;
-    }
-
-    /**
-     * Split by tokens (simple splitting)
-     */
-    async chunkByTokens(content: string, metadata: ChunkMetadata = {}): Promise<ContentChunk[]> {
-        const chunks: ContentChunk[] = [];
-        const lines = content.split('\n');
-        let currentChunk = '';
-        let currentTokens = 0;
-        let chunkIndex = 0;
-        const maxTokens = this.options.maxTokens || 8000;
-        const chunkOverlap = this.options.chunkOverlap || 200;
-
-        for (const line of lines) {
-            const lineTokens = this.countTokens(line + '\n');
-
-            if (currentTokens + lineTokens > maxTokens && currentChunk) {
-                // Save current chunk
-                chunks.push({
-                    content: currentChunk.trim(),
-                    tokens: currentTokens,
-                    chunkIndex: chunkIndex++,
-                    totalChunks: 0,
-                    metadata: { ...metadata, type: 'token-based' }
-                });
-
-                // Start new chunk with overlap
-                if (chunkOverlap > 0) {
-                    const overlapLines = this.getOverlapLines(currentChunk, chunkOverlap);
-                    currentChunk = overlapLines;
-                    currentTokens = this.countTokens(overlapLines);
-                } else {
-                    currentChunk = '';
-                    currentTokens = 0;
-                }
-            }
-
-            currentChunk += line + '\n';
-            currentTokens += lineTokens;
-        }
-
-        // Add last chunk
-        if (currentChunk.trim()) {
-            chunks.push({
-                content: currentChunk.trim(),
-                tokens: currentTokens,
-                chunkIndex: chunkIndex++,
-                totalChunks: 0,
-                metadata: { ...metadata, type: 'token-based' }
-            });
-        }
-
-        // Update totalChunks
-        chunks.forEach(chunk => {
-            chunk.totalChunks = chunks.length;
-        });
-
-        return chunks;
-    }
-
-    /**
-     * Split large file
-     */
-    async chunkLargeFile(fileContent: string, filePath: string): Promise<ContentChunk[]> {
-        const lines = fileContent.split('\n');
-        const chunks: ContentChunk[] = [];
-        let currentChunk = '';
-        let currentTokens = 0;
-        let chunkIndex = 0;
-        const maxTokens = this.options.maxTokens || 8000;
-
-        for (const line of lines) {
-            const lineTokens = this.countTokens(line + '\n');
-
-            if (currentTokens + lineTokens > maxTokens && currentChunk) {
-                chunks.push({
-                    content: currentChunk.trim(),
-                    tokens: this.countTokens(currentChunk.trim()),
-                    chunkIndex: chunkIndex++,
-                    totalChunks: 0,
-                    metadata: {
-                        filePath,
-                        partNumber: chunkIndex + 1,
-                        type: 'large-file-part'
-                    }
-                });
-
-                currentChunk = '';
-                currentTokens = 0;
-            }
-
-            currentChunk += line + '\n';
-            currentTokens += lineTokens;
-        }
-
-        // Add last chunk
-        if (currentChunk.trim()) {
-            chunks.push({
-                content: currentChunk.trim(),
-                tokens: this.countTokens(currentChunk.trim()),
-                chunkIndex: chunkIndex++,
-                totalChunks: 0,
-                metadata: {
-                    filePath,
-                    partNumber: chunkIndex + 1,
-                    type: 'large-file-part'
-                }
-            });
-        }
-
-        // Update totalChunks
-        chunks.forEach(chunk => {
-            chunk.totalChunks = chunks.length;
-        });
-
-        return chunks;
-    }
-
-    /**
-     * Find file boundaries in content
-     */
-    findFileBoundaries(lines: string[]): { start: number; end: number; filePath: string }[] {
-        const boundaries: { start: number; end: number; filePath: string }[] = [];
-        let currentStart = 0;
-        let currentFilePath: string | null = null;
-
-        for (let i = 0; i < lines.length; i++) {
-            const line = lines[i];
-
-            // Find file header pattern
-            if (line.includes('================================================================================')) {
-                if (i + 1 < lines.length && lines[i + 1].startsWith('File: ')) {
-                    // Save previous boundary
-                    if (currentFilePath) {
-                        boundaries.push({
-                            start: currentStart,
-                            end: i,
-                            filePath: currentFilePath
-                        });
-                    }
-
-                    // Start new file
-                    currentStart = i;
-                    currentFilePath = lines[i + 1].replace('File: ', '').trim();
-                }
-            }
-        }
-
-        // Add last boundary
-        if (currentFilePath) {
-            boundaries.push({
-                start: currentStart,
-                end: lines.length,
-                filePath: currentFilePath
-            });
-        }
-
-        return boundaries;
-    }
-
-    /**
-     * Get overlap lines
-     */
-    getOverlapLines(content: string, maxTokens: number): string {
-        const lines = content.split('\n');
-        let overlapContent = '';
-        let tokens = 0;
-
-        // Get from end
-        for (let i = lines.length - 1; i >= 0; i--) {
-            const line = lines[i] + '\n';
-            const lineTokens = this.countTokens(line);
-
-            if (tokens + lineTokens > maxTokens) {
-                break;
-            }
-
-            overlapContent = line + overlapContent;
-            tokens += lineTokens;
-        }
-
-        return overlapContent;
-    }
-
-    /**
-     * Generate file header
-     */
-    generateFileHeader(file: { relativePath: string; size: number; lines: number }): string {
-        return `
-================================================================================
-File: ${file.relativePath}
-Size: ${file.size} bytes | Lines: ${file.lines}
-================================================================================`;
-    }
-
-    /**
-     * Cleanup encoding
-     */
-    cleanup(): void {
-        if (this.encoding) {
-            this.encoding.free();
-        }
-    }
-}

package/vg/packages/core/src/lib/utils/bash-executor.ts

@@ -1,146 +0,0 @@
-import { exec } from 'child_process';
-import * as path from 'path';
-import * as fs from 'fs-extra';
-import { promisify } from 'util';
-
-const execAsync = promisify(exec);
-
-export interface ExecutionResult {
-    success: boolean;
-    stdout?: string;
-    stderr?: string;
-    exitCode?: number;
-    error?: string;
-    details?: string;
-    executionTime: number;
-    syntaxError?: boolean;
-}
-
-export interface ValidationResult {
-    valid: boolean;
-    error: string | null;
-}
-
-/**
- * Bash Script Executor
- * Validates and executes bash scripts safely
- */
-export class BashExecutor {
-    private workingDir: string;
-    private tempDir: string;
-
-    constructor(workingDir?: string) {
-        this.workingDir = workingDir || process.cwd();
-        this.tempDir = path.join(this.workingDir, '.vg', 'temp-execute');
-    }
-
-    /**
-     * Ensure temp directory exists
-     */
-    async ensureTempDir(): Promise<void> {
-        await fs.ensureDir(this.tempDir);
-    }
-
-    /**
-     * Cleanup temp directory
-     */
-    async cleanup(): Promise<void> {
-        try {
-            if (await fs.pathExists(this.tempDir)) {
-                await fs.remove(this.tempDir);
-            }
-        } catch (error: any) {
-            console.error('Cleanup error:', error.message);
-        }
-    }
-
-    /**
-     * Validate bash script syntax
-     * @param bashScript - The bash script to validate
-     */
-    async validateSyntax(bashScript: string): Promise<ValidationResult> {
-        await this.ensureTempDir();
-
-        const scriptPath = path.join(this.tempDir, 'validate.sh');
-
-        try {
-            // Write script to temp file
-            await fs.writeFile(scriptPath, bashScript, 'utf8');
-
-            // Validate syntax using bash -n
-            await execAsync(`bash -n "${scriptPath}"`);
-
-            return { valid: true, error: null };
-
-        } catch (error: any) {
-            return {
-                valid: false,
-                error: error.stderr || error.message
-            };
-        }
-    }
-
-    /**
-     * Execute bash script in working directory
-     * @param bashScript - The bash script to execute
-     */
-    async execute(bashScript: string): Promise<ExecutionResult> {
-        const startTime = Date.now();
-
-        try {
-            // First validate syntax
-            const validation = await this.validateSyntax(bashScript);
-
-            if (!validation.valid) {
-                return {
-                    success: false,
-                    error: 'Syntax validation failed',
-                    details: validation.error || 'Unknown syntax error',
-                    executionTime: Date.now() - startTime,
-                    syntaxError: true
-                };
-            }
-
-            // Execute script in working directory
-            const result = await execAsync(bashScript, {
-                cwd: this.workingDir,
-                maxBuffer: 10 * 1024 * 1024, // 10MB buffer
-                shell: '/bin/bash'
-            });
-
-            // Cleanup temp directory
-            await this.cleanup();
-
-            return {
-                success: true,
-                stdout: result.stdout || '',
-                stderr: result.stderr || '',
-                exitCode: 0,
-                executionTime: Date.now() - startTime
-            };
-
-        } catch (error: any) {
-            // Cleanup even on error
-            await this.cleanup();
-
-            // Check if it's an execution error (not syntax)
-            if (error.code !== undefined) {
-                return {
-                    success: false,
-                    stdout: error.stdout || '',
-                    stderr: error.stderr || error.message,
-                    exitCode: error.code || 1,
-                    executionTime: Date.now() - startTime
-                };
-            }
-
-            // Unknown error
-            return {
-                success: false,
-                error: 'Execution failed',
-                details: error.message,
-                executionTime: Date.now() - startTime
-            };
-        }
-    }
-}

package/vg/packages/shared/data-types/README.md

@@ -1,11 +0,0 @@
-# data-types
-
-This library was generated with [Nx](https://nx.dev).
-
-## Building
-
-Run `nx build data-types` to build the library.
-
-## Running unit tests
-
-Run `nx test data-types` to execute the unit tests via [Jest](https://jestjs.io).

package/vg/packages/shared/data-types/jest.config.ts

@@ -1,10 +0,0 @@
-export default {
-  displayName: 'data-types',
-  preset: '../../../jest.preset.js',
-  testEnvironment: 'node',
-  transform: {
-    '^.+\\.[tj]s$': ['ts-jest', { tsconfig: '<rootDir>/tsconfig.spec.json' }],
-  },
-  moduleFileExtensions: ['ts', 'js', 'html'],
-  coverageDirectory: '../../../coverage/packages/shared/data-types',
-};

package/vg/packages/shared/data-types/package.json

@@ -1,11 +0,0 @@
-{
-  "name": "@vg/data-types",
-  "version": "0.0.1",
-  "private": true,
-  "type": "commonjs",
-  "main": "./src/index.js",
-  "types": "./src/index.d.ts",
-  "dependencies": {
-    "tslib": "^2.3.0"
-  }
-}

package/vg/packages/shared/data-types/project.json

@@ -1,26 +0,0 @@
-{
-  "name": "data-types",
-  "$schema": "../../../node_modules/nx/schemas/project-schema.json",
-  "sourceRoot": "packages/shared/data-types/src",
-  "projectType": "library",
-  "tags": ["scope:shared", "type:lib"],
-  "targets": {
-    "build": {
-      "executor": "@nx/js:tsc",
-      "outputs": ["{options.outputPath}"],
-      "options": {
-        "outputPath": "dist/packages/shared/data-types",
-        "main": "packages/shared/data-types/src/index.ts",
-        "tsConfig": "packages/shared/data-types/tsconfig.lib.json",
-        "assets": ["packages/shared/data-types/*.md"]
-      }
-    },
-    "test": {
-      "executor": "@nx/jest:jest",
-      "outputs": ["{workspaceRoot}/coverage/{projectRoot}"],
-      "options": {
-        "jestConfig": "packages/shared/data-types/jest.config.ts"
-      }
-    }
-  }
-}

package/vg/packages/shared/data-types/src/index.ts

@@ -1 +0,0 @@
-export * from './lib/data-types';

package/vg/packages/shared/data-types/src/lib/data-types.ts

@@ -1,3 +0,0 @@
-export function dataTypes(): string {
-  return 'data-types';
-}

package/vg/start-dev.sh

@@ -1,22 +0,0 @@
-#!/bin/bash
-
-# VG Coder - Start Development Servers
-# This script starts both the NestJS API and Angular frontend
-
-echo "🚀 Starting VG Coder Development Environment..."
-echo ""
-
-# Check if we're in the vg directory
-if [ ! -f "nx.json" ]; then
-    echo "❌ Error: Please run this script from the vg directory"
-    exit 1
-fi
-
-# Start both services
-echo "📦 Starting API backend (port 3000)..."
-echo "🎨 Starting Angular frontend (port 4200)..."
-echo ""
-echo "Press Ctrl+C to stop all services"
-echo ""
-
-nx run-many --target=serve --projects=api,ng-app --parallel=2