vg-coder-cli 1.0.0

package/src/tokenizer/token-manager.js

@@ -0,0 +1,389 @@
+const { encoding_for_model, get_encoding } = require('tiktoken');
+
+/**
+ * Quản lý token counting và chunking
+ */
+class TokenManager {
+  constructor(options = {}) {
+    this.options = {
+      model: options.model || 'gpt-4',
+      maxTokens: options.maxTokens || 8000,
+      chunkOverlap: options.chunkOverlap || 200,
+      preserveStructure: options.preserveStructure !== false,
+      ...options
+    };
+    
+    this.encoding = null;
+    this.initializeEncoding();
+  }
+
+  /**
+   * Khởi tạo encoding
+   */
+  initializeEncoding() {
+    try {
+      // Thử sử dụng encoding cho model cụ thể
+      this.encoding = encoding_for_model(this.options.model);
+    } catch (error) {
+      try {
+        // Fallback to cl100k_base (GPT-4, GPT-3.5-turbo)
+        this.encoding = get_encoding('cl100k_base');
+      } catch (fallbackError) {
+        // Fallback to p50k_base (GPT-3)
+        this.encoding = get_encoding('p50k_base');
+      }
+    }
+  }
+
+  /**
+   * Đếm tokens trong text
+   */
+  countTokens(text) {
+    if (!text || typeof text !== 'string') {
+      return 0;
+    }
+    
+    try {
+      const tokens = this.encoding.encode(text);
+      return tokens.length;
+    } catch (error) {
+      // Fallback: estimate based on characters (rough approximation)
+      return Math.ceil(text.length / 4);
+    }
+  }
+
+  /**
+   * Phân tích token usage cho một file
+   */
+  analyzeFile(file) {
+    const headerTokens = this.countTokens(this.generateFileHeader(file));
+    const contentTokens = this.countTokens(file.content);
+    const totalTokens = headerTokens + contentTokens;
+    
+    return {
+      file: file.relativePath,
+      headerTokens,
+      contentTokens,
+      totalTokens,
+      exceedsLimit: totalTokens > this.options.maxTokens,
+      chunksNeeded: Math.ceil(totalTokens / this.options.maxTokens)
+    };
+  }
+
+  /**
+   * Phân tích token usage cho tất cả files
+   */
+  analyzeFiles(files) {
+    const analyses = files.map(file => this.analyzeFile(file));
+    
+    const totalTokens = analyses.reduce((sum, analysis) => sum + analysis.totalTokens, 0);
+    const filesExceedingLimit = analyses.filter(analysis => analysis.exceedsLimit);
+    const totalChunks = analyses.reduce((sum, analysis) => sum + analysis.chunksNeeded, 0);
+    
+    return {
+      files: analyses,
+      summary: {
+        totalFiles: files.length,
+        totalTokens,
+        averageTokensPerFile: Math.round(totalTokens / files.length),
+        filesExceedingLimit: filesExceedingLimit.length,
+        totalChunks,
+        estimatedChunks: Math.ceil(totalTokens / this.options.maxTokens)
+      }
+    };
+  }
+
+  /**
+   * Chia nhỏ content thành chunks
+   */
+  async chunkContent(content, metadata = {}) {
+    const totalTokens = this.countTokens(content);
+    
+    if (totalTokens <= this.options.maxTokens) {
+      return [{
+        content,
+        tokens: totalTokens,
+        chunkIndex: 0,
+        totalChunks: 1,
+        metadata
+      }];
+    }
+    
+    return this.options.preserveStructure 
+      ? await this.chunkByStructure(content, metadata)
+      : await this.chunkByTokens(content, metadata);
+  }
+
+  /**
+   * Chia nhỏ theo cấu trúc (ưu tiên giữ nguyên files)
+   */
+  async chunkByStructure(content, metadata = {}) {
+    const chunks = [];
+    const lines = content.split('\n');
+    let currentChunk = '';
+    let currentTokens = 0;
+    let chunkIndex = 0;
+    
+    // Tìm file boundaries
+    const fileBoundaries = this.findFileBoundaries(lines);
+    
+    for (let i = 0; i < fileBoundaries.length; i++) {
+      const boundary = fileBoundaries[i];
+      const fileContent = lines.slice(boundary.start, boundary.end).join('\n');
+      const fileTokens = this.countTokens(fileContent);
+      
+      // Nếu file này làm chunk vượt quá limit
+      if (currentTokens + fileTokens > this.options.maxTokens && currentChunk) {
+        // Lưu chunk hiện tại
+        chunks.push({
+          content: currentChunk.trim(),
+          tokens: currentTokens,
+          chunkIndex: chunkIndex++,
+          totalChunks: 0, // Sẽ update sau
+          metadata: { ...metadata, type: 'structure' }
+        });
+        
+        currentChunk = '';
+        currentTokens = 0;
+      }
+      
+      // Nếu file quá lớn, chia nhỏ file này
+      if (fileTokens > this.options.maxTokens) {
+        const fileChunks = await this.chunkLargeFile(fileContent, boundary.filePath);
+        chunks.push(...fileChunks.map(chunk => ({
+          ...chunk,
+          chunkIndex: chunkIndex++,
+          metadata: { ...metadata, ...chunk.metadata, type: 'large-file' }
+        })));
+      } else {
+        currentChunk += fileContent + '\n';
+        currentTokens += fileTokens;
+      }
+    }
+    
+    // Thêm chunk cuối cùng
+    if (currentChunk.trim()) {
+      chunks.push({
+        content: currentChunk.trim(),
+        tokens: currentTokens,
+        chunkIndex: chunkIndex++,
+        totalChunks: 0,
+        metadata: { ...metadata, type: 'structure' }
+      });
+    }
+    
+    // Update totalChunks
+    chunks.forEach(chunk => {
+      chunk.totalChunks = chunks.length;
+    });
+    
+    return chunks;
+  }
+
+  /**
+   * Chia nhỏ theo tokens (simple splitting)
+   */
+  async chunkByTokens(content, metadata = {}) {
+    const chunks = [];
+    const lines = content.split('\n');
+    let currentChunk = '';
+    let currentTokens = 0;
+    let chunkIndex = 0;
+    
+    for (const line of lines) {
+      const lineTokens = this.countTokens(line + '\n');
+      
+      if (currentTokens + lineTokens > this.options.maxTokens && currentChunk) {
+        // Lưu chunk hiện tại
+        chunks.push({
+          content: currentChunk.trim(),
+          tokens: currentTokens,
+          chunkIndex: chunkIndex++,
+          totalChunks: 0,
+          metadata: { ...metadata, type: 'token-based' }
+        });
+        
+        // Bắt đầu chunk mới với overlap
+        if (this.options.chunkOverlap > 0) {
+          const overlapLines = this.getOverlapLines(currentChunk, this.options.chunkOverlap);
+          currentChunk = overlapLines;
+          currentTokens = this.countTokens(overlapLines);
+        } else {
+          currentChunk = '';
+          currentTokens = 0;
+        }
+      }
+      
+      currentChunk += line + '\n';
+      currentTokens += lineTokens;
+    }
+    
+    // Thêm chunk cuối cùng
+    if (currentChunk.trim()) {
+      chunks.push({
+        content: currentChunk.trim(),
+        tokens: currentTokens,
+        chunkIndex: chunkIndex++,
+        totalChunks: 0,
+        metadata: { ...metadata, type: 'token-based' }
+      });
+    }
+    
+    // Update totalChunks
+    chunks.forEach(chunk => {
+      chunk.totalChunks = chunks.length;
+    });
+    
+    return chunks;
+  }
+
+  /**
+   * Chia nhỏ file lớn
+   */
+  async chunkLargeFile(fileContent, filePath) {
+    const lines = fileContent.split('\n');
+    const chunks = [];
+    let currentChunk = '';
+    let currentTokens = 0;
+    let chunkIndex = 0;
+    
+    // Thêm header cho file
+    const header = `\n=== Large File: ${filePath} (Part {part}) ===\n`;
+    
+    for (const line of lines) {
+      const lineTokens = this.countTokens(line + '\n');
+      
+      if (currentTokens + lineTokens > this.options.maxTokens - 100 && currentChunk) { // Reserve 100 tokens for header
+        const partHeader = header.replace('{part}', (chunkIndex + 1).toString());
+        chunks.push({
+          content: partHeader + currentChunk.trim(),
+          tokens: this.countTokens(partHeader + currentChunk.trim()),
+          chunkIndex: chunkIndex++,
+          totalChunks: 0,
+          metadata: { 
+            filePath,
+            partNumber: chunkIndex + 1,
+            type: 'large-file-part'
+          }
+        });
+        
+        currentChunk = '';
+        currentTokens = 0;
+      }
+      
+      currentChunk += line + '\n';
+      currentTokens += lineTokens;
+    }
+    
+    // Thêm chunk cuối cùng
+    if (currentChunk.trim()) {
+      const partHeader = header.replace('{part}', (chunkIndex + 1).toString());
+      chunks.push({
+        content: partHeader + currentChunk.trim(),
+        tokens: this.countTokens(partHeader + currentChunk.trim()),
+        chunkIndex: chunkIndex++,
+        totalChunks: 0,
+        metadata: { 
+          filePath,
+          partNumber: chunkIndex + 1,
+          type: 'large-file-part'
+        }
+      });
+    }
+    
+    // Update totalChunks
+    chunks.forEach(chunk => {
+      chunk.totalChunks = chunks.length;
+    });
+    
+    return chunks;
+  }
+
+  /**
+   * Tìm file boundaries trong content
+   */
+  findFileBoundaries(lines) {
+    const boundaries = [];
+    let currentStart = 0;
+    let currentFilePath = null;
+    
+    for (let i = 0; i < lines.length; i++) {
+      const line = lines[i];
+      
+      // Tìm file header pattern
+      if (line.includes('================================================================================')) {
+        if (i + 1 < lines.length && lines[i + 1].startsWith('File: ')) {
+          // Lưu boundary trước đó
+          if (currentFilePath) {
+            boundaries.push({
+              start: currentStart,
+              end: i,
+              filePath: currentFilePath
+            });
+          }
+          
+          // Bắt đầu file mới
+          currentStart = i;
+          currentFilePath = lines[i + 1].replace('File: ', '').trim();
+        }
+      }
+    }
+    
+    // Thêm boundary cuối cùng
+    if (currentFilePath) {
+      boundaries.push({
+        start: currentStart,
+        end: lines.length,
+        filePath: currentFilePath
+      });
+    }
+    
+    return boundaries;
+  }
+
+  /**
+   * Lấy overlap lines
+   */
+  getOverlapLines(content, maxTokens) {
+    const lines = content.split('\n');
+    let overlapContent = '';
+    let tokens = 0;
+    
+    // Lấy từ cuối lên
+    for (let i = lines.length - 1; i >= 0; i--) {
+      const line = lines[i] + '\n';
+      const lineTokens = this.countTokens(line);
+      
+      if (tokens + lineTokens > maxTokens) {
+        break;
+      }
+      
+      overlapContent = line + overlapContent;
+      tokens += lineTokens;
+    }
+    
+    return overlapContent;
+  }
+
+  /**
+   * Tạo file header
+   */
+  generateFileHeader(file) {
+    return `
+================================================================================
+File: ${file.relativePath}
+Size: ${file.size} bytes | Lines: ${file.lines}
+================================================================================`;
+  }
+
+  /**
+   * Cleanup encoding
+   */
+  cleanup() {
+    if (this.encoding) {
+      this.encoding.free();
+    }
+  }
+}
+
+module.exports = TokenManager;

package/src/utils/helpers.js

@@ -0,0 +1,128 @@
+/**
+ * Utility helpers
+ */
+
+/**
+ * Format bytes to human readable string
+ */
+function formatBytes(bytes) {
+  if (bytes === 0) return '0 Bytes';
+  const k = 1024;
+  const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB'];
+  const i = Math.floor(Math.log(bytes) / Math.log(k));
+  return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
+}
+
+/**
+ * Format number with commas
+ */
+function formatNumber(num) {
+  return num.toLocaleString();
+}
+
+/**
+ * Get file extension from filename
+ */
+function getFileExtension(filename) {
+  return filename.slice((filename.lastIndexOf('.') - 1 >>> 0) + 2);
+}
+
+/**
+ * Check if path is hidden (starts with dot)
+ */
+function isHidden(filePath) {
+  return filePath.split('/').some(part => part.startsWith('.'));
+}
+
+/**
+ * Sanitize filename for safe usage
+ */
+function sanitizeFilename(filename) {
+  return filename.replace(/[^a-z0-9]/gi, '_').toLowerCase();
+}
+
+/**
+ * Deep merge objects
+ */
+function deepMerge(target, source) {
+  const output = Object.assign({}, target);
+  if (isObject(target) && isObject(source)) {
+    Object.keys(source).forEach(key => {
+      if (isObject(source[key])) {
+        if (!(key in target))
+          Object.assign(output, { [key]: source[key] });
+        else
+          output[key] = deepMerge(target[key], source[key]);
+      } else {
+        Object.assign(output, { [key]: source[key] });
+      }
+    });
+  }
+  return output;
+}
+
+/**
+ * Check if value is object
+ */
+function isObject(item) {
+  return item && typeof item === 'object' && !Array.isArray(item);
+}
+
+/**
+ * Debounce function
+ */
+function debounce(func, wait, immediate) {
+  let timeout;
+  return function executedFunction(...args) {
+    const later = () => {
+      timeout = null;
+      if (!immediate) func(...args);
+    };
+    const callNow = immediate && !timeout;
+    clearTimeout(timeout);
+    timeout = setTimeout(later, wait);
+    if (callNow) func(...args);
+  };
+}
+
+/**
+ * Throttle function
+ */
+function throttle(func, limit) {
+  let inThrottle;
+  return function(...args) {
+    if (!inThrottle) {
+      func.apply(this, args);
+      inThrottle = true;
+      setTimeout(() => inThrottle = false, limit);
+    }
+  };
+}
+
+/**
+ * Generate unique ID
+ */
+function generateId() {
+  return Math.random().toString(36).substr(2, 9);
+}
+
+/**
+ * Sleep for specified milliseconds
+ */
+function sleep(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+module.exports = {
+  formatBytes,
+  formatNumber,
+  getFileExtension,
+  isHidden,
+  sanitizeFilename,
+  deepMerge,
+  isObject,
+  debounce,
+  throttle,
+  generateId,
+  sleep
+};

package/test-project/package.json

@@ -0,0 +1,21 @@
+{
+  "name": "test-project",
+  "version": "1.0.0",
+  "description": "A test project for VG Coder CLI",
+  "main": "src/index.js",
+  "scripts": {
+    "start": "node src/index.js",
+    "test": "jest"
+  },
+  "dependencies": {
+    "express": "^4.18.0",
+    "lodash": "^4.17.21"
+  },
+  "devDependencies": {
+    "jest": "^29.0.0",
+    "nodemon": "^3.0.0"
+  },
+  "keywords": ["test", "demo", "vg-coder"],
+  "author": "VG Coder",
+  "license": "MIT"
+}

package/test-project/src/controllers/userController.js

@@ -0,0 +1,129 @@
+const express = require('express');
+const _ = require('lodash');
+const User = require('../models/User');
+const { validateUser, validateEmail } = require('../utils/validation');
+
+const router = express.Router();
+
+// Get all users
+router.get('/', async (req, res) => {
+  try {
+    const { page = 1, limit = 10, search } = req.query;
+    const offset = (page - 1) * limit;
+    
+    let users = await User.findAll({
+      limit: parseInt(limit),
+      offset: parseInt(offset),
+      where: search ? {
+        name: { [Op.iLike]: `%${search}%` }
+      } : {}
+    });
+
+    users = users.map(user => _.omit(user.toJSON(), ['password']));
+    
+    res.json({
+      users,
+      pagination: {
+        page: parseInt(page),
+        limit: parseInt(limit),
+        total: await User.count()
+      }
+    });
+  } catch (error) {
+    res.status(500).json({ error: error.message });
+  }
+});
+
+// Get user by ID
+router.get('/:id', async (req, res) => {
+  try {
+    const { id } = req.params;
+    const user = await User.findByPk(id);
+    
+    if (!user) {
+      return res.status(404).json({ error: 'User not found' });
+    }
+    
+    res.json(_.omit(user.toJSON(), ['password']));
+  } catch (error) {
+    res.status(500).json({ error: error.message });
+  }
+});
+
+// Create new user
+router.post('/', async (req, res) => {
+  try {
+    const userData = req.body;
+    
+    // Validate input
+    const validation = validateUser(userData);
+    if (!validation.isValid) {
+      return res.status(400).json({ 
+        error: 'Validation failed',
+        details: validation.errors 
+      });
+    }
+    
+    // Check if email already exists
+    const existingUser = await User.findOne({ 
+      where: { email: userData.email } 
+    });
+    
+    if (existingUser) {
+      return res.status(409).json({ 
+        error: 'Email already exists' 
+      });
+    }
+    
+    const user = await User.create(userData);
+    res.status(201).json(_.omit(user.toJSON(), ['password']));
+  } catch (error) {
+    res.status(500).json({ error: error.message });
+  }
+});
+
+// Update user
+router.put('/:id', async (req, res) => {
+  try {
+    const { id } = req.params;
+    const updateData = req.body;
+    
+    const user = await User.findByPk(id);
+    if (!user) {
+      return res.status(404).json({ error: 'User not found' });
+    }
+    
+    // Validate update data
+    const validation = validateUser(updateData, true);
+    if (!validation.isValid) {
+      return res.status(400).json({ 
+        error: 'Validation failed',
+        details: validation.errors 
+      });
+    }
+    
+    await user.update(updateData);
+    res.json(_.omit(user.toJSON(), ['password']));
+  } catch (error) {
+    res.status(500).json({ error: error.message });
+  }
+});
+
+// Delete user
+router.delete('/:id', async (req, res) => {
+  try {
+    const { id } = req.params;
+    const user = await User.findByPk(id);
+    
+    if (!user) {
+      return res.status(404).json({ error: 'User not found' });
+    }
+    
+    await user.destroy();
+    res.status(204).send();
+  } catch (error) {
+    res.status(500).json({ error: error.message });
+  }
+});
+
+module.exports = router;

package/test-project/src/index.js

@@ -0,0 +1,46 @@
+const express = require('express');
+const _ = require('lodash');
+const userController = require('./controllers/userController');
+const authMiddleware = require('./middleware/auth');
+
+const app = express();
+const PORT = process.env.PORT || 3000;
+
+// Middleware
+app.use(express.json());
+app.use(express.urlencoded({ extended: true }));
+
+// Routes
+app.get('/', (req, res) => {
+  res.json({ 
+    message: 'Welcome to Test API',
+    version: '1.0.0',
+    timestamp: new Date().toISOString()
+  });
+});
+
+app.use('/api/users', authMiddleware, userController);
+
+// Error handling middleware
+app.use((err, req, res, next) => {
+  console.error(err.stack);
+  res.status(500).json({ 
+    error: 'Something went wrong!',
+    message: err.message 
+  });
+});
+
+// 404 handler
+app.use('*', (req, res) => {
+  res.status(404).json({ 
+    error: 'Route not found',
+    path: req.originalUrl 
+  });
+});
+
+app.listen(PORT, () => {
+  console.log(`Server is running on port ${PORT}`);
+  console.log(`Environment: ${process.env.NODE_ENV || 'development'}`);
+});
+
+module.exports = app;