scai 0.1.85 → 0.1.86

package/dist/CHANGELOG.md

@@ -99,4 +99,12 @@ Type handling with the module pipeline
 • Update .gitignore to ignore Node and build artifacts, editor/IDE files, and project-specific ignores.
 • Added scripts for building cli, server, and dashboard
 • Restructure the workspace into 3 modules, cli, dashboard, server
-• Improved `askChangelogApproval` to support editing and skipping of changelog entries
+• Improved `askChangelogApproval` to support editing and skipping of changelog entries
+
+## 2025-08-12
+
+* Added support for backing up SCAI folder with timestamped directory.
+* Split file processing into logical chunks with logging.
+* Improved commit suggestion generation logic.
+* Removed unnecessary files and updated index.ts to use handleAgentRun instead.
+* Fetch download statistics from NPM API.

package/dist/agentManager.js

@@ -0,0 +1,46 @@
+import fs from 'fs/promises';
+import chalk from 'chalk';
+import { runModulePipeline } from './pipeline/runModulePipeline.js';
+import { normalizePath } from './utils/normalizePath.js';
+import { readConfig } from './config.js';
+import { countTokens, splitCodeIntoChunks } from './utils/splitCodeIntoChunk.js';
+export async function handleAgentRun(filepath, modules) {
+    try {
+        filepath = normalizePath(filepath);
+        const content = await fs.readFile(filepath, 'utf-8');
+        const totalTokens = countTokens(content);
+        console.log(chalk.blue(`🧮 Total tokens in file:`), chalk.yellow(totalTokens.toString()));
+        const config = readConfig();
+        const maxTokens = 1500;
+        const chunks = splitCodeIntoChunks(content, maxTokens);
+        console.log(chalk.magenta(`📦 Split into ${chunks.length} chunks`));
+        const processedChunks = [];
+        for (const [i, chunk] of chunks.entries()) {
+            const chunkTokens = countTokens(chunk);
+            if (i === 0) {
+                console.log(chalk.cyan(`🔍 Processing ${chunks.length} chunks of file:`), chalk.white(filepath));
+            }
+            console.log(chalk.gray(`   - Chunk ${i + 1} tokens:`), chalk.yellow(chunkTokens.toString()));
+            const chunkInput = {
+                content: chunk,
+                filepath,
+                chunkIndex: i,
+                chunkCount: chunks.length,
+            };
+            const response = await runModulePipeline(modules, chunkInput);
+            if (!response.content.trim()) {
+                throw new Error(`⚠️ Model returned empty result on chunk ${i + 1}`);
+            }
+            processedChunks.push(response.content);
+            console.log(chalk.green(`✅ Finished chunk ${i + 1}/${chunks.length}`));
+        }
+        // Join all chunk outputs into one string
+        const finalOutput = processedChunks.join('\n\n');
+        // Overwrite original file here:
+        await fs.writeFile(filepath, finalOutput, 'utf-8');
+        console.log(chalk.green(`✅ Original file overwritten: ${filepath}`));
+    }
+    catch (err) {
+        console.error(chalk.red('❌ Error in agent run:'), err instanceof Error ? err.message : err);
+    }
+}

package/dist/db/backup.js

@@ -1,28 +1,40 @@
-import fsp from 'fs/promises';
+// src/db/backup.ts
+import fs from 'fs';
 import path from 'path';
-import { SCAI_HOME } from '../constants.js';
-function getBackupDir() {
-    const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
-    return path.join(SCAI_HOME, `backup-${timestamp}`);
-}
+import os from 'os';
 export async function backupScaiFolder() {
-    const backupDir = getBackupDir();
     try {
-        await fsp.mkdir(backupDir, { recursive: true });
-        const files = await fsp.readdir(SCAI_HOME);
-        for (const file of files) {
-            const srcPath = path.join(SCAI_HOME, file);
-            const destPath = path.join(backupDir, file);
-            const stat = await fsp.stat(srcPath);
-            if (stat.isFile()) {
-                await fsp.copyFile(srcPath, destPath);
-            }
+        const homeDir = os.homedir();
+        const scaiDir = path.join(homeDir, '.scai');
+        // Ensure the .scai folder exists
+        if (!fs.existsSync(scaiDir)) {
+            console.error(`⚠️ No .scai folder found at ${scaiDir}`);
+            return false;
         }
-        console.log(`📦 Backed up .scai folder to: ${backupDir}`);
-        return backupDir;
+        // Backup path: ~/.scai_backup_<timestamp>
+        const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+        const backupDir = path.join(homeDir, `.scai_backup_${timestamp}`);
+        // Recursive copy
+        copyRecursiveSync(scaiDir, backupDir);
+        return true;
     }
     catch (err) {
-        console.warn('⚠️ Failed to back up .scai folder:', err instanceof Error ? err.message : err);
-        return null;
+        console.error(`⚠️ Failed to back up .scai folder:`, err);
+        return false;
+    }
+}
+function copyRecursiveSync(src, dest) {
+    const stats = fs.statSync(src);
+    if (stats.isDirectory()) {
+        fs.mkdirSync(dest, { recursive: true });
+        const entries = fs.readdirSync(src);
+        for (const entry of entries) {
+            const srcPath = path.join(src, entry);
+            const destPath = path.join(dest, entry);
+            copyRecursiveSync(srcPath, destPath);
+        }
+    }
+    else {
+        fs.copyFileSync(src, dest);
     }
 }

package/dist/index.js

@@ -5,8 +5,6 @@ const { version } = require('../package.json');
 import { Command } from "commander";
 import { Config } from './config.js';
 import { suggestCommitMessage } from "./commands/CommitSuggesterCmd.js";
-import { handleRefactor } from "./commands/RefactorCmd.js";
-import { generateTests } from "./commands/TestGenCmd.js";
 import { bootstrap } from './modelSetup.js';
 import { summarizeFile } from "./commands/SummaryCmd.js";
 import { handleStandaloneChangelogUpdate } from './commands/ChangeLogUpdateCmd.js';
@@ -28,6 +26,10 @@ import { runInteractiveSwitch } from "./commands/SwitchCmd.js";
 import { execSync } from "child_process";
 import { fileURLToPath } from "url";
 import { dirname, resolve } from "path";
+import { handleAgentRun } from './agentManager.js';
+import { addCommentsModule } from './pipeline/modules/commentModule.js';
+import { generateTestsModule } from './pipeline/modules/generateTestsModule.js';
+import { cleanupModule } from './pipeline/modules/cleanupModule.js';
 // 🎛️ CLI Setup
 const cmd = new Command('scai')
     .version(version)
@@ -103,8 +105,9 @@ const gen = cmd.command('gen').description('Generate code-related output');
 gen
     .command('comm <file>')
     .description('Write comments for the given file')
-    .option('-a, --apply', 'Apply the refactored version to the original file')
-    .action((file, options) => handleRefactor(file, options));
+    .action((file) => {
+    handleAgentRun(file, [addCommentsModule, cleanupModule]);
+});
 gen
     .command('changelog')
     .description('Update or create the CHANGELOG.md based on current Git diff')
@@ -116,9 +119,12 @@ gen
     .description('Print a summary of the given file to the terminal')
     .action((file) => summarizeFile(file));
 gen
-    .command('tests <file>')
-    .description('Generate a Jest test file for the specified JS/TS module')
-    .action((file) => generateTests(file));
+    .command('testgen <file>')
+    .description('Generate tests for the given file')
+    .option('-a, --apply', 'Apply the output to the original file')
+    .action((file) => {
+    handleAgentRun(file, [generateTestsModule]);
+});
 // ⚙️ Group: Configuration settings
 const config = cmd.command('config').description('Manage SCAI configuration');
 config

package/dist/pipeline/modules/cleanupModule.js

@@ -1,42 +1,90 @@
-function isNaturalLanguageNoise(line) {
-    const trimmed = line.trim().toLowerCase();
-    return (trimmed.startsWith('i ') ||
-        trimmed.startsWith('here') ||
-        trimmed.startsWith('this') ||
-        trimmed.startsWith('the following') ||
-        trimmed.startsWith('below') ||
-        trimmed.startsWith('in this') ||
-        trimmed.startsWith('we have') ||
-        trimmed.includes('the code above') ||
-        trimmed.includes('ensures that') ||
-        trimmed.includes('it handles') ||
-        trimmed.includes('used to'));
+import chalk from 'chalk';
+function isTopOrBottomNoise(line) {
+    const trimmed = line.trim();
+    if (/^```(?:\w+)?$/.test(trimmed))
+        return true;
+    if (/^<!--.*-->$/.test(trimmed))
+        return true;
+    const lower = trimmed.toLowerCase();
+    if (!trimmed.startsWith('//') && !trimmed.startsWith('/*')) {
+        return [
+            /^i\s/i,
+            /^here/,
+            /^this/,
+            /^the following/,
+            /^below/,
+            /^in this/,
+            /^we have/,
+            /the code above/,
+            /ensures that/,
+            /it handles/,
+            /used to/,
+            /note that/,
+            /example/,
+            /summary/,
+            /added comments/,
+        ].some(pattern => pattern.test(lower));
+    }
+    return false;
 }
 export const cleanupModule = {
     name: 'cleanup',
-    description: 'Remove markdown fences and natural language noise from top/bottom of code',
-    async run({ content }) {
-        let lines = content.trim().split('\n');
-        // ───── Clean top ─────
-        while (lines.length) {
-            const line = lines[0].trim();
-            if (line === '' || line.startsWith('```') || isNaturalLanguageNoise(line)) {
-                lines.shift();
-            }
-            else {
-                break;
-            }
+    description: 'Remove markdown fences and fluff from top/bottom of each chunk with colored logging',
+    async run(input) {
+        // Normalize line endings to \n to avoid issues with \r\n
+        let content = input.content.replace(/\r\n/g, '\n');
+        let lines = content.split('\n');
+        // --- CLEAN TOP ---
+        // Remove noise lines before the first triple tick or end
+        while (lines.length && (lines[0].trim() === '' || isTopOrBottomNoise(lines[0]))) {
+            if (/^```(?:\w+)?$/.test(lines[0].trim()))
+                break; // Stop if opening fence found
+            console.log(chalk.red(`[cleanupModule] Removing noise from top:`), chalk.yellow(`"${lines[0].trim()}"`));
+            lines.shift();
         }
-        // ───── Clean bottom ─────
-        while (lines.length) {
-            const line = lines[lines.length - 1].trim();
-            if (line === '' || line.startsWith('```') || isNaturalLanguageNoise(line)) {
-                lines.pop();
+        // If opening fence found at top, find matching closing fence
+        if (lines.length && /^```(?:\w+)?$/.test(lines[0].trim())) {
+            console.log(chalk.red(`[cleanupModule] Found opening fenced block at top.`));
+            // Remove opening fence line
+            lines.shift();
+            // Find closing fence index
+            let closingIndex = -1;
+            for (let i = 0; i < lines.length; i++) {
+                if (/^```(?:\w+)?$/.test(lines[i].trim())) {
+                    closingIndex = i;
+                    break;
+                }
+            }
+            if (closingIndex !== -1) {
+                console.log(chalk.red(`[cleanupModule] Found closing fenced block at line ${closingIndex + 1}, removing fence lines.`));
+                // Remove closing fence line
+                lines.splice(closingIndex, 1);
             }
             else {
-                break;
+                console.log(chalk.yellow(`[cleanupModule] No closing fenced block found, only removed opening fence.`));
             }
+            // NO removal of noise lines after fenced block here (to keep new comments intact)
+        }
+        // --- CLEAN BOTTOM ---
+        // If closing fence found at bottom, remove only that triple tick line
+        if (lines.length && /^```(?:\w+)?$/.test(lines[lines.length - 1].trim())) {
+            console.log(chalk.red(`[cleanupModule] Removing closing fenced block line at bottom.`));
+            lines.pop();
         }
+        // Remove noise lines after closing fence (now bottom)
+        while (lines.length && (lines[lines.length - 1].trim() === '' || isTopOrBottomNoise(lines[lines.length - 1]))) {
+            console.log(chalk.red(`[cleanupModule] Removing noise from bottom after fenced block:`), chalk.yellow(`"${lines[lines.length - 1].trim()}"`));
+            lines.pop();
+        }
+        // --- FINAL CLEANUP: REMOVE ANY LINGERING TRIPLE TICK LINES ANYWHERE ---
+        lines = lines.filter(line => {
+            const trimmed = line.trim();
+            if (/^```(?:\w+)?$/.test(trimmed)) {
+                console.log(chalk.red(`[cleanupModule] Removing lingering triple tick line:`), chalk.yellow(`"${line}"`));
+                return false;
+            }
+            return true;
+        });
         return { content: lines.join('\n').trim() };
     }
 };

package/dist/pipeline/modules/commentModule.js

@@ -2,7 +2,7 @@ import { Config } from '../../config.js';
 import { generate } from '../../lib/generate.js';
 import { detectFileType } from '../../fileRules/detectFileType.js';
 export const addCommentsModule = {
-    name: 'addComments',
+    name: 'comments',
     description: 'Adds meaningful comments to any file type (code, config, or data)',
     async run(input) {
         const model = Config.getModel();
@@ -37,23 +37,37 @@ export const addCommentsModule = {
             text: '#',
         };
         const commentSyntax = commentMap[fileType] || '//';
+        const chunkLabel = (input.chunkIndex && input.chunkCount && input.chunkCount > 1)
+            ? `CHUNK ${input.chunkIndex + 1}/${input.chunkCount} of ${input.filepath}`
+            : `FULL FILE: ${input.filepath}`;
         const prompt = `
 You are a senior engineer reviewing a ${fileType} file.
 
-Your task is to add clear, helpful, and accurate comments using "${commentSyntax}" syntax.
-This applies to *every* section where an explanation could help — even for simple code or configuration.
+Please:
+
+1. Add a concise summary comment above each function describing its purpose.  
+   - Do NOT write trivial summaries like "..." or empty comments.  
+   - Do NOT overwrite or remove any existing comments.
+
+2. Add clear, helpful inline comments to explain non-obvious logic inside functions.  
+   - Preserve all existing comments as-is.
+
+3. Use "${commentSyntax}" as the comment syntax appropriate for ${fileType}.
+
+4. Preserve all original formatting, whitespace, and code exactly.
 
 Rules:
-- Always return the full original file with comments added.
-- Preserve all formatting, whitespace, and code exactly.
-- Use only "${commentSyntax}" comments that are valid for ${fileType}.
-- Do not wrap the output in markdown or code fences.
+- Return the full original chunk of code with added comments only.
+- Do NOT remove, change, or overwrite any existing code or comments.
+- Summaries should be brief but meaningful.
+- Inline comments should clarify complex or tricky parts only.
 
---- FILE START ---
+--- ${chunkLabel} START ---
 ${input.content}
---- FILE END ---
+--- ${chunkLabel} END ---
 `.trim();
         const response = await generate({ content: prompt }, model);
-        return { content: response.content === 'NO UPDATE' ? '' : response.content };
+        const contentToReturn = (response.content && response.content !== 'NO UPDATE') ? response.content : input.content;
+        return { content: contentToReturn };
     },
 };

package/dist/pipeline/modules/commitSuggesterModule.js

@@ -7,11 +7,17 @@ export const commitSuggesterModule = {
         const model = Config.getModel();
         const prompt = `
 Suggest ALWAYS 3 concise, conventional Git commit messages based on the input code diff.
-Use this format ONLY:
 
-1. feat: ...
-2. fix: ...
-3. refactor: ...
+- Follow the Conventional Commits specification for prefixes (e.g., feat:, fix:, refactor:, docs:, style:, test:, chore:).
+- Choose the most appropriate prefix for each message based on the changes in the diff.
+- Do NOT repeat the same prefix unless it is clearly the most appropriate.
+- Keep each message short, clear, and action-oriented.
+
+Format your response exactly as:
+
+1. <type>: <message>
+2. <type>: <message>
+3. <type>: <message>
 
 Here is the diff:
 ${content}

package/dist/pipeline/modules/generateTestsModule.js

@@ -1,10 +1,9 @@
-// src/pipeline/modules/generateTestsModule.ts
 import fs from 'fs/promises';
 import path from 'path';
 import { Config } from '../../config.js';
 import { generate } from '../../lib/generate.js';
 export const generateTestsModule = {
-    name: 'generateTests',
+    name: 'tests',
     description: 'Generate a Jest test file for the class/module',
     async run({ content, filepath }) {
         const model = Config.getModel();

package/dist/pipeline/modules/summaryModule.js

@@ -8,7 +8,6 @@ export const summaryModule = {
         const model = Config.getModel();
         const ext = filepath ? path.extname(filepath).toLowerCase() : '';
         const filename = filepath ? path.basename(filepath) : '';
-        // More neutral prompt for general-purpose content
         const prompt = `
 You are an assistant specialized in summarizing files.
 

package/dist/pipeline/registry/moduleRegistry.js

@@ -16,6 +16,10 @@ const builtInModules = {
 export function getModuleByName(name) {
     return builtInModules[name];
 }
+// Return module metadata for CLI or UI
 export function listAvailableModules() {
-    return Object.keys(builtInModules);
+    return Object.values(builtInModules).map(({ name, description }) => ({
+        name,
+        description: description || 'No description available',
+    }));
 }

package/dist/pipeline/runModulePipeline.js

@@ -1,24 +1,24 @@
+import chalk from 'chalk';
 export async function runModulePipeline(modules, input) {
     let current = input;
-    // Add flag or condition for logging (optional)
-    const isDebug = false;
-    if (isDebug) {
-        console.log('Input: ', input);
-    }
-    let response = { content: '' };
+    const isDebug = true;
     for (const mod of modules) {
         try {
-            response = await mod.run(current);
             if (isDebug) {
-                console.log(`⚙️ Running: ${mod.name}`);
-                console.log("Current: ", response.content);
+                console.log(chalk.green('➡️ Input:', current.content));
+            }
+            const response = await mod.run(current);
+            console.log(`⚙️ Running: ${mod.name}`);
+            if (isDebug) {
+                console.log(chalk.red('➡️ Output:', response.content));
             }
+            // Feed this output into the next module
+            current = response;
         }
         catch (error) {
             console.error(`❌ Error in ${mod.name}:`, error instanceof Error ? error.message : error);
             throw new Error(`Pipeline failed at module ${mod.name}`);
         }
     }
-    // Return the output, assuming 'code' holds the relevant transformed content
-    return response; // Ensure the return type matches PromptOutput
+    return current;
 }

package/dist/utils/splitCodeIntoChunk.js

@@ -0,0 +1,38 @@
+import { encode } from 'gpt-3-encoder';
+export function splitCodeIntoChunks(text, maxTokens) {
+    const lines = text.split('\n');
+    const chunks = [];
+    let currentChunkLines = [];
+    let currentTokens = 0;
+    for (const line of lines) {
+        const lineTokens = encode(line + '\n').length;
+        if (currentTokens + lineTokens > maxTokens) {
+            // Try to split at a more natural point
+            let splitIndex = currentChunkLines.length;
+            for (let i = currentChunkLines.length - 1; i >= 0; i--) {
+                const trimmed = currentChunkLines[i].trim();
+                if (trimmed === '' ||
+                    trimmed.startsWith('function ') ||
+                    trimmed.startsWith('class ') ||
+                    trimmed.endsWith('}') ||
+                    trimmed.endsWith(';')) {
+                    splitIndex = i + 1;
+                    break;
+                }
+            }
+            chunks.push(currentChunkLines.slice(0, splitIndex).join('\n'));
+            // Move leftover lines into the next chunk
+            currentChunkLines = currentChunkLines.slice(splitIndex);
+            currentTokens = encode(currentChunkLines.join('\n')).length;
+        }
+        currentChunkLines.push(line);
+        currentTokens += lineTokens;
+    }
+    if (currentChunkLines.length > 0) {
+        chunks.push(currentChunkLines.join('\n'));
+    }
+    return chunks;
+}
+export function countTokens(text) {
+    return encode(text).length;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "scai",
-  "version": "0.1.85",
+  "version": "0.1.86",
   "type": "module",
   "bin": {
     "scai": "./dist/index.js"
@@ -44,6 +44,7 @@
     "columnify": "^1.6.0",
     "commander": "^11.0.0",
     "fast-glob": "^3.3.3",
+    "gpt-3-encoder": "^1.1.4",
     "proper-lockfile": "^4.1.2",
     "string-similarity-js": "^2.1.4",
     "ts-morph": "^26.0.0"

package/dist/commands/RefactorCmd.js

@@ -1,35 +0,0 @@
-import fs from 'fs/promises';
-import path from 'path';
-import { runModulePipeline } from '../pipeline/runModulePipeline.js';
-import { addCommentsModule } from '../pipeline/modules/commentModule.js';
-import { normalizePath } from '../utils/normalizePath.js';
-export async function handleRefactor(filepath, options = {}) {
-    try {
-        // Normalize and resolve filepath (includes expanding ~ and consistent slashes)
-        filepath = normalizePath(filepath);
-        const { dir, name, ext } = path.parse(filepath);
-        const refactoredPath = path.join(dir, `${name}.refactored${ext}`);
-        if (options.apply) {
-            try {
-                const refactoredCode = await fs.readFile(refactoredPath, 'utf-8');
-                await fs.writeFile(filepath, refactoredCode, 'utf-8');
-                await fs.unlink(refactoredPath);
-                console.log(`♻️ Applied refactor: Overwrote ${filepath} and removed ${refactoredPath}`);
-            }
-            catch {
-                console.error(`❌ No saved refactor found at ${refactoredPath}`);
-            }
-            return;
-        }
-        const content = await fs.readFile(filepath, 'utf-8');
-        const response = await runModulePipeline([addCommentsModule], { content });
-        if (!response.content.trim())
-            throw new Error('⚠️ Model returned empty result');
-        await fs.writeFile(refactoredPath, response.content, 'utf-8');
-        console.log(`✅ Refactored code saved to: ${refactoredPath}`);
-        console.log(`ℹ️ Run again with '--apply' to overwrite the original.`);
-    }
-    catch (err) {
-        console.error('❌ Error in refactor command:', err instanceof Error ? err.message : err);
-    }
-}

package/dist/commands/TestGenCmd.js

@@ -1,14 +0,0 @@
-import fs from 'fs/promises';
-import { generateTestsModule } from '../pipeline/modules/generateTestsModule.js';
-import { cleanupModule } from '../pipeline/modules/cleanupModule.js';
-import { runModulePipeline } from '../pipeline/runModulePipeline.js';
-export async function generateTests(filepath) {
-    try {
-        const content = await fs.readFile(filepath, 'utf-8');
-        const result = await runModulePipeline([generateTestsModule, cleanupModule], { content, filepath });
-        console.log('✅ Test generated and cleaned up.');
-    }
-    catch (err) {
-        console.error('❌ Error generating tests:', err.message);
-    }
-}