scai 0.1.117 → 0.1.119

package/dist/index.js

@@ -1,341 +1,144 @@
 #!/usr/bin/env node
+import readline from 'readline';
+import { spawn } from 'child_process';
 import { createRequire } from 'module';
 const require = createRequire(import.meta.url);
-const { version } = require('../package.json');
-import { Command } from "commander";
-import { Config } from './config.js';
-import { suggestCommitMessage } from "./commands/CommitSuggesterCmd.js";
-import { bootstrap } from './modelSetup.js';
-import { summarizeFile } from "./commands/SummaryCmd.js";
-import { handleStandaloneChangelogUpdate } from './commands/ChangeLogUpdateCmd.js';
-import { runIndexCommand } from './commands/IndexCmd.js';
-import { resetDatabase } from './commands/ResetDbCmd.js';
-import { runFindCommand } from './commands/FindCmd.js';
-import { startDaemon } from './commands/DaemonCmd.js';
-import { runStopDaemonCommand } from "./commands/StopDaemonCmd.js";
+const shellQuote = require('shell-quote');
+import { createProgram as cmdFactory, withContext } from './commands/factory.js';
 import { runAskCommand } from './commands/AskCmd.js';
-import { runBackupCommand } from './commands/BackupCmd.js';
-import { runInspectCommand } from "./commands/InspectCmd.js";
-import { reviewPullRequestCmd } from "./commands/ReviewCmd.js";
-import { promptForToken } from "./github/token.js";
-import { validateGitHubTokenAgainstRepo } from "./github/githubAuthCheck.js";
-import { checkGit } from "./commands/GitCmd.js";
-import { runInteractiveSwitch } from "./commands/SwitchCmd.js";
-import { execSync } from "child_process";
-import { fileURLToPath } from "url";
-import { dirname, resolve } from "path";
-import { runInteractiveDelete } from './commands/DeleteIndex.js';
-import { resolveTargetsToFiles } from './utils/resolveTargetsToFiles.js';
-import { updateContext } from './context.js';
-import { Agent } from './agent/agentManager.js';
-// 🎛️ CLI Setup
-const cmd = new Command('scai')
-    .version(version, '-v, --version', 'output the current version');
-// 🔧 Main command group
-cmd
-    .command('init')
-    .description('Initialize the model and download required models')
-    .action(async () => {
-    await bootstrap();
-    console.log('✅ Model initialization completed!');
-});
-// 🔧 Group: Agent-related commands
-const agent = cmd
-    .command('agent')
-    .description(`Run an agent workflow. Example available tools:\n` +
-    `  - summary\n` +
-    `  - comments\n` +
-    `  - tests\n\n` +
-    `Example usage:\n` +
-    `  $ scai agent run comments tests -f path/to/myfile\n` +
-    `  This will run the agent with the goals: comments → tests\n`);
-// Run workflow subcommand
-const runCmd = agent
-    .command('run <goals...>')
-    .description('Run an agent workflow with a list of goals')
-    .option('-f, --file <filepath>', 'File to process', 'example.txt')
-    .action(async (cmdGoals, cmd) => {
-    await withContext(async () => {
-        const goals = cmdGoals;
-        const file = cmd.file;
-        console.log('Agent will execute:', goals.join(' → '));
-        console.log('On file:', file);
-        const agentInstance = new Agent(goals);
-        await agentInstance.execute(file);
-    });
-});
-// Optional: show example modules on --help
-runCmd.on('--help', () => {
-    console.log('\nExample tools:');
-    console.log('  - summary');
-    console.log('  - comments');
-    console.log('  - tests');
-});
-// 🔧 Group: Git-related commands
-const git = cmd.command('git').description('Git utilities');
-git
-    .command('review')
-    .description('Review an open pull request using AI')
-    .option('-a, --all', 'Show all PRs requiring a review (not just for the current user)', false)
-    .action(async (cmd) => {
-    await withContext(async () => {
-        const showAll = cmd.all;
-        await reviewPullRequestCmd('main', showAll);
-    });
-});
-git
-    .command('commit')
-    .description('Suggest a commit message from staged changes and optionally commit')
-    .option('-l, --changelog', 'Generate and optionally stage a changelog entry')
-    .action(async (options) => {
-    await withContext(async () => {
-        suggestCommitMessage(options);
-    });
-});
-git
-    .command('check')
-    .description('Check Git working directory and branch status')
-    .action(async () => {
-    await withContext(async () => {
-        checkGit();
-    });
-});
-// Add auth-related commands
-const auth = cmd.command('auth').description('GitHub authentication commands');
-// ⚡ Auth commands
-auth
-    .command('check')
-    .description('Check if GitHub authentication is set up and valid')
-    .action(async () => {
-    await withContext(async () => {
+import { setRl } from './commands/ReadlineSingleton.js';
+const program = cmdFactory(); // single Commander instance
+let inShell = false;
+const customCommands = {};
+// =====================================================
+// TEST QUERIES
+// =====================================================
+const testQueries = [
+    'please write me comprehensive comments for spatialmap.js and typescript.ts files',
+    'refactor spatialmap.js to improve readability and reduce nesting',
+    'explain the intent and architecture of the semantic analysis module',
+    'add validation and error handling to the context review step',
+    'summarize this repo architecture and identify weak coupling points',
+    'Where are all the database queries defined?',
+    'Which functions have no tests written for them yet?',
+    'What could cause memory leaks in this codebase?',
+    'How does authentication work here?',
+    'Is there a clear separation between frontend and backend code?',
+    'please refactor the buildContextualPrompt into smaller functions',
+    'Is error handling consistent across the codebase?',
+    'What’s missing from the README?',
+    'How do I run the test suite?',
+    'Are there any flaky tests in this repo?',
+    'Are there any security vulnerabilities in our dependencies?',
+    'Is there any dead code we can safely remove?'
+];
+// =====================================================
+// HELPERS
+// =====================================================
+function pickRandom(items) {
+    return items[Math.floor(Math.random() * items.length)];
+}
+async function runQuery(query) {
+    await withContext(() => runAskCommand(query));
+}
+// =====================================================
+// BUILT-IN COMMANDS
+// =====================================================
+customCommands.test = async () => {
+    await runQuery(testQueries[0]);
+};
+customCommands['test-random'] = async () => {
+    const query = pickRandom(testQueries);
+    console.log(`\n🎲 [test-random] Selected query:\n→ ${query}\n`);
+    await runQuery(query);
+};
+customCommands.exit = async () => {
+    console.log('Exiting...');
+    process.exit(0);
+};
+customCommands.q = customCommands.exit;
+customCommands.quit = customCommands.exit;
+// =====================================================
+// EXTENSION API
+// =====================================================
+export function registerCommand(name, fn) {
+    customCommands[name] = fn;
+}
+async function startShell() {
+    if (inShell)
+        return;
+    inShell = true;
+    const rl = readline.createInterface({
+        input: process.stdin,
+        output: process.stdout,
+        prompt: 'scai> ',
+        historySize: 200,
+    });
+    setRl(rl);
+    rl.prompt();
+    rl.on('line', async (line) => {
         try {
-            const token = Config.getGitHubToken();
-            if (!token) {
-                console.log('❌ GitHub authentication not found. Please set your token.');
+            const trimmed = line.trim();
+            if (!trimmed) {
+                rl.prompt();
                 return;
             }
-            const result = await validateGitHubTokenAgainstRepo();
-            console.log(result);
+            // --- Shell command
+            if (trimmed.startsWith('!')) {
+                const child = spawn(trimmed.slice(1).trim(), { shell: true, stdio: 'inherit' });
+                child.on('exit', () => rl.prompt());
+                child.on('error', (err) => { console.error('Shell command error:', err); rl.prompt(); });
+                return;
+            }
+            // --- Slash commands
+            if (trimmed.startsWith('/')) {
+                const argvParts = shellQuote.parse(trimmed.slice(1))
+                    .map((tok) => typeof tok === 'object' ? tok.op ?? tok.pattern ?? '' : String(tok))
+                    .filter(Boolean);
+                const cmdName = argvParts[0];
+                if (customCommands[cmdName]) {
+                    await customCommands[cmdName]();
+                }
+                else {
+                    try {
+                        await program.parseAsync(argvParts, { from: 'user' });
+                    }
+                    catch (err) {
+                        console.error('Command error:', err instanceof Error ? err.message : err);
+                    }
+                }
+                rl.prompt();
+                return;
+            }
+            // --- Bare input → LLM
+            await withContext(() => runAskCommand(trimmed));
+            rl.prompt();
         }
         catch (err) {
-            console.error(typeof err === 'string' ? err : err.message);
-        }
-    });
-});
-auth
-    .command('reset')
-    .description('Reset GitHub authentication credentials')
-    .action(async () => {
-    await withContext(async () => {
-        Config.setGitHubToken('');
-        console.log('🔄 GitHub authentication has been reset.');
-        const token = Config.getGitHubToken();
-        console.log(token ? '❌ Token still exists in the configuration.' : '✅ Token successfully removed.');
-    });
-});
-auth
-    .command('set')
-    .description('Set your GitHub Personal Access Token')
-    .action(async () => {
-    await withContext(async () => {
-        const token = await promptForToken();
-        Config.setGitHubToken(token.trim());
-        console.log('🔑 GitHub token set successfully.');
-    });
-});
-// 🛠️ Group: `gen` commands for content generation
-const gen = cmd.command('gen').description('Generate code-related output');
-gen
-    .command("comm <targets...>")
-    .description("Write comments for the given file(s) or folder(s)")
-    .action(async (targets) => {
-    await withContext(async () => {
-        const files = await resolveTargetsToFiles(targets);
-        for (const file of files) {
-            const agent = new Agent(["comments"]); // goals: "comments"
-            await agent.execute(file); // run on the file
-        }
-    });
-});
-gen
-    .command("test <targets...>")
-    .description("Generate tests for the given file(s) or folder(s)")
-    .action(async (targets) => {
-    await withContext(async () => {
-        const files = await resolveTargetsToFiles(targets);
-        for (const file of files) {
-            const agent = new Agent(["tests"]); // goals: "tests"
-            await agent.execute(file);
-        }
-    });
-});
-gen
-    .command('changelog')
-    .description('Update or create the CHANGELOG.md based on current Git diff')
-    .action(async () => {
-    await withContext(async () => {
-        await handleStandaloneChangelogUpdate();
-    });
-});
-gen
-    .command('summ [file]')
-    .description('Print a summary of the given file to the terminal')
-    .action(async (file) => {
-    await withContext(async () => {
-        summarizeFile(file);
-    });
-});
-// ⚙️ Group: Configuration settings
-const config = cmd.command('config').description('Manage SCAI configuration');
-config
-    .command('set-model <model>')
-    .description('Set the model to use')
-    .option('-g, --global', 'Set the global default model instead of the active repo')
-    .action(async (model, options) => {
-    await withContext(async () => {
-        const scope = options.global ? 'global' : 'repo';
-        Config.setModel(model, scope);
-        Config.show();
-    });
-});
-config
-    .command('set-lang <lang>')
-    .description('Set the programming language')
-    .action(async (lang) => {
-    await withContext(async () => {
-        Config.setLanguage(lang);
-        Config.show();
-    });
-});
-config
-    .command("show")
-    .option("--raw", "Show full raw config")
-    .description("Display current configuration")
-    .action(async (options) => {
-    await withContext(async () => {
-        if (options.raw) {
-            console.log(JSON.stringify(Config.getRaw(), null, 2));
-        }
-        else {
-            Config.show();
+            console.error('REPL error:', err instanceof Error ? err.stack : err);
+            rl.prompt();
         }
     });
-});
-const index = cmd.command('index').description('index operations');
-index
-    .command('start')
-    .description('Index supported files in the configured index directory')
-    .action(async () => await withContext(async () => {
-    await runIndexCommand();
-}));
-index
-    .command('set [dir]')
-    .description('Set and activate index directory')
-    .action(async (dir = process.cwd()) => {
-    await Config.setIndexDir(dir);
-    Config.show();
-});
-index
-    .command('list')
-    .description('List all indexed repositories')
-    .action(async () => await withContext(async () => {
-    await Config.printAllRepos();
-}));
-index
-    .command('switch')
-    .description('Switch active repository (by interactive list only)')
-    .action(async () => await withContext(async () => {
-    await runInteractiveSwitch();
-}));
-index
-    .command('delete')
-    .description('Delete a repository from the index (interactive)')
-    .action(async () => await withContext(async () => {
-    await runInteractiveDelete();
-}));
-const db = cmd.command('db').description('Database operations');
-db
-    .command('check')
-    .description('Run the dbcheck script to check the database status')
-    .action(async () => await withContext(async () => {
-    const __filename = fileURLToPath(import.meta.url);
-    const __dirname = dirname(__filename);
-    const scriptPath = resolve(__dirname, '..', 'dist/scripts', 'dbcheck.js');
-    console.log(`🚀 Running database check script: ${scriptPath}`);
+    rl.on('close', () => { console.log('Bye!'); process.exit(0); });
+    process.on('SIGINT', () => { console.log('\nExiting REPL...'); rl.close(); });
+}
+// ---------------- Main -----------------
+async function main() {
+    process.on('unhandledRejection', (reason) => console.error('Unhandled Rejection:', reason));
+    process.on('uncaughtException', (err) => { console.error('Uncaught Exception:', err.stack ?? err); process.exit(1); });
+    const isInteractive = process.stdin.isTTY && process.stdout.isTTY;
+    const args = process.argv.slice(2);
+    // Run REPL if no args OR user typed `scai shell`
+    if (isInteractive && (args.length === 0 || (args.length === 1 && args[0] === 'shell'))) {
+        await startShell();
+        return;
+    }
     try {
-        execSync(`node "${scriptPath}"`, { stdio: 'inherit' });
+        await program.parseAsync(process.argv);
     }
     catch (err) {
-        console.error('❌ Error running dbcheck script:', err instanceof Error ? err.message : err);
+        console.error('CLI Error:', err instanceof Error ? err.message : err);
+        process.exit(1);
     }
-}));
-db
-    .command('reset')
-    .description('Delete and reset the SQLite database')
-    .action(async () => await withContext(async () => {
-    await resetDatabase();
-}));
-db
-    .command('inspect')
-    .argument('<filepath>', 'Path to the file to inspect')
-    .description('Inspect a specific file and print its indexed summary and functions')
-    .action(async (filepath) => await withContext(async () => {
-    await runInspectCommand(filepath);
-}));
-const daemon = cmd
-    .command('daemon')
-    .description('Background summarizer operations');
-// Start the daemon
-daemon
-    .command('start')
-    .description('Run background summarization of indexed files')
-    .action(async () => {
-    await withContext(async () => {
-        await startDaemon();
-    });
-});
-// Stop the daemon
-daemon
-    .command('stop')
-    .description('Stop the background summarizer daemon')
-    .action(async () => {
-    await withContext(async () => {
-        await runStopDaemonCommand();
-    });
-});
-cmd
-    .command('backup')
-    .description('Backup the current .scai folder')
-    .action(async () => await withContext(async () => {
-    await runBackupCommand();
-}));
-cmd
-    .command('find <query>')
-    .description('Search indexed files by keyword')
-    .action(async (query) => await withContext(async () => {
-    await runFindCommand(query);
-}));
-cmd
-    .command('ask [question...]')
-    .description('Ask a question based on indexed files')
-    .action(async (questionParts) => await withContext(async () => {
-    const fullQuery = questionParts?.join(' ');
-    await runAskCommand(fullQuery);
-}));
-cmd.addHelpText('after', `
-🚨 Alpha Features:
-  - The "index", "daemon", "stop-daemon", "reset-db" commands are considered alpha features.
-  - These commands are in active development and may change in the future.
-
-💡 Use with caution and expect possible changes or instability.
-`);
-async function withContext(action) {
-    const ok = await updateContext();
-    //if (!ok) process.exit(1);
-    await action();
 }
-// 👇 this should be the very last line
-cmd.parseAsync(process.argv).catch((err) => {
-    console.error(err.message); // only show our styled message
-    process.exit(1);
-});
+main();

package/dist/lib/generate.js

@@ -2,24 +2,36 @@
 import { Spinner } from './spinner.js';
 import { Config, readConfig } from '../config.js';
 import { startModelProcess } from '../utils/checkModel.js';
+/**
+ * The generate module uses the local model API to produce output
+ * based on the `content` of the input. It returns the response
+ * on the `data` property of ModuleIO.
+ */
 export async function generate(input) {
     const model = Config.getModel();
-    const contextLength = readConfig().contextLength ?? 8192;
-    let prompt = input.content;
-    const spinner = new Spinner(`🧠 Thinking with ${model}...`);
+    const { contextLength } = readConfig();
+    // Safely build prompt
+    const queryPart = input.query ? `User query:\n${input.query}\n\n` : "";
+    const contentPart = input.content
+        ? (typeof input.content === "string"
+            ? input.content
+            : JSON.stringify(input.content, null, 2))
+        : "";
+    const prompt = `${queryPart}${contentPart}`.trim();
+    const spinner = new Spinner();
     spinner.start();
     try {
-        return await doGenerate(prompt, model, spinner, false);
+        const data = await doGenerate(prompt, model, contextLength, spinner, false);
+        return { query: input.query, data };
     }
-    catch (err) {
-        spinner.fail('Model request failed. Attempting to start model...');
-        process.stdout.write('\n');
+    catch {
+        spinner.fail('Model request failed. Attempting restart...');
         await startModelProcess();
-        spinner.update(`🧠 Retrying with ${model}...`);
-        return await doGenerate(prompt, model, spinner, true); // retry once
+        const data = await doGenerate(prompt, model, contextLength, spinner, true);
+        return { query: input.query, data };
     }
 }
-async function doGenerate(prompt, model, spinner, retrying) {
+async function doGenerate(prompt, model, contextLength, spinner, retrying) {
     const res = await fetch('http://localhost:11434/api/generate', {
         method: 'POST',
         headers: { 'Content-Type': 'application/json' },
@@ -27,12 +39,23 @@ async function doGenerate(prompt, model, spinner, retrying) {
             model,
             prompt,
             stream: false,
+            options: {
+                num_ctx: contextLength,
+            },
         }),
     });
+    if (!res.ok) {
+        throw new Error(`Model request failed with status ${res.status}`);
+    }
     const data = await res.json();
-    spinner.succeed(retrying ? 'Model response received after restart.' : 'Model response received.');
+    if (retrying) {
+        spinner.succeed('Model response received after restart.');
+    }
+    else {
+        spinner.succeed();
+    }
+    process.stdout.write('\n');
+    return data.response?.trim() ?? '';
     process.stdout.write('\n');
-    return {
-        content: data.response?.trim() ?? '',
-    };
+    return data.response?.trim() ?? '';
 }

package/dist/lib/generateFolderCapsules.js

@@ -0,0 +1,109 @@
+import { getDbForRepo } from "../db/client.js";
+import { generate } from "../lib/generate.js";
+import { cleanupModule } from "../pipeline/modules/cleanupModule.js";
+import path from "path";
+export async function generateFolderCapsules() {
+    const db = getDbForRepo();
+    // --- Load files (paths only) ---
+    const files = db
+        .prepare(`SELECT path FROM files`)
+        .all();
+    // --- Group files by folder ---
+    const folders = new Map();
+    for (const { path: filePath } of files) {
+        const dir = path.dirname(filePath) || "/";
+        const depth = dir === "/" ? 0 : dir.split("/").length;
+        if (!folders.has(dir)) {
+            folders.set(dir, {
+                path: dir,
+                depth,
+                files: [],
+            });
+        }
+        folders.get(dir).files.push({
+            name: path.basename(filePath),
+            ext: path.extname(filePath).replace(".", "") || "unknown",
+        });
+    }
+    // --- Generate one capsule per folder ---
+    for (const folder of folders.values()) {
+        const byType = {};
+        for (const f of folder.files) {
+            byType[f.ext] = (byType[f.ext] ?? 0) + 1;
+        }
+        const inputPayload = {
+            path: folder.path,
+            depth: folder.depth,
+            fileCount: folder.files.length,
+            byType,
+            files: folder.files.map((f) => f.name),
+        };
+        const input = {
+            query: `Generate a FolderCapsule for ${folder.path}`,
+            content: `
+You are generating a structured FolderCapsule describing the architectural role of a folder.
+
+Return ONLY valid JSON matching this TypeScript interface:
+
+interface FolderCapsule {
+  path: string;
+  depth: number;
+  stats: {
+    fileCount: number;
+    byType: Record<string, number>;
+  };
+  roles: string[];
+  concerns: string[];
+  keyFiles: { path: string; reason: string }[];
+  dependencies: {
+    importsFrom: string[];
+    usedBy: string[];
+  };
+  confidence: number;
+}
+
+Folder facts:
+${JSON.stringify(inputPayload, null, 2)}
+      `.trim(),
+        };
+        const response = await generate(input);
+        // --- Cleanup model output ---
+        const cleaned = await cleanupModule.run({
+            query: input.query,
+            content: response.content,
+        });
+        let capsule = null;
+        try {
+            capsule = JSON.parse(String(cleaned.content));
+        }
+        catch {
+            console.warn(`⚠️ Failed to parse FolderCapsule for ${folder.path}`);
+            continue;
+        }
+        db.prepare(`
+      INSERT INTO folder_capsules
+        (path, depth, capsule_json, confidence, last_generated, source_file_count)
+      VALUES
+        (
+          @path,
+          @depth,
+          @capsule_json,
+          @confidence,
+          datetime('now'),
+          @source_file_count
+        )
+      ON CONFLICT(path) DO UPDATE SET
+        capsule_json = excluded.capsule_json,
+        confidence = excluded.confidence,
+        last_generated = datetime('now'),
+        source_file_count = excluded.source_file_count
+    `).run({
+            path: folder.path,
+            depth: folder.depth,
+            capsule_json: JSON.stringify(capsule),
+            confidence: capsule?.confidence ?? null,
+            source_file_count: folder.files.length,
+        });
+        console.log(`📦 Folder capsule generated for ${folder.path}`);
+    }
+}