scythe-context-mcp 0.1.0

package/dist/indexing/indexStatus.js

@@ -0,0 +1,224 @@
+import fs from "node:fs";
+import path from "node:path";
+import Database from "better-sqlite3";
+import { scanProject } from "./scanner.js";
+import { sha256Hex } from "./hash.js";
+export function recommendedNextActions(status, options) {
+    if (!status.exists) {
+        return [
+            "Run repo_reindex with dry_run=false to create the metadata index.",
+            "Then run repo_reindex with dry_run=false and index_embeddings=true when semantic search or context packs need vectors.",
+        ];
+    }
+    const actions = [];
+    if (options.freshness?.status === "stale") {
+        actions.push("Run repo_reindex with dry_run=false to refresh stale, new, missing, or metadata-changed files.");
+    }
+    if (status.files === 0 || status.chunks === 0) {
+        actions.push("Run repo_reindex with dry_run=false to index files and chunks.");
+    }
+    if (status.ftsRows < status.chunks) {
+        actions.push("Run repo_reindex with dry_run=false to repair missing keyword-search rows.");
+    }
+    if (status.symbols === 0 && status.files > 0) {
+        actions.push("Run repo_reindex with dry_run=false to populate symbol metadata.");
+    }
+    if (status.dependencies === 0 && status.files > 1) {
+        actions.push("Run repo_reindex with dry_run=false to populate dependency metadata.");
+    }
+    const matchingEmbeddingSet = status.embeddingSets.find((set) => set.dimensions === options.desiredDimensions);
+    if (!matchingEmbeddingSet || matchingEmbeddingSet.embeddings === 0) {
+        actions.push(`Run repo_reindex with dry_run=false and index_embeddings=true to create ${options.desiredDimensions}-dimension embeddings for semantic search.`);
+    }
+    else if (matchingEmbeddingSet.embeddings < status.chunks) {
+        actions.push("Run repo_reindex with dry_run=false and index_embeddings=true to fill missing embeddings for newly indexed chunks.");
+    }
+    if (actions.length === 0) {
+        actions.push("Index is ready. Prefer repo_context_pack for task-oriented lookup.");
+    }
+    return actions;
+}
+function tableExists(db, name) {
+    const row = db
+        .prepare("select 1 as existsFlag from sqlite_master where type in ('table', 'virtual table') and name = ?")
+        .get(name);
+    return Boolean(row);
+}
+function countTableRows(db, name) {
+    if (!tableExists(db, name))
+        return 0;
+    return db.prepare(`select count(*) as count from ${name}`).get().count;
+}
+export function readDetailedIndexStatus(dbPath) {
+    if (!fs.existsSync(dbPath)) {
+        return {
+            exists: false,
+            dbPath,
+            files: 0,
+            chunks: 0,
+            ftsRows: 0,
+            symbols: 0,
+            dependencies: 0,
+            embeddingSets: [],
+        };
+    }
+    const db = new Database(dbPath, { readonly: true });
+    try {
+        const files = countTableRows(db, "files");
+        const chunks = countTableRows(db, "chunks");
+        const ftsRows = countTableRows(db, "chunk_fts");
+        const symbols = countTableRows(db, "file_symbols");
+        const dependencies = countTableRows(db, "file_dependencies");
+        const embeddingSets = tableExists(db, "embedding_sets")
+            ? db
+                .prepare(`
+        select embedding_sets.id,
+               embedding_sets.provider,
+               embedding_sets.model,
+               embedding_sets.dimensions,
+               count(embeddings.id) as embeddings
+        from embedding_sets
+        left join embeddings on embeddings.embedding_set_id = embedding_sets.id
+        group by embedding_sets.id
+        order by embedding_sets.id
+      `)
+                .all()
+            : [];
+        return { exists: true, dbPath, files, chunks, ftsRows, symbols, dependencies, embeddingSets };
+    }
+    finally {
+        db.close();
+    }
+}
+function readIndexedProjectPaths(db) {
+    if (!tableExists(db, "files"))
+        return [];
+    return db.prepare("select distinct project_path as projectPath from files order by project_path").all().map((row) => row.projectPath);
+}
+function safeRealpath(value) {
+    try {
+        return fs.realpathSync(value);
+    }
+    catch {
+        return undefined;
+    }
+}
+function projectPathAliases(db, projectPath) {
+    const aliases = new Set([projectPath]);
+    const projectRealpath = safeRealpath(projectPath);
+    const indexedProjectPaths = readIndexedProjectPaths(db);
+    for (const indexedProjectPath of indexedProjectPaths) {
+        if (projectRealpath && safeRealpath(indexedProjectPath) === projectRealpath) {
+            aliases.add(indexedProjectPath);
+        }
+    }
+    if (aliases.size === 1 && indexedProjectPaths.length === 1) {
+        aliases.add(indexedProjectPaths[0]);
+    }
+    return Array.from(aliases);
+}
+function readIndexedFileRows(db, projectPaths) {
+    if (!tableExists(db, "files"))
+        return [];
+    if (projectPaths.length === 0)
+        return [];
+    const placeholders = projectPaths.map(() => "?").join(", ");
+    return db
+        .prepare(`
+      select project_path as projectPath, path, size, mtime_ms as mtimeMs, hash
+      from files
+      where project_path in (${placeholders})
+      order by path
+    `)
+        .all(...projectPaths);
+}
+function pushSample(samples, sample, maxSamples) {
+    if (samples.length < maxSamples) {
+        samples.push(sample);
+    }
+}
+export async function readIndexFreshness(options) {
+    const checkedAt = new Date().toISOString();
+    const projectPath = path.resolve(options.projectPath);
+    const maxSamples = options.maxSamples ?? 20;
+    if (!fs.existsSync(options.dbPath)) {
+        return {
+            checked: false,
+            status: "index_missing",
+            checkedAt,
+            indexedFiles: 0,
+            indexedProjectPaths: [],
+            scannedFiles: 0,
+            staleFiles: 0,
+            newFiles: 0,
+            modifiedFiles: 0,
+            metadataChangedFiles: 0,
+            missingFiles: 0,
+            skippedFiles: 0,
+            samples: [],
+        };
+    }
+    const db = new Database(options.dbPath, { readonly: true });
+    let indexedRows;
+    let indexedProjectPaths;
+    try {
+        indexedProjectPaths = projectPathAliases(db, projectPath);
+        indexedRows = readIndexedFileRows(db, indexedProjectPaths);
+    }
+    finally {
+        db.close();
+    }
+    const scan = await scanProject(projectPath, options.limits);
+    const indexedByPath = new Map(indexedRows.map((row) => [row.path, row]));
+    const scannedByPath = new Map(scan.files.map((file) => [file.relativePath, file]));
+    const samples = [];
+    let newFiles = 0;
+    let modifiedFiles = 0;
+    let metadataChangedFiles = 0;
+    let missingFiles = 0;
+    for (const file of scan.files) {
+        if (!indexedByPath.has(file.relativePath)) {
+            newFiles += 1;
+            pushSample(samples, { path: file.relativePath, reason: "new" }, maxSamples);
+        }
+    }
+    for (const indexed of indexedRows) {
+        const file = scannedByPath.get(indexed.path);
+        if (!file) {
+            missingFiles += 1;
+            pushSample(samples, { path: indexed.path, reason: "missing" }, maxSamples);
+            continue;
+        }
+        const sizeChanged = indexed.size !== file.size;
+        const mtimeChanged = Math.abs(indexed.mtimeMs - file.mtimeMs) > 1;
+        if (!sizeChanged && !mtimeChanged) {
+            continue;
+        }
+        const content = fs.readFileSync(file.absolutePath);
+        const currentHash = sha256Hex(content);
+        if (currentHash !== indexed.hash) {
+            modifiedFiles += 1;
+            pushSample(samples, { path: indexed.path, reason: "modified" }, maxSamples);
+        }
+        else {
+            metadataChangedFiles += 1;
+            pushSample(samples, { path: indexed.path, reason: "metadata_changed" }, maxSamples);
+        }
+    }
+    const staleFiles = newFiles + modifiedFiles + metadataChangedFiles + missingFiles;
+    return {
+        checked: true,
+        status: staleFiles === 0 ? "fresh" : "stale",
+        checkedAt,
+        indexedFiles: indexedRows.length,
+        indexedProjectPaths,
+        scannedFiles: scan.files.length,
+        staleFiles,
+        newFiles,
+        modifiedFiles,
+        metadataChangedFiles,
+        missingFiles,
+        skippedFiles: scan.skipped.length,
+        samples,
+    };
+}

package/dist/indexing/indexWriter.js

@@ -0,0 +1,106 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import Database from "better-sqlite3";
+import { chunkText } from "./chunker.js";
+import { resolveIndexingLimits } from "./dryRun.js";
+import { sha256Hex } from "./hash.js";
+import { scanProject } from "./scanner.js";
+import { extractFileGraph, resolveDependencyPath } from "./symbolGraph.js";
+import { deleteChunksForFile, initializeStorageSchema, insertChunk, rebuildChunkFtsForFile, replaceSymbolGraphForFile, upsertFile, } from "../storage/schema.js";
+export async function persistentReindexMetadata(options) {
+    const projectPath = path.resolve(options.projectPath);
+    const limits = resolveIndexingLimits(options);
+    const indexPath = path.join(projectPath, options.indexDirName);
+    const dbPath = path.join(indexPath, "index.sqlite");
+    await fs.mkdir(indexPath, { recursive: true });
+    const scan = await scanProject(projectPath, limits);
+    const activePathSet = new Set(scan.files.map((file) => file.relativePath));
+    const db = new Database(dbPath);
+    let chunkCount = 0;
+    let byteCount = 0;
+    let symbolCount = 0;
+    let dependencyCount = 0;
+    try {
+        initializeStorageSchema(db, { vectorDimensions: options.vectorDimensions });
+        const writeFileMetadata = db.transaction((input) => {
+            const existing = db
+                .prepare("select id, hash from files where project_path = ? and path = ?")
+                .get(projectPath, input.relativePath);
+            const fileId = upsertFile(db, {
+                projectPath,
+                path: input.relativePath,
+                mtimeMs: input.mtimeMs,
+                size: input.size,
+                hash: input.hash,
+            });
+            const graph = extractFileGraph(input.relativePath, input.content);
+            replaceSymbolGraphForFile(db, fileId, graph.symbols.map((symbol) => ({ fileId, ...symbol })), graph.dependencies.map((dependency) => ({
+                fileId,
+                specifier: dependency.specifier,
+                line: dependency.line,
+                resolvedPath: resolveDependencyPath(input.relativePath, dependency.specifier, activePathSet),
+            })));
+            if (existing?.hash === input.hash) {
+                rebuildChunkFtsForFile(db, fileId, input.relativePath);
+                return graph;
+            }
+            deleteChunksForFile(db, fileId);
+            for (const chunk of input.chunks) {
+                insertChunk(db, {
+                    fileId,
+                    path: input.relativePath,
+                    startLine: chunk.startLine,
+                    endLine: chunk.endLine,
+                    title: input.relativePath,
+                    text: chunk.text,
+                    hash: chunk.hash,
+                });
+            }
+            return graph;
+        });
+        for (const file of scan.files) {
+            const content = await fs.readFile(file.absolutePath, "utf8");
+            const chunks = chunkText(file.relativePath, content, limits);
+            const graph = writeFileMetadata({
+                relativePath: file.relativePath,
+                size: file.size,
+                mtimeMs: file.mtimeMs,
+                hash: sha256Hex(content),
+                chunks,
+                content,
+            });
+            chunkCount += chunks.length;
+            byteCount += file.size;
+            symbolCount += graph.symbols.length;
+            dependencyCount += graph.dependencies.length;
+        }
+        const activePaths = scan.files.map((file) => file.relativePath);
+        if (activePaths.length === 0) {
+            db.prepare("delete from files where project_path = ?").run(projectPath);
+        }
+        else {
+            const placeholders = activePaths.map(() => "?").join(", ");
+            db.prepare(`delete from files where project_path = ? and path not in (${placeholders})`).run(projectPath, ...activePaths);
+        }
+    }
+    finally {
+        db.close();
+    }
+    return {
+        projectPath,
+        dryRun: false,
+        status: "metadata_indexed",
+        dbPath,
+        limits,
+        stats: {
+            scannedFiles: scan.files.length + scan.skipped.length,
+            indexedFiles: scan.files.length,
+            skippedFiles: scan.skipped.length,
+            chunks: chunkCount,
+            bytes: byteCount,
+            symbols: symbolCount,
+            dependencies: dependencyCount,
+        },
+        skipped: scan.skipped,
+    };
+}

package/dist/indexing/keywordSearch.js

@@ -0,0 +1,86 @@
+import Database from "better-sqlite3";
+function compactSnippet(text, maxChars) {
+    const normalized = text.replace(/\s+$/g, "");
+    if (normalized.length <= maxChars)
+        return normalized;
+    return `${normalized.slice(0, Math.max(0, maxChars - 3))}...`;
+}
+export function keywordTerms(query) {
+    return Array.from(new Set(query.match(/[\p{L}\p{N}_.$/-]{2,}/gu) || []));
+}
+function ftsQuery(query) {
+    const terms = keywordTerms(query).filter((term) => term.length >= 3);
+    if (terms.length === 0)
+        return undefined;
+    return terms.map((term) => `"${term.replace(/"/g, '""')}"`).join(" OR ");
+}
+export function searchByKeyword(options) {
+    if (!Number.isInteger(options.maxResults) || options.maxResults <= 0) {
+        throw new Error("maxResults must be a positive integer");
+    }
+    if (!Number.isInteger(options.maxSnippetChars) || options.maxSnippetChars <= 0) {
+        throw new Error("maxSnippetChars must be a positive integer");
+    }
+    const db = new Database(options.dbPath, { readonly: true });
+    try {
+        const query = ftsQuery(options.query);
+        const rows = query
+            ? db
+                .prepare(`
+            select files.path,
+                   chunks.start_line as startLine,
+                   chunks.end_line as endLine,
+                   chunks.text,
+                   bm25(chunk_fts) as score
+            from chunk_fts
+            join chunks on chunks.id = chunk_fts.rowid
+            join files on files.id = chunks.file_id
+            where chunk_fts match ?
+            order by score
+            limit ?
+          `)
+                .all(query, options.maxResults)
+            : [];
+        if (rows.length > 0) {
+            return rows.map((row) => ({
+                path: row.path,
+                startLine: row.startLine,
+                endLine: row.endLine,
+                score: row.score,
+                snippet: compactSnippet(row.text, options.maxSnippetChars),
+                matchType: "keyword",
+            }));
+        }
+        const fallbackTerm = keywordTerms(options.query)[0];
+        if (!fallbackTerm)
+            return [];
+        const fallbackRows = db
+            .prepare(`
+        select files.path,
+               chunks.start_line as startLine,
+               chunks.end_line as endLine,
+               chunks.text,
+               case
+                 when files.path like @pattern then -10.0
+                 else -1.0
+               end as score
+        from chunks
+        join files on files.id = chunks.file_id
+        where files.path like @pattern or chunks.text like @pattern
+        order by score, files.path, chunks.start_line
+        limit @maxResults
+      `)
+            .all({ pattern: `%${fallbackTerm}%`, maxResults: options.maxResults });
+        return fallbackRows.map((row) => ({
+            path: row.path,
+            startLine: row.startLine,
+            endLine: row.endLine,
+            score: row.score,
+            snippet: compactSnippet(row.text, options.maxSnippetChars),
+            matchType: "keyword",
+        }));
+    }
+    finally {
+        db.close();
+    }
+}

package/dist/indexing/relatedFiles.js

@@ -0,0 +1,137 @@
+import fs from "node:fs";
+import Database from "better-sqlite3";
+function tableExists(db, name) {
+    const row = db
+        .prepare("select 1 as existsFlag from sqlite_master where type in ('table', 'virtual table') and name = ?")
+        .get(name);
+    return Boolean(row);
+}
+export function classifyRelatedPath(path) {
+    const normalized = path.toLowerCase();
+    if (/(^|\/)(test|tests|__tests__|spec|specs)(\/|$)|\.(test|spec)\.[^.]+$/.test(normalized))
+        return "test";
+    if (/(^|\/)(__mocks__|mocks?)(\/|$)|\bmock\b/.test(normalized))
+        return "mock";
+    if (/(^|\/)(fixtures?|samples?)(\/|$)/.test(normalized))
+        return "fixture";
+    if (/(\.generated\.|\.gen\.|\/generated\/|\/dist\/|\/build\/)/.test(normalized))
+        return "generated";
+    if (/\.(md|mdx|rst|txt)$|(^|\/)docs?\//.test(normalized))
+        return "docs";
+    return "source";
+}
+function relatedPathScore(path) {
+    switch (classifyRelatedPath(path)) {
+        case "source":
+            return 0;
+        case "docs":
+            return 1;
+        case "test":
+            return 2;
+        case "mock":
+            return 3;
+        case "fixture":
+            return 4;
+        case "generated":
+            return 5;
+    }
+}
+function sortRelatedPaths(paths) {
+    return paths
+        .map((path, index) => ({ path, index }))
+        .sort((a, b) => relatedPathScore(a.path) - relatedPathScore(b.path) || a.index - b.index || a.path.localeCompare(b.path))
+        .map((item) => item.path);
+}
+function sortQueue(queue) {
+    queue.sort((a, b) => a.depth - b.depth ||
+        relatedPathScore(a.path) - relatedPathScore(b.path) ||
+        a.order - b.order ||
+        a.path.localeCompare(b.path));
+}
+export function readRelatedFiles(options) {
+    if (!fs.existsSync(options.dbPath)) {
+        throw new Error(`Index database not found: ${options.dbPath}`);
+    }
+    const db = new Database(options.dbPath, { readonly: true });
+    try {
+        if (!tableExists(db, "files") || !tableExists(db, "file_symbols") || !tableExists(db, "file_dependencies")) {
+            return { path: options.filePath, symbols: [], imports: [], importedBy: [] };
+        }
+        const file = db.prepare("select id, path from files where path = ?").get(options.filePath);
+        if (!file) {
+            return { path: options.filePath, symbols: [], imports: [], importedBy: [] };
+        }
+        const symbols = db
+            .prepare(`
+        select name, kind, line, signature, exported
+        from file_symbols
+        where file_id = ?
+        order by line, name
+        limit ?
+      `)
+            .all(file.id, options.maxResults);
+        const imports = db
+            .prepare(`
+        select specifier, resolved_path as resolvedPath, line
+        from file_dependencies
+        where file_id = ?
+        order by line, specifier
+        limit ?
+      `)
+            .all(file.id, options.maxResults);
+        const importedBy = db
+            .prepare(`
+        select files.path, file_dependencies.specifier, file_dependencies.line
+        from file_dependencies
+        join files on files.id = file_dependencies.file_id
+        where file_dependencies.resolved_path = ?
+        order by files.path, file_dependencies.line
+        limit ?
+      `)
+            .all(file.path, options.maxResults);
+        return {
+            path: file.path,
+            symbols: symbols.map((symbol) => ({ ...symbol, exported: Boolean(symbol.exported) })),
+            imports,
+            importedBy,
+        };
+    }
+    finally {
+        db.close();
+    }
+}
+export function readRelatedFileGraph(options) {
+    const maxDepth = Math.max(0, options.maxDepth);
+    const maxFiles = Math.max(0, options.maxFiles);
+    if (maxFiles === 0)
+        return [];
+    let order = 0;
+    const queue = options.seedPaths.map((path) => ({ path, depth: 0, via: null, order: order++ }));
+    const visited = new Set();
+    const nodes = [];
+    while (queue.length > 0 && nodes.length < maxFiles) {
+        sortQueue(queue);
+        const next = queue.shift();
+        if (!next || visited.has(next.path))
+            continue;
+        visited.add(next.path);
+        const related = readRelatedFiles({
+            dbPath: options.dbPath,
+            filePath: next.path,
+            maxResults: options.maxResultsPerFile,
+        });
+        nodes.push({ ...related, depth: next.depth, via: next.via });
+        if (next.depth >= maxDepth)
+            continue;
+        const neighbors = sortRelatedPaths([
+            ...related.imports.map((item) => item.resolvedPath).filter((path) => Boolean(path)),
+            ...related.importedBy.map((item) => item.path),
+        ]);
+        for (const neighbor of neighbors) {
+            if (!visited.has(neighbor) && !queue.some((queued) => queued.path === neighbor)) {
+                queue.push({ path: neighbor, depth: next.depth + 1, via: next.path, order: order++ });
+            }
+        }
+    }
+    return nodes;
+}

package/dist/indexing/relatedSnippets.js

@@ -0,0 +1,105 @@
+import fs from "node:fs";
+import Database from "better-sqlite3";
+const truncationMarker = "... [truncated]";
+const truncationMarkerWithBreak = `\n${truncationMarker}`;
+function emptyRelatedSnippetPack(maxRelatedContextChars) {
+    return {
+        snippets: [],
+        summary: {
+            maxRelatedContextChars: Math.max(0, maxRelatedContextChars),
+            usedRelatedContextChars: 0,
+            relatedSnippetCount: 0,
+            truncatedRelatedSnippets: 0,
+        },
+    };
+}
+function tableExists(db, name) {
+    const row = db
+        .prepare("select 1 as existsFlag from sqlite_master where type in ('table', 'virtual table') and name = ?")
+        .get(name);
+    return Boolean(row);
+}
+function compactSnippet(text, maxChars) {
+    const normalized = text.replace(/\s+$/g, "");
+    if (normalized.length <= maxChars)
+        return { snippet: normalized, truncated: false };
+    if (maxChars <= truncationMarker.length)
+        return { snippet: truncationMarker.slice(0, maxChars), truncated: true };
+    return {
+        snippet: `${normalized.slice(0, Math.max(0, maxChars - truncationMarkerWithBreak.length)).trimEnd()}${truncationMarkerWithBreak}`,
+        truncated: true,
+    };
+}
+function fitBudget(snippet, remaining) {
+    if (snippet.length <= remaining)
+        return { snippet, truncated: false };
+    if (remaining <= 0)
+        return { snippet: "", truncated: true };
+    if (remaining <= truncationMarker.length)
+        return { snippet: truncationMarker.slice(0, remaining), truncated: true };
+    return {
+        snippet: `${snippet.slice(0, remaining - truncationMarkerWithBreak.length).trimEnd()}${truncationMarkerWithBreak}`,
+        truncated: true,
+    };
+}
+export function readRelatedSnippets(options) {
+    if (!fs.existsSync(options.dbPath) || options.maxRelatedContextChars <= 0 || options.maxSnippetsPerFile <= 0) {
+        return emptyRelatedSnippetPack(options.maxRelatedContextChars);
+    }
+    const db = new Database(options.dbPath, { readonly: true });
+    const snippets = [];
+    let usedRelatedContextChars = 0;
+    let truncatedRelatedSnippets = 0;
+    try {
+        if (!tableExists(db, "files") || !tableExists(db, "chunks")) {
+            return emptyRelatedSnippetPack(options.maxRelatedContextChars);
+        }
+        const selectChunks = db.prepare(`
+      select files.path,
+             chunks.start_line as startLine,
+             chunks.end_line as endLine,
+             chunks.text
+      from chunks
+      join files on files.id = chunks.file_id
+      where files.path = ?
+      order by chunks.start_line
+      limit ?
+    `);
+        for (const path of options.paths) {
+            if (usedRelatedContextChars >= options.maxRelatedContextChars)
+                break;
+            const rows = selectChunks.all(path, options.maxSnippetsPerFile);
+            for (const row of rows) {
+                if (usedRelatedContextChars >= options.maxRelatedContextChars)
+                    break;
+                const compacted = compactSnippet(row.text, options.maxSnippetChars);
+                const fitted = fitBudget(compacted.snippet, options.maxRelatedContextChars - usedRelatedContextChars);
+                if (fitted.snippet.length === 0)
+                    break;
+                const snippetTruncated = compacted.truncated || fitted.truncated;
+                if (snippetTruncated)
+                    truncatedRelatedSnippets += 1;
+                usedRelatedContextChars += fitted.snippet.length;
+                snippets.push({
+                    path: row.path,
+                    startLine: row.startLine,
+                    endLine: row.endLine,
+                    snippet: fitted.snippet,
+                    ...(snippetTruncated ? { snippetTruncated } : {}),
+                });
+            }
+        }
+        return {
+            snippets,
+            summary: {
+                maxRelatedContextChars: options.maxRelatedContextChars,
+                usedRelatedContextChars,
+                relatedSnippetCount: snippets.length,
+                truncatedRelatedSnippets,
+            },
+        };
+    }
+    finally {
+        db.close();
+    }
+}