scythe-context-mcp 0.1.0

package/dist/storage/schema.js

@@ -0,0 +1,187 @@
+import { loadSqliteVec } from "./sqliteVec.js";
+export function vectorTableName(dimensions) {
+    if (!Number.isInteger(dimensions) || dimensions <= 0) {
+        throw new Error("Vector dimensions must be a positive integer");
+    }
+    return `vec_embeddings_${dimensions}`;
+}
+export function initializeStorageSchema(db, options) {
+    loadSqliteVec(db);
+    db.pragma("foreign_keys = ON");
+    if (db.name !== ":memory:") {
+        db.pragma("journal_mode = WAL");
+    }
+    db.exec(`
+    create table if not exists schema_migrations (
+      version integer primary key,
+      applied_at text not null default (datetime('now'))
+    );
+
+    create table if not exists files (
+      id integer primary key,
+      project_path text not null,
+      path text not null,
+      mtime_ms real not null,
+      size integer not null,
+      hash text not null,
+      unique(project_path, path)
+    );
+
+    create table if not exists chunks (
+      id integer primary key,
+      file_id integer not null references files(id) on delete cascade,
+      start_line integer not null,
+      end_line integer not null,
+      language text,
+      title text,
+      text text not null,
+      hash text not null,
+      unique(file_id, start_line, end_line, hash)
+    );
+
+    create virtual table if not exists chunk_fts using fts5(
+      path,
+      title,
+      text,
+      tokenize='trigram'
+    );
+
+    create table if not exists embedding_sets (
+      id integer primary key,
+      provider text not null,
+      base_url_hash text not null,
+      model text not null,
+      dimensions integer not null,
+      created_at text not null default (datetime('now')),
+      unique(provider, base_url_hash, model, dimensions)
+    );
+
+    create table if not exists embeddings (
+      id integer primary key,
+      chunk_id integer not null references chunks(id) on delete cascade,
+      embedding_set_id integer not null references embedding_sets(id) on delete cascade,
+      created_at text not null default (datetime('now')),
+      unique(chunk_id, embedding_set_id)
+    );
+
+    create table if not exists file_symbols (
+      id integer primary key,
+      file_id integer not null references files(id) on delete cascade,
+      name text not null,
+      kind text not null,
+      line integer not null,
+      signature text not null,
+      exported integer not null default 0
+    );
+
+    create index if not exists idx_file_symbols_name on file_symbols(name);
+    create index if not exists idx_file_symbols_file_id on file_symbols(file_id);
+
+    create table if not exists file_dependencies (
+      id integer primary key,
+      file_id integer not null references files(id) on delete cascade,
+      specifier text not null,
+      resolved_path text,
+      line integer not null
+    );
+
+    create index if not exists idx_file_dependencies_file_id on file_dependencies(file_id);
+    create index if not exists idx_file_dependencies_resolved_path on file_dependencies(resolved_path);
+
+    insert or ignore into schema_migrations(version) values (1);
+    insert or ignore into schema_migrations(version) values (2);
+  `);
+    const tableName = vectorTableName(options.vectorDimensions);
+    db.exec(`create virtual table if not exists ${tableName} using vec0(embedding float[${options.vectorDimensions}]);`);
+}
+export function upsertFile(db, input) {
+    db.prepare(`
+    insert into files(project_path, path, mtime_ms, size, hash)
+    values (@projectPath, @path, @mtimeMs, @size, @hash)
+    on conflict(project_path, path) do update set
+      mtime_ms = excluded.mtime_ms,
+      size = excluded.size,
+      hash = excluded.hash
+  `).run(input);
+    const row = db
+        .prepare("select id from files where project_path = ? and path = ?")
+        .get(input.projectPath, input.path);
+    return row.id;
+}
+export function insertChunk(db, input) {
+    db.prepare(`
+    insert or ignore into chunks(file_id, start_line, end_line, language, title, text, hash)
+    values (@fileId, @startLine, @endLine, @language, @title, @text, @hash)
+  `).run({
+        ...input,
+        language: input.language ?? null,
+        title: input.title ?? null,
+    });
+    const row = db
+        .prepare("select id from chunks where file_id = ? and start_line = ? and end_line = ? and hash = ?")
+        .get(input.fileId, input.startLine, input.endLine, input.hash);
+    db.prepare("delete from chunk_fts where rowid = ?").run(row.id);
+    db.prepare("insert into chunk_fts(rowid, path, title, text) values (?, ?, ?, ?)").run(row.id, input.path, input.title ?? "", input.text);
+    return row.id;
+}
+export function deleteChunksForFile(db, fileId) {
+    const rows = db.prepare("select id from chunks where file_id = ?").all(fileId);
+    for (const row of rows) {
+        db.prepare("delete from chunk_fts where rowid = ?").run(row.id);
+    }
+    db.prepare("delete from chunks where file_id = ?").run(fileId);
+}
+export function rebuildChunkFtsForFile(db, fileId, filePath) {
+    const rows = db.prepare("select id, title, text from chunks where file_id = ?").all(fileId);
+    for (const row of rows) {
+        db.prepare("delete from chunk_fts where rowid = ?").run(row.id);
+        db.prepare("insert into chunk_fts(rowid, path, title, text) values (?, ?, ?, ?)").run(row.id, filePath, row.title ?? "", row.text);
+    }
+}
+export function getOrCreateEmbeddingSet(db, input) {
+    db.prepare(`
+    insert or ignore into embedding_sets(provider, base_url_hash, model, dimensions)
+    values (@provider, @baseUrlHash, @model, @dimensions)
+  `).run(input);
+    const row = db
+        .prepare(`
+      select id from embedding_sets
+      where provider = ? and base_url_hash = ? and model = ? and dimensions = ?
+    `)
+        .get(input.provider, input.baseUrlHash, input.model, input.dimensions);
+    return row.id;
+}
+export function getOrCreateEmbeddingRecord(db, input) {
+    db.prepare(`
+    insert or ignore into embeddings(chunk_id, embedding_set_id)
+    values (@chunkId, @embeddingSetId)
+  `).run(input);
+    const row = db
+        .prepare("select id from embeddings where chunk_id = ? and embedding_set_id = ?")
+        .get(input.chunkId, input.embeddingSetId);
+    return row.id;
+}
+export function replaceSymbolGraphForFile(db, fileId, symbols, dependencies) {
+    db.prepare("delete from file_symbols where file_id = ?").run(fileId);
+    db.prepare("delete from file_dependencies where file_id = ?").run(fileId);
+    const insertSymbol = db.prepare(`
+    insert into file_symbols(file_id, name, kind, line, signature, exported)
+    values (@fileId, @name, @kind, @line, @signature, @exported)
+  `);
+    for (const symbol of symbols) {
+        insertSymbol.run({
+            ...symbol,
+            exported: symbol.exported ? 1 : 0,
+        });
+    }
+    const insertDependency = db.prepare(`
+    insert into file_dependencies(file_id, specifier, resolved_path, line)
+    values (@fileId, @specifier, @resolvedPath, @line)
+  `);
+    for (const dependency of dependencies) {
+        insertDependency.run({
+            ...dependency,
+            resolvedPath: dependency.resolvedPath ?? null,
+        });
+    }
+}

package/dist/storage/sqliteVec.js

@@ -0,0 +1,17 @@
+import * as sqliteVec from "sqlite-vec";
+export function loadSqliteVec(db) {
+    sqliteVec.load(db);
+    const row = db.prepare("select vec_version() as version").get();
+    return row.version;
+}
+export function vectorToFloat32Buffer(vector, expectedDimensions) {
+    if (vector.length !== expectedDimensions) {
+        throw new Error(`Expected vector with ${expectedDimensions} dimensions, received ${vector.length}`);
+    }
+    for (const value of vector) {
+        if (!Number.isFinite(value)) {
+            throw new Error("Vector contains a non-finite value");
+        }
+    }
+    return Buffer.from(new Float32Array(vector).buffer);
+}

package/dist/tools/registerTools.js

@@ -0,0 +1,364 @@
+import path from "node:path";
+import fs from "node:fs";
+import { z } from "zod";
+import { buildContextPack } from "../indexing/contextPack.js";
+import { reindexDryRun } from "../indexing/dryRun.js";
+import { indexMissingEmbeddings } from "../indexing/embeddingWriter.js";
+import { searchHybrid } from "../indexing/hybridSearch.js";
+import { readDetailedIndexStatus, readIndexFreshness, recommendedNextActions } from "../indexing/indexStatus.js";
+import { persistentReindexMetadata } from "../indexing/indexWriter.js";
+import { readRelatedFileGraph, readRelatedFiles } from "../indexing/relatedFiles.js";
+import { readRelatedSnippets } from "../indexing/relatedSnippets.js";
+import { formatSearchResults } from "../indexing/resultFormat.js";
+import { searchByVector } from "../indexing/semanticSearch.js";
+import { buildGeminiEndpoint, GeminiEmbeddingError, GeminiEmbeddingProvider, normalizeGeminiBaseUrl } from "../providers/gemini.js";
+function asJsonText(value) {
+    return {
+        content: [{ type: "text", text: JSON.stringify(value, null, 2) }],
+    };
+}
+function searchIndexedChunks(options) {
+    return options.mode === "semantic"
+        ? searchByVector({
+            dbPath: options.dbPath,
+            dimensions: options.dimensions,
+            queryVector: options.queryVector,
+            maxResults: options.maxResults,
+            maxSnippetChars: options.maxSnippetChars,
+        })
+        : searchHybrid({
+            dbPath: options.dbPath,
+            query: options.query,
+            dimensions: options.dimensions,
+            queryVector: options.queryVector,
+            maxResults: options.maxResults,
+            maxSnippetChars: options.maxSnippetChars,
+        });
+}
+function buildGeminiDiagnostics(config, expectedDimensions) {
+    const diagnostics = {
+        baseUrl: config.baseUrl,
+        model: config.model,
+        expectedDimensions,
+        authMode: config.authMode,
+        hasApiKey: Boolean(config.apiKey),
+    };
+    try {
+        diagnostics.normalizedBaseUrl = normalizeGeminiBaseUrl(config.baseUrl);
+        diagnostics.endpoint = buildGeminiEndpoint(config.baseUrl, config.model, "embedContent").toString();
+    }
+    catch (error) {
+        diagnostics.configError = error instanceof Error ? error.message : String(error);
+    }
+    return diagnostics;
+}
+export function registerTools(server, config) {
+    const embeddingProvider = new GeminiEmbeddingProvider(config.gemini);
+    const expectedDimensions = config.gemini.outputDimensionality ?? 1536;
+    server.registerTool("repo_index_status", {
+        title: "Repo Index Status",
+        description: "Show Scythe Context configuration and current index status.",
+        inputSchema: {
+            project_path: z.string().optional(),
+        },
+    }, async ({ project_path }) => {
+        const projectPath = path.resolve(project_path || config.defaultProjectPath);
+        const dbPath = path.join(projectPath, config.indexDirName, "index.sqlite");
+        const index = readDetailedIndexStatus(dbPath);
+        const freshness = await readIndexFreshness({
+            projectPath,
+            dbPath,
+            limits: { maxFileBytes: config.indexing.maxFileBytes },
+        });
+        return asJsonText({
+            projectPath,
+            indexPath: path.join(projectPath, config.indexDirName),
+            index,
+            recommendedNextActions: recommendedNextActions(index, {
+                desiredDimensions: expectedDimensions,
+                freshness,
+            }),
+            freshness,
+            status: "usable_mvp",
+            implemented: [
+                "mcp_server",
+                "gemini_embedding_provider",
+                "config",
+                "file_scanner",
+                "chunker",
+                "reindex_dry_run",
+                "sqlite_schema",
+                "persistent_metadata_index",
+                "embedding_index_writer",
+                "semantic_vector_search",
+                "symbol_graph",
+                "related_files",
+                "context_budgeting",
+                "context_packer",
+                "multi_hop_related_files",
+                "related_snippet_packing",
+            ],
+            pending: ["tree_sitter_symbols"],
+            indexing: config.indexing,
+            gemini: {
+                baseUrl: config.gemini.baseUrl,
+                model: config.gemini.model,
+                outputDimensionality: config.gemini.outputDimensionality,
+                authMode: config.gemini.authMode,
+                hasApiKey: Boolean(config.gemini.apiKey),
+            },
+        });
+    });
+    server.registerTool("gemini_embedding_probe", {
+        title: "Gemini Embedding Probe",
+        description: "Send one embedding request and return diagnostics for official Gemini or proxy compatibility.",
+        inputSchema: {
+            text: z.string().min(1),
+        },
+    }, async ({ text }) => {
+        const startedAt = Date.now();
+        const diagnostics = buildGeminiDiagnostics(config.gemini, expectedDimensions);
+        try {
+            const result = await embeddingProvider.embed({ kind: "query", text });
+            return asJsonText({
+                status: "ok",
+                latencyMs: Date.now() - startedAt,
+                diagnostics,
+                model: result.model,
+                dimensions: result.dimensions,
+                dimensionsMatchExpected: result.dimensions === expectedDimensions,
+                sample: result.vector.slice(0, 8),
+            });
+        }
+        catch (error) {
+            const geminiError = error instanceof GeminiEmbeddingError ? error : undefined;
+            return asJsonText({
+                status: "embedding_probe_failed",
+                latencyMs: Date.now() - startedAt,
+                diagnostics,
+                error: {
+                    type: error instanceof Error ? error.name : "UnknownError",
+                    message: error instanceof Error ? error.message : String(error),
+                    httpStatus: geminiError?.status,
+                    retryable: geminiError?.retryable ?? false,
+                    bodySnippet: geminiError?.bodySnippet,
+                },
+                recommendedNextActions: [
+                    "Verify GEMINI_API_KEY is present in the environment Codex launches from.",
+                    "Verify GEMINI_BASE_URL points to the provider root and can include or omit /v1beta.",
+                    "Verify GEMINI_AUTH_MODE matches the proxy requirement: x-goog-api-key, bearer, or query.",
+                    "Verify the provider supports models/{model}:embedContent and the requested output dimensionality.",
+                ],
+            });
+        }
+    });
+    server.registerTool("repo_reindex", {
+        title: "Repo Reindex",
+        description: "Scan a project, write metadata, and optionally index embeddings.",
+        inputSchema: {
+            project_path: z.string().optional(),
+            dry_run: z.boolean().default(true),
+            max_file_bytes: z.number().int().positive().optional(),
+            target_chunk_chars: z.number().int().positive().optional(),
+            chunk_overlap_chars: z.number().int().nonnegative().optional(),
+            max_chunks_per_file: z.number().int().positive().optional(),
+            index_embeddings: z.boolean().default(false),
+            embedding_batch_size: z.number().int().positive().max(128).optional(),
+            max_embedding_chunks: z.number().int().positive().max(10000).optional(),
+        },
+    }, async ({ project_path, dry_run, max_file_bytes, target_chunk_chars, chunk_overlap_chars, max_chunks_per_file, index_embeddings, embedding_batch_size, max_embedding_chunks, }) => {
+        const commonOptions = {
+            projectPath: path.resolve(project_path || config.defaultProjectPath),
+            maxFileBytes: max_file_bytes ?? config.indexing.maxFileBytes,
+            targetChunkChars: target_chunk_chars ?? config.indexing.targetChunkChars,
+            chunkOverlapChars: chunk_overlap_chars ?? config.indexing.chunkOverlapChars,
+            maxChunksPerFile: max_chunks_per_file ?? config.indexing.maxChunksPerFile,
+        };
+        if (dry_run) {
+            return asJsonText(await reindexDryRun(commonOptions));
+        }
+        const metadataResult = await persistentReindexMetadata({
+            ...commonOptions,
+            indexDirName: config.indexDirName,
+            vectorDimensions: expectedDimensions,
+        });
+        if (!index_embeddings) {
+            return asJsonText(metadataResult);
+        }
+        const embeddingResult = await indexMissingEmbeddings({
+            dbPath: metadataResult.dbPath,
+            providerName: "gemini",
+            providerBaseUrl: config.gemini.baseUrl,
+            model: config.gemini.model,
+            dimensions: expectedDimensions,
+            batchSize: embedding_batch_size ?? config.indexing.embeddingBatchSize,
+            maxChunks: max_embedding_chunks ?? config.indexing.maxEmbeddingChunks,
+            provider: embeddingProvider,
+        });
+        return asJsonText({
+            ...metadataResult,
+            status: "metadata_and_embeddings_indexed",
+            embeddings: embeddingResult,
+        });
+    });
+    server.registerTool("repo_semantic_search", {
+        title: "Repo Semantic Search",
+        description: "Search indexed code chunks by semantic similarity. Requires repo_reindex with index_embeddings=true first.",
+        inputSchema: {
+            query: z.string().min(1),
+            project_path: z.string().optional(),
+            max_results: z.number().int().positive().max(50).default(8),
+            max_snippet_chars: z.number().int().positive().max(4000).default(1200),
+            max_context_chars: z.number().int().positive().max(100000).default(12000),
+            mode: z.enum(["hybrid", "semantic"]).default("hybrid"),
+        },
+    }, async ({ query, project_path, max_results, max_snippet_chars, max_context_chars, mode }) => {
+        const projectPath = path.resolve(project_path || config.defaultProjectPath);
+        const dbPath = path.join(projectPath, config.indexDirName, "index.sqlite");
+        if (!fs.existsSync(dbPath)) {
+            return asJsonText({
+                query,
+                projectPath,
+                status: "index_missing",
+                message: "Run repo_reindex with dry_run=false and index_embeddings=true before semantic search.",
+            });
+        }
+        const queryEmbedding = await embeddingProvider.embed({ kind: "query", text: query });
+        const dimensions = expectedDimensions;
+        if (queryEmbedding.dimensions !== dimensions) {
+            throw new Error(`Query embedding dimensions mismatch: expected ${dimensions}, got ${queryEmbedding.dimensions}`);
+        }
+        const rawResults = searchIndexedChunks({
+            dbPath,
+            query,
+            dimensions,
+            queryVector: queryEmbedding.vector,
+            maxResults: max_results,
+            maxSnippetChars: max_snippet_chars,
+            mode,
+        });
+        const formatted = formatSearchResults(query, rawResults, { maxContextChars: max_context_chars });
+        return asJsonText({
+            query,
+            projectPath,
+            dbPath,
+            dimensions,
+            mode,
+            results: formatted.results,
+            context: formatted.summary,
+            resultCount: rawResults.length,
+        });
+    });
+    server.registerTool("repo_related_files", {
+        title: "Repo Related Files",
+        description: "Show symbols, imports, and reverse imports for an indexed file.",
+        inputSchema: {
+            path: z.string().min(1),
+            project_path: z.string().optional(),
+            max_results: z.number().int().positive().max(100).default(24),
+        },
+    }, async ({ path: filePath, project_path, max_results }) => {
+        const projectPath = path.resolve(project_path || config.defaultProjectPath);
+        const dbPath = path.join(projectPath, config.indexDirName, "index.sqlite");
+        if (!fs.existsSync(dbPath)) {
+            return asJsonText({
+                path: filePath,
+                projectPath,
+                status: "index_missing",
+                message: "Run repo_reindex with dry_run=false before related file lookup.",
+            });
+        }
+        return asJsonText({
+            projectPath,
+            dbPath,
+            ...readRelatedFiles({
+                dbPath,
+                filePath,
+                maxResults: max_results,
+            }),
+        });
+    });
+    server.registerTool("repo_context_pack", {
+        title: "Repo Context Pack",
+        description: "Search code and package primary snippets with symbols/imports/reverse imports for matched files.",
+        inputSchema: {
+            query: z.string().min(1),
+            project_path: z.string().optional(),
+            max_results: z.number().int().positive().max(30).default(6),
+            max_snippet_chars: z.number().int().positive().max(4000).default(1200),
+            max_context_chars: z.number().int().positive().max(100000).default(16000),
+            max_seed_files: z.number().int().positive().max(10).default(3),
+            max_related_files: z.number().int().nonnegative().max(30).default(10),
+            max_related_items: z.number().int().positive().max(50).default(8),
+            include_related_snippets: z.boolean().default(false),
+            max_related_snippets_per_file: z.number().int().positive().max(5).default(1),
+            max_related_snippet_chars: z.number().int().positive().max(2000).default(600),
+            max_related_context_chars: z.number().int().nonnegative().max(50000).default(4000),
+            related_depth: z.number().int().nonnegative().max(3).default(1),
+            mode: z.enum(["hybrid", "semantic"]).default("hybrid"),
+        },
+    }, async ({ query, project_path, max_results, max_snippet_chars, max_context_chars, max_seed_files, max_related_files, max_related_items, include_related_snippets, max_related_snippets_per_file, max_related_snippet_chars, max_related_context_chars, related_depth, mode, }) => {
+        const projectPath = path.resolve(project_path || config.defaultProjectPath);
+        const dbPath = path.join(projectPath, config.indexDirName, "index.sqlite");
+        if (!fs.existsSync(dbPath)) {
+            return asJsonText({
+                query,
+                projectPath,
+                status: "index_missing",
+                message: "Run repo_reindex with dry_run=false and index_embeddings=true before building a context pack.",
+            });
+        }
+        const queryEmbedding = await embeddingProvider.embed({ kind: "query", text: query });
+        const dimensions = expectedDimensions;
+        if (queryEmbedding.dimensions !== dimensions) {
+            throw new Error(`Query embedding dimensions mismatch: expected ${dimensions}, got ${queryEmbedding.dimensions}`);
+        }
+        const rawResults = searchIndexedChunks({
+            dbPath,
+            query,
+            dimensions,
+            queryVector: queryEmbedding.vector,
+            maxResults: max_results,
+            maxSnippetChars: max_snippet_chars,
+            mode,
+        });
+        const relatedPaths = Array.from(new Set(rawResults.map((result) => result.path))).slice(0, Math.min(max_seed_files, max_related_files));
+        const relatedFiles = readRelatedFileGraph({
+            dbPath,
+            seedPaths: relatedPaths,
+            maxDepth: related_depth,
+            maxFiles: max_related_files,
+            maxResultsPerFile: max_related_items,
+        });
+        const primaryPathSet = new Set(rawResults.map((result) => result.path));
+        const relatedSnippetPaths = relatedFiles
+            .map((file) => file.path)
+            .filter((filePath) => !primaryPathSet.has(filePath));
+        const relatedSnippets = include_related_snippets && max_related_context_chars > 0
+            ? readRelatedSnippets({
+                dbPath,
+                paths: relatedSnippetPaths,
+                maxSnippetsPerFile: max_related_snippets_per_file,
+                maxSnippetChars: max_related_snippet_chars,
+                maxRelatedContextChars: max_related_context_chars,
+            })
+            : undefined;
+        const pack = buildContextPack(query, rawResults, relatedFiles, {
+            maxContextChars: max_context_chars,
+            maxRelatedFiles: max_related_files,
+            maxRelatedItems: max_related_items,
+            relatedSnippets,
+        });
+        return asJsonText({
+            query,
+            projectPath,
+            dbPath,
+            dimensions,
+            mode,
+            relatedDepth: related_depth,
+            relatedSeedCount: relatedPaths.length,
+            includeRelatedSnippets: include_related_snippets,
+            ...pack,
+        });
+    });
+}