scythe-context-mcp 0.1.0

package/dist/indexing/resultFormat.js

@@ -0,0 +1,69 @@
+import { keywordTerms } from "./keywordSearch.js";
+const truncationMarker = "... [truncated]";
+const truncationMarkerWithBreak = `\n${truncationMarker}`;
+export function grepKeywords(query, result) {
+    const pathParts = result.path
+        .split(/[\/._-]+/g)
+        .filter((part) => part.length >= 3);
+    return Array.from(new Set([...keywordTerms(query), ...pathParts])).slice(0, 8);
+}
+export function matchReason(result) {
+    const matchTypes = result.matchTypes || [];
+    if (matchTypes.includes("semantic") && matchTypes.includes("keyword")) {
+        return "semantic similarity plus keyword/path match";
+    }
+    if (matchTypes.includes("keyword")) {
+        return "keyword/path match";
+    }
+    if (typeof result.distance === "number") {
+        return `semantic similarity distance ${result.distance.toFixed(4)}`;
+    }
+    if (typeof result.score === "number") {
+        return `ranked score ${result.score.toFixed(4)}`;
+    }
+    return "ranked match";
+}
+export function formatSearchResults(query, results, options = {}) {
+    const maxContextChars = options.maxContextChars ?? null;
+    let usedContextChars = 0;
+    let truncatedResults = 0;
+    const formatted = results.map((result) => {
+        let snippet = result.snippet;
+        let snippetTruncated = false;
+        if (typeof snippet === "string" && maxContextChars !== null) {
+            const remaining = Math.max(0, maxContextChars - usedContextChars);
+            if (snippet.length > remaining) {
+                if (remaining >= truncationMarkerWithBreak.length) {
+                    snippet = `${snippet.slice(0, remaining - truncationMarkerWithBreak.length).trimEnd()}${truncationMarkerWithBreak}`;
+                }
+                else if (remaining >= truncationMarker.length) {
+                    snippet = truncationMarker;
+                }
+                else {
+                    snippet = "";
+                }
+                snippetTruncated = true;
+                truncatedResults += 1;
+            }
+            usedContextChars += snippet.length;
+        }
+        else if (typeof snippet === "string") {
+            usedContextChars += snippet.length;
+        }
+        return {
+            ...result,
+            snippet,
+            grepKeywords: grepKeywords(query, result),
+            matchReason: matchReason(result),
+            ...(snippetTruncated ? { snippetTruncated } : {}),
+        };
+    });
+    return {
+        results: formatted,
+        summary: {
+            maxContextChars,
+            usedContextChars,
+            truncatedResults,
+        },
+    };
+}

package/dist/indexing/scanner.js

@@ -0,0 +1,123 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import fg from "fast-glob";
+import ignore from "ignore";
+import { isProbablyBinary } from "./binary.js";
+const BUILT_IN_IGNORES = [
+    ".git/**",
+    ".scythe-context/**",
+    ".repo-beacon/**",
+    "node_modules/**",
+    "dist/**",
+    "build/**",
+    ".next/**",
+    ".turbo/**",
+    "coverage/**",
+    "package-lock.json",
+    "npm-shrinkwrap.json",
+    "yarn.lock",
+    "pnpm-lock.yaml",
+    "bun.lock",
+    "bun.lockb",
+    "*.png",
+    "*.jpg",
+    "*.jpeg",
+    "*.gif",
+    "*.webp",
+    "*.ico",
+    "*.pdf",
+    "*.zip",
+    "*.gz",
+    "*.tar",
+    "*.7z",
+    "*.woff",
+    "*.woff2",
+    "*.ttf",
+];
+async function loadGitignore(projectPath) {
+    const matcher = ignore();
+    try {
+        const content = await fs.readFile(path.join(projectPath, ".gitignore"), "utf8");
+        matcher.add(content);
+    }
+    catch (error) {
+        if (error.code !== "ENOENT") {
+            throw error;
+        }
+    }
+    return matcher;
+}
+function toPosixPath(value) {
+    return value.split(path.sep).join("/");
+}
+async function hasBinaryHeader(absolutePath) {
+    const handle = await fs.open(absolutePath, "r");
+    try {
+        const buffer = Buffer.alloc(8192);
+        const { bytesRead } = await handle.read(buffer, 0, buffer.length, 0);
+        return isProbablyBinary(buffer.subarray(0, bytesRead));
+    }
+    finally {
+        await handle.close();
+    }
+}
+export async function scanProject(projectPath, limits) {
+    const resolvedProjectPath = path.resolve(projectPath);
+    const gitignore = await loadGitignore(resolvedProjectPath);
+    const entries = await fg("**/*", {
+        cwd: resolvedProjectPath,
+        dot: true,
+        onlyFiles: true,
+        followSymbolicLinks: false,
+        ignore: BUILT_IN_IGNORES,
+        unique: true,
+    });
+    const files = [];
+    const skipped = [];
+    for (const entry of entries.sort()) {
+        const relativePath = toPosixPath(entry);
+        if (gitignore.ignores(relativePath)) {
+            skipped.push({ relativePath, reason: "ignored" });
+            continue;
+        }
+        const absolutePath = path.resolve(resolvedProjectPath, relativePath);
+        if (!absolutePath.startsWith(resolvedProjectPath + path.sep)) {
+            skipped.push({ relativePath, reason: "outside_project" });
+            continue;
+        }
+        try {
+            const stat = await fs.stat(absolutePath);
+            if (!stat.isFile()) {
+                skipped.push({ relativePath, reason: "not_file" });
+                continue;
+            }
+            if (stat.size > limits.maxFileBytes) {
+                skipped.push({
+                    relativePath,
+                    reason: "too_large",
+                    size: stat.size,
+                    detail: `size ${stat.size} exceeds maxFileBytes ${limits.maxFileBytes}`,
+                });
+                continue;
+            }
+            if (await hasBinaryHeader(absolutePath)) {
+                skipped.push({ relativePath, reason: "binary", size: stat.size });
+                continue;
+            }
+            files.push({
+                relativePath,
+                absolutePath,
+                size: stat.size,
+                mtimeMs: stat.mtimeMs,
+            });
+        }
+        catch (error) {
+            skipped.push({
+                relativePath,
+                reason: "read_error",
+                detail: error instanceof Error ? error.message : String(error),
+            });
+        }
+    }
+    return { projectPath: resolvedProjectPath, files, skipped };
+}

package/dist/indexing/semanticSearch.js

@@ -0,0 +1,48 @@
+import Database from "better-sqlite3";
+import { vectorTableName } from "../storage/schema.js";
+import { loadSqliteVec, vectorToFloat32Buffer } from "../storage/sqliteVec.js";
+function compactSnippet(text, maxChars) {
+    const normalized = text.replace(/\s+$/g, "");
+    if (normalized.length <= maxChars)
+        return normalized;
+    return `${normalized.slice(0, Math.max(0, maxChars - 3))}...`;
+}
+export function searchByVector(options) {
+    if (!Number.isInteger(options.maxResults) || options.maxResults <= 0) {
+        throw new Error("maxResults must be a positive integer");
+    }
+    if (!Number.isInteger(options.maxSnippetChars) || options.maxSnippetChars <= 0) {
+        throw new Error("maxSnippetChars must be a positive integer");
+    }
+    const db = new Database(options.dbPath, { readonly: true });
+    try {
+        loadSqliteVec(db);
+        const rows = db
+            .prepare(`
+        select files.path,
+               chunks.start_line as startLine,
+               chunks.end_line as endLine,
+               chunks.text,
+               vec.distance as distance
+        from ${vectorTableName(options.dimensions)} vec
+        join embeddings on embeddings.id = vec.rowid
+        join chunks on chunks.id = embeddings.chunk_id
+        join files on files.id = chunks.file_id
+        where vec.embedding match ?
+          and vec.k = ?
+        order by vec.distance
+      `)
+            .all(vectorToFloat32Buffer(options.queryVector, options.dimensions), options.maxResults);
+        return rows.map((row) => ({
+            path: row.path,
+            startLine: row.startLine,
+            endLine: row.endLine,
+            distance: row.distance,
+            snippet: compactSnippet(row.text, options.maxSnippetChars),
+            matchType: "semantic",
+        }));
+    }
+    finally {
+        db.close();
+    }
+}

package/dist/indexing/symbolGraph.js

@@ -0,0 +1,121 @@
+const symbolPatterns = [
+    { kind: "function", regex: /^(\s*export\s+)?(?:async\s+)?function\s+([A-Za-z_$][\w$]*)\b/, nameGroup: 2, exportedGroup: 1 },
+    { kind: "class", regex: /^(\s*export\s+)?(?:abstract\s+)?class\s+([A-Za-z_$][\w$]*)\b/, nameGroup: 2, exportedGroup: 1 },
+    { kind: "interface", regex: /^(\s*export\s+)?interface\s+([A-Za-z_$][\w$]*)\b/, nameGroup: 2, exportedGroup: 1 },
+    { kind: "type", regex: /^(\s*export\s+)?type\s+([A-Za-z_$][\w$]*)\b/, nameGroup: 2, exportedGroup: 1 },
+    { kind: "enum", regex: /^(\s*export\s+)?enum\s+([A-Za-z_$][\w$]*)\b/, nameGroup: 2, exportedGroup: 1 },
+    { kind: "const", regex: /^(\s*export\s+)?const\s+([A-Za-z_$][\w$]*)\b/, nameGroup: 2, exportedGroup: 1 },
+    { kind: "variable", regex: /^\s*(?:let|var)\s+([A-Za-z_$][\w$]*)\b/, nameGroup: 1 },
+    { kind: "function", regex: /^\s*def\s+([A-Za-z_]\w*)\b/, nameGroup: 1 },
+    { kind: "class", regex: /^\s*class\s+([A-Za-z_]\w*)\b/, nameGroup: 1 },
+    { kind: "function", regex: /^\s*func\s+(?:\([^)]*\)\s*)?([A-Za-z_]\w*)\b/, nameGroup: 1 },
+    { kind: "type", regex: /^\s*type\s+([A-Za-z_]\w*)\b/, nameGroup: 1 },
+    { kind: "function", regex: /^\s*(?:pub\s+)?(?:async\s+)?fn\s+([A-Za-z_]\w*)\b/, nameGroup: 1, exportedGroup: 0 },
+    { kind: "struct", regex: /^\s*(?:pub\s+)?struct\s+([A-Za-z_]\w*)\b/, nameGroup: 1, exportedGroup: 0 },
+    { kind: "enum", regex: /^\s*(?:pub\s+)?enum\s+([A-Za-z_]\w*)\b/, nameGroup: 1, exportedGroup: 0 },
+    { kind: "trait", regex: /^\s*(?:pub\s+)?trait\s+([A-Za-z_]\w*)\b/, nameGroup: 1, exportedGroup: 0 },
+    { kind: "module", regex: /^\s*(?:pub\s+)?mod\s+([A-Za-z_]\w*)\b/, nameGroup: 1, exportedGroup: 0 },
+];
+const javascriptImportPatterns = [
+    /\bimport\s+(?:[^'"`"]+\s+from\s+)?["'`]([^"'`]+)["'`]/g,
+    /\bexport\s+[^"'`]+\s+from\s+["'`]([^"'`]+)["'`]/g,
+    /\brequire\(\s*["'`]([^"'`]+)["'`]\s*\)/g,
+];
+const pythonImportPatterns = [
+    /\bfrom\s+([A-Za-z_][\w.]+)\s+import\b/g,
+    /^\s*import\s+([A-Za-z_][\w.]+)/g,
+];
+const rustImportPatterns = [
+    /^\s*use\s+([^;]+);/g,
+    /^\s*mod\s+([A-Za-z_]\w*)\s*;/g,
+];
+const goImportPatterns = [/^\s*import\s+["`]([^"`]+)["`]/g, /^\s*["`]([^"`]+)["`]/g];
+function dependencyPatternsForPath(relativePath) {
+    if (/\.(?:js|jsx|mjs|cjs|ts|tsx|mts|cts)$/.test(relativePath))
+        return javascriptImportPatterns;
+    if (/\.py$/.test(relativePath))
+        return pythonImportPatterns;
+    if (/\.rs$/.test(relativePath))
+        return rustImportPatterns;
+    if (/\.go$/.test(relativePath))
+        return goImportPatterns;
+    return javascriptImportPatterns;
+}
+function normalizeSignature(line) {
+    return line.trim().replace(/\s+/g, " ").slice(0, 240);
+}
+function addUnique(items, item, key) {
+    if (!items.some((existing) => key(existing) === key(item))) {
+        items.push(item);
+    }
+}
+function extractDependenciesFromLine(patterns, line, lineNumber) {
+    const dependencies = [];
+    for (const pattern of patterns) {
+        pattern.lastIndex = 0;
+        let match;
+        while ((match = pattern.exec(line))) {
+            const specifier = match[1]?.trim();
+            if (specifier) {
+                addUnique(dependencies, { specifier, line: lineNumber }, (dep) => `${dep.specifier}:${dep.line}`);
+            }
+        }
+    }
+    return dependencies;
+}
+export function extractFileGraph(relativePath, content) {
+    const symbols = [];
+    const dependencies = [];
+    const dependencyPatterns = dependencyPatternsForPath(relativePath);
+    const lines = content.split(/\r?\n/);
+    lines.forEach((line, index) => {
+        const lineNumber = index + 1;
+        for (const pattern of symbolPatterns) {
+            const match = pattern.regex.exec(line);
+            if (!match)
+                continue;
+            const name = match[pattern.nameGroup ?? 1];
+            if (!name)
+                continue;
+            addUnique(symbols, {
+                name,
+                kind: pattern.kind,
+                line: lineNumber,
+                signature: normalizeSignature(line),
+                exported: pattern.exportedGroup === 0 ? /\bpub\b/.test(line) : Boolean(match[pattern.exportedGroup ?? -1]),
+            }, (symbol) => `${symbol.name}:${symbol.kind}:${symbol.line}`);
+            break;
+        }
+        for (const dependency of extractDependenciesFromLine(dependencyPatterns, line, lineNumber)) {
+            addUnique(dependencies, dependency, (dep) => `${dep.specifier}:${dep.line}`);
+        }
+    });
+    return { symbols, dependencies };
+}
+export function resolveDependencyPath(fromPath, specifier, activePaths) {
+    if (!specifier.startsWith("."))
+        return null;
+    const fromDir = fromPath.includes("/") ? fromPath.slice(0, fromPath.lastIndexOf("/")) : "";
+    const normalized = new URL(specifier, `file:///${fromDir ? `${fromDir}/` : ""}`).pathname.replace(/^\/+/, "");
+    const withoutRuntimeExtension = normalized.replace(/\.(?:js|jsx|mjs|cjs)$/, "");
+    const candidates = [
+        normalized,
+        withoutRuntimeExtension,
+        `${withoutRuntimeExtension}.ts`,
+        `${withoutRuntimeExtension}.tsx`,
+        `${withoutRuntimeExtension}.mts`,
+        `${withoutRuntimeExtension}.cts`,
+        `${normalized}.ts`,
+        `${normalized}.tsx`,
+        `${normalized}.js`,
+        `${normalized}.jsx`,
+        `${normalized}.mjs`,
+        `${normalized}.cjs`,
+        `${normalized}.py`,
+        `${normalized}/index.ts`,
+        `${normalized}/index.tsx`,
+        `${normalized}/index.js`,
+        `${normalized}/index.jsx`,
+    ];
+    return candidates.find((candidate) => activePaths.has(candidate)) ?? null;
+}

package/dist/indexing/types.js

@@ -0,0 +1 @@
+export {};

package/dist/providers/gemini.js

@@ -0,0 +1,149 @@
+export class GeminiEmbeddingError extends Error {
+    status;
+    bodySnippet;
+    retryable;
+    constructor(message, options = {}) {
+        super(message);
+        this.name = "GeminiEmbeddingError";
+        this.status = options.status;
+        this.bodySnippet = options.bodySnippet;
+        this.retryable = options.retryable ?? false;
+    }
+}
+function trimTrailingSlash(value) {
+    return value.replace(/\/+$/, "");
+}
+export function normalizeGeminiBaseUrl(baseUrl) {
+    const url = new URL(baseUrl);
+    const pathParts = url.pathname.split("/").filter(Boolean);
+    const lastPart = pathParts.at(-1);
+    if (lastPart !== "v1" && lastPart !== "v1beta") {
+        pathParts.push("v1beta");
+    }
+    url.pathname = `/${pathParts.join("/")}`;
+    url.search = "";
+    url.hash = "";
+    return trimTrailingSlash(url.toString());
+}
+function modelResource(model) {
+    return model.startsWith("models/") ? model : `models/${model}`;
+}
+export function buildGeminiEndpoint(baseUrl, model, method) {
+    return new URL(`${normalizeGeminiBaseUrl(baseUrl)}/${modelResource(model)}:${method}`);
+}
+function formatEmbeddingText(input) {
+    if (input.kind === "query") {
+        return `task: code retrieval | query: ${input.text}`;
+    }
+    return `title: ${input.title || "none"} | text: ${input.text}`;
+}
+function isRetryableStatus(status) {
+    return status === 408 || status === 409 || status === 425 || status === 429 || status >= 500;
+}
+function safeSnippet(text, secret) {
+    const collapsed = text.replace(/\s+/g, " ").trim();
+    const redacted = secret ? collapsed.split(secret).join("[REDACTED]") : collapsed;
+    return redacted.length > 1000 ? `${redacted.slice(0, 1000)}...` : redacted;
+}
+export class GeminiEmbeddingProvider {
+    config;
+    constructor(config) {
+        this.config = config;
+    }
+    async embed(input) {
+        const response = await this.post("embedContent", {
+            model: modelResource(this.config.model),
+            content: { parts: [{ text: formatEmbeddingText(input) }] },
+            ...(this.config.outputDimensionality
+                ? { output_dimensionality: this.config.outputDimensionality }
+                : {}),
+        });
+        const vector = this.extractSingleVector(response);
+        return {
+            vector,
+            model: this.config.model,
+            dimensions: vector.length,
+        };
+    }
+    async embedBatch(inputs) {
+        if (inputs.length === 0)
+            return [];
+        const response = await this.post("batchEmbedContents", {
+            requests: inputs.map((input) => ({
+                model: modelResource(this.config.model),
+                content: { parts: [{ text: formatEmbeddingText(input) }] },
+                ...(this.config.outputDimensionality
+                    ? { output_dimensionality: this.config.outputDimensionality }
+                    : {}),
+            })),
+        });
+        const vectors = this.extractVectors(response);
+        if (vectors.length !== inputs.length) {
+            throw new Error(`Gemini returned ${vectors.length} embeddings for ${inputs.length} inputs`);
+        }
+        return vectors.map((vector) => ({
+            vector,
+            model: this.config.model,
+            dimensions: vector.length,
+        }));
+    }
+    async post(method, body) {
+        if (!this.config.apiKey) {
+            throw new GeminiEmbeddingError("GEMINI_API_KEY is required for embedding calls");
+        }
+        let url;
+        try {
+            url = buildGeminiEndpoint(this.config.baseUrl, this.config.model, method);
+        }
+        catch (error) {
+            throw new GeminiEmbeddingError(`Invalid GEMINI_BASE_URL: ${error instanceof Error ? error.message : String(error)}`);
+        }
+        const headers = {
+            "content-type": "application/json",
+        };
+        if (this.config.authMode === "bearer") {
+            headers.authorization = `Bearer ${this.config.apiKey}`;
+        }
+        else if (this.config.authMode === "query") {
+            url.searchParams.set(this.config.apiKeyQueryParam, this.config.apiKey);
+        }
+        else {
+            headers[this.config.apiKeyHeader] = this.config.apiKey;
+        }
+        const response = await fetch(url, {
+            method: "POST",
+            headers,
+            body: JSON.stringify(body),
+        });
+        const text = await response.text();
+        if (!response.ok) {
+            throw new GeminiEmbeddingError(`Gemini embedding request failed with HTTP ${response.status}`, {
+                status: response.status,
+                bodySnippet: safeSnippet(text, this.config.apiKey),
+                retryable: isRetryableStatus(response.status),
+            });
+        }
+        try {
+            return JSON.parse(text);
+        }
+        catch {
+            throw new GeminiEmbeddingError("Gemini embedding response was not valid JSON", {
+                bodySnippet: safeSnippet(text, this.config.apiKey),
+            });
+        }
+    }
+    extractSingleVector(response) {
+        const vector = response.embedding?.values || response.embeddings?.[0]?.values;
+        if (!vector || vector.length === 0) {
+            throw new Error("Gemini response did not include an embedding vector");
+        }
+        return vector;
+    }
+    extractVectors(response) {
+        const vectors = response.embeddings?.map((embedding) => embedding.values || []) || [];
+        if (vectors.some((vector) => vector.length === 0)) {
+            throw new Error("Gemini batch response included an empty embedding vector");
+        }
+        return vectors;
+    }
+}

package/dist/providers/types.js

@@ -0,0 +1 @@
+export {};