recker 1.0.68 → 1.0.70-next.9b4eebc

package/dist/mcp/search/hybrid-search.js

@@ -1,402 +0,0 @@
-import Fuse from 'fuse.js';
-import { cosineSimilarity, levenshtein } from './math.js';
-import { loadEmbeddings } from '../embeddings-loader.js';
-import { StateError } from '../../core/errors.js';
-let cachedEmbeddings = null;
-export class HybridSearch {
-    fuse = null;
-    docs = [];
-    vectors = new Map();
-    embeddingsData = null;
-    initialized = false;
-    config;
-    constructor(config = {}) {
-        this.config = {
-            fuzzyThreshold: config.fuzzyThreshold ?? 0.3,
-            fuzzyWeight: config.fuzzyWeight ?? 0.5,
-            semanticWeight: config.semanticWeight ?? 0.5,
-            debug: config.debug ?? false,
-            offline: config.offline ?? false,
-            embedder: config.embedder,
-        };
-    }
-    setEmbedder(embedder) {
-        this.config.embedder = embedder;
-    }
-    async initialize(docs) {
-        this.docs = docs;
-        this.fuse = new Fuse(docs, {
-            keys: [
-                { name: 'keywords', weight: 10 },
-                { name: 'title', weight: 6 },
-                { name: 'section', weight: 4 },
-                { name: 'path', weight: 2 },
-                { name: 'content', weight: 0.5 },
-            ],
-            includeScore: true,
-            threshold: this.config.fuzzyThreshold,
-            ignoreLocation: true,
-            useExtendedSearch: true,
-            findAllMatches: true,
-            minMatchCharLength: 2,
-        });
-        await this.loadPrecomputedEmbeddings();
-        this.initialized = true;
-        this.log(`Initialized with ${docs.length} docs, ${this.vectors.size} embeddings`);
-    }
-    async loadPrecomputedEmbeddings() {
-        try {
-            if (cachedEmbeddings) {
-                this.embeddingsData = cachedEmbeddings;
-            }
-            else {
-                const data = await loadEmbeddings({
-                    debug: this.config.debug,
-                    offline: this.config.offline,
-                });
-                if (data) {
-                    this.embeddingsData = data;
-                    cachedEmbeddings = data;
-                }
-            }
-            if (this.embeddingsData) {
-                for (const entry of this.embeddingsData.documents) {
-                    if (entry.vector) {
-                        const vec = Array.isArray(entry.vector)
-                            ? entry.vector
-                            : Object.values(entry.vector);
-                        if (vec.length > 0) {
-                            this.vectors.set(entry.id, vec);
-                        }
-                    }
-                }
-                this.log(`Loaded ${this.vectors.size} pre-computed embeddings (model: ${this.embeddingsData.model})`);
-            }
-        }
-        catch (error) {
-            this.log(`No pre-computed embeddings found: ${error}`);
-        }
-    }
-    async search(query, options = {}) {
-        const { limit = 10, category, mode = 'hybrid', minScore = 0 } = options;
-        if (!this.initialized) {
-            throw new StateError('HybridSearch not initialized. Call initialize() first.', {
-                expectedState: 'initialized',
-                actualState: 'not-initialized',
-            });
-        }
-        const cleanedQuery = this.cleanQuery(query);
-        this.log(`Original query: "${query}" → Cleaned: "${cleanedQuery}"`);
-        const searchQuery = cleanedQuery.length > 0 ? cleanedQuery : query;
-        const results = new Map();
-        if (mode === 'hybrid' || mode === 'fuzzy') {
-            const fuzzyResults = this.fuzzySearch(searchQuery, limit * 2, category);
-            for (const result of fuzzyResults) {
-                results.set(result.id, result);
-            }
-            this.log(`Fuzzy search found ${fuzzyResults.length} results`);
-        }
-        if ((mode === 'hybrid' || mode === 'semantic') && this.vectors.size > 0) {
-            const semanticResults = await this.semanticSearch(searchQuery, limit * 2, category);
-            for (const result of semanticResults) {
-                const existing = results.get(result.id);
-                if (existing) {
-                    const maxScore = Math.max(existing.score, result.score);
-                    const bonus = Math.min(existing.score, result.score) * 0.3;
-                    existing.score = Math.min(1.0, maxScore + bonus);
-                    existing.source = 'hybrid';
-                }
-                else {
-                    results.set(result.id, result);
-                }
-            }
-            this.log(`Semantic search found ${semanticResults.length} results`);
-        }
-        return Array.from(results.values())
-            .filter((r) => r.score >= minScore)
-            .sort((a, b) => b.score - a.score)
-            .slice(0, limit);
-    }
-    fuzzySearch(query, limit, category) {
-        if (!this.fuse)
-            return [];
-        let results = this.fuse.search(query, { limit: limit * 2 });
-        if (category) {
-            results = results.filter((r) => r.item.category.toLowerCase().includes(category.toLowerCase()));
-        }
-        const queryTerms = this.tokenize(query);
-        const scored = results.slice(0, limit).map((r) => {
-            const fuseScore = r.score || 0;
-            const baseScore = 1 - fuseScore;
-            let boost = 0;
-            if (fuseScore < 0.3) {
-                const keywords = r.item.keywords || [];
-                const titleLower = r.item.title.toLowerCase();
-                for (const term of queryTerms) {
-                    if (keywords.some(k => k.toLowerCase() === term)) {
-                        boost += 0.15;
-                    }
-                }
-                for (const term of queryTerms) {
-                    if (titleLower.includes(term)) {
-                        boost += 0.10;
-                    }
-                }
-            }
-            else {
-                boost = -0.3;
-            }
-            return {
-                item: r.item,
-                baseScore,
-                boost,
-                finalScore: baseScore + boost,
-            };
-        });
-        scored.sort((a, b) => b.finalScore - a.finalScore);
-        return scored.map((r) => ({
-            id: r.item.id,
-            path: r.item.path,
-            title: r.item.title,
-            content: r.item.content,
-            snippet: this.extractSnippet(r.item.content, query),
-            score: Math.max(0, Math.min(1, r.finalScore)),
-            source: 'fuzzy',
-        }));
-    }
-    async semanticSearch(query, limit, category) {
-        if (!this.embeddingsData || this.vectors.size === 0) {
-            return [];
-        }
-        if (this.config.embedder) {
-            try {
-                const model = this.embeddingsData.model;
-                const queryVector = await this.config.embedder(query, model);
-                this.log(`Generated query vector using provided embedder (model: ${model})`);
-                const scores = [];
-                for (const [id, vector] of this.vectors) {
-                    if (category) {
-                        const entry = this.embeddingsData.documents.find((e) => e.id === id);
-                        if (!entry || !entry.category.toLowerCase().includes(category.toLowerCase())) {
-                            continue;
-                        }
-                    }
-                    if (vector.length !== queryVector.length)
-                        continue;
-                    const score = cosineSimilarity(queryVector, vector);
-                    if (score > 0.05) {
-                        scores.push({ id, score });
-                    }
-                }
-                const results = [];
-                for (const s of scores.sort((a, b) => b.score - a.score).slice(0, limit)) {
-                    const doc = this.docs.find((d) => d.id === s.id);
-                    if (!doc)
-                        continue;
-                    results.push({
-                        id: s.id,
-                        path: doc.path,
-                        title: doc.title,
-                        content: doc.content,
-                        snippet: this.extractSnippet(doc.content, query),
-                        score: s.score,
-                        source: 'semantic',
-                    });
-                }
-                return results;
-            }
-            catch (error) {
-                this.log(`Embedder failed: ${error}. Falling back to synthetic vectors.`);
-            }
-        }
-        const queryTerms = this.tokenize(query);
-        const scores = [];
-        for (const entry of this.embeddingsData.documents) {
-            if (category && !entry.category.toLowerCase().includes(category.toLowerCase())) {
-                continue;
-            }
-            let termScore = 0;
-            const docText = `${entry.title} ${entry.keywords?.join(' ') || ''}`.toLowerCase();
-            const docTerms = this.tokenize(docText);
-            for (const queryTerm of queryTerms) {
-                if (docTerms.includes(queryTerm)) {
-                    termScore += 2;
-                    continue;
-                }
-                for (const docTerm of docTerms) {
-                    const distance = levenshtein(queryTerm, docTerm);
-                    const maxLen = Math.max(queryTerm.length, docTerm.length);
-                    if (distance <= Math.min(2, maxLen * 0.3)) {
-                        termScore += 1 - distance / maxLen;
-                        break;
-                    }
-                }
-            }
-            const normalizedScore = Math.min(1, termScore / (queryTerms.length * 2));
-            if (normalizedScore > 0.1) {
-                scores.push({ id: entry.id, score: normalizedScore });
-            }
-        }
-        const topDocs = scores.sort((a, b) => b.score - a.score).slice(0, 3);
-        if (topDocs.length > 0) {
-            const topVectors = topDocs
-                .map((d) => this.vectors.get(d.id))
-                .filter((v) => Array.isArray(v) && v.length > 0);
-            if (topVectors.length > 0 && topVectors.every((v) => v.length === topVectors[0].length)) {
-                const avgVector = this.averageVectors(topVectors);
-                const expectedDimensions = avgVector.length;
-                for (const entry of this.embeddingsData.documents) {
-                    if (scores.some((s) => s.id === entry.id))
-                        continue;
-                    if (category && !entry.category.toLowerCase().includes(category.toLowerCase()))
-                        continue;
-                    const vector = this.vectors.get(entry.id);
-                    if (!vector || vector.length === 0 || vector.length !== expectedDimensions)
-                        continue;
-                    const similarity = cosineSimilarity(avgVector, vector);
-                    if (similarity > 0.7) {
-                        scores.push({ id: entry.id, score: similarity * 0.5 });
-                    }
-                }
-            }
-        }
-        const results = [];
-        for (const s of scores.sort((a, b) => b.score - a.score).slice(0, limit)) {
-            const doc = this.docs.find((d) => d.id === s.id);
-            if (!doc) {
-                continue;
-            }
-            results.push({
-                id: s.id,
-                path: doc.path,
-                title: doc.title,
-                content: doc.content,
-                snippet: this.extractSnippet(doc.content, query),
-                score: s.score,
-                source: 'semantic',
-            });
-        }
-        return results;
-    }
-    extractSnippet(content, query) {
-        if (!content)
-            return '';
-        const lowerContent = content.toLowerCase();
-        const queryTerms = this.tokenize(query);
-        let bestIndex = -1;
-        let bestScore = 0;
-        for (const term of queryTerms) {
-            const idx = lowerContent.indexOf(term);
-            if (idx !== -1) {
-                bestIndex = idx;
-                bestScore = term.length;
-                break;
-            }
-        }
-        if (bestIndex === -1) {
-            const words = lowerContent.split(/\s+/).slice(0, 500);
-            for (let i = 0; i < words.length; i++) {
-                for (const term of queryTerms) {
-                    const dist = levenshtein(words[i].slice(0, 20), term.slice(0, 20));
-                    if (dist <= 2 && dist < bestScore) {
-                        bestScore = dist;
-                        bestIndex = lowerContent.indexOf(words[i]);
-                    }
-                }
-            }
-        }
-        if (bestIndex === -1) {
-            return content.slice(0, 200).trim() + (content.length > 200 ? '...' : '');
-        }
-        const start = Math.max(0, bestIndex - 50);
-        const end = Math.min(content.length, bestIndex + 150);
-        let snippet = content.slice(start, end).trim();
-        if (start > 0)
-            snippet = '...' + snippet;
-        if (end < content.length)
-            snippet = snippet + '...';
-        return snippet.replace(/\n+/g, ' ').replace(/\s+/g, ' ');
-    }
-    static STOP_WORDS = new Set([
-        'the', 'a', 'an', 'is', 'are', 'was', 'were', 'be', 'been', 'being',
-        'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could', 'should',
-        'may', 'might', 'must', 'shall', 'can', 'need', 'dare', 'ought', 'used',
-        'and', 'but', 'or', 'nor', 'for', 'yet', 'so', 'both', 'either', 'neither',
-        'not', 'only', 'own', 'same', 'than', 'too', 'very', 'just', 'also',
-        'how', 'what', 'when', 'where', 'who', 'which', 'why', 'whom', 'whose',
-        'this', 'that', 'these', 'those', 'here', 'there', 'all', 'each', 'every',
-        'any', 'some', 'no', 'none', 'one', 'two', 'other', 'another', 'such',
-        'to', 'of', 'in', 'on', 'at', 'by', 'with', 'from', 'as', 'into', 'through',
-        'about', 'above', 'below', 'between', 'under', 'over', 'out', 'up', 'down',
-        'if', 'then', 'else', 'because', 'while', 'although', 'though', 'unless',
-        'my', 'your', 'his', 'her', 'its', 'our', 'their', 'me', 'you', 'him', 'us', 'them',
-        'o', 'a', 'os', 'as', 'um', 'uma', 'uns', 'umas', 'de', 'do', 'da', 'dos', 'das',
-        'em', 'no', 'na', 'nos', 'nas', 'por', 'para', 'com', 'sem', 'sob', 'sobre',
-        'e', 'ou', 'mas', 'porem', 'todavia', 'contudo', 'entretanto',
-        'que', 'qual', 'quais', 'quanto', 'quem', 'como', 'onde', 'quando', 'porque',
-        'eu', 'tu', 'ele', 'ela', 'nos', 'vos', 'eles', 'elas', 'voce', 'voces',
-        'meu', 'minha', 'meus', 'minhas', 'seu', 'sua', 'seus', 'suas',
-        'este', 'esta', 'estes', 'estas', 'esse', 'essa', 'esses', 'essas',
-        'isso', 'isto', 'aquilo', 'aquele', 'aquela', 'aqueles', 'aquelas',
-        'ser', 'estar', 'ter', 'haver', 'fazer', 'ir', 'vir', 'poder', 'dever',
-        'sim', 'nao', 'ja', 'ainda', 'sempre', 'nunca', 'tambem', 'so', 'apenas',
-        'muito', 'pouco', 'mais', 'menos', 'bem', 'mal', 'assim', 'entao', 'logo',
-        'yo', 'hey', 'oi', 'ola', 'bom', 'boa', 'obrigado', 'por favor',
-        'configure', 'configuro', 'configurar', 'configurando', 'configura',
-        'use', 'usar', 'using', 'uso', 'usa',
-        'create', 'criar', 'creating', 'criando', 'cria', 'crio',
-        'setup', 'setar', 'setting', 'setando', 'seta', 'seto',
-        'add', 'adicionar', 'adding', 'adicionando', 'adiciona', 'adiciono',
-        'get', 'getting', 'pegar', 'pegando', 'pega', 'pego',
-        'set', 'setting', 'definir', 'definindo', 'define', 'defino',
-        'make', 'making', 'fazer', 'fazendo', 'faz', 'faço', 'faco',
-    ]);
-    cleanQuery(query) {
-        const words = query
-            .toLowerCase()
-            .split(/[\s\-_.,;:!?()[\]{}'"]+/)
-            .filter((w) => w.length > 1 && !HybridSearch.STOP_WORDS.has(w));
-        return words.join(' ');
-    }
-    tokenize(text) {
-        return text
-            .toLowerCase()
-            .split(/[\s\-_.,;:!?()[\]{}'"]+/)
-            .filter((t) => t.length > 2 && !HybridSearch.STOP_WORDS.has(t));
-    }
-    averageVectors(vectors) {
-        if (vectors.length === 0)
-            return [];
-        if (vectors.length === 1)
-            return vectors[0];
-        const result = new Array(vectors[0].length).fill(0);
-        for (const vec of vectors) {
-            for (let i = 0; i < vec.length; i++) {
-                result[i] += vec[i];
-            }
-        }
-        for (let i = 0; i < result.length; i++) {
-            result[i] /= vectors.length;
-        }
-        return result;
-    }
-    hasEmbeddings() {
-        return this.vectors.size > 0;
-    }
-    getStats() {
-        return {
-            documents: this.docs.length,
-            embeddings: this.vectors.size,
-            model: this.embeddingsData?.model,
-            dimensions: this.embeddingsData?.dimensions,
-        };
-    }
-    log(message) {
-        if (this.config.debug) {
-            console.log(`[HybridSearch] ${message}`);
-        }
-    }
-}
-export function createHybridSearch(config) {
-    return new HybridSearch(config);
-}

package/dist/mcp/search/index.d.ts

@@ -1,4 +0,0 @@
-export { HybridSearch, createHybridSearch } from './hybrid-search.js';
-export { cosineSimilarity, levenshtein, stringSimilarity, reciprocalRankFusion, combineScores, } from './math.js';
-export { embed, embedBatch, createEmbedder, isFastembedAvailable, getModelInfo, unloadEmbedder, } from './embedder.js';
-export type { IndexedDoc, SearchResult, SearchOptions, HybridSearchConfig, EmbeddingsData, EmbeddingEntry, } from './types.js';

package/dist/mcp/search/index.js

@@ -1,3 +0,0 @@
-export { HybridSearch, createHybridSearch } from './hybrid-search.js';
-export { cosineSimilarity, levenshtein, stringSimilarity, reciprocalRankFusion, combineScores, } from './math.js';
-export { embed, embedBatch, createEmbedder, isFastembedAvailable, getModelInfo, unloadEmbedder, } from './embedder.js';

package/dist/mcp/search/math.d.ts

@@ -1,5 +0,0 @@
-export declare function cosineSimilarity(a: number[], b: number[]): number;
-export declare function levenshtein(a: string, b: string): number;
-export declare function stringSimilarity(a: string, b: string): number;
-export declare function reciprocalRankFusion(ranks: number[], k?: number): number;
-export declare function combineScores(score1: number, score2: number, k?: number): number;

package/dist/mcp/search/math.js

@@ -1,63 +0,0 @@
-import { ValidationError } from '../../core/errors.js';
-export function cosineSimilarity(a, b) {
-    if (a.length !== b.length) {
-        throw new ValidationError(`Vector length mismatch: ${a.length} vs ${b.length}`, {
-            field: 'vectorLength',
-            value: { a: a.length, b: b.length },
-        });
-    }
-    if (a.length === 0) {
-        return 0;
-    }
-    let dot = 0;
-    let normA = 0;
-    let normB = 0;
-    for (let i = 0; i < a.length; i++) {
-        dot += a[i] * b[i];
-        normA += a[i] * a[i];
-        normB += b[i] * b[i];
-    }
-    const magnitude = Math.sqrt(normA) * Math.sqrt(normB);
-    return magnitude === 0 ? 0 : dot / magnitude;
-}
-export function levenshtein(a, b) {
-    if (a === b)
-        return 0;
-    if (a.length === 0)
-        return b.length;
-    if (b.length === 0)
-        return a.length;
-    if (a.length > b.length) {
-        [a, b] = [b, a];
-    }
-    const aLen = a.length;
-    const bLen = b.length;
-    let prevRow = new Array(aLen + 1);
-    let currRow = new Array(aLen + 1);
-    for (let i = 0; i <= aLen; i++) {
-        prevRow[i] = i;
-    }
-    for (let j = 1; j <= bLen; j++) {
-        currRow[0] = j;
-        for (let i = 1; i <= aLen; i++) {
-            const cost = a[i - 1] === b[j - 1] ? 0 : 1;
-            currRow[i] = Math.min(prevRow[i] + 1, currRow[i - 1] + 1, prevRow[i - 1] + cost);
-        }
-        [prevRow, currRow] = [currRow, prevRow];
-    }
-    return prevRow[aLen];
-}
-export function stringSimilarity(a, b) {
-    const maxLen = Math.max(a.length, b.length);
-    if (maxLen === 0)
-        return 1;
-    return 1 - levenshtein(a, b) / maxLen;
-}
-export function reciprocalRankFusion(ranks, k = 60) {
-    return ranks.reduce((sum, rank) => sum + 1 / (k + rank), 0);
-}
-export function combineScores(score1, score2, k = 60) {
-    const rank1 = 1 + (1 - score1) * 100;
-    const rank2 = 1 + (1 - score2) * 100;
-    return reciprocalRankFusion([rank1, rank2], k);
-}

package/dist/mcp/search/types.d.ts

@@ -1,51 +0,0 @@
-export interface IndexedDoc {
-    id: string;
-    path: string;
-    title: string;
-    content: string;
-    category: string;
-    keywords: string[];
-    section?: string;
-    parentPath?: string;
-}
-export interface SearchResult {
-    id: string;
-    path: string;
-    title: string;
-    content: string;
-    snippet: string;
-    score: number;
-    source: 'fuzzy' | 'semantic' | 'hybrid';
-}
-export interface EmbeddingsData {
-    version: string;
-    model: string;
-    dimensions: number;
-    generatedAt: string;
-    documents: EmbeddingEntry[];
-}
-export interface EmbeddingEntry {
-    id: string;
-    path: string;
-    title: string;
-    category: string;
-    keywords: string[];
-    section?: string;
-    parentPath?: string;
-    content?: string;
-    vector: number[];
-}
-export interface SearchOptions {
-    limit?: number;
-    category?: string;
-    mode?: 'hybrid' | 'fuzzy' | 'semantic';
-    minScore?: number;
-}
-export interface HybridSearchConfig {
-    fuzzyThreshold?: number;
-    fuzzyWeight?: number;
-    semanticWeight?: number;
-    debug?: boolean;
-    offline?: boolean;
-    embedder?: (text: string, model?: string) => Promise<number[]>;
-}

package/dist/mcp/search/types.js

@@ -1 +0,0 @@
-export {};