raggrep 0.1.0

package/dist/types.d.ts

@@ -0,0 +1,84 @@
+/**
+ * Type definitions for the RAG system
+ *
+ * This file re-exports domain entities and defines module interfaces.
+ * For new code, prefer importing directly from 'domain/entities'.
+ */
+export type { Chunk, ChunkType, FileIndex, FileManifestEntry, ModuleManifest, GlobalManifest, FileSummary, Tier1Manifest, SearchResult, SearchOptions, Config, ModuleConfig, } from './domain/entities';
+export { createChunkId, DEFAULT_SEARCH_OPTIONS, DEFAULT_IGNORE_PATHS, DEFAULT_EXTENSIONS, createDefaultConfig, } from './domain/entities';
+import type { Config, FileIndex, SearchResult, SearchOptions, ModuleConfig } from './domain/entities';
+/**
+ * Context provided to modules during indexing
+ */
+export interface IndexContext {
+    rootDir: string;
+    config: Config;
+    /** Get the content of a file */
+    readFile: (filepath: string) => Promise<string>;
+    /** Get file stats */
+    getFileStats: (filepath: string) => Promise<{
+        lastModified: string;
+    }>;
+}
+/**
+ * Context provided to modules during search
+ */
+export interface SearchContext {
+    rootDir: string;
+    config: Config;
+    /** Load index data for a specific file */
+    loadFileIndex: (filepath: string) => Promise<FileIndex | null>;
+    /** List all indexed files */
+    listIndexedFiles: () => Promise<string[]>;
+}
+/**
+ * Base interface for index modules
+ *
+ * Modules provide different strategies for indexing and retrieving code.
+ * Examples:
+ * - SemanticModule: Uses text embeddings for natural language search
+ * - SymbolModule: Uses TypeScript/LSP symbol information
+ * - ASTModule: Uses AST-based code structure analysis
+ */
+export interface IndexModule {
+    /** Unique identifier for this module */
+    readonly id: string;
+    /** Human-readable name */
+    readonly name: string;
+    /** Description of what this module indexes */
+    readonly description: string;
+    /** Version of the module (for index compatibility) */
+    readonly version: string;
+    /**
+     * Index a single file
+     * @returns FileIndex with module-specific data, or null if file should be skipped
+     */
+    indexFile(filepath: string, content: string, ctx: IndexContext): Promise<FileIndex | null>;
+    /**
+     * Search the index with a query
+     * @returns Ranked search results
+     */
+    search(query: string, ctx: SearchContext, options?: SearchOptions): Promise<SearchResult[]>;
+    /**
+     * Optional: Initialize the module (e.g., load models, connect to services)
+     */
+    initialize?(config: ModuleConfig): Promise<void>;
+    /**
+     * Optional: Called after all files have been indexed.
+     * Use for building secondary indexes (e.g., Tier 1 summaries, BM25 index).
+     */
+    finalize?(ctx: IndexContext): Promise<void>;
+    /**
+     * Optional: Cleanup resources
+     */
+    dispose?(): Promise<void>;
+}
+/**
+ * Registry for managing available modules
+ */
+export interface ModuleRegistry {
+    register(module: IndexModule): void;
+    get(id: string): IndexModule | undefined;
+    list(): IndexModule[];
+    getEnabled(config: Config): IndexModule[];
+}

package/dist/utils/bm25.d.ts

@@ -0,0 +1,9 @@
+/**
+ * BM25 Search Utilities
+ *
+ * Re-exports BM25 functionality from the domain layer.
+ * This file exists for backwards compatibility with existing code.
+ *
+ * For new code, import directly from 'domain/services'.
+ */
+export { BM25Index, tokenize, normalizeScore, type BM25Document, type BM25Result, } from '../domain/services/bm25';

package/dist/utils/config.d.ts

@@ -0,0 +1,45 @@
+/**
+ * Configuration utilities
+ *
+ * Provides functions for loading, saving, and managing RAGgrep configuration.
+ */
+import type { Config, ModuleConfig } from '../domain/entities';
+import { EmbeddingConfig } from './embeddings';
+/** Default configuration instance */
+export declare const DEFAULT_CONFIG: Config;
+/**
+ * Get the root .raggrep directory path
+ */
+export declare function getRaggrepDir(rootDir: string, config?: Config): string;
+/**
+ * Get the index data directory for a specific module
+ */
+export declare function getModuleIndexPath(rootDir: string, moduleId: string, config?: Config): string;
+/**
+ * Get the manifest path for a specific module
+ */
+export declare function getModuleManifestPath(rootDir: string, moduleId: string, config?: Config): string;
+/**
+ * Get the global manifest path
+ */
+export declare function getGlobalManifestPath(rootDir: string, config?: Config): string;
+/**
+ * Get the config file path
+ */
+export declare function getConfigPath(rootDir: string, config?: Config): string;
+/**
+ * Load config from file or return default
+ */
+export declare function loadConfig(rootDir: string): Promise<Config>;
+/**
+ * Save config to file
+ */
+export declare function saveConfig(rootDir: string, config: Config): Promise<void>;
+/**
+ * Get module config by ID
+ */
+export declare function getModuleConfig(config: Config, moduleId: string): ModuleConfig | undefined;
+/**
+ * Extract embedding config from module options
+ */
+export declare function getEmbeddingConfigFromModule(moduleConfig: ModuleConfig): EmbeddingConfig;

package/dist/utils/embeddings.d.ts

@@ -0,0 +1,46 @@
+export declare const EMBEDDING_MODELS: {
+    readonly 'all-MiniLM-L6-v2': "Xenova/all-MiniLM-L6-v2";
+    readonly 'all-MiniLM-L12-v2': "Xenova/all-MiniLM-L12-v2";
+    readonly 'bge-small-en-v1.5': "Xenova/bge-small-en-v1.5";
+    readonly 'paraphrase-MiniLM-L3-v2': "Xenova/paraphrase-MiniLM-L3-v2";
+};
+export type EmbeddingModelName = keyof typeof EMBEDDING_MODELS;
+export interface EmbeddingConfig {
+    model: EmbeddingModelName;
+    /** Show progress during model download */
+    showProgress?: boolean;
+}
+/**
+ * Configure the embedding model
+ */
+export declare function configureEmbeddings(config: Partial<EmbeddingConfig>): void;
+/**
+ * Get embedding for a single text
+ */
+export declare function getEmbedding(text: string): Promise<number[]>;
+/**
+ * Get embeddings for multiple texts (batched for efficiency)
+ *
+ * Processes texts in batches of BATCH_SIZE for better performance
+ * while avoiding memory issues with very large batches.
+ *
+ * @param texts - Array of texts to embed
+ * @returns Array of embedding vectors
+ */
+export declare function getEmbeddings(texts: string[]): Promise<number[][]>;
+/**
+ * Calculate cosine similarity between two vectors
+ */
+export declare function cosineSimilarity(a: number[], b: number[]): number;
+/**
+ * Get current embedding configuration
+ */
+export declare function getEmbeddingConfig(): EmbeddingConfig;
+/**
+ * Get the cache directory path
+ */
+export declare function getCacheDir(): string;
+/**
+ * Check if a model is already cached
+ */
+export declare function isModelCached(model?: EmbeddingModelName): Promise<boolean>;

package/dist/utils/embeddings.test.d.ts

@@ -0,0 +1,4 @@
+/**
+ * Tests for embedding utilities
+ */
+export {};

package/dist/utils/tieredIndex.d.ts

@@ -0,0 +1,100 @@
+/**
+ * Symbolic Index System
+ *
+ * Provides fast keyword-based filtering using BM25 before semantic search.
+ *
+ * Structure:
+ *   .raggrep/index/<module>/symbolic/
+ *   ├── _meta.json    (BM25 statistics)
+ *   └── <filepath>.json (per-file summaries)
+ *
+ * This approach scales well because:
+ * - Each file summary is stored separately
+ * - BM25 metadata is small and loads quickly
+ * - Summaries are loaded on-demand during search
+ */
+import type { FileSummary, SymbolicIndexMeta } from '../domain/entities';
+export type { FileSummary, SymbolicIndexMeta } from '../domain/entities';
+export { extractKeywords } from '../domain/services/keywords';
+/** @deprecated Use SymbolicIndexMeta */
+export type Tier1Manifest = SymbolicIndexMeta & {
+    files: Record<string, FileSummary>;
+};
+/**
+ * Symbolic Index Manager
+ *
+ * Manages the keyword-based index for fast file filtering.
+ */
+export declare class SymbolicIndex {
+    private meta;
+    private fileSummaries;
+    private bm25Index;
+    private symbolicPath;
+    private moduleId;
+    constructor(indexDir: string, moduleId: string);
+    /**
+     * Initialize or load the symbolic index
+     */
+    initialize(): Promise<void>;
+    /**
+     * Add or update a file summary
+     */
+    addFile(summary: FileSummary): void;
+    /**
+     * Remove a file from the index
+     */
+    removeFile(filepath: string): boolean;
+    /**
+     * Build BM25 index from file summaries
+     */
+    buildBM25Index(): void;
+    /**
+     * Find candidate files using BM25 keyword search
+     */
+    findCandidates(query: string, maxCandidates?: number): string[];
+    /**
+     * Get all file paths in the index
+     */
+    getAllFiles(): string[];
+    /**
+     * Get summary for a specific file
+     */
+    getFileSummary(filepath: string): FileSummary | undefined;
+    /**
+     * Save the index to disk (per-file structure)
+     */
+    save(): Promise<void>;
+    /**
+     * Load the index from disk
+     */
+    load(): Promise<void>;
+    /**
+     * Recursively load file summaries from the symbolic directory
+     */
+    private loadFileSummariesRecursive;
+    /**
+     * Get the path for a file summary
+     */
+    private getFileSummaryPath;
+    /**
+     * Delete a file summary from disk
+     */
+    deleteFileSummary(filepath: string): Promise<void>;
+    /**
+     * Check if the index exists on disk
+     */
+    exists(): Promise<boolean>;
+    /**
+     * Get the number of indexed files
+     */
+    get size(): number;
+    /**
+     * Clear the index
+     */
+    clear(): void;
+}
+/** @deprecated Use SymbolicIndex instead */
+export declare const Tier1Index: typeof SymbolicIndex;
+/** @deprecated Use SymbolicIndex instead */
+export declare function getTier1Path(rootDir: string, moduleId: string, indexDir?: string): string;
+export declare function getSymbolicPath(rootDir: string, moduleId: string, indexDir?: string): string;

package/package.json

@@ -0,0 +1,66 @@
+{
+  "name": "raggrep",
+  "version": "0.1.0",
+  "description": "Local filesystem-based RAG system for codebases - semantic search using local embeddings",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    }
+  },
+  "bin": {
+    "raggrep": "./dist/cli/main.js"
+  },
+  "files": [
+    "dist",
+    "README.md",
+    "LICENSE"
+  ],
+  "scripts": {
+    "build": "bun run build:clean && bun run build:bundle && bun run build:types && bun run build:shebang",
+    "build:clean": "rm -rf dist",
+    "build:bundle": "bun build src/index.ts --outdir dist --target node --sourcemap=external --external '@xenova/transformers' --external 'glob' --external 'typescript' && bun build src/cli/main.ts --outdir dist/cli --target node --sourcemap=external --external '@xenova/transformers' --external 'glob' --external 'typescript'",
+    "build:types": "tsc --emitDeclarationOnly --outDir dist",
+    "build:shebang": "echo '#!/usr/bin/env node' | cat - dist/cli/main.js > temp && mv temp dist/cli/main.js",
+    "prepublishOnly": "bun run build",
+    "raggrep": "bun run src/cli/main.ts",
+    "test": "bun test",
+    "dev": "bun run src/cli/main.ts"
+  },
+  "keywords": [
+    "rag",
+    "search",
+    "semantic-search",
+    "embeddings",
+    "codebase",
+    "local",
+    "ai",
+    "code-search",
+    "transformers"
+  ],
+  "author": "",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/conradkoh/raggrep.git"
+  },
+  "bugs": {
+    "url": "https://github.com/conradkoh/raggrep/issues"
+  },
+  "homepage": "https://github.com/conradkoh/raggrep#readme",
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "dependencies": {
+    "@xenova/transformers": "^2.17.0",
+    "glob": "^10.0.0",
+    "typescript": "^5.0.0"
+  },
+  "devDependencies": {
+    "@types/bun": "latest",
+    "@types/node": "^20.0.0"
+  }
+}