rag-lite-ts 1.0.1

package/dist/ingestion.d.ts

@@ -0,0 +1,175 @@
+import { type FileProcessorOptions } from './file-processor.js';
+import { type ChunkConfig } from './chunker.js';
+import { type EmbeddingEngine } from './embedder.js';
+import { Config } from './config.js';
+/**
+ * User-friendly error class with actionable suggestions
+ */
+export declare class IngestionError extends Error {
+    code: string;
+    suggestions: string[];
+    constructor(message: string, code: string, suggestions: string[]);
+}
+/**
+ * Options for the ingestion pipeline
+ */
+export interface IngestionOptions {
+    /** File processing options */
+    fileOptions?: FileProcessorOptions;
+    /** Chunking configuration */
+    chunkConfig?: ChunkConfig;
+    /** Whether to force rebuild the index */
+    forceRebuild?: boolean;
+}
+/**
+ * Result of the ingestion process
+ */
+export interface IngestionResult {
+    /** Total documents processed */
+    documentsProcessed: number;
+    /** Total chunks created */
+    chunksCreated: number;
+    /** Total embeddings generated */
+    embeddingsGenerated: number;
+    /** Number of documents that failed processing */
+    documentErrors: number;
+    /** Number of chunks that failed embedding */
+    embeddingErrors: number;
+    /** Processing time in milliseconds */
+    processingTimeMs: number;
+}
+/**
+ * Main ingestion pipeline class
+ * Coordinates the entire process from file discovery to vector storage
+ */
+export declare class IngestionPipeline {
+    private static instances;
+    private static cleanupHandlersSet;
+    private db;
+    private indexManager;
+    private embeddingEngine;
+    private pathManager;
+    private isInitialized;
+    private dbPath;
+    private indexPath;
+    private basePath;
+    private configOverrides;
+    /**
+     * Creates a new IngestionPipeline with simplified constructor
+     * Pipeline is ready to use immediately without requiring initialization calls (Requirement 1.5)
+     * @param basePath - Base directory path for database and index files (defaults to current directory)
+     * @param embedder - Pre-initialized embedding engine (optional, will use default if not provided)
+     */
+    constructor(basePath?: string, embedder?: EmbeddingEngine);
+    /**
+     * Set configuration overrides (for internal use)
+     * @param overrides - Configuration overrides to apply
+     */
+    setConfigOverrides(overrides: Partial<Config>): void;
+    /**
+     * Set path storage strategy
+     * @param strategy - Path storage strategy ('absolute' or 'relative')
+     * @param basePath - Base path for relative paths (optional, defaults to current base path)
+     */
+    setPathStorageStrategy(strategy: 'absolute' | 'relative', basePath?: string): void;
+    /**
+     * Get effective configuration with overrides applied
+     */
+    private getEffectiveConfig;
+    /**
+     * Automatically initialize resources on first use with user-friendly error handling
+     * Implements lazy initialization as required by 5.2
+     */
+    private ensureInitialized;
+    /**
+     * Create user-friendly error messages with actionable suggestions
+     * Implements requirement 5.3: Clear, actionable error messages with specific next steps
+     */
+    private createUserFriendlyError;
+    /**
+     * Initialize the ingestion pipeline (public method for backward compatibility)
+     * Sets up database, index manager, and embedding engine
+     */
+    initialize(): Promise<void>;
+    /**
+     * Ingest documents from a directory (matches README API)
+     * Automatically initializes resources on first use (Requirements 2.1, 2.3, 5.2)
+     * @param directoryPath - Path to directory containing documents
+     * @param options - Optional ingestion configuration
+     * @returns Promise resolving to ingestion results
+     */
+    ingestDirectory(directoryPath: string, options?: IngestionOptions): Promise<IngestionResult>;
+    /**
+     * Ingest a single file (matches README API)
+     * Automatically initializes resources on first use (Requirements 2.2, 2.3, 5.2)
+     * @param filePath - Path to the file to ingest
+     * @param options - Optional ingestion configuration
+     * @returns Promise resolving to ingestion results
+     */
+    ingestFile(filePath: string, options?: IngestionOptions): Promise<IngestionResult>;
+    /**
+     * Ingest documents from a path (file or directory)
+     * Implements the complete pipeline: file processing → chunking → embedding → storage
+     *
+     * Requirements addressed:
+     * - 7.5: Single-threaded write processing to avoid SQLite lock contention
+     * - 3.3: Graceful handling of embedding failures without stopping ingestion
+     * - 10.1: Progress logging and error reporting during batch ingestion
+     * - 2.3: Automatic creation of database and index files in appropriate locations
+     */
+    ingestPath(path: string, options?: IngestionOptions): Promise<IngestionResult>;
+    /**
+     * Chunk all documents and organize results
+     */
+    private chunkDocuments;
+    /**
+     * Generate embeddings for all chunks with error handling
+     * Requirement 3.3: Graceful handling of embedding failures without stopping ingestion
+     */
+    private generateEmbeddings;
+    /**
+     * Store documents and chunks in database with single-threaded writes
+     * Requirement 7.5: Single-threaded write processing to avoid SQLite lock contention
+     */
+    private storeDocumentsAndChunks;
+    /**
+     * Update vector index with new embeddings
+     */
+    private updateVectorIndex;
+    /**
+     * Initialize the pipeline for rebuild (skips model compatibility check)
+     */
+    private initializeForRebuild;
+    /**
+     * Rebuild the entire index from scratch
+     * Useful when model version changes or for maintenance
+     * Automatically initializes resources if needed (Requirement 5.2)
+     */
+    rebuildIndex(): Promise<void>;
+    /**
+     * Get pipeline statistics
+     */
+    getStats(): Promise<{
+        indexStats: any;
+        isInitialized: boolean;
+    }>;
+    /**
+     * Set up automatic cleanup on process exit (Requirement 5.5)
+     */
+    private setupAutomaticCleanup;
+    /**
+     * Clean up resources
+     */
+    cleanup(): Promise<void>;
+}
+/**
+ * Convenience function to ingest documents from a path
+ * Creates a pipeline instance, runs ingestion, and cleans up
+ */
+export declare function ingestDocuments(path: string, options?: IngestionOptions): Promise<IngestionResult>;
+/**
+ * Convenience function to rebuild the index
+ * Creates a pipeline instance, rebuilds index, and cleans up
+ */
+export declare function rebuildIndex(): Promise<void>;
+//# sourceMappingURL=ingestion.d.ts.map

package/dist/ingestion.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ingestion.d.ts","sourceRoot":"","sources":["../src/ingestion.ts"],"names":[],"mappings":"AAAA,OAAO,EAA2B,KAAK,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AACzF,OAAO,EAAiB,KAAK,WAAW,EAAE,MAAM,cAAc,CAAC;AAC/D,OAAO,EAA6B,KAAK,eAAe,EAAE,MAAM,eAAe,CAAC;AAGhF,OAAO,EAA0B,MAAM,EAAoB,MAAM,aAAa,CAAC;AAO/E;;GAEG;AACH,qBAAa,cAAe,SAAQ,KAAK;IAG9B,IAAI,EAAE,MAAM;IACZ,WAAW,EAAE,MAAM,EAAE;gBAF5B,OAAO,EAAE,MAAM,EACR,IAAI,EAAE,MAAM,EACZ,WAAW,EAAE,MAAM,EAAE;CAK/B;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,8BAA8B;IAC9B,WAAW,CAAC,EAAE,oBAAoB,CAAC;IACnC,6BAA6B;IAC7B,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,yCAAyC;IACzC,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,gCAAgC;IAChC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,2BAA2B;IAC3B,aAAa,EAAE,MAAM,CAAC;IACtB,iCAAiC;IACjC,mBAAmB,EAAE,MAAM,CAAC;IAC5B,iDAAiD;IACjD,cAAc,EAAE,MAAM,CAAC;IACvB,6CAA6C;IAC7C,eAAe,EAAE,MAAM,CAAC;IACxB,sCAAsC;IACtC,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AA4BD;;;GAGG;AACH,qBAAa,iBAAiB;IAE5B,OAAO,CAAC,MAAM,CAAC,SAAS,CAAgC;IACxD,OAAO,CAAC,MAAM,CAAC,kBAAkB,CAAS;IAE1C,OAAO,CAAC,EAAE,CAAmC;IAC7C,OAAO,CAAC,YAAY,CAA6B;IACjD,OAAO,CAAC,eAAe,CAAgC;IACvD,OAAO,CAAC,WAAW,CAAoC;IACvD,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,eAAe,CAAuB;IAE9C;;;;;OAKG;gBACS,QAAQ,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,eAAe;IAgCzD;;;OAGG;IACH,kBAAkB,CAAC,SAAS,EAAE,OAAO,CAAC,MAAM,CAAC,GAAG,IAAI;IAIpD;;;;OAIG;IACH,sBAAsB,CAAC,QAAQ,EAAE,UAAU,GAAG,UAAU,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI;IAKlF;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAuB1B;;;OAGG;YACW,iBAAiB;IA+C/B;;;OAGG;IACH,OAAO,CAAC,uBAAuB;IAuG/B;;;OAGG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAIjC;;;;;;OAMG;IACG,eAAe,CAAC,aAAa,EAAE,MAAM,EAAE,OAAO,GAAE,gBAAqB,GAAG,OAAO,CAAC,eAAe,CAAC;IAetG;;;;;;OAMG;IACG,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,gBAAqB,GAAG,OAAO,CAAC,eAAe,CAAC;IAe5F;;;;;;;;;OASG;IACG,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,gBAAqB,GAAG,OAAO,CAAC,eAAe,CAAC;IA+FxF;;OAEG;YACW,cAAc;IA2C5B;;;OAGG;YACW,kBAAkB;IA+BhC;;;OAGG;YACW,uBAAuB;IA4ErC;;OAEG;YACW,iBAAiB;IAqB/B;;OAEG;YACW,oBAAoB;IA2ClC;;;;OAIG;IACG,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAuBnC;;OAEG;IACG,QAAQ,IAAI,OAAO,CAAC;QACxB,UAAU,EAAE,GAAG,CAAC;QAChB,aAAa,EAAE,OAAO,CAAC;KACxB,CAAC;IAiBF;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAyD7B;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAuB/B;AAED;;;GAGG;AACH,wBAAsB,eAAe,CACnC,IAAI,EAAE,MAAM,EACZ,OAAO,GAAE,gBAAqB,GAC7B,OAAO,CAAC,eAAe,CAAC,CAU1B;AAED;;;GAGG;AACH,wBAAsB,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC,CAmClD"}