tea-rags 0.9.0

package/build/core/domains/ingest/pipeline/infra/worker-pool.js

@@ -0,0 +1,212 @@
+/**
+ * WorkerPool - Bounded concurrency executor with retry support
+ *
+ * Features:
+ * - Fixed number of concurrent workers
+ * - Exponential backoff retry on failures
+ * - Queue management with backpressure signaling
+ * - Graceful shutdown
+ */
+import { isDebug } from "./runtime.js";
+export class WorkerPool {
+    config;
+    onCompletion;
+    onQueueChange;
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any -- heterogeneous batch queue requires existential type
+    queue = [];
+    activeWorkers = 0;
+    isShuttingDown = false;
+    totalProcessed = 0;
+    totalErrors = 0;
+    totalTimeMs = 0;
+    startTime;
+    constructor(config, onCompletion, onQueueChange) {
+        this.config = config;
+        this.onCompletion = onCompletion;
+        this.onQueueChange = onQueueChange;
+        this.startTime = Date.now();
+    }
+    /**
+     * Submit a batch for processing
+     * @returns Promise that resolves when batch is processed
+     */
+    async submit(batch, handler) {
+        if (this.isShuttingDown) {
+            return Promise.reject(new Error("WorkerPool is shutting down"));
+        }
+        return new Promise((resolve, reject) => {
+            const queuedBatch = {
+                batch,
+                handler,
+                retryCount: 0,
+                resolve,
+                reject,
+            };
+            this.queue.push(queuedBatch);
+            this.notifyQueueChange();
+            this.tryProcessNext();
+        });
+    }
+    /**
+     * Get current queue depth
+     */
+    getQueueDepth() {
+        return this.queue.length;
+    }
+    /**
+     * Get number of active workers
+     */
+    getActiveWorkers() {
+        return this.activeWorkers;
+    }
+    /**
+     * Check if pool is at capacity
+     */
+    isAtCapacity() {
+        return this.activeWorkers >= this.config.concurrency;
+    }
+    /**
+     * Get pool statistics
+     */
+    getStats() {
+        const uptimeMs = Date.now() - this.startTime;
+        return {
+            processed: this.totalProcessed,
+            errors: this.totalErrors,
+            avgTimeMs: this.totalProcessed > 0 ? this.totalTimeMs / this.totalProcessed : 0,
+            queueDepth: this.queue.length,
+            activeWorkers: this.activeWorkers,
+            throughput: uptimeMs > 0 ? (this.totalProcessed / uptimeMs) * 1000 : 0,
+        };
+    }
+    /**
+     * Wait for all queued batches to complete
+     */
+    async drain() {
+        while (this.queue.length > 0 || this.activeWorkers > 0) {
+            await new Promise((resolve) => setTimeout(resolve, 50));
+        }
+    }
+    /**
+     * Gracefully shutdown the pool
+     * Waits for in-progress work to complete
+     */
+    async shutdown() {
+        this.isShuttingDown = true;
+        await this.drain();
+        if (isDebug()) {
+            const stats = this.getStats();
+            console.error(`[WorkerPool] Shutdown complete: processed=${stats.processed}, errors=${stats.errors}, ` +
+                `avgTime=${stats.avgTimeMs.toFixed(1)}ms`);
+        }
+    }
+    /**
+     * Force immediate shutdown (cancels pending work)
+     */
+    forceShutdown() {
+        this.isShuttingDown = true;
+        // Resolve all pending batches with cancelled status (don't reject to avoid unhandled rejections)
+        for (const queuedBatch of this.queue) {
+            const result = {
+                batchId: queuedBatch.batch.id,
+                type: queuedBatch.batch.type,
+                success: false,
+                itemCount: queuedBatch.batch.items.length,
+                durationMs: 0,
+                error: "WorkerPool force shutdown",
+            };
+            this.totalErrors++;
+            this.onCompletion?.(result);
+            queuedBatch.resolve(result);
+        }
+        this.queue = [];
+        this.notifyQueueChange();
+    }
+    tryProcessNext() {
+        if (this.isShuttingDown) {
+            return;
+        }
+        if (this.activeWorkers >= this.config.concurrency) {
+            return; // At capacity
+        }
+        const queuedBatch = this.queue.shift();
+        if (!queuedBatch) {
+            return; // Queue empty
+        }
+        this.notifyQueueChange();
+        this.activeWorkers++;
+        void this.processWithRetry(queuedBatch);
+    }
+    async processWithRetry(queuedBatch) {
+        const { batch, handler, resolve } = queuedBatch;
+        const startTime = Date.now();
+        try {
+            await handler(batch);
+            const durationMs = Date.now() - startTime;
+            this.totalProcessed++;
+            this.totalTimeMs += durationMs;
+            const result = {
+                batchId: batch.id,
+                type: batch.type,
+                success: true,
+                itemCount: batch.items.length,
+                durationMs,
+                retryCount: queuedBatch.retryCount,
+            };
+            if (isDebug()) {
+                console.error(`[WorkerPool] Batch ${batch.id} completed in ${durationMs}ms ` +
+                    `(${batch.items.length} items, retry=${queuedBatch.retryCount})`);
+            }
+            this.onCompletion?.(result);
+            resolve(result);
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            if (queuedBatch.retryCount < this.config.maxRetries) {
+                // Retry with exponential backoff
+                queuedBatch.retryCount++;
+                const delay = this.calculateBackoff(queuedBatch.retryCount);
+                if (isDebug()) {
+                    console.error(`[WorkerPool] Batch ${batch.id} failed (attempt ${queuedBatch.retryCount}/${this.config.maxRetries}), ` +
+                        `retrying in ${delay}ms: ${errorMessage}`);
+                }
+                setTimeout(() => {
+                    this.queue.unshift(queuedBatch); // Add to front of queue
+                    this.notifyQueueChange();
+                    this.tryProcessNext();
+                }, delay);
+            }
+            else {
+                // Max retries exceeded
+                this.totalErrors++;
+                const durationMs = Date.now() - startTime;
+                const result = {
+                    batchId: batch.id,
+                    type: batch.type,
+                    success: false,
+                    itemCount: batch.items.length,
+                    durationMs,
+                    error: errorMessage,
+                    retryCount: queuedBatch.retryCount,
+                };
+                console.error(`[WorkerPool] Batch ${batch.id} failed after ${queuedBatch.retryCount} retries: ${errorMessage}`);
+                this.onCompletion?.(result);
+                resolve(result); // Resolve with failure result (don't reject)
+            }
+        }
+        finally {
+            this.activeWorkers--;
+            this.tryProcessNext();
+        }
+    }
+    calculateBackoff(retryCount) {
+        // Exponential backoff with jitter
+        const baseDelay = this.config.retryBaseDelayMs * Math.pow(2, retryCount - 1);
+        const jitter = Math.random() * 0.3 * baseDelay; // 0-30% jitter
+        return Math.min(baseDelay + jitter, this.config.retryMaxDelayMs);
+    }
+    notifyQueueChange() {
+        this.onQueueChange?.(this.queue.length);
+    }
+}
+//# sourceMappingURL=worker-pool.js.map

package/build/core/domains/ingest/pipeline/infra/worker-pool.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"worker-pool.js","sourceRoot":"","sources":["../../../../../../src/core/domains/ingest/pipeline/infra/worker-pool.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAUH,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAUvC,MAAM,OAAO,UAAU;IACJ,MAAM,CAAmB;IACzB,YAAY,CAA2B;IACvC,aAAa,CAA+B;IAE7D,qHAAqH;IAC7G,KAAK,GAAuB,EAAE,CAAC;IAC/B,aAAa,GAAG,CAAC,CAAC;IAClB,cAAc,GAAG,KAAK,CAAC;IACvB,cAAc,GAAG,CAAC,CAAC;IACnB,WAAW,GAAG,CAAC,CAAC;IAChB,WAAW,GAAG,CAAC,CAAC;IACP,SAAS,CAAS;IAEnC,YACE,MAAwB,EACxB,YAAsC,EACtC,aAA2C;QAE3C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC9B,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,MAAM,CAAqB,KAAe,EAAE,OAAwB;QACxE,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC,CAAC;QAClE,CAAC;QAED,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,WAAW,GAAmB;gBAClC,KAAK;gBACL,OAAO;gBACP,UAAU,EAAE,CAAC;gBACb,OAAO;gBACP,MAAM;aACP,CAAC;YAEF,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAC7B,IAAI,CAAC,iBAAiB,EAAE,CAAC;YACzB,IAAI,CAAC,cAAc,EAAE,CAAC;QACxB,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,aAAa;QACX,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,gBAAgB;QACd,OAAO,IAAI,CAAC,aAAa,CAAC;IAC5B,CAAC;IAED;;OAEG;IACH,YAAY;QACV,OAAO,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC;IACvD,CAAC;IAED;;OAEG;IACH,QAAQ;QAQN,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC;QAC7C,OAAO;YACL,SAAS,EAAE,IAAI,CAAC,cAAc;YAC9B,MAAM,EAAE,IAAI,CAAC,WAAW;YACxB,SAAS,EAAE,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;YAC/E,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM;YAC7B,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,UAAU,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,cAAc,GAAG,QAAQ,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;SACvE,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,aAAa,GAAG,CAAC,EAAE,CAAC;YACvD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ;QACZ,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAC3B,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QAEnB,IAAI,OAAO,EAAE,EAAE,CAAC;YACd,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC9B,OAAO,CAAC,KAAK,CACX,6CAA6C,KAAK,CAAC,SAAS,YAAY,KAAK,CAAC,MAAM,IAAI;gBACtF,WAAW,KAAK,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAC5C,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;OAEG;IACH,aAAa;QACX,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAE3B,iGAAiG;QACjG,KAAK,MAAM,WAAW,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACrC,MAAM,MAAM,GAAgB;gBAC1B,OAAO,EAAE,WAAW,CAAC,KAAK,CAAC,EAAE;gBAC7B,IAAI,EAAE,WAAW,CAAC,KAAK,CAAC,IAAI;gBAC5B,OAAO,EAAE,KAAK;gBACd,SAAS,EAAE,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM;gBACzC,UAAU,EAAE,CAAC;gBACb,KAAK,EAAE,2BAA2B;aACnC,CAAC;YACF,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,IAAI,CAAC,YAAY,EAAE,CAAC,MAAM,CAAC,CAAC;YAC5B,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC9B,CAAC;QACD,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QAChB,IAAI,CAAC,iBAAiB,EAAE,CAAC;IAC3B,CAAC;IAEO,cAAc;QACpB,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,OAAO;QACT,CAAC;QAED,IAAI,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;YAClD,OAAO,CAAC,cAAc;QACxB,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACvC,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,OAAO,CAAC,cAAc;QACxB,CAAC;QAED,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACzB,IAAI,CAAC,aAAa,EAAE,CAAC;QACrB,KAAK,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC;IAC1C,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAAqB,WAA2B;QAC5E,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,WAAW,CAAC;QAChD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE7B,IAAI,CAAC;YACH,MAAM,OAAO,CAAC,KAAK,CAAC,CAAC;YAErB,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YAC1C,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,IAAI,CAAC,WAAW,IAAI,UAAU,CAAC;YAE/B,MAAM,MAAM,GAAgB;gBAC1B,OAAO,EAAE,KAAK,CAAC,EAAE;gBACjB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,OAAO,EAAE,IAAI;gBACb,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM;gBAC7B,UAAU;gBACV,UAAU,EAAE,WAAW,CAAC,UAAU;aACnC,CAAC;YAEF,IAAI,OAAO,EAAE,EAAE,CAAC;gBACd,OAAO,CAAC,KAAK,CACX,sBAAsB,KAAK,CAAC,EAAE,iBAAiB,UAAU,KAAK;oBAC5D,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,iBAAiB,WAAW,CAAC,UAAU,GAAG,CACnE,CAAC;YACJ,CAAC;YAED,IAAI,CAAC,YAAY,EAAE,CAAC,MAAM,CAAC,CAAC;YAC5B,OAAO,CAAC,MAAM,CAAC,CAAC;QAClB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAE5E,IAAI,WAAW,CAAC,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;gBACpD,iCAAiC;gBACjC,WAAW,CAAC,UAAU,EAAE,CAAC;gBACzB,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;gBAE5D,IAAI,OAAO,EAAE,EAAE,CAAC;oBACd,OAAO,CAAC,KAAK,CACX,sBAAsB,KAAK,CAAC,EAAE,oBAAoB,WAAW,CAAC,UAAU,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,KAAK;wBACrG,eAAe,KAAK,OAAO,YAAY,EAAE,CAC5C,CAAC;gBACJ,CAAC;gBAED,UAAU,CAAC,GAAG,EAAE;oBACd,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,wBAAwB;oBACzD,IAAI,CAAC,iBAAiB,EAAE,CAAC;oBACzB,IAAI,CAAC,cAAc,EAAE,CAAC;gBACxB,CAAC,EAAE,KAAK,CAAC,CAAC;YACZ,CAAC;iBAAM,CAAC;gBACN,uBAAuB;gBACvB,IAAI,CAAC,WAAW,EAAE,CAAC;gBAEnB,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;gBAC1C,MAAM,MAAM,GAAgB;oBAC1B,OAAO,EAAE,KAAK,CAAC,EAAE;oBACjB,IAAI,EAAE,KAAK,CAAC,IAAI;oBAChB,OAAO,EAAE,KAAK;oBACd,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM;oBAC7B,UAAU;oBACV,KAAK,EAAE,YAAY;oBACnB,UAAU,EAAE,WAAW,CAAC,UAAU;iBACnC,CAAC;gBAEF,OAAO,CAAC,KAAK,CAAC,sBAAsB,KAAK,CAAC,EAAE,iBAAiB,WAAW,CAAC,UAAU,aAAa,YAAY,EAAE,CAAC,CAAC;gBAEhH,IAAI,CAAC,YAAY,EAAE,CAAC,MAAM,CAAC,CAAC;gBAC5B,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,6CAA6C;YAChE,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,aAAa,EAAE,CAAC;YACrB,IAAI,CAAC,cAAc,EAAE,CAAC;QACxB,CAAC;IACH,CAAC;IAEO,gBAAgB,CAAC,UAAkB;QACzC,kCAAkC;QAClC,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,GAAG,CAAC,CAAC,CAAC;QAC7E,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,GAAG,SAAS,CAAC,CAAC,eAAe;QAC/D,OAAO,IAAI,CAAC,GAAG,CAAC,SAAS,GAAG,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IACnE,CAAC;IAEO,iBAAiB;QACvB,IAAI,CAAC,aAAa,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC1C,CAAC;CACF"}

package/build/core/domains/ingest/pipeline/pipeline-manager.d.ts

@@ -0,0 +1,108 @@
+/**
+ * PipelineManager - Main coordinator for batched operations
+ *
+ * Orchestrates:
+ * - BatchAccumulators for upsert and delete operations
+ * - WorkerPool for bounded concurrent execution
+ * - Backpressure management between producer and consumer
+ *
+ * Usage:
+ *   const pipeline = new PipelineManager(config, handlers);
+ *   await pipeline.start();
+ *
+ *   // Add items (returns false if backpressure active)
+ *   pipeline.addUpsert(point);
+ *   pipeline.addDelete(path);
+ *
+ *   // Wait for all work to complete
+ *   await pipeline.flush();
+ *   await pipeline.shutdown();
+ */
+import type { QdrantManager } from "../../../adapters/qdrant/client.js";
+import type { Batch, DeleteItem, PipelineConfig, PipelineStats, UpsertItem } from "./types.js";
+/**
+ * Handlers for processing different operation types
+ */
+export interface PipelineHandlers {
+    /** Process a batch of upsert operations */
+    handleUpsertBatch: (batch: Batch<UpsertItem>) => Promise<void>;
+    /** Process a batch of delete operations */
+    handleDeleteBatch: (batch: Batch<DeleteItem>) => Promise<void>;
+}
+export declare class PipelineManager {
+    private readonly config;
+    private readonly handlers;
+    private readonly workerPool;
+    private readonly upsertAccumulator;
+    private readonly deleteAccumulator;
+    private isRunning;
+    private pendingBatches;
+    private readonly stats;
+    constructor(handlers: PipelineHandlers, config?: Partial<PipelineConfig>);
+    /**
+     * Start the pipeline
+     */
+    start(): void;
+    /**
+     * Add a point for upsert
+     * @returns true if accepted, false if backpressure active
+     */
+    addUpsert(point: UpsertItem["point"]): boolean;
+    /**
+     * Add multiple points for upsert
+     * @returns number of items accepted
+     */
+    addUpsertMany(points: UpsertItem["point"][]): number;
+    /**
+     * Add a path for deletion
+     * @returns true if accepted, false if backpressure active
+     */
+    addDelete(relativePath: string): boolean;
+    /**
+     * Add multiple paths for deletion
+     * @returns number of items accepted
+     */
+    addDeleteMany(paths: string[]): number;
+    /**
+     * Flush all pending items and wait for completion
+     */
+    flush(): Promise<void>;
+    /**
+     * Gracefully shutdown the pipeline
+     */
+    shutdown(): Promise<void>;
+    /**
+     * Force shutdown (cancel pending work)
+     */
+    forceShutdown(): void;
+    /**
+     * Get pipeline statistics
+     */
+    getStats(): PipelineStats;
+    /**
+     * Check if backpressure is active for upserts
+     */
+    isUpsertBackpressured(): boolean;
+    /**
+     * Check if backpressure is active for deletes
+     */
+    isDeleteBackpressured(): boolean;
+    /**
+     * Wait for backpressure to release
+     * @param timeout Maximum time to wait (ms)
+     * @returns true if released, false if timeout
+     */
+    waitForBackpressure(timeout?: number): Promise<boolean>;
+    private submitBatch;
+    private onBatchComplete;
+    private onQueueChange;
+    private isPromiseResolved;
+}
+/**
+ * Factory function to create a PipelineManager with Qdrant handlers
+ */
+export declare function createQdrantPipeline(qdrant: QdrantManager, collectionName: string, options?: {
+    config?: Partial<PipelineConfig>;
+    enableHybrid?: boolean;
+}): PipelineManager;
+//# sourceMappingURL=pipeline-manager.d.ts.map

package/build/core/domains/ingest/pipeline/pipeline-manager.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pipeline-manager.d.ts","sourceRoot":"","sources":["../../../../../src/core/domains/ingest/pipeline/pipeline-manager.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,oCAAoC,CAAC;AAIxE,OAAO,KAAK,EAAE,KAAK,EAAe,UAAU,EAAE,cAAc,EAAE,aAAa,EAAE,UAAU,EAAY,MAAM,YAAY,CAAC;AAEtH;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,2CAA2C;IAC3C,iBAAiB,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC,UAAU,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/D,2CAA2C;IAC3C,iBAAiB,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC,UAAU,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAChE;AAED,qBAAa,eAAe;IAC1B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiB;IACxC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAmB;IAE5C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAa;IACxC,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAA+B;IACjE,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAA+B;IAEjE,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,cAAc,CAA8B;IACpD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAKpB;gBAEU,QAAQ,EAAE,gBAAgB,EAAE,MAAM,CAAC,EAAE,OAAO,CAAC,cAAc,CAAC;IAkCxE;;OAEG;IACH,KAAK,IAAI,IAAI;IAkBb;;;OAGG;IACH,SAAS,CAAC,KAAK,EAAE,UAAU,CAAC,OAAO,CAAC,GAAG,OAAO;IAc9C;;;OAGG;IACH,aAAa,CAAC,MAAM,EAAE,UAAU,CAAC,OAAO,CAAC,EAAE,GAAG,MAAM;IAYpD;;;OAGG;IACH,SAAS,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO;IAcxC;;;OAGG;IACH,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM;IAYtC;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAQ5B;;OAEG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAmB/B;;OAEG;IACH,aAAa,IAAI,IAAI;IAQrB;;OAEG;IACH,QAAQ,IAAI,aAAa;IAezB;;OAEG;IACH,qBAAqB,IAAI,OAAO;IAIhC;;OAEG;IACH,qBAAqB,IAAI,OAAO;IAIhC;;;;OAIG;IACG,mBAAmB,CAAC,OAAO,SAAQ,GAAG,OAAO,CAAC,OAAO,CAAC;IAa5D,OAAO,CAAC,WAAW;IAUnB,OAAO,CAAC,eAAe;IASvB,OAAO,CAAC,aAAa;IAcrB,OAAO,CAAC,iBAAiB;CAM1B;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAClC,MAAM,EAAE,aAAa,EACrB,cAAc,EAAE,MAAM,EACtB,OAAO,CAAC,EAAE;IACR,MAAM,CAAC,EAAE,OAAO,CAAC,cAAc,CAAC,CAAC;IACjC,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB,GACA,eAAe,CAgCjB"}

package/build/core/domains/ingest/pipeline/pipeline-manager.js

@@ -0,0 +1,290 @@
+/**
+ * PipelineManager - Main coordinator for batched operations
+ *
+ * Orchestrates:
+ * - BatchAccumulators for upsert and delete operations
+ * - WorkerPool for bounded concurrent execution
+ * - Backpressure management between producer and consumer
+ *
+ * Usage:
+ *   const pipeline = new PipelineManager(config, handlers);
+ *   await pipeline.start();
+ *
+ *   // Add items (returns false if backpressure active)
+ *   pipeline.addUpsert(point);
+ *   pipeline.addDelete(path);
+ *
+ *   // Wait for all work to complete
+ *   await pipeline.flush();
+ *   await pipeline.shutdown();
+ */
+import { BatchAccumulator } from "./infra/batch-accumulator.js";
+import { isDebug } from "./infra/runtime.js";
+import { WorkerPool } from "./infra/worker-pool.js";
+export class PipelineManager {
+    config;
+    handlers;
+    workerPool;
+    upsertAccumulator;
+    deleteAccumulator;
+    isRunning = false;
+    pendingBatches = [];
+    stats = {
+        itemsProcessed: 0,
+        batchesProcessed: 0,
+        errors: 0,
+        startTime: 0,
+    };
+    constructor(handlers, config) {
+        const fallbackWorkerPool = { concurrency: 1, maxRetries: 3, retryBaseDelayMs: 100, retryMaxDelayMs: 5000 };
+        const fallbackUpsert = { batchSize: 1024, flushTimeoutMs: 2000, maxQueueSize: 2 };
+        const fallbackDelete = { batchSize: 500, flushTimeoutMs: 1000, maxQueueSize: 16 };
+        this.config = {
+            workerPool: { ...fallbackWorkerPool, ...config?.workerPool },
+            deleteWorkerPool: { ...fallbackWorkerPool, concurrency: 8, ...config?.deleteWorkerPool },
+            upsertAccumulator: { ...fallbackUpsert, ...config?.upsertAccumulator },
+            deleteAccumulator: { ...fallbackDelete, ...config?.deleteAccumulator },
+        };
+        this.handlers = handlers;
+        // Initialize worker pool
+        this.workerPool = new WorkerPool(this.config.workerPool, (result) => {
+            this.onBatchComplete(result);
+        }, (queueSize) => {
+            this.onQueueChange(queueSize);
+        });
+        // Initialize accumulators
+        this.upsertAccumulator = new BatchAccumulator(this.config.upsertAccumulator, "upsert", (batch) => {
+            this.submitBatch(batch, this.handlers.handleUpsertBatch);
+        });
+        this.deleteAccumulator = new BatchAccumulator(this.config.deleteAccumulator, "delete", (batch) => {
+            this.submitBatch(batch, this.handlers.handleDeleteBatch);
+        });
+    }
+    /**
+     * Start the pipeline
+     */
+    start() {
+        if (this.isRunning) {
+            return;
+        }
+        this.isRunning = true;
+        this.stats.startTime = Date.now();
+        if (isDebug()) {
+            console.error(`[PipelineManager] Started with config: ` +
+                `workers=${this.config.workerPool.concurrency}, ` +
+                `upsertBatch=${this.config.upsertAccumulator.batchSize}, ` +
+                `deleteBatch=${this.config.deleteAccumulator.batchSize}`);
+        }
+    }
+    /**
+     * Add a point for upsert
+     * @returns true if accepted, false if backpressure active
+     */
+    addUpsert(point) {
+        if (!this.isRunning) {
+            throw new Error("Pipeline not started");
+        }
+        const item = {
+            type: "upsert",
+            id: point.id,
+            point,
+        };
+        return this.upsertAccumulator.add(item);
+    }
+    /**
+     * Add multiple points for upsert
+     * @returns number of items accepted
+     */
+    addUpsertMany(points) {
+        let accepted = 0;
+        for (const point of points) {
+            if (this.addUpsert(point)) {
+                accepted++;
+            }
+            else {
+                break;
+            }
+        }
+        return accepted;
+    }
+    /**
+     * Add a path for deletion
+     * @returns true if accepted, false if backpressure active
+     */
+    addDelete(relativePath) {
+        if (!this.isRunning) {
+            throw new Error("Pipeline not started");
+        }
+        const item = {
+            type: "delete",
+            id: relativePath,
+            relativePath,
+        };
+        return this.deleteAccumulator.add(item);
+    }
+    /**
+     * Add multiple paths for deletion
+     * @returns number of items accepted
+     */
+    addDeleteMany(paths) {
+        let accepted = 0;
+        for (const path of paths) {
+            if (this.addDelete(path)) {
+                accepted++;
+            }
+            else {
+                break;
+            }
+        }
+        return accepted;
+    }
+    /**
+     * Flush all pending items and wait for completion
+     */
+    async flush() {
+        this.upsertAccumulator.drain();
+        this.deleteAccumulator.drain();
+        await this.workerPool.drain();
+        await Promise.all(this.pendingBatches);
+        this.pendingBatches = [];
+    }
+    /**
+     * Gracefully shutdown the pipeline
+     */
+    async shutdown() {
+        if (!this.isRunning) {
+            return;
+        }
+        await this.flush();
+        await this.workerPool.shutdown();
+        this.isRunning = false;
+        if (isDebug()) {
+            const stats = this.getStats();
+            console.error(`[PipelineManager] Shutdown: ${stats.itemsProcessed} items, ` +
+                `${stats.batchesProcessed} batches, ${stats.errors} errors ` +
+                `in ${(stats.uptimeMs / 1000).toFixed(1)}s (${stats.throughput.toFixed(1)} items/s)`);
+        }
+    }
+    /**
+     * Force shutdown (cancel pending work)
+     */
+    forceShutdown() {
+        this.isRunning = false;
+        this.upsertAccumulator.clear();
+        this.deleteAccumulator.clear();
+        this.workerPool.forceShutdown();
+        this.pendingBatches = [];
+    }
+    /**
+     * Get pipeline statistics
+     */
+    getStats() {
+        const poolStats = this.workerPool.getStats();
+        const uptimeMs = this.stats.startTime > 0 ? Date.now() - this.stats.startTime : 0;
+        return {
+            itemsProcessed: this.stats.itemsProcessed,
+            batchesProcessed: this.stats.batchesProcessed,
+            errors: this.stats.errors,
+            queueDepth: poolStats.queueDepth,
+            avgBatchTimeMs: poolStats.avgTimeMs,
+            throughput: uptimeMs > 0 ? (this.stats.itemsProcessed / uptimeMs) * 1000 : 0,
+            uptimeMs,
+        };
+    }
+    /**
+     * Check if backpressure is active for upserts
+     */
+    isUpsertBackpressured() {
+        return this.upsertAccumulator.isPausedState();
+    }
+    /**
+     * Check if backpressure is active for deletes
+     */
+    isDeleteBackpressured() {
+        return this.deleteAccumulator.isPausedState();
+    }
+    /**
+     * Wait for backpressure to release
+     * @param timeout Maximum time to wait (ms)
+     * @returns true if released, false if timeout
+     */
+    async waitForBackpressure(timeout = 30000) {
+        const startTime = Date.now();
+        while (this.isUpsertBackpressured() || this.isDeleteBackpressured()) {
+            if (Date.now() - startTime > timeout) {
+                return false;
+            }
+            await new Promise((resolve) => setTimeout(resolve, 50));
+        }
+        return true;
+    }
+    submitBatch(batch, handler) {
+        const promise = this.workerPool.submit(batch, handler);
+        this.pendingBatches.push(promise);
+        // Cleanup completed promises periodically
+        if (this.pendingBatches.length > 100) {
+            this.pendingBatches = this.pendingBatches.filter((p) => !this.isPromiseResolved(p));
+        }
+    }
+    onBatchComplete(result) {
+        this.stats.batchesProcessed++;
+        this.stats.itemsProcessed += result.itemCount;
+        if (!result.success) {
+            this.stats.errors++;
+        }
+    }
+    onQueueChange(queueSize) {
+        const maxQueue = this.config.upsertAccumulator.maxQueueSize;
+        if (queueSize >= maxQueue) {
+            // Apply backpressure
+            this.upsertAccumulator.pause();
+            this.deleteAccumulator.pause();
+        }
+        else if (queueSize < maxQueue * 0.5) {
+            // Release backpressure when queue is half empty
+            this.upsertAccumulator.resume();
+            this.deleteAccumulator.resume();
+        }
+    }
+    isPromiseResolved(promise) {
+        // Check if promise is resolved by racing with a known resolved promise
+        let resolved = false;
+        Promise.race([promise.then(() => (resolved = true)), Promise.resolve()]).catch(() => { });
+        return resolved;
+    }
+}
+/**
+ * Factory function to create a PipelineManager with Qdrant handlers
+ */
+export function createQdrantPipeline(qdrant, collectionName, options) {
+    const handlers = {
+        handleUpsertBatch: async (batch) => {
+            const points = batch.items.map((item) => ({
+                id: item.point.id,
+                vector: item.point.vector,
+                payload: item.point.payload,
+            }));
+            if (options?.enableHybrid) {
+                const hybridPoints = batch.items.map((item) => ({
+                    id: item.point.id,
+                    vector: item.point.vector,
+                    sparseVector: item.point.sparseVector || { indices: [], values: [] },
+                    payload: item.point.payload,
+                }));
+                await qdrant.addPointsWithSparse(collectionName, hybridPoints);
+            }
+            else {
+                await qdrant.addPointsOptimized(collectionName, points, {
+                    wait: false,
+                    ordering: "weak",
+                });
+            }
+        },
+        handleDeleteBatch: async (batch) => {
+            const paths = batch.items.map((item) => item.relativePath);
+            await qdrant.deletePointsByPaths(collectionName, paths);
+        },
+    };
+    return new PipelineManager(handlers, options?.config);
+}
+//# sourceMappingURL=pipeline-manager.js.map

package/build/core/domains/ingest/pipeline/pipeline-manager.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pipeline-manager.js","sourceRoot":"","sources":["../../../../../src/core/domains/ingest/pipeline/pipeline-manager.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAGH,OAAO,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAChE,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAapD,MAAM,OAAO,eAAe;IACT,MAAM,CAAiB;IACvB,QAAQ,CAAmB;IAE3B,UAAU,CAAa;IACvB,iBAAiB,CAA+B;IAChD,iBAAiB,CAA+B;IAEzD,SAAS,GAAG,KAAK,CAAC;IAClB,cAAc,GAA2B,EAAE,CAAC;IACnC,KAAK,GAAG;QACvB,cAAc,EAAE,CAAC;QACjB,gBAAgB,EAAE,CAAC;QACnB,MAAM,EAAE,CAAC;QACT,SAAS,EAAE,CAAC;KACb,CAAC;IAEF,YAAY,QAA0B,EAAE,MAAgC;QACtE,MAAM,kBAAkB,GAAG,EAAE,WAAW,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,gBAAgB,EAAE,GAAG,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC;QAC3G,MAAM,cAAc,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC;QAClF,MAAM,cAAc,GAAG,EAAE,SAAS,EAAE,GAAG,EAAE,cAAc,EAAE,IAAI,EAAE,YAAY,EAAE,EAAE,EAAE,CAAC;QAElF,IAAI,CAAC,MAAM,GAAG;YACZ,UAAU,EAAE,EAAE,GAAG,kBAAkB,EAAE,GAAG,MAAM,EAAE,UAAU,EAAE;YAC5D,gBAAgB,EAAE,EAAE,GAAG,kBAAkB,EAAE,WAAW,EAAE,CAAC,EAAE,GAAG,MAAM,EAAE,gBAAgB,EAAE;YACxF,iBAAiB,EAAE,EAAE,GAAG,cAAc,EAAE,GAAG,MAAM,EAAE,iBAAiB,EAAE;YACtE,iBAAiB,EAAE,EAAE,GAAG,cAAc,EAAE,GAAG,MAAM,EAAE,iBAAiB,EAAE;SACvE,CAAC;QACF,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QAEzB,yBAAyB;QACzB,IAAI,CAAC,UAAU,GAAG,IAAI,UAAU,CAC9B,IAAI,CAAC,MAAM,CAAC,UAAU,EACtB,CAAC,MAAM,EAAE,EAAE;YACT,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;QAC/B,CAAC,EACD,CAAC,SAAS,EAAE,EAAE;YACZ,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;QAChC,CAAC,CACF,CAAC;QAEF,0BAA0B;QAC1B,IAAI,CAAC,iBAAiB,GAAG,IAAI,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,iBAAiB,EAAE,QAAQ,EAAE,CAAC,KAAK,EAAE,EAAE;YAC/F,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC;QAC3D,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,iBAAiB,GAAG,IAAI,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,iBAAiB,EAAE,QAAQ,EAAE,CAAC,KAAK,EAAE,EAAE;YAC/F,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC;QAC3D,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK;QACH,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,OAAO;QACT,CAAC;QAED,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAElC,IAAI,OAAO,EAAE,EAAE,CAAC;YACd,OAAO,CAAC,KAAK,CACX,yCAAyC;gBACvC,WAAW,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,WAAW,IAAI;gBACjD,eAAe,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,SAAS,IAAI;gBAC1D,eAAe,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,SAAS,EAAE,CAC3D,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,SAAS,CAAC,KAA0B;QAClC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;QAC1C,CAAC;QAED,MAAM,IAAI,GAAe;YACvB,IAAI,EAAE,QAAQ;YACd,EAAE,EAAE,KAAK,CAAC,EAAE;YACZ,KAAK;SACN,CAAC;QAEF,OAAO,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC1C,CAAC;IAED;;;OAGG;IACH,aAAa,CAAC,MAA6B;QACzC,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC1B,QAAQ,EAAE,CAAC;YACb,CAAC;iBAAM,CAAC;gBACN,MAAM;YACR,CAAC;QACH,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;;OAGG;IACH,SAAS,CAAC,YAAoB;QAC5B,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;QAC1C,CAAC;QAED,MAAM,IAAI,GAAe;YACvB,IAAI,EAAE,QAAQ;YACd,EAAE,EAAE,YAAY;YAChB,YAAY;SACb,CAAC;QAEF,OAAO,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC1C,CAAC;IAED;;;OAGG;IACH,aAAa,CAAC,KAAe;QAC3B,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;gBACzB,QAAQ,EAAE,CAAC;YACb,CAAC;iBAAM,CAAC;gBACN,MAAM;YACR,CAAC;QACH,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAC;QAC/B,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAC;QAC/B,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;QAC9B,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QACvC,IAAI,CAAC,cAAc,GAAG,EAAE,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ;QACZ,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACpB,OAAO;QACT,CAAC;QAED,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACnB,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC;QACjC,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QAEvB,IAAI,OAAO,EAAE,EAAE,CAAC;YACd,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC9B,OAAO,CAAC,KAAK,CACX,+BAA+B,KAAK,CAAC,cAAc,UAAU;gBAC3D,GAAG,KAAK,CAAC,gBAAgB,aAAa,KAAK,CAAC,MAAM,UAAU;gBAC5D,MAAM,CAAC,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,CACvF,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;OAEG;IACH,aAAa;QACX,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACvB,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAC;QAC/B,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAC;QAC/B,IAAI,CAAC,UAAU,CAAC,aAAa,EAAE,CAAC;QAChC,IAAI,CAAC,cAAc,GAAG,EAAE,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,QAAQ;QACN,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC;QAC7C,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;QAElF,OAAO;YACL,cAAc,EAAE,IAAI,CAAC,KAAK,CAAC,cAAc;YACzC,gBAAgB,EAAE,IAAI,CAAC,KAAK,CAAC,gBAAgB;YAC7C,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM;YACzB,UAAU,EAAE,SAAS,CAAC,UAAU;YAChC,cAAc,EAAE,SAAS,CAAC,SAAS;YACnC,UAAU,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,QAAQ,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;YAC5E,QAAQ;SACT,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,qBAAqB;QACnB,OAAO,IAAI,CAAC,iBAAiB,CAAC,aAAa,EAAE,CAAC;IAChD,CAAC;IAED;;OAEG;IACH,qBAAqB;QACnB,OAAO,IAAI,CAAC,iBAAiB,CAAC,aAAa,EAAE,CAAC;IAChD,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,mBAAmB,CAAC,OAAO,GAAG,KAAK;QACvC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE7B,OAAO,IAAI,CAAC,qBAAqB,EAAE,IAAI,IAAI,CAAC,qBAAqB,EAAE,EAAE,CAAC;YACpE,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,GAAG,OAAO,EAAE,CAAC;gBACrC,OAAO,KAAK,CAAC;YACf,CAAC;YACD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;QAC1D,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,WAAW,CAAqB,KAAe,EAAE,OAA2C;QAClG,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QACvD,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAElC,0CAA0C;QAC1C,IAAI,IAAI,CAAC,cAAc,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;YACrC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC;QACtF,CAAC;IACH,CAAC;IAEO,eAAe,CAAC,MAAmB;QACzC,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC;QAC9B,IAAI,CAAC,KAAK,CAAC,cAAc,IAAI,MAAM,CAAC,SAAS,CAAC;QAE9C,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QACtB,CAAC;IACH,CAAC;IAEO,aAAa,CAAC,SAAiB;QACrC,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,YAAY,CAAC;QAE5D,IAAI,SAAS,IAAI,QAAQ,EAAE,CAAC;YAC1B,qBAAqB;YACrB,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAC;YAC/B,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAC;QACjC,CAAC;aAAM,IAAI,SAAS,GAAG,QAAQ,GAAG,GAAG,EAAE,CAAC;YACtC,gDAAgD;YAChD,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE,CAAC;YAChC,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE,CAAC;QAClC,CAAC;IACH,CAAC;IAEO,iBAAiB,CAAC,OAAyB;QACjD,uEAAuE;QACvE,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACzF,OAAO,QAAQ,CAAC;IAClB,CAAC;CACF;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAClC,MAAqB,EACrB,cAAsB,EACtB,OAGC;IAED,MAAM,QAAQ,GAAqB;QACjC,iBAAiB,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;YACjC,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;gBACxC,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE;gBACjB,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM;gBACzB,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO;aAC5B,CAAC,CAAC,CAAC;YAEJ,IAAI,OAAO,EAAE,YAAY,EAAE,CAAC;gBAC1B,MAAM,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;oBAC9C,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE;oBACjB,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM;oBACzB,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,YAAY,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE;oBACpE,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO;iBAC5B,CAAC,CAAC,CAAC;gBACJ,MAAM,MAAM,CAAC,mBAAmB,CAAC,cAAc,EAAE,YAAY,CAAC,CAAC;YACjE,CAAC;iBAAM,CAAC;gBACN,MAAM,MAAM,CAAC,kBAAkB,CAAC,cAAc,EAAE,MAAM,EAAE;oBACtD,IAAI,EAAE,KAAK;oBACX,QAAQ,EAAE,MAAM;iBACjB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,iBAAiB,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;YACjC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAC3D,MAAM,MAAM,CAAC,mBAAmB,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;QAC1D,CAAC;KACF,CAAC;IAEF,OAAO,IAAI,eAAe,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;AACxD,CAAC"}

package/build/core/domains/ingest/pipeline/scanner.d.ts

@@ -0,0 +1,41 @@
+/**
+ * FileScanner - Discovers code files in a directory while respecting ignore patterns
+ */
+import { type Ignore } from "ignore";
+import type { ScannerConfig } from "../../../types.js";
+export declare class FileScanner {
+    private readonly config;
+    private readonly ig;
+    private readonly supportedExts;
+    constructor(config: ScannerConfig);
+    /**
+     * Load ignore patterns from .gitignore, .dockerignore, .npmignore, and .contextignore
+     */
+    loadIgnorePatterns(rootPath: string): Promise<void>;
+    /**
+     * Scan directory recursively and return all code files
+     */
+    scanDirectory(rootPath: string): Promise<string[]>;
+    /**
+     * Check if a file should be ignored based on patterns
+     */
+    shouldIgnore(filePath: string, rootPath: string): boolean;
+    /**
+     * Get the configured ignore filter instance.
+     * Used by enrichment module to filter git log results.
+     */
+    getIgnoreFilter(): Ignore;
+    /**
+     * Get list of supported file extensions
+     */
+    getSupportedExtensions(): string[];
+    /**
+     * Recursively walk directory tree
+     */
+    private walkDirectory;
+    /**
+     * Check if a file exists
+     */
+    private fileExists;
+}
+//# sourceMappingURL=scanner.d.ts.map