tea-rags 0.9.0

package/build/core/ingest/pipeline/base.d.ts

@@ -0,0 +1,56 @@
+/**
+ * BaseIndexingPipeline - Template Method base for indexing pipelines.
+ *
+ * Provides shared infrastructure: scanner, chunker pool, chunk pipeline,
+ * enrichment hooks, flush/shutdown, and enrichment completion.
+ * Subclasses compose these building blocks in their own orchestration flow.
+ */
+import type { EmbeddingProvider } from "../../adapters/embeddings/base.js";
+import type { QdrantManager } from "../../adapters/qdrant/client.js";
+import type { ChunkLookupEntry, IngestCodeConfig } from "../../types.js";
+import type { IngestDependencies } from "../factory.js";
+import { ChunkerPool } from "./chunker/infra/pool.js";
+import type { EnrichmentCoordinator } from "./enrichment/coordinator.js";
+import { ChunkPipeline } from "./index.js";
+import { FileScanner } from "./scanner.js";
+import type { PipelineConfig } from "./types.js";
+export interface ProcessingContext {
+    chunkerPool: ChunkerPool;
+    chunkPipeline: ChunkPipeline;
+}
+export interface PipelineTuning {
+    pipelineConfig: PipelineConfig;
+    chunkerPoolSize: number;
+    fileConcurrency: number;
+}
+export declare abstract class BaseIndexingPipeline {
+    protected readonly qdrant: QdrantManager;
+    protected readonly embeddings: EmbeddingProvider;
+    protected readonly config: IngestCodeConfig;
+    protected readonly enrichment: EnrichmentCoordinator;
+    protected readonly deps: IngestDependencies;
+    protected readonly tuning: PipelineTuning;
+    constructor(qdrant: QdrantManager, embeddings: EmbeddingProvider, config: IngestCodeConfig, enrichment: EnrichmentCoordinator, deps: IngestDependencies, tuning?: PipelineTuning);
+    protected get snapshotDir(): string;
+    protected resolveContext(path: string): Promise<{
+        absolutePath: string;
+        collectionName: string;
+    }>;
+    protected createScanner(overrides?: {
+        extensions?: string[];
+        customIgnorePatterns?: string[];
+    }): FileScanner;
+    protected scanFiles(absolutePath: string, scanner: FileScanner): Promise<string[]>;
+    protected initProcessing(collectionName: string, absolutePath: string, scanner: FileScanner): ProcessingContext;
+    protected finalizeProcessing(ctx: ProcessingContext, chunkMap: Map<string, ChunkLookupEntry[]>, collectionName: string, absolutePath: string): Promise<() => "completed" | "background" | "skipped">;
+    private createChunkerPool;
+    private createChunkPipeline;
+    private setupEnrichmentHooks;
+    private flushAndShutdown;
+    /**
+     * Starts background enrichment and returns a status getter.
+     * Call the returned function after snapshot/finalization to get current status.
+     */
+    private startEnrichment;
+}
+//# sourceMappingURL=base.d.ts.map

package/build/core/ingest/pipeline/base.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"base.d.ts","sourceRoot":"","sources":["../../../../src/core/ingest/pipeline/base.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAKH,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,mCAAmC,CAAC;AAC3E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iCAAiC,CAAC;AACrE,OAAO,KAAK,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAEzE,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACxD,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC;AACzE,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAE3C,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAEjD,MAAM,WAAW,iBAAiB;IAChC,WAAW,EAAE,WAAW,CAAC;IACzB,aAAa,EAAE,aAAa,CAAC;CAC9B;AAED,MAAM,WAAW,cAAc;IAC7B,cAAc,EAAE,cAAc,CAAC;IAC/B,eAAe,EAAE,MAAM,CAAC;IACxB,eAAe,EAAE,MAAM,CAAC;CACzB;AAcD,8BAAsB,oBAAoB;IAItC,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,aAAa;IACxC,SAAS,CAAC,QAAQ,CAAC,UAAU,EAAE,iBAAiB;IAChD,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,gBAAgB;IAC3C,SAAS,CAAC,QAAQ,CAAC,UAAU,EAAE,qBAAqB;IACpD,SAAS,CAAC,QAAQ,CAAC,IAAI,EAAE,kBAAkB;IAP7C,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;gBAGrB,MAAM,EAAE,aAAa,EACrB,UAAU,EAAE,iBAAiB,EAC7B,MAAM,EAAE,gBAAgB,EACxB,UAAU,EAAE,qBAAqB,EACjC,IAAI,EAAE,kBAAkB,EAC3C,MAAM,CAAC,EAAE,cAAc;IAOzB,SAAS,KAAK,WAAW,IAAI,MAAM,CAElC;cAEe,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC;QACpD,YAAY,EAAE,MAAM,CAAC;QACrB,cAAc,EAAE,MAAM,CAAC;KACxB,CAAC;IAQF,SAAS,CAAC,aAAa,CAAC,SAAS,CAAC,EAAE;QAAE,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;QAAC,oBAAoB,CAAC,EAAE,MAAM,EAAE,CAAA;KAAE,GAAG,WAAW;cAQ5F,SAAS,CAAC,YAAY,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAWxF,SAAS,CAAC,cAAc,CAAC,cAAc,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,GAAG,iBAAiB;cAQ/F,kBAAkB,CAChC,GAAG,EAAE,iBAAiB,EACtB,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,gBAAgB,EAAE,CAAC,EACzC,cAAc,EAAE,MAAM,EACtB,YAAY,EAAE,MAAM,GACnB,OAAO,CAAC,MAAM,WAAW,GAAG,YAAY,GAAG,SAAS,CAAC;IAOxD,OAAO,CAAC,iBAAiB;IAQzB,OAAO,CAAC,mBAAmB;IAQ3B,OAAO,CAAC,oBAAoB;YAcd,gBAAgB;IAK9B;;;OAGG;IACH,OAAO,CAAC,eAAe;CAoBxB"}

package/build/core/ingest/pipeline/base.js

@@ -0,0 +1,123 @@
+/**
+ * BaseIndexingPipeline - Template Method base for indexing pipelines.
+ *
+ * Provides shared infrastructure: scanner, chunker pool, chunk pipeline,
+ * enrichment hooks, flush/shutdown, and enrichment completion.
+ * Subclasses compose these building blocks in their own orchestration flow.
+ */
+import { snapshotsDir } from "../../../bootstrap/config/paths.js";
+import { resolveCollectionName, validatePath } from "../collection.js";
+import { ChunkerPool } from "./chunker/infra/pool.js";
+import { ChunkPipeline } from "./index.js";
+import { pipelineLog } from "./infra/debug-logger.js";
+import { FileScanner } from "./scanner.js";
+/** Fallback tuning when no config is injected (tests, legacy callers) */
+const DEFAULT_TUNING = {
+    pipelineConfig: {
+        workerPool: { concurrency: 1, maxRetries: 3, retryBaseDelayMs: 100, retryMaxDelayMs: 5000 },
+        deleteWorkerPool: { concurrency: 8, maxRetries: 3, retryBaseDelayMs: 100, retryMaxDelayMs: 5000 },
+        upsertAccumulator: { batchSize: 1024, flushTimeoutMs: 2000, maxQueueSize: 2 },
+        deleteAccumulator: { batchSize: 500, flushTimeoutMs: 1000, maxQueueSize: 16 },
+    },
+    chunkerPoolSize: 4,
+    fileConcurrency: 50,
+};
+export class BaseIndexingPipeline {
+    qdrant;
+    embeddings;
+    config;
+    enrichment;
+    deps;
+    tuning;
+    constructor(qdrant, embeddings, config, enrichment, deps, tuning) {
+        this.qdrant = qdrant;
+        this.embeddings = embeddings;
+        this.config = config;
+        this.enrichment = enrichment;
+        this.deps = deps;
+        this.tuning = tuning ?? DEFAULT_TUNING;
+    }
+    // ── Shared context ─────────────────────────────────────────
+    get snapshotDir() {
+        return snapshotsDir();
+    }
+    async resolveContext(path) {
+        const absolutePath = await validatePath(path);
+        const collectionName = resolveCollectionName(absolutePath);
+        return { absolutePath, collectionName };
+    }
+    // ── Scanner ──────────────────────────────────────────────
+    createScanner(overrides) {
+        return new FileScanner({
+            supportedExtensions: overrides?.extensions || this.config.supportedExtensions,
+            ignorePatterns: this.config.ignorePatterns,
+            customIgnorePatterns: overrides?.customIgnorePatterns || this.config.customIgnorePatterns,
+        });
+    }
+    async scanFiles(absolutePath, scanner) {
+        await scanner.loadIgnorePatterns(absolutePath);
+        pipelineLog.resetProfiler();
+        pipelineLog.stageStart("scan");
+        const files = await scanner.scanDirectory(absolutePath);
+        pipelineLog.stageEnd("scan");
+        return files;
+    }
+    // ── Processing lifecycle ─────────────────────────────────
+    initProcessing(collectionName, absolutePath, scanner) {
+        const chunkerPool = this.createChunkerPool();
+        const chunkPipeline = this.createChunkPipeline(collectionName);
+        this.setupEnrichmentHooks(chunkPipeline, absolutePath, collectionName, scanner.getIgnoreFilter());
+        chunkPipeline.start();
+        return { chunkerPool, chunkPipeline };
+    }
+    async finalizeProcessing(ctx, chunkMap, collectionName, absolutePath) {
+        await this.flushAndShutdown(ctx.chunkPipeline, ctx.chunkerPool);
+        return this.startEnrichment(chunkMap, collectionName, absolutePath);
+    }
+    // ── Processing components (private) ────────────────────
+    createChunkerPool() {
+        return new ChunkerPool(this.tuning.chunkerPoolSize, {
+            chunkSize: this.config.chunkSize,
+            chunkOverlap: this.config.chunkOverlap,
+            maxChunkSize: this.config.chunkSize * 2,
+        });
+    }
+    createChunkPipeline(collectionName) {
+        return new ChunkPipeline(this.qdrant, this.embeddings, collectionName, this.deps.payloadBuilder, {
+            workerPool: this.tuning.pipelineConfig.workerPool,
+            accumulator: this.tuning.pipelineConfig.upsertAccumulator,
+            enableHybrid: this.config.enableHybridSearch,
+        });
+    }
+    setupEnrichmentHooks(chunkPipeline, absolutePath, collectionName, ignoreFilter) {
+        this.enrichment.prefetch(absolutePath, collectionName, ignoreFilter);
+        chunkPipeline.setOnBatchUpserted((items) => {
+            this.enrichment.onChunksStored(collectionName, absolutePath, items);
+        });
+    }
+    // ── Teardown ─────────────────────────────────────────────
+    async flushAndShutdown(chunkPipeline, chunkerPool) {
+        await chunkPipeline.flush();
+        await Promise.all([chunkPipeline.shutdown(), chunkerPool.shutdown()]);
+    }
+    /**
+     * Starts background enrichment and returns a status getter.
+     * Call the returned function after snapshot/finalization to get current status.
+     */
+    startEnrichment(chunkMap, collectionName, absolutePath) {
+        if (chunkMap.size === 0)
+            return () => "skipped";
+        let done = false;
+        this.enrichment.startChunkEnrichment(collectionName, absolutePath, chunkMap);
+        this.enrichment
+            .awaitCompletion(collectionName)
+            .then(() => {
+            done = true;
+        })
+            .catch((error) => {
+            console.error("[Pipeline] Background enrichment failed:", error);
+        });
+        return () => (done ? "completed" : "background");
+    }
+}
+//# sourceMappingURL=base.js.map

package/build/core/ingest/pipeline/base.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"base.js","sourceRoot":"","sources":["../../../../src/core/ingest/pipeline/base.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAIH,OAAO,EAAE,YAAY,EAAE,MAAM,oCAAoC,CAAC;AAIlE,OAAO,EAAE,qBAAqB,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAEvE,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAEtD,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAc3C,yEAAyE;AACzE,MAAM,cAAc,GAAmB;IACrC,cAAc,EAAE;QACd,UAAU,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,gBAAgB,EAAE,GAAG,EAAE,eAAe,EAAE,IAAI,EAAE;QAC3F,gBAAgB,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,gBAAgB,EAAE,GAAG,EAAE,eAAe,EAAE,IAAI,EAAE;QACjG,iBAAiB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC,EAAE;QAC7E,iBAAiB,EAAE,EAAE,SAAS,EAAE,GAAG,EAAE,cAAc,EAAE,IAAI,EAAE,YAAY,EAAE,EAAE,EAAE;KAC9E;IACD,eAAe,EAAE,CAAC;IAClB,eAAe,EAAE,EAAE;CACpB,CAAC;AAEF,MAAM,OAAgB,oBAAoB;IAInB;IACA;IACA;IACA;IACA;IAPF,MAAM,CAAiB;IAE1C,YACqB,MAAqB,EACrB,UAA6B,EAC7B,MAAwB,EACxB,UAAiC,EACjC,IAAwB,EAC3C,MAAuB;QALJ,WAAM,GAAN,MAAM,CAAe;QACrB,eAAU,GAAV,UAAU,CAAmB;QAC7B,WAAM,GAAN,MAAM,CAAkB;QACxB,eAAU,GAAV,UAAU,CAAuB;QACjC,SAAI,GAAJ,IAAI,CAAoB;QAG3C,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,cAAc,CAAC;IACzC,CAAC;IAED,8DAA8D;IAE9D,IAAc,WAAW;QACvB,OAAO,YAAY,EAAE,CAAC;IACxB,CAAC;IAES,KAAK,CAAC,cAAc,CAAC,IAAY;QAIzC,MAAM,YAAY,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,CAAC;QAC9C,MAAM,cAAc,GAAG,qBAAqB,CAAC,YAAY,CAAC,CAAC;QAC3D,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,CAAC;IAC1C,CAAC;IAED,4DAA4D;IAElD,aAAa,CAAC,SAAsE;QAC5F,OAAO,IAAI,WAAW,CAAC;YACrB,mBAAmB,EAAE,SAAS,EAAE,UAAU,IAAI,IAAI,CAAC,MAAM,CAAC,mBAAmB;YAC7E,cAAc,EAAE,IAAI,CAAC,MAAM,CAAC,cAAc;YAC1C,oBAAoB,EAAE,SAAS,EAAE,oBAAoB,IAAI,IAAI,CAAC,MAAM,CAAC,oBAAoB;SAC1F,CAAC,CAAC;IACL,CAAC;IAES,KAAK,CAAC,SAAS,CAAC,YAAoB,EAAE,OAAoB;QAClE,MAAM,OAAO,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAC;QAC/C,WAAW,CAAC,aAAa,EAAE,CAAC;QAC5B,WAAW,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QAC/B,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,aAAa,CAAC,YAAY,CAAC,CAAC;QACxD,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC7B,OAAO,KAAK,CAAC;IACf,CAAC;IAED,4DAA4D;IAElD,cAAc,CAAC,cAAsB,EAAE,YAAoB,EAAE,OAAoB;QACzF,MAAM,WAAW,GAAG,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC7C,MAAM,aAAa,GAAG,IAAI,CAAC,mBAAmB,CAAC,cAAc,CAAC,CAAC;QAC/D,IAAI,CAAC,oBAAoB,CAAC,aAAa,EAAE,YAAY,EAAE,cAAc,EAAE,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC;QAClG,aAAa,CAAC,KAAK,EAAE,CAAC;QACtB,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,CAAC;IACxC,CAAC;IAES,KAAK,CAAC,kBAAkB,CAChC,GAAsB,EACtB,QAAyC,EACzC,cAAsB,EACtB,YAAoB;QAEpB,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,aAAa,EAAE,GAAG,CAAC,WAAW,CAAC,CAAC;QAChE,OAAO,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,cAAc,EAAE,YAAY,CAAC,CAAC;IACtE,CAAC;IAED,0DAA0D;IAElD,iBAAiB;QACvB,OAAO,IAAI,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE;YAClD,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS;YAChC,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,YAAY;YACtC,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,GAAG,CAAC;SACxC,CAAC,CAAC;IACL,CAAC;IAEO,mBAAmB,CAAC,cAAsB;QAChD,OAAO,IAAI,aAAa,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,cAAc,EAAE,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE;YAC/F,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,UAAU;YACjD,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,iBAAiB;YACzD,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,kBAAkB;SAC7C,CAAC,CAAC;IACL,CAAC;IAEO,oBAAoB,CAC1B,aAA4B,EAC5B,YAAoB,EACpB,cAAsB,EACtB,YAAoB;QAEpB,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,YAAY,EAAE,cAAc,EAAE,YAAY,CAAC,CAAC;QACrE,aAAa,CAAC,kBAAkB,CAAC,CAAC,KAAK,EAAE,EAAE;YACzC,IAAI,CAAC,UAAU,CAAC,cAAc,CAAC,cAAc,EAAE,YAAY,EAAE,KAAK,CAAC,CAAC;QACtE,CAAC,CAAC,CAAC;IACL,CAAC;IAED,4DAA4D;IAEpD,KAAK,CAAC,gBAAgB,CAAC,aAA4B,EAAE,WAAwB;QACnF,MAAM,aAAa,CAAC,KAAK,EAAE,CAAC;QAC5B,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,aAAa,CAAC,QAAQ,EAAE,EAAE,WAAW,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;IACxE,CAAC;IAED;;;OAGG;IACK,eAAe,CACrB,QAAyC,EACzC,cAAsB,EACtB,YAAoB;QAEpB,IAAI,QAAQ,CAAC,IAAI,KAAK,CAAC;YAAE,OAAO,GAAG,EAAE,CAAC,SAAS,CAAC;QAEhD,IAAI,IAAI,GAAG,KAAK,CAAC;QACjB,IAAI,CAAC,UAAU,CAAC,oBAAoB,CAAC,cAAc,EAAE,YAAY,EAAE,QAAQ,CAAC,CAAC;QAC7E,IAAI,CAAC,UAAU;aACZ,eAAe,CAAC,cAAc,CAAC;aAC/B,IAAI,CAAC,GAAG,EAAE;YACT,IAAI,GAAG,IAAI,CAAC;QACd,CAAC,CAAC;aACD,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;YACf,OAAO,CAAC,KAAK,CAAC,0CAA0C,EAAE,KAAK,CAAC,CAAC;QACnE,CAAC,CAAC,CAAC;QAEL,OAAO,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;IACnD,CAAC;CACF"}

package/build/core/ingest/pipeline/chunk-pipeline.d.ts

@@ -0,0 +1,102 @@
+/**
+ * ChunkPipeline - Specialized pipeline for chunk → embedding → Qdrant flow
+ *
+ * Solves the problem of uneven server load by:
+ * 1. Accumulating chunks from multiple file processing threads
+ * 2. Forming optimal batches (5s timeout for batch formation)
+ * 3. Dispatching to bounded worker pool for embedding + storage
+ *
+ * Architecture:
+ *   [File Thread 1] ─┐
+ *   [File Thread 2] ─┼→ ChunkPipeline → BatchAccumulator → WorkerPool → Ollama/Qdrant
+ *   [File Thread N] ─┘      (5s timeout)    (1s flush)     (bounded)
+ */
+import type { EmbeddingProvider } from "../../adapters/embeddings/base.js";
+import type { QdrantManager } from "../../adapters/qdrant/client.js";
+import type { PayloadBuilder } from "../../contracts/types/provider.js";
+import type { BatchAccumulatorConfig, ChunkItem, PipelineStats, WorkerPoolConfig } from "./types.js";
+export interface ChunkPipelineConfig {
+    /** Worker pool settings */
+    workerPool: WorkerPoolConfig;
+    /** Batch accumulator settings */
+    accumulator: BatchAccumulatorConfig;
+    /** Enable hybrid search (sparse vectors) */
+    enableHybrid: boolean;
+}
+export declare class ChunkPipeline {
+    private readonly config;
+    private readonly qdrant;
+    private readonly embeddings;
+    private readonly collectionName;
+    private readonly payloadBuilder;
+    private readonly sparseGenerator;
+    private readonly workerPool;
+    private readonly accumulator;
+    private pendingBatches;
+    private onBatchUpsertedCb?;
+    private isRunning;
+    private readonly stats;
+    constructor(qdrant: QdrantManager, embeddings: EmbeddingProvider, collectionName: string, payloadBuilder: PayloadBuilder, config?: Partial<ChunkPipelineConfig>);
+    /**
+     * Register a callback that fires after each successful batch upsert.
+     * Used by EnrichmentModule to stream git metadata as chunks are stored.
+     */
+    setOnBatchUpserted(cb: (items: ChunkItem[]) => void): void;
+    /**
+     * Start the pipeline
+     */
+    start(): void;
+    /**
+     * Add a chunk for processing
+     * @returns true if accepted, false if backpressure active
+     */
+    addChunk(chunk: ChunkItem["chunk"], chunkId: string, codebasePath: string): boolean;
+    /**
+     * Add multiple chunks for processing
+     * @returns number of items accepted
+     */
+    addChunks(chunks: {
+        chunk: ChunkItem["chunk"];
+        chunkId: string;
+        codebasePath: string;
+    }[]): number;
+    /**
+     * Check if backpressure is active
+     */
+    isBackpressured(): boolean;
+    /**
+     * Wait for backpressure to release
+     * @param timeout Maximum time to wait (ms)
+     * @returns true if released, false if timeout
+     */
+    waitForBackpressure(timeout?: number): Promise<boolean>;
+    /**
+     * Flush all pending chunks and wait for completion
+     */
+    flush(): Promise<void>;
+    /**
+     * Gracefully shutdown the pipeline
+     */
+    shutdown(): Promise<void>;
+    /**
+     * Force shutdown (cancel pending work)
+     */
+    forceShutdown(): void;
+    /**
+     * Get pipeline statistics
+     */
+    getStats(): PipelineStats;
+    /**
+     * Get pending count (chunks waiting to be batched)
+     */
+    getPendingCount(): number;
+    private submitBatch;
+    /**
+     * Create a batch handler that embeds chunks and stores to Qdrant
+     */
+    private createBatchHandler;
+    private onBatchComplete;
+    private onQueueChange;
+    private isPromiseResolved;
+}
+//# sourceMappingURL=chunk-pipeline.d.ts.map

package/build/core/ingest/pipeline/chunk-pipeline.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunk-pipeline.d.ts","sourceRoot":"","sources":["../../../../src/core/ingest/pipeline/chunk-pipeline.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,mCAAmC,CAAC;AAE3E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iCAAiC,CAAC;AACrE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mCAAmC,CAAC;AAKxE,OAAO,KAAK,EAEV,sBAAsB,EAEtB,SAAS,EACT,aAAa,EACb,gBAAgB,EACjB,MAAM,YAAY,CAAC;AAIpB,MAAM,WAAW,mBAAmB;IAClC,2BAA2B;IAC3B,UAAU,EAAE,gBAAgB,CAAC;IAC7B,iCAAiC;IACjC,WAAW,EAAE,sBAAsB,CAAC;IACpC,4CAA4C;IAC5C,YAAY,EAAE,OAAO,CAAC;CACvB;AAED,qBAAa,aAAa;IACxB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAsB;IAC7C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgB;IACvC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAoB;IAC/C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAiB;IAChD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAmC;IAEnE,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAa;IACxC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAA8B;IAC1D,OAAO,CAAC,cAAc,CAA8B;IAEpD,OAAO,CAAC,iBAAiB,CAAC,CAA+B;IACzD,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAKpB;gBAGA,MAAM,EAAE,aAAa,EACrB,UAAU,EAAE,iBAAiB,EAC7B,cAAc,EAAE,MAAM,EACtB,cAAc,EAAE,cAAc,EAC9B,MAAM,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC;IAyCvC;;;OAGG;IACH,kBAAkB,CAAC,EAAE,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,KAAK,IAAI,GAAG,IAAI;IAI1D;;OAEG;IACH,KAAK,IAAI,IAAI;IAwBb;;;OAGG;IACH,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO;IAgBnF;;;OAGG;IACH,SAAS,CACP,MAAM,EAAE;QACN,KAAK,EAAE,SAAS,CAAC,OAAO,CAAC,CAAC;QAC1B,OAAO,EAAE,MAAM,CAAC;QAChB,YAAY,EAAE,MAAM,CAAC;KACtB,EAAE,GACF,MAAM;IAYT;;OAEG;IACH,eAAe,IAAI,OAAO;IAI1B;;;;OAIG;IACG,mBAAmB,CAAC,OAAO,SAAQ,GAAG,OAAO,CAAC,OAAO,CAAC;IAa5D;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAO5B;;OAEG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IA6B/B;;OAEG;IACH,aAAa,IAAI,IAAI;IAOrB;;OAEG;IACH,QAAQ,IAAI,aAAa;IAezB;;OAEG;IACH,eAAe,IAAI,MAAM;IAIzB,OAAO,CAAC,WAAW;IAWnB;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAiD1B,OAAO,CAAC,eAAe;IA4BvB,OAAO,CAAC,aAAa;IAkBrB,OAAO,CAAC,iBAAiB;CAK1B"}

package/build/core/ingest/pipeline/chunk-pipeline.js

@@ -0,0 +1,313 @@
+/**
+ * ChunkPipeline - Specialized pipeline for chunk → embedding → Qdrant flow
+ *
+ * Solves the problem of uneven server load by:
+ * 1. Accumulating chunks from multiple file processing threads
+ * 2. Forming optimal batches (5s timeout for batch formation)
+ * 3. Dispatching to bounded worker pool for embedding + storage
+ *
+ * Architecture:
+ *   [File Thread 1] ─┐
+ *   [File Thread 2] ─┼→ ChunkPipeline → BatchAccumulator → WorkerPool → Ollama/Qdrant
+ *   [File Thread N] ─┘      (5s timeout)    (1s flush)     (bounded)
+ */
+import { BM25SparseVectorGenerator } from "../../adapters/qdrant/sparse.js";
+import { BatchAccumulator } from "./infra/batch-accumulator.js";
+import { pipelineLog } from "./infra/debug-logger.js";
+import { isDebug } from "./infra/runtime.js";
+import { WorkerPool } from "./infra/worker-pool.js";
+const LOG_CTX = { component: "ChunkPipeline" };
+export class ChunkPipeline {
+    config;
+    qdrant;
+    embeddings;
+    collectionName;
+    payloadBuilder;
+    sparseGenerator;
+    workerPool;
+    accumulator;
+    pendingBatches = [];
+    onBatchUpsertedCb;
+    isRunning = false;
+    stats = {
+        chunksProcessed: 0,
+        batchesProcessed: 0,
+        errors: 0,
+        startTime: 0,
+    };
+    constructor(qdrant, embeddings, collectionName, payloadBuilder, config) {
+        this.qdrant = qdrant;
+        this.embeddings = embeddings;
+        this.collectionName = collectionName;
+        this.payloadBuilder = payloadBuilder;
+        this.config = {
+            workerPool: config?.workerPool ?? {
+                concurrency: 1,
+                maxRetries: 3,
+                retryBaseDelayMs: 100,
+                retryMaxDelayMs: 5000,
+            },
+            accumulator: config?.accumulator ?? {
+                batchSize: 1024,
+                flushTimeoutMs: 2000,
+                maxQueueSize: 2,
+            },
+            enableHybrid: config?.enableHybrid ?? false,
+        };
+        this.sparseGenerator = this.config.enableHybrid ? new BM25SparseVectorGenerator() : null;
+        // Initialize worker pool
+        this.workerPool = new WorkerPool(this.config.workerPool, (result) => {
+            this.onBatchComplete(result);
+        }, (queueSize) => {
+            this.onQueueChange(queueSize);
+        });
+        // Initialize accumulator
+        this.accumulator = new BatchAccumulator(this.config.accumulator, "upsert", (batch) => {
+            this.submitBatch(batch);
+        });
+    }
+    /**
+     * Register a callback that fires after each successful batch upsert.
+     * Used by EnrichmentModule to stream git metadata as chunks are stored.
+     */
+    setOnBatchUpserted(cb) {
+        this.onBatchUpsertedCb = cb;
+    }
+    /**
+     * Start the pipeline
+     */
+    start() {
+        if (this.isRunning)
+            return;
+        this.isRunning = true;
+        this.stats.startTime = Date.now();
+        pipelineLog.step(LOG_CTX, "PIPELINE_START", {
+            workers: this.config.workerPool.concurrency,
+            batchSize: this.config.accumulator.batchSize,
+            flushTimeoutMs: this.config.accumulator.flushTimeoutMs,
+            hybrid: this.config.enableHybrid,
+            collection: this.collectionName,
+        });
+        if (isDebug()) {
+            console.error(`[ChunkPipeline] Started: ` +
+                `workers=${this.config.workerPool.concurrency}, ` +
+                `batchSize=${this.config.accumulator.batchSize}, ` +
+                `flushTimeout=${this.config.accumulator.flushTimeoutMs}ms`);
+        }
+    }
+    /**
+     * Add a chunk for processing
+     * @returns true if accepted, false if backpressure active
+     */
+    addChunk(chunk, chunkId, codebasePath) {
+        if (!this.isRunning) {
+            throw new Error("ChunkPipeline not started");
+        }
+        const item = {
+            type: "upsert",
+            id: chunkId,
+            chunk,
+            chunkId,
+            codebasePath,
+        };
+        return this.accumulator.add(item);
+    }
+    /**
+     * Add multiple chunks for processing
+     * @returns number of items accepted
+     */
+    addChunks(chunks) {
+        let accepted = 0;
+        for (const { chunk, chunkId, codebasePath } of chunks) {
+            if (this.addChunk(chunk, chunkId, codebasePath)) {
+                accepted++;
+            }
+            else {
+                break;
+            }
+        }
+        return accepted;
+    }
+    /**
+     * Check if backpressure is active
+     */
+    isBackpressured() {
+        return this.accumulator.isPausedState();
+    }
+    /**
+     * Wait for backpressure to release
+     * @param timeout Maximum time to wait (ms)
+     * @returns true if released, false if timeout
+     */
+    async waitForBackpressure(timeout = 30000) {
+        const startTime = Date.now();
+        while (this.isBackpressured()) {
+            if (Date.now() - startTime > timeout) {
+                return false;
+            }
+            await new Promise((resolve) => setTimeout(resolve, 50));
+        }
+        return true;
+    }
+    /**
+     * Flush all pending chunks and wait for completion
+     */
+    async flush() {
+        this.accumulator.drain();
+        await this.workerPool.drain();
+        await Promise.all(this.pendingBatches);
+        this.pendingBatches = [];
+    }
+    /**
+     * Gracefully shutdown the pipeline
+     */
+    async shutdown() {
+        if (!this.isRunning)
+            return;
+        pipelineLog.step(LOG_CTX, "PIPELINE_SHUTDOWN_START");
+        await this.flush();
+        await this.workerPool.shutdown();
+        this.isRunning = false;
+        const stats = this.getStats();
+        pipelineLog.summary(LOG_CTX, {
+            chunksProcessed: stats.itemsProcessed,
+            batchesProcessed: stats.batchesProcessed,
+            errors: stats.errors,
+            uptimeMs: stats.uptimeMs,
+            throughput: stats.throughput,
+            avgBatchTimeMs: stats.avgBatchTimeMs,
+        });
+        if (isDebug()) {
+            console.error(`[ChunkPipeline] Shutdown: ${stats.itemsProcessed} chunks, ` +
+                `${stats.batchesProcessed} batches, ${stats.errors} errors ` +
+                `in ${(stats.uptimeMs / 1000).toFixed(1)}s ` +
+                `(${stats.throughput.toFixed(1)} chunks/s)`);
+        }
+    }
+    /**
+     * Force shutdown (cancel pending work)
+     */
+    forceShutdown() {
+        this.isRunning = false;
+        this.accumulator.clear();
+        this.workerPool.forceShutdown();
+        this.pendingBatches = [];
+    }
+    /**
+     * Get pipeline statistics
+     */
+    getStats() {
+        const poolStats = this.workerPool.getStats();
+        const uptimeMs = this.stats.startTime > 0 ? Date.now() - this.stats.startTime : 0;
+        return {
+            itemsProcessed: this.stats.chunksProcessed,
+            batchesProcessed: this.stats.batchesProcessed,
+            errors: this.stats.errors,
+            queueDepth: poolStats.queueDepth,
+            avgBatchTimeMs: poolStats.avgTimeMs,
+            throughput: uptimeMs > 0 ? (this.stats.chunksProcessed / uptimeMs) * 1000 : 0,
+            uptimeMs,
+        };
+    }
+    /**
+     * Get pending count (chunks waiting to be batched)
+     */
+    getPendingCount() {
+        return this.accumulator.getPendingCount();
+    }
+    submitBatch(batch) {
+        const handler = this.createBatchHandler();
+        const promise = this.workerPool.submit(batch, handler);
+        this.pendingBatches.push(promise);
+        // Cleanup completed promises periodically
+        if (this.pendingBatches.length > 100) {
+            this.pendingBatches = this.pendingBatches.filter((p) => !this.isPromiseResolved(p));
+        }
+    }
+    /**
+     * Create a batch handler that embeds chunks and stores to Qdrant
+     */
+    createBatchHandler() {
+        return async (batch) => {
+            const ctx = { ...LOG_CTX, batchId: batch.id };
+            pipelineLog.batchStart(ctx, batch.id, batch.items.length);
+            // 1. Extract texts for embedding
+            const texts = batch.items.map((item) => item.chunk.content);
+            // 2. Generate embeddings
+            const embedStart = Date.now();
+            const embeddings = await this.embeddings.embedBatch(texts);
+            const embedDuration = Date.now() - embedStart;
+            pipelineLog.embedCall(ctx, texts.length, embedDuration);
+            pipelineLog.addStageTime("embed", embedDuration);
+            // 3. Build points
+            const points = batch.items.map((item, idx) => ({
+                id: item.chunkId,
+                vector: embeddings[idx].embedding,
+                payload: this.payloadBuilder.buildPayload(item.chunk, item.codebasePath),
+            }));
+            // 4. Store to Qdrant
+            const qdrantStart = Date.now();
+            if (this.sparseGenerator) {
+                const hybridPoints = points.map((point, idx) => ({
+                    ...point,
+                    sparseVector: this.sparseGenerator?.generate(batch.items[idx].chunk.content) ?? { indices: [], values: [] },
+                }));
+                await this.qdrant.addPointsWithSparse(this.collectionName, hybridPoints);
+                const qdrantDurationHybrid = Date.now() - qdrantStart;
+                pipelineLog.qdrantCall(ctx, "UPSERT_HYBRID", points.length, qdrantDurationHybrid);
+                pipelineLog.addStageTime("qdrant", qdrantDurationHybrid);
+            }
+            else {
+                await this.qdrant.addPointsOptimized(this.collectionName, points, {
+                    wait: false,
+                    ordering: "weak",
+                });
+                const qdrantDuration = Date.now() - qdrantStart;
+                pipelineLog.qdrantCall(ctx, "UPSERT", points.length, qdrantDuration);
+                pipelineLog.addStageTime("qdrant", qdrantDuration);
+            }
+            // 5. Notify callback after successful upsert (for streaming enrichment)
+            this.onBatchUpsertedCb?.(batch.items);
+        };
+    }
+    onBatchComplete(result) {
+        this.stats.batchesProcessed++;
+        this.stats.chunksProcessed += result.itemCount;
+        const ctx = { ...LOG_CTX, batchId: result.batchId };
+        if (!result.success) {
+            this.stats.errors++;
+            pipelineLog.batchFailed(ctx, result.batchId, result.error || "Unknown error", result.retryCount || 0, this.config.workerPool.maxRetries);
+            if (isDebug()) {
+                console.error(`[ChunkPipeline] Batch ${result.batchId} failed: ${result.error}`);
+            }
+        }
+        else {
+            pipelineLog.batchComplete(ctx, result.batchId, result.itemCount, result.durationMs, result.retryCount || 0);
+            if (isDebug()) {
+                console.error(`[ChunkPipeline] Batch ${result.batchId} complete: ${result.itemCount} chunks in ${result.durationMs}ms`);
+            }
+        }
+    }
+    onQueueChange(queueSize) {
+        const maxQueue = this.config.accumulator.maxQueueSize;
+        const activeWorkers = this.workerPool.getActiveWorkers();
+        const pendingItems = this.accumulator.getPendingCount();
+        pipelineLog.queueState(LOG_CTX, queueSize, activeWorkers, pendingItems);
+        if (queueSize >= maxQueue) {
+            pipelineLog.backpressure(LOG_CTX, true, `queueSize(${queueSize}) >= maxQueue(${maxQueue})`);
+            this.accumulator.pause();
+        }
+        else if (queueSize < maxQueue * 0.5) {
+            if (this.accumulator.isPausedState()) {
+                pipelineLog.backpressure(LOG_CTX, false, `queueSize(${queueSize}) < threshold(${maxQueue * 0.5})`);
+            }
+            this.accumulator.resume();
+        }
+    }
+    isPromiseResolved(promise) {
+        let resolved = false;
+        Promise.race([promise.then(() => (resolved = true)), Promise.resolve()]).catch(() => { });
+        return resolved;
+    }
+}
+//# sourceMappingURL=chunk-pipeline.js.map

package/build/core/ingest/pipeline/chunk-pipeline.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunk-pipeline.js","sourceRoot":"","sources":["../../../../src/core/ingest/pipeline/chunk-pipeline.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAGH,OAAO,EAAE,yBAAyB,EAAE,MAAM,iCAAiC,CAAC;AAG5E,OAAO,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAChE,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAUpD,MAAM,OAAO,GAAG,EAAE,SAAS,EAAE,eAAe,EAAE,CAAC;AAW/C,MAAM,OAAO,aAAa;IACP,MAAM,CAAsB;IAC5B,MAAM,CAAgB;IACtB,UAAU,CAAoB;IAC9B,cAAc,CAAS;IACvB,cAAc,CAAiB;IAC/B,eAAe,CAAmC;IAElD,UAAU,CAAa;IACvB,WAAW,CAA8B;IAClD,cAAc,GAA2B,EAAE,CAAC;IAE5C,iBAAiB,CAAgC;IACjD,SAAS,GAAG,KAAK,CAAC;IACT,KAAK,GAAG;QACvB,eAAe,EAAE,CAAC;QAClB,gBAAgB,EAAE,CAAC;QACnB,MAAM,EAAE,CAAC;QACT,SAAS,EAAE,CAAC;KACb,CAAC;IAEF,YACE,MAAqB,EACrB,UAA6B,EAC7B,cAAsB,EACtB,cAA8B,EAC9B,MAAqC;QAErC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QAErC,IAAI,CAAC,MAAM,GAAG;YACZ,UAAU,EAAE,MAAM,EAAE,UAAU,IAAI;gBAChC,WAAW,EAAE,CAAC;gBACd,UAAU,EAAE,CAAC;gBACb,gBAAgB,EAAE,GAAG;gBACrB,eAAe,EAAE,IAAI;aACtB;YACD,WAAW,EAAE,MAAM,EAAE,WAAW,IAAI;gBAClC,SAAS,EAAE,IAAI;gBACf,cAAc,EAAE,IAAI;gBACpB,YAAY,EAAE,CAAC;aAChB;YACD,YAAY,EAAE,MAAM,EAAE,YAAY,IAAI,KAAK;SAC5C,CAAC;QAEF,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,yBAAyB,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QAEzF,yBAAyB;QACzB,IAAI,CAAC,UAAU,GAAG,IAAI,UAAU,CAC9B,IAAI,CAAC,MAAM,CAAC,UAAU,EACtB,CAAC,MAAM,EAAE,EAAE;YACT,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;QAC/B,CAAC,EACD,CAAC,SAAS,EAAE,EAAE;YACZ,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;QAChC,CAAC,CACF,CAAC;QAEF,yBAAyB;QACzB,IAAI,CAAC,WAAW,GAAG,IAAI,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,QAAQ,EAAE,CAAC,KAAK,EAAE,EAAE;YACnF,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,kBAAkB,CAAC,EAAgC;QACjD,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC;IAC9B,CAAC;IAED;;OAEG;IACH,KAAK;QACH,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO;QAE3B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAElC,WAAW,CAAC,IAAI,CAAC,OAAO,EAAE,gBAAgB,EAAE;YAC1C,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,WAAW;YAC3C,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,SAAS;YAC5C,cAAc,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,cAAc;YACtD,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,YAAY;YAChC,UAAU,EAAE,IAAI,CAAC,cAAc;SAChC,CAAC,CAAC;QAEH,IAAI,OAAO,EAAE,EAAE,CAAC;YACd,OAAO,CAAC,KAAK,CACX,2BAA2B;gBACzB,WAAW,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,WAAW,IAAI;gBACjD,aAAa,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,SAAS,IAAI;gBAClD,gBAAgB,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,cAAc,IAAI,CAC7D,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,QAAQ,CAAC,KAAyB,EAAE,OAAe,EAAE,YAAoB;QACvE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC/C,CAAC;QAED,MAAM,IAAI,GAAc;YACtB,IAAI,EAAE,QAAQ;YACd,EAAE,EAAE,OAAO;YACX,KAAK;YACL,OAAO;YACP,YAAY;SACb,CAAC;QAEF,OAAO,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACpC,CAAC;IAED;;;OAGG;IACH,SAAS,CACP,MAIG;QAEH,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,KAAK,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,YAAY,EAAE,IAAI,MAAM,EAAE,CAAC;YACtD,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,EAAE,YAAY,CAAC,EAAE,CAAC;gBAChD,QAAQ,EAAE,CAAC;YACb,CAAC;iBAAM,CAAC;gBACN,MAAM;YACR,CAAC;QACH,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;OAEG;IACH,eAAe;QACb,OAAO,IAAI,CAAC,WAAW,CAAC,aAAa,EAAE,CAAC;IAC1C,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,mBAAmB,CAAC,OAAO,GAAG,KAAK;QACvC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE7B,OAAO,IAAI,CAAC,eAAe,EAAE,EAAE,CAAC;YAC9B,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,GAAG,OAAO,EAAE,CAAC;gBACrC,OAAO,KAAK,CAAC;YACf,CAAC;YACD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;QAC1D,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;QACzB,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;QAC9B,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QACvC,IAAI,CAAC,cAAc,GAAG,EAAE,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ;QACZ,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,OAAO;QAE5B,WAAW,CAAC,IAAI,CAAC,OAAO,EAAE,yBAAyB,CAAC,CAAC;QAErD,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACnB,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC;QACjC,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QAEvB,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC9B,WAAW,CAAC,OAAO,CAAC,OAAO,EAAE;YAC3B,eAAe,EAAE,KAAK,CAAC,cAAc;YACrC,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;YACxC,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,UAAU,EAAE,KAAK,CAAC,UAAU;YAC5B,cAAc,EAAE,KAAK,CAAC,cAAc;SACrC,CAAC,CAAC;QAEH,IAAI,OAAO,EAAE,EAAE,CAAC;YACd,OAAO,CAAC,KAAK,CACX,6BAA6B,KAAK,CAAC,cAAc,WAAW;gBAC1D,GAAG,KAAK,CAAC,gBAAgB,aAAa,KAAK,CAAC,MAAM,UAAU;gBAC5D,MAAM,CAAC,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI;gBAC5C,IAAI,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,YAAY,CAC9C,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;OAEG;IACH,aAAa;QACX,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACvB,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,UAAU,CAAC,aAAa,EAAE,CAAC;QAChC,IAAI,CAAC,cAAc,GAAG,EAAE,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,QAAQ;QACN,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC;QAC7C,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;QAElF,OAAO;YACL,cAAc,EAAE,IAAI,CAAC,KAAK,CAAC,eAAe;YAC1C,gBAAgB,EAAE,IAAI,CAAC,KAAK,CAAC,gBAAgB;YAC7C,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM;YACzB,UAAU,EAAE,SAAS,CAAC,UAAU;YAChC,cAAc,EAAE,SAAS,CAAC,SAAS;YACnC,UAAU,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,eAAe,GAAG,QAAQ,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;YAC7E,QAAQ;SACT,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,eAAe;QACb,OAAO,IAAI,CAAC,WAAW,CAAC,eAAe,EAAE,CAAC;IAC5C,CAAC;IAEO,WAAW,CAAC,KAAuB;QACzC,MAAM,OAAO,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAC1C,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QACvD,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAElC,0CAA0C;QAC1C,IAAI,IAAI,CAAC,cAAc,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;YACrC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC;QACtF,CAAC;IACH,CAAC;IAED;;OAEG;IACK,kBAAkB;QACxB,OAAO,KAAK,EAAE,KAAuB,EAAE,EAAE;YACvC,MAAM,GAAG,GAAG,EAAE,GAAG,OAAO,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC;YAE9C,WAAW,CAAC,UAAU,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YAE1D,iCAAiC;YACjC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAE5D,yBAAyB;YACzB,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC9B,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YAC3D,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC;YAC9C,WAAW,CAAC,SAAS,CAAC,GAAG,EAAE,KAAK,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;YACxD,WAAW,CAAC,YAAY,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;YAEjD,kBAAkB;YAClB,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;gBAC7C,EAAE,EAAE,IAAI,CAAC,OAAO;gBAChB,MAAM,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS;gBACjC,OAAO,EAAE,IAAI,CAAC,cAAc,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,YAAY,CAAC;aACzE,CAAC,CAAC,CAAC;YAEJ,qBAAqB;YACrB,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC/B,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;gBACzB,MAAM,YAAY,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;oBAC/C,GAAG,KAAK;oBACR,YAAY,EAAE,IAAI,CAAC,eAAe,EAAE,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE;iBAC5G,CAAC,CAAC,CAAC;gBACJ,MAAM,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,IAAI,CAAC,cAAc,EAAE,YAAY,CAAC,CAAC;gBACzE,MAAM,oBAAoB,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW,CAAC;gBACtD,WAAW,CAAC,UAAU,CAAC,GAAG,EAAE,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,oBAAoB,CAAC,CAAC;gBAClF,WAAW,CAAC,YAAY,CAAC,QAAQ,EAAE,oBAAoB,CAAC,CAAC;YAC3D,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,IAAI,CAAC,cAAc,EAAE,MAAM,EAAE;oBAChE,IAAI,EAAE,KAAK;oBACX,QAAQ,EAAE,MAAM;iBACjB,CAAC,CAAC;gBACH,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW,CAAC;gBAChD,WAAW,CAAC,UAAU,CAAC,GAAG,EAAE,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;gBACrE,WAAW,CAAC,YAAY,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;YACrD,CAAC;YAED,wEAAwE;YACxE,IAAI,CAAC,iBAAiB,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACxC,CAAC,CAAC;IACJ,CAAC;IAEO,eAAe,CAAC,MAAmB;QACzC,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC;QAC9B,IAAI,CAAC,KAAK,CAAC,eAAe,IAAI,MAAM,CAAC,SAAS,CAAC;QAE/C,MAAM,GAAG,GAAG,EAAE,GAAG,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC;QAEpD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;YACpB,WAAW,CAAC,WAAW,CACrB,GAAG,EACH,MAAM,CAAC,OAAO,EACd,MAAM,CAAC,KAAK,IAAI,eAAe,EAC/B,MAAM,CAAC,UAAU,IAAI,CAAC,EACtB,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,UAAU,CAClC,CAAC;YACF,IAAI,OAAO,EAAE,EAAE,CAAC;gBACd,OAAO,CAAC,KAAK,CAAC,yBAAyB,MAAM,CAAC,OAAO,YAAY,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;YACnF,CAAC;QACH,CAAC;aAAM,CAAC;YACN,WAAW,CAAC,aAAa,CAAC,GAAG,EAAE,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,CAAC,CAAC,CAAC;YAC5G,IAAI,OAAO,EAAE,EAAE,CAAC;gBACd,OAAO,CAAC,KAAK,CACX,yBAAyB,MAAM,CAAC,OAAO,cAAc,MAAM,CAAC,SAAS,cAAc,MAAM,CAAC,UAAU,IAAI,CACzG,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAEO,aAAa,CAAC,SAAiB;QACrC,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,YAAY,CAAC;QACtD,MAAM,aAAa,GAAG,IAAI,CAAC,UAAU,CAAC,gBAAgB,EAAE,CAAC;QACzD,MAAM,YAAY,GAAG,IAAI,CAAC,WAAW,CAAC,eAAe,EAAE,CAAC;QAExD,WAAW,CAAC,UAAU,CAAC,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,YAAY,CAAC,CAAC;QAExE,IAAI,SAAS,IAAI,QAAQ,EAAE,CAAC;YAC1B,WAAW,CAAC,YAAY,CAAC,OAAO,EAAE,IAAI,EAAE,aAAa,SAAS,iBAAiB,QAAQ,GAAG,CAAC,CAAC;YAC5F,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;QAC3B,CAAC;aAAM,IAAI,SAAS,GAAG,QAAQ,GAAG,GAAG,EAAE,CAAC;YACtC,IAAI,IAAI,CAAC,WAAW,CAAC,aAAa,EAAE,EAAE,CAAC;gBACrC,WAAW,CAAC,YAAY,CAAC,OAAO,EAAE,KAAK,EAAE,aAAa,SAAS,iBAAiB,QAAQ,GAAG,GAAG,GAAG,CAAC,CAAC;YACrG,CAAC;YACD,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;QAC5B,CAAC;IACH,CAAC;IAEO,iBAAiB,CAAC,OAAyB;QACjD,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACzF,OAAO,QAAQ,CAAC;IAClB,CAAC;CACF"}

package/build/core/ingest/pipeline/chunker/base.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Base interface for code chunkers
+ */
+import type { CodeChunk } from "../../../types.js";
+export interface CodeChunker {
+    /**
+     * Split code into semantic chunks
+     */
+    chunk: (code: string, filePath: string, language: string) => Promise<CodeChunk[]>;
+    /**
+     * Check if language is supported by this chunker
+     */
+    supportsLanguage: (language: string) => boolean;
+    /**
+     * Get chunking strategy name
+     */
+    getStrategyName: () => string;
+}
+//# sourceMappingURL=base.d.ts.map

package/build/core/ingest/pipeline/chunker/base.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"base.d.ts","sourceRoot":"","sources":["../../../../../src/core/ingest/pipeline/chunker/base.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAEnD,MAAM,WAAW,WAAW;IAC1B;;OAEG;IACH,KAAK,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;IAElF;;OAEG;IACH,gBAAgB,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC;IAEhD;;OAEG;IACH,eAAe,EAAE,MAAM,MAAM,CAAC;CAC/B"}

package/build/core/ingest/pipeline/chunker/base.js

@@ -0,0 +1,5 @@
+/**
+ * Base interface for code chunkers
+ */
+export {};
+//# sourceMappingURL=base.js.map