tea-rags 0.9.0

package/build/core/domains/ingest/reindexing.js

@@ -0,0 +1,166 @@
+/**
+ * ReindexPipeline - Incremental re-indexing of changed files.
+ *
+ * Orchestrates: scan → detect changes → delete old → process new/modified → snapshot.
+ * File processing logic is delegated to FileProcessor.
+ */
+import { BaseIndexingPipeline } from "./pipeline/base.js";
+import { processRelativeFiles } from "./pipeline/file-processor.js";
+import { pipelineLog } from "./pipeline/infra/debug-logger.js";
+import { isDebug } from "./pipeline/infra/runtime.js";
+import { performDeletion } from "./sync/deletion-strategy.js";
+export class ReindexPipeline extends BaseIndexingPipeline {
+    deleteConfig;
+    constructor(qdrant, embeddings, config, enrichment, deps, deleteConfig = { batchSize: 500, concurrency: 8 }, tuning) {
+        super(qdrant, embeddings, config, enrichment, deps, tuning);
+        this.deleteConfig = deleteConfig;
+    }
+    async reindexChanges(path, progressCallback) {
+        const startTime = Date.now();
+        const stats = {
+            filesAdded: 0,
+            filesModified: 0,
+            filesDeleted: 0,
+            chunksAdded: 0,
+            chunksDeleted: 0,
+            durationMs: 0,
+            status: "completed",
+        };
+        try {
+            const { absolutePath, collectionName } = await this.resolveContext(path);
+            // Check if collection exists
+            const exists = await this.qdrant.collectionExists(collectionName);
+            if (!exists) {
+                throw new Error(`Codebase not indexed: ${path}`);
+            }
+            // AUTO-MIGRATE: snapshots and schema
+            const migrator = this.deps.createMigrator(collectionName, absolutePath);
+            await migrator.ensureMigrated();
+            const schemaManager = this.deps.createSchemaManager();
+            const schemaMigration = await schemaManager.ensureCurrentSchema(collectionName);
+            if (schemaMigration.migrationsApplied.length > 0) {
+                pipelineLog.reindexPhase("schema_migration", {
+                    fromVersion: schemaMigration.fromVersion,
+                    toVersion: schemaMigration.toVersion,
+                    migrations: schemaMigration.migrationsApplied,
+                });
+            }
+            // Initialize synchronizer
+            const synchronizer = this.deps.createSynchronizer(absolutePath, collectionName);
+            const hasSnapshot = await synchronizer.initialize();
+            if (!hasSnapshot) {
+                throw new Error("No previous snapshot found. Use index_codebase for initial indexing.");
+            }
+            // Check for checkpoint
+            const checkpoint = await synchronizer.loadCheckpoint();
+            let resumeFromCheckpoint = false;
+            if (checkpoint) {
+                resumeFromCheckpoint = true;
+                console.error(`[Reindex] Resuming from checkpoint: ${checkpoint.processedFiles.length} files already processed`);
+            }
+            // Scan current files
+            progressCallback?.({
+                phase: "scanning",
+                current: 0,
+                total: 100,
+                percentage: 0,
+                message: resumeFromCheckpoint ? "Resuming from checkpoint..." : "Scanning for changes...",
+            });
+            const scanner = this.createScanner();
+            const currentFiles = await this.scanFiles(absolutePath, scanner);
+            // Detect changes
+            pipelineLog.stageStart("scan");
+            const changes = await synchronizer.detectChanges(currentFiles);
+            pipelineLog.stageEnd("scan");
+            stats.filesAdded = changes.added.length;
+            stats.filesModified = changes.modified.length;
+            stats.filesDeleted = changes.deleted.length;
+            if (stats.filesAdded === 0 && stats.filesModified === 0 && stats.filesDeleted === 0) {
+                await synchronizer.deleteCheckpoint();
+                stats.durationMs = Date.now() - startTime;
+                return stats;
+            }
+            // Initialize processing components
+            const ctx = this.initProcessing(collectionName, absolutePath, scanner);
+            const chunkMap = new Map();
+            const filesToDelete = [...changes.modified, ...changes.deleted];
+            const addedFiles = [...changes.added];
+            const modifiedFiles = [...changes.modified];
+            const processOpts = {
+                enableGitMetadata: this.config.enableGitMetadata === true,
+                concurrency: this.tuning.fileConcurrency,
+            };
+            // PARALLEL PIPELINES: delete + add simultaneously, then modified after delete
+            const parallelStart = Date.now();
+            pipelineLog.reindexPhase("PARALLEL_START", {
+                deleted: filesToDelete.length,
+                added: addedFiles.length,
+                modified: modifiedFiles.length,
+            });
+            if (isDebug()) {
+                console.error(`[Reindex] Starting parallel pipelines: ` +
+                    `delete=${filesToDelete.length}, added=${addedFiles.length}, modified=${modifiedFiles.length}`);
+            }
+            const deleteStartTime = Date.now();
+            const deletePromise = performDeletion(this.qdrant, collectionName, filesToDelete, this.deleteConfig, progressCallback);
+            const addPromise = processRelativeFiles(addedFiles, absolutePath, ctx.chunkerPool, ctx.chunkPipeline, processOpts, chunkMap, "added");
+            pipelineLog.reindexPhase("DELETE_AND_ADD_STARTED", {
+                deleteFiles: filesToDelete.length,
+                addFiles: addedFiles.length,
+            });
+            await deletePromise;
+            pipelineLog.reindexPhase("DELETE_COMPLETE", {
+                durationMs: Date.now() - deleteStartTime,
+                deleted: filesToDelete.length,
+            });
+            if (isDebug()) {
+                console.error(`[Reindex] Delete complete, starting modified indexing (add still running in parallel)`);
+            }
+            const modifiedStartTime = Date.now();
+            const modifiedPromise = processRelativeFiles(modifiedFiles, absolutePath, ctx.chunkerPool, ctx.chunkPipeline, processOpts, chunkMap, "modified");
+            pipelineLog.reindexPhase("MODIFIED_STARTED", {
+                modifiedFiles: modifiedFiles.length,
+                addStillRunning: true,
+            });
+            const [addedChunks, modifiedChunks] = await Promise.all([addPromise, modifiedPromise]);
+            pipelineLog.reindexPhase("ADD_AND_MODIFIED_COMPLETE", {
+                addedChunks,
+                modifiedChunks,
+                addDurationMs: Date.now() - parallelStart,
+                modifiedDurationMs: Date.now() - modifiedStartTime,
+            });
+            // Flush, shutdown, and enrichment
+            if (isDebug()) {
+                const pipelineStats = ctx.chunkPipeline.getStats();
+                console.error(`[Reindex] ChunkPipeline before flush: ` +
+                    `pending=${ctx.chunkPipeline.getPendingCount()}, ` +
+                    `processed=${pipelineStats.itemsProcessed}, ` +
+                    `batches=${pipelineStats.batchesProcessed}`);
+            }
+            const getEnrichmentStatus = await this.finalizeProcessing(ctx, chunkMap, collectionName, absolutePath);
+            const pipelineStats = ctx.chunkPipeline.getStats();
+            if (isDebug()) {
+                console.error(`[Reindex] Parallel pipelines completed in ${Date.now() - parallelStart}ms ` +
+                    `(pipeline: ${pipelineStats.itemsProcessed} chunks in ${pipelineStats.batchesProcessed} batches, ` +
+                    `${pipelineStats.throughput.toFixed(1)} chunks/s)`);
+            }
+            stats.chunksAdded = addedChunks + modifiedChunks;
+            // Update snapshot
+            await synchronizer.updateSnapshot(currentFiles);
+            await synchronizer.deleteCheckpoint();
+            stats.enrichmentStatus = getEnrichmentStatus();
+            stats.durationMs = Date.now() - startTime;
+            if (isDebug()) {
+                console.error(`[Reindex] Complete: ${stats.filesAdded} added, ` +
+                    `${stats.filesModified} modified, ${stats.filesDeleted} deleted. ` +
+                    `Created ${stats.chunksAdded} chunks in ${(stats.durationMs / 1000).toFixed(1)}s`);
+            }
+            return stats;
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            throw new Error(`Incremental re-indexing failed: ${errorMessage}`);
+        }
+    }
+}
+//# sourceMappingURL=reindexing.js.map

package/build/core/domains/ingest/reindexing.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"reindexing.js","sourceRoot":"","sources":["../../../../src/core/domains/ingest/reindexing.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EAAE,oBAAoB,EAAuB,MAAM,oBAAoB,CAAC;AAC/E,OAAO,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,EAAE,WAAW,EAAE,MAAM,kCAAkC,CAAC;AAC/D,OAAO,EAAE,OAAO,EAAE,MAAM,6BAA6B,CAAC;AACtD,OAAO,EAAE,eAAe,EAAuB,MAAM,6BAA6B,CAAC;AAEnF,MAAM,OAAO,eAAgB,SAAQ,oBAAoB;IAOpC;IANnB,YACE,MAA6D,EAC7D,UAAiE,EACjE,MAA6D,EAC7D,UAAiE,EACjE,IAA2D,EAC1C,eAA+B,EAAE,SAAS,EAAE,GAAG,EAAE,WAAW,EAAE,CAAC,EAAE,EAClF,MAAuB;QAEvB,KAAK,CAAC,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;QAH3C,iBAAY,GAAZ,YAAY,CAAqD;IAIpF,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,IAAY,EAAE,gBAAmC;QACpE,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,KAAK,GAAgB;YACzB,UAAU,EAAE,CAAC;YACb,aAAa,EAAE,CAAC;YAChB,YAAY,EAAE,CAAC;YACf,WAAW,EAAE,CAAC;YACd,aAAa,EAAE,CAAC;YAChB,UAAU,EAAE,CAAC;YACb,MAAM,EAAE,WAAW;SACpB,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,EAAE,YAAY,EAAE,cAAc,EAAE,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;YAEzE,6BAA6B;YAC7B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,cAAc,CAAC,CAAC;YAClE,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,MAAM,IAAI,KAAK,CAAC,yBAAyB,IAAI,EAAE,CAAC,CAAC;YACnD,CAAC;YAED,qCAAqC;YACrC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,YAAY,CAAC,CAAC;YACxE,MAAM,QAAQ,CAAC,cAAc,EAAE,CAAC;YAEhC,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,CAAC;YACtD,MAAM,eAAe,GAAG,MAAM,aAAa,CAAC,mBAAmB,CAAC,cAAc,CAAC,CAAC;YAChF,IAAI,eAAe,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACjD,WAAW,CAAC,YAAY,CAAC,kBAAkB,EAAE;oBAC3C,WAAW,EAAE,eAAe,CAAC,WAAW;oBACxC,SAAS,EAAE,eAAe,CAAC,SAAS;oBACpC,UAAU,EAAE,eAAe,CAAC,iBAAiB;iBAC9C,CAAC,CAAC;YACL,CAAC;YAED,0BAA0B;YAC1B,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,YAAY,EAAE,cAAc,CAAC,CAAC;YAChF,MAAM,WAAW,GAAG,MAAM,YAAY,CAAC,UAAU,EAAE,CAAC;YAEpD,IAAI,CAAC,WAAW,EAAE,CAAC;gBACjB,MAAM,IAAI,KAAK,CAAC,sEAAsE,CAAC,CAAC;YAC1F,CAAC;YAED,uBAAuB;YACvB,MAAM,UAAU,GAAG,MAAM,YAAY,CAAC,cAAc,EAAE,CAAC;YACvD,IAAI,oBAAoB,GAAG,KAAK,CAAC;YAEjC,IAAI,UAAU,EAAE,CAAC;gBACf,oBAAoB,GAAG,IAAI,CAAC;gBAC5B,OAAO,CAAC,KAAK,CACX,uCAAuC,UAAU,CAAC,cAAc,CAAC,MAAM,0BAA0B,CAClG,CAAC;YACJ,CAAC;YAED,qBAAqB;YACrB,gBAAgB,EAAE,CAAC;gBACjB,KAAK,EAAE,UAAU;gBACjB,OAAO,EAAE,CAAC;gBACV,KAAK,EAAE,GAAG;gBACV,UAAU,EAAE,CAAC;gBACb,OAAO,EAAE,oBAAoB,CAAC,CAAC,CAAC,6BAA6B,CAAC,CAAC,CAAC,yBAAyB;aAC1F,CAAC,CAAC;YAEH,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;YACrC,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;YAEjE,iBAAiB;YACjB,WAAW,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;YAC/B,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,aAAa,CAAC,YAAY,CAAC,CAAC;YAC/D,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAC7B,KAAK,CAAC,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC;YACxC,KAAK,CAAC,aAAa,GAAG,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;YAC9C,KAAK,CAAC,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC;YAE5C,IAAI,KAAK,CAAC,UAAU,KAAK,CAAC,IAAI,KAAK,CAAC,aAAa,KAAK,CAAC,IAAI,KAAK,CAAC,YAAY,KAAK,CAAC,EAAE,CAAC;gBACpF,MAAM,YAAY,CAAC,gBAAgB,EAAE,CAAC;gBACtC,KAAK,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;gBAC1C,OAAO,KAAK,CAAC;YACf,CAAC;YAED,mCAAmC;YACnC,MAAM,GAAG,GAAG,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,YAAY,EAAE,OAAO,CAAC,CAAC;YAEvE,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA8B,CAAC;YACvD,MAAM,aAAa,GAAG,CAAC,GAAG,OAAO,CAAC,QAAQ,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;YAChE,MAAM,UAAU,GAAG,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;YACtC,MAAM,aAAa,GAAG,CAAC,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;YAE5C,MAAM,WAAW,GAAG;gBAClB,iBAAiB,EAAE,IAAI,CAAC,MAAM,CAAC,iBAAiB,KAAK,IAAI;gBACzD,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,eAAe;aACzC,CAAC;YAEF,8EAA8E;YAC9E,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAEjC,WAAW,CAAC,YAAY,CAAC,gBAAgB,EAAE;gBACzC,OAAO,EAAE,aAAa,CAAC,MAAM;gBAC7B,KAAK,EAAE,UAAU,CAAC,MAAM;gBACxB,QAAQ,EAAE,aAAa,CAAC,MAAM;aAC/B,CAAC,CAAC;YAEH,IAAI,OAAO,EAAE,EAAE,CAAC;gBACd,OAAO,CAAC,KAAK,CACX,yCAAyC;oBACvC,UAAU,aAAa,CAAC,MAAM,WAAW,UAAU,CAAC,MAAM,cAAc,aAAa,CAAC,MAAM,EAAE,CACjG,CAAC;YACJ,CAAC;YAED,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACnC,MAAM,aAAa,GAAG,eAAe,CACnC,IAAI,CAAC,MAAM,EACX,cAAc,EACd,aAAa,EACb,IAAI,CAAC,YAAY,EACjB,gBAAgB,CACjB,CAAC;YACF,MAAM,UAAU,GAAG,oBAAoB,CACrC,UAAU,EACV,YAAY,EACZ,GAAG,CAAC,WAAW,EACf,GAAG,CAAC,aAAa,EACjB,WAAW,EACX,QAAQ,EACR,OAAO,CACR,CAAC;YAEF,WAAW,CAAC,YAAY,CAAC,wBAAwB,EAAE;gBACjD,WAAW,EAAE,aAAa,CAAC,MAAM;gBACjC,QAAQ,EAAE,UAAU,CAAC,MAAM;aAC5B,CAAC,CAAC;YAEH,MAAM,aAAa,CAAC;YAEpB,WAAW,CAAC,YAAY,CAAC,iBAAiB,EAAE;gBAC1C,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,eAAe;gBACxC,OAAO,EAAE,aAAa,CAAC,MAAM;aAC9B,CAAC,CAAC;YAEH,IAAI,OAAO,EAAE,EAAE,CAAC;gBACd,OAAO,CAAC,KAAK,CAAC,uFAAuF,CAAC,CAAC;YACzG,CAAC;YAED,MAAM,iBAAiB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACrC,MAAM,eAAe,GAAG,oBAAoB,CAC1C,aAAa,EACb,YAAY,EACZ,GAAG,CAAC,WAAW,EACf,GAAG,CAAC,aAAa,EACjB,WAAW,EACX,QAAQ,EACR,UAAU,CACX,CAAC;YAEF,WAAW,CAAC,YAAY,CAAC,kBAAkB,EAAE;gBAC3C,aAAa,EAAE,aAAa,CAAC,MAAM;gBACnC,eAAe,EAAE,IAAI;aACtB,CAAC,CAAC;YAEH,MAAM,CAAC,WAAW,EAAE,cAAc,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,eAAe,CAAC,CAAC,CAAC;YAEvF,WAAW,CAAC,YAAY,CAAC,2BAA2B,EAAE;gBACpD,WAAW;gBACX,cAAc;gBACd,aAAa,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,aAAa;gBACzC,kBAAkB,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,iBAAiB;aACnD,CAAC,CAAC;YAEH,kCAAkC;YAClC,IAAI,OAAO,EAAE,EAAE,CAAC;gBACd,MAAM,aAAa,GAAG,GAAG,CAAC,aAAa,CAAC,QAAQ,EAAE,CAAC;gBACnD,OAAO,CAAC,KAAK,CACX,wCAAwC;oBACtC,WAAW,GAAG,CAAC,aAAa,CAAC,eAAe,EAAE,IAAI;oBAClD,aAAa,aAAa,CAAC,cAAc,IAAI;oBAC7C,WAAW,aAAa,CAAC,gBAAgB,EAAE,CAC9C,CAAC;YACJ,CAAC;YAED,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,GAAG,EAAE,QAAQ,EAAE,cAAc,EAAE,YAAY,CAAC,CAAC;YAEvG,MAAM,aAAa,GAAG,GAAG,CAAC,aAAa,CAAC,QAAQ,EAAE,CAAC;YACnD,IAAI,OAAO,EAAE,EAAE,CAAC;gBACd,OAAO,CAAC,KAAK,CACX,6CAA6C,IAAI,CAAC,GAAG,EAAE,GAAG,aAAa,KAAK;oBAC1E,cAAc,aAAa,CAAC,cAAc,cAAc,aAAa,CAAC,gBAAgB,YAAY;oBAClG,GAAG,aAAa,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,YAAY,CACrD,CAAC;YACJ,CAAC;YAED,KAAK,CAAC,WAAW,GAAG,WAAW,GAAG,cAAc,CAAC;YAEjD,kBAAkB;YAClB,MAAM,YAAY,CAAC,cAAc,CAAC,YAAY,CAAC,CAAC;YAChD,MAAM,YAAY,CAAC,gBAAgB,EAAE,CAAC;YAEtC,KAAK,CAAC,gBAAgB,GAAG,mBAAmB,EAAE,CAAC;YAC/C,KAAK,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YAE1C,IAAI,OAAO,EAAE,EAAE,CAAC;gBACd,OAAO,CAAC,KAAK,CACX,uBAAuB,KAAK,CAAC,UAAU,UAAU;oBAC/C,GAAG,KAAK,CAAC,aAAa,cAAc,KAAK,CAAC,YAAY,YAAY;oBAClE,WAAW,KAAK,CAAC,WAAW,cAAc,CAAC,KAAK,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CACpF,CAAC;YACJ,CAAC;YAED,OAAO,KAAK,CAAC;QACf,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC5E,MAAM,IAAI,KAAK,CAAC,mCAAmC,YAAY,EAAE,CAAC,CAAC;QACrE,CAAC;IACH,CAAC;CACF"}

package/build/core/domains/ingest/sync/consistent-hash.d.ts

@@ -0,0 +1,39 @@
+/**
+ * ConsistentHash - Consistent hashing ring for shard distribution
+ *
+ * Uses virtual nodes to ensure even distribution and minimal redistribution
+ * when shard count changes.
+ */
+export interface ConsistentHashOptions {
+    virtualNodesPerShard?: number;
+}
+export declare class ConsistentHash {
+    private readonly ring;
+    private sortedPositions;
+    private readonly shardCount;
+    private readonly virtualNodesPerShard;
+    constructor(shardCount: number, options?: ConsistentHashOptions);
+    /**
+     * Build the consistent hash ring with virtual nodes
+     */
+    private buildRing;
+    /**
+     * Get shard index for a file path
+     */
+    getShard(filePath: string): number;
+    /**
+     * Get the number of shards
+     */
+    getShardCount(): number;
+    /**
+     * Hash a string to unsigned 32-bit integer
+     * Using MD5 for speed (not cryptographic use)
+     */
+    private hash;
+    /**
+     * Binary search for first position >= target
+     * Returns 0 if target is greater than all positions (wrap around)
+     */
+    private binarySearchCeil;
+}
+//# sourceMappingURL=consistent-hash.d.ts.map

package/build/core/domains/ingest/sync/consistent-hash.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"consistent-hash.d.ts","sourceRoot":"","sources":["../../../../../src/core/domains/ingest/sync/consistent-hash.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,MAAM,WAAW,qBAAqB;IACpC,oBAAoB,CAAC,EAAE,MAAM,CAAC;CAC/B;AAED,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAkC;IACvD,OAAO,CAAC,eAAe,CAAgB;IACvC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAS;gBAElC,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,qBAAqB;IAW/D;;OAEG;IACH,OAAO,CAAC,SAAS;IAYjB;;OAEG;IACH,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM;IAelC;;OAEG;IACH,aAAa,IAAI,MAAM;IAIvB;;;OAGG;IACH,OAAO,CAAC,IAAI;IAMZ;;;OAGG;IACH,OAAO,CAAC,gBAAgB;CAiBzB"}

package/build/core/domains/ingest/sync/consistent-hash.js

@@ -0,0 +1,86 @@
+/**
+ * ConsistentHash - Consistent hashing ring for shard distribution
+ *
+ * Uses virtual nodes to ensure even distribution and minimal redistribution
+ * when shard count changes.
+ */
+import { createHash } from "node:crypto";
+export class ConsistentHash {
+    ring = new Map();
+    sortedPositions = [];
+    shardCount;
+    virtualNodesPerShard;
+    constructor(shardCount, options) {
+        if (shardCount < 1) {
+            throw new Error("Shard count must be at least 1");
+        }
+        this.shardCount = shardCount;
+        this.virtualNodesPerShard = options?.virtualNodesPerShard ?? 150;
+        this.buildRing();
+    }
+    /**
+     * Build the consistent hash ring with virtual nodes
+     */
+    buildRing() {
+        for (let shard = 0; shard < this.shardCount; shard++) {
+            for (let v = 0; v < this.virtualNodesPerShard; v++) {
+                const key = `shard-${shard}-vnode-${v}`;
+                const position = this.hash(key);
+                this.ring.set(position, shard);
+            }
+        }
+        this.sortedPositions = [...this.ring.keys()].sort((a, b) => a - b);
+    }
+    /**
+     * Get shard index for a file path
+     */
+    getShard(filePath) {
+        if (this.shardCount === 1) {
+            return 0;
+        }
+        const hash = this.hash(filePath);
+        const idx = this.binarySearchCeil(hash);
+        const position = this.sortedPositions[idx];
+        const shard = this.ring.get(position);
+        if (shard === undefined) {
+            throw new Error(`No shard found for position ${position}`);
+        }
+        return shard;
+    }
+    /**
+     * Get the number of shards
+     */
+    getShardCount() {
+        return this.shardCount;
+    }
+    /**
+     * Hash a string to unsigned 32-bit integer
+     * Using MD5 for speed (not cryptographic use)
+     */
+    hash(key) {
+        const hash = createHash("md5").update(key).digest();
+        // Read first 4 bytes as unsigned 32-bit integer
+        return hash.readUInt32BE(0);
+    }
+    /**
+     * Binary search for first position >= target
+     * Returns 0 if target is greater than all positions (wrap around)
+     */
+    binarySearchCeil(target) {
+        const positions = this.sortedPositions;
+        let left = 0;
+        let right = positions.length;
+        while (left < right) {
+            const mid = Math.floor((left + right) / 2);
+            if (positions[mid] < target) {
+                left = mid + 1;
+            }
+            else {
+                right = mid;
+            }
+        }
+        // Wrap around if target is greater than all positions
+        return left === positions.length ? 0 : left;
+    }
+}
+//# sourceMappingURL=consistent-hash.js.map

package/build/core/domains/ingest/sync/consistent-hash.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"consistent-hash.js","sourceRoot":"","sources":["../../../../../src/core/domains/ingest/sync/consistent-hash.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAMzC,MAAM,OAAO,cAAc;IACR,IAAI,GAAwB,IAAI,GAAG,EAAE,CAAC;IAC/C,eAAe,GAAa,EAAE,CAAC;IACtB,UAAU,CAAS;IACnB,oBAAoB,CAAS;IAE9C,YAAY,UAAkB,EAAE,OAA+B;QAC7D,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;QACpD,CAAC;QAED,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,oBAAoB,GAAG,OAAO,EAAE,oBAAoB,IAAI,GAAG,CAAC;QAEjE,IAAI,CAAC,SAAS,EAAE,CAAC;IACnB,CAAC;IAED;;OAEG;IACK,SAAS;QACf,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,UAAU,EAAE,KAAK,EAAE,EAAE,CAAC;YACrD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,oBAAoB,EAAE,CAAC,EAAE,EAAE,CAAC;gBACnD,MAAM,GAAG,GAAG,SAAS,KAAK,UAAU,CAAC,EAAE,CAAC;gBACxC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAChC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;YACjC,CAAC;QACH,CAAC;QAED,IAAI,CAAC,eAAe,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACrE,CAAC;IAED;;OAEG;IACH,QAAQ,CAAC,QAAgB;QACvB,IAAI,IAAI,CAAC,UAAU,KAAK,CAAC,EAAE,CAAC;YAC1B,OAAO,CAAC,CAAC;QACX,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACjC,MAAM,GAAG,GAAG,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;QACxC,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;QAC3C,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACtC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,+BAA+B,QAAQ,EAAE,CAAC,CAAC;QAC7D,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;OAEG;IACH,aAAa;QACX,OAAO,IAAI,CAAC,UAAU,CAAC;IACzB,CAAC;IAED;;;OAGG;IACK,IAAI,CAAC,GAAW;QACtB,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC;QACpD,gDAAgD;QAChD,OAAO,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IAC9B,CAAC;IAED;;;OAGG;IACK,gBAAgB,CAAC,MAAc;QACrC,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC;QACvC,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,IAAI,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC;QAE7B,OAAO,IAAI,GAAG,KAAK,EAAE,CAAC;YACpB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;YAC3C,IAAI,SAAS,CAAC,GAAG,CAAC,GAAG,MAAM,EAAE,CAAC;gBAC5B,IAAI,GAAG,GAAG,GAAG,CAAC,CAAC;YACjB,CAAC;iBAAM,CAAC;gBACN,KAAK,GAAG,GAAG,CAAC;YACd,CAAC;QACH,CAAC;QAED,sDAAsD;QACtD,OAAO,IAAI,KAAK,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC9C,CAAC;CACF"}

package/build/core/domains/ingest/sync/deletion-strategy.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Deletion Strategy - 3-level fallback cascade for removing old chunks from Qdrant.
+ *
+ * Level 0: Batched path deletion (fast, concurrent)
+ * Level 1: Bulk path deletion (single call fallback)
+ * Level 2: Individual filter-based deletion (slow, last resort)
+ */
+import type { QdrantManager } from "../../../adapters/qdrant/client.js";
+import type { ProgressCallback } from "../../../types.js";
+export interface DeletionConfig {
+    batchSize: number;
+    concurrency: number;
+}
+/**
+ * Delete chunks for a list of relative file paths using a 3-level fallback cascade.
+ */
+export declare function performDeletion(qdrant: QdrantManager, collectionName: string, filesToDelete: string[], deleteConfig: DeletionConfig, progressCallback?: ProgressCallback): Promise<void>;
+//# sourceMappingURL=deletion-strategy.d.ts.map

package/build/core/domains/ingest/sync/deletion-strategy.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"deletion-strategy.d.ts","sourceRoot":"","sources":["../../../../../src/core/domains/ingest/sync/deletion-strategy.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,oCAAoC,CAAC;AACxE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAI1D,MAAM,WAAW,cAAc;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,wBAAsB,eAAe,CACnC,MAAM,EAAE,aAAa,EACrB,cAAc,EAAE,MAAM,EACtB,aAAa,EAAE,MAAM,EAAE,EACvB,YAAY,EAAE,cAAc,EAC5B,gBAAgB,CAAC,EAAE,gBAAgB,GAClC,OAAO,CAAC,IAAI,CAAC,CAoFf"}

package/build/core/domains/ingest/sync/deletion-strategy.js

@@ -0,0 +1,88 @@
+/**
+ * Deletion Strategy - 3-level fallback cascade for removing old chunks from Qdrant.
+ *
+ * Level 0: Batched path deletion (fast, concurrent)
+ * Level 1: Bulk path deletion (single call fallback)
+ * Level 2: Individual filter-based deletion (slow, last resort)
+ */
+import { pipelineLog } from "../pipeline/infra/debug-logger.js";
+import { isDebug } from "../pipeline/infra/runtime.js";
+/**
+ * Delete chunks for a list of relative file paths using a 3-level fallback cascade.
+ */
+export async function performDeletion(qdrant, collectionName, filesToDelete, deleteConfig, progressCallback) {
+    if (filesToDelete.length === 0)
+        return;
+    progressCallback?.({
+        phase: "scanning",
+        current: 0,
+        total: filesToDelete.length,
+        percentage: 5,
+        message: `Deleting old chunks for ${filesToDelete.length} files...`,
+    });
+    try {
+        const deleteResult = await qdrant.deletePointsByPathsBatched(collectionName, filesToDelete, {
+            batchSize: deleteConfig.batchSize,
+            concurrency: deleteConfig.concurrency,
+            onProgress: (deleted, total) => {
+                progressCallback?.({
+                    phase: "scanning",
+                    current: deleted,
+                    total,
+                    percentage: 5 + Math.floor((deleted / total) * 5),
+                    message: `Deleting old chunks: ${deleted}/${total} files...`,
+                });
+            },
+        });
+        if (isDebug()) {
+            console.error(`[Reindex] Deleted ${deleteResult.deletedPaths} paths in ${deleteResult.batchCount} batches (${deleteResult.durationMs}ms)`);
+        }
+    }
+    catch (error) {
+        // FALLBACK LEVEL 1
+        const errorMsg = error instanceof Error ? error.message : String(error);
+        pipelineLog.fallback({ component: "Reindex" }, 1, `deletePointsByPathsBatched failed: ${errorMsg}`);
+        console.error(`[Reindex] FALLBACK L1: deletePointsByPathsBatched failed for ${filesToDelete.length} paths:`, errorMsg);
+        try {
+            const fallbackStart = Date.now();
+            await qdrant.deletePointsByPaths(collectionName, filesToDelete);
+            pipelineLog.step({ component: "Reindex" }, "FALLBACK_L1_SUCCESS", {
+                durationMs: Date.now() - fallbackStart,
+                paths: filesToDelete.length,
+            });
+            console.error(`[Reindex] FALLBACK L1 SUCCESS: deletePointsByPaths completed in ${Date.now() - fallbackStart}ms`);
+        }
+        catch (fallbackError) {
+            // FALLBACK LEVEL 2
+            const fallbackErrorMsg = fallbackError instanceof Error ? fallbackError.message : String(fallbackError);
+            pipelineLog.fallback({ component: "Reindex" }, 2, `deletePointsByPaths failed: ${fallbackErrorMsg}`);
+            console.error(`[Reindex] FALLBACK L2: deletePointsByPaths also failed:`, fallbackErrorMsg);
+            console.error(`[Reindex] FALLBACK L2: Starting INDIVIDUAL deletions for ${filesToDelete.length} paths (SLOW!)`);
+            let deleted = 0;
+            let failed = 0;
+            const individualStart = Date.now();
+            for (const relativePath of filesToDelete) {
+                try {
+                    const filter = {
+                        must: [{ key: "relativePath", match: { value: relativePath } }],
+                    };
+                    await qdrant.deletePointsByFilter(collectionName, filter);
+                    deleted++;
+                }
+                catch (innerError) {
+                    failed++;
+                    if (isDebug()) {
+                        console.error(`[Reindex] FALLBACK L2: Failed to delete ${relativePath}:`, innerError);
+                    }
+                }
+            }
+            pipelineLog.step({ component: "Reindex" }, "FALLBACK_L2_COMPLETE", {
+                deleted,
+                failed,
+                durationMs: Date.now() - individualStart,
+            });
+            console.error(`[Reindex] FALLBACK L2 COMPLETE: ${deleted} deleted, ${failed} failed in ${Date.now() - individualStart}ms`);
+        }
+    }
+}
+//# sourceMappingURL=deletion-strategy.js.map

package/build/core/domains/ingest/sync/deletion-strategy.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"deletion-strategy.js","sourceRoot":"","sources":["../../../../../src/core/domains/ingest/sync/deletion-strategy.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAIH,OAAO,EAAE,WAAW,EAAE,MAAM,mCAAmC,CAAC;AAChE,OAAO,EAAE,OAAO,EAAE,MAAM,8BAA8B,CAAC;AAOvD;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,MAAqB,EACrB,cAAsB,EACtB,aAAuB,EACvB,YAA4B,EAC5B,gBAAmC;IAEnC,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAEvC,gBAAgB,EAAE,CAAC;QACjB,KAAK,EAAE,UAAU;QACjB,OAAO,EAAE,CAAC;QACV,KAAK,EAAE,aAAa,CAAC,MAAM;QAC3B,UAAU,EAAE,CAAC;QACb,OAAO,EAAE,2BAA2B,aAAa,CAAC,MAAM,WAAW;KACpE,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC,0BAA0B,CAAC,cAAc,EAAE,aAAa,EAAE;YAC1F,SAAS,EAAE,YAAY,CAAC,SAAS;YACjC,WAAW,EAAE,YAAY,CAAC,WAAW;YACrC,UAAU,EAAE,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE;gBAC7B,gBAAgB,EAAE,CAAC;oBACjB,KAAK,EAAE,UAAU;oBACjB,OAAO,EAAE,OAAO;oBAChB,KAAK;oBACL,UAAU,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC;oBACjD,OAAO,EAAE,wBAAwB,OAAO,IAAI,KAAK,WAAW;iBAC7D,CAAC,CAAC;YACL,CAAC;SACF,CAAC,CAAC;QAEH,IAAI,OAAO,EAAE,EAAE,CAAC;YACd,OAAO,CAAC,KAAK,CACX,qBAAqB,YAAY,CAAC,YAAY,aAAa,YAAY,CAAC,UAAU,aAAa,YAAY,CAAC,UAAU,KAAK,CAC5H,CAAC;QACJ,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,mBAAmB;QACnB,MAAM,QAAQ,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACxE,WAAW,CAAC,QAAQ,CAAC,EAAE,SAAS,EAAE,SAAS,EAAE,EAAE,CAAC,EAAE,sCAAsC,QAAQ,EAAE,CAAC,CAAC;QACpG,OAAO,CAAC,KAAK,CACX,gEAAgE,aAAa,CAAC,MAAM,SAAS,EAC7F,QAAQ,CACT,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACjC,MAAM,MAAM,CAAC,mBAAmB,CAAC,cAAc,EAAE,aAAa,CAAC,CAAC;YAChE,WAAW,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,SAAS,EAAE,EAAE,qBAAqB,EAAE;gBAChE,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,aAAa;gBACtC,KAAK,EAAE,aAAa,CAAC,MAAM;aAC5B,CAAC,CAAC;YACH,OAAO,CAAC,KAAK,CAAC,mEAAmE,IAAI,CAAC,GAAG,EAAE,GAAG,aAAa,IAAI,CAAC,CAAC;QACnH,CAAC;QAAC,OAAO,aAAa,EAAE,CAAC;YACvB,mBAAmB;YACnB,MAAM,gBAAgB,GAAG,aAAa,YAAY,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;YACxG,WAAW,CAAC,QAAQ,CAAC,EAAE,SAAS,EAAE,SAAS,EAAE,EAAE,CAAC,EAAE,+BAA+B,gBAAgB,EAAE,CAAC,CAAC;YACrG,OAAO,CAAC,KAAK,CAAC,yDAAyD,EAAE,gBAAgB,CAAC,CAAC;YAC3F,OAAO,CAAC,KAAK,CAAC,4DAA4D,aAAa,CAAC,MAAM,gBAAgB,CAAC,CAAC;YAEhH,IAAI,OAAO,GAAG,CAAC,CAAC;YAChB,IAAI,MAAM,GAAG,CAAC,CAAC;YACf,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAEnC,KAAK,MAAM,YAAY,IAAI,aAAa,EAAE,CAAC;gBACzC,IAAI,CAAC;oBACH,MAAM,MAAM,GAAG;wBACb,IAAI,EAAE,CAAC,EAAE,GAAG,EAAE,cAAc,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,CAAC;qBAChE,CAAC;oBACF,MAAM,MAAM,CAAC,oBAAoB,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;oBAC1D,OAAO,EAAE,CAAC;gBACZ,CAAC;gBAAC,OAAO,UAAU,EAAE,CAAC;oBACpB,MAAM,EAAE,CAAC;oBACT,IAAI,OAAO,EAAE,EAAE,CAAC;wBACd,OAAO,CAAC,KAAK,CAAC,2CAA2C,YAAY,GAAG,EAAE,UAAU,CAAC,CAAC;oBACxF,CAAC;gBACH,CAAC;YACH,CAAC;YAED,WAAW,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,SAAS,EAAE,EAAE,sBAAsB,EAAE;gBACjE,OAAO;gBACP,MAAM;gBACN,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,eAAe;aACzC,CAAC,CAAC;YACH,OAAO,CAAC,KAAK,CACX,mCAAmC,OAAO,aAAa,MAAM,cAAc,IAAI,CAAC,GAAG,EAAE,GAAG,eAAe,IAAI,CAC5G,CAAC;QACJ,CAAC;IACH,CAAC;AACH,CAAC"}

package/build/core/domains/ingest/sync/merkle.d.ts

@@ -0,0 +1,45 @@
+/**
+ * MerkleTree - Efficient change detection using Merkle trees
+ * Enables incremental updates by comparing file hashes
+ */
+export declare class MerkleNode {
+    hash: string;
+    left?: MerkleNode | undefined;
+    right?: MerkleNode | undefined;
+    constructor(hash: string, left?: MerkleNode | undefined, right?: MerkleNode | undefined);
+}
+export declare class MerkleTree {
+    root: MerkleNode | undefined;
+    /**
+     * Build Merkle tree from file hashes
+     * @param fileHashes Map of file path to content hash
+     */
+    build(fileHashes: Map<string, string>): void;
+    /**
+     * Recursively build tree from leaf nodes
+     */
+    private buildRecursive;
+    /**
+     * Compare two trees and return file differences
+     */
+    static compare(oldHashes: Map<string, string>, newHashes: Map<string, string>): {
+        added: string[];
+        modified: string[];
+        deleted: string[];
+    };
+    /**
+     * Get root hash (quick comparison)
+     */
+    getRootHash(): string | undefined;
+    /**
+     * Serialize tree for storage
+     */
+    serialize(): string;
+    private serializeNode;
+    /**
+     * Deserialize tree from storage
+     */
+    static deserialize(data: string): MerkleTree;
+    private deserializeNode;
+}
+//# sourceMappingURL=merkle.d.ts.map

package/build/core/domains/ingest/sync/merkle.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"merkle.d.ts","sourceRoot":"","sources":["../../../../../src/core/domains/ingest/sync/merkle.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,qBAAa,UAAU;IAEZ,IAAI,EAAE,MAAM;IACZ,IAAI,CAAC,EAAE,UAAU;IACjB,KAAK,CAAC,EAAE,UAAU;gBAFlB,IAAI,EAAE,MAAM,EACZ,IAAI,CAAC,EAAE,UAAU,YAAA,EACjB,KAAK,CAAC,EAAE,UAAU,YAAA;CAE5B;AAED,qBAAa,UAAU;IACrB,IAAI,EAAE,UAAU,GAAG,SAAS,CAAa;IAEzC;;;OAGG;IACH,KAAK,CAAC,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI;IAkB5C;;OAEG;IACH,OAAO,CAAC,cAAc;IAmBtB;;OAEG;IACH,MAAM,CAAC,OAAO,CACZ,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAC9B,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GAC7B;QAAE,KAAK,EAAE,MAAM,EAAE,CAAC;QAAC,QAAQ,EAAE,MAAM,EAAE,CAAC;QAAC,OAAO,EAAE,MAAM,EAAE,CAAA;KAAE;IAwB7D;;OAEG;IACH,WAAW,IAAI,MAAM,GAAG,SAAS;IAIjC;;OAEG;IACH,SAAS,IAAI,MAAM;IAMnB,OAAO,CAAC,aAAa;IASrB;;OAEG;IACH,MAAM,CAAC,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,UAAU;IAO5C,OAAO,CAAC,eAAe;CAIxB"}

package/build/core/domains/ingest/sync/merkle.js

@@ -0,0 +1,116 @@
+/**
+ * MerkleTree - Efficient change detection using Merkle trees
+ * Enables incremental updates by comparing file hashes
+ */
+import { createHash } from "node:crypto";
+export class MerkleNode {
+    hash;
+    left;
+    right;
+    constructor(hash, left, right) {
+        this.hash = hash;
+        this.left = left;
+        this.right = right;
+    }
+}
+export class MerkleTree {
+    root = undefined;
+    /**
+     * Build Merkle tree from file hashes
+     * @param fileHashes Map of file path to content hash
+     */
+    build(fileHashes) {
+        if (fileHashes.size === 0) {
+            this.root = undefined;
+            return;
+        }
+        // Sort files for deterministic tree structure
+        const leaves = Array.from(fileHashes.entries())
+            .sort(([pathA], [pathB]) => pathA.localeCompare(pathB))
+            .map(([path, hash]) => {
+            const combined = `${path}:${hash}`;
+            const leafHash = createHash("sha256").update(combined).digest("hex");
+            return new MerkleNode(leafHash);
+        });
+        this.root = this.buildRecursive(leaves);
+    }
+    /**
+     * Recursively build tree from leaf nodes
+     */
+    buildRecursive(nodes) {
+        if (nodes.length === 1) {
+            return nodes[0];
+        }
+        const parents = [];
+        for (let i = 0; i < nodes.length; i += 2) {
+            const left = nodes[i];
+            const right = nodes[i + 1] || left; // Duplicate if odd number
+            const combined = left.hash + right.hash;
+            const parentHash = createHash("sha256").update(combined).digest("hex");
+            parents.push(new MerkleNode(parentHash, left, right));
+        }
+        return this.buildRecursive(parents);
+    }
+    /**
+     * Compare two trees and return file differences
+     */
+    static compare(oldHashes, newHashes) {
+        const added = [];
+        const modified = [];
+        const deleted = [];
+        // Find added and modified files
+        for (const [path, hash] of newHashes) {
+            if (!oldHashes.has(path)) {
+                added.push(path);
+            }
+            else if (oldHashes.get(path) !== hash) {
+                modified.push(path);
+            }
+        }
+        // Find deleted files
+        for (const [path] of oldHashes) {
+            if (!newHashes.has(path)) {
+                deleted.push(path);
+            }
+        }
+        return { added, modified, deleted };
+    }
+    /**
+     * Get root hash (quick comparison)
+     */
+    getRootHash() {
+        return this.root?.hash;
+    }
+    /**
+     * Serialize tree for storage
+     */
+    serialize() {
+        return JSON.stringify({
+            root: this.serializeNode(this.root),
+        });
+    }
+    serializeNode(node) {
+        if (!node)
+            return null;
+        return {
+            hash: node.hash,
+            left: this.serializeNode(node.left),
+            right: this.serializeNode(node.right),
+        };
+    }
+    /**
+     * Deserialize tree from storage
+     */
+    static deserialize(data) {
+        const tree = new MerkleTree();
+        const obj = JSON.parse(data);
+        tree.root = tree.deserializeNode(obj.root);
+        return tree;
+    }
+    deserializeNode(obj) {
+        if (!obj)
+            return undefined;
+        return new MerkleNode(obj.hash, this.deserializeNode(obj.left), this.deserializeNode(obj.right));
+    }
+}
+//# sourceMappingURL=merkle.js.map

package/build/core/domains/ingest/sync/merkle.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"merkle.js","sourceRoot":"","sources":["../../../../../src/core/domains/ingest/sync/merkle.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,MAAM,OAAO,UAAU;IAEZ;IACA;IACA;IAHT,YACS,IAAY,EACZ,IAAiB,EACjB,KAAkB;QAFlB,SAAI,GAAJ,IAAI,CAAQ;QACZ,SAAI,GAAJ,IAAI,CAAa;QACjB,UAAK,GAAL,KAAK,CAAa;IACxB,CAAC;CACL;AAED,MAAM,OAAO,UAAU;IACrB,IAAI,GAA2B,SAAS,CAAC;IAEzC;;;OAGG;IACH,KAAK,CAAC,UAA+B;QACnC,IAAI,UAAU,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YAC1B,IAAI,CAAC,IAAI,GAAG,SAAS,CAAC;YACtB,OAAO;QACT,CAAC;QAED,8CAA8C;QAC9C,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;aAC5C,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;aACtD,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE;YACpB,MAAM,QAAQ,GAAG,GAAG,IAAI,IAAI,IAAI,EAAE,CAAC;YACnC,MAAM,QAAQ,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACrE,OAAO,IAAI,UAAU,CAAC,QAAQ,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;QAEL,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;IAC1C,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,KAAmB;QACxC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,MAAM,OAAO,GAAiB,EAAE,CAAC;QACjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;YACzC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACtB,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,0BAA0B;YAE9D,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC;YACxC,MAAM,UAAU,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAEvE,OAAO,CAAC,IAAI,CAAC,IAAI,UAAU,CAAC,UAAU,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;QACxD,CAAC;QAED,OAAO,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,OAAO,CACZ,SAA8B,EAC9B,SAA8B;QAE9B,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,MAAM,QAAQ,GAAa,EAAE,CAAC;QAC9B,MAAM,OAAO,GAAa,EAAE,CAAC;QAE7B,gCAAgC;QAChC,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,SAAS,EAAE,CAAC;YACrC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACzB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnB,CAAC;iBAAM,IAAI,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;gBACxC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;QAED,qBAAqB;QACrB,KAAK,MAAM,CAAC,IAAI,CAAC,IAAI,SAAS,EAAE,CAAC;YAC/B,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACzB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACrB,CAAC;QACH,CAAC;QAED,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,WAAW;QACT,OAAO,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC;IACzB,CAAC;IAED;;OAEG;IACH,SAAS;QACP,OAAO,IAAI,CAAC,SAAS,CAAC;YACpB,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC;SACpC,CAAC,CAAC;IACL,CAAC;IAEO,aAAa,CAAC,IAA4B;QAChD,IAAI,CAAC,IAAI;YAAE,OAAO,IAAI,CAAC;QACvB,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC;YACnC,KAAK,EAAE,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC;SACtC,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,WAAW,CAAC,IAAY;QAC7B,MAAM,IAAI,GAAG,IAAI,UAAU,EAAE,CAAC;QAC9B,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAoC,CAAC;QAChE,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC3C,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,eAAe,CAAC,GAA0B;QAChD,IAAI,CAAC,GAAG;YAAE,OAAO,SAAS,CAAC;QAC3B,OAAO,IAAI,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;IACnG,CAAC;CACF"}

package/build/core/domains/ingest/sync/migration.d.ts

@@ -0,0 +1,51 @@
+/**
+ * SnapshotMigrator - Migrates v1/v2 snapshots to v3 sharded format
+ *
+ * Migration strategy:
+ * 1. Detect old format (single JSON file)
+ * 2. Read and parse old snapshot
+ * 3. Create backup of old file
+ * 4. Write new sharded format
+ * 5. Delete old file (keep backup)
+ */
+/**
+ * Migration result
+ */
+export interface MigrationResult {
+    success: boolean;
+    filesCount: number;
+    skippedCount: number;
+    alreadyMigrated: boolean;
+    error?: string;
+}
+export declare class SnapshotMigrator {
+    private readonly snapshotDir;
+    private readonly collectionName;
+    private readonly codebasePath;
+    private readonly shardCount;
+    private readonly oldSnapshotPath;
+    private readonly newSnapshotDir;
+    private readonly backupPath;
+    constructor(snapshotDir: string, collectionName: string, codebasePath: string, shardCount?: number);
+    /**
+     * Check if migration is needed
+     */
+    needsMigration(): Promise<boolean>;
+    /**
+     * Migrate old snapshot to new format
+     */
+    migrate(): Promise<MigrationResult>;
+    /**
+     * Ensure snapshot is migrated (auto-migrate if needed)
+     */
+    ensureMigrated(): Promise<void>;
+    /**
+     * Check if file exists
+     */
+    private fileExists;
+    /**
+     * Get file stats safely
+     */
+    private getFileStats;
+}
+//# sourceMappingURL=migration.d.ts.map