rowpipe 2.0.0 → 2.1.0

package/dist/transforms/unique.d.ts

@@ -0,0 +1,13 @@
+import type { TransformFunction } from "../core/types.js";
+export interface UniqueOptions {
+    by?: string | string[];
+    keep?: "first" | "last";
+    memoryLimit?: number | string;
+    tempDir?: string;
+    batchSize?: number;
+}
+/**
+ * Deduplicates rows by key columns (or whole row) with spillable disk backup for large cardinalities.
+ */
+export declare function uniqueRows(options?: UniqueOptions): TransformFunction;
+//# sourceMappingURL=unique.d.ts.map

package/dist/transforms/unique.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"unique.d.ts","sourceRoot":"","sources":["../../src/transforms/unique.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAA8B,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAGtF,MAAM,WAAW,aAAa;IAC5B,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IACvB,IAAI,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;IACxB,WAAW,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC9B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AASD;;GAEG;AACH,wBAAgB,UAAU,CAAC,OAAO,GAAE,aAAkB,GAAG,iBAAiB,CAoFzE"}

package/dist/transforms/unique.js

@@ -0,0 +1,91 @@
+import { encodeCompositeKey } from "../diff/key.js";
+import { SpillableKeyStore } from "../utils/keystore.js";
+function getRowUniqueKey(row, byCols) {
+    if (byCols && byCols.length > 0) {
+        return encodeCompositeKey(row, byCols).encoded;
+    }
+    return JSON.stringify(row);
+}
+/**
+ * Deduplicates rows by key columns (or whole row) with spillable disk backup for large cardinalities.
+ */
+export function uniqueRows(options = {}) {
+    const keep = options.keep || "first";
+    const byCols = options.by
+        ? (Array.isArray(options.by) ? options.by : options.by.split(",").map((s) => s.trim()).filter(Boolean))
+        : undefined;
+    const effectiveBatchSize = options.batchSize || 1000;
+    return (stream) => {
+        return (async function* () {
+            const keyStore = new SpillableKeyStore({
+                memoryLimit: options.memoryLimit,
+                tempDir: options.tempDir,
+            });
+            try {
+                if (keep === "first") {
+                    // Streaming mode: Emit immediately on first encounter
+                    let currentBatchRows = [];
+                    let globalOffset = 0;
+                    for await (const batch of stream) {
+                        for (let i = 0; i < batch.rows.length; i++) {
+                            const row = batch.rows[i];
+                            const key = getRowUniqueKey(row, byCols);
+                            const isNew = await keyStore.add(key);
+                            if (isNew) {
+                                currentBatchRows.push(row);
+                                if (currentBatchRows.length >= effectiveBatchSize) {
+                                    yield {
+                                        rows: currentBatchRows,
+                                        offset: globalOffset,
+                                    };
+                                    globalOffset += currentBatchRows.length;
+                                    currentBatchRows = [];
+                                }
+                            }
+                        }
+                    }
+                    if (currentBatchRows.length > 0) {
+                        yield {
+                            rows: currentBatchRows,
+                            offset: globalOffset,
+                        };
+                    }
+                }
+                else {
+                    // keep: "last" mode: Retain latest row per key
+                    const latestRows = new Map();
+                    let globalOffset = 0;
+                    for await (const batch of stream) {
+                        for (let i = 0; i < batch.rows.length; i++) {
+                            const row = batch.rows[i];
+                            const key = getRowUniqueKey(row, byCols);
+                            latestRows.set(key, row);
+                        }
+                    }
+                    let currentBatchRows = [];
+                    for (const row of latestRows.values()) {
+                        currentBatchRows.push(row);
+                        if (currentBatchRows.length >= effectiveBatchSize) {
+                            yield {
+                                rows: currentBatchRows,
+                                offset: globalOffset,
+                            };
+                            globalOffset += currentBatchRows.length;
+                            currentBatchRows = [];
+                        }
+                    }
+                    if (currentBatchRows.length > 0) {
+                        yield {
+                            rows: currentBatchRows,
+                            offset: globalOffset,
+                        };
+                    }
+                }
+            }
+            finally {
+                await keyStore.close();
+            }
+        })();
+    };
+}
+//# sourceMappingURL=unique.js.map

package/dist/transforms/unique.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"unique.js","sourceRoot":"","sources":["../../src/transforms/unique.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AAEpD,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAUzD,SAAS,eAAe,CAAC,GAAQ,EAAE,MAAiB;IAClD,IAAI,MAAM,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAChC,OAAO,kBAAkB,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC;IACjD,CAAC;IACD,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;AAC7B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,UAAU,CAAC,UAAyB,EAAE;IACpD,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC;IACrC,MAAM,MAAM,GAAG,OAAO,CAAC,EAAE;QACvB,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACvG,CAAC,CAAC,SAAS,CAAC;IACd,MAAM,kBAAkB,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC;IAErD,OAAO,CAAC,MAAkB,EAAc,EAAE;QACxC,OAAO,CAAC,KAAK,SAAS,CAAC;YACrB,MAAM,QAAQ,GAAG,IAAI,iBAAiB,CAAC;gBACrC,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,OAAO,EAAE,OAAO,CAAC,OAAO;aACzB,CAAC,CAAC;YAEH,IAAI,CAAC;gBACH,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;oBACrB,sDAAsD;oBACtD,IAAI,gBAAgB,GAAU,EAAE,CAAC;oBACjC,IAAI,YAAY,GAAG,CAAC,CAAC;oBAErB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;wBACjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;4BAC3C,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC;4BAC3B,MAAM,GAAG,GAAG,eAAe,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;4BAEzC,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;4BACtC,IAAI,KAAK,EAAE,CAAC;gCACV,gBAAgB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gCAC3B,IAAI,gBAAgB,CAAC,MAAM,IAAI,kBAAkB,EAAE,CAAC;oCAClD,MAAM;wCACJ,IAAI,EAAE,gBAAgB;wCACtB,MAAM,EAAE,YAAY;qCACrB,CAAC;oCACF,YAAY,IAAI,gBAAgB,CAAC,MAAM,CAAC;oCACxC,gBAAgB,GAAG,EAAE,CAAC;gCACxB,CAAC;4BACH,CAAC;wBACH,CAAC;oBACH,CAAC;oBAED,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAChC,MAAM;4BACJ,IAAI,EAAE,gBAAgB;4BACtB,MAAM,EAAE,YAAY;yBACrB,CAAC;oBACJ,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,+CAA+C;oBAC/C,MAAM,UAAU,GAAG,IAAI,GAAG,EAAe,CAAC;oBAC1C,IAAI,YAAY,GAAG,CAAC,CAAC;oBAErB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;wBACjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;4BAC3C,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC;4BAC3B,MAAM,GAAG,GAAG,eAAe,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;4BACzC,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;wBAC3B,CAAC;oBACH,CAAC;oBAED,IAAI,gBAAgB,GAAU,EAAE,CAAC;oBACjC,KAAK,MAAM,GAAG,IAAI,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC;wBACtC,gBAAgB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;wBAC3B,IAAI,gBAAgB,CAAC,MAAM,IAAI,kBAAkB,EAAE,CAAC;4BAClD,MAAM;gCACJ,IAAI,EAAE,gBAAgB;gCACtB,MAAM,EAAE,YAAY;6BACrB,CAAC;4BACF,YAAY,IAAI,gBAAgB,CAAC,MAAM,CAAC;4BACxC,gBAAgB,GAAG,EAAE,CAAC;wBACxB,CAAC;oBACH,CAAC;oBAED,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAChC,MAAM;4BACJ,IAAI,EAAE,gBAAgB;4BACtB,MAAM,EAAE,YAAY;yBACrB,CAAC;oBACJ,CAAC;gBACH,CAAC;YACH,CAAC;oBAAS,CAAC;gBACT,MAAM,QAAQ,CAAC,KAAK,EAAE,CAAC;YACzB,CAAC;QACH,CAAC,CAAC,EAAE,CAAC;IACP,CAAC,CAAC;AACJ,CAAC"}

package/dist/utils/heap.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Generic Binary Heap implementation supporting MinHeap, MaxHeap, and Priority Queue.
+ */
+export type Comparator<T> = (a: T, b: T) => number;
+export declare class Heap<T> {
+    private data;
+    private comparator;
+    constructor(comparator: Comparator<T>);
+    get size(): number;
+    isEmpty(): boolean;
+    peek(): T | undefined;
+    push(value: T): void;
+    pop(): T | undefined;
+    /**
+     * Replaces the root element with a new element and re-balances the heap.
+     */
+    replace(value: T): T | undefined;
+    /**
+     * Drains all elements from the heap in sorted order.
+     */
+    drain(): T[];
+    private bubbleUp;
+    private bubbleDown;
+}
+//# sourceMappingURL=heap.d.ts.map

package/dist/utils/heap.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"heap.d.ts","sourceRoot":"","sources":["../../src/utils/heap.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,MAAM,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,KAAK,MAAM,CAAC;AAEnD,qBAAa,IAAI,CAAC,CAAC;IACjB,OAAO,CAAC,IAAI,CAAW;IACvB,OAAO,CAAC,UAAU,CAAgB;gBAEtB,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC;IAIrC,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED,OAAO,IAAI,OAAO;IAIlB,IAAI,IAAI,CAAC,GAAG,SAAS;IAIrB,IAAI,CAAC,KAAK,EAAE,CAAC,GAAG,IAAI;IAKpB,GAAG,IAAI,CAAC,GAAG,SAAS;IAWpB;;OAEG;IACH,OAAO,CAAC,KAAK,EAAE,CAAC,GAAG,CAAC,GAAG,SAAS;IAOhC;;OAEG;IACH,KAAK,IAAI,CAAC,EAAE;IAQZ,OAAO,CAAC,QAAQ;IAchB,OAAO,CAAC,UAAU;CA+BnB"}

package/dist/utils/heap.js

@@ -0,0 +1,90 @@
+export class Heap {
+    data = [];
+    comparator;
+    constructor(comparator) {
+        this.comparator = comparator;
+    }
+    get size() {
+        return this.data.length;
+    }
+    isEmpty() {
+        return this.data.length === 0;
+    }
+    peek() {
+        return this.data[0];
+    }
+    push(value) {
+        this.data.push(value);
+        this.bubbleUp(this.data.length - 1);
+    }
+    pop() {
+        if (this.data.length === 0)
+            return undefined;
+        const top = this.data[0];
+        const bottom = this.data.pop();
+        if (this.data.length > 0) {
+            this.data[0] = bottom;
+            this.bubbleDown(0);
+        }
+        return top;
+    }
+    /**
+     * Replaces the root element with a new element and re-balances the heap.
+     */
+    replace(value) {
+        const top = this.data[0];
+        this.data[0] = value;
+        this.bubbleDown(0);
+        return top;
+    }
+    /**
+     * Drains all elements from the heap in sorted order.
+     */
+    drain() {
+        const result = [];
+        while (!this.isEmpty()) {
+            result.push(this.pop());
+        }
+        return result;
+    }
+    bubbleUp(index) {
+        while (index > 0) {
+            const parentIdx = (index - 1) >>> 1;
+            if (this.comparator(this.data[index], this.data[parentIdx]) < 0) {
+                const temp = this.data[index];
+                this.data[index] = this.data[parentIdx];
+                this.data[parentIdx] = temp;
+                index = parentIdx;
+            }
+            else {
+                break;
+            }
+        }
+    }
+    bubbleDown(index) {
+        const length = this.data.length;
+        while (true) {
+            const leftIdx = (index << 1) + 1;
+            const rightIdx = leftIdx + 1;
+            let smallest = index;
+            if (leftIdx < length &&
+                this.comparator(this.data[leftIdx], this.data[smallest]) < 0) {
+                smallest = leftIdx;
+            }
+            if (rightIdx < length &&
+                this.comparator(this.data[rightIdx], this.data[smallest]) < 0) {
+                smallest = rightIdx;
+            }
+            if (smallest !== index) {
+                const temp = this.data[index];
+                this.data[index] = this.data[smallest];
+                this.data[smallest] = temp;
+                index = smallest;
+            }
+            else {
+                break;
+            }
+        }
+    }
+}
+//# sourceMappingURL=heap.js.map

package/dist/utils/heap.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"heap.js","sourceRoot":"","sources":["../../src/utils/heap.ts"],"names":[],"mappings":"AAKA,MAAM,OAAO,IAAI;IACP,IAAI,GAAQ,EAAE,CAAC;IACf,UAAU,CAAgB;IAElC,YAAY,UAAyB;QACnC,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IAC/B,CAAC;IAED,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;IAC1B,CAAC;IAED,OAAO;QACL,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC;IAChC,CAAC;IAED,IAAI;QACF,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACtB,CAAC;IAED,IAAI,CAAC,KAAQ;QACX,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACtB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACtC,CAAC;IAED,GAAG;QACD,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,SAAS,CAAC;QAC7C,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACzB,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAG,CAAC;QAChC,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzB,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC;YACtB,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QACrB,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED;;OAEG;IACH,OAAO,CAAC,KAAQ;QACd,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACzB,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC;QACrB,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QACnB,OAAO,GAAG,CAAC;IACb,CAAC;IAED;;OAEG;IACH,KAAK;QACH,MAAM,MAAM,GAAQ,EAAE,CAAC;QACvB,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;YACvB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAG,CAAC,CAAC;QAC3B,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,QAAQ,CAAC,KAAa;QAC5B,OAAO,KAAK,GAAG,CAAC,EAAE,CAAC;YACjB,MAAM,SAAS,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;YACpC,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAE,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,CAAE,CAAC,GAAG,CAAC,EAAE,CAAC;gBAClE,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAE,CAAC;gBAC/B,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAE,CAAC;gBACzC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC;gBAC5B,KAAK,GAAG,SAAS,CAAC;YACpB,CAAC;iBAAM,CAAC;gBACN,MAAM;YACR,CAAC;QACH,CAAC;IACH,CAAC;IAEO,UAAU,CAAC,KAAa;QAC9B,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;QAChC,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,OAAO,GAAG,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;YACjC,MAAM,QAAQ,GAAG,OAAO,GAAG,CAAC,CAAC;YAC7B,IAAI,QAAQ,GAAG,KAAK,CAAC;YAErB,IACE,OAAO,GAAG,MAAM;gBAChB,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAE,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAE,CAAC,GAAG,CAAC,EAC9D,CAAC;gBACD,QAAQ,GAAG,OAAO,CAAC;YACrB,CAAC;YAED,IACE,QAAQ,GAAG,MAAM;gBACjB,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAE,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAE,CAAC,GAAG,CAAC,EAC/D,CAAC;gBACD,QAAQ,GAAG,QAAQ,CAAC;YACtB,CAAC;YAED,IAAI,QAAQ,KAAK,KAAK,EAAE,CAAC;gBACvB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAE,CAAC;gBAC/B,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAE,CAAC;gBACxC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,IAAI,CAAC;gBAC3B,KAAK,GAAG,QAAQ,CAAC;YACnB,CAAC;iBAAM,CAAC;gBACN,MAAM;YACR,CAAC;QACH,CAAC;IACH,CAAC;CACF"}

package/dist/utils/keystore.d.ts

@@ -0,0 +1,80 @@
+export interface KeyStore {
+    has(key: string): Promise<boolean>;
+    add(key: string): Promise<boolean>;
+    delete(key: string): Promise<boolean>;
+    count(): number;
+    close(): Promise<void>;
+    getSpillStats?(): {
+        isSpilled: boolean;
+        spilledBytes: number;
+        entryCount: number;
+    };
+}
+export declare class MemoryKeyStore implements KeyStore {
+    private set;
+    private estimatedBytes;
+    has(key: string): Promise<boolean>;
+    add(key: string): Promise<boolean>;
+    delete(key: string): Promise<boolean>;
+    count(): number;
+    getEstimatedBytes(): number;
+    keys(): AsyncIterable<string>;
+    close(): Promise<void>;
+    getSpillStats(): {
+        isSpilled: boolean;
+        spilledBytes: number;
+        entryCount: number;
+    };
+}
+/**
+ * Disk-backed KeyStore using 256 hash partitions.
+ */
+export declare class DiskKeyStore implements KeyStore {
+    private tempDir;
+    private partitionSets;
+    private totalCount;
+    private spilledBytes;
+    private isClosed;
+    constructor(customTempDir?: string);
+    private ensureDir;
+    private getPartition;
+    private getPartitionFilePath;
+    private loadPartition;
+    private flushPartition;
+    has(key: string): Promise<boolean>;
+    add(key: string): Promise<boolean>;
+    delete(key: string): Promise<boolean>;
+    count(): number;
+    getSpillStats(): {
+        isSpilled: boolean;
+        spilledBytes: number;
+        entryCount: number;
+    };
+    close(): Promise<void>;
+}
+/**
+ * Hybrid Spillable KeyStore.
+ */
+export declare class SpillableKeyStore implements KeyStore {
+    private memoryStore;
+    private diskStore?;
+    private memoryLimitBytes;
+    private tempDir?;
+    private isSpilled;
+    constructor(options?: {
+        memoryLimit?: number | string;
+        tempDir?: string;
+    });
+    private spillToDisk;
+    has(key: string): Promise<boolean>;
+    add(key: string): Promise<boolean>;
+    delete(key: string): Promise<boolean>;
+    count(): number;
+    getSpillStats(): {
+        isSpilled: boolean;
+        spilledBytes: number;
+        entryCount: number;
+    };
+    close(): Promise<void>;
+}
+//# sourceMappingURL=keystore.d.ts.map

package/dist/utils/keystore.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"keystore.d.ts","sourceRoot":"","sources":["../../src/utils/keystore.ts"],"names":[],"mappings":"AAKA,MAAM,WAAW,QAAQ;IACvB,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IACnC,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IACnC,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IACtC,KAAK,IAAI,MAAM,CAAC;IAChB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACvB,aAAa,CAAC,IAAI;QAAE,SAAS,EAAE,OAAO,CAAC;QAAC,YAAY,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC;CACpF;AAED,qBAAa,cAAe,YAAW,QAAQ;IAC7C,OAAO,CAAC,GAAG,CAAqB;IAChC,OAAO,CAAC,cAAc,CAAK;IAErB,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAIlC,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IASlC,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAQ3C,KAAK,IAAI,MAAM;IAIf,iBAAiB,IAAI,MAAM;IAIpB,IAAI,IAAI,aAAa,CAAC,MAAM,CAAC;IAM9B,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAK5B,aAAa;;;;;CAGd;AAED;;GAEG;AACH,qBAAa,YAAa,YAAW,QAAQ;IAC3C,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,aAAa,CAAkC;IACvD,OAAO,CAAC,UAAU,CAAK;IACvB,OAAO,CAAC,YAAY,CAAK;IACzB,OAAO,CAAC,QAAQ,CAAS;gBAEb,aAAa,CAAC,EAAE,MAAM;YAQpB,SAAS;IAIvB,OAAO,CAAC,YAAY;IAQpB,OAAO,CAAC,oBAAoB;YAId,aAAa;YAgCb,cAAc;IAUtB,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAMlC,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAWlC,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAU3C,KAAK,IAAI,MAAM;IAIf,aAAa;;;;;IAIP,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAU7B;AAED;;GAEG;AACH,qBAAa,iBAAkB,YAAW,QAAQ;IAChD,OAAO,CAAC,WAAW,CAAwB;IAC3C,OAAO,CAAC,SAAS,CAAC,CAAe;IACjC,OAAO,CAAC,gBAAgB,CAAS;IACjC,OAAO,CAAC,OAAO,CAAC,CAAS;IACzB,OAAO,CAAC,SAAS,CAAS;gBAEd,OAAO,GAAE;QAAE,WAAW,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAO;YAK/D,WAAW;IAUnB,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAOlC,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAYlC,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAO3C,KAAK,IAAI,MAAM;IAOf,aAAa;;;;;IAOP,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAM7B"}

package/dist/utils/keystore.js

@@ -0,0 +1,217 @@
+import { promises as fsPromises } from "node:fs";
+import * as os from "node:os";
+import * as path from "node:path";
+import { parseMemoryLimit } from "../diff/storage/spillable-index.js";
+export class MemoryKeyStore {
+    set = new Set();
+    estimatedBytes = 0;
+    async has(key) {
+        return this.set.has(key);
+    }
+    async add(key) {
+        if (this.set.has(key)) {
+            return false;
+        }
+        this.set.add(key);
+        this.estimatedBytes += key.length * 2 + 16;
+        return true;
+    }
+    async delete(key) {
+        if (this.set.delete(key)) {
+            this.estimatedBytes = Math.max(0, this.estimatedBytes - (key.length * 2 + 16));
+            return true;
+        }
+        return false;
+    }
+    count() {
+        return this.set.size;
+    }
+    getEstimatedBytes() {
+        return this.estimatedBytes;
+    }
+    async *keys() {
+        for (const key of this.set) {
+            yield key;
+        }
+    }
+    async close() {
+        this.set.clear();
+        this.estimatedBytes = 0;
+    }
+    getSpillStats() {
+        return { isSpilled: false, spilledBytes: 0, entryCount: this.set.size };
+    }
+}
+/**
+ * Disk-backed KeyStore using 256 hash partitions.
+ */
+export class DiskKeyStore {
+    tempDir;
+    partitionSets = new Map();
+    totalCount = 0;
+    spilledBytes = 0;
+    isClosed = false;
+    constructor(customTempDir) {
+        const baseDir = customTempDir || os.tmpdir();
+        this.tempDir = path.join(baseDir, `rowpipe-keystore-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`);
+    }
+    async ensureDir() {
+        await fsPromises.mkdir(this.tempDir, { recursive: true });
+    }
+    getPartition(key) {
+        let hash = 0;
+        for (let i = 0; i < key.length; i++) {
+            hash = (hash * 31 + key.charCodeAt(i)) >>> 0;
+        }
+        return hash % 256;
+    }
+    getPartitionFilePath(partition) {
+        return path.join(this.tempDir, `part_${partition}.txt`);
+    }
+    async loadPartition(partition) {
+        if (this.partitionSets.has(partition)) {
+            return this.partitionSets.get(partition);
+        }
+        const set = new Set();
+        const filePath = this.getPartitionFilePath(partition);
+        try {
+            const content = await fsPromises.readFile(filePath, "utf-8");
+            const lines = content.split("\n");
+            for (const line of lines) {
+                if (line.length > 0) {
+                    set.add(line);
+                }
+            }
+        }
+        catch {
+            // File does not exist yet
+        }
+        // Keep active partitions bounded in memory
+        if (this.partitionSets.size >= 16) {
+            const firstKey = this.partitionSets.keys().next().value;
+            if (firstKey !== undefined) {
+                await this.flushPartition(firstKey);
+                this.partitionSets.delete(firstKey);
+            }
+        }
+        this.partitionSets.set(partition, set);
+        return set;
+    }
+    async flushPartition(partition) {
+        const set = this.partitionSets.get(partition);
+        if (!set)
+            return;
+        await this.ensureDir();
+        const filePath = this.getPartitionFilePath(partition);
+        const content = Array.from(set).join("\n") + "\n";
+        await fsPromises.writeFile(filePath, content, "utf-8");
+        this.spilledBytes += Buffer.byteLength(content, "utf-8");
+    }
+    async has(key) {
+        const part = this.getPartition(key);
+        const set = await this.loadPartition(part);
+        return set.has(key);
+    }
+    async add(key) {
+        const part = this.getPartition(key);
+        const set = await this.loadPartition(part);
+        if (set.has(key)) {
+            return false;
+        }
+        set.add(key);
+        this.totalCount++;
+        return true;
+    }
+    async delete(key) {
+        const part = this.getPartition(key);
+        const set = await this.loadPartition(part);
+        if (set.delete(key)) {
+            this.totalCount--;
+            return true;
+        }
+        return false;
+    }
+    count() {
+        return this.totalCount;
+    }
+    getSpillStats() {
+        return { isSpilled: true, spilledBytes: this.spilledBytes, entryCount: this.totalCount };
+    }
+    async close() {
+        if (this.isClosed)
+            return;
+        this.isClosed = true;
+        this.partitionSets.clear();
+        try {
+            await fsPromises.rm(this.tempDir, { recursive: true, force: true });
+        }
+        catch {
+            // Ignore cleanup error
+        }
+    }
+}
+/**
+ * Hybrid Spillable KeyStore.
+ */
+export class SpillableKeyStore {
+    memoryStore = new MemoryKeyStore();
+    diskStore;
+    memoryLimitBytes;
+    tempDir;
+    isSpilled = false;
+    constructor(options = {}) {
+        this.memoryLimitBytes = parseMemoryLimit(options.memoryLimit, 256 * 1024 * 1024);
+        this.tempDir = options.tempDir;
+    }
+    async spillToDisk() {
+        if (this.isSpilled)
+            return;
+        this.diskStore = new DiskKeyStore(this.tempDir);
+        for await (const key of this.memoryStore.keys()) {
+            await this.diskStore.add(key);
+        }
+        await this.memoryStore.close();
+        this.isSpilled = true;
+    }
+    async has(key) {
+        if (this.isSpilled && this.diskStore) {
+            return this.diskStore.has(key);
+        }
+        return this.memoryStore.has(key);
+    }
+    async add(key) {
+        if (this.isSpilled && this.diskStore) {
+            return this.diskStore.add(key);
+        }
+        const added = await this.memoryStore.add(key);
+        if (this.memoryStore.getEstimatedBytes() >= this.memoryLimitBytes) {
+            await this.spillToDisk();
+        }
+        return added;
+    }
+    async delete(key) {
+        if (this.isSpilled && this.diskStore) {
+            return this.diskStore.delete(key);
+        }
+        return this.memoryStore.delete(key);
+    }
+    count() {
+        if (this.isSpilled && this.diskStore) {
+            return this.diskStore.count();
+        }
+        return this.memoryStore.count();
+    }
+    getSpillStats() {
+        if (this.isSpilled && this.diskStore) {
+            return this.diskStore.getSpillStats();
+        }
+        return this.memoryStore.getSpillStats();
+    }
+    async close() {
+        await this.memoryStore.close();
+        if (this.diskStore) {
+            await this.diskStore.close();
+        }
+    }
+}
+//# sourceMappingURL=keystore.js.map

package/dist/utils/keystore.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"keystore.js","sourceRoot":"","sources":["../../src/utils/keystore.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,IAAI,UAAU,EAAE,MAAM,SAAS,CAAC;AACjD,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAC;AAWtE,MAAM,OAAO,cAAc;IACjB,GAAG,GAAG,IAAI,GAAG,EAAU,CAAC;IACxB,cAAc,GAAG,CAAC,CAAC;IAE3B,KAAK,CAAC,GAAG,CAAC,GAAW;QACnB,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,GAAW;QACnB,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACtB,OAAO,KAAK,CAAC;QACf,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAClB,IAAI,CAAC,cAAc,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,GAAG,EAAE,CAAC;QAC3C,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,GAAW;QACtB,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YACzB,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,cAAc,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;YAC/E,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,KAAK;QACH,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;IACvB,CAAC;IAED,iBAAiB;QACf,OAAO,IAAI,CAAC,cAAc,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,CAAC,IAAI;QACT,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;YAC3B,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;QACjB,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC;IAC1B,CAAC;IAED,aAAa;QACX,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;IAC1E,CAAC;CACF;AAED;;GAEG;AACH,MAAM,OAAO,YAAY;IACf,OAAO,CAAS;IAChB,aAAa,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC/C,UAAU,GAAG,CAAC,CAAC;IACf,YAAY,GAAG,CAAC,CAAC;IACjB,QAAQ,GAAG,KAAK,CAAC;IAEzB,YAAY,aAAsB;QAChC,MAAM,OAAO,GAAG,aAAa,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC;QAC7C,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,IAAI,CACtB,OAAO,EACP,oBAAoB,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAC3E,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,SAAS;QACrB,MAAM,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5D,CAAC;IAEO,YAAY,CAAC,GAAW;QAC9B,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACpC,IAAI,GAAG,CAAC,IAAI,GAAG,EAAE,GAAG,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QAC/C,CAAC;QACD,OAAO,IAAI,GAAG,GAAG,CAAC;IACpB,CAAC;IAEO,oBAAoB,CAAC,SAAiB;QAC5C,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,SAAS,MAAM,CAAC,CAAC;IAC1D,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,SAAiB;QAC3C,IAAI,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;YACtC,OAAO,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAE,CAAC;QAC5C,CAAC;QAED,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAC;QAC9B,MAAM,QAAQ,GAAG,IAAI,CAAC,oBAAoB,CAAC,SAAS,CAAC,CAAC;QACtD,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAC7D,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAClC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACpB,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBAChB,CAAC;YACH,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,0BAA0B;QAC5B,CAAC;QAED,2CAA2C;QAC3C,IAAI,IAAI,CAAC,aAAa,CAAC,IAAI,IAAI,EAAE,EAAE,CAAC;YAClC,MAAM,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;YACxD,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;gBAC3B,MAAM,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;gBACpC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;QAED,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;QACvC,OAAO,GAAG,CAAC;IACb,CAAC;IAEO,KAAK,CAAC,cAAc,CAAC,SAAiB;QAC5C,MAAM,GAAG,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAC9C,IAAI,CAAC,GAAG;YAAE,OAAO;QACjB,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;QACvB,MAAM,QAAQ,GAAG,IAAI,CAAC,oBAAoB,CAAC,SAAS,CAAC,CAAC;QACtD,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;QAClD,MAAM,UAAU,CAAC,SAAS,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;QACvD,IAAI,CAAC,YAAY,IAAI,MAAM,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC3D,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,GAAW;QACnB,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;QACpC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;QAC3C,OAAO,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACtB,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,GAAW;QACnB,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;QACpC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;QAC3C,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACjB,OAAO,KAAK,CAAC;QACf,CAAC;QACD,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACb,IAAI,CAAC,UAAU,EAAE,CAAC;QAClB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,GAAW;QACtB,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;QACpC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;QAC3C,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YACpB,IAAI,CAAC,UAAU,EAAE,CAAC;YAClB,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,KAAK;QACH,OAAO,IAAI,CAAC,UAAU,CAAC;IACzB,CAAC;IAED,aAAa;QACX,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC;IAC3F,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,QAAQ;YAAE,OAAO;QAC1B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,CAAC;QAC3B,IAAI,CAAC;YACH,MAAM,UAAU,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACtE,CAAC;QAAC,MAAM,CAAC;YACP,uBAAuB;QACzB,CAAC;IACH,CAAC;CACF;AAED;;GAEG;AACH,MAAM,OAAO,iBAAiB;IACpB,WAAW,GAAG,IAAI,cAAc,EAAE,CAAC;IACnC,SAAS,CAAgB;IACzB,gBAAgB,CAAS;IACzB,OAAO,CAAU;IACjB,SAAS,GAAG,KAAK,CAAC;IAE1B,YAAY,UAA+D,EAAE;QAC3E,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC,OAAO,CAAC,WAAW,EAAE,GAAG,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC;QACjF,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IACjC,CAAC;IAEO,KAAK,CAAC,WAAW;QACvB,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO;QAC3B,IAAI,CAAC,SAAS,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAChD,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,EAAE,CAAC;YAChD,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAChC,CAAC;QACD,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;QAC/B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,GAAW;QACnB,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACrC,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACjC,CAAC;QACD,OAAO,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACnC,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,GAAW;QACnB,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACrC,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACjC,CAAC;QAED,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC9C,IAAI,IAAI,CAAC,WAAW,CAAC,iBAAiB,EAAE,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAClE,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QAC3B,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,GAAW;QACtB,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACrC,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACpC,CAAC;QACD,OAAO,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACtC,CAAC;IAED,KAAK;QACH,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACrC,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;QAChC,CAAC;QACD,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;IAClC,CAAC;IAED,aAAa;QACX,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACrC,OAAO,IAAI,CAAC,SAAS,CAAC,aAAa,EAAE,CAAC;QACxC,CAAC;QACD,OAAO,IAAI,CAAC,WAAW,CAAC,aAAa,EAAE,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;QAC/B,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;QAC/B,CAAC;IACH,CAAC;CACF"}

package/dist/utils/ring-buffer.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Fixed-capacity circular buffer with O(1) push and bounded O(N) memory complexity.
+ */
+export declare class RingBuffer<T> {
+    private buffer;
+    private capacity;
+    private head;
+    private length;
+    constructor(capacity: number);
+    get size(): number;
+    push(item: T): void;
+    toArray(): T[];
+    clear(): void;
+}
+//# sourceMappingURL=ring-buffer.d.ts.map

package/dist/utils/ring-buffer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ring-buffer.d.ts","sourceRoot":"","sources":["../../src/utils/ring-buffer.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,qBAAa,UAAU,CAAC,CAAC;IACvB,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,IAAI,CAAK;IACjB,OAAO,CAAC,MAAM,CAAK;gBAEP,QAAQ,EAAE,MAAM;IAK5B,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED,IAAI,CAAC,IAAI,EAAE,CAAC,GAAG,IAAI;IAUnB,OAAO,IAAI,CAAC,EAAE;IASd,KAAK,IAAI,IAAI;CAKd"}

package/dist/utils/ring-buffer.js

@@ -0,0 +1,40 @@
+/**
+ * Fixed-capacity circular buffer with O(1) push and bounded O(N) memory complexity.
+ */
+export class RingBuffer {
+    buffer;
+    capacity;
+    head = 0;
+    length = 0;
+    constructor(capacity) {
+        this.capacity = Math.max(1, capacity);
+        this.buffer = new Array(this.capacity);
+    }
+    get size() {
+        return this.length;
+    }
+    push(item) {
+        const index = (this.head + this.length) % this.capacity;
+        this.buffer[index] = item;
+        if (this.length < this.capacity) {
+            this.length++;
+        }
+        else {
+            this.head = (this.head + 1) % this.capacity;
+        }
+    }
+    toArray() {
+        const result = new Array(this.length);
+        for (let i = 0; i < this.length; i++) {
+            const index = (this.head + i) % this.capacity;
+            result[i] = this.buffer[index];
+        }
+        return result;
+    }
+    clear() {
+        this.head = 0;
+        this.length = 0;
+        this.buffer.fill(undefined);
+    }
+}
+//# sourceMappingURL=ring-buffer.js.map

package/dist/utils/ring-buffer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ring-buffer.js","sourceRoot":"","sources":["../../src/utils/ring-buffer.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,OAAO,UAAU;IACb,MAAM,CAAuB;IAC7B,QAAQ,CAAS;IACjB,IAAI,GAAG,CAAC,CAAC;IACT,MAAM,GAAG,CAAC,CAAC;IAEnB,YAAY,QAAgB;QAC1B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;QACtC,IAAI,CAAC,MAAM,GAAG,IAAI,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACzC,CAAC;IAED,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED,IAAI,CAAC,IAAO;QACV,MAAM,KAAK,GAAG,CAAC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC;QACxD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC;QAC1B,IAAI,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;YAChC,IAAI,CAAC,MAAM,EAAE,CAAC;QAChB,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC9C,CAAC;IACH,CAAC;IAED,OAAO;QACL,MAAM,MAAM,GAAQ,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,MAAM,KAAK,GAAG,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC;YAC9C,MAAM,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAM,CAAC;QACtC,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK;QACH,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC;QACd,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;QAChB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC9B,CAAC;CACF"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rowpipe",
-  "version": "2.0.0",
+  "version": "2.1.0",
   "description": "Stream-first tabular data toolkit for CSV, TSV, JSON, JSONL, XLSX, Apache Parquet, and Markdown",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -20,7 +20,8 @@
     "test:watch": "vitest",
     "bench": "tsx benchmarks/memory-bench.ts",
     "benchmark:diff": "tsx benchmarks/diff-bench.ts",
-    "benchmark:files": "tsx benchmarks/files-bench.ts"
+    "benchmark:files": "tsx benchmarks/files-bench.ts",
+    "benchmark:ops": "tsx benchmarks/operations-bench.ts"
   },
   "keywords": [
     "csv",

package/skills/rowpipe/SKILL.md

@@ -20,6 +20,15 @@ Rowpipe is a high-performance, stream-first command-line toolkit and Node.js lib
 | **Map** | `rowpipe map <file> "<col1=expr1>" "<col2=expr2>"` | Derive and compute columns per row with arithmetic/JSON dot-notation |
 | **Filter** | `rowpipe filter <file> "<expression>"` | Filter rows using safe expression engine and pipe syntax (`\|`) |
 | **Reduce** | `rowpipe reduce <file> "<agg1=func(col)>" [--by <cols>]` | Global & group-by streaming aggregations (`sum`, `avg`, `min`, `max`, `count`, `countDistinct`, `stddev`) |
+| **Limit / Head** | `rowpipe limit <file> <n>` / `rowpipe head <file> [-n <n>]` | Emit first N rows with early upstream stream cancellation |
+| **Offset** | `rowpipe offset <file> <n>` | Skip first N rows without scanning remainder |
+| **Tail** | `rowpipe tail <file> [-n <n>]` | Emit last N rows using bounded $O(N)$ circular ring buffer |
+| **Sort** | `rowpipe sort <file> --by <cols> [--memory-limit <m>]` | Multi-column typed external merge sort with disk run spilling and $K$-way min-heap |
+| **Top** | `rowpipe top <file> --by <cols> [-n <n>]` | Extract Top-K largest/smallest records using bounded $O(K)$ heap |
+| **Unique** | `rowpipe unique <file> [--by <cols>] [--keep first\|last]` | Deduplicate rows by key columns or entire rows with spillable hash key store |
+| **Count** | `rowpipe count <file> [--by <cols>] [--distinct <col>]` | Ultra-fast row total, distinct count (HyperLogLog), or group-count |
+| **Group** | `rowpipe group <file> --by <cols> [--count] [--sum <c>]` | Group tabular streams with hash accumulation and multi-column aggregations |
+| **Explain** | `rowpipe explain <file> [pipeline flags]` | Visualize planned execution pipeline and rule-based optimizations |
 | **Select** | `rowpipe select <file> <col1,col2,...>` | Project subset of columns in order |
 | **Rename** | `rowpipe rename <file> <old1=new1> <old2=new2>` | Rename columns in stream |
 | **Cast** | `rowpipe cast <file> <col1:type> [--on-error null]` | Stream cast types (`integer`, `number`, `boolean`, `date`) |