rowpipe 1.2.0 → 2.1.0

package/dist/transforms/sort/comparator.js

@@ -0,0 +1,131 @@
+/**
+ * Parses sort specifications from strings (e.g. "country,revenue:desc", "age:desc") or objects.
+ */
+export function parseSortSpecs(input, defaults = {}) {
+    const fallbackDir = defaults.defaultDirection || "asc";
+    if (Array.isArray(input) && input.length > 0 && typeof input[0] === "object") {
+        return input.map((spec) => ({
+            ...spec,
+            direction: spec.direction || fallbackDir,
+            nulls: spec.nulls || defaults.nulls || "last",
+            ignoreCase: spec.ignoreCase ?? defaults.ignoreCase ?? false,
+            natural: spec.natural ?? defaults.natural ?? false,
+        }));
+    }
+    const rawSpecs = [];
+    if (typeof input === "string") {
+        rawSpecs.push(...input.split(",").map((s) => s.trim()).filter(Boolean));
+    }
+    else if (Array.isArray(input)) {
+        for (const item of input) {
+            if (typeof item === "string") {
+                rawSpecs.push(...item.split(",").map((s) => s.trim()).filter(Boolean));
+            }
+        }
+    }
+    return rawSpecs.map((spec) => {
+        const parts = spec.split(":");
+        const column = parts[0].trim();
+        let direction;
+        if (parts[1]) {
+            const dirStr = parts[1].trim().toLowerCase();
+            direction = dirStr === "desc" ? "desc" : "asc";
+        }
+        else {
+            direction = fallbackDir;
+        }
+        return {
+            column,
+            direction,
+            nulls: defaults.nulls || "last",
+            ignoreCase: defaults.ignoreCase ?? false,
+            natural: defaults.natural ?? false,
+        };
+    });
+}
+/**
+ * Compares two primitive values with typed semantics (numbers, dates, booleans, strings).
+ */
+export function compareValues(a, b, spec) {
+    const isANull = a === null || a === undefined || a === "";
+    const isBNull = b === null || b === undefined || b === "";
+    if (isANull && isBNull)
+        return 0;
+    if (isANull)
+        return spec.nulls === "first" ? -1 : 1;
+    if (isBNull)
+        return spec.nulls === "first" ? 1 : -1;
+    // Number comparison (including numeric strings)
+    const isANum = typeof a === "number" || (typeof a === "string" && /^-?\d+(\.\d+)?$/.test(a.trim()));
+    const isBNum = typeof b === "number" || (typeof b === "string" && /^-?\d+(\.\d+)?$/.test(b.trim()));
+    if (isANum && isBNum) {
+        const numA = typeof a === "number" ? a : Number.parseFloat(a);
+        const numB = typeof b === "number" ? b : Number.parseFloat(b);
+        if (!Number.isNaN(numA) && !Number.isNaN(numB)) {
+            return numA < numB ? -1 : numA > numB ? 1 : 0;
+        }
+    }
+    // Boolean comparison
+    if (typeof a === "boolean" || typeof b === "boolean") {
+        const boolA = Boolean(a);
+        const boolB = Boolean(b);
+        return boolA === boolB ? 0 : boolA ? 1 : -1;
+    }
+    // Date comparison
+    if (a instanceof Date && b instanceof Date) {
+        return a.getTime() - b.getTime();
+    }
+    // String comparison
+    const strA = String(a);
+    const strB = String(b);
+    if (spec.natural) {
+        return strA.localeCompare(strB, undefined, {
+            numeric: true,
+            sensitivity: spec.ignoreCase ? "base" : "variant",
+        });
+    }
+    if (spec.ignoreCase) {
+        const lowerA = strA.toLowerCase();
+        const lowerB = strB.toLowerCase();
+        return lowerA < lowerB ? -1 : lowerA > lowerB ? 1 : 0;
+    }
+    return strA < strB ? -1 : strA > strB ? 1 : 0;
+}
+/**
+ * Creates a deterministic, multi-column, stable row comparator.
+ */
+export function createRowComparator(specs) {
+    return (aObj, bObj) => {
+        const a = aObj.row;
+        const b = bObj.row;
+        for (let i = 0; i < specs.length; i++) {
+            const spec = specs[i];
+            const valA = a[spec.column];
+            const valB = b[spec.column];
+            const cmp = compareValues(valA, valB, spec);
+            if (cmp !== 0) {
+                return spec.direction === "desc" ? -cmp : cmp;
+            }
+        }
+        // Preserve original insertion sequence for stability
+        return aObj.seq - bObj.seq;
+    };
+}
+/**
+ * Creates a raw Row comparator (without seq).
+ */
+export function createRawRowComparator(specs) {
+    return (a, b) => {
+        for (let i = 0; i < specs.length; i++) {
+            const spec = specs[i];
+            const valA = a[spec.column];
+            const valB = b[spec.column];
+            const cmp = compareValues(valA, valB, spec);
+            if (cmp !== 0) {
+                return spec.direction === "desc" ? -cmp : cmp;
+            }
+        }
+        return 0;
+    };
+}
+//# sourceMappingURL=comparator.js.map

package/dist/transforms/sort/comparator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"comparator.js","sourceRoot":"","sources":["../../../src/transforms/sort/comparator.ts"],"names":[],"mappings":"AAyBA;;GAEG;AACH,MAAM,UAAU,cAAc,CAC5B,KAAwC,EACxC,WAKI,EAAE;IAEN,MAAM,WAAW,GAAG,QAAQ,CAAC,gBAAgB,IAAI,KAAK,CAAC;IACvD,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,QAAQ,EAAE,CAAC;QAC7E,OAAQ,KAAuB,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAC7C,GAAG,IAAI;YACP,SAAS,EAAE,IAAI,CAAC,SAAS,IAAI,WAAW;YACxC,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,QAAQ,CAAC,KAAK,IAAI,MAAM;YAC7C,UAAU,EAAE,IAAI,CAAC,UAAU,IAAI,QAAQ,CAAC,UAAU,IAAI,KAAK;YAC3D,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,QAAQ,CAAC,OAAO,IAAI,KAAK;SACnD,CAAC,CAAC,CAAC;IACN,CAAC;IAED,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,QAAQ,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;IAC1E,CAAC;SAAM,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QAChC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAC7B,QAAQ,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;YACzE,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC9B,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,CAAC;QAChC,IAAI,SAAyB,CAAC;QAC9B,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;YACb,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YAC7C,SAAS,GAAG,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC;QACjD,CAAC;aAAM,CAAC;YACN,SAAS,GAAG,WAAW,CAAC;QAC1B,CAAC;QAED,OAAO;YACL,MAAM;YACN,SAAS;YACT,KAAK,EAAE,QAAQ,CAAC,KAAK,IAAI,MAAM;YAC/B,UAAU,EAAE,QAAQ,CAAC,UAAU,IAAI,KAAK;YACxC,OAAO,EAAE,QAAQ,CAAC,OAAO,IAAI,KAAK;SACnC,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,aAAa,CAC3B,CAAU,EACV,CAAU,EACV,IAAiB;IAEjB,MAAM,OAAO,GAAG,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,EAAE,CAAC;IAC1D,MAAM,OAAO,GAAG,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,EAAE,CAAC;IAE1D,IAAI,OAAO,IAAI,OAAO;QAAE,OAAO,CAAC,CAAC;IACjC,IAAI,OAAO;QAAE,OAAO,IAAI,CAAC,KAAK,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACpD,IAAI,OAAO;QAAE,OAAO,IAAI,CAAC,KAAK,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAEpD,gDAAgD;IAChD,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IACpG,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAEpG,IAAI,MAAM,IAAI,MAAM,EAAE,CAAC;QACrB,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAW,CAAC,CAAC;QACxE,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAW,CAAC,CAAC;QACxE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YAC/C,OAAO,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IAED,qBAAqB;IACrB,IAAI,OAAO,CAAC,KAAK,SAAS,IAAI,OAAO,CAAC,KAAK,SAAS,EAAE,CAAC;QACrD,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QACzB,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QACzB,OAAO,KAAK,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9C,CAAC;IAED,kBAAkB;IAClB,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,YAAY,IAAI,EAAE,CAAC;QAC3C,OAAO,CAAC,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,OAAO,EAAE,CAAC;IACnC,CAAC;IAED,oBAAoB;IACpB,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;IACvB,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;IAEvB,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;QACjB,OAAO,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,SAAS,EAAE;YACzC,OAAO,EAAE,IAAI;YACb,WAAW,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;SAClD,CAAC,CAAC;IACL,CAAC;IAED,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;QACpB,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QAClC,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QAClC,OAAO,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACxD,CAAC;IAED,OAAO,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAChD,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAAoB;IACtD,OAAO,CAAC,IAAkB,EAAE,IAAkB,EAAU,EAAE;QACxD,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC;QACnB,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC;QAEnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC;YACvB,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC5B,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAE5B,MAAM,GAAG,GAAG,aAAa,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;YAC5C,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC;gBACd,OAAO,IAAI,CAAC,SAAS,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;YAChD,CAAC;QACH,CAAC;QAED,qDAAqD;QACrD,OAAO,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC;IAC7B,CAAC,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,sBAAsB,CAAC,KAAoB;IACzD,OAAO,CAAC,CAAM,EAAE,CAAM,EAAU,EAAE;QAChC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC;YACvB,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC5B,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAE5B,MAAM,GAAG,GAAG,aAAa,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;YAC5C,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC;gBACd,OAAO,IAAI,CAAC,SAAS,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;YAChD,CAAC;QACH,CAAC;QACD,OAAO,CAAC,CAAC;IACX,CAAC,CAAC;AACJ,CAAC"}

package/dist/transforms/sort/external-sort.d.ts

@@ -0,0 +1,8 @@
+import type { TransformFunction } from "../../core/types.js";
+import { SortOptions } from "./comparator.js";
+/**
+ * External Merge Sort Engine.
+ * Sorts arbitrary-scale datasets with bounded RAM and K-Way Heap merge.
+ */
+export declare function externalSort(options: SortOptions): TransformFunction;
+//# sourceMappingURL=external-sort.d.ts.map

package/dist/transforms/sort/external-sort.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"external-sort.d.ts","sourceRoot":"","sources":["../../../src/transforms/sort/external-sort.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAA8B,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAEzF,OAAO,EAIL,WAAW,EACZ,MAAM,iBAAiB,CAAC;AAQzB;;;GAGG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE,WAAW,GAAG,iBAAiB,CAwKpE"}

package/dist/transforms/sort/external-sort.js

@@ -0,0 +1,159 @@
+import { createReadStream, createWriteStream, promises as fsPromises } from "node:fs";
+import * as os from "node:os";
+import * as path from "node:path";
+import * as readline from "node:readline";
+import { parseMemoryLimit } from "../../diff/storage/spillable-index.js";
+import { Heap } from "../../utils/heap.js";
+import { createRowComparator, parseSortSpecs, } from "./comparator.js";
+/**
+ * External Merge Sort Engine.
+ * Sorts arbitrary-scale datasets with bounded RAM and K-Way Heap merge.
+ */
+export function externalSort(options) {
+    const specs = parseSortSpecs(options.by, {
+        nulls: options.nulls,
+        ignoreCase: options.ignoreCase,
+        natural: options.natural,
+    });
+    const rowComparator = createRowComparator(specs);
+    const memoryLimitBytes = parseMemoryLimit(options.memoryLimit, 256 * 1024 * 1024);
+    const effectiveBatchSize = options.batchSize || 1000;
+    const baseTempDir = options.tempDir || os.tmpdir();
+    return (stream) => {
+        return (async function* () {
+            let currentRun = [];
+            let currentRunBytes = 0;
+            let totalSeq = 0;
+            const runFiles = [];
+            const sessionDir = path.join(baseTempDir, `rowpipe-sort-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`);
+            const cleanup = async () => {
+                try {
+                    await fsPromises.rm(sessionDir, { recursive: true, force: true });
+                }
+                catch {
+                    // Ignore temp cleanup error
+                }
+            };
+            try {
+                for await (const batch of stream) {
+                    for (let i = 0; i < batch.rows.length; i++) {
+                        const row = batch.rows[i];
+                        currentRun.push({ row, seq: totalSeq++ });
+                        // Estimate row memory (~64 bytes overhead + keys/values)
+                        currentRunBytes += 64 + Object.keys(row).length * 24;
+                        if (currentRunBytes >= memoryLimitBytes) {
+                            // Spill current run
+                            await fsPromises.mkdir(sessionDir, { recursive: true });
+                            currentRun.sort(rowComparator);
+                            const runFilePath = path.join(sessionDir, `run_${runFiles.length}.jsonl`);
+                            const writeStream = createWriteStream(runFilePath, { encoding: "utf-8" });
+                            for (let r = 0; r < currentRun.length; r++) {
+                                const item = currentRun[r];
+                                writeStream.write(JSON.stringify({ row: item.row, seq: item.seq }) + "\n");
+                            }
+                            await new Promise((resolve, reject) => {
+                                writeStream.end((err) => {
+                                    if (err)
+                                        reject(err);
+                                    else
+                                        resolve();
+                                });
+                            });
+                            runFiles.push(runFilePath);
+                            currentRun = [];
+                            currentRunBytes = 0;
+                        }
+                    }
+                }
+                // Case 1: All data fit in memory without spilling
+                if (runFiles.length === 0) {
+                    currentRun.sort(rowComparator);
+                    let offset = 0;
+                    for (let i = 0; i < currentRun.length; i += effectiveBatchSize) {
+                        const slice = currentRun.slice(i, i + effectiveBatchSize).map((item) => item.row);
+                        yield {
+                            rows: slice,
+                            offset,
+                        };
+                        offset += slice.length;
+                    }
+                    return;
+                }
+                // Case 2: Spill remainder run if any rows are in memory
+                if (currentRun.length > 0) {
+                    await fsPromises.mkdir(sessionDir, { recursive: true });
+                    currentRun.sort(rowComparator);
+                    const runFilePath = path.join(sessionDir, `run_${runFiles.length}.jsonl`);
+                    const writeStream = createWriteStream(runFilePath, { encoding: "utf-8" });
+                    for (let r = 0; r < currentRun.length; r++) {
+                        const item = currentRun[r];
+                        writeStream.write(JSON.stringify({ row: item.row, seq: item.seq }) + "\n");
+                    }
+                    await new Promise((resolve, reject) => {
+                        writeStream.end((err) => {
+                            if (err)
+                                reject(err);
+                            else
+                                resolve();
+                        });
+                    });
+                    runFiles.push(runFilePath);
+                    currentRun = [];
+                    currentRunBytes = 0;
+                }
+                // Case 3: K-Way Merge using Min-Heap
+                const k = runFiles.length;
+                const iterators = new Array(k);
+                for (let i = 0; i < k; i++) {
+                    const rl = readline.createInterface({
+                        input: createReadStream(runFiles[i], { encoding: "utf-8" }),
+                        crlfDelay: Infinity,
+                    });
+                    iterators[i] = rl[Symbol.asyncIterator]();
+                }
+                const heapComparator = (a, b) => {
+                    return rowComparator({ row: a.row, seq: a.seq }, { row: b.row, seq: b.seq });
+                };
+                const heap = new Heap(heapComparator);
+                // Prime the heap with 1 item from each run
+                for (let i = 0; i < k; i++) {
+                    const next = await iterators[i].next();
+                    if (!next.done && next.value.trim().length > 0) {
+                        const parsed = JSON.parse(next.value);
+                        heap.push({ row: parsed.row, seq: parsed.seq, runIndex: i });
+                    }
+                }
+                let outputRows = [];
+                let globalOffset = 0;
+                while (!heap.isEmpty()) {
+                    const smallest = heap.pop();
+                    outputRows.push(smallest.row);
+                    if (outputRows.length >= effectiveBatchSize) {
+                        yield {
+                            rows: outputRows,
+                            offset: globalOffset,
+                        };
+                        globalOffset += outputRows.length;
+                        outputRows = [];
+                    }
+                    // Advance iterator for smallest.runIndex
+                    const next = await iterators[smallest.runIndex].next();
+                    if (!next.done && next.value.trim().length > 0) {
+                        const parsed = JSON.parse(next.value);
+                        heap.push({ row: parsed.row, seq: parsed.seq, runIndex: smallest.runIndex });
+                    }
+                }
+                if (outputRows.length > 0) {
+                    yield {
+                        rows: outputRows,
+                        offset: globalOffset,
+                    };
+                }
+            }
+            finally {
+                await cleanup();
+            }
+        })();
+    };
+}
+//# sourceMappingURL=external-sort.js.map

package/dist/transforms/sort/external-sort.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"external-sort.js","sourceRoot":"","sources":["../../../src/transforms/sort/external-sort.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,QAAQ,IAAI,UAAU,EAAE,MAAM,SAAS,CAAC;AACtF,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,KAAK,QAAQ,MAAM,eAAe,CAAC;AAC1C,OAAO,EAAE,gBAAgB,EAAE,MAAM,uCAAuC,CAAC;AAEzE,OAAO,EAAE,IAAI,EAAE,MAAM,qBAAqB,CAAC;AAC3C,OAAO,EACL,mBAAmB,EACnB,cAAc,GAGf,MAAM,iBAAiB,CAAC;AAQzB;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,OAAoB;IAC/C,MAAM,KAAK,GAAG,cAAc,CAAC,OAAO,CAAC,EAAE,EAAE;QACvC,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,OAAO,EAAE,OAAO,CAAC,OAAO;KACzB,CAAC,CAAC;IAEH,MAAM,aAAa,GAAG,mBAAmB,CAAC,KAAK,CAAC,CAAC;IACjD,MAAM,gBAAgB,GAAG,gBAAgB,CAAC,OAAO,CAAC,WAAW,EAAE,GAAG,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC;IAClF,MAAM,kBAAkB,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC;IACrD,MAAM,WAAW,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC;IAEnD,OAAO,CAAC,MAAkB,EAAc,EAAE;QACxC,OAAO,CAAC,KAAK,SAAS,CAAC;YACrB,IAAI,UAAU,GAAmB,EAAE,CAAC;YACpC,IAAI,eAAe,GAAG,CAAC,CAAC;YACxB,IAAI,QAAQ,GAAG,CAAC,CAAC;YACjB,MAAM,QAAQ,GAAa,EAAE,CAAC;YAC9B,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAC1B,WAAW,EACX,gBAAgB,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CACvE,CAAC;YAEF,MAAM,OAAO,GAAG,KAAK,IAAI,EAAE;gBACzB,IAAI,CAAC;oBACH,MAAM,UAAU,CAAC,EAAE,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;gBACpE,CAAC;gBAAC,MAAM,CAAC;oBACP,4BAA4B;gBAC9B,CAAC;YACH,CAAC,CAAC;YAEF,IAAI,CAAC;gBACH,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;oBACjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;wBAC3C,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC;wBAC3B,UAAU,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE,CAAC,CAAC;wBAC1C,yDAAyD;wBACzD,eAAe,IAAI,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,GAAG,EAAE,CAAC;wBAErD,IAAI,eAAe,IAAI,gBAAgB,EAAE,CAAC;4BACxC,oBAAoB;4BACpB,MAAM,UAAU,CAAC,KAAK,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;4BACxD,UAAU,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;4BAE/B,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,OAAO,QAAQ,CAAC,MAAM,QAAQ,CAAC,CAAC;4BAC1E,MAAM,WAAW,GAAG,iBAAiB,CAAC,WAAW,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;4BAE1E,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gCAC3C,MAAM,IAAI,GAAG,UAAU,CAAC,CAAC,CAAE,CAAC;gCAC5B,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;4BAC7E,CAAC;4BAED,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gCAC1C,WAAW,CAAC,GAAG,CAAC,CAAC,GAAkB,EAAE,EAAE;oCACrC,IAAI,GAAG;wCAAE,MAAM,CAAC,GAAG,CAAC,CAAC;;wCAChB,OAAO,EAAE,CAAC;gCACjB,CAAC,CAAC,CAAC;4BACL,CAAC,CAAC,CAAC;4BAEH,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;4BAC3B,UAAU,GAAG,EAAE,CAAC;4BAChB,eAAe,GAAG,CAAC,CAAC;wBACtB,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,kDAAkD;gBAClD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAC1B,UAAU,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;oBAC/B,IAAI,MAAM,GAAG,CAAC,CAAC;oBACf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,IAAI,kBAAkB,EAAE,CAAC;wBAC/D,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,kBAAkB,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;wBAClF,MAAM;4BACJ,IAAI,EAAE,KAAK;4BACX,MAAM;yBACP,CAAC;wBACF,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC;oBACzB,CAAC;oBACD,OAAO;gBACT,CAAC;gBAED,wDAAwD;gBACxD,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC1B,MAAM,UAAU,CAAC,KAAK,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;oBACxD,UAAU,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;oBAE/B,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,OAAO,QAAQ,CAAC,MAAM,QAAQ,CAAC,CAAC;oBAC1E,MAAM,WAAW,GAAG,iBAAiB,CAAC,WAAW,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;oBAE1E,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;wBAC3C,MAAM,IAAI,GAAG,UAAU,CAAC,CAAC,CAAE,CAAC;wBAC5B,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;oBAC7E,CAAC;oBAED,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;wBAC1C,WAAW,CAAC,GAAG,CAAC,CAAC,GAAkB,EAAE,EAAE;4BACrC,IAAI,GAAG;gCAAE,MAAM,CAAC,GAAG,CAAC,CAAC;;gCAChB,OAAO,EAAE,CAAC;wBACjB,CAAC,CAAC,CAAC;oBACL,CAAC,CAAC,CAAC;oBAEH,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;oBAC3B,UAAU,GAAG,EAAE,CAAC;oBAChB,eAAe,GAAG,CAAC,CAAC;gBACtB,CAAC;gBAED,qCAAqC;gBACrC,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC;gBAC1B,MAAM,SAAS,GAA4B,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC;gBAExD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC3B,MAAM,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC;wBAClC,KAAK,EAAE,gBAAgB,CAAC,QAAQ,CAAC,CAAC,CAAE,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;wBAC5D,SAAS,EAAE,QAAQ;qBACpB,CAAC,CAAC;oBACH,SAAS,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC;gBAC5C,CAAC;gBAED,MAAM,cAAc,GAAG,CAAC,CAAe,EAAE,CAAe,EAAE,EAAE;oBAC1D,OAAO,aAAa,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;gBAC/E,CAAC,CAAC;gBAEF,MAAM,IAAI,GAAG,IAAI,IAAI,CAAe,cAAc,CAAC,CAAC;gBAEpD,2CAA2C;gBAC3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC3B,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,CAAC;oBACxC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAC/C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAA8B,CAAC;wBACnE,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;oBAC/D,CAAC;gBACH,CAAC;gBAED,IAAI,UAAU,GAAU,EAAE,CAAC;gBAC3B,IAAI,YAAY,GAAG,CAAC,CAAC;gBAErB,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;oBACvB,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAG,CAAC;oBAC7B,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;oBAE9B,IAAI,UAAU,CAAC,MAAM,IAAI,kBAAkB,EAAE,CAAC;wBAC5C,MAAM;4BACJ,IAAI,EAAE,UAAU;4BAChB,MAAM,EAAE,YAAY;yBACrB,CAAC;wBACF,YAAY,IAAI,UAAU,CAAC,MAAM,CAAC;wBAClC,UAAU,GAAG,EAAE,CAAC;oBAClB,CAAC;oBAED,yCAAyC;oBACzC,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAE,CAAC,IAAI,EAAE,CAAC;oBACxD,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAC/C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAA8B,CAAC;wBACnE,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC;oBAC/E,CAAC;gBACH,CAAC;gBAED,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC1B,MAAM;wBACJ,IAAI,EAAE,UAAU;wBAChB,MAAM,EAAE,YAAY;qBACrB,CAAC;gBACJ,CAAC;YACH,CAAC;oBAAS,CAAC;gBACT,MAAM,OAAO,EAAE,CAAC;YAClB,CAAC;QACH,CAAC,CAAC,EAAE,CAAC;IACP,CAAC,CAAC;AACJ,CAAC"}

package/dist/transforms/sort/index.d.ts

@@ -0,0 +1,10 @@
+import type { TransformFunction } from "../../core/types.js";
+import type { SortOptions } from "./comparator.js";
+export * from "./comparator.js";
+export * from "./external-sort.js";
+/**
+ * Sorts tabular stream by one or more columns with typed comparisons,
+ * stable ordering, and automatic external merge sort on large streams.
+ */
+export declare function sortRows(options: SortOptions): TransformFunction;
+//# sourceMappingURL=index.d.ts.map

package/dist/transforms/sort/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/transforms/sort/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAE7D,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAEnD,cAAc,iBAAiB,CAAC;AAChC,cAAc,oBAAoB,CAAC;AAEnC;;;GAGG;AACH,wBAAgB,QAAQ,CAAC,OAAO,EAAE,WAAW,GAAG,iBAAiB,CAEhE"}

package/dist/transforms/sort/index.js

@@ -0,0 +1,11 @@
+import { externalSort } from "./external-sort.js";
+export * from "./comparator.js";
+export * from "./external-sort.js";
+/**
+ * Sorts tabular stream by one or more columns with typed comparisons,
+ * stable ordering, and automatic external merge sort on large streams.
+ */
+export function sortRows(options) {
+    return externalSort(options);
+}
+//# sourceMappingURL=index.js.map

package/dist/transforms/sort/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/transforms/sort/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAGlD,cAAc,iBAAiB,CAAC;AAChC,cAAc,oBAAoB,CAAC;AAEnC;;;GAGG;AACH,MAAM,UAAU,QAAQ,CAAC,OAAoB;IAC3C,OAAO,YAAY,CAAC,OAAO,CAAC,CAAC;AAC/B,CAAC"}

package/dist/transforms/tail.d.ts

@@ -0,0 +1,6 @@
+import type { TransformFunction } from "../core/types.js";
+/**
+ * Retains and emits only the last `count` rows of a stream using a bounded O(N) Ring Buffer.
+ */
+export declare function tailRows(count?: number): TransformFunction;
+//# sourceMappingURL=tail.d.ts.map

package/dist/transforms/tail.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tail.d.ts","sourceRoot":"","sources":["../../src/transforms/tail.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAA8B,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAGtF;;GAEG;AACH,wBAAgB,QAAQ,CAAC,KAAK,SAAK,GAAG,iBAAiB,CAwBtD"}

package/dist/transforms/tail.js

@@ -0,0 +1,27 @@
+import { RingBuffer } from "../utils/ring-buffer.js";
+/**
+ * Retains and emits only the last `count` rows of a stream using a bounded O(N) Ring Buffer.
+ */
+export function tailRows(count = 10) {
+    const tailCount = Math.max(1, count);
+    return (stream) => {
+        return (async function* () {
+            const ringBuffer = new RingBuffer(tailCount);
+            let totalRowsSeen = 0;
+            for await (const batch of stream) {
+                for (let i = 0; i < batch.rows.length; i++) {
+                    ringBuffer.push(batch.rows[i]);
+                    totalRowsSeen++;
+                }
+            }
+            if (ringBuffer.size > 0) {
+                const rows = ringBuffer.toArray();
+                yield {
+                    rows,
+                    offset: Math.max(0, totalRowsSeen - rows.length),
+                };
+            }
+        })();
+    };
+}
+//# sourceMappingURL=tail.js.map

package/dist/transforms/tail.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tail.js","sourceRoot":"","sources":["../../src/transforms/tail.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAErD;;GAEG;AACH,MAAM,UAAU,QAAQ,CAAC,KAAK,GAAG,EAAE;IACjC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAErC,OAAO,CAAC,MAAkB,EAAc,EAAE;QACxC,OAAO,CAAC,KAAK,SAAS,CAAC;YACrB,MAAM,UAAU,GAAG,IAAI,UAAU,CAAM,SAAS,CAAC,CAAC;YAClD,IAAI,aAAa,GAAG,CAAC,CAAC;YAEtB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBACjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC3C,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC,CAAC;oBAChC,aAAa,EAAE,CAAC;gBAClB,CAAC;YACH,CAAC;YAED,IAAI,UAAU,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;gBACxB,MAAM,IAAI,GAAG,UAAU,CAAC,OAAO,EAAE,CAAC;gBAClC,MAAM;oBACJ,IAAI;oBACJ,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC;iBACjD,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,EAAE,CAAC;IACP,CAAC,CAAC;AACJ,CAAC"}

package/dist/transforms/top.d.ts

@@ -0,0 +1,17 @@
+import type { TransformFunction } from "../core/types.js";
+import { SortKeySpec } from "./sort/comparator.js";
+export interface TopOptions {
+    by: string | string[] | SortKeySpec[];
+    count?: number;
+    order?: "asc" | "desc";
+    smallest?: boolean;
+    nulls?: "first" | "last";
+    ignoreCase?: boolean;
+    natural?: boolean;
+}
+/**
+ * Retains only the Top-K (or Bottom-K) rows using a bounded O(K) Binary Heap.
+ * Time complexity: O(N * log K), Memory complexity: O(K).
+ */
+export declare function topRows(options: TopOptions): TransformFunction;
+//# sourceMappingURL=top.d.ts.map

package/dist/transforms/top.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"top.d.ts","sourceRoot":"","sources":["../../src/transforms/top.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAA8B,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAEtF,OAAO,EAGL,WAAW,EACZ,MAAM,sBAAsB,CAAC;AAE9B,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,WAAW,EAAE,CAAC;IACtC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;IACvB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,KAAK,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;IACzB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED;;;GAGG;AACH,wBAAgB,OAAO,CAAC,OAAO,EAAE,UAAU,GAAG,iBAAiB,CAkD9D"}

package/dist/transforms/top.js

@@ -0,0 +1,52 @@
+import { Heap } from "../utils/heap.js";
+import { createRawRowComparator, parseSortSpecs, } from "./sort/comparator.js";
+/**
+ * Retains only the Top-K (or Bottom-K) rows using a bounded O(K) Binary Heap.
+ * Time complexity: O(N * log K), Memory complexity: O(K).
+ */
+export function topRows(options) {
+    const k = Math.max(1, options.count ?? 10);
+    const isSmallest = options.smallest || options.order === "asc";
+    const defaultDirection = isSmallest ? "asc" : "desc";
+    const rawSpecs = parseSortSpecs(options.by, {
+        defaultDirection,
+        nulls: options.nulls,
+        ignoreCase: options.ignoreCase,
+        natural: options.natural,
+    });
+    // Desired output comparator: desiredComparator(a, b) < 0 means 'a' comes before 'b'
+    const desiredComparator = createRawRowComparator(rawSpecs);
+    return (stream) => {
+        return (async function* () {
+            // Min-Heap where root is the LEAST preferred element among current top K
+            // If desiredComparator(a, b) > 0 ('a' is worse than 'b'), then heapComparator(a, b) < 0 (floats to root)
+            const heapComparator = (a, b) => -desiredComparator(a, b);
+            const heap = new Heap(heapComparator);
+            for await (const batch of stream) {
+                for (let i = 0; i < batch.rows.length; i++) {
+                    const row = batch.rows[i];
+                    if (heap.size < k) {
+                        heap.push(row);
+                    }
+                    else {
+                        const worst = heap.peek();
+                        // If incoming row is more preferred than the current worst, replace it
+                        if (desiredComparator(row, worst) < 0) {
+                            heap.replace(row);
+                        }
+                    }
+                }
+            }
+            // Drain and sort results in exact desired comparator order
+            const drained = heap.drain();
+            drained.sort(desiredComparator);
+            if (drained.length > 0) {
+                yield {
+                    rows: drained,
+                    offset: 0,
+                };
+            }
+        })();
+    };
+}
+//# sourceMappingURL=top.js.map

package/dist/transforms/top.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"top.js","sourceRoot":"","sources":["../../src/transforms/top.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AACxC,OAAO,EACL,sBAAsB,EACtB,cAAc,GAEf,MAAM,sBAAsB,CAAC;AAY9B;;;GAGG;AACH,MAAM,UAAU,OAAO,CAAC,OAAmB;IACzC,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;IAC3C,MAAM,UAAU,GAAG,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,KAAK,KAAK,KAAK,CAAC;IAC/D,MAAM,gBAAgB,GAAmB,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC;IAErE,MAAM,QAAQ,GAAG,cAAc,CAAC,OAAO,CAAC,EAAE,EAAE;QAC1C,gBAAgB;QAChB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,OAAO,EAAE,OAAO,CAAC,OAAO;KACzB,CAAC,CAAC;IAEH,oFAAoF;IACpF,MAAM,iBAAiB,GAAG,sBAAsB,CAAC,QAAQ,CAAC,CAAC;IAE3D,OAAO,CAAC,MAAkB,EAAc,EAAE;QACxC,OAAO,CAAC,KAAK,SAAS,CAAC;YACrB,yEAAyE;YACzE,yGAAyG;YACzG,MAAM,cAAc,GAAG,CAAC,CAAM,EAAE,CAAM,EAAE,EAAE,CAAC,CAAC,iBAAiB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACpE,MAAM,IAAI,GAAG,IAAI,IAAI,CAAM,cAAc,CAAC,CAAC;YAE3C,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBACjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC3C,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC;oBAE3B,IAAI,IAAI,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;wBAClB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBACjB,CAAC;yBAAM,CAAC;wBACN,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,EAAG,CAAC;wBAC3B,uEAAuE;wBACvE,IAAI,iBAAiB,CAAC,GAAG,EAAE,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;4BACtC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;wBACpB,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;YAED,2DAA2D;YAC3D,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;YAC7B,OAAO,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;YAEhC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvB,MAAM;oBACJ,IAAI,EAAE,OAAO;oBACb,MAAM,EAAE,CAAC;iBACV,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,EAAE,CAAC;IACP,CAAC,CAAC;AACJ,CAAC"}

package/dist/transforms/unique.d.ts

@@ -0,0 +1,13 @@
+import type { TransformFunction } from "../core/types.js";
+export interface UniqueOptions {
+    by?: string | string[];
+    keep?: "first" | "last";
+    memoryLimit?: number | string;
+    tempDir?: string;
+    batchSize?: number;
+}
+/**
+ * Deduplicates rows by key columns (or whole row) with spillable disk backup for large cardinalities.
+ */
+export declare function uniqueRows(options?: UniqueOptions): TransformFunction;
+//# sourceMappingURL=unique.d.ts.map

package/dist/transforms/unique.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"unique.d.ts","sourceRoot":"","sources":["../../src/transforms/unique.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAA8B,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAGtF,MAAM,WAAW,aAAa;IAC5B,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IACvB,IAAI,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;IACxB,WAAW,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC9B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AASD;;GAEG;AACH,wBAAgB,UAAU,CAAC,OAAO,GAAE,aAAkB,GAAG,iBAAiB,CAoFzE"}

package/dist/transforms/unique.js

@@ -0,0 +1,91 @@
+import { encodeCompositeKey } from "../diff/key.js";
+import { SpillableKeyStore } from "../utils/keystore.js";
+function getRowUniqueKey(row, byCols) {
+    if (byCols && byCols.length > 0) {
+        return encodeCompositeKey(row, byCols).encoded;
+    }
+    return JSON.stringify(row);
+}
+/**
+ * Deduplicates rows by key columns (or whole row) with spillable disk backup for large cardinalities.
+ */
+export function uniqueRows(options = {}) {
+    const keep = options.keep || "first";
+    const byCols = options.by
+        ? (Array.isArray(options.by) ? options.by : options.by.split(",").map((s) => s.trim()).filter(Boolean))
+        : undefined;
+    const effectiveBatchSize = options.batchSize || 1000;
+    return (stream) => {
+        return (async function* () {
+            const keyStore = new SpillableKeyStore({
+                memoryLimit: options.memoryLimit,
+                tempDir: options.tempDir,
+            });
+            try {
+                if (keep === "first") {
+                    // Streaming mode: Emit immediately on first encounter
+                    let currentBatchRows = [];
+                    let globalOffset = 0;
+                    for await (const batch of stream) {
+                        for (let i = 0; i < batch.rows.length; i++) {
+                            const row = batch.rows[i];
+                            const key = getRowUniqueKey(row, byCols);
+                            const isNew = await keyStore.add(key);
+                            if (isNew) {
+                                currentBatchRows.push(row);
+                                if (currentBatchRows.length >= effectiveBatchSize) {
+                                    yield {
+                                        rows: currentBatchRows,
+                                        offset: globalOffset,
+                                    };
+                                    globalOffset += currentBatchRows.length;
+                                    currentBatchRows = [];
+                                }
+                            }
+                        }
+                    }
+                    if (currentBatchRows.length > 0) {
+                        yield {
+                            rows: currentBatchRows,
+                            offset: globalOffset,
+                        };
+                    }
+                }
+                else {
+                    // keep: "last" mode: Retain latest row per key
+                    const latestRows = new Map();
+                    let globalOffset = 0;
+                    for await (const batch of stream) {
+                        for (let i = 0; i < batch.rows.length; i++) {
+                            const row = batch.rows[i];
+                            const key = getRowUniqueKey(row, byCols);
+                            latestRows.set(key, row);
+                        }
+                    }
+                    let currentBatchRows = [];
+                    for (const row of latestRows.values()) {
+                        currentBatchRows.push(row);
+                        if (currentBatchRows.length >= effectiveBatchSize) {
+                            yield {
+                                rows: currentBatchRows,
+                                offset: globalOffset,
+                            };
+                            globalOffset += currentBatchRows.length;
+                            currentBatchRows = [];
+                        }
+                    }
+                    if (currentBatchRows.length > 0) {
+                        yield {
+                            rows: currentBatchRows,
+                            offset: globalOffset,
+                        };
+                    }
+                }
+            }
+            finally {
+                await keyStore.close();
+            }
+        })();
+    };
+}
+//# sourceMappingURL=unique.js.map

package/dist/transforms/unique.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"unique.js","sourceRoot":"","sources":["../../src/transforms/unique.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AAEpD,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAUzD,SAAS,eAAe,CAAC,GAAQ,EAAE,MAAiB;IAClD,IAAI,MAAM,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAChC,OAAO,kBAAkB,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC;IACjD,CAAC;IACD,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;AAC7B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,UAAU,CAAC,UAAyB,EAAE;IACpD,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC;IACrC,MAAM,MAAM,GAAG,OAAO,CAAC,EAAE;QACvB,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACvG,CAAC,CAAC,SAAS,CAAC;IACd,MAAM,kBAAkB,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC;IAErD,OAAO,CAAC,MAAkB,EAAc,EAAE;QACxC,OAAO,CAAC,KAAK,SAAS,CAAC;YACrB,MAAM,QAAQ,GAAG,IAAI,iBAAiB,CAAC;gBACrC,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,OAAO,EAAE,OAAO,CAAC,OAAO;aACzB,CAAC,CAAC;YAEH,IAAI,CAAC;gBACH,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;oBACrB,sDAAsD;oBACtD,IAAI,gBAAgB,GAAU,EAAE,CAAC;oBACjC,IAAI,YAAY,GAAG,CAAC,CAAC;oBAErB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;wBACjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;4BAC3C,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC;4BAC3B,MAAM,GAAG,GAAG,eAAe,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;4BAEzC,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;4BACtC,IAAI,KAAK,EAAE,CAAC;gCACV,gBAAgB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gCAC3B,IAAI,gBAAgB,CAAC,MAAM,IAAI,kBAAkB,EAAE,CAAC;oCAClD,MAAM;wCACJ,IAAI,EAAE,gBAAgB;wCACtB,MAAM,EAAE,YAAY;qCACrB,CAAC;oCACF,YAAY,IAAI,gBAAgB,CAAC,MAAM,CAAC;oCACxC,gBAAgB,GAAG,EAAE,CAAC;gCACxB,CAAC;4BACH,CAAC;wBACH,CAAC;oBACH,CAAC;oBAED,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAChC,MAAM;4BACJ,IAAI,EAAE,gBAAgB;4BACtB,MAAM,EAAE,YAAY;yBACrB,CAAC;oBACJ,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,+CAA+C;oBAC/C,MAAM,UAAU,GAAG,IAAI,GAAG,EAAe,CAAC;oBAC1C,IAAI,YAAY,GAAG,CAAC,CAAC;oBAErB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;wBACjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;4BAC3C,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC;4BAC3B,MAAM,GAAG,GAAG,eAAe,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;4BACzC,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;wBAC3B,CAAC;oBACH,CAAC;oBAED,IAAI,gBAAgB,GAAU,EAAE,CAAC;oBACjC,KAAK,MAAM,GAAG,IAAI,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC;wBACtC,gBAAgB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;wBAC3B,IAAI,gBAAgB,CAAC,MAAM,IAAI,kBAAkB,EAAE,CAAC;4BAClD,MAAM;gCACJ,IAAI,EAAE,gBAAgB;gCACtB,MAAM,EAAE,YAAY;6BACrB,CAAC;4BACF,YAAY,IAAI,gBAAgB,CAAC,MAAM,CAAC;4BACxC,gBAAgB,GAAG,EAAE,CAAC;wBACxB,CAAC;oBACH,CAAC;oBAED,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAChC,MAAM;4BACJ,IAAI,EAAE,gBAAgB;4BACtB,MAAM,EAAE,YAAY;yBACrB,CAAC;oBACJ,CAAC;gBACH,CAAC;YACH,CAAC;oBAAS,CAAC;gBACT,MAAM,QAAQ,CAAC,KAAK,EAAE,CAAC;YACzB,CAAC;QACH,CAAC,CAAC,EAAE,CAAC;IACP,CAAC,CAAC;AACJ,CAAC"}

package/dist/utils/formatting.d.ts

@@ -1,6 +1,7 @@
 export declare function formatNumber(num: number | null | undefined): string;
 export declare function formatDecimal(num: number | null | undefined, digits?: number): string;
 export declare function formatBytes(bytes: number | null | undefined): string;
+export declare function parseBytes(input: string | number | null | undefined): number;
 export declare function formatTable(headers: string[], rows: string[][], alignments?: Array<"left" | "right">): string;
 export declare function logMemoryDebug(): void;
 //# sourceMappingURL=formatting.d.ts.map

package/dist/utils/formatting.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"formatting.d.ts","sourceRoot":"","sources":["../../src/utils/formatting.ts"],"names":[],"mappings":"AAAA,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,MAAM,CAKnE;AAED,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,EAAE,MAAM,SAAI,GAAG,MAAM,CAQhF;AAED,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,MAAM,CAMpE;AAED,wBAAgB,WAAW,CACzB,OAAO,EAAE,MAAM,EAAE,EACjB,IAAI,EAAE,MAAM,EAAE,EAAE,EAChB,UAAU,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,OAAO,CAAC,GACnC,MAAM,CAoCR;AAED,wBAAgB,cAAc,IAAI,IAAI,CAOrC"}
+{"version":3,"file":"formatting.d.ts","sourceRoot":"","sources":["../../src/utils/formatting.ts"],"names":[],"mappings":"AAAA,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,MAAM,CAKnE;AAED,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,EAAE,MAAM,SAAI,GAAG,MAAM,CAQhF;AAED,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,MAAM,CAQpE;AAED,wBAAgB,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,MAAM,CAyB5E;AAED,wBAAgB,WAAW,CACzB,OAAO,EAAE,MAAM,EAAE,EACjB,IAAI,EAAE,MAAM,EAAE,EAAE,EAChB,UAAU,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,OAAO,CAAC,GACnC,MAAM,CAoCR;AAED,wBAAgB,cAAc,IAAI,IAAI,CAOrC"}