rowpipe 1.1.0 → 2.0.0

package/dist/diff/reporter.js

@@ -0,0 +1,133 @@
+import { formatNumber } from "../utils/formatting.js";
+import { formatKeyForDisplay } from "./key.js";
+function formatValueForDisplay(val) {
+    if (val === undefined)
+        return "<missing>";
+    if (val === null)
+        return "null";
+    if (typeof val === "object")
+        return JSON.stringify(val);
+    return String(val);
+}
+/**
+ * Formats a single DiffEvent for --format rows output.
+ */
+export function formatRowEvent(event) {
+    const keyStr = formatKeyForDisplay(event.key);
+    switch (event.type) {
+        case "added":
+            return `ADDED ${keyStr}\n`;
+        case "removed":
+            return `REMOVED ${keyStr}\n`;
+        case "changed": {
+            let output = `CHANGED ${keyStr}\n`;
+            for (const ch of event.changes) {
+                const oldDisplay = formatValueForDisplay(ch.oldVal);
+                const newDisplay = formatValueForDisplay(ch.newVal);
+                output += `  ${ch.column}: ${oldDisplay} -> ${newDisplay}\n`;
+            }
+            return output;
+        }
+        case "unchanged":
+            return `UNCHANGED ${keyStr}\n`;
+        default:
+            return null;
+    }
+}
+/**
+ * Formats a single DiffEvent for --format patch (JSONL) output.
+ */
+export function formatPatchEvent(event) {
+    switch (event.type) {
+        case "added":
+            return (JSON.stringify({
+                op: "insert",
+                key: event.key,
+                row: event.row,
+            }) + "\n");
+        case "removed":
+            return (JSON.stringify({
+                op: "delete",
+                key: event.key,
+                row: event.row,
+            }) + "\n");
+        case "changed": {
+            const changesObj = {};
+            for (const ch of event.changes) {
+                changesObj[ch.column] = {
+                    old: ch.oldVal,
+                    new: ch.newVal,
+                };
+            }
+            return (JSON.stringify({
+                op: "update",
+                key: event.key,
+                changes: changesObj,
+            }) + "\n");
+        }
+        default:
+            return null;
+    }
+}
+/**
+ * Formats complete DiffSummary as human-readable ASCII text.
+ */
+export function formatSummary(summary) {
+    const lines = [];
+    lines.push("Comparing:");
+    lines.push(`  ${summary.leftSource}`);
+    lines.push(`  ${summary.rightSource}`);
+    lines.push("");
+    lines.push(`Key: ${summary.keyColumns.join(", ")}`);
+    lines.push("");
+    lines.push("Rows");
+    const maxNumLen = Math.max(formatNumber(summary.rows.added).length, formatNumber(summary.rows.removed).length, formatNumber(summary.rows.changed).length, formatNumber(summary.rows.unchanged).length);
+    lines.push(`  added      ${formatNumber(summary.rows.added).padStart(maxNumLen)}`);
+    lines.push(`  removed    ${formatNumber(summary.rows.removed).padStart(maxNumLen)}`);
+    lines.push(`  changed    ${formatNumber(summary.rows.changed).padStart(maxNumLen)}`);
+    lines.push(`  unchanged  ${formatNumber(summary.rows.unchanged).padStart(maxNumLen)}`);
+    // Changed columns breakdown
+    const changedCols = Object.entries(summary.columns).sort((a, b) => b[1].changed - a[1].changed);
+    if (changedCols.length > 0) {
+        lines.push("");
+        lines.push("Changed columns");
+        const maxColLen = Math.max(...changedCols.map(([c]) => c.length), 10);
+        for (const [col, info] of changedCols) {
+            lines.push(`  ${col.padEnd(maxColLen)}  ${formatNumber(info.changed)}`);
+        }
+    }
+    // Schema diff
+    if (summary.schema) {
+        const { added, removed, changed } = summary.schema;
+        if (added.length > 0 || removed.length > 0 || changed.length > 0) {
+            lines.push("");
+            lines.push("Schema");
+            for (const col of added) {
+                lines.push(`  + ${col.name}: ${col.type}`);
+            }
+            for (const col of changed) {
+                lines.push(`  ~ ${col.name}: ${col.leftType} -> ${col.rightType}`);
+            }
+            for (const col of removed) {
+                lines.push(`  - ${col.name}: ${col.type}`);
+            }
+        }
+    }
+    return lines.join("\n") + "\n";
+}
+/**
+ * Formats complete DiffSummary as JSON string.
+ */
+export function formatJson(summary) {
+    const jsonObject = {
+        left: summary.leftSource,
+        right: summary.rightSource,
+        key: summary.keyColumns,
+        rows: summary.rows,
+        columns: summary.columns,
+        schema: summary.schema,
+        executionTimeMs: summary.executionTimeMs,
+    };
+    return JSON.stringify(jsonObject, null, 2) + "\n";
+}
+//# sourceMappingURL=reporter.js.map

package/dist/diff/reporter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"reporter.js","sourceRoot":"","sources":["../../src/diff/reporter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,EAAE,mBAAmB,EAAE,MAAM,UAAU,CAAC;AAG/C,SAAS,qBAAqB,CAAC,GAAY;IACzC,IAAI,GAAG,KAAK,SAAS;QAAE,OAAO,WAAW,CAAC;IAC1C,IAAI,GAAG,KAAK,IAAI;QAAE,OAAO,MAAM,CAAC;IAChC,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACxD,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC;AACrB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,KAAgB;IAC7C,MAAM,MAAM,GAAG,mBAAmB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAE9C,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;QACnB,KAAK,OAAO;YACV,OAAO,SAAS,MAAM,IAAI,CAAC;QAE7B,KAAK,SAAS;YACZ,OAAO,WAAW,MAAM,IAAI,CAAC;QAE/B,KAAK,SAAS,CAAC,CAAC,CAAC;YACf,IAAI,MAAM,GAAG,WAAW,MAAM,IAAI,CAAC;YACnC,KAAK,MAAM,EAAE,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;gBAC/B,MAAM,UAAU,GAAG,qBAAqB,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;gBACpD,MAAM,UAAU,GAAG,qBAAqB,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;gBACpD,MAAM,IAAI,KAAK,EAAE,CAAC,MAAM,KAAK,UAAU,OAAO,UAAU,IAAI,CAAC;YAC/D,CAAC;YACD,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,KAAK,WAAW;YACd,OAAO,aAAa,MAAM,IAAI,CAAC;QAEjC;YACE,OAAO,IAAI,CAAC;IAChB,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAAgB;IAC/C,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;QACnB,KAAK,OAAO;YACV,OAAO,CACL,IAAI,CAAC,SAAS,CAAC;gBACb,EAAE,EAAE,QAAQ;gBACZ,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;aACf,CAAC,GAAG,IAAI,CACV,CAAC;QAEJ,KAAK,SAAS;YACZ,OAAO,CACL,IAAI,CAAC,SAAS,CAAC;gBACb,EAAE,EAAE,QAAQ;gBACZ,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;aACf,CAAC,GAAG,IAAI,CACV,CAAC;QAEJ,KAAK,SAAS,CAAC,CAAC,CAAC;YACf,MAAM,UAAU,GAAmD,EAAE,CAAC;YACtE,KAAK,MAAM,EAAE,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;gBAC/B,UAAU,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG;oBACtB,GAAG,EAAE,EAAE,CAAC,MAAM;oBACd,GAAG,EAAE,EAAE,CAAC,MAAM;iBACf,CAAC;YACJ,CAAC;YACD,OAAO,CACL,IAAI,CAAC,SAAS,CAAC;gBACb,EAAE,EAAE,QAAQ;gBACZ,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,OAAO,EAAE,UAAU;aACpB,CAAC,GAAG,IAAI,CACV,CAAC;QACJ,CAAC;QAED;YACE,OAAO,IAAI,CAAC;IAChB,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,aAAa,CAAC,OAAoB;IAChD,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IACzB,KAAK,CAAC,IAAI,CAAC,KAAK,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC;IACtC,KAAK,CAAC,IAAI,CAAC,KAAK,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;IACvC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,QAAQ,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACpD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACnB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CACxB,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,EACvC,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,EACzC,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,EACzC,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,CAC5C,CAAC;IAEF,KAAK,CAAC,IAAI,CAAC,gBAAgB,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;IACnF,KAAK,CAAC,IAAI,CAAC,gBAAgB,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;IACrF,KAAK,CAAC,IAAI,CAAC,gBAAgB,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;IACrF,KAAK,CAAC,IAAI,CAAC,gBAAgB,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;IAEvF,4BAA4B;IAC5B,MAAM,WAAW,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,CACtD,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CACtC,CAAC;IAEF,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAC9B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC;QACtE,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,WAAW,EAAE,CAAC;YACtC,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC1E,CAAC;IACH,CAAC;IAED,cAAc;IACd,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;QACnD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACjE,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACrB,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;gBACxB,KAAK,CAAC,IAAI,CAAC,OAAO,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;YAC7C,CAAC;YACD,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;gBAC1B,KAAK,CAAC,IAAI,CAAC,OAAO,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,QAAQ,OAAO,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC;YACrE,CAAC;YACD,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;gBAC1B,KAAK,CAAC,IAAI,CAAC,OAAO,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;YAC7C,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;AACjC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,UAAU,CAAC,OAAoB;IAC7C,MAAM,UAAU,GAAG;QACjB,IAAI,EAAE,OAAO,CAAC,UAAU;QACxB,KAAK,EAAE,OAAO,CAAC,WAAW;QAC1B,GAAG,EAAE,OAAO,CAAC,UAAU;QACvB,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,eAAe,EAAE,OAAO,CAAC,eAAe;KACzC,CAAC;IACF,OAAO,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC;AACpD,CAAC"}

package/dist/diff/schema.d.ts

@@ -0,0 +1,7 @@
+import type { TabularReader } from "../core/types.js";
+import type { SchemaColumnDiff } from "./types.js";
+/**
+ * Computes the schema differences between two tabular readers.
+ */
+export declare function computeSchemaDiff(leftReader: TabularReader, rightReader: TabularReader): Promise<SchemaColumnDiff>;
+//# sourceMappingURL=schema.d.ts.map

package/dist/diff/schema.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/diff/schema.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAc,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAElE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AA2CnD;;GAEG;AACH,wBAAsB,iBAAiB,CACrC,UAAU,EAAE,aAAa,EACzB,WAAW,EAAE,aAAa,GACzB,OAAO,CAAC,gBAAgB,CAAC,CA4B3B"}

package/dist/diff/schema.js

@@ -0,0 +1,65 @@
+import { classifyPrimitiveType } from "../analytics/schema-inference.js";
+async function extractReaderColumns(reader) {
+    const map = new Map();
+    // 1. Try reader.inspect()
+    if (typeof reader.inspect === "function") {
+        try {
+            const meta = await reader.inspect({ maxRows: 1000 });
+            if (meta.columns && meta.columns.length > 0) {
+                for (const col of meta.columns) {
+                    map.set(col.name, col.type);
+                }
+                return map;
+            }
+        }
+        catch {
+            // Fallback to reading first batch
+        }
+    }
+    // 2. Read first batch
+    try {
+        for await (const batch of reader.read({ batchSize: 100 })) {
+            if (batch.rows.length > 0) {
+                const firstRow = batch.rows[0];
+                for (const [k, v] of Object.entries(firstRow)) {
+                    map.set(k, classifyPrimitiveType(v));
+                }
+                break;
+            }
+        }
+    }
+    catch {
+        // Ignore error
+    }
+    return map;
+}
+/**
+ * Computes the schema differences between two tabular readers.
+ */
+export async function computeSchemaDiff(leftReader, rightReader) {
+    const leftCols = await extractReaderColumns(leftReader);
+    const rightCols = await extractReaderColumns(rightReader);
+    const added = [];
+    const removed = [];
+    const changed = [];
+    // Detect removed & changed
+    for (const [name, leftType] of leftCols.entries()) {
+        if (!rightCols.has(name)) {
+            removed.push({ name, type: leftType });
+        }
+        else {
+            const rightType = rightCols.get(name);
+            if (leftType !== rightType && leftType !== "mixed" && rightType !== "mixed") {
+                changed.push({ name, leftType, rightType });
+            }
+        }
+    }
+    // Detect added
+    for (const [name, rightType] of rightCols.entries()) {
+        if (!leftCols.has(name)) {
+            added.push({ name, type: rightType });
+        }
+    }
+    return { added, removed, changed };
+}
+//# sourceMappingURL=schema.js.map

package/dist/diff/schema.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/diff/schema.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,qBAAqB,EAAE,MAAM,kCAAkC,CAAC;AAQzE,KAAK,UAAU,oBAAoB,CAAC,MAAqB;IACvD,MAAM,GAAG,GAAG,IAAI,GAAG,EAAsB,CAAC;IAE1C,0BAA0B;IAC1B,IAAI,OAAO,MAAM,CAAC,OAAO,KAAK,UAAU,EAAE,CAAC;QACzC,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;YACrD,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5C,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;oBAC/B,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;gBAC9B,CAAC;gBACD,OAAO,GAAG,CAAC;YACb,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,kCAAkC;QACpC,CAAC;IACH,CAAC;IAED,sBAAsB;IACtB,IAAI,CAAC;QACH,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;YAC1D,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC;gBAChC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAC9C,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,qBAAqB,CAAC,CAAC,CAAC,CAAC,CAAC;gBACvC,CAAC;gBACD,MAAM;YACR,CAAC;QACH,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,eAAe;IACjB,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,UAAyB,EACzB,WAA0B;IAE1B,MAAM,QAAQ,GAAG,MAAM,oBAAoB,CAAC,UAAU,CAAC,CAAC;IACxD,MAAM,SAAS,GAAG,MAAM,oBAAoB,CAAC,WAAW,CAAC,CAAC;IAE1D,MAAM,KAAK,GAA8C,EAAE,CAAC;IAC5D,MAAM,OAAO,GAA8C,EAAE,CAAC;IAC9D,MAAM,OAAO,GAAyE,EAAE,CAAC;IAEzF,2BAA2B;IAC3B,KAAK,MAAM,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;QAClD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACzB,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;QACzC,CAAC;aAAM,CAAC;YACN,MAAM,SAAS,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC;YACvC,IAAI,QAAQ,KAAK,SAAS,IAAI,QAAQ,KAAK,OAAO,IAAI,SAAS,KAAK,OAAO,EAAE,CAAC;gBAC5E,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,CAAC;YAC9C,CAAC;QACH,CAAC;IACH,CAAC;IAED,eAAe;IACf,KAAK,MAAM,CAAC,IAAI,EAAE,SAAS,CAAC,IAAI,SAAS,CAAC,OAAO,EAAE,EAAE,CAAC;QACpD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACxB,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;QACxC,CAAC;IACH,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;AACrC,CAAC"}

package/dist/diff/storage/disk-index.d.ts

@@ -0,0 +1,28 @@
+import type { DiffIndex, IndexedRow } from "../types.js";
+/**
+ * High-performance Disk-backed Diff Index using an append-only temp file and offset index.
+ * Offloads entire row object memory to disk with microsecond random-access lookups.
+ */
+export declare class DiskDiffIndex implements DiffIndex {
+    private tempFilePath;
+    private fd;
+    private index;
+    private currentOffset;
+    private isClosed;
+    private cleanupHandler?;
+    constructor();
+    set(key: string, indexedRow: IndexedRow): Promise<void>;
+    get(key: string): Promise<IndexedRow | undefined>;
+    has(key: string): Promise<boolean>;
+    delete(key: string): Promise<boolean>;
+    entries(): AsyncIterable<[string, IndexedRow]>;
+    count(): number;
+    getSpillStats(): {
+        isSpilled: boolean;
+        spilledBytes: number;
+        entryCount: number;
+    };
+    private cleanupSync;
+    close(): Promise<void>;
+}
+//# sourceMappingURL=disk-index.d.ts.map

package/dist/diff/storage/disk-index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"disk-index.d.ts","sourceRoot":"","sources":["../../../src/diff/storage/disk-index.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AASzD;;;GAGG;AACH,qBAAa,aAAc,YAAW,SAAS;IAC7C,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,EAAE,CAAS;IACnB,OAAO,CAAC,KAAK,CAAsC;IACnD,OAAO,CAAC,aAAa,CAAK;IAC1B,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,cAAc,CAAC,CAAa;;IAe9B,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAgBvD,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,SAAS,CAAC;IAejD,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAIlC,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAIpC,OAAO,IAAI,aAAa,CAAC,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IASrD,KAAK,IAAI,MAAM;IAIf,aAAa,IAAI;QAAE,SAAS,EAAE,OAAO,CAAC;QAAC,YAAY,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE;IAQjF,OAAO,CAAC,WAAW;IAiBb,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAQ7B"}

package/dist/diff/storage/disk-index.js

@@ -0,0 +1,105 @@
+import { closeSync, existsSync, openSync, readSync, unlinkSync, writeSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { randomBytes } from "node:crypto";
+/**
+ * High-performance Disk-backed Diff Index using an append-only temp file and offset index.
+ * Offloads entire row object memory to disk with microsecond random-access lookups.
+ */
+export class DiskDiffIndex {
+    tempFilePath;
+    fd;
+    index = new Map();
+    currentOffset = 0;
+    isClosed = false;
+    cleanupHandler;
+    constructor() {
+        const randomId = randomBytes(8).toString("hex");
+        this.tempFilePath = join(tmpdir(), `rowpipe-diff-${process.pid}-${randomId}.tmp`);
+        this.fd = openSync(this.tempFilePath, "w+");
+        this.cleanupHandler = () => {
+            this.cleanupSync();
+        };
+        process.once("exit", this.cleanupHandler);
+        process.once("SIGINT", this.cleanupHandler);
+        process.once("SIGTERM", this.cleanupHandler);
+    }
+    async set(key, indexedRow) {
+        const jsonStr = JSON.stringify(indexedRow.row);
+        const buf = Buffer.from(jsonStr, "utf-8");
+        const offset = this.currentOffset;
+        writeSync(this.fd, buf, 0, buf.length, offset);
+        this.currentOffset += buf.length;
+        this.index.set(key, {
+            offset,
+            length: buf.length,
+            hash: indexedRow.hash,
+            rowNumber: indexedRow.rowNumber,
+        });
+    }
+    async get(key) {
+        const meta = this.index.get(key);
+        if (!meta)
+            return undefined;
+        const buf = Buffer.allocUnsafe(meta.length);
+        readSync(this.fd, buf, 0, meta.length, meta.offset);
+        const row = JSON.parse(buf.toString("utf-8"));
+        return {
+            rowNumber: meta.rowNumber,
+            row,
+            hash: meta.hash,
+        };
+    }
+    async has(key) {
+        return this.index.has(key);
+    }
+    async delete(key) {
+        return this.index.delete(key);
+    }
+    async *entries() {
+        for (const [key, meta] of this.index.entries()) {
+            const buf = Buffer.allocUnsafe(meta.length);
+            readSync(this.fd, buf, 0, meta.length, meta.offset);
+            const row = JSON.parse(buf.toString("utf-8"));
+            yield [key, { rowNumber: meta.rowNumber, row, hash: meta.hash }];
+        }
+    }
+    count() {
+        return this.index.size;
+    }
+    getSpillStats() {
+        return {
+            isSpilled: true,
+            spilledBytes: this.currentOffset,
+            entryCount: this.index.size,
+        };
+    }
+    cleanupSync() {
+        if (this.isClosed)
+            return;
+        this.isClosed = true;
+        try {
+            closeSync(this.fd);
+        }
+        catch {
+            // Ignore
+        }
+        try {
+            if (existsSync(this.tempFilePath)) {
+                unlinkSync(this.tempFilePath);
+            }
+        }
+        catch {
+            // Ignore
+        }
+    }
+    async close() {
+        if (this.cleanupHandler) {
+            process.removeListener("exit", this.cleanupHandler);
+            process.removeListener("SIGINT", this.cleanupHandler);
+            process.removeListener("SIGTERM", this.cleanupHandler);
+        }
+        this.cleanupSync();
+    }
+}
+//# sourceMappingURL=disk-index.js.map

package/dist/diff/storage/disk-index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"disk-index.js","sourceRoot":"","sources":["../../../src/diff/storage/disk-index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAC3F,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAU1C;;;GAGG;AACH,MAAM,OAAO,aAAa;IAChB,YAAY,CAAS;IACrB,EAAE,CAAS;IACX,KAAK,GAAG,IAAI,GAAG,EAA2B,CAAC;IAC3C,aAAa,GAAG,CAAC,CAAC;IAClB,QAAQ,GAAG,KAAK,CAAC;IACjB,cAAc,CAAc;IAEpC;QACE,MAAM,QAAQ,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QAChD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,MAAM,EAAE,EAAE,gBAAgB,OAAO,CAAC,GAAG,IAAI,QAAQ,MAAM,CAAC,CAAC;QAClF,IAAI,CAAC,EAAE,GAAG,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;QAE5C,IAAI,CAAC,cAAc,GAAG,GAAG,EAAE;YACzB,IAAI,CAAC,WAAW,EAAE,CAAC;QACrB,CAAC,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC;QAC1C,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC;QAC5C,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC;IAC/C,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,GAAW,EAAE,UAAsB;QAC3C,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;QAC/C,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAC1C,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC;QAElC,SAAS,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC/C,IAAI,CAAC,aAAa,IAAI,GAAG,CAAC,MAAM,CAAC;QAEjC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE;YAClB,MAAM;YACN,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,IAAI,EAAE,UAAU,CAAC,IAAI;YACrB,SAAS,EAAE,UAAU,CAAC,SAAS;SAChC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,GAAW;QACnB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACjC,IAAI,CAAC,IAAI;YAAE,OAAO,SAAS,CAAC;QAE5B,MAAM,GAAG,GAAG,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC5C,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QACpD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;QAE9C,OAAO;YACL,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,GAAG;YACH,IAAI,EAAE,IAAI,CAAC,IAAI;SAChB,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,GAAW;QACnB,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,GAAW;QACtB,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAChC,CAAC;IAED,KAAK,CAAC,CAAC,OAAO;QACZ,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;YAC/C,MAAM,GAAG,GAAG,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC5C,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YACpD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;YAC9C,MAAM,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QACnE,CAAC;IACH,CAAC;IAED,KAAK;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;IACzB,CAAC;IAED,aAAa;QACX,OAAO;YACL,SAAS,EAAE,IAAI;YACf,YAAY,EAAE,IAAI,CAAC,aAAa;YAChC,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI;SAC5B,CAAC;IACJ,CAAC;IAEO,WAAW;QACjB,IAAI,IAAI,CAAC,QAAQ;YAAE,OAAO;QAC1B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,CAAC;YACH,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACrB,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;QACD,IAAI,CAAC;YACH,IAAI,UAAU,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC;gBAClC,UAAU,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAChC,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC;YACpD,OAAO,CAAC,cAAc,CAAC,QAAQ,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC;YACtD,OAAO,CAAC,cAAc,CAAC,SAAS,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC;QACzD,CAAC;QACD,IAAI,CAAC,WAAW,EAAE,CAAC;IACrB,CAAC;CACF"}

package/dist/diff/storage/memory-index.d.ts

@@ -0,0 +1,23 @@
+import type { DiffIndex, IndexedRow } from "../types.js";
+/**
+ * High-performance in-memory Diff Index backed by JavaScript Map.
+ */
+export declare class MemoryDiffIndex implements DiffIndex {
+    private store;
+    private estimatedBytes;
+    set(key: string, row: IndexedRow): Promise<void>;
+    get(key: string): Promise<IndexedRow | undefined>;
+    has(key: string): Promise<boolean>;
+    delete(key: string): Promise<boolean>;
+    entries(): AsyncIterable<[string, IndexedRow]>;
+    count(): number;
+    getEstimatedBytes(): number;
+    getSpillStats(): {
+        isSpilled: boolean;
+        spilledBytes: number;
+        entryCount: number;
+    };
+    close(): Promise<void>;
+    private estimateRowBytes;
+}
+//# sourceMappingURL=memory-index.d.ts.map

package/dist/diff/storage/memory-index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"memory-index.d.ts","sourceRoot":"","sources":["../../../src/diff/storage/memory-index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzD;;GAEG;AACH,qBAAa,eAAgB,YAAW,SAAS;IAC/C,OAAO,CAAC,KAAK,CAAiC;IAC9C,OAAO,CAAC,cAAc,CAAK;IAErB,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAShD,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,SAAS,CAAC;IAIjD,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAIlC,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IASpC,OAAO,IAAI,aAAa,CAAC,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IAMrD,KAAK,IAAI,MAAM;IAIf,iBAAiB,IAAI,MAAM;IAI3B,aAAa,IAAI;QAAE,SAAS,EAAE,OAAO,CAAC;QAAC,YAAY,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE;IAQ3E,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAK5B,OAAO,CAAC,gBAAgB;CAezB"}

package/dist/diff/storage/memory-index.js

@@ -0,0 +1,69 @@
+/**
+ * High-performance in-memory Diff Index backed by JavaScript Map.
+ */
+export class MemoryDiffIndex {
+    store = new Map();
+    estimatedBytes = 0;
+    async set(key, row) {
+        const existing = this.store.get(key);
+        if (existing) {
+            this.estimatedBytes -= this.estimateRowBytes(key, existing);
+        }
+        this.store.set(key, row);
+        this.estimatedBytes += this.estimateRowBytes(key, row);
+    }
+    async get(key) {
+        return this.store.get(key);
+    }
+    async has(key) {
+        return this.store.has(key);
+    }
+    async delete(key) {
+        const existing = this.store.get(key);
+        if (existing) {
+            this.estimatedBytes -= this.estimateRowBytes(key, existing);
+            return this.store.delete(key);
+        }
+        return false;
+    }
+    async *entries() {
+        for (const entry of this.store.entries()) {
+            yield entry;
+        }
+    }
+    count() {
+        return this.store.size;
+    }
+    getEstimatedBytes() {
+        return this.estimatedBytes;
+    }
+    getSpillStats() {
+        return {
+            isSpilled: false,
+            spilledBytes: 0,
+            entryCount: this.store.size,
+        };
+    }
+    async close() {
+        this.store.clear();
+        this.estimatedBytes = 0;
+    }
+    estimateRowBytes(key, item) {
+        // Key string bytes + row keys/values + Map object entry overhead in V8 (~128 bytes)
+        let bytes = key.length * 2 + 128;
+        for (const [k, v] of Object.entries(item.row)) {
+            bytes += k.length * 2 + 48;
+            if (typeof v === "string") {
+                bytes += v.length * 2 + 32;
+            }
+            else if (typeof v === "number" || typeof v === "boolean") {
+                bytes += 16;
+            }
+            else if (v !== null && v !== undefined) {
+                bytes += JSON.stringify(v).length * 2 + 48;
+            }
+        }
+        return bytes;
+    }
+}
+//# sourceMappingURL=memory-index.js.map

package/dist/diff/storage/memory-index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"memory-index.js","sourceRoot":"","sources":["../../../src/diff/storage/memory-index.ts"],"names":[],"mappings":"AAEA;;GAEG;AACH,MAAM,OAAO,eAAe;IAClB,KAAK,GAAG,IAAI,GAAG,EAAsB,CAAC;IACtC,cAAc,GAAG,CAAC,CAAC;IAE3B,KAAK,CAAC,GAAG,CAAC,GAAW,EAAE,GAAe;QACpC,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACrC,IAAI,QAAQ,EAAE,CAAC;YACb,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,gBAAgB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;QAC9D,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QACzB,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,gBAAgB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IACzD,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,GAAW;QACnB,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,GAAW;QACnB,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,GAAW;QACtB,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACrC,IAAI,QAAQ,EAAE,CAAC;YACb,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,gBAAgB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;YAC5D,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAChC,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,KAAK,CAAC,CAAC,OAAO;QACZ,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;YACzC,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,KAAK;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;IACzB,CAAC;IAED,iBAAiB;QACf,OAAO,IAAI,CAAC,cAAc,CAAC;IAC7B,CAAC;IAED,aAAa;QACX,OAAO;YACL,SAAS,EAAE,KAAK;YAChB,YAAY,EAAE,CAAC;YACf,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI;SAC5B,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC;IAC1B,CAAC;IAEO,gBAAgB,CAAC,GAAW,EAAE,IAAgB;QACpD,oFAAoF;QACpF,IAAI,KAAK,GAAG,GAAG,CAAC,MAAM,GAAG,CAAC,GAAG,GAAG,CAAC;QACjC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YAC9C,KAAK,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,GAAG,EAAE,CAAC;YAC3B,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;gBAC1B,KAAK,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,GAAG,EAAE,CAAC;YAC7B,CAAC;iBAAM,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,SAAS,EAAE,CAAC;gBAC3D,KAAK,IAAI,EAAE,CAAC;YACd,CAAC;iBAAM,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,SAAS,EAAE,CAAC;gBACzC,KAAK,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,GAAG,EAAE,CAAC;YAC7C,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;CACF"}

package/dist/diff/storage/spillable-index.d.ts

@@ -0,0 +1,30 @@
+import type { DiffIndex, IndexedRow } from "../types.js";
+/**
+ * Parses a human-readable memory limit string (e.g. "256mb", "1gb", "64MB") into bytes.
+ */
+export declare function parseMemoryLimit(input?: string | number, defaultBytes?: number): number;
+/**
+ * Hybrid Spillable Diff Index that stays in memory for high speed,
+ * and automatically spills to partitioned disk storage when memory exceeds the threshold.
+ */
+export declare class SpillableDiffIndex implements DiffIndex {
+    private memoryIndex;
+    private diskIndex?;
+    private memoryLimitBytes;
+    private isSpilled;
+    constructor(memoryLimit?: number | string);
+    private spillToDisk;
+    set(key: string, row: IndexedRow): Promise<void>;
+    get(key: string): Promise<IndexedRow | undefined>;
+    has(key: string): Promise<boolean>;
+    delete(key: string): Promise<boolean>;
+    entries(): AsyncIterable<[string, IndexedRow]>;
+    count(): number;
+    getSpillStats(): {
+        isSpilled: boolean;
+        spilledBytes: number;
+        entryCount: number;
+    };
+    close(): Promise<void>;
+}
+//# sourceMappingURL=spillable-index.d.ts.map

package/dist/diff/storage/spillable-index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"spillable-index.d.ts","sourceRoot":"","sources":["../../../src/diff/storage/spillable-index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAIzD;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,YAAY,SAAoB,GAAG,MAAM,CAwBlG;AAED;;;GAGG;AACH,qBAAa,kBAAmB,YAAW,SAAS;IAClD,OAAO,CAAC,WAAW,CAAyB;IAC5C,OAAO,CAAC,SAAS,CAAC,CAAgB;IAClC,OAAO,CAAC,gBAAgB,CAAS;IACjC,OAAO,CAAC,SAAS,CAAS;gBAEd,WAAW,GAAE,MAAM,GAAG,MAA0B;YAI9C,WAAW;IAYnB,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAahD,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,SAAS,CAAC;IAOjD,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAOlC,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAOpC,OAAO,IAAI,aAAa,CAAC,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IAQrD,KAAK,IAAI,MAAM;IAOf,aAAa,IAAI;QAAE,SAAS,EAAE,OAAO,CAAC;QAAC,YAAY,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE;IAO3E,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAM7B"}

package/dist/diff/storage/spillable-index.js

@@ -0,0 +1,108 @@
+import { DiskDiffIndex } from "./disk-index.js";
+import { MemoryDiffIndex } from "./memory-index.js";
+/**
+ * Parses a human-readable memory limit string (e.g. "256mb", "1gb", "64MB") into bytes.
+ */
+export function parseMemoryLimit(input, defaultBytes = 256 * 1024 * 1024) {
+    if (!input)
+        return defaultBytes;
+    if (typeof input === "number")
+        return input;
+    const str = input.trim().toLowerCase();
+    const match = /^(\d+(?:\.\d+)?)\s*([a-z]+)?$/.exec(str);
+    if (!match)
+        return defaultBytes;
+    const val = Number.parseFloat(match[1]);
+    const unit = match[2] || "b";
+    switch (unit) {
+        case "gb":
+        case "g":
+            return Math.round(val * 1024 * 1024 * 1024);
+        case "mb":
+        case "m":
+            return Math.round(val * 1024 * 1024);
+        case "kb":
+        case "k":
+            return Math.round(val * 1024);
+        default:
+            return Math.round(val);
+    }
+}
+/**
+ * Hybrid Spillable Diff Index that stays in memory for high speed,
+ * and automatically spills to partitioned disk storage when memory exceeds the threshold.
+ */
+export class SpillableDiffIndex {
+    memoryIndex = new MemoryDiffIndex();
+    diskIndex;
+    memoryLimitBytes;
+    isSpilled = false;
+    constructor(memoryLimit = 256 * 1024 * 1024) {
+        this.memoryLimitBytes = parseMemoryLimit(memoryLimit);
+    }
+    async spillToDisk() {
+        if (this.isSpilled)
+            return;
+        this.diskIndex = new DiskDiffIndex();
+        for await (const [key, row] of this.memoryIndex.entries()) {
+            await this.diskIndex.set(key, row);
+        }
+        await this.memoryIndex.close();
+        this.isSpilled = true;
+    }
+    async set(key, row) {
+        if (this.isSpilled && this.diskIndex) {
+            await this.diskIndex.set(key, row);
+            return;
+        }
+        await this.memoryIndex.set(key, row);
+        if (this.memoryIndex.getEstimatedBytes() >= this.memoryLimitBytes) {
+            await this.spillToDisk();
+        }
+    }
+    async get(key) {
+        if (this.isSpilled && this.diskIndex) {
+            return this.diskIndex.get(key);
+        }
+        return this.memoryIndex.get(key);
+    }
+    async has(key) {
+        if (this.isSpilled && this.diskIndex) {
+            return this.diskIndex.has(key);
+        }
+        return this.memoryIndex.has(key);
+    }
+    async delete(key) {
+        if (this.isSpilled && this.diskIndex) {
+            return this.diskIndex.delete(key);
+        }
+        return this.memoryIndex.delete(key);
+    }
+    async *entries() {
+        if (this.isSpilled && this.diskIndex) {
+            yield* this.diskIndex.entries();
+        }
+        else {
+            yield* this.memoryIndex.entries();
+        }
+    }
+    count() {
+        if (this.isSpilled && this.diskIndex) {
+            return this.diskIndex.count();
+        }
+        return this.memoryIndex.count();
+    }
+    getSpillStats() {
+        if (this.isSpilled && this.diskIndex) {
+            return this.diskIndex.getSpillStats();
+        }
+        return this.memoryIndex.getSpillStats();
+    }
+    async close() {
+        await this.memoryIndex.close();
+        if (this.diskIndex) {
+            await this.diskIndex.close();
+        }
+    }
+}
+//# sourceMappingURL=spillable-index.js.map

package/dist/diff/storage/spillable-index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"spillable-index.js","sourceRoot":"","sources":["../../../src/diff/storage/spillable-index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAChD,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAEpD;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAAuB,EAAE,YAAY,GAAG,GAAG,GAAG,IAAI,GAAG,IAAI;IACxF,IAAI,CAAC,KAAK;QAAE,OAAO,YAAY,CAAC;IAChC,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAE5C,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACvC,MAAM,KAAK,GAAG,+BAA+B,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACxD,IAAI,CAAC,KAAK;QAAE,OAAO,YAAY,CAAC;IAEhC,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAC;IACzC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC;IAE7B,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,IAAI,CAAC;QACV,KAAK,GAAG;YACN,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC;QAC9C,KAAK,IAAI,CAAC;QACV,KAAK,GAAG;YACN,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC;QACvC,KAAK,IAAI,CAAC;QACV,KAAK,GAAG;YACN,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC,CAAC;QAChC;YACE,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC3B,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,OAAO,kBAAkB;IACrB,WAAW,GAAG,IAAI,eAAe,EAAE,CAAC;IACpC,SAAS,CAAiB;IAC1B,gBAAgB,CAAS;IACzB,SAAS,GAAG,KAAK,CAAC;IAE1B,YAAY,cAA+B,GAAG,GAAG,IAAI,GAAG,IAAI;QAC1D,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC,WAAW,CAAC,CAAC;IACxD,CAAC;IAEO,KAAK,CAAC,WAAW;QACvB,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO;QAE3B,IAAI,CAAC,SAAS,GAAG,IAAI,aAAa,EAAE,CAAC;QACrC,IAAI,KAAK,EAAE,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,EAAE,CAAC;YAC1D,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QACrC,CAAC;QAED,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;QAC/B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,GAAW,EAAE,GAAe;QACpC,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACrC,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;YACnC,OAAO;QACT,CAAC;QAED,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QAErC,IAAI,IAAI,CAAC,WAAW,CAAC,iBAAiB,EAAE,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAClE,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,GAAW;QACnB,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACrC,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACjC,CAAC;QACD,OAAO,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACnC,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,GAAW;QACnB,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACrC,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACjC,CAAC;QACD,OAAO,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACnC,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,GAAW;QACtB,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACrC,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACpC,CAAC;QACD,OAAO,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,CAAC,OAAO;QACZ,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACrC,KAAK,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;QAClC,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;QACpC,CAAC;IACH,CAAC;IAED,KAAK;QACH,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACrC,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;QAChC,CAAC;QACD,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;IAClC,CAAC;IAED,aAAa;QACX,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACrC,OAAO,IAAI,CAAC,SAAS,CAAC,aAAa,EAAE,CAAC;QACxC,CAAC;QACD,OAAO,IAAI,CAAC,WAAW,CAAC,aAAa,EAAE,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;QAC/B,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;QAC/B,CAAC;IACH,CAAC;CACF"}