rowpipe 1.1.0 → 2.0.0

package/dist/writers/csv.js.map

@@ -1 +1 @@
-{"version":3,"file":"csv.js","sourceRoot":"","sources":["../../src/writers/csv.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,yBAAyB,EAAE,MAAM,yBAAyB,CAAC;AASzF,MAAM,OAAO,SAAS;IACZ,MAAM,CAAoB;IAC1B,OAAO,CAAmB;IAC1B,aAAa,CAAY;IAEjC,YAAY,MAAyB,EAAE,UAA4B,EAAE;QACnE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;IAChC,CAAC;IAEO,eAAe,CAAC,IAAoB;QAC1C,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;QAC7C,MAAM,MAAM,GAAG,yBAAyB,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;QAChE,IAAI,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YACpC,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC;QAC9B,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,WAAW,CAAC,KAAc,EAAE,SAAiB;QACnD,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YAC1C,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,MAAM,GAAG,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC9E,MAAM,SAAS,GACb,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC;YACvB,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC;YACjB,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC;YAClB,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAErB,IAAI,SAAS,EAAE,CAAC;YACd,OAAO,IAAI,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC;QACxC,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,MAAgB,EAAE,IAAY;QACrD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;QACtE,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,UAAsB,EAAE,OAAuB;QACzD,MAAM,aAAa,GAAqB;YACtC,GAAG,IAAI,CAAC,OAAO;YACf,GAAG,OAAO;SACX,CAAC;QACF,MAAM,SAAS,GAAG,aAAa,CAAC,SAAS,IAAI,GAAG,CAAC;QACjD,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,KAAK,KAAK,CAAC;QAEnD,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,aAAa,CAAC,CAAC;QACtD,IAAI,OAAO,GAAoB,IAAI,CAAC;QAEpC,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;YACrC,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAEtC,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;gBACrB,4DAA4D;gBAC5D,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;gBACjC,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;oBAC7B,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;wBACnC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;oBAClB,CAAC;gBACH,CAAC;gBACD,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBAE7B,IAAI,WAAW,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACtC,MAAM,UAAU,GACd,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC;wBAClE,IAAI,CAAC;oBACP,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;gBAC/C,CAAC;YACH,CAAC;YAED,IAAI,SAAS,GAAG,EAAE,CAAC;YACnB,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;gBAC7B,MAAM,IAAI,GACR,OAAO;qBACJ,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;qBAC/C,IAAI,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC;gBAC5B,SAAS,IAAI,IAAI,CAAC;YACpB,CAAC;YAED,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACzB,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;YAC9C,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACvB,MAAM,mBAAmB,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;CACF"}
+{"version":3,"file":"csv.js","sourceRoot":"","sources":["../../src/writers/csv.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,yBAAyB,EAAE,MAAM,yBAAyB,CAAC;AASzF,MAAM,OAAO,SAAS;IACZ,MAAM,CAAoB;IAC1B,OAAO,CAAmB;IAC1B,aAAa,CAAY;IAEjC,YAAY,MAAyB,EAAE,UAA4B,EAAE;QACnE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;IAChC,CAAC;IAEO,eAAe,CAAC,IAAoB;QAC1C,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;QAC7C,MAAM,MAAM,GAAG,yBAAyB,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;QAChE,IAAI,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YACpC,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC;QAC9B,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,WAAW,CAAC,KAAc,EAAE,SAAiB;QACnD,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YAC1C,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9B,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;QACvB,CAAC;QACD,IAAI,OAAO,KAAK,KAAK,SAAS,EAAE,CAAC;YAC/B,OAAO,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC;QAClC,CAAC;QACD,MAAM,GAAG,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QACpH,MAAM,SAAS,GACb,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC;YACvB,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC;YACjB,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC;YAClB,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAErB,IAAI,SAAS,EAAE,CAAC;YACd,OAAO,IAAI,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC;QACxC,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,MAAgB,EAAE,IAAY;QACrD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;QACtE,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,UAAsB,EAAE,OAAuB;QACzD,MAAM,aAAa,GAAqB;YACtC,GAAG,IAAI,CAAC,OAAO;YACf,GAAG,OAAO;SACX,CAAC;QACF,MAAM,SAAS,GAAG,aAAa,CAAC,SAAS,IAAI,GAAG,CAAC;QACjD,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,KAAK,KAAK,CAAC;QAEnD,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,aAAa,CAAC,CAAC;QACtD,IAAI,OAAO,GAAoB,IAAI,CAAC;QAEpC,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;YACrC,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAEtC,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;gBACrB,4DAA4D;gBAC5D,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;gBACjC,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;oBAC7B,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;wBACnC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;oBAClB,CAAC;gBACH,CAAC;gBACD,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBAE7B,IAAI,WAAW,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACtC,IAAI,UAAU,GAAG,EAAE,CAAC;oBACpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;wBACxC,IAAI,CAAC,GAAG,CAAC;4BAAE,UAAU,IAAI,SAAS,CAAC;wBACnC,UAAU,IAAI,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;oBACxD,CAAC;oBACD,UAAU,IAAI,IAAI,CAAC;oBACnB,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;gBAC/C,CAAC;YACH,CAAC;YAED,IAAI,SAAS,GAAG,EAAE,CAAC;YACnB,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC;YAC5B,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;gBAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC9B,IAAI,CAAC,GAAG,CAAC;wBAAE,SAAS,IAAI,SAAS,CAAC;oBAClC,SAAS,IAAI,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC,EAAE,SAAS,CAAC,CAAC;gBAC7D,CAAC;gBACD,SAAS,IAAI,IAAI,CAAC;YACpB,CAAC;YAED,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACzB,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;YAC9C,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACvB,MAAM,mBAAmB,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;CACF"}

package/dist/writers/markdown.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"markdown.d.ts","sourceRoot":"","sources":["../../src/writers/markdown.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjF,MAAM,WAAW,qBAAsB,SAAQ,aAAa;IAC1D,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;CACpB;AAED;;;GAGG;AACH,qBAAa,cAAe,YAAW,aAAa;IAClD,OAAO,CAAC,MAAM,CAAoB;IAClC,OAAO,CAAC,OAAO,CAAwB;IACvC,OAAO,CAAC,aAAa,CAAC,CAAW;gBAErB,MAAM,EAAE,QAAQ,GAAG,MAAM,EAAE,OAAO,GAAE,qBAA0B;IAK1E,OAAO,CAAC,eAAe;IASvB,OAAO,CAAC,UAAU;YAUJ,UAAU;IAMlB,KAAK,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IA+CrE,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAK7B"}
+{"version":3,"file":"markdown.d.ts","sourceRoot":"","sources":["../../src/writers/markdown.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjF,MAAM,WAAW,qBAAsB,SAAQ,aAAa;IAC1D,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;CACpB;AAED;;;GAGG;AACH,qBAAa,cAAe,YAAW,aAAa;IAClD,OAAO,CAAC,MAAM,CAAoB;IAClC,OAAO,CAAC,OAAO,CAAwB;IACvC,OAAO,CAAC,aAAa,CAAC,CAAW;gBAErB,MAAM,EAAE,QAAQ,GAAG,MAAM,EAAE,OAAO,GAAE,qBAA0B;IAK1E,OAAO,CAAC,eAAe;IASvB,OAAO,CAAC,UAAU;YAUJ,UAAU;IAMlB,KAAK,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IAsFrE,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAK7B"}

package/dist/writers/markdown.js

@@ -40,6 +40,7 @@ export class MarkdownWriter {
         };
         const outStream = this.getOutputStream(mergedOptions);
         let headers = mergedOptions.headers ?? null;
+        const allRows = [];
         for await (const batch of dataStream) {
             if (batch.rows.length === 0)
                 continue;
@@ -48,13 +49,7 @@ export class MarkdownWriter {
                 if (headers.length === 0) {
                     headers = ["value"];
                 }
-                // Write header row: | Col1 | Col2 |
-                const headerLine = `| ${headers.join(" | ")} |\n`;
-                // Write separator row: |---|---|
-                const separatorLine = `| ${headers.map(() => "---").join(" | ")} |\n`;
-                await this.writeChunk(outStream, headerLine + separatorLine);
             }
-            let chunk = "";
             for (let r = 0; r < batch.rows.length; r++) {
                 const row = batch.rows[r];
                 const cells = new Array(headers.length);
@@ -62,15 +57,49 @@ export class MarkdownWriter {
                     const colName = headers[i];
                     cells[i] = this.escapeCell(row[colName]);
                 }
-                chunk += `| ${cells.join(" | ")} |\n`;
-            }
-            if (chunk.length > 0) {
-                await this.writeChunk(outStream, chunk);
+                allRows.push(cells);
             }
         }
-        if (headers === null) {
+        if (headers === null || headers.length === 0) {
             // Empty dataset
-            await this.writeChunk(outStream, "| (empty) |\n|---|\n");
+            await this.writeChunk(outStream, "| (empty) |\n| ------- |\n");
+            return;
+        }
+        // Compute maximum width per column
+        const colWidths = headers.map((h, colIdx) => {
+            let maxLen = Math.max(3, h.length);
+            for (let r = 0; r < allRows.length; r++) {
+                const cell = allRows[r]?.[colIdx] ?? "";
+                if (cell.length > maxLen) {
+                    maxLen = cell.length;
+                }
+            }
+            return maxLen;
+        });
+        // Detect numeric / byte columns for clean right-alignment
+        const isNumericCol = headers.map((_, colIdx) => {
+            if (allRows.length === 0)
+                return false;
+            return allRows.every((row) => {
+                const val = row[colIdx]?.trim();
+                return !val || /^-?\d+(\.\d+)?(\s*[KMGTPE]?B)?$/.test(val);
+            });
+        });
+        // 1. Render Header Row
+        const headerCells = headers.map((h, i) => isNumericCol[i] ? h.padStart(colWidths[i]) : h.padEnd(colWidths[i]));
+        const headerLine = `| ${headerCells.join(" | ")} |\n`;
+        // 2. Render Separator Row (e.g. |:---|---:|)
+        const separatorCells = colWidths.map((w, i) => isNumericCol[i] ? "-".repeat(Math.max(2, w - 1)) + ":" : "-".repeat(Math.max(3, w)));
+        const separatorLine = `| ${separatorCells.join(" | ")} |\n`;
+        let buffer = headerLine + separatorLine;
+        // 3. Render Data Rows
+        for (let r = 0; r < allRows.length; r++) {
+            const row = allRows[r];
+            const rowCells = row.map((cell, i) => isNumericCol[i] ? cell.padStart(colWidths[i]) : cell.padEnd(colWidths[i]));
+            buffer += `| ${rowCells.join(" | ")} |\n`;
+        }
+        if (buffer.length > 0) {
+            await this.writeChunk(outStream, buffer);
         }
     }
     async close() {

package/dist/writers/markdown.js.map

@@ -1 +1 @@
-{"version":3,"file":"markdown.js","sourceRoot":"","sources":["../../src/writers/markdown.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,yBAAyB,EAAE,MAAM,yBAAyB,CAAC;AAQzF;;;GAGG;AACH,MAAM,OAAO,cAAc;IACjB,MAAM,CAAoB;IAC1B,OAAO,CAAwB;IAC/B,aAAa,CAAY;IAEjC,YAAY,MAAyB,EAAE,UAAiC,EAAE;QACxE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;IAChC,CAAC;IAEO,eAAe,CAAC,IAAoB;QAC1C,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;QAC7C,MAAM,MAAM,GAAG,yBAAyB,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;QAChE,IAAI,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YACpC,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC;QAC9B,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,UAAU,CAAC,KAAc;QAC/B,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YAC1C,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,MAAM,GAAG,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC9E,OAAO,GAAG;aACP,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC;aACrB,OAAO,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC/B,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,MAAgB,EAAE,IAAY;QACrD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;QACtE,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,UAAsB,EAAE,OAAuB;QACzD,MAAM,aAAa,GAA0B;YAC3C,GAAG,IAAI,CAAC,OAAO;YACf,GAAG,OAAO;SACX,CAAC;QAEF,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,aAAa,CAAC,CAAC;QACtD,IAAI,OAAO,GAAoB,aAAa,CAAC,OAAO,IAAI,IAAI,CAAC;QAE7D,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;YACrC,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAEtC,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;gBACrB,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC3C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBACzB,OAAO,GAAG,CAAC,OAAO,CAAC,CAAC;gBACtB,CAAC;gBAED,oCAAoC;gBACpC,MAAM,UAAU,GAAG,KAAK,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;gBAClD,iCAAiC;gBACjC,MAAM,aAAa,GAAG,KAAK,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;gBACtE,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,UAAU,GAAG,aAAa,CAAC,CAAC;YAC/D,CAAC;YAED,IAAI,KAAK,GAAG,EAAE,CAAC;YACf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC3C,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC;gBAC3B,MAAM,KAAK,GAAa,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;gBAClD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBACxC,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,CAAE,CAAC;oBAC5B,KAAK,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;gBAC3C,CAAC;gBACD,KAAK,IAAI,KAAK,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;YACxC,CAAC;YAED,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACrB,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;YAC1C,CAAC;QACH,CAAC;QAED,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;YACrB,gBAAgB;YAChB,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,sBAAsB,CAAC,CAAC;QAC3D,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACvB,MAAM,mBAAmB,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;CACF"}
+{"version":3,"file":"markdown.js","sourceRoot":"","sources":["../../src/writers/markdown.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,yBAAyB,EAAE,MAAM,yBAAyB,CAAC;AAQzF;;;GAGG;AACH,MAAM,OAAO,cAAc;IACjB,MAAM,CAAoB;IAC1B,OAAO,CAAwB;IAC/B,aAAa,CAAY;IAEjC,YAAY,MAAyB,EAAE,UAAiC,EAAE;QACxE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;IAChC,CAAC;IAEO,eAAe,CAAC,IAAoB;QAC1C,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;QAC7C,MAAM,MAAM,GAAG,yBAAyB,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;QAChE,IAAI,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YACpC,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC;QAC9B,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,UAAU,CAAC,KAAc;QAC/B,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YAC1C,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,MAAM,GAAG,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC9E,OAAO,GAAG;aACP,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC;aACrB,OAAO,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC/B,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,MAAgB,EAAE,IAAY;QACrD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;QACtE,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,UAAsB,EAAE,OAAuB;QACzD,MAAM,aAAa,GAA0B;YAC3C,GAAG,IAAI,CAAC,OAAO;YACf,GAAG,OAAO;SACX,CAAC;QAEF,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,aAAa,CAAC,CAAC;QACtD,IAAI,OAAO,GAAoB,aAAa,CAAC,OAAO,IAAI,IAAI,CAAC;QAC7D,MAAM,OAAO,GAAe,EAAE,CAAC;QAE/B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;YACrC,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAEtC,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;gBACrB,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC3C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBACzB,OAAO,GAAG,CAAC,OAAO,CAAC,CAAC;gBACtB,CAAC;YACH,CAAC;YAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC3C,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC;gBAC3B,MAAM,KAAK,GAAa,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;gBAClD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBACxC,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,CAAE,CAAC;oBAC5B,KAAK,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;gBAC3C,CAAC;gBACD,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;QAED,IAAI,OAAO,KAAK,IAAI,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7C,gBAAgB;YAChB,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,4BAA4B,CAAC,CAAC;YAC/D,OAAO;QACT,CAAC;QAED,mCAAmC;QACnC,MAAM,SAAS,GAAa,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;YACpD,IAAI,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;YACnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACxC,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;gBACxC,IAAI,IAAI,CAAC,MAAM,GAAG,MAAM,EAAE,CAAC;oBACzB,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;gBACvB,CAAC;YACH,CAAC;YACD,OAAO,MAAM,CAAC;QAChB,CAAC,CAAC,CAAC;QAEH,0DAA0D;QAC1D,MAAM,YAAY,GAAc,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;YACxD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,KAAK,CAAC;YACvC,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBAC3B,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC;gBAChC,OAAO,CAAC,GAAG,IAAI,iCAAiC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC7D,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,uBAAuB;QACvB,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACvC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAE,CAAC,CACtE,CAAC;QACF,MAAM,UAAU,GAAG,KAAK,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;QAEtD,6CAA6C;QAC7C,MAAM,cAAc,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAC5C,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CACpF,CAAC;QACF,MAAM,aAAa,GAAG,KAAK,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;QAE5D,IAAI,MAAM,GAAG,UAAU,GAAG,aAAa,CAAC;QAExC,sBAAsB;QACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACxC,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAE,CAAC;YACxB,MAAM,QAAQ,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CACnC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAE,CAAC,CAC5E,CAAC;YACF,MAAM,IAAI,KAAK,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;QAC5C,CAAC;QAED,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAC3C,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACvB,MAAM,mBAAmB,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;CACF"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rowpipe",
-  "version": "1.1.0",
+  "version": "2.0.0",
   "description": "Stream-first tabular data toolkit for CSV, TSV, JSON, JSONL, XLSX, Apache Parquet, and Markdown",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -18,7 +18,9 @@
     "dev": "tsx src/cli/index.ts",
     "test": "vitest run",
     "test:watch": "vitest",
-    "bench": "tsx benchmarks/memory-bench.ts"
+    "bench": "tsx benchmarks/memory-bench.ts",
+    "benchmark:diff": "tsx benchmarks/diff-bench.ts",
+    "benchmark:files": "tsx benchmarks/files-bench.ts"
   },
   "keywords": [
     "csv",

package/skills/rowpipe/SKILL.md

@@ -25,6 +25,8 @@ Rowpipe is a high-performance, stream-first command-line toolkit and Node.js lib
 | **Cast** | `rowpipe cast <file> <col1:type> [--on-error null]` | Stream cast types (`integer`, `number`, `boolean`, `date`) |
 | **Sample** | `rowpipe sample <file> --rows <n> [--seed <s>]` | Reservoir sampling with bounded memory |
 | **Validate** | `rowpipe validate <file> --schema <schema.json>` | Validate stream against JSON schema definition |
+| **Diff** | `rowpipe diff <left> <right> --key <cols> [opts]` | Stream compare datasets by key across formats with bounded RAM and disk spilling |
+| **Files** | `rowpipe files <path> [--include <glob>] [opts]` | Stream filesystem directory as tabular records with bounded memory, globs, hashes, and MIME |
 
 ---
 
@@ -138,6 +140,66 @@ cat transactions.csv.gz | \
   rowpipe convert - --to markdown
 ```
 
+### 7. Dataset Diffing & Change Feeds (`diff`)
+Compare two datasets by key across formats with bounded memory ($O(1)$) and automatic disk spilling:
+
+```bash
+# 1. Basic Keyed Diff
+rowpipe diff yesterday.csv today.csv --key id
+
+# 2. Composite Key & Cross-Format (CSV vs Parquet)
+rowpipe diff old.csv new.parquet --key country,user_id
+
+# 3. Multi-Sheet Excel Workbook Diff
+rowpipe diff old.xlsx new.xlsx --sheet Users --key id
+
+# 4. Filter Columns & String Normalization
+rowpipe diff old.csv new.csv \
+  --key id \
+  --ignore updated_at,last_seen \
+  --trim \
+  --ignore-case \
+  --epsilon 0.001
+
+# 5. Output Modes: Rows, JSONL Patch, and JSON
+rowpipe diff old.csv new.csv --key id --format rows --only changed --limit 50
+rowpipe diff old.csv new.csv --key id --format patch > changes.jsonl
+rowpipe diff old.csv new.csv --key id --json
+
+# 6. CI/CD Validation (Fails with exit code 1 if differences found)
+rowpipe diff expected.csv actual.csv --key id --fail-on-diff
+```
+
+### 8. Filesystem as Streaming Tabular Records (`files`) & Pretty Bytes
+Stream files and directory trees with bounded memory, globs, streaming hashes, and MIME detection:
+
+```bash
+# 1. Stream directory records with human-readable sizes (-h / --human)
+rowpipe files ./src -h --include "**/*.ts"
+
+# 2. Filter files by human byte string (parsebytes) and derive pretty size (formatbytes)
+rowpipe files ./uploads | \
+  rowpipe filter - 'size > parsebytes("10MB")' | \
+  rowpipe map - 'size_h = formatbytes(size)' | \
+  rowpipe select - relative_path,size_h
+
+# 3. Group by extension with pretty total size in Markdown table
+rowpipe files . | \
+  rowpipe reduce - 'count=count()' 'total_bytes=sum(size)' --by extension | \
+  rowpipe map - 'size_pretty=formatbytes(total_bytes)' | \
+  rowpipe convert - --to markdown
+
+# 4. Stream SHA-256 or fast hashes
+rowpipe files ./assets --hash sha256 --to jsonl
+rowpipe files ./dataset --hash fast
+
+# 5. Generate deterministic build snapshot
+rowpipe files snapshot ./dist --hash sha256 --to jsonl > dist.snapshot.jsonl
+
+# 6. Direct directory diffing
+rowpipe files diff ./dist-v1 ./dist-v2 --hash sha256
+```
+
 ---
 
 ## 💻 Node.js Library API
@@ -153,9 +215,12 @@ import {
   ParquetWriter,
   MarkdownWriter,
   XLSXReader,
+  FileSystemReader,
   mapRows,
   filterRows,
   reduceRows,
+  diffRows,
+  computeDiff,
 } from "rowpipe";
 import { createReadStream, createWriteStream } from "node:fs";
 
@@ -178,4 +243,30 @@ await createPipeline(reader)
     ],
   }))
   .to(writer);
+
+// 2. Stream Filesystem Directory as Tabular Rows
+const fsReader = new FileSystemReader({
+  root: "./src",
+  include: ["**/*.ts"],
+  hash: "sha256",
+  mime: true,
+});
+
+for await (const batch of fsReader.read()) {
+  for (const file of batch.rows) {
+    console.log(`${file.relative_path} (${file.size} bytes) - Hash: ${file.hash}`);
+  }
+}
+
+// 3. Streaming Diff Events
+for await (const event of diffRows({
+  leftPath: "old.csv",
+  rightPath: "new.parquet",
+  keys: ["id"],
+  ignore: ["updated_at"],
+})) {
+  if (event.type === "changed") {
+    console.log(`Row ${JSON.stringify(event.key)} changed:`, event.changes);
+  }
+}
 ```