rowpipe 1.2.0 → 2.1.0

package/dist/writers/markdown.js.map

@@ -1 +1 @@
-{"version":3,"file":"markdown.js","sourceRoot":"","sources":["../../src/writers/markdown.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,yBAAyB,EAAE,MAAM,yBAAyB,CAAC;AAQzF;;;GAGG;AACH,MAAM,OAAO,cAAc;IACjB,MAAM,CAAoB;IAC1B,OAAO,CAAwB;IAC/B,aAAa,CAAY;IAEjC,YAAY,MAAyB,EAAE,UAAiC,EAAE;QACxE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;IAChC,CAAC;IAEO,eAAe,CAAC,IAAoB;QAC1C,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;QAC7C,MAAM,MAAM,GAAG,yBAAyB,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;QAChE,IAAI,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YACpC,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC;QAC9B,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,UAAU,CAAC,KAAc;QAC/B,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YAC1C,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,MAAM,GAAG,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC9E,OAAO,GAAG;aACP,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC;aACrB,OAAO,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC/B,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,MAAgB,EAAE,IAAY;QACrD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;QACtE,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,UAAsB,EAAE,OAAuB;QACzD,MAAM,aAAa,GAA0B;YAC3C,GAAG,IAAI,CAAC,OAAO;YACf,GAAG,OAAO;SACX,CAAC;QAEF,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,aAAa,CAAC,CAAC;QACtD,IAAI,OAAO,GAAoB,aAAa,CAAC,OAAO,IAAI,IAAI,CAAC;QAE7D,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;YACrC,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAEtC,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;gBACrB,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC3C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBACzB,OAAO,GAAG,CAAC,OAAO,CAAC,CAAC;gBACtB,CAAC;gBAED,oCAAoC;gBACpC,MAAM,UAAU,GAAG,KAAK,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;gBAClD,iCAAiC;gBACjC,MAAM,aAAa,GAAG,KAAK,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;gBACtE,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,UAAU,GAAG,aAAa,CAAC,CAAC;YAC/D,CAAC;YAED,IAAI,KAAK,GAAG,EAAE,CAAC;YACf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC3C,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC;gBAC3B,MAAM,KAAK,GAAa,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;gBAClD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBACxC,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,CAAE,CAAC;oBAC5B,KAAK,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;gBAC3C,CAAC;gBACD,KAAK,IAAI,KAAK,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;YACxC,CAAC;YAED,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACrB,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;YAC1C,CAAC;QACH,CAAC;QAED,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;YACrB,gBAAgB;YAChB,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,sBAAsB,CAAC,CAAC;QAC3D,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACvB,MAAM,mBAAmB,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;CACF"}
+{"version":3,"file":"markdown.js","sourceRoot":"","sources":["../../src/writers/markdown.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,yBAAyB,EAAE,MAAM,yBAAyB,CAAC;AAQzF;;;GAGG;AACH,MAAM,OAAO,cAAc;IACjB,MAAM,CAAoB;IAC1B,OAAO,CAAwB;IAC/B,aAAa,CAAY;IAEjC,YAAY,MAAyB,EAAE,UAAiC,EAAE;QACxE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;IAChC,CAAC;IAEO,eAAe,CAAC,IAAoB;QAC1C,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;QAC7C,MAAM,MAAM,GAAG,yBAAyB,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;QAChE,IAAI,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YACpC,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC;QAC9B,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,UAAU,CAAC,KAAc;QAC/B,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YAC1C,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,MAAM,GAAG,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC9E,OAAO,GAAG;aACP,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC;aACrB,OAAO,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC/B,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,MAAgB,EAAE,IAAY;QACrD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;QACtE,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,UAAsB,EAAE,OAAuB;QACzD,MAAM,aAAa,GAA0B;YAC3C,GAAG,IAAI,CAAC,OAAO;YACf,GAAG,OAAO;SACX,CAAC;QAEF,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,aAAa,CAAC,CAAC;QACtD,IAAI,OAAO,GAAoB,aAAa,CAAC,OAAO,IAAI,IAAI,CAAC;QAC7D,MAAM,OAAO,GAAe,EAAE,CAAC;QAE/B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;YACrC,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAEtC,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;gBACrB,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC3C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBACzB,OAAO,GAAG,CAAC,OAAO,CAAC,CAAC;gBACtB,CAAC;YACH,CAAC;YAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC3C,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC;gBAC3B,MAAM,KAAK,GAAa,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;gBAClD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBACxC,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,CAAE,CAAC;oBAC5B,KAAK,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;gBAC3C,CAAC;gBACD,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;QAED,IAAI,OAAO,KAAK,IAAI,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7C,gBAAgB;YAChB,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,4BAA4B,CAAC,CAAC;YAC/D,OAAO;QACT,CAAC;QAED,mCAAmC;QACnC,MAAM,SAAS,GAAa,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;YACpD,IAAI,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;YACnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACxC,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;gBACxC,IAAI,IAAI,CAAC,MAAM,GAAG,MAAM,EAAE,CAAC;oBACzB,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;gBACvB,CAAC;YACH,CAAC;YACD,OAAO,MAAM,CAAC;QAChB,CAAC,CAAC,CAAC;QAEH,0DAA0D;QAC1D,MAAM,YAAY,GAAc,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;YACxD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,KAAK,CAAC;YACvC,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBAC3B,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC;gBAChC,OAAO,CAAC,GAAG,IAAI,iCAAiC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC7D,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,uBAAuB;QACvB,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACvC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAE,CAAC,CACtE,CAAC;QACF,MAAM,UAAU,GAAG,KAAK,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;QAEtD,6CAA6C;QAC7C,MAAM,cAAc,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAC5C,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CACpF,CAAC;QACF,MAAM,aAAa,GAAG,KAAK,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;QAE5D,IAAI,MAAM,GAAG,UAAU,GAAG,aAAa,CAAC;QAExC,sBAAsB;QACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACxC,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAE,CAAC;YACxB,MAAM,QAAQ,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CACnC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAE,CAAC,CAC5E,CAAC;YACF,MAAM,IAAI,KAAK,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;QAC5C,CAAC;QAED,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAC3C,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACvB,MAAM,mBAAmB,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;CACF"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rowpipe",
-  "version": "1.2.0",
+  "version": "2.1.0",
   "description": "Stream-first tabular data toolkit for CSV, TSV, JSON, JSONL, XLSX, Apache Parquet, and Markdown",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -20,7 +20,8 @@
     "test:watch": "vitest",
     "bench": "tsx benchmarks/memory-bench.ts",
     "benchmark:diff": "tsx benchmarks/diff-bench.ts",
-    "benchmark:files": "tsx benchmarks/files-bench.ts"
+    "benchmark:files": "tsx benchmarks/files-bench.ts",
+    "benchmark:ops": "tsx benchmarks/operations-bench.ts"
   },
   "keywords": [
     "csv",

package/skills/rowpipe/SKILL.md

@@ -20,6 +20,15 @@ Rowpipe is a high-performance, stream-first command-line toolkit and Node.js lib
 | **Map** | `rowpipe map <file> "<col1=expr1>" "<col2=expr2>"` | Derive and compute columns per row with arithmetic/JSON dot-notation |
 | **Filter** | `rowpipe filter <file> "<expression>"` | Filter rows using safe expression engine and pipe syntax (`\|`) |
 | **Reduce** | `rowpipe reduce <file> "<agg1=func(col)>" [--by <cols>]` | Global & group-by streaming aggregations (`sum`, `avg`, `min`, `max`, `count`, `countDistinct`, `stddev`) |
+| **Limit / Head** | `rowpipe limit <file> <n>` / `rowpipe head <file> [-n <n>]` | Emit first N rows with early upstream stream cancellation |
+| **Offset** | `rowpipe offset <file> <n>` | Skip first N rows without scanning remainder |
+| **Tail** | `rowpipe tail <file> [-n <n>]` | Emit last N rows using bounded $O(N)$ circular ring buffer |
+| **Sort** | `rowpipe sort <file> --by <cols> [--memory-limit <m>]` | Multi-column typed external merge sort with disk run spilling and $K$-way min-heap |
+| **Top** | `rowpipe top <file> --by <cols> [-n <n>]` | Extract Top-K largest/smallest records using bounded $O(K)$ heap |
+| **Unique** | `rowpipe unique <file> [--by <cols>] [--keep first\|last]` | Deduplicate rows by key columns or entire rows with spillable hash key store |
+| **Count** | `rowpipe count <file> [--by <cols>] [--distinct <col>]` | Ultra-fast row total, distinct count (HyperLogLog), or group-count |
+| **Group** | `rowpipe group <file> --by <cols> [--count] [--sum <c>]` | Group tabular streams with hash accumulation and multi-column aggregations |
+| **Explain** | `rowpipe explain <file> [pipeline flags]` | Visualize planned execution pipeline and rule-based optimizations |
 | **Select** | `rowpipe select <file> <col1,col2,...>` | Project subset of columns in order |
 | **Rename** | `rowpipe rename <file> <old1=new1> <old2=new2>` | Rename columns in stream |
 | **Cast** | `rowpipe cast <file> <col1:type> [--on-error null]` | Stream cast types (`integer`, `number`, `boolean`, `date`) |
@@ -170,24 +179,33 @@ rowpipe diff old.csv new.csv --key id --json
 rowpipe diff expected.csv actual.csv --key id --fail-on-diff
 ```
 
-### 8. Filesystem as Streaming Tabular Records (`files`)
+### 8. Filesystem as Streaming Tabular Records (`files`) & Pretty Bytes
 Stream files and directory trees with bounded memory, globs, streaming hashes, and MIME detection:
 
 ```bash
-# 1. Stream directory records
-rowpipe files ./src --include "**/*.ts"
-
-# 2. Filter large files and select columns via Unix pipe
-rowpipe files ./uploads | rowpipe filter - 'size > 1000000' | rowpipe select - relative_path,size
+# 1. Stream directory records with human-readable sizes (-h / --human)
+rowpipe files ./src -h --include "**/*.ts"
+
+# 2. Filter files by human byte string (parsebytes) and derive pretty size (formatbytes)
+rowpipe files ./uploads | \
+  rowpipe filter - 'size > parsebytes("10MB")' | \
+  rowpipe map - 'size_h = formatbytes(size)' | \
+  rowpipe select - relative_path,size_h
+
+# 3. Group by extension with pretty total size in Markdown table
+rowpipe files . | \
+  rowpipe reduce - 'count=count()' 'total_bytes=sum(size)' --by extension | \
+  rowpipe map - 'size_pretty=formatbytes(total_bytes)' | \
+  rowpipe convert - --to markdown
 
-# 3. Stream SHA-256 or fast hashes
+# 4. Stream SHA-256 or fast hashes
 rowpipe files ./assets --hash sha256 --to jsonl
 rowpipe files ./dataset --hash fast
 
-# 4. Generate deterministic build snapshot
+# 5. Generate deterministic build snapshot
 rowpipe files snapshot ./dist --hash sha256 --to jsonl > dist.snapshot.jsonl
 
-# 5. Direct directory diffing
+# 6. Direct directory diffing
 rowpipe files diff ./dist-v1 ./dist-v2 --hash sha256
 ```