rowpipe 2.0.0 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +420 -59
- package/dist/analytics/profiler.d.ts +56 -0
- package/dist/analytics/profiler.d.ts.map +1 -0
- package/dist/analytics/profiler.js +277 -0
- package/dist/analytics/profiler.js.map +1 -0
- package/dist/analytics/schema-inference.d.ts.map +1 -1
- package/dist/analytics/schema-inference.js +14 -0
- package/dist/analytics/schema-inference.js.map +1 -1
- package/dist/cli/commands/clean.d.ts +15 -0
- package/dist/cli/commands/clean.d.ts.map +1 -0
- package/dist/cli/commands/clean.js +108 -0
- package/dist/cli/commands/clean.js.map +1 -0
- package/dist/cli/commands/convert.d.ts +6 -0
- package/dist/cli/commands/convert.d.ts.map +1 -1
- package/dist/cli/commands/convert.js +6 -6
- package/dist/cli/commands/convert.js.map +1 -1
- package/dist/cli/commands/count.d.ts +12 -0
- package/dist/cli/commands/count.d.ts.map +1 -0
- package/dist/cli/commands/count.js +49 -0
- package/dist/cli/commands/count.js.map +1 -0
- package/dist/cli/commands/db.d.ts +35 -0
- package/dist/cli/commands/db.d.ts.map +1 -0
- package/dist/cli/commands/db.js +173 -0
- package/dist/cli/commands/db.js.map +1 -0
- package/dist/cli/commands/explain.d.ts +10 -0
- package/dist/cli/commands/explain.d.ts.map +1 -0
- package/dist/cli/commands/explain.js +98 -0
- package/dist/cli/commands/explain.js.map +1 -0
- package/dist/cli/commands/files.d.ts +2 -1
- package/dist/cli/commands/files.d.ts.map +1 -1
- package/dist/cli/commands/files.js +5 -2
- package/dist/cli/commands/files.js.map +1 -1
- package/dist/cli/commands/group.d.ts +12 -0
- package/dist/cli/commands/group.d.ts.map +1 -0
- package/dist/cli/commands/group.js +51 -0
- package/dist/cli/commands/group.js.map +1 -0
- package/dist/cli/commands/head.d.ts +7 -0
- package/dist/cli/commands/head.d.ts.map +1 -0
- package/dist/cli/commands/head.js +6 -0
- package/dist/cli/commands/head.js.map +1 -0
- package/dist/cli/commands/join.d.ts +23 -0
- package/dist/cli/commands/join.d.ts.map +1 -0
- package/dist/cli/commands/join.js +97 -0
- package/dist/cli/commands/join.js.map +1 -0
- package/dist/cli/commands/limit.d.ts +11 -0
- package/dist/cli/commands/limit.d.ts.map +1 -0
- package/dist/cli/commands/limit.js +40 -0
- package/dist/cli/commands/limit.js.map +1 -0
- package/dist/cli/commands/offset.d.ts +11 -0
- package/dist/cli/commands/offset.d.ts.map +1 -0
- package/dist/cli/commands/offset.js +39 -0
- package/dist/cli/commands/offset.js.map +1 -0
- package/dist/cli/commands/pipeline.d.ts +63 -0
- package/dist/cli/commands/pipeline.d.ts.map +1 -0
- package/dist/cli/commands/pipeline.js +239 -0
- package/dist/cli/commands/pipeline.js.map +1 -0
- package/dist/cli/commands/profile.d.ts +22 -0
- package/dist/cli/commands/profile.d.ts.map +1 -0
- package/dist/cli/commands/profile.js +53 -0
- package/dist/cli/commands/profile.js.map +1 -0
- package/dist/cli/commands/sort.d.ts +17 -0
- package/dist/cli/commands/sort.d.ts.map +1 -0
- package/dist/cli/commands/sort.js +55 -0
- package/dist/cli/commands/sort.js.map +1 -0
- package/dist/cli/commands/tail.d.ts +13 -0
- package/dist/cli/commands/tail.d.ts.map +1 -0
- package/dist/cli/commands/tail.js +39 -0
- package/dist/cli/commands/tail.js.map +1 -0
- package/dist/cli/commands/top.d.ts +19 -0
- package/dist/cli/commands/top.d.ts.map +1 -0
- package/dist/cli/commands/top.js +51 -0
- package/dist/cli/commands/top.js.map +1 -0
- package/dist/cli/commands/unique.d.ts +15 -0
- package/dist/cli/commands/unique.d.ts.map +1 -0
- package/dist/cli/commands/unique.js +44 -0
- package/dist/cli/commands/unique.js.map +1 -0
- package/dist/cli/commands/view.d.ts +12 -0
- package/dist/cli/commands/view.d.ts.map +1 -0
- package/dist/cli/commands/view.js +52 -0
- package/dist/cli/commands/view.js.map +1 -0
- package/dist/cli/commands/window.d.ts +10 -0
- package/dist/cli/commands/window.d.ts.map +1 -0
- package/dist/cli/commands/window.js +87 -0
- package/dist/cli/commands/window.js.map +1 -0
- package/dist/cli/index.js +448 -3
- package/dist/cli/index.js.map +1 -1
- package/dist/core/batch.d.ts +1 -1
- package/dist/core/batch.d.ts.map +1 -1
- package/dist/core/batch.js.map +1 -1
- package/dist/core/pipeline.d.ts +42 -2
- package/dist/core/pipeline.d.ts.map +1 -1
- package/dist/core/pipeline.js +70 -2
- package/dist/core/pipeline.js.map +1 -1
- package/dist/core/types.d.ts +21 -1
- package/dist/core/types.d.ts.map +1 -1
- package/dist/db/adapters/index.d.ts +6 -0
- package/dist/db/adapters/index.d.ts.map +1 -0
- package/dist/db/adapters/index.js +21 -0
- package/dist/db/adapters/index.js.map +1 -0
- package/dist/db/adapters/mysql.d.ts +16 -0
- package/dist/db/adapters/mysql.d.ts.map +1 -0
- package/dist/db/adapters/mysql.js +230 -0
- package/dist/db/adapters/mysql.js.map +1 -0
- package/dist/db/adapters/postgres.d.ts +17 -0
- package/dist/db/adapters/postgres.d.ts.map +1 -0
- package/dist/db/adapters/postgres.js +239 -0
- package/dist/db/adapters/postgres.js.map +1 -0
- package/dist/db/adapters/sqlite.d.ts +18 -0
- package/dist/db/adapters/sqlite.d.ts.map +1 -0
- package/dist/db/adapters/sqlite.js +210 -0
- package/dist/db/adapters/sqlite.js.map +1 -0
- package/dist/db/index.d.ts +11 -0
- package/dist/db/index.d.ts.map +1 -0
- package/dist/db/index.js +11 -0
- package/dist/db/index.js.map +1 -0
- package/dist/db/mapping.d.ts +45 -0
- package/dist/db/mapping.d.ts.map +1 -0
- package/dist/db/mapping.js +222 -0
- package/dist/db/mapping.js.map +1 -0
- package/dist/db/pushdown.d.ts +25 -0
- package/dist/db/pushdown.d.ts.map +1 -0
- package/dist/db/pushdown.js +165 -0
- package/dist/db/pushdown.js.map +1 -0
- package/dist/db/sink.d.ts +15 -0
- package/dist/db/sink.d.ts.map +1 -0
- package/dist/db/sink.js +47 -0
- package/dist/db/sink.js.map +1 -0
- package/dist/db/source.d.ts +14 -0
- package/dist/db/source.d.ts.map +1 -0
- package/dist/db/source.js +69 -0
- package/dist/db/source.js.map +1 -0
- package/dist/db/types.d.ts +80 -0
- package/dist/db/types.d.ts.map +1 -0
- package/dist/db/types.js +2 -0
- package/dist/db/types.js.map +1 -0
- package/dist/db/url.d.ts +15 -0
- package/dist/db/url.d.ts.map +1 -0
- package/dist/db/url.js +145 -0
- package/dist/db/url.js.map +1 -0
- package/dist/diff/hash.d.ts.map +1 -1
- package/dist/diff/hash.js +4 -3
- package/dist/diff/hash.js.map +1 -1
- package/dist/diff/key.d.ts.map +1 -1
- package/dist/diff/key.js +8 -1
- package/dist/diff/key.js.map +1 -1
- package/dist/diff/reporter.d.ts.map +1 -1
- package/dist/diff/reporter.js +5 -3
- package/dist/diff/reporter.js.map +1 -1
- package/dist/diff/storage/disk-index.d.ts.map +1 -1
- package/dist/diff/storage/disk-index.js +2 -1
- package/dist/diff/storage/disk-index.js.map +1 -1
- package/dist/diff/storage/memory-index.d.ts.map +1 -1
- package/dist/diff/storage/memory-index.js +3 -2
- package/dist/diff/storage/memory-index.js.map +1 -1
- package/dist/index.d.ts +20 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +22 -1
- package/dist/index.js.map +1 -1
- package/dist/planner/executor.d.ts +8 -0
- package/dist/planner/executor.d.ts.map +1 -0
- package/dist/planner/executor.js +93 -0
- package/dist/planner/executor.js.map +1 -0
- package/dist/planner/explain.d.ts +6 -0
- package/dist/planner/explain.d.ts.map +1 -0
- package/dist/planner/explain.js +91 -0
- package/dist/planner/explain.js.map +1 -0
- package/dist/planner/index.d.ts +5 -0
- package/dist/planner/index.d.ts.map +1 -0
- package/dist/planner/index.js +5 -0
- package/dist/planner/index.js.map +1 -0
- package/dist/planner/optimizer.d.ts +9 -0
- package/dist/planner/optimizer.d.ts.map +1 -0
- package/dist/planner/optimizer.js +78 -0
- package/dist/planner/optimizer.js.map +1 -0
- package/dist/planner/types.d.ts +67 -0
- package/dist/planner/types.d.ts.map +1 -0
- package/dist/planner/types.js +2 -0
- package/dist/planner/types.js.map +1 -0
- package/dist/readers/csv.d.ts.map +1 -1
- package/dist/readers/csv.js +1 -2
- package/dist/readers/csv.js.map +1 -1
- package/dist/readers/index.d.ts +5 -1
- package/dist/readers/index.d.ts.map +1 -1
- package/dist/readers/index.js +43 -0
- package/dist/readers/index.js.map +1 -1
- package/dist/readers/json.d.ts.map +1 -1
- package/dist/readers/json.js +1 -2
- package/dist/readers/json.js.map +1 -1
- package/dist/readers/jsonl.d.ts.map +1 -1
- package/dist/readers/jsonl.js +1 -2
- package/dist/readers/jsonl.js.map +1 -1
- package/dist/readers/multi-file.d.ts +29 -0
- package/dist/readers/multi-file.d.ts.map +1 -0
- package/dist/readers/multi-file.js +169 -0
- package/dist/readers/multi-file.js.map +1 -0
- package/dist/transforms/clean.d.ts +18 -0
- package/dist/transforms/clean.d.ts.map +1 -0
- package/dist/transforms/clean.js +118 -0
- package/dist/transforms/clean.js.map +1 -0
- package/dist/transforms/count.d.ts +21 -0
- package/dist/transforms/count.d.ts.map +1 -0
- package/dist/transforms/count.js +65 -0
- package/dist/transforms/count.js.map +1 -0
- package/dist/transforms/expression.d.ts.map +1 -1
- package/dist/transforms/expression.js +12 -0
- package/dist/transforms/expression.js.map +1 -1
- package/dist/transforms/group.d.ts +25 -0
- package/dist/transforms/group.d.ts.map +1 -0
- package/dist/transforms/group.js +71 -0
- package/dist/transforms/group.js.map +1 -0
- package/dist/transforms/jit-compiler.d.ts +50 -0
- package/dist/transforms/jit-compiler.d.ts.map +1 -0
- package/dist/transforms/jit-compiler.js +275 -0
- package/dist/transforms/jit-compiler.js.map +1 -0
- package/dist/transforms/join/index-storage.d.ts +33 -0
- package/dist/transforms/join/index-storage.d.ts.map +1 -0
- package/dist/transforms/join/index-storage.js +139 -0
- package/dist/transforms/join/index-storage.js.map +1 -0
- package/dist/transforms/join/index.d.ts +4 -0
- package/dist/transforms/join/index.d.ts.map +1 -0
- package/dist/transforms/join/index.js +4 -0
- package/dist/transforms/join/index.js.map +1 -0
- package/dist/transforms/join/join.d.ts +15 -0
- package/dist/transforms/join/join.d.ts.map +1 -0
- package/dist/transforms/join/join.js +216 -0
- package/dist/transforms/join/join.js.map +1 -0
- package/dist/transforms/join/types.d.ts +29 -0
- package/dist/transforms/join/types.d.ts.map +1 -0
- package/dist/transforms/join/types.js +2 -0
- package/dist/transforms/join/types.js.map +1 -0
- package/dist/transforms/limit.d.ts +7 -0
- package/dist/transforms/limit.d.ts.map +1 -0
- package/dist/transforms/limit.js +40 -0
- package/dist/transforms/limit.js.map +1 -0
- package/dist/transforms/offset.d.ts +7 -0
- package/dist/transforms/offset.d.ts.map +1 -0
- package/dist/transforms/offset.js +38 -0
- package/dist/transforms/offset.js.map +1 -0
- package/dist/transforms/sort/comparator.d.ts +43 -0
- package/dist/transforms/sort/comparator.d.ts.map +1 -0
- package/dist/transforms/sort/comparator.js +131 -0
- package/dist/transforms/sort/comparator.js.map +1 -0
- package/dist/transforms/sort/external-sort.d.ts +8 -0
- package/dist/transforms/sort/external-sort.d.ts.map +1 -0
- package/dist/transforms/sort/external-sort.js +159 -0
- package/dist/transforms/sort/external-sort.js.map +1 -0
- package/dist/transforms/sort/index.d.ts +10 -0
- package/dist/transforms/sort/index.d.ts.map +1 -0
- package/dist/transforms/sort/index.js +11 -0
- package/dist/transforms/sort/index.js.map +1 -0
- package/dist/transforms/tail.d.ts +6 -0
- package/dist/transforms/tail.d.ts.map +1 -0
- package/dist/transforms/tail.js +27 -0
- package/dist/transforms/tail.js.map +1 -0
- package/dist/transforms/top.d.ts +17 -0
- package/dist/transforms/top.d.ts.map +1 -0
- package/dist/transforms/top.js +52 -0
- package/dist/transforms/top.js.map +1 -0
- package/dist/transforms/unique.d.ts +13 -0
- package/dist/transforms/unique.d.ts.map +1 -0
- package/dist/transforms/unique.js +91 -0
- package/dist/transforms/unique.js.map +1 -0
- package/dist/transforms/window.d.ts +24 -0
- package/dist/transforms/window.d.ts.map +1 -0
- package/dist/transforms/window.js +317 -0
- package/dist/transforms/window.js.map +1 -0
- package/dist/ui/colors.d.ts +57 -0
- package/dist/ui/colors.d.ts.map +1 -0
- package/dist/ui/colors.js +94 -0
- package/dist/ui/colors.js.map +1 -0
- package/dist/ui/viewer.d.ts +67 -0
- package/dist/ui/viewer.d.ts.map +1 -0
- package/dist/ui/viewer.js +673 -0
- package/dist/ui/viewer.js.map +1 -0
- package/dist/utils/compression.d.ts +68 -19
- package/dist/utils/compression.d.ts.map +1 -1
- package/dist/utils/compression.js +222 -34
- package/dist/utils/compression.js.map +1 -1
- package/dist/utils/formatting.d.ts +5 -0
- package/dist/utils/formatting.d.ts.map +1 -1
- package/dist/utils/formatting.js +15 -0
- package/dist/utils/formatting.js.map +1 -1
- package/dist/utils/heap.d.ts +25 -0
- package/dist/utils/heap.d.ts.map +1 -0
- package/dist/utils/heap.js +90 -0
- package/dist/utils/heap.js.map +1 -0
- package/dist/utils/keystore.d.ts +80 -0
- package/dist/utils/keystore.d.ts.map +1 -0
- package/dist/utils/keystore.js +217 -0
- package/dist/utils/keystore.js.map +1 -0
- package/dist/utils/ring-buffer.d.ts +15 -0
- package/dist/utils/ring-buffer.d.ts.map +1 -0
- package/dist/utils/ring-buffer.js +40 -0
- package/dist/utils/ring-buffer.js.map +1 -0
- package/dist/writers/csv.d.ts.map +1 -1
- package/dist/writers/csv.js +13 -14
- package/dist/writers/csv.js.map +1 -1
- package/dist/writers/index.d.ts +1 -0
- package/dist/writers/index.d.ts.map +1 -1
- package/dist/writers/index.js +35 -0
- package/dist/writers/index.js.map +1 -1
- package/dist/writers/json.d.ts.map +1 -1
- package/dist/writers/json.js +6 -5
- package/dist/writers/json.js.map +1 -1
- package/dist/writers/jsonl.d.ts.map +1 -1
- package/dist/writers/jsonl.js +6 -5
- package/dist/writers/jsonl.js.map +1 -1
- package/dist/writers/markdown.d.ts.map +1 -1
- package/dist/writers/markdown.js +5 -4
- package/dist/writers/markdown.js.map +1 -1
- package/dist/writers/parquet.d.ts +1 -1
- package/dist/writers/parquet.d.ts.map +1 -1
- package/package.json +11 -2
- package/skills/rowpipe/SKILL.md +16 -3
package/README.md
CHANGED
|
@@ -13,11 +13,15 @@ Designed around stream backpressure and batch processing, Rowpipe processes mult
|
|
|
13
13
|
## Highlights
|
|
14
14
|
|
|
15
15
|
- **Stream-First Architecture**: Datasets are never buffered entirely in memory; rows flow in configurable batches through async generator pipelines.
|
|
16
|
-
- **Rich Format Ecosystem**: Native streaming support for **CSV, TSV / PSV, JSON, JSONL, XLSX, Apache Parquet (`.parquet`), Markdown tables (`.md`), and Gzip (`.gz`)**.
|
|
16
|
+
- **Rich Format Ecosystem**: Native streaming support for **PostgreSQL, MySQL, SQLite, CSV, TSV / PSV, JSON, JSONL, XLSX, Apache Parquet (`.parquet`), Markdown tables (`.md`), and Gzip (`.gz`)**.
|
|
17
|
+
- **First-Class Database Streaming (`rowpipe db`)**: True cursor-based streaming sources and sinks for PostgreSQL, MySQL, and SQLite. "Databases are just another Rowpipe source and sink" — rows normalize directly into standard `DataBatch` objects with zero parallel data models.
|
|
18
|
+
- **Intelligent Pushdown Optimizer**: Automatically pushes compatible operations (`SELECT`, `WHERE`, `ORDER BY`, `LIMIT`, `OFFSET`) down into database queries in `--table` mode while seamlessly executing non-pushdownable transforms in the local streaming pipeline.
|
|
19
|
+
- **Database Writing, Migration & DDL**: Stream directly into database tables (`--to-db`) with automatic table creation (`--create-table`), batched inserts, transactions (`--transaction`), dialect-specific upsert (`--upsert --conflict <cols>`), and truncate (`--truncate`).
|
|
20
|
+
- **Lossless Type Precision**: Preserves full `BIGINT` and arbitrary precision `DECIMAL` / `NUMERIC` values without floating point corruption. Safely maps `JSONB`, `BYTEA`/`BLOB`, dates, and timestamps.
|
|
21
|
+
- **Connection Security & Credential Masking**: Automatically sanitizes database passwords across all logs, explain plans, errors, and metadata (`postgres://user:***@host/db`).
|
|
17
22
|
- **Apache Parquet Support**: High-performance columnar binary format reader (Snappy, Zstd, Gzip decompression via pure JS) and streaming writer.
|
|
18
|
-
- **Cross-Format Conversion & Multi-Sheet Export**: Convert seamlessly across all formats. Export individual Excel sheets or bulk export all worksheets with `--all-sheets`.
|
|
19
23
|
- **Filesystem as Data (`rowpipe files`)**: First-class streaming filesystem source adapter. Directories and files become streams of tabular records (`path`, `size`, `type`, `extension`, `modified_at`, streaming `hash`, `mime`) composable with `filter`, `select`, `stats`, `convert`, and `diff`.
|
|
20
|
-
- **Cross-Format Dataset Diffing (`rowpipe diff`)**: High-performance, stream-first comparison across CSV, JSONL, XLSX, and
|
|
24
|
+
- **Cross-Format Dataset Diffing (`rowpipe diff`)**: High-performance, stream-first comparison across CSV, JSONL, XLSX, Parquet, and Database tables with bounded memory and automatic spill-to-disk. Supports composite keys, duplicate policies, value tolerance, column filtering, schema diffing, JSONL patch generation, and CI `--fail-on-diff` mode.
|
|
21
25
|
|
|
22
26
|
- **Zero-`eval` Safe Expression Engine**: Ultra-fast JIT closure-compiled evaluators supporting arithmetic (`revenue - cost`), string/date pipe functions (`email | lower | trim`), and direct JSON dot-notation (`payload.user.city`).
|
|
23
27
|
- **Map & Reduce Streaming Engine**: Compute row derivations (`rowpipe map`) and global or group-by aggregations (`rowpipe reduce --by country`) in a single pass.
|
|
@@ -549,8 +553,390 @@ rowpipe diff dist.snapshot.v1.jsonl dist.snapshot.v2.jsonl \
|
|
|
549
553
|
#### 5. Native Directory Diffing (`rowpipe files diff`)
|
|
550
554
|
Compare two directory trees directly:
|
|
551
555
|
|
|
556
|
+
---
|
|
557
|
+
|
|
558
|
+
## Databases
|
|
559
|
+
|
|
560
|
+
Rowpipe provides first-class streaming database support for **PostgreSQL, MySQL, and SQLite**.
|
|
561
|
+
|
|
562
|
+
The key architectural rule is:
|
|
563
|
+
|
|
564
|
+
> **Databases are just another Rowpipe source and sink.**
|
|
565
|
+
|
|
566
|
+
Database rows normalize directly into standard Rowpipe `DataBatch` and `Row` objects. There is no parallel execution engine or secondary row type — existing operations (`filter`, `select`, `sort`, `group`, `reduce`, `stats`, `diff`) compose natively without intermediate files or memory buffering.
|
|
567
|
+
|
|
568
|
+
```text
|
|
569
|
+
PostgreSQL / MySQL / SQLite / CSV / JSONL / Parquet / Files
|
|
570
|
+
↓
|
|
571
|
+
DataBatch / Row
|
|
572
|
+
↓
|
|
573
|
+
filter / select / map / sort
|
|
574
|
+
group / reduce / stats / diff
|
|
575
|
+
↓
|
|
576
|
+
TabularWriter
|
|
577
|
+
↓
|
|
578
|
+
PostgreSQL / MySQL / SQLite / CSV / JSONL / Parquet / Markdown
|
|
579
|
+
```
|
|
580
|
+
|
|
581
|
+
### 1. Database Streaming Sources
|
|
582
|
+
|
|
583
|
+
Read directly from database tables or custom queries in bounded, cursor-based streams:
|
|
584
|
+
|
|
585
|
+
```bash
|
|
586
|
+
# Read from PostgreSQL table in streaming batches
|
|
587
|
+
rowpipe db postgres://localhost/mydb --table users
|
|
588
|
+
|
|
589
|
+
# Read from MySQL database
|
|
590
|
+
rowpipe db mysql://root@localhost/app --table orders
|
|
591
|
+
|
|
592
|
+
# Read from SQLite database file
|
|
593
|
+
rowpipe db sqlite://./data.db --table events
|
|
594
|
+
# or direct file path
|
|
595
|
+
rowpipe db ./data.db --table events
|
|
596
|
+
|
|
597
|
+
# Execute custom SQL query
|
|
598
|
+
rowpipe db postgres://localhost/mydb \
|
|
599
|
+
--query 'SELECT id, email, created_at FROM users WHERE active = true'
|
|
600
|
+
|
|
601
|
+
# Parameterized queries (driver-level binding)
|
|
602
|
+
rowpipe db postgres://localhost/mydb \
|
|
603
|
+
--query 'SELECT * FROM users WHERE age > $1 AND status = $2' \
|
|
604
|
+
--params '[30, "ACTIVE"]'
|
|
605
|
+
```
|
|
606
|
+
|
|
607
|
+
### 2. Output to Any Rowpipe Format
|
|
608
|
+
|
|
609
|
+
Database streams flow directly through Rowpipe writers to CSV, JSONL, Parquet, or Markdown tables:
|
|
610
|
+
|
|
552
611
|
```bash
|
|
553
|
-
|
|
612
|
+
# Stream PostgreSQL table to JSONL
|
|
613
|
+
rowpipe db postgres://localhost/app --table users --to jsonl > users.jsonl
|
|
614
|
+
|
|
615
|
+
# Stream MySQL query to CSV
|
|
616
|
+
rowpipe db mysql://localhost/app --query 'SELECT * FROM orders' --to csv > orders.csv
|
|
617
|
+
|
|
618
|
+
# Stream SQLite table to GitHub Markdown table
|
|
619
|
+
rowpipe db sqlite://./sales.db --table sales --limit 20 --to markdown
|
|
620
|
+
```
|
|
621
|
+
|
|
622
|
+
### 3. Pipeline Operations & Pushdown Optimizer
|
|
623
|
+
|
|
624
|
+
When querying tables with `--table`, Rowpipe's **Intelligent Pushdown Optimizer** analyzes the pipeline and pushes compatible operations directly into database SQL:
|
|
625
|
+
|
|
626
|
+
* `select` $\rightarrow$ `SELECT col1, col2`
|
|
627
|
+
* `filter` $\rightarrow$ `WHERE condition`
|
|
628
|
+
* `sort` $\rightarrow$ `ORDER BY col DESC`
|
|
629
|
+
* `limit` $\rightarrow$ `LIMIT N`
|
|
630
|
+
* `offset` $\rightarrow$ `OFFSET N`
|
|
631
|
+
|
|
632
|
+
Unsupported or complex operations safely remain in the local streaming pipeline.
|
|
633
|
+
|
|
634
|
+
```bash
|
|
635
|
+
# Pushes down WHERE, SELECT, ORDER BY, and LIMIT to database SQL
|
|
636
|
+
rowpipe db postgres://localhost/app \
|
|
637
|
+
--table users \
|
|
638
|
+
--filter 'active == true and age > 25' \
|
|
639
|
+
--select id,email,created_at \
|
|
640
|
+
--sort created_at:desc \
|
|
641
|
+
--limit 100 \
|
|
642
|
+
--to jsonl
|
|
643
|
+
|
|
644
|
+
# Mixed execution: pushdown WHERE + local aggregations
|
|
645
|
+
rowpipe db sqlite://./sales.db \
|
|
646
|
+
--table sales \
|
|
647
|
+
--where 'year = 2026' \
|
|
648
|
+
--group-by country \
|
|
649
|
+
--sum revenue \
|
|
650
|
+
--sort revenue_sum:desc
|
|
651
|
+
```
|
|
652
|
+
|
|
653
|
+
### 4. Database Sinks & File-to-Database Loading
|
|
654
|
+
|
|
655
|
+
Load CSV, JSONL, XLSX, Parquet, or other databases directly into database tables in batched transactions:
|
|
656
|
+
|
|
657
|
+
```bash
|
|
658
|
+
# Load CSV into PostgreSQL table with automatic DDL table creation
|
|
659
|
+
rowpipe users.csv \
|
|
660
|
+
--to-db postgres://localhost/app \
|
|
661
|
+
--to-table users \
|
|
662
|
+
--create-table
|
|
663
|
+
|
|
664
|
+
# Load JSONL into SQLite database
|
|
665
|
+
rowpipe events.jsonl \
|
|
666
|
+
--to-db sqlite://./app.db \
|
|
667
|
+
--to-table events \
|
|
668
|
+
--create-table
|
|
669
|
+
|
|
670
|
+
# Dialect-specific Upsert on conflict keys
|
|
671
|
+
rowpipe users.csv \
|
|
672
|
+
--to-db postgres://localhost/app \
|
|
673
|
+
--to-table users \
|
|
674
|
+
--upsert \
|
|
675
|
+
--conflict id
|
|
676
|
+
|
|
677
|
+
# Destructive Truncate before bulk inserting
|
|
678
|
+
rowpipe new_catalog.parquet \
|
|
679
|
+
--to-db mysql://localhost/store \
|
|
680
|
+
--to-table catalog \
|
|
681
|
+
--truncate \
|
|
682
|
+
--transaction
|
|
683
|
+
```
|
|
684
|
+
|
|
685
|
+
### 5. Database-to-Database Migrations (Zero Intermediate Files)
|
|
686
|
+
|
|
687
|
+
Stream data directly between different database dialects with on-the-fly transformations:
|
|
688
|
+
|
|
689
|
+
```bash
|
|
690
|
+
# Migrate PostgreSQL table directly to SQLite with filtering & projection
|
|
691
|
+
rowpipe db postgres://production/app \
|
|
692
|
+
--table users \
|
|
693
|
+
--filter 'deleted_at == null' \
|
|
694
|
+
--select id,email,created_at \
|
|
695
|
+
--to-db sqlite://./backup.db \
|
|
696
|
+
--to-table active_users \
|
|
697
|
+
--create-table
|
|
698
|
+
```
|
|
699
|
+
|
|
700
|
+
### 6. Cross-Format Diffing with Databases
|
|
701
|
+
|
|
702
|
+
Diff a database table directly against a CSV, JSONL, or Parquet file using Rowpipe's diff engine:
|
|
703
|
+
|
|
704
|
+
```bash
|
|
705
|
+
# Compare PostgreSQL table against CSV export
|
|
706
|
+
rowpipe diff \
|
|
707
|
+
'postgres://localhost/app?table=users' \
|
|
708
|
+
users_backup.csv \
|
|
709
|
+
--key id \
|
|
710
|
+
--coerce
|
|
711
|
+
```
|
|
712
|
+
|
|
713
|
+
### 7. Database Introspection
|
|
714
|
+
|
|
715
|
+
Inspect tables and schema types without third-party database clients:
|
|
716
|
+
|
|
717
|
+
```bash
|
|
718
|
+
# List all tables in database
|
|
719
|
+
rowpipe db postgres://localhost/app --tables
|
|
720
|
+
|
|
721
|
+
# Inspect column schema, data types, and nullability
|
|
722
|
+
rowpipe db postgres://localhost/app --schema users
|
|
723
|
+
|
|
724
|
+
# Output schema as machine-readable JSON
|
|
725
|
+
rowpipe db postgres://localhost/app --schema users --json
|
|
726
|
+
```
|
|
727
|
+
|
|
728
|
+
### 8. Lossless Type Mapping & Security
|
|
729
|
+
|
|
730
|
+
- **BIGINT**: Values outside JavaScript safe integer range (e.g. `9223372036854775807`) are preserved losslessly as `bigint` without silent precision loss.
|
|
731
|
+
- **DECIMAL / NUMERIC**: High-precision numbers survive database $\leftrightarrow$ file workflows without floating point truncation.
|
|
732
|
+
- **JSON / JSONB**: Native objects and arrays are preserved and parsed appropriately.
|
|
733
|
+
- **BYTEA / BLOB**: Binary buffers are mapped cleanly without dumping corrupted text to stdout.
|
|
734
|
+
- **Credential Masking**: Connection URLs containing passwords (e.g. `postgres://admin:secret@host/db`) are automatically masked (`postgres://admin:***@host/db`) across all logs, explain plans, progress outputs, and errors.
|
|
735
|
+
|
|
736
|
+
---
|
|
737
|
+
|
|
738
|
+
## Tabular Operations & Execution Planner
|
|
739
|
+
|
|
740
|
+
Rowpipe 2.0 introduces a comprehensive set of Unix-like tabular data operations, powered by a rule-based execution planner that optimizes multi-transform pipelines and enforces strict memory bounds.
|
|
741
|
+
|
|
742
|
+
> **Streaming is the default. Global state is explicit, bounded, and spillable.**
|
|
743
|
+
|
|
744
|
+
### Streaming Safety Classification
|
|
745
|
+
|
|
746
|
+
| Classification | Operations | Memory Complexity | Description |
|
|
747
|
+
|----------------|------------|-------------------|-------------|
|
|
748
|
+
| **Fully Streaming** | `limit`, `offset`, `head`, `filter`, `select`, `rename`, `cast`, `map`, `count` | $O(1)$ | Consumes stream in batches, cancels upstream sources immediately upon completion. |
|
|
749
|
+
| **Bounded State** | `tail`, `top`, `sample` | $O(K)$ / $O(N)$ | Maintains a fixed-size ring buffer ($O(N)$) or binary min/max heap ($O(K)$). |
|
|
750
|
+
| **Spillable Global State** | `sort`, `unique`, `group`, `reduce`, `diff` | $O(1)$ RAM threshold + Disk Spilling | Buffers in-memory runs up to `--memory-limit` and transparently spills to disk partitions or runs. |
|
|
751
|
+
|
|
752
|
+
---
|
|
753
|
+
|
|
754
|
+
### Operations Reference
|
|
755
|
+
|
|
756
|
+
#### 1. Limit & Head
|
|
757
|
+
Emit the first $N$ rows and immediately cancel upstream stream reading:
|
|
758
|
+
|
|
759
|
+
```bash
|
|
760
|
+
# Explicit limit command
|
|
761
|
+
rowpipe limit users.csv 100
|
|
762
|
+
|
|
763
|
+
# Pipeline flag
|
|
764
|
+
rowpipe users.csv --limit 100
|
|
765
|
+
|
|
766
|
+
# Head alias (default: 10 rows)
|
|
767
|
+
rowpipe head users.csv
|
|
768
|
+
rowpipe head users.csv -n 50
|
|
769
|
+
|
|
770
|
+
# Streaming through Unix pipe
|
|
771
|
+
cat data.csv | rowpipe limit - 100
|
|
772
|
+
```
|
|
773
|
+
|
|
774
|
+
#### 2. Offset
|
|
775
|
+
Skip the first $N$ rows and stream everything after without scanning unneeded rows:
|
|
776
|
+
|
|
777
|
+
```bash
|
|
778
|
+
rowpipe offset users.csv 1000
|
|
779
|
+
rowpipe users.csv --offset 1000
|
|
780
|
+
|
|
781
|
+
# Combine with limit (automatically halts reading after 1,100 rows)
|
|
782
|
+
rowpipe users.csv --offset 1000 --limit 100
|
|
783
|
+
```
|
|
784
|
+
|
|
785
|
+
#### 3. Tail
|
|
786
|
+
Emit the last $N$ rows using a bounded circular ring buffer without buffering the entire dataset in RAM:
|
|
787
|
+
|
|
788
|
+
```bash
|
|
789
|
+
# Default: last 10 rows
|
|
790
|
+
rowpipe tail server_logs.jsonl
|
|
791
|
+
|
|
792
|
+
# Specify row count
|
|
793
|
+
rowpipe tail server_logs.jsonl -n 100
|
|
794
|
+
```
|
|
795
|
+
|
|
796
|
+
#### 4. External Merge Sort (`sort`)
|
|
797
|
+
Multi-column typed sorting with deterministic sequence stability, natural sorting, case-insensitivity, null handling, and disk run spilling:
|
|
798
|
+
|
|
799
|
+
```bash
|
|
800
|
+
# Single column sort (numeric, boolean, date, string typed)
|
|
801
|
+
rowpipe sort users.csv --by age:desc
|
|
802
|
+
|
|
803
|
+
# Multi-column sort
|
|
804
|
+
rowpipe sort sales.csv --by country --by revenue:desc
|
|
805
|
+
|
|
806
|
+
# Compact syntax
|
|
807
|
+
rowpipe sort sales.csv --by "country,revenue:desc"
|
|
808
|
+
|
|
809
|
+
# Natural filename / alphanumeric sort
|
|
810
|
+
rowpipe sort files.csv --by name --natural
|
|
811
|
+
|
|
812
|
+
# Null value positioning
|
|
813
|
+
rowpipe sort users.csv --by age --nulls first # or --nulls last
|
|
814
|
+
|
|
815
|
+
# Memory threshold before disk run spilling
|
|
816
|
+
rowpipe sort huge.csv --by created_at --memory-limit 256mb --temp-dir /tmp/rowpipe
|
|
817
|
+
```
|
|
818
|
+
|
|
819
|
+
#### 5. Top-K Bounded Heap (`top`)
|
|
820
|
+
Extract the top (or bottom) $K$ records using a bounded $O(K)$ binary heap:
|
|
821
|
+
|
|
822
|
+
```bash
|
|
823
|
+
# Top 10 largest by revenue
|
|
824
|
+
rowpipe top sales.csv --by revenue -n 10
|
|
825
|
+
|
|
826
|
+
# Top 20 largest files
|
|
827
|
+
rowpipe files ./src --sort size:desc --limit 20 # Automatically rewritten to TopK(20)!
|
|
828
|
+
|
|
829
|
+
# Bottom 5 smallest
|
|
830
|
+
rowpipe top users.csv --by score -n 5 --smallest
|
|
831
|
+
```
|
|
832
|
+
|
|
833
|
+
#### 6. Unique Deduplication (`unique`)
|
|
834
|
+
Deduplicate rows by key columns or entire rows with spillable hash partitions:
|
|
835
|
+
|
|
836
|
+
```bash
|
|
837
|
+
# Unique by single field (keeps first occurrence by default)
|
|
838
|
+
rowpipe unique users.csv --by email
|
|
839
|
+
|
|
840
|
+
# Composite uniqueness
|
|
841
|
+
rowpipe unique users.csv --by country,email
|
|
842
|
+
|
|
843
|
+
# Keep last occurrence
|
|
844
|
+
rowpipe unique logs.jsonl --by user_id --keep last
|
|
845
|
+
|
|
846
|
+
# Entire row deduplication
|
|
847
|
+
rowpipe unique dataset.csv
|
|
848
|
+
```
|
|
849
|
+
|
|
850
|
+
#### 7. Count Analyzer (`count`)
|
|
851
|
+
Ultra-lightweight streaming row, distinct, and group counting:
|
|
852
|
+
|
|
853
|
+
```bash
|
|
854
|
+
# Total row count
|
|
855
|
+
rowpipe count dataset.parquet
|
|
856
|
+
# Output: 8,531,221
|
|
857
|
+
|
|
858
|
+
# Distinct column count (exact or approximate HyperLogLog)
|
|
859
|
+
rowpipe count users.csv --distinct email
|
|
860
|
+
rowpipe count users.csv --distinct email --approx
|
|
861
|
+
|
|
862
|
+
# Group-by count
|
|
863
|
+
rowpipe count users.csv --by country
|
|
864
|
+
|
|
865
|
+
# Machine-readable JSON output
|
|
866
|
+
rowpipe count users.csv --by country --json
|
|
867
|
+
```
|
|
868
|
+
|
|
869
|
+
#### 8. Group & Aggregations (`group`)
|
|
870
|
+
Group tabular streams with single-pass hash accumulators and multi-column aggregations:
|
|
871
|
+
|
|
872
|
+
```bash
|
|
873
|
+
# Group count
|
|
874
|
+
rowpipe group sales.csv --by country --count
|
|
875
|
+
|
|
876
|
+
# Multi-field aggregations
|
|
877
|
+
rowpipe group sales.csv \
|
|
878
|
+
--by country \
|
|
879
|
+
--count \
|
|
880
|
+
--sum revenue \
|
|
881
|
+
--avg revenue \
|
|
882
|
+
--max quantity
|
|
883
|
+
|
|
884
|
+
# Compact aggregation spec
|
|
885
|
+
rowpipe group sales.csv \
|
|
886
|
+
--by country,category \
|
|
887
|
+
--agg "orders=count(),total_rev=sum(revenue),avg_rev=avg(revenue)" \
|
|
888
|
+
--to markdown
|
|
889
|
+
```
|
|
890
|
+
|
|
891
|
+
---
|
|
892
|
+
|
|
893
|
+
### Combined Multi-Transform Pipeline
|
|
894
|
+
|
|
895
|
+
Chain multiple operations in a **single process** without writing temporary intermediate files:
|
|
896
|
+
|
|
897
|
+
```bash
|
|
898
|
+
# Reader -> Filter -> Select -> TopK -> Writer
|
|
899
|
+
rowpipe users.csv \
|
|
900
|
+
--filter "age >= 18 && active == true" \
|
|
901
|
+
--select id,name,age,country,revenue \
|
|
902
|
+
--sort revenue:desc \
|
|
903
|
+
--limit 50 \
|
|
904
|
+
--to markdown
|
|
905
|
+
```
|
|
906
|
+
|
|
907
|
+
### Execution Plan Visualizer (`rowpipe explain`)
|
|
908
|
+
|
|
909
|
+
Inspect the planned execution pipeline and see automatic optimizations in action:
|
|
910
|
+
|
|
911
|
+
```bash
|
|
912
|
+
rowpipe explain sales.csv \
|
|
913
|
+
--filter "active == true" \
|
|
914
|
+
--sort revenue:desc \
|
|
915
|
+
--limit 10 \
|
|
916
|
+
--to jsonl
|
|
917
|
+
```
|
|
918
|
+
|
|
919
|
+
Output:
|
|
920
|
+
```text
|
|
921
|
+
======================================================
|
|
922
|
+
Rowpipe Execution Plan
|
|
923
|
+
======================================================
|
|
924
|
+
|
|
925
|
+
CSVReader (sales.csv)
|
|
926
|
+
↓
|
|
927
|
+
Filter (active == true)
|
|
928
|
+
↓
|
|
929
|
+
TopK (revenue DESC, 10) [Bounded Heap]
|
|
930
|
+
↓
|
|
931
|
+
JSONLWriter (stdout)
|
|
932
|
+
|
|
933
|
+
------------------------------------------------------
|
|
934
|
+
Optimizations Applied:
|
|
935
|
+
• sort(revenue desc) + limit(10) -> top-k (10)
|
|
936
|
+
------------------------------------------------------
|
|
937
|
+
Memory Classification:
|
|
938
|
+
Bounded State (O(K) fixed memory bound)
|
|
939
|
+
======================================================
|
|
554
940
|
```
|
|
555
941
|
|
|
556
942
|
---
|
|
@@ -569,59 +955,34 @@ import {
|
|
|
569
955
|
FileSystemReader,
|
|
570
956
|
filterRows,
|
|
571
957
|
selectColumns,
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
958
|
+
limitRows,
|
|
959
|
+
offsetRows,
|
|
960
|
+
tailRows,
|
|
961
|
+
sortRows,
|
|
962
|
+
topRows,
|
|
963
|
+
uniqueRows,
|
|
964
|
+
groupRows,
|
|
965
|
+
countStream,
|
|
575
966
|
diffRows,
|
|
576
967
|
computeDiff,
|
|
577
968
|
} from "rowpipe";
|
|
578
969
|
import { createReadStream, createWriteStream } from "node:fs";
|
|
579
970
|
|
|
580
|
-
//
|
|
971
|
+
// Fluent Pipeline API
|
|
581
972
|
const reader = new CSVReader(createReadStream("sales.csv"));
|
|
582
|
-
const writer = new JSONLWriter(createWriteStream("
|
|
973
|
+
const writer = new JSONLWriter(createWriteStream("top_sales.jsonl"));
|
|
583
974
|
|
|
584
975
|
await createPipeline(reader)
|
|
585
|
-
.
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
}))
|
|
589
|
-
.pipe(filterRows("margin >= 15"))
|
|
590
|
-
.pipe(reduceRows({
|
|
591
|
-
by: ["country"],
|
|
592
|
-
aggregations: [
|
|
593
|
-
"total_profit = sum(profit)",
|
|
594
|
-
"avg_margin = avg(margin)",
|
|
595
|
-
"count = count()",
|
|
596
|
-
],
|
|
597
|
-
}))
|
|
976
|
+
.filter("status == 'completed'")
|
|
977
|
+
.sort({ by: "revenue:desc" })
|
|
978
|
+
.limit(100)
|
|
598
979
|
.to(writer);
|
|
599
980
|
|
|
600
|
-
//
|
|
601
|
-
const fsReader = new FileSystemReader({
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
mime: true,
|
|
606
|
-
});
|
|
607
|
-
|
|
608
|
-
for await (const batch of fsReader.read()) {
|
|
609
|
-
for (const file of batch.rows) {
|
|
610
|
-
console.log(`${file.relative_path} (${file.size} bytes) - Hash: ${file.hash}`);
|
|
611
|
-
}
|
|
612
|
-
}
|
|
613
|
-
|
|
614
|
-
// 3. Streaming Dataset Diffing (Async Iterable Events)
|
|
615
|
-
for await (const event of diffRows({
|
|
616
|
-
leftPath: "old.csv",
|
|
617
|
-
rightPath: "new.parquet",
|
|
618
|
-
keys: ["id"],
|
|
619
|
-
ignore: ["updated_at"],
|
|
620
|
-
})) {
|
|
621
|
-
if (event.type === "changed") {
|
|
622
|
-
console.log(`Key ${JSON.stringify(event.key)} changed:`, event.changes);
|
|
623
|
-
}
|
|
624
|
-
}
|
|
981
|
+
// Filesystem as Tabular Source + Top 10 Largest Files
|
|
982
|
+
const fsReader = new FileSystemReader({ root: "./src" });
|
|
983
|
+
const largestFiles = await createPipeline(fsReader)
|
|
984
|
+
.top({ by: "size:desc", count: 10 })
|
|
985
|
+
.toArray();
|
|
625
986
|
```
|
|
626
987
|
|
|
627
988
|
---
|
|
@@ -643,20 +1004,19 @@ Rowpipe uses standardized POSIX exit codes:
|
|
|
643
1004
|
|
|
644
1005
|
## Memory & Performance Benchmarks
|
|
645
1006
|
|
|
646
|
-
### 1.
|
|
647
|
-
|
|
648
|
-
| Dataset Size | Throughput | Execution Time | Peak RSS Memory | Memory Scaling |
|
|
649
|
-
|--------------|------------|----------------|-----------------|----------------|
|
|
650
|
-
| **100,000 rows** | **~318,000 rows/s** | 0.31s | ~106 MB | $O(1)$ Bounded |
|
|
651
|
-
| **1,000,000 rows** | **~500,000 rows/s** | 2.00s | ~153 MB | $O(1)$ Bounded |
|
|
652
|
-
|
|
653
|
-
### 2. Diff Engine Benchmark (`rowpipe diff`):
|
|
1007
|
+
### 1. Operations Benchmark (`1,000,000 rows`, 47.1 MB CSV):
|
|
654
1008
|
|
|
655
|
-
|
|
|
656
|
-
|
|
657
|
-
| **
|
|
1009
|
+
| Scenario | Throughput | Elapsed Time | Peak RSS Memory | Memory Scaling |
|
|
1010
|
+
|----------|------------|--------------|-----------------|----------------|
|
|
1011
|
+
| **Limit 100** (Early Stream Cancel) | **Instant** | 0.01s | ~492 MB | $O(1)$ Early Termination |
|
|
1012
|
+
| **Tail 100** (Bounded RingBuffer) | **1,029,866 rows/s** (48.5 MB/s) | 0.97s | ~514 MB | $O(N)$ Bounded |
|
|
1013
|
+
| **Top-100** (Bounded Min-Heap) | **731,529 rows/s** (34.4 MB/s) | 1.37s | ~530 MB | $O(K)$ Bounded |
|
|
1014
|
+
| **Count + Distinct** (Streaming HLL) | **764,526 rows/s** (36.0 MB/s) | 1.31s | ~530 MB | $O(1)$ Bounded |
|
|
1015
|
+
| **Group-by** (Sum/Avg/Min/Max) | **611,995 rows/s** (28.8 MB/s) | 1.63s | ~516 MB | $O(1)$ Stream Hash |
|
|
1016
|
+
| **Unique** (Composite Deduplication) | **532,481 rows/s** (25.1 MB/s) | 1.88s | ~516 MB | Spillable Hash |
|
|
1017
|
+
| **External Merge Sort** (16MB threshold) | **73,185 rows/s** (3.4 MB/s) | 13.66s | ~606 MB | Spillable $K$-Way Merge |
|
|
658
1018
|
|
|
659
|
-
###
|
|
1019
|
+
### 2. Filesystem Streaming Benchmark (`rowpipe files`):
|
|
660
1020
|
|
|
661
1021
|
| File Count | Operation | Throughput | Peak RSS Memory | Memory Scaling |
|
|
662
1022
|
|------------|-----------|------------|-----------------|----------------|
|
|
@@ -668,6 +1028,7 @@ To run the benchmarks locally:
|
|
|
668
1028
|
|
|
669
1029
|
```bash
|
|
670
1030
|
npm run bench
|
|
1031
|
+
npm run benchmark:ops
|
|
671
1032
|
npm run benchmark:diff
|
|
672
1033
|
npm run benchmark:files
|
|
673
1034
|
```
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import type { Aggregator, ColumnType, Row, SemanticType } from "../core/types.js";
|
|
2
|
+
export interface ColumnProfileResult {
|
|
3
|
+
name: string;
|
|
4
|
+
inferredType: ColumnType;
|
|
5
|
+
typeConfidence: number;
|
|
6
|
+
typeBreakdown: Record<string, number>;
|
|
7
|
+
anomalies: unknown[];
|
|
8
|
+
totalRows: number;
|
|
9
|
+
nullCount: number;
|
|
10
|
+
nullPercentage: number;
|
|
11
|
+
emptyStringCount: number;
|
|
12
|
+
approxDistinctCount: number;
|
|
13
|
+
distinctPercentage: number;
|
|
14
|
+
semanticType?: SemanticType;
|
|
15
|
+
semanticConfidence?: number;
|
|
16
|
+
isNumeric: boolean;
|
|
17
|
+
min?: number;
|
|
18
|
+
max?: number;
|
|
19
|
+
sum?: number;
|
|
20
|
+
mean?: number;
|
|
21
|
+
stddev?: number;
|
|
22
|
+
zerosCount?: number;
|
|
23
|
+
negativeCount?: number;
|
|
24
|
+
isString: boolean;
|
|
25
|
+
minLength?: number;
|
|
26
|
+
maxLength?: number;
|
|
27
|
+
avgLength?: number;
|
|
28
|
+
topValues?: Array<{
|
|
29
|
+
value: string;
|
|
30
|
+
count: number;
|
|
31
|
+
percentage: number;
|
|
32
|
+
}>;
|
|
33
|
+
}
|
|
34
|
+
export interface DatasetProfileResult {
|
|
35
|
+
totalRows: number;
|
|
36
|
+
totalColumns: number;
|
|
37
|
+
columns: ColumnProfileResult[];
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Single-pass Streaming Dataset Profiler.
|
|
41
|
+
*/
|
|
42
|
+
export declare class DatasetProfiler implements Aggregator<DatasetProfileResult> {
|
|
43
|
+
private columnProfilers;
|
|
44
|
+
private totalRows;
|
|
45
|
+
add(row: Row): void;
|
|
46
|
+
result(): DatasetProfileResult;
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Formats profile results as an aligned terminal table.
|
|
50
|
+
*/
|
|
51
|
+
export declare function formatProfileTerminal(profile: DatasetProfileResult): string;
|
|
52
|
+
/**
|
|
53
|
+
* Formats profile results as a GitHub Flavored Markdown document.
|
|
54
|
+
*/
|
|
55
|
+
export declare function formatProfileMarkdown(profile: DatasetProfileResult): string;
|
|
56
|
+
//# sourceMappingURL=profiler.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"profiler.d.ts","sourceRoot":"","sources":["../../src/analytics/profiler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,GAAG,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAMlF,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,UAAU,CAAC;IACzB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtC,SAAS,EAAE,OAAO,EAAE,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;IACvB,gBAAgB,EAAE,MAAM,CAAC;IACzB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAG5B,SAAS,EAAE,OAAO,CAAC;IACnB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,MAAM,CAAC;IAGvB,QAAQ,EAAE,OAAO,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACzE;AAED,MAAM,WAAW,oBAAoB;IACnC,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,mBAAmB,EAAE,CAAC;CAChC;AA+LD;;GAEG;AACH,qBAAa,eAAgB,YAAW,UAAU,CAAC,oBAAoB,CAAC;IACtE,OAAO,CAAC,eAAe,CAAqC;IAC5D,OAAO,CAAC,SAAS,CAAK;IAEtB,GAAG,CAAC,GAAG,EAAE,GAAG,GAAG,IAAI;IAYnB,MAAM,IAAI,oBAAoB;CAY/B;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,oBAAoB,GAAG,MAAM,CAsC3E;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,oBAAoB,GAAG,MAAM,CA+B3E"}
|