rowpipe 2.0.0 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +420 -59
- package/dist/analytics/profiler.d.ts +56 -0
- package/dist/analytics/profiler.d.ts.map +1 -0
- package/dist/analytics/profiler.js +277 -0
- package/dist/analytics/profiler.js.map +1 -0
- package/dist/analytics/schema-inference.d.ts.map +1 -1
- package/dist/analytics/schema-inference.js +14 -0
- package/dist/analytics/schema-inference.js.map +1 -1
- package/dist/cli/commands/clean.d.ts +15 -0
- package/dist/cli/commands/clean.d.ts.map +1 -0
- package/dist/cli/commands/clean.js +108 -0
- package/dist/cli/commands/clean.js.map +1 -0
- package/dist/cli/commands/convert.d.ts +6 -0
- package/dist/cli/commands/convert.d.ts.map +1 -1
- package/dist/cli/commands/convert.js +6 -6
- package/dist/cli/commands/convert.js.map +1 -1
- package/dist/cli/commands/count.d.ts +12 -0
- package/dist/cli/commands/count.d.ts.map +1 -0
- package/dist/cli/commands/count.js +49 -0
- package/dist/cli/commands/count.js.map +1 -0
- package/dist/cli/commands/db.d.ts +35 -0
- package/dist/cli/commands/db.d.ts.map +1 -0
- package/dist/cli/commands/db.js +173 -0
- package/dist/cli/commands/db.js.map +1 -0
- package/dist/cli/commands/explain.d.ts +10 -0
- package/dist/cli/commands/explain.d.ts.map +1 -0
- package/dist/cli/commands/explain.js +98 -0
- package/dist/cli/commands/explain.js.map +1 -0
- package/dist/cli/commands/files.d.ts +2 -1
- package/dist/cli/commands/files.d.ts.map +1 -1
- package/dist/cli/commands/files.js +5 -2
- package/dist/cli/commands/files.js.map +1 -1
- package/dist/cli/commands/group.d.ts +12 -0
- package/dist/cli/commands/group.d.ts.map +1 -0
- package/dist/cli/commands/group.js +51 -0
- package/dist/cli/commands/group.js.map +1 -0
- package/dist/cli/commands/head.d.ts +7 -0
- package/dist/cli/commands/head.d.ts.map +1 -0
- package/dist/cli/commands/head.js +6 -0
- package/dist/cli/commands/head.js.map +1 -0
- package/dist/cli/commands/join.d.ts +23 -0
- package/dist/cli/commands/join.d.ts.map +1 -0
- package/dist/cli/commands/join.js +97 -0
- package/dist/cli/commands/join.js.map +1 -0
- package/dist/cli/commands/limit.d.ts +11 -0
- package/dist/cli/commands/limit.d.ts.map +1 -0
- package/dist/cli/commands/limit.js +40 -0
- package/dist/cli/commands/limit.js.map +1 -0
- package/dist/cli/commands/offset.d.ts +11 -0
- package/dist/cli/commands/offset.d.ts.map +1 -0
- package/dist/cli/commands/offset.js +39 -0
- package/dist/cli/commands/offset.js.map +1 -0
- package/dist/cli/commands/pipeline.d.ts +63 -0
- package/dist/cli/commands/pipeline.d.ts.map +1 -0
- package/dist/cli/commands/pipeline.js +239 -0
- package/dist/cli/commands/pipeline.js.map +1 -0
- package/dist/cli/commands/profile.d.ts +22 -0
- package/dist/cli/commands/profile.d.ts.map +1 -0
- package/dist/cli/commands/profile.js +53 -0
- package/dist/cli/commands/profile.js.map +1 -0
- package/dist/cli/commands/sort.d.ts +17 -0
- package/dist/cli/commands/sort.d.ts.map +1 -0
- package/dist/cli/commands/sort.js +55 -0
- package/dist/cli/commands/sort.js.map +1 -0
- package/dist/cli/commands/tail.d.ts +13 -0
- package/dist/cli/commands/tail.d.ts.map +1 -0
- package/dist/cli/commands/tail.js +39 -0
- package/dist/cli/commands/tail.js.map +1 -0
- package/dist/cli/commands/top.d.ts +19 -0
- package/dist/cli/commands/top.d.ts.map +1 -0
- package/dist/cli/commands/top.js +51 -0
- package/dist/cli/commands/top.js.map +1 -0
- package/dist/cli/commands/unique.d.ts +15 -0
- package/dist/cli/commands/unique.d.ts.map +1 -0
- package/dist/cli/commands/unique.js +44 -0
- package/dist/cli/commands/unique.js.map +1 -0
- package/dist/cli/commands/view.d.ts +12 -0
- package/dist/cli/commands/view.d.ts.map +1 -0
- package/dist/cli/commands/view.js +52 -0
- package/dist/cli/commands/view.js.map +1 -0
- package/dist/cli/commands/window.d.ts +10 -0
- package/dist/cli/commands/window.d.ts.map +1 -0
- package/dist/cli/commands/window.js +87 -0
- package/dist/cli/commands/window.js.map +1 -0
- package/dist/cli/index.js +448 -3
- package/dist/cli/index.js.map +1 -1
- package/dist/core/batch.d.ts +1 -1
- package/dist/core/batch.d.ts.map +1 -1
- package/dist/core/batch.js.map +1 -1
- package/dist/core/pipeline.d.ts +42 -2
- package/dist/core/pipeline.d.ts.map +1 -1
- package/dist/core/pipeline.js +70 -2
- package/dist/core/pipeline.js.map +1 -1
- package/dist/core/types.d.ts +21 -1
- package/dist/core/types.d.ts.map +1 -1
- package/dist/db/adapters/index.d.ts +6 -0
- package/dist/db/adapters/index.d.ts.map +1 -0
- package/dist/db/adapters/index.js +21 -0
- package/dist/db/adapters/index.js.map +1 -0
- package/dist/db/adapters/mysql.d.ts +16 -0
- package/dist/db/adapters/mysql.d.ts.map +1 -0
- package/dist/db/adapters/mysql.js +230 -0
- package/dist/db/adapters/mysql.js.map +1 -0
- package/dist/db/adapters/postgres.d.ts +17 -0
- package/dist/db/adapters/postgres.d.ts.map +1 -0
- package/dist/db/adapters/postgres.js +239 -0
- package/dist/db/adapters/postgres.js.map +1 -0
- package/dist/db/adapters/sqlite.d.ts +18 -0
- package/dist/db/adapters/sqlite.d.ts.map +1 -0
- package/dist/db/adapters/sqlite.js +210 -0
- package/dist/db/adapters/sqlite.js.map +1 -0
- package/dist/db/index.d.ts +11 -0
- package/dist/db/index.d.ts.map +1 -0
- package/dist/db/index.js +11 -0
- package/dist/db/index.js.map +1 -0
- package/dist/db/mapping.d.ts +45 -0
- package/dist/db/mapping.d.ts.map +1 -0
- package/dist/db/mapping.js +222 -0
- package/dist/db/mapping.js.map +1 -0
- package/dist/db/pushdown.d.ts +25 -0
- package/dist/db/pushdown.d.ts.map +1 -0
- package/dist/db/pushdown.js +165 -0
- package/dist/db/pushdown.js.map +1 -0
- package/dist/db/sink.d.ts +15 -0
- package/dist/db/sink.d.ts.map +1 -0
- package/dist/db/sink.js +47 -0
- package/dist/db/sink.js.map +1 -0
- package/dist/db/source.d.ts +14 -0
- package/dist/db/source.d.ts.map +1 -0
- package/dist/db/source.js +69 -0
- package/dist/db/source.js.map +1 -0
- package/dist/db/types.d.ts +80 -0
- package/dist/db/types.d.ts.map +1 -0
- package/dist/db/types.js +2 -0
- package/dist/db/types.js.map +1 -0
- package/dist/db/url.d.ts +15 -0
- package/dist/db/url.d.ts.map +1 -0
- package/dist/db/url.js +145 -0
- package/dist/db/url.js.map +1 -0
- package/dist/diff/hash.d.ts.map +1 -1
- package/dist/diff/hash.js +4 -3
- package/dist/diff/hash.js.map +1 -1
- package/dist/diff/key.d.ts.map +1 -1
- package/dist/diff/key.js +8 -1
- package/dist/diff/key.js.map +1 -1
- package/dist/diff/reporter.d.ts.map +1 -1
- package/dist/diff/reporter.js +5 -3
- package/dist/diff/reporter.js.map +1 -1
- package/dist/diff/storage/disk-index.d.ts.map +1 -1
- package/dist/diff/storage/disk-index.js +2 -1
- package/dist/diff/storage/disk-index.js.map +1 -1
- package/dist/diff/storage/memory-index.d.ts.map +1 -1
- package/dist/diff/storage/memory-index.js +3 -2
- package/dist/diff/storage/memory-index.js.map +1 -1
- package/dist/index.d.ts +20 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +22 -1
- package/dist/index.js.map +1 -1
- package/dist/planner/executor.d.ts +8 -0
- package/dist/planner/executor.d.ts.map +1 -0
- package/dist/planner/executor.js +93 -0
- package/dist/planner/executor.js.map +1 -0
- package/dist/planner/explain.d.ts +6 -0
- package/dist/planner/explain.d.ts.map +1 -0
- package/dist/planner/explain.js +91 -0
- package/dist/planner/explain.js.map +1 -0
- package/dist/planner/index.d.ts +5 -0
- package/dist/planner/index.d.ts.map +1 -0
- package/dist/planner/index.js +5 -0
- package/dist/planner/index.js.map +1 -0
- package/dist/planner/optimizer.d.ts +9 -0
- package/dist/planner/optimizer.d.ts.map +1 -0
- package/dist/planner/optimizer.js +78 -0
- package/dist/planner/optimizer.js.map +1 -0
- package/dist/planner/types.d.ts +67 -0
- package/dist/planner/types.d.ts.map +1 -0
- package/dist/planner/types.js +2 -0
- package/dist/planner/types.js.map +1 -0
- package/dist/readers/csv.d.ts.map +1 -1
- package/dist/readers/csv.js +1 -2
- package/dist/readers/csv.js.map +1 -1
- package/dist/readers/index.d.ts +5 -1
- package/dist/readers/index.d.ts.map +1 -1
- package/dist/readers/index.js +43 -0
- package/dist/readers/index.js.map +1 -1
- package/dist/readers/json.d.ts.map +1 -1
- package/dist/readers/json.js +1 -2
- package/dist/readers/json.js.map +1 -1
- package/dist/readers/jsonl.d.ts.map +1 -1
- package/dist/readers/jsonl.js +1 -2
- package/dist/readers/jsonl.js.map +1 -1
- package/dist/readers/multi-file.d.ts +29 -0
- package/dist/readers/multi-file.d.ts.map +1 -0
- package/dist/readers/multi-file.js +169 -0
- package/dist/readers/multi-file.js.map +1 -0
- package/dist/transforms/clean.d.ts +18 -0
- package/dist/transforms/clean.d.ts.map +1 -0
- package/dist/transforms/clean.js +118 -0
- package/dist/transforms/clean.js.map +1 -0
- package/dist/transforms/count.d.ts +21 -0
- package/dist/transforms/count.d.ts.map +1 -0
- package/dist/transforms/count.js +65 -0
- package/dist/transforms/count.js.map +1 -0
- package/dist/transforms/expression.d.ts.map +1 -1
- package/dist/transforms/expression.js +12 -0
- package/dist/transforms/expression.js.map +1 -1
- package/dist/transforms/group.d.ts +25 -0
- package/dist/transforms/group.d.ts.map +1 -0
- package/dist/transforms/group.js +71 -0
- package/dist/transforms/group.js.map +1 -0
- package/dist/transforms/jit-compiler.d.ts +50 -0
- package/dist/transforms/jit-compiler.d.ts.map +1 -0
- package/dist/transforms/jit-compiler.js +275 -0
- package/dist/transforms/jit-compiler.js.map +1 -0
- package/dist/transforms/join/index-storage.d.ts +33 -0
- package/dist/transforms/join/index-storage.d.ts.map +1 -0
- package/dist/transforms/join/index-storage.js +139 -0
- package/dist/transforms/join/index-storage.js.map +1 -0
- package/dist/transforms/join/index.d.ts +4 -0
- package/dist/transforms/join/index.d.ts.map +1 -0
- package/dist/transforms/join/index.js +4 -0
- package/dist/transforms/join/index.js.map +1 -0
- package/dist/transforms/join/join.d.ts +15 -0
- package/dist/transforms/join/join.d.ts.map +1 -0
- package/dist/transforms/join/join.js +216 -0
- package/dist/transforms/join/join.js.map +1 -0
- package/dist/transforms/join/types.d.ts +29 -0
- package/dist/transforms/join/types.d.ts.map +1 -0
- package/dist/transforms/join/types.js +2 -0
- package/dist/transforms/join/types.js.map +1 -0
- package/dist/transforms/limit.d.ts +7 -0
- package/dist/transforms/limit.d.ts.map +1 -0
- package/dist/transforms/limit.js +40 -0
- package/dist/transforms/limit.js.map +1 -0
- package/dist/transforms/offset.d.ts +7 -0
- package/dist/transforms/offset.d.ts.map +1 -0
- package/dist/transforms/offset.js +38 -0
- package/dist/transforms/offset.js.map +1 -0
- package/dist/transforms/sort/comparator.d.ts +43 -0
- package/dist/transforms/sort/comparator.d.ts.map +1 -0
- package/dist/transforms/sort/comparator.js +131 -0
- package/dist/transforms/sort/comparator.js.map +1 -0
- package/dist/transforms/sort/external-sort.d.ts +8 -0
- package/dist/transforms/sort/external-sort.d.ts.map +1 -0
- package/dist/transforms/sort/external-sort.js +159 -0
- package/dist/transforms/sort/external-sort.js.map +1 -0
- package/dist/transforms/sort/index.d.ts +10 -0
- package/dist/transforms/sort/index.d.ts.map +1 -0
- package/dist/transforms/sort/index.js +11 -0
- package/dist/transforms/sort/index.js.map +1 -0
- package/dist/transforms/tail.d.ts +6 -0
- package/dist/transforms/tail.d.ts.map +1 -0
- package/dist/transforms/tail.js +27 -0
- package/dist/transforms/tail.js.map +1 -0
- package/dist/transforms/top.d.ts +17 -0
- package/dist/transforms/top.d.ts.map +1 -0
- package/dist/transforms/top.js +52 -0
- package/dist/transforms/top.js.map +1 -0
- package/dist/transforms/unique.d.ts +13 -0
- package/dist/transforms/unique.d.ts.map +1 -0
- package/dist/transforms/unique.js +91 -0
- package/dist/transforms/unique.js.map +1 -0
- package/dist/transforms/window.d.ts +24 -0
- package/dist/transforms/window.d.ts.map +1 -0
- package/dist/transforms/window.js +317 -0
- package/dist/transforms/window.js.map +1 -0
- package/dist/ui/colors.d.ts +57 -0
- package/dist/ui/colors.d.ts.map +1 -0
- package/dist/ui/colors.js +94 -0
- package/dist/ui/colors.js.map +1 -0
- package/dist/ui/viewer.d.ts +67 -0
- package/dist/ui/viewer.d.ts.map +1 -0
- package/dist/ui/viewer.js +673 -0
- package/dist/ui/viewer.js.map +1 -0
- package/dist/utils/compression.d.ts +68 -19
- package/dist/utils/compression.d.ts.map +1 -1
- package/dist/utils/compression.js +222 -34
- package/dist/utils/compression.js.map +1 -1
- package/dist/utils/formatting.d.ts +5 -0
- package/dist/utils/formatting.d.ts.map +1 -1
- package/dist/utils/formatting.js +15 -0
- package/dist/utils/formatting.js.map +1 -1
- package/dist/utils/heap.d.ts +25 -0
- package/dist/utils/heap.d.ts.map +1 -0
- package/dist/utils/heap.js +90 -0
- package/dist/utils/heap.js.map +1 -0
- package/dist/utils/keystore.d.ts +80 -0
- package/dist/utils/keystore.d.ts.map +1 -0
- package/dist/utils/keystore.js +217 -0
- package/dist/utils/keystore.js.map +1 -0
- package/dist/utils/ring-buffer.d.ts +15 -0
- package/dist/utils/ring-buffer.d.ts.map +1 -0
- package/dist/utils/ring-buffer.js +40 -0
- package/dist/utils/ring-buffer.js.map +1 -0
- package/dist/writers/csv.d.ts.map +1 -1
- package/dist/writers/csv.js +13 -14
- package/dist/writers/csv.js.map +1 -1
- package/dist/writers/index.d.ts +1 -0
- package/dist/writers/index.d.ts.map +1 -1
- package/dist/writers/index.js +35 -0
- package/dist/writers/index.js.map +1 -1
- package/dist/writers/json.d.ts.map +1 -1
- package/dist/writers/json.js +6 -5
- package/dist/writers/json.js.map +1 -1
- package/dist/writers/jsonl.d.ts.map +1 -1
- package/dist/writers/jsonl.js +6 -5
- package/dist/writers/jsonl.js.map +1 -1
- package/dist/writers/markdown.d.ts.map +1 -1
- package/dist/writers/markdown.js +5 -4
- package/dist/writers/markdown.js.map +1 -1
- package/dist/writers/parquet.d.ts +1 -1
- package/dist/writers/parquet.d.ts.map +1 -1
- package/package.json +11 -2
- package/skills/rowpipe/SKILL.md +16 -3
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"jit-compiler.js","sourceRoot":"","sources":["../../src/transforms/jit-compiler.ts"],"names":[],"mappings":"AAGA;;GAEG;AACH,SAAS,cAAc,CAAC,GAAY,EAAE,IAAY;IAChD,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IACnD,IAAI,OAAO,GAAY,GAAG,CAAC;IAE3B,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;QAChC,IAAI,CAAC;YACH,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAChC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC9B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,OAAO,KAAK,IAAI,IAAI,OAAO,KAAK,SAAS,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;YAC7E,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO,GAAI,OAAmC,CAAC,IAAI,CAAC,CAAC;IACvD,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAY,EAAE,GAAQ;IAC/C,IAAI,IAAI,IAAI,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,SAAS,EAAE,CAAC;QAC3C,MAAM,GAAG,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC;QACtB,OAAO,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC;IACjD,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC5C,IAAI,OAAO,IAAI,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,KAAK,SAAS,EAAE,CAAC;YACjD,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC1C,IAAI,QAAQ,KAAK,EAAE,EAAE,CAAC;gBACpB,MAAM,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,CAAC;gBACzB,OAAO,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC;YACjD,CAAC;YACD,MAAM,MAAM,GAAG,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,QAAQ,CAAC,CAAC;YACtD,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,KAAK,IAAI,IAAI,MAAM,KAAK,EAAE,EAAE,CAAC;gBAC7D,OAAO,MAAM,CAAC;YAChB,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;IAC3B,IAAI,SAAS,KAAK,SAAS,IAAI,SAAS,KAAK,KAAK,IAAI,SAAS,KAAK,MAAM,EAAE,CAAC;QAC3E,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC1C,MAAM,MAAM,GAAG,cAAc,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;QAC7C,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,KAAK,IAAI,IAAI,MAAM,KAAK,EAAE,EAAE,CAAC;YAC7D,OAAO,MAAM,CAAC;QAChB,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG;IACzB,GAAG,EAAE,CAAC,GAAQ,EAAE,IAAY,EAAW,EAAE,CAAC,iBAAiB,CAAC,IAAI,EAAE,GAAG,CAAC;IACtE,EAAE,EAAE,CAAC,CAAU,EAAE,CAAU,EAAW,EAAE;QACtC,IAAI,CAAC,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QACzB,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,SAAS;YAAE,OAAO,CAAC,IAAI,CAAC,CAAC;QAClF,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;YACnD,OAAO,MAAM,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC;QACjC,CAAC;QACD,OAAO,MAAM,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC;IACjC,CAAC;IACD,GAAG,EAAE,CAAC,CAAU,EAAE,CAAU,EAAW,EAAE;QACvC,IAAI,CAAC,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QAC1B,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,SAAS;YAAE,OAAO,CAAC,IAAI,CAAC,CAAC;QAClF,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;YACnD,OAAO,MAAM,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC;QACjC,CAAC;QACD,OAAO,MAAM,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC;IACjC,CAAC;IACD,EAAE,EAAE,CAAC,CAAU,EAAE,CAAU,EAAW,EAAE;QACtC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,SAAS;YAAE,OAAO,KAAK,CAAC;QACjF,OAAO,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;IAC/B,CAAC;IACD,GAAG,EAAE,CAAC,CAAU,EAAE,CAAU,EAAW,EAAE;QACvC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,SAAS;YAAE,OAAO,KAAK,CAAC;QACjF,OAAO,MAAM,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;IAChC,CAAC;IACD,EAAE,EAAE,CAAC,CAAU,EAAE,CAAU,EAAW,EAAE;QACtC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,SAAS;YAAE,OAAO,KAAK,CAAC;QACjF,OAAO,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;IAC/B,CAAC;IACD,GAAG,EAAE,CAAC,CAAU,EAAE,CAAU,EAAW,EAAE;QACvC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,SAAS;YAAE,OAAO,KAAK,CAAC;QACjF,OAAO,MAAM,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;IAChC,CAAC;IACD,GAAG,EAAE,CAAC,CAAU,EAAE,CAAU,EAAW,EAAE;QACvC,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;YACnD,OAAO,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAC3C,CAAC;QACD,OAAO,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACzC,CAAC;IACD,QAAQ,EAAE,CAAC,CAAU,EAAE,CAAU,EAAW,EAAE;QAC5C,OAAO,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;IAC/E,CAAC;IACD,UAAU,EAAE,CAAC,CAAU,EAAE,CAAU,EAAW,EAAE;QAC9C,OAAO,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IACrD,CAAC;IACD,QAAQ,EAAE,CAAC,CAAU,EAAE,CAAU,EAAW,EAAE;QAC5C,OAAO,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IACnD,CAAC;IACD,KAAK,EAAE,CAAC,CAAU,EAAU,EAAE,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE;IAC5D,KAAK,EAAE,CAAC,CAAU,EAAU,EAAE,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE;IAC5D,IAAI,EAAE,CAAC,CAAU,EAAU,EAAE,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE;IACpD,MAAM,EAAE,CAAC,CAAU,EAAU,EAAE,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM;IACtD,MAAM,EAAE,CAAC,GAAG,IAAe,EAAU,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;IAClH,MAAM,EAAE,CAAC,CAAU,EAAE,KAAK,GAAG,CAAC,EAAE,GAAY,EAAU,EAAE;QACtD,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAC5B,OAAO,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IAC9G,CAAC;IACD,SAAS,EAAE,CAAC,CAAU,EAAE,KAAK,GAAG,CAAC,EAAE,GAAY,EAAU,EAAE;QACzD,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAC5B,OAAO,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IAC9G,CAAC;IACD,OAAO,EAAE,CAAC,CAAU,EAAE,MAAe,EAAE,WAAoB,EAAU,EAAE;QACrE,OAAO,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC,EAAE,MAAM,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,CAAC;IACrF,CAAC;IACD,KAAK,EAAE,CAAC,CAAU,EAAE,QAAQ,GAAG,CAAC,EAAU,EAAE;QAC1C,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QACtB,IAAI,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC;YAAE,OAAO,CAAC,CAAC;QAChC,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;QACnD,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,MAAM,CAAC,GAAG,MAAM,CAAC;IAC3C,CAAC;IACD,GAAG,EAAE,CAAC,CAAU,EAAU,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACrD,IAAI,EAAE,CAAC,CAAU,EAAU,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACvD,KAAK,EAAE,CAAC,CAAU,EAAU,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACzD,IAAI,EAAE,CAAC,CAAU,EAAU,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACvD,GAAG,EAAE,CAAC,GAAG,IAAe,EAAU,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACjF,GAAG,EAAE,CAAC,GAAG,IAAe,EAAU,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACjF,QAAQ,EAAE,CAAC,GAAG,IAAe,EAAW,EAAE;QACxC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,EAAE;gBAAE,OAAO,GAAG,CAAC;QAClE,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,EAAE,EAAE,CAAC,IAAa,EAAE,OAAgB,EAAE,OAAgB,EAAW,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC;IAC9F,GAAG,EAAE,CAAC,IAAa,EAAE,OAAgB,EAAE,OAAgB,EAAW,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC;IAC/F,MAAM,EAAE,CAAC,CAAU,EAAW,EAAE,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,EAAE;IAC1E,SAAS,EAAE,CAAC,CAAU,EAAW,EAAE,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,EAAE;IAC7E,OAAO,EAAE,CAAC,GAAY,EAAE,IAAa,EAAW,EAAE,CAAC,cAAc,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;CAC3F,CAAC;AAEF;;;GAGG;AACH,SAAS,cAAc,CAAC,IAAa;IACnC,QAAQ,IAAI,CAAC,IAAI,EAAE,CAAC;QAClB,KAAK,SAAS;YACZ,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAEpC,KAAK,YAAY,CAAC,CAAC,CAAC;YAClB,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC7B,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC7C,OAAO,SAAS,UAAU,0BAA0B,UAAU,mBAAmB,UAAU,WAAW,CAAC;YACzG,CAAC;YACD,OAAO,eAAe,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;QACtD,CAAC;QAED,KAAK,SAAS,CAAC,CAAC,CAAC;YACf,MAAM,OAAO,GAAG,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC9C,IAAI,CAAC,OAAO;gBAAE,OAAO,IAAI,CAAC;YAC1B,IAAI,IAAI,CAAC,QAAQ,KAAK,GAAG;gBAAE,OAAO,aAAa,OAAO,IAAI,CAAC;YAC3D,IAAI,IAAI,CAAC,QAAQ,KAAK,GAAG;gBAAE,OAAO,YAAY,OAAO,SAAS,CAAC;YAC/D,IAAI,IAAI,CAAC,QAAQ,KAAK,GAAG;gBAAE,OAAO,YAAY,OAAO,SAAS,CAAC;YAC/D,OAAO,IAAI,CAAC;QACd,CAAC;QAED,KAAK,UAAU,CAAC,CAAC,CAAC;YAChB,MAAM,QAAQ,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC3C,MAAM,SAAS,GAAG,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC7C,IAAI,CAAC,QAAQ,IAAI,CAAC,SAAS;gBAAE,OAAO,IAAI,CAAC;YAEzC,QAAQ,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACtB,KAAK,IAAI;oBACP,OAAO,YAAY,QAAQ,gBAAgB,SAAS,IAAI,CAAC;gBAC3D,KAAK,IAAI;oBACP,OAAO,YAAY,QAAQ,gBAAgB,SAAS,IAAI,CAAC;gBAC3D,KAAK,IAAI;oBACP,OAAO,SAAS,QAAQ,KAAK,SAAS,IAAI,CAAC;gBAC7C,KAAK,IAAI;oBACP,OAAO,UAAU,QAAQ,KAAK,SAAS,IAAI,CAAC;gBAC9C,KAAK,GAAG;oBACN,OAAO,SAAS,QAAQ,KAAK,SAAS,IAAI,CAAC;gBAC7C,KAAK,IAAI;oBACP,OAAO,UAAU,QAAQ,KAAK,SAAS,IAAI,CAAC;gBAC9C,KAAK,GAAG;oBACN,OAAO,SAAS,QAAQ,KAAK,SAAS,IAAI,CAAC;gBAC7C,KAAK,IAAI;oBACP,OAAO,UAAU,QAAQ,KAAK,SAAS,IAAI,CAAC;gBAC9C,KAAK,GAAG;oBACN,OAAO,UAAU,QAAQ,KAAK,SAAS,IAAI,CAAC;gBAC9C,KAAK,GAAG;oBACN,OAAO,YAAY,QAAQ,qBAAqB,SAAS,UAAU,CAAC;gBACtE,KAAK,GAAG;oBACN,OAAO,YAAY,QAAQ,qBAAqB,SAAS,UAAU,CAAC;gBACtE,KAAK,GAAG;oBACN,OAAO,YAAY,QAAQ,qBAAqB,SAAS,UAAU,CAAC;gBACtE,KAAK,GAAG;oBACN,OAAO,YAAY,QAAQ,qBAAqB,SAAS,UAAU,CAAC;gBACtE;oBACE,OAAO,IAAI,CAAC;YAChB,CAAC;QACH,CAAC;QAED,KAAK,UAAU,CAAC,CAAC,CAAC;YAChB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;YACzC,MAAM,QAAQ,GAAa,EAAE,CAAC;YAC9B,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;gBAC5B,MAAM,CAAC,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;gBAC9B,IAAI,CAAC,CAAC;oBAAE,OAAO,IAAI,CAAC;gBACpB,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACnB,CAAC;YAED,IAAI,MAAM,IAAI,WAAW,EAAE,CAAC;gBAC1B,OAAO,MAAM,MAAM,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;YACjD,CAAC;YAED,oFAAoF;YACpF,OAAO,IAAI,CAAC;QACd,CAAC;QAED;YACE,OAAO,IAAI,CAAC;IAChB,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,sBAAsB,CAAC,GAAY;IACjD,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;QACrC,IAAI,CAAC,QAAQ;YAAE,OAAO,IAAI,CAAC;QAE3B,qEAAqE;QACrE,MAAM,aAAa,GAAG,IAAI,QAAQ,CAAC,GAAG,EAAE,yCAAyC,QAAQ,OAAO,CAAC,CAAC;QAClG,MAAM,QAAQ,GAAG,aAAa,CAAC,WAAW,CAA0B,CAAC;QACrE,OAAO,QAAQ,CAAC;IAClB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,GAAY;IAC7C,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;QACrC,IAAI,CAAC,QAAQ;YAAE,OAAO,IAAI,CAAC;QAE3B,MAAM,aAAa,GAAG,IAAI,QAAQ,CAAC,GAAG,EAAE,iCAAiC,QAAQ,MAAM,CAAC,CAAC;QACzF,MAAM,QAAQ,GAAG,aAAa,CAAC,WAAW,CAA0B,CAAC;QACrE,OAAO,QAAQ,CAAC;IAClB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import type { Row } from "../../core/types.js";
|
|
2
|
+
import type { JoinIndex } from "./types.js";
|
|
3
|
+
/**
|
|
4
|
+
* High-performance Spillable Join Index that stores build-side rows in memory,
|
|
5
|
+
* and transparently spills to disk when memory threshold is reached.
|
|
6
|
+
*/
|
|
7
|
+
export declare class SpillableJoinIndex implements JoinIndex {
|
|
8
|
+
private memoryMap;
|
|
9
|
+
private matchedKeys;
|
|
10
|
+
private estimatedBytes;
|
|
11
|
+
private memoryLimitBytes;
|
|
12
|
+
private isSpilled;
|
|
13
|
+
private tempDir?;
|
|
14
|
+
private dbPath?;
|
|
15
|
+
private db?;
|
|
16
|
+
private insertStmt?;
|
|
17
|
+
private selectStmt?;
|
|
18
|
+
private hasStmt?;
|
|
19
|
+
private totalCount;
|
|
20
|
+
constructor(options?: {
|
|
21
|
+
memoryLimit?: number | string;
|
|
22
|
+
tempDir?: string;
|
|
23
|
+
});
|
|
24
|
+
private spillToDisk;
|
|
25
|
+
set(key: string, row: Row): Promise<void>;
|
|
26
|
+
get(key: string): Promise<Row[] | undefined>;
|
|
27
|
+
has(key: string): Promise<boolean>;
|
|
28
|
+
markMatched(key: string): void;
|
|
29
|
+
getUnmatched(): AsyncIterable<Row>;
|
|
30
|
+
count(): number;
|
|
31
|
+
close(): Promise<void>;
|
|
32
|
+
}
|
|
33
|
+
//# sourceMappingURL=index-storage.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index-storage.d.ts","sourceRoot":"","sources":["../../../src/transforms/join/index-storage.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,qBAAqB,CAAC;AAG/C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAE5C;;;GAGG;AACH,qBAAa,kBAAmB,YAAW,SAAS;IAClD,OAAO,CAAC,SAAS,CAA4B;IAC7C,OAAO,CAAC,WAAW,CAAqB;IACxC,OAAO,CAAC,cAAc,CAAK;IAC3B,OAAO,CAAC,gBAAgB,CAAS;IACjC,OAAO,CAAC,SAAS,CAAS;IAE1B,OAAO,CAAC,OAAO,CAAC,CAAS;IACzB,OAAO,CAAC,MAAM,CAAC,CAAS;IACxB,OAAO,CAAC,EAAE,CAAC,CAAe;IAC1B,OAAO,CAAC,UAAU,CAAC,CAAM;IACzB,OAAO,CAAC,UAAU,CAAC,CAAM;IACzB,OAAO,CAAC,OAAO,CAAC,CAAM;IACtB,OAAO,CAAC,UAAU,CAAK;gBAEX,OAAO,CAAC,EAAE;QAAE,WAAW,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE;YAK3D,WAAW;IAiCnB,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC;IAuBzC,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;IAS5C,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAQxC,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;IAIvB,YAAY,IAAI,aAAa,CAAC,GAAG,CAAC;IAmBzC,KAAK,IAAI,MAAM;IAIT,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAuB7B"}
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
import { DatabaseSync } from "node:sqlite";
|
|
2
|
+
import * as fs from "node:fs";
|
|
3
|
+
import * as os from "node:os";
|
|
4
|
+
import * as path from "node:path";
|
|
5
|
+
import { parseMemoryLimit } from "../../diff/storage/spillable-index.js";
|
|
6
|
+
import { safeJsonStringify } from "../../utils/formatting.js";
|
|
7
|
+
/**
|
|
8
|
+
* High-performance Spillable Join Index that stores build-side rows in memory,
|
|
9
|
+
* and transparently spills to disk when memory threshold is reached.
|
|
10
|
+
*/
|
|
11
|
+
export class SpillableJoinIndex {
|
|
12
|
+
memoryMap = new Map();
|
|
13
|
+
matchedKeys = new Set();
|
|
14
|
+
estimatedBytes = 0;
|
|
15
|
+
memoryLimitBytes;
|
|
16
|
+
isSpilled = false;
|
|
17
|
+
tempDir;
|
|
18
|
+
dbPath;
|
|
19
|
+
db;
|
|
20
|
+
insertStmt;
|
|
21
|
+
selectStmt;
|
|
22
|
+
hasStmt;
|
|
23
|
+
totalCount = 0;
|
|
24
|
+
constructor(options) {
|
|
25
|
+
this.memoryLimitBytes = parseMemoryLimit(options?.memoryLimit, 256 * 1024 * 1024);
|
|
26
|
+
this.tempDir = options?.tempDir;
|
|
27
|
+
}
|
|
28
|
+
async spillToDisk() {
|
|
29
|
+
if (this.isSpilled)
|
|
30
|
+
return;
|
|
31
|
+
const tmpFolder = this.tempDir || os.tmpdir();
|
|
32
|
+
this.dbPath = path.join(tmpFolder, `rowpipe_join_${Date.now()}_${Math.random().toString(36).slice(2)}.db`);
|
|
33
|
+
this.db = new DatabaseSync(this.dbPath);
|
|
34
|
+
this.db.exec(`
|
|
35
|
+
PRAGMA synchronous = OFF;
|
|
36
|
+
PRAGMA journal_mode = OFF;
|
|
37
|
+
CREATE TABLE join_store (
|
|
38
|
+
k TEXT NOT NULL,
|
|
39
|
+
data TEXT NOT NULL
|
|
40
|
+
);
|
|
41
|
+
CREATE INDEX idx_join_k ON join_store (k);
|
|
42
|
+
`);
|
|
43
|
+
this.insertStmt = this.db.prepare("INSERT INTO join_store (k, data) VALUES (?, ?)");
|
|
44
|
+
this.selectStmt = this.db.prepare("SELECT data FROM join_store WHERE k = ?");
|
|
45
|
+
this.hasStmt = this.db.prepare("SELECT 1 FROM join_store WHERE k = ? LIMIT 1");
|
|
46
|
+
this.db.exec("BEGIN TRANSACTION;");
|
|
47
|
+
for (const [key, rows] of this.memoryMap.entries()) {
|
|
48
|
+
for (const r of rows) {
|
|
49
|
+
this.insertStmt.run(key, safeJsonStringify(r));
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
this.db.exec("COMMIT;");
|
|
53
|
+
this.memoryMap.clear();
|
|
54
|
+
this.isSpilled = true;
|
|
55
|
+
}
|
|
56
|
+
async set(key, row) {
|
|
57
|
+
this.totalCount++;
|
|
58
|
+
if (this.isSpilled && this.insertStmt) {
|
|
59
|
+
this.insertStmt.run(key, safeJsonStringify(row));
|
|
60
|
+
return;
|
|
61
|
+
}
|
|
62
|
+
let existing = this.memoryMap.get(key);
|
|
63
|
+
if (!existing) {
|
|
64
|
+
existing = [];
|
|
65
|
+
this.memoryMap.set(key, existing);
|
|
66
|
+
}
|
|
67
|
+
existing.push(row);
|
|
68
|
+
// Rough size estimate: key bytes + row keys/values
|
|
69
|
+
this.estimatedBytes += key.length * 2 + 128 + Object.keys(row).length * 32;
|
|
70
|
+
if (this.estimatedBytes >= this.memoryLimitBytes) {
|
|
71
|
+
await this.spillToDisk();
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
async get(key) {
|
|
75
|
+
if (this.isSpilled && this.selectStmt) {
|
|
76
|
+
const records = this.selectStmt.all(key);
|
|
77
|
+
if (records.length === 0)
|
|
78
|
+
return undefined;
|
|
79
|
+
return records.map((rec) => JSON.parse(rec.data));
|
|
80
|
+
}
|
|
81
|
+
return this.memoryMap.get(key);
|
|
82
|
+
}
|
|
83
|
+
async has(key) {
|
|
84
|
+
if (this.isSpilled && this.hasStmt) {
|
|
85
|
+
const res = this.hasStmt.get(key);
|
|
86
|
+
return res !== undefined;
|
|
87
|
+
}
|
|
88
|
+
return this.memoryMap.has(key);
|
|
89
|
+
}
|
|
90
|
+
markMatched(key) {
|
|
91
|
+
this.matchedKeys.add(key);
|
|
92
|
+
}
|
|
93
|
+
async *getUnmatched() {
|
|
94
|
+
if (this.isSpilled && this.db) {
|
|
95
|
+
const stmt = this.db.prepare("SELECT k, data FROM join_store");
|
|
96
|
+
for (const item of stmt.iterate()) {
|
|
97
|
+
if (!this.matchedKeys.has(item.k)) {
|
|
98
|
+
yield JSON.parse(item.data);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
else {
|
|
103
|
+
for (const [k, rows] of this.memoryMap.entries()) {
|
|
104
|
+
if (!this.matchedKeys.has(k)) {
|
|
105
|
+
for (const row of rows) {
|
|
106
|
+
yield row;
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
count() {
|
|
113
|
+
return this.totalCount;
|
|
114
|
+
}
|
|
115
|
+
async close() {
|
|
116
|
+
this.memoryMap.clear();
|
|
117
|
+
this.matchedKeys.clear();
|
|
118
|
+
if (this.db) {
|
|
119
|
+
try {
|
|
120
|
+
this.db.close();
|
|
121
|
+
}
|
|
122
|
+
catch {
|
|
123
|
+
// Ignore
|
|
124
|
+
}
|
|
125
|
+
this.db = undefined;
|
|
126
|
+
}
|
|
127
|
+
if (this.dbPath) {
|
|
128
|
+
try {
|
|
129
|
+
if (fs.existsSync(this.dbPath)) {
|
|
130
|
+
fs.unlinkSync(this.dbPath);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
catch {
|
|
134
|
+
// Ignore unlink error on exit
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
//# sourceMappingURL=index-storage.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index-storage.js","sourceRoot":"","sources":["../../../src/transforms/join/index-storage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAElC,OAAO,EAAE,gBAAgB,EAAE,MAAM,uCAAuC,CAAC;AACzE,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAG9D;;;GAGG;AACH,MAAM,OAAO,kBAAkB;IACrB,SAAS,GAAG,IAAI,GAAG,EAAiB,CAAC;IACrC,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;IAChC,cAAc,GAAG,CAAC,CAAC;IACnB,gBAAgB,CAAS;IACzB,SAAS,GAAG,KAAK,CAAC;IAElB,OAAO,CAAU;IACjB,MAAM,CAAU;IAChB,EAAE,CAAgB;IAClB,UAAU,CAAO;IACjB,UAAU,CAAO;IACjB,OAAO,CAAO;IACd,UAAU,GAAG,CAAC,CAAC;IAEvB,YAAY,OAA6D;QACvE,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC,OAAO,EAAE,WAAW,EAAE,GAAG,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC;QAClF,IAAI,CAAC,OAAO,GAAG,OAAO,EAAE,OAAO,CAAC;IAClC,CAAC;IAEO,KAAK,CAAC,WAAW;QACvB,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO;QAE3B,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC;QAC9C,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,gBAAgB,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QAC3G,IAAI,CAAC,EAAE,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAExC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC;;;;;;;;KAQZ,CAAC,CAAC;QAEH,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,gDAAgD,CAAC,CAAC;QACpF,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,yCAAyC,CAAC,CAAC;QAC7E,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,8CAA8C,CAAC,CAAC;QAE/E,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;QACnC,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,CAAC;YACnD,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;gBACrB,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC;YACjD,CAAC;QACH,CAAC;QACD,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAExB,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;QACvB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,GAAW,EAAE,GAAQ;QAC7B,IAAI,CAAC,UAAU,EAAE,CAAC;QAElB,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACtC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC;YACjD,OAAO;QACT,CAAC;QAED,IAAI,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACvC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,QAAQ,GAAG,EAAE,CAAC;YACd,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;QACpC,CAAC;QACD,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEnB,mDAAmD;QACnD,IAAI,CAAC,cAAc,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,GAAG,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,GAAG,EAAE,CAAC;QAE3E,IAAI,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACjD,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,GAAW;QACnB,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACtC,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAA4B,CAAC;YACpE,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,SAAS,CAAC;YAC3C,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;QACpD,CAAC;QACD,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACjC,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,GAAW;QACnB,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACnC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAClC,OAAO,GAAG,KAAK,SAAS,CAAC;QAC3B,CAAC;QACD,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACjC,CAAC;IAED,WAAW,CAAC,GAAW;QACrB,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC5B,CAAC;IAED,KAAK,CAAC,CAAC,YAAY;QACjB,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC;YAC9B,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,gCAAgC,CAAC,CAAC;YAC/D,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,OAAO,EAA2C,EAAE,CAAC;gBAC3E,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;oBAClC,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC9B,CAAC;YACH,CAAC;QACH,CAAC;aAAM,CAAC;YACN,KAAK,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,CAAC;gBACjD,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;oBAC7B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;wBACvB,MAAM,GAAG,CAAC;oBACZ,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK;QACH,OAAO,IAAI,CAAC,UAAU,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;QACvB,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;QAEzB,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC;YACZ,IAAI,CAAC;gBACH,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;YAClB,CAAC;YAAC,MAAM,CAAC;gBACP,SAAS;YACX,CAAC;YACD,IAAI,CAAC,EAAE,GAAG,SAAS,CAAC;QACtB,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,IAAI,CAAC;gBACH,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;oBAC/B,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBAC7B,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,8BAA8B;YAChC,CAAC;QACH,CAAC;IACH,CAAC;CACF"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/transforms/join/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,cAAc,oBAAoB,CAAC;AACnC,cAAc,WAAW,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/transforms/join/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,cAAc,oBAAoB,CAAC;AACnC,cAAc,WAAW,CAAC"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { DataStream, TabularReader, TransformFunction } from "../../core/types.js";
|
|
2
|
+
import type { JoinKeyMapping, JoinOptions } from "./types.js";
|
|
3
|
+
/**
|
|
4
|
+
* Parses user join key options into structured left/right column mappings.
|
|
5
|
+
*/
|
|
6
|
+
export declare function parseJoinKeys(options: JoinOptions): JoinKeyMapping;
|
|
7
|
+
/**
|
|
8
|
+
* Joins two tabular streams with bounded memory and spill-to-disk index support.
|
|
9
|
+
*/
|
|
10
|
+
export declare function joinStreams(leftSource: DataStream | TabularReader, rightSource: DataStream | TabularReader, options?: JoinOptions): DataStream;
|
|
11
|
+
/**
|
|
12
|
+
* Returns a TransformFunction for pipeline chaining (e.g. pipeline.pipe(joinRows(rightReader, options))).
|
|
13
|
+
*/
|
|
14
|
+
export declare function joinRows(rightSource: DataStream | TabularReader, options?: JoinOptions): TransformFunction;
|
|
15
|
+
//# sourceMappingURL=join.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"join.d.ts","sourceRoot":"","sources":["../../../src/transforms/join/join.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAa,UAAU,EAAO,aAAa,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAExG,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAY,MAAM,YAAY,CAAC;AAExE;;GAEG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,WAAW,GAAG,cAAc,CAgClE;AA8ED;;GAEG;AACH,wBAAuB,WAAW,CAChC,UAAU,EAAE,UAAU,GAAG,aAAa,EACtC,WAAW,EAAE,UAAU,GAAG,aAAa,EACvC,OAAO,GAAE,WAAgB,GACxB,UAAU,CA2GZ;AAED;;GAEG;AACH,wBAAgB,QAAQ,CACtB,WAAW,EAAE,UAAU,GAAG,aAAa,EACvC,OAAO,GAAE,WAAgB,GACxB,iBAAiB,CAInB"}
|
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
import { encodeCompositeKey } from "../../diff/key.js";
|
|
2
|
+
import { InvalidArgumentError } from "../../core/errors.js";
|
|
3
|
+
import { SpillableJoinIndex } from "./index-storage.js";
|
|
4
|
+
/**
|
|
5
|
+
* Parses user join key options into structured left/right column mappings.
|
|
6
|
+
*/
|
|
7
|
+
export function parseJoinKeys(options) {
|
|
8
|
+
if (options.leftKey && options.rightKey) {
|
|
9
|
+
const leftCols = Array.isArray(options.leftKey) ? options.leftKey : options.leftKey.split(",").map((s) => s.trim());
|
|
10
|
+
const rightCols = Array.isArray(options.rightKey) ? options.rightKey : options.rightKey.split(",").map((s) => s.trim());
|
|
11
|
+
if (leftCols.length !== rightCols.length) {
|
|
12
|
+
throw new InvalidArgumentError(`Mismatched join keys: left has ${leftCols.length} columns (${leftCols.join(",")}), right has ${rightCols.length} columns (${rightCols.join(",")})`);
|
|
13
|
+
}
|
|
14
|
+
return { left: leftCols, right: rightCols };
|
|
15
|
+
}
|
|
16
|
+
if (options.on) {
|
|
17
|
+
const specs = Array.isArray(options.on) ? options.on : options.on.split(",").map((s) => s.trim());
|
|
18
|
+
const leftCols = [];
|
|
19
|
+
const rightCols = [];
|
|
20
|
+
for (const spec of specs) {
|
|
21
|
+
if (spec.includes("=")) {
|
|
22
|
+
const parts = spec.split("=");
|
|
23
|
+
leftCols.push(parts[0].trim());
|
|
24
|
+
rightCols.push(parts[1].trim());
|
|
25
|
+
}
|
|
26
|
+
else {
|
|
27
|
+
leftCols.push(spec.trim());
|
|
28
|
+
rightCols.push(spec.trim());
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
return { left: leftCols, right: rightCols };
|
|
32
|
+
}
|
|
33
|
+
// Default fallback to 'id'
|
|
34
|
+
return { left: ["id"], right: ["id"] };
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Generates an encoded lookup key for a row given key columns.
|
|
38
|
+
*/
|
|
39
|
+
function getRowKey(row, keyCols) {
|
|
40
|
+
return encodeCompositeKey(row, keyCols).encoded;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Merges a left row and right row, handling colliding column names.
|
|
44
|
+
*/
|
|
45
|
+
function mergeRows(leftRow, rightRow, joinKeys, options, sampleLeftCols, sampleRightCols) {
|
|
46
|
+
const result = {};
|
|
47
|
+
const prefixRight = options.prefixRight || "";
|
|
48
|
+
const suffixRight = options.suffixRight !== undefined ? options.suffixRight : (prefixRight ? "" : "_right");
|
|
49
|
+
const prefixLeft = options.prefixLeft || "";
|
|
50
|
+
const suffixLeft = options.suffixLeft || "";
|
|
51
|
+
if (leftRow) {
|
|
52
|
+
for (const [k, v] of Object.entries(leftRow)) {
|
|
53
|
+
const isKey = joinKeys.left.includes(k);
|
|
54
|
+
const outKey = isKey ? k : `${prefixLeft}${k}${suffixLeft}`;
|
|
55
|
+
result[outKey] = v;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
else if (sampleLeftCols) {
|
|
59
|
+
for (const col of sampleLeftCols) {
|
|
60
|
+
const isKey = joinKeys.left.includes(col);
|
|
61
|
+
const outKey = isKey ? col : `${prefixLeft}${col}${suffixLeft}`;
|
|
62
|
+
result[outKey] = null;
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
if (rightRow) {
|
|
66
|
+
for (const [k, v] of Object.entries(rightRow)) {
|
|
67
|
+
const rightKeyIdx = joinKeys.right.indexOf(k);
|
|
68
|
+
if (rightKeyIdx >= 0) {
|
|
69
|
+
// This is a join key column
|
|
70
|
+
const leftKeyName = joinKeys.left[rightKeyIdx];
|
|
71
|
+
if (!leftRow) {
|
|
72
|
+
result[leftKeyName] = v;
|
|
73
|
+
}
|
|
74
|
+
continue;
|
|
75
|
+
}
|
|
76
|
+
// Check if collides with left row non-key column
|
|
77
|
+
let targetKey = k;
|
|
78
|
+
if (leftRow && k in leftRow) {
|
|
79
|
+
targetKey = `${prefixRight}${k}${suffixRight}`;
|
|
80
|
+
}
|
|
81
|
+
else if (prefixRight || (suffixRight && suffixRight !== "_right")) {
|
|
82
|
+
targetKey = `${prefixRight}${k}${suffixRight}`;
|
|
83
|
+
}
|
|
84
|
+
result[targetKey] = v;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
else if (sampleRightCols) {
|
|
88
|
+
for (const col of sampleRightCols) {
|
|
89
|
+
const rightKeyIdx = joinKeys.right.indexOf(col);
|
|
90
|
+
if (rightKeyIdx >= 0)
|
|
91
|
+
continue;
|
|
92
|
+
let targetKey = col;
|
|
93
|
+
if (leftRow && col in leftRow) {
|
|
94
|
+
targetKey = `${prefixRight}${col}${suffixRight}`;
|
|
95
|
+
}
|
|
96
|
+
else if (prefixRight || (suffixRight && suffixRight !== "_right")) {
|
|
97
|
+
targetKey = `${prefixRight}${col}${suffixRight}`;
|
|
98
|
+
}
|
|
99
|
+
result[targetKey] = null;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
return result;
|
|
103
|
+
}
|
|
104
|
+
/**
|
|
105
|
+
* Joins two tabular streams with bounded memory and spill-to-disk index support.
|
|
106
|
+
*/
|
|
107
|
+
export async function* joinStreams(leftSource, rightSource, options = {}) {
|
|
108
|
+
const joinType = options.type || "inner";
|
|
109
|
+
const keys = parseJoinKeys(options);
|
|
110
|
+
const effectiveBatchSize = Math.max(1, options.batchSize || 1000);
|
|
111
|
+
const rightIndex = new SpillableJoinIndex({
|
|
112
|
+
memoryLimit: options.memoryLimit,
|
|
113
|
+
tempDir: options.tempDir,
|
|
114
|
+
});
|
|
115
|
+
const rightStream = "read" in rightSource ? rightSource.read({ batchSize: effectiveBatchSize }) : rightSource;
|
|
116
|
+
const leftStream = "read" in leftSource ? leftSource.read({ batchSize: effectiveBatchSize }) : leftSource;
|
|
117
|
+
let rightSampleCols = [];
|
|
118
|
+
let leftSampleCols = [];
|
|
119
|
+
try {
|
|
120
|
+
// 1. Build Phase: Ingest right dataset into SpillableJoinIndex
|
|
121
|
+
for await (const batch of rightStream) {
|
|
122
|
+
for (const row of batch.rows) {
|
|
123
|
+
if (rightSampleCols.length === 0) {
|
|
124
|
+
rightSampleCols = Object.keys(row);
|
|
125
|
+
}
|
|
126
|
+
const key = getRowKey(row, keys.right);
|
|
127
|
+
await rightIndex.set(key, row);
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
// 2. Probe Phase: Stream left dataset and match against rightIndex
|
|
131
|
+
let currentBatch = [];
|
|
132
|
+
let globalOffset = 0;
|
|
133
|
+
for await (const batch of leftStream) {
|
|
134
|
+
for (const leftRow of batch.rows) {
|
|
135
|
+
if (leftSampleCols.length === 0) {
|
|
136
|
+
leftSampleCols = Object.keys(leftRow);
|
|
137
|
+
}
|
|
138
|
+
const key = getRowKey(leftRow, keys.left);
|
|
139
|
+
if (joinType === "semi") {
|
|
140
|
+
const exists = await rightIndex.has(key);
|
|
141
|
+
if (exists) {
|
|
142
|
+
currentBatch.push(leftRow);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
else if (joinType === "anti") {
|
|
146
|
+
const exists = await rightIndex.has(key);
|
|
147
|
+
if (!exists) {
|
|
148
|
+
currentBatch.push(leftRow);
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
else {
|
|
152
|
+
const matchedRightRows = await rightIndex.get(key);
|
|
153
|
+
if (matchedRightRows && matchedRightRows.length > 0) {
|
|
154
|
+
rightIndex.markMatched(key);
|
|
155
|
+
for (const rightRow of matchedRightRows) {
|
|
156
|
+
const merged = mergeRows(leftRow, rightRow, keys, options, leftSampleCols, rightSampleCols);
|
|
157
|
+
currentBatch.push(merged);
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
else if (joinType === "left" || joinType === "full") {
|
|
161
|
+
const merged = mergeRows(leftRow, null, keys, options, leftSampleCols, rightSampleCols);
|
|
162
|
+
currentBatch.push(merged);
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
if (currentBatch.length >= effectiveBatchSize) {
|
|
166
|
+
yield {
|
|
167
|
+
rows: currentBatch,
|
|
168
|
+
offset: globalOffset,
|
|
169
|
+
};
|
|
170
|
+
globalOffset += currentBatch.length;
|
|
171
|
+
currentBatch = [];
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
// 3. Post-Probe Phase: For right and full outer joins, emit unmatched right rows
|
|
176
|
+
if (joinType === "right" || joinType === "full") {
|
|
177
|
+
for await (const unmatchedRightRow of rightIndex.getUnmatched()) {
|
|
178
|
+
const merged = mergeRows(null, unmatchedRightRow, keys, options, leftSampleCols, rightSampleCols);
|
|
179
|
+
currentBatch.push(merged);
|
|
180
|
+
if (currentBatch.length >= effectiveBatchSize) {
|
|
181
|
+
yield {
|
|
182
|
+
rows: currentBatch,
|
|
183
|
+
offset: globalOffset,
|
|
184
|
+
};
|
|
185
|
+
globalOffset += currentBatch.length;
|
|
186
|
+
currentBatch = [];
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
if (currentBatch.length > 0) {
|
|
191
|
+
yield {
|
|
192
|
+
rows: currentBatch,
|
|
193
|
+
offset: globalOffset,
|
|
194
|
+
};
|
|
195
|
+
globalOffset += currentBatch.length;
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
finally {
|
|
199
|
+
await rightIndex.close();
|
|
200
|
+
if ("close" in rightSource && typeof rightSource.close === "function") {
|
|
201
|
+
await rightSource.close();
|
|
202
|
+
}
|
|
203
|
+
if ("close" in leftSource && typeof leftSource.close === "function") {
|
|
204
|
+
await leftSource.close();
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
/**
|
|
209
|
+
* Returns a TransformFunction for pipeline chaining (e.g. pipeline.pipe(joinRows(rightReader, options))).
|
|
210
|
+
*/
|
|
211
|
+
export function joinRows(rightSource, options = {}) {
|
|
212
|
+
return (leftStream) => {
|
|
213
|
+
return joinStreams(leftStream, rightSource, options);
|
|
214
|
+
};
|
|
215
|
+
}
|
|
216
|
+
//# sourceMappingURL=join.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"join.js","sourceRoot":"","sources":["../../../src/transforms/join/join.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AACvD,OAAO,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAE5D,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAGxD;;GAEG;AACH,MAAM,UAAU,aAAa,CAAC,OAAoB;IAChD,IAAI,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACxC,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACpH,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACxH,IAAI,QAAQ,CAAC,MAAM,KAAK,SAAS,CAAC,MAAM,EAAE,CAAC;YACzC,MAAM,IAAI,oBAAoB,CAC5B,kCAAkC,QAAQ,CAAC,MAAM,aAAa,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,gBAAgB,SAAS,CAAC,MAAM,aAAa,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CACpJ,CAAC;QACJ,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;IAC9C,CAAC;IAED,IAAI,OAAO,CAAC,EAAE,EAAE,CAAC;QACf,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAClG,MAAM,QAAQ,GAAa,EAAE,CAAC;QAC9B,MAAM,SAAS,GAAa,EAAE,CAAC;QAE/B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBACvB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAC9B,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,CAAC,CAAC;gBAChC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,CAAC,CAAC;YACnC,CAAC;iBAAM,CAAC;gBACN,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC3B,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;IAC9C,CAAC;IAED,2BAA2B;IAC3B,OAAO,EAAE,IAAI,EAAE,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;AACzC,CAAC;AAED;;GAEG;AACH,SAAS,SAAS,CAAC,GAAQ,EAAE,OAAiB;IAC5C,OAAO,kBAAkB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,OAAO,CAAC;AAClD,CAAC;AAED;;GAEG;AACH,SAAS,SAAS,CAChB,OAAmB,EACnB,QAAoB,EACpB,QAAwB,EACxB,OAAoB,EACpB,cAAyB,EACzB,eAA0B;IAE1B,MAAM,MAAM,GAAQ,EAAE,CAAC;IACvB,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,EAAE,CAAC;IAC9C,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IAC5G,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,EAAE,CAAC;IAC5C,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,EAAE,CAAC;IAE5C,IAAI,OAAO,EAAE,CAAC;QACZ,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;YAC7C,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;YACxC,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,UAAU,GAAG,CAAC,GAAG,UAAU,EAAE,CAAC;YAC5D,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;SAAM,IAAI,cAAc,EAAE,CAAC;QAC1B,KAAK,MAAM,GAAG,IAAI,cAAc,EAAE,CAAC;YACjC,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;YAC1C,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,UAAU,GAAG,GAAG,GAAG,UAAU,EAAE,CAAC;YAChE,MAAM,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC;QACxB,CAAC;IACH,CAAC;IAED,IAAI,QAAQ,EAAE,CAAC;QACb,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC9C,MAAM,WAAW,GAAG,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YAC9C,IAAI,WAAW,IAAI,CAAC,EAAE,CAAC;gBACrB,4BAA4B;gBAC5B,MAAM,WAAW,GAAG,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAE,CAAC;gBAChD,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;gBAC1B,CAAC;gBACD,SAAS;YACX,CAAC;YAED,iDAAiD;YACjD,IAAI,SAAS,GAAG,CAAC,CAAC;YAClB,IAAI,OAAO,IAAI,CAAC,IAAI,OAAO,EAAE,CAAC;gBAC5B,SAAS,GAAG,GAAG,WAAW,GAAG,CAAC,GAAG,WAAW,EAAE,CAAC;YACjD,CAAC;iBAAM,IAAI,WAAW,IAAI,CAAC,WAAW,IAAI,WAAW,KAAK,QAAQ,CAAC,EAAE,CAAC;gBACpE,SAAS,GAAG,GAAG,WAAW,GAAG,CAAC,GAAG,WAAW,EAAE,CAAC;YACjD,CAAC;YACD,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;SAAM,IAAI,eAAe,EAAE,CAAC;QAC3B,KAAK,MAAM,GAAG,IAAI,eAAe,EAAE,CAAC;YAClC,MAAM,WAAW,GAAG,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAChD,IAAI,WAAW,IAAI,CAAC;gBAAE,SAAS;YAC/B,IAAI,SAAS,GAAG,GAAG,CAAC;YACpB,IAAI,OAAO,IAAI,GAAG,IAAI,OAAO,EAAE,CAAC;gBAC9B,SAAS,GAAG,GAAG,WAAW,GAAG,GAAG,GAAG,WAAW,EAAE,CAAC;YACnD,CAAC;iBAAM,IAAI,WAAW,IAAI,CAAC,WAAW,IAAI,WAAW,KAAK,QAAQ,CAAC,EAAE,CAAC;gBACpE,SAAS,GAAG,GAAG,WAAW,GAAG,GAAG,GAAG,WAAW,EAAE,CAAC;YACnD,CAAC;YACD,MAAM,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,WAAW,CAChC,UAAsC,EACtC,WAAuC,EACvC,UAAuB,EAAE;IAEzB,MAAM,QAAQ,GAAa,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC;IACnD,MAAM,IAAI,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC;IACpC,MAAM,kBAAkB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,CAAC;IAElE,MAAM,UAAU,GAAG,IAAI,kBAAkB,CAAC;QACxC,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,OAAO,EAAE,OAAO,CAAC,OAAO;KACzB,CAAC,CAAC;IAEH,MAAM,WAAW,GAAe,MAAM,IAAI,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,kBAAkB,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC;IAC1H,MAAM,UAAU,GAAe,MAAM,IAAI,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,kBAAkB,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC;IAEtH,IAAI,eAAe,GAAa,EAAE,CAAC;IACnC,IAAI,cAAc,GAAa,EAAE,CAAC;IAElC,IAAI,CAAC;QACH,+DAA+D;QAC/D,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,WAAW,EAAE,CAAC;YACtC,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;gBAC7B,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBACjC,eAAe,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACrC,CAAC;gBACD,MAAM,GAAG,GAAG,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;gBACvC,MAAM,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;YACjC,CAAC;QACH,CAAC;QAED,mEAAmE;QACnE,IAAI,YAAY,GAAU,EAAE,CAAC;QAC7B,IAAI,YAAY,GAAG,CAAC,CAAC;QAErB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;YACrC,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;gBACjC,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAChC,cAAc,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACxC,CAAC;gBACD,MAAM,GAAG,GAAG,SAAS,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;gBAE1C,IAAI,QAAQ,KAAK,MAAM,EAAE,CAAC;oBACxB,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;oBACzC,IAAI,MAAM,EAAE,CAAC;wBACX,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;oBAC7B,CAAC;gBACH,CAAC;qBAAM,IAAI,QAAQ,KAAK,MAAM,EAAE,CAAC;oBAC/B,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;oBACzC,IAAI,CAAC,MAAM,EAAE,CAAC;wBACZ,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;oBAC7B,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,MAAM,gBAAgB,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;oBACnD,IAAI,gBAAgB,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBACpD,UAAU,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;wBAC5B,KAAK,MAAM,QAAQ,IAAI,gBAAgB,EAAE,CAAC;4BACxC,MAAM,MAAM,GAAG,SAAS,CAAC,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,cAAc,EAAE,eAAe,CAAC,CAAC;4BAC5F,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;wBAC5B,CAAC;oBACH,CAAC;yBAAM,IAAI,QAAQ,KAAK,MAAM,IAAI,QAAQ,KAAK,MAAM,EAAE,CAAC;wBACtD,MAAM,MAAM,GAAG,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,cAAc,EAAE,eAAe,CAAC,CAAC;wBACxF,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;oBAC5B,CAAC;gBACH,CAAC;gBAED,IAAI,YAAY,CAAC,MAAM,IAAI,kBAAkB,EAAE,CAAC;oBAC9C,MAAM;wBACJ,IAAI,EAAE,YAAY;wBAClB,MAAM,EAAE,YAAY;qBACrB,CAAC;oBACF,YAAY,IAAI,YAAY,CAAC,MAAM,CAAC;oBACpC,YAAY,GAAG,EAAE,CAAC;gBACpB,CAAC;YACH,CAAC;QACH,CAAC;QAED,iFAAiF;QACjF,IAAI,QAAQ,KAAK,OAAO,IAAI,QAAQ,KAAK,MAAM,EAAE,CAAC;YAChD,IAAI,KAAK,EAAE,MAAM,iBAAiB,IAAI,UAAU,CAAC,YAAY,EAAE,EAAE,CAAC;gBAChE,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,EAAE,iBAAiB,EAAE,IAAI,EAAE,OAAO,EAAE,cAAc,EAAE,eAAe,CAAC,CAAC;gBAClG,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBAE1B,IAAI,YAAY,CAAC,MAAM,IAAI,kBAAkB,EAAE,CAAC;oBAC9C,MAAM;wBACJ,IAAI,EAAE,YAAY;wBAClB,MAAM,EAAE,YAAY;qBACrB,CAAC;oBACF,YAAY,IAAI,YAAY,CAAC,MAAM,CAAC;oBACpC,YAAY,GAAG,EAAE,CAAC;gBACpB,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM;gBACJ,IAAI,EAAE,YAAY;gBAClB,MAAM,EAAE,YAAY;aACrB,CAAC;YACF,YAAY,IAAI,YAAY,CAAC,MAAM,CAAC;QACtC,CAAC;IACH,CAAC;YAAS,CAAC;QACT,MAAM,UAAU,CAAC,KAAK,EAAE,CAAC;QACzB,IAAI,OAAO,IAAI,WAAW,IAAI,OAAO,WAAW,CAAC,KAAK,KAAK,UAAU,EAAE,CAAC;YACtE,MAAM,WAAW,CAAC,KAAK,EAAE,CAAC;QAC5B,CAAC;QACD,IAAI,OAAO,IAAI,UAAU,IAAI,OAAO,UAAU,CAAC,KAAK,KAAK,UAAU,EAAE,CAAC;YACpE,MAAM,UAAU,CAAC,KAAK,EAAE,CAAC;QAC3B,CAAC;IACH,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,QAAQ,CACtB,WAAuC,EACvC,UAAuB,EAAE;IAEzB,OAAO,CAAC,UAAsB,EAAc,EAAE;QAC5C,OAAO,WAAW,CAAC,UAAU,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;IACvD,CAAC,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { Row } from "../../core/types.js";
|
|
2
|
+
export type JoinType = "inner" | "left" | "right" | "full" | "semi" | "anti";
|
|
3
|
+
export interface JoinKeyMapping {
|
|
4
|
+
left: string[];
|
|
5
|
+
right: string[];
|
|
6
|
+
}
|
|
7
|
+
export interface JoinOptions {
|
|
8
|
+
on?: string | string[];
|
|
9
|
+
leftKey?: string | string[];
|
|
10
|
+
rightKey?: string | string[];
|
|
11
|
+
type?: JoinType;
|
|
12
|
+
prefixLeft?: string;
|
|
13
|
+
suffixLeft?: string;
|
|
14
|
+
prefixRight?: string;
|
|
15
|
+
suffixRight?: string;
|
|
16
|
+
memoryLimit?: number | string;
|
|
17
|
+
tempDir?: string;
|
|
18
|
+
batchSize?: number;
|
|
19
|
+
}
|
|
20
|
+
export interface JoinIndex {
|
|
21
|
+
set(key: string, row: Row): Promise<void>;
|
|
22
|
+
get(key: string): Promise<Row[] | undefined>;
|
|
23
|
+
has(key: string): Promise<boolean>;
|
|
24
|
+
markMatched(key: string): void;
|
|
25
|
+
getUnmatched(): AsyncIterable<Row>;
|
|
26
|
+
count(): number;
|
|
27
|
+
close(): Promise<void>;
|
|
28
|
+
}
|
|
29
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/transforms/join/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,qBAAqB,CAAC;AAE/C,MAAM,MAAM,QAAQ,GAAG,OAAO,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,MAAM,CAAC;AAE7E,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,KAAK,EAAE,MAAM,EAAE,CAAC;CACjB;AAED,MAAM,WAAW,WAAW;IAC1B,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IACvB,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC5B,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC7B,IAAI,CAAC,EAAE,QAAQ,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC9B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,SAAS;IACxB,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1C,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,CAAC;IAC7C,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IACnC,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,YAAY,IAAI,aAAa,CAAC,GAAG,CAAC,CAAC;IACnC,KAAK,IAAI,MAAM,CAAC;IAChB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/transforms/join/types.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { TransformFunction } from "../core/types.js";
|
|
2
|
+
/**
|
|
3
|
+
* Limits the number of rows emitted to `maxCount` and cancels the upstream stream immediately.
|
|
4
|
+
* Zero memory overhead ($O(1)$) and true early cancellation.
|
|
5
|
+
*/
|
|
6
|
+
export declare function limitRows(maxCount: number): TransformFunction;
|
|
7
|
+
//# sourceMappingURL=limit.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"limit.d.ts","sourceRoot":"","sources":["../../src/transforms/limit.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAA8B,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAEtF;;;GAGG;AACH,wBAAgB,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,iBAAiB,CAqC7D"}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Limits the number of rows emitted to `maxCount` and cancels the upstream stream immediately.
|
|
3
|
+
* Zero memory overhead ($O(1)$) and true early cancellation.
|
|
4
|
+
*/
|
|
5
|
+
export function limitRows(maxCount) {
|
|
6
|
+
const limit = Math.max(0, maxCount);
|
|
7
|
+
return (stream) => {
|
|
8
|
+
return (async function* () {
|
|
9
|
+
if (limit === 0)
|
|
10
|
+
return;
|
|
11
|
+
let emitted = 0;
|
|
12
|
+
for await (const batch of stream) {
|
|
13
|
+
if (batch.rows.length === 0)
|
|
14
|
+
continue;
|
|
15
|
+
const remaining = limit - emitted;
|
|
16
|
+
if (remaining <= 0) {
|
|
17
|
+
break;
|
|
18
|
+
}
|
|
19
|
+
if (batch.rows.length <= remaining) {
|
|
20
|
+
yield batch;
|
|
21
|
+
emitted += batch.rows.length;
|
|
22
|
+
}
|
|
23
|
+
else {
|
|
24
|
+
// Slice partial batch to meet exact limit
|
|
25
|
+
const sliceRows = batch.rows.slice(0, remaining);
|
|
26
|
+
yield {
|
|
27
|
+
rows: sliceRows,
|
|
28
|
+
offset: batch.offset,
|
|
29
|
+
};
|
|
30
|
+
emitted += sliceRows.length;
|
|
31
|
+
break;
|
|
32
|
+
}
|
|
33
|
+
if (emitted >= limit) {
|
|
34
|
+
break;
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
})();
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
//# sourceMappingURL=limit.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"limit.js","sourceRoot":"","sources":["../../src/transforms/limit.ts"],"names":[],"mappings":"AAEA;;;GAGG;AACH,MAAM,UAAU,SAAS,CAAC,QAAgB;IACxC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IAEpC,OAAO,CAAC,MAAkB,EAAc,EAAE;QACxC,OAAO,CAAC,KAAK,SAAS,CAAC;YACrB,IAAI,KAAK,KAAK,CAAC;gBAAE,OAAO;YAExB,IAAI,OAAO,GAAG,CAAC,CAAC;YAEhB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBACjC,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC;oBAAE,SAAS;gBAEtC,MAAM,SAAS,GAAG,KAAK,GAAG,OAAO,CAAC;gBAClC,IAAI,SAAS,IAAI,CAAC,EAAE,CAAC;oBACnB,MAAM;gBACR,CAAC;gBAED,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;oBACnC,MAAM,KAAK,CAAC;oBACZ,OAAO,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC;gBAC/B,CAAC;qBAAM,CAAC;oBACN,0CAA0C;oBAC1C,MAAM,SAAS,GAAU,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;oBACxD,MAAM;wBACJ,IAAI,EAAE,SAAS;wBACf,MAAM,EAAE,KAAK,CAAC,MAAM;qBACrB,CAAC;oBACF,OAAO,IAAI,SAAS,CAAC,MAAM,CAAC;oBAC5B,MAAM;gBACR,CAAC;gBAED,IAAI,OAAO,IAAI,KAAK,EAAE,CAAC;oBACrB,MAAM;gBACR,CAAC;YACH,CAAC;QACH,CAAC,CAAC,EAAE,CAAC;IACP,CAAC,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { TransformFunction } from "../core/types.js";
|
|
2
|
+
/**
|
|
3
|
+
* Skips the first `skipCount` rows in the stream without buffering.
|
|
4
|
+
* Zero memory overhead ($O(1)$).
|
|
5
|
+
*/
|
|
6
|
+
export declare function offsetRows(skipCount: number): TransformFunction;
|
|
7
|
+
//# sourceMappingURL=offset.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"offset.d.ts","sourceRoot":"","sources":["../../src/transforms/offset.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAA8B,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAEtF;;;GAGG;AACH,wBAAgB,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,iBAAiB,CAqC/D"}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Skips the first `skipCount` rows in the stream without buffering.
|
|
3
|
+
* Zero memory overhead ($O(1)$).
|
|
4
|
+
*/
|
|
5
|
+
export function offsetRows(skipCount) {
|
|
6
|
+
const offsetToSkip = Math.max(0, skipCount);
|
|
7
|
+
return (stream) => {
|
|
8
|
+
return (async function* () {
|
|
9
|
+
if (offsetToSkip === 0) {
|
|
10
|
+
yield* stream;
|
|
11
|
+
return;
|
|
12
|
+
}
|
|
13
|
+
let skipped = 0;
|
|
14
|
+
for await (const batch of stream) {
|
|
15
|
+
if (batch.rows.length === 0)
|
|
16
|
+
continue;
|
|
17
|
+
if (skipped < offsetToSkip) {
|
|
18
|
+
const remainingToSkip = offsetToSkip - skipped;
|
|
19
|
+
if (batch.rows.length <= remainingToSkip) {
|
|
20
|
+
skipped += batch.rows.length;
|
|
21
|
+
continue;
|
|
22
|
+
}
|
|
23
|
+
// Partial batch skip
|
|
24
|
+
const remainingRows = batch.rows.slice(remainingToSkip);
|
|
25
|
+
skipped += remainingToSkip;
|
|
26
|
+
yield {
|
|
27
|
+
rows: remainingRows,
|
|
28
|
+
offset: batch.offset + remainingToSkip,
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
else {
|
|
32
|
+
yield batch;
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
})();
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
//# sourceMappingURL=offset.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"offset.js","sourceRoot":"","sources":["../../src/transforms/offset.ts"],"names":[],"mappings":"AAEA;;;GAGG;AACH,MAAM,UAAU,UAAU,CAAC,SAAiB;IAC1C,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;IAE5C,OAAO,CAAC,MAAkB,EAAc,EAAE;QACxC,OAAO,CAAC,KAAK,SAAS,CAAC;YACrB,IAAI,YAAY,KAAK,CAAC,EAAE,CAAC;gBACvB,KAAK,CAAC,CAAC,MAAM,CAAC;gBACd,OAAO;YACT,CAAC;YAED,IAAI,OAAO,GAAG,CAAC,CAAC;YAEhB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBACjC,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC;oBAAE,SAAS;gBAEtC,IAAI,OAAO,GAAG,YAAY,EAAE,CAAC;oBAC3B,MAAM,eAAe,GAAG,YAAY,GAAG,OAAO,CAAC;oBAE/C,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,IAAI,eAAe,EAAE,CAAC;wBACzC,OAAO,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC;wBAC7B,SAAS;oBACX,CAAC;oBAED,qBAAqB;oBACrB,MAAM,aAAa,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;oBACxD,OAAO,IAAI,eAAe,CAAC;oBAE3B,MAAM;wBACJ,IAAI,EAAE,aAAa;wBACnB,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,eAAe;qBACvC,CAAC;gBACJ,CAAC;qBAAM,CAAC;oBACN,MAAM,KAAK,CAAC;gBACd,CAAC;YACH,CAAC;QACH,CAAC,CAAC,EAAE,CAAC;IACP,CAAC,CAAC;AACJ,CAAC"}
|