deltacat 2.0.0b10__py3-none-any.whl → 2.0.0b11__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- deltacat/__init__.py +19 -15
- deltacat/benchmarking/benchmark_engine.py +4 -2
- deltacat/benchmarking/test_benchmark_pipeline.py +6 -4
- deltacat/catalog/__init__.py +62 -5
- deltacat/catalog/main/impl.py +18 -8
- deltacat/catalog/model/catalog.py +111 -73
- deltacat/catalog/model/properties.py +25 -22
- deltacat/compute/jobs/client.py +7 -5
- deltacat/constants.py +1 -2
- deltacat/env.py +10 -0
- deltacat/examples/basic_logging.py +1 -3
- deltacat/examples/{iceberg → experimental/iceberg}/iceberg_bucket_writer.py +3 -5
- deltacat/examples/{iceberg → experimental/iceberg}/iceberg_reader.py +2 -4
- deltacat/examples/indexer/indexer.py +2 -2
- deltacat/examples/indexer/job_runner.py +1 -2
- deltacat/experimental/catalog/iceberg/__init__.py +6 -0
- deltacat/{catalog → experimental/catalog}/iceberg/iceberg_catalog_config.py +1 -1
- deltacat/{catalog → experimental/catalog}/iceberg/impl.py +27 -9
- deltacat/{storage → experimental/storage}/iceberg/iceberg_scan_planner.py +1 -1
- deltacat/{storage → experimental/storage}/iceberg/impl.py +1 -1
- deltacat/experimental/storage/rivulet/__init__.py +11 -0
- deltacat/{storage → experimental/storage}/rivulet/arrow/serializer.py +7 -4
- deltacat/{storage → experimental/storage}/rivulet/dataset.py +13 -9
- deltacat/{storage → experimental/storage}/rivulet/dataset_executor.py +12 -20
- deltacat/experimental/storage/rivulet/feather/__init__.py +7 -0
- deltacat/{storage → experimental/storage}/rivulet/feather/file_reader.py +7 -5
- deltacat/{storage → experimental/storage}/rivulet/feather/serializer.py +4 -4
- deltacat/{storage → experimental/storage}/rivulet/fs/file_provider.py +3 -3
- deltacat/{storage → experimental/storage}/rivulet/fs/file_store.py +2 -2
- deltacat/{storage → experimental/storage}/rivulet/fs/output_file.py +1 -1
- deltacat/{storage → experimental/storage}/rivulet/logical_plan.py +4 -4
- deltacat/{storage → experimental/storage}/rivulet/metastore/delta.py +1 -1
- deltacat/{storage → experimental/storage}/rivulet/metastore/json_sst.py +3 -3
- deltacat/{storage → experimental/storage}/rivulet/metastore/sst.py +2 -2
- deltacat/{storage → experimental/storage}/rivulet/metastore/sst_interval_tree.py +3 -3
- deltacat/experimental/storage/rivulet/parquet/__init__.py +7 -0
- deltacat/{storage → experimental/storage}/rivulet/parquet/file_reader.py +7 -5
- deltacat/{storage → experimental/storage}/rivulet/parquet/serializer.py +4 -4
- deltacat/{storage → experimental/storage}/rivulet/reader/block_scanner.py +20 -9
- deltacat/{storage → experimental/storage}/rivulet/reader/data_reader.py +3 -3
- deltacat/{storage → experimental/storage}/rivulet/reader/data_scan.py +5 -3
- deltacat/{storage → experimental/storage}/rivulet/reader/dataset_metastore.py +4 -4
- deltacat/{storage → experimental/storage}/rivulet/reader/dataset_reader.py +8 -6
- deltacat/{storage → experimental/storage}/rivulet/reader/pyarrow_data_reader.py +4 -1
- deltacat/{storage → experimental/storage}/rivulet/reader/reader_type_registrar.py +4 -4
- deltacat/{storage → experimental/storage}/rivulet/schema/schema.py +1 -1
- deltacat/{storage → experimental/storage}/rivulet/serializer.py +1 -1
- deltacat/{storage → experimental/storage}/rivulet/serializer_factory.py +9 -5
- deltacat/experimental/storage/rivulet/shard/range_shard.py +129 -0
- deltacat/{storage → experimental/storage}/rivulet/writer/memtable_dataset_writer.py +20 -9
- deltacat/io/reader/deltacat_read_api.py +1 -1
- deltacat/storage/model/shard.py +6 -2
- deltacat/tests/catalog/main/test_catalog_impl_namespace_operations.py +130 -0
- deltacat/tests/catalog/main/test_catalog_impl_table_operations.py +436 -0
- deltacat/tests/catalog/model/__init__.py +0 -0
- deltacat/tests/catalog/model/test_table_definition.py +16 -0
- deltacat/tests/catalog/test_catalogs.py +52 -98
- deltacat/tests/catalog/test_default_catalog_impl.py +1 -2
- deltacat/tests/daft/__init__.py +0 -0
- deltacat/tests/daft/test_model.py +97 -0
- deltacat/tests/experimental/__init__.py +0 -0
- deltacat/tests/experimental/catalog/__init__.py +0 -0
- deltacat/tests/experimental/catalog/iceberg/__init__.py +0 -0
- deltacat/tests/experimental/catalog/iceberg/test_iceberg_catalog.py +71 -0
- deltacat/tests/experimental/daft/__init__.py +0 -0
- deltacat/tests/experimental/daft/test_deltacat_daft_integration.py +136 -0
- deltacat/tests/experimental/storage/__init__.py +0 -0
- deltacat/tests/experimental/storage/rivulet/__init__.py +0 -0
- deltacat/tests/{storage → experimental/storage}/rivulet/conftest.py +3 -3
- deltacat/tests/experimental/storage/rivulet/fs/__init__.py +0 -0
- deltacat/tests/{storage → experimental/storage}/rivulet/fs/test_file_location_provider.py +3 -3
- deltacat/tests/experimental/storage/rivulet/reader/__init__.py +0 -0
- deltacat/tests/experimental/storage/rivulet/reader/query_expression.py +80 -0
- deltacat/tests/experimental/storage/rivulet/reader/test_data_scan.py +119 -0
- deltacat/tests/experimental/storage/rivulet/reader/test_dataset_metastore.py +71 -0
- deltacat/tests/experimental/storage/rivulet/schema/__init__.py +0 -0
- deltacat/tests/{storage → experimental/storage}/rivulet/schema/test_schema.py +1 -1
- deltacat/tests/experimental/storage/rivulet/shard/__init__.py +0 -0
- deltacat/tests/experimental/storage/rivulet/shard/test_range_shard.py +162 -0
- deltacat/tests/{storage → experimental/storage}/rivulet/test_dataset.py +5 -3
- deltacat/tests/{storage → experimental/storage}/rivulet/test_manifest.py +5 -5
- deltacat/tests/{storage → experimental/storage}/rivulet/test_sst_interval_tree.py +5 -5
- deltacat/tests/{storage → experimental/storage}/rivulet/test_utils.py +8 -6
- deltacat/tests/experimental/storage/rivulet/writer/__init__.py +0 -0
- deltacat/tests/{storage → experimental/storage}/rivulet/writer/test_dataset_write_then_read.py +11 -9
- deltacat/tests/{storage → experimental/storage}/rivulet/writer/test_dataset_writer.py +2 -2
- deltacat/tests/{storage → experimental/storage}/rivulet/writer/test_memtable_dataset_writer.py +7 -7
- deltacat/tests/storage/model/test_shard.py +3 -1
- deltacat/types/media.py +3 -3
- deltacat/utils/daft.py +530 -4
- deltacat/utils/export.py +3 -1
- deltacat/utils/url.py +1 -1
- {deltacat-2.0.0b10.dist-info → deltacat-2.0.0b11.dist-info}/METADATA +4 -5
- {deltacat-2.0.0b10.dist-info → deltacat-2.0.0b11.dist-info}/RECORD +120 -100
- deltacat/catalog/iceberg/__init__.py +0 -4
- deltacat/daft/daft_scan.py +0 -115
- deltacat/daft/model.py +0 -258
- deltacat/daft/translator.py +0 -126
- deltacat/examples/common/fixtures.py +0 -15
- deltacat/storage/rivulet/__init__.py +0 -11
- deltacat/storage/rivulet/feather/__init__.py +0 -5
- deltacat/storage/rivulet/parquet/__init__.py +0 -5
- /deltacat/{daft → examples/experimental}/__init__.py +0 -0
- /deltacat/examples/{common → experimental/iceberg}/__init__.py +0 -0
- /deltacat/{examples/iceberg → experimental/catalog}/__init__.py +0 -0
- /deltacat/{catalog → experimental/catalog}/iceberg/overrides.py +0 -0
- /deltacat/{storage/iceberg → experimental/storage}/__init__.py +0 -0
- /deltacat/{storage/rivulet/arrow → experimental/storage/iceberg}/__init__.py +0 -0
- /deltacat/{storage → experimental/storage}/iceberg/model.py +0 -0
- /deltacat/{storage/rivulet/fs → experimental/storage/rivulet/arrow}/__init__.py +0 -0
- /deltacat/{storage/rivulet/metastore → experimental/storage/rivulet/fs}/__init__.py +0 -0
- /deltacat/{storage → experimental/storage}/rivulet/fs/input_file.py +0 -0
- /deltacat/{storage/rivulet/reader → experimental/storage/rivulet/metastore}/__init__.py +0 -0
- /deltacat/{storage → experimental/storage}/rivulet/mvp/Table.py +0 -0
- /deltacat/{storage → experimental/storage}/rivulet/mvp/__init__.py +0 -0
- /deltacat/{storage → experimental/storage}/rivulet/parquet/data_reader.py +0 -0
- /deltacat/{storage/rivulet/schema → experimental/storage/rivulet/reader}/__init__.py +0 -0
- /deltacat/{storage → experimental/storage}/rivulet/reader/query_expression.py +0 -0
- /deltacat/{storage/rivulet/writer → experimental/storage/rivulet/schema}/__init__.py +0 -0
- /deltacat/{storage → experimental/storage}/rivulet/schema/datatype.py +0 -0
- /deltacat/{tests/storage/rivulet → experimental/storage/rivulet/shard}/__init__.py +0 -0
- /deltacat/{tests/storage/rivulet/fs → experimental/storage/rivulet/writer}/__init__.py +0 -0
- /deltacat/{storage → experimental/storage}/rivulet/writer/dataset_writer.py +0 -0
- /deltacat/tests/{storage/rivulet/schema → catalog/data}/__init__.py +0 -0
- /deltacat/tests/{storage/rivulet/writer → catalog/main}/__init__.py +0 -0
- {deltacat-2.0.0b10.dist-info → deltacat-2.0.0b11.dist-info}/LICENSE +0 -0
- {deltacat-2.0.0b10.dist-info → deltacat-2.0.0b11.dist-info}/WHEEL +0 -0
- {deltacat-2.0.0b10.dist-info → deltacat-2.0.0b11.dist-info}/top_level.txt +0 -0
@@ -1,8 +1,8 @@
|
|
1
|
-
deltacat/__init__.py,sha256=
|
1
|
+
deltacat/__init__.py,sha256=_r2iNgTB-UbRKhQooHH8Q_Ul_sb6dScSkIHSYsrZ_7E,2835
|
2
2
|
deltacat/annotations.py,sha256=9lBi34DpIV_RPjCCK2Aiz_6nMyd-e-_CfQ1XtdRQQlM,1196
|
3
3
|
deltacat/api.py,sha256=8wXFzNWDUYqy40ZtRex4outDPiFy8K8KHF-PSxMsZsw,18396
|
4
|
-
deltacat/constants.py,sha256=
|
5
|
-
deltacat/env.py,sha256=
|
4
|
+
deltacat/constants.py,sha256=ggHddITClnrcrzeGQkSas81UeyXGrF1DmLZQG-oNpik,3395
|
5
|
+
deltacat/env.py,sha256=BJdTt8od3IVR4RMLjBxy4oRUHM7Lb16AzMOz8-hpwOI,2303
|
6
6
|
deltacat/exceptions.py,sha256=K44xpmWNptIpM2gEWX6cy8Ko0vU1G7qDwwILDW8NWKM,13938
|
7
7
|
deltacat/logs.py,sha256=vlKFsid9D6dpX9JlIP67fRZy18SjA08XMEfn0QWrxt0,9513
|
8
8
|
deltacat/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
@@ -10,27 +10,23 @@ deltacat/aws/clients.py,sha256=4eQvpkV1PzFfxog7EriuglOGGwNFHR5hbGYpjsNNPxk,6949
|
|
10
10
|
deltacat/aws/constants.py,sha256=hcYAUot4ahq9GXCMClQiuYCtiDs5XaOebdUoKg4V84k,1222
|
11
11
|
deltacat/aws/s3u.py,sha256=OQYDh0MqwH6UPSU62fzy0g226OiPeFndjsnwFkd8GGc,28572
|
12
12
|
deltacat/benchmarking/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
13
|
-
deltacat/benchmarking/benchmark_engine.py,sha256=
|
13
|
+
deltacat/benchmarking/benchmark_engine.py,sha256=cLoENLTJNAO01Xfsyf14B7xvE4KW1FCyDNcyZp3AUdE,3377
|
14
14
|
deltacat/benchmarking/benchmark_parquet_reads.py,sha256=2BctkvXAYcAxokLwMSTu4TQ6-HGqzkgYcVEAzPN2QQo,1709
|
15
15
|
deltacat/benchmarking/benchmark_report.py,sha256=Bm-7NQFiQM0f47tQ9jMjcaQ8GBNxZ-m8Z130wgwTldk,2719
|
16
16
|
deltacat/benchmarking/benchmark_suite.py,sha256=RbZR1p0U0MU0HkGR06MosV6TKysooQD0HQne2rUAIds,316
|
17
17
|
deltacat/benchmarking/conftest.py,sha256=WRfEz46G8PyLp9E3RNpv_jpys7AyGiC-GFpUYs08f9M,2414
|
18
|
-
deltacat/benchmarking/test_benchmark_pipeline.py,sha256=
|
18
|
+
deltacat/benchmarking/test_benchmark_pipeline.py,sha256=MD6Af3EJ1V80l9xqCbl4N5uEjI8thMb2zXgJIrs2Ofg,3851
|
19
19
|
deltacat/benchmarking/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
20
20
|
deltacat/benchmarking/data/random_row_generator.py,sha256=-D1RFIQ6jE6KBmWGN6We8smhYxWQ19N2Pi8Z3iwBCiM,3227
|
21
21
|
deltacat/benchmarking/data/row_generator.py,sha256=PqtYdbgznqKVfidnR40gwET7yHov2N7Esd8bEY90ALU,333
|
22
|
-
deltacat/catalog/__init__.py,sha256=
|
22
|
+
deltacat/catalog/__init__.py,sha256=8tC0n1Y1LSs-tePt1TAPg9i_kNGxo0WgyFSlGFYxxhw,1500
|
23
23
|
deltacat/catalog/delegate.py,sha256=x3jj_T61gyExuAnbDqhU6smbaAbIN4UxrVMZuBEOg0A,10447
|
24
24
|
deltacat/catalog/interface.py,sha256=YB-qNBFsWupqyWJuHr7eQ-_MshhZZ5HpLphoZ64yn2g,12244
|
25
|
-
deltacat/catalog/iceberg/__init__.py,sha256=LOENcLTQQlu_694MvRhMd2TQDLzwfg2vz0D8DuVO3M8,190
|
26
|
-
deltacat/catalog/iceberg/iceberg_catalog_config.py,sha256=LfHxv8pk-YmTRQy5LvKFzwSqZ8ek2Y6v0KY7xihhIN0,786
|
27
|
-
deltacat/catalog/iceberg/impl.py,sha256=c_ONnLLyh8Vyqo5PusQSHySQ92iM4Qgk-rucHMfdd7s,14288
|
28
|
-
deltacat/catalog/iceberg/overrides.py,sha256=WmM2mxf7ihDl8anb5GzBxo5-sxBkot8ZSRTxDpaauRA,2687
|
29
25
|
deltacat/catalog/main/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
30
|
-
deltacat/catalog/main/impl.py,sha256
|
26
|
+
deltacat/catalog/main/impl.py,sha256=-thwtB19f45Hh2zlenXMcRRrKDWvjbGU10OagiVpYsI,23681
|
31
27
|
deltacat/catalog/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
32
|
-
deltacat/catalog/model/catalog.py,sha256=
|
33
|
-
deltacat/catalog/model/properties.py,sha256=
|
28
|
+
deltacat/catalog/model/catalog.py,sha256=qgIw5faIOdy1Tbn6JJv0J-3gXXGmVY7tTOp5IDvg5Og,11148
|
29
|
+
deltacat/catalog/model/properties.py,sha256=UraiKC99UlZh1ev4UbydQbBaa8-L5QhgSxiMaMbxSE0,3894
|
34
30
|
deltacat/catalog/model/table_definition.py,sha256=mKmwFd2qKlj7ZH4AdZwU3bEcFtzpGvl0mXuWjqLUgBw,1842
|
35
31
|
deltacat/compute/__init__.py,sha256=Yopyz3mokk-HtV9CDjQFCluvHbI7UiOPZYVtapUu13A,247
|
36
32
|
deltacat/compute/compactor/__init__.py,sha256=ivpOPve1yKi3Vz3tVgp-eeFMNEeUSf-dlRJNSCM85sE,1022
|
@@ -113,7 +109,7 @@ deltacat/compute/converter/utils/iceberg_columns.py,sha256=k7ng1LOCU8VUfEHMj6z0l
|
|
113
109
|
deltacat/compute/converter/utils/io.py,sha256=8ymAFaciRpIhFbe_bl6D-Ozn5HehO5sCKV24DeenNbQ,4218
|
114
110
|
deltacat/compute/converter/utils/s3u.py,sha256=Du5UKB0hdaQmT-yGaFtUVCXB2KTBVXWmKRL2er49KKg,4434
|
115
111
|
deltacat/compute/jobs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
116
|
-
deltacat/compute/jobs/client.py,sha256=
|
112
|
+
deltacat/compute/jobs/client.py,sha256=xWhZUpBNoSFa7s0MxTbqJ0OKHHiB1O3SIbM8gVJZaq8,14750
|
117
113
|
deltacat/compute/merge_on_read/__init__.py,sha256=ckbgngmqPjYBYz_NySsR1vNTOb_hNpeL1sYkZKvBI9M,214
|
118
114
|
deltacat/compute/merge_on_read/daft.py,sha256=1oC38u5ig_aTrq7EzyWBo8Ui54rb6yERYMk-vEFbpxM,1400
|
119
115
|
deltacat/compute/merge_on_read/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
@@ -133,26 +129,75 @@ deltacat/compute/stats/models/delta_stats.py,sha256=hBith8_hbF9TVr6HocLAt6RJ_kZZ
|
|
133
129
|
deltacat/compute/stats/models/delta_stats_cache_result.py,sha256=mbJYxpZd5jaER_BWrCD2hROFy3p1nNdBrj66nUpc6io,1624
|
134
130
|
deltacat/compute/stats/models/manifest_entry_stats.py,sha256=NCDAe2nPDEI4kOkuwNkRFgGPS-rqQaQqLuaLoKk20KQ,2419
|
135
131
|
deltacat/compute/stats/models/stats_result.py,sha256=XQAlmzhUqRmg4jzEMUAOqcYn1HUOBTMryBH1CCVlet8,3820
|
136
|
-
deltacat/daft/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
137
|
-
deltacat/daft/daft_scan.py,sha256=30S6OgKx3eBovpAVomHcr8xjjJVf_vwhJ9FA6dKQKYc,3981
|
138
|
-
deltacat/daft/model.py,sha256=6NaKkp9R0ruE0K2x-moyARNrQisswUl6TjMeA6YHtBM,9078
|
139
|
-
deltacat/daft/translator.py,sha256=Tm2K1Zcik2TjV_vKCd8Jc2IEA5MvSKLz1FvGODKrWR8,3771
|
140
132
|
deltacat/examples/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
141
|
-
deltacat/examples/basic_logging.py,sha256=
|
133
|
+
deltacat/examples/basic_logging.py,sha256=Umrum-gvY3gJjDNJ4hOMslMMq9bzeTM-s_DO4dGqJiw,2833
|
142
134
|
deltacat/examples/hello_world.py,sha256=FvxkEDB1qVPJv55Fe1I7Coy0VLYJIisU7ZFYYkw9U2g,525
|
143
|
-
deltacat/examples/
|
144
|
-
deltacat/examples/
|
145
|
-
deltacat/examples/iceberg/
|
146
|
-
deltacat/examples/iceberg/
|
147
|
-
deltacat/examples/iceberg/iceberg_reader.py,sha256=mlF-277vT04at-2jibAjgRJG6Y-zle_NNy1-pXwS2YQ,5023
|
135
|
+
deltacat/examples/experimental/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
136
|
+
deltacat/examples/experimental/iceberg/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
137
|
+
deltacat/examples/experimental/iceberg/iceberg_bucket_writer.py,sha256=7v4y_qWLpjB1oT_ZfUXORTF7kWZygUmxmKJoGakLsio,6573
|
138
|
+
deltacat/examples/experimental/iceberg/iceberg_reader.py,sha256=ygnfXiZkVI4MGnbkrAFfHl6eECjVUW2jqz6N5tjJ4Bk,4997
|
148
139
|
deltacat/examples/indexer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
149
|
-
deltacat/examples/indexer/indexer.py,sha256=
|
150
|
-
deltacat/examples/indexer/job_runner.py,sha256=
|
140
|
+
deltacat/examples/indexer/indexer.py,sha256=A5gfWomVtfu9uRrAB6UbitQ158ZFkviUscSveri1ggs,6334
|
141
|
+
deltacat/examples/indexer/job_runner.py,sha256=M3ZKTcPL5gy1u_E3aZPal2u5INWZlBGDrsRhb5-F7n4,5921
|
151
142
|
deltacat/examples/indexer/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
152
143
|
deltacat/examples/indexer/gcp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
153
144
|
deltacat/experimental/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
145
|
+
deltacat/experimental/catalog/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
146
|
+
deltacat/experimental/catalog/iceberg/__init__.py,sha256=rKZk4_5Yh4k50DVxOMb91VUqE18dn3k8XML80ZX_Z60,225
|
147
|
+
deltacat/experimental/catalog/iceberg/iceberg_catalog_config.py,sha256=W3916LCtSP3IPjOzyY-OKH5sSfqixVjVoANb6WVPhB4,799
|
148
|
+
deltacat/experimental/catalog/iceberg/impl.py,sha256=YDNRa6363XCPQXcPJkLmoq5cc-qIB1wh_BGou0y2t7c,14753
|
149
|
+
deltacat/experimental/catalog/iceberg/overrides.py,sha256=WmM2mxf7ihDl8anb5GzBxo5-sxBkot8ZSRTxDpaauRA,2687
|
154
150
|
deltacat/experimental/daft/__init__.py,sha256=0d1SsgjbDher8TKgS0gSBBdy5TGi01fewiwpG0BMwck,108
|
155
151
|
deltacat/experimental/daft/daft_catalog.py,sha256=112wDqqzdtxmtZVwiZW59MektbRsFMjSRgqYHrUOuok,8396
|
152
|
+
deltacat/experimental/storage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
153
|
+
deltacat/experimental/storage/iceberg/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
154
|
+
deltacat/experimental/storage/iceberg/iceberg_scan_planner.py,sha256=kraWLzN9zC0_7F7jgtWXo6utPDGaez4CR7JgOgmT-R8,1082
|
155
|
+
deltacat/experimental/storage/iceberg/impl.py,sha256=vF8n_W6fY8SNhYS3abj2k36k3dUruYZm8Ol0L-H2_sg,26326
|
156
|
+
deltacat/experimental/storage/iceberg/model.py,sha256=PbEnOEY5Ni7E5pJA2xL2d-rwLGOWw6eTARBAYFgCzm8,24288
|
157
|
+
deltacat/experimental/storage/rivulet/__init__.py,sha256=X7bYog7c4NFjoGw16AsCasBRNqKH9N7Z9Pc8aFUT2sA,353
|
158
|
+
deltacat/experimental/storage/rivulet/dataset.py,sha256=5eNkHZEK9f1t4R-IatUFtGD-x7jWiRE5aJKOKP6YA6g,28740
|
159
|
+
deltacat/experimental/storage/rivulet/dataset_executor.py,sha256=8KVkXLdnyDVfbtFFGw4x3o-1EHUQt8j8DZkfpI-0jaQ,2772
|
160
|
+
deltacat/experimental/storage/rivulet/logical_plan.py,sha256=jvF4fyrIWHuLnMX-_tBytiGekPRqW1dwB6pbL3agF8s,3316
|
161
|
+
deltacat/experimental/storage/rivulet/serializer.py,sha256=LeyPOKzSa0m5mI__i7OUaVvBTT9T78NQe1hmlgvij_8,1690
|
162
|
+
deltacat/experimental/storage/rivulet/serializer_factory.py,sha256=XDntdBRtHRK9T3lZ0KNVlb5hLudfBXJhgZyPM-4G6OY,1920
|
163
|
+
deltacat/experimental/storage/rivulet/arrow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
164
|
+
deltacat/experimental/storage/rivulet/arrow/serializer.py,sha256=HlMaNlshxYjCq5wgWVI5ETKtf6vQtl0fpv6Y7X3_hfo,2736
|
165
|
+
deltacat/experimental/storage/rivulet/feather/__init__.py,sha256=rR91gf2z-iux8WNV5equ9FEfenrl_eu3F0PJnxDCnKs,321
|
166
|
+
deltacat/experimental/storage/rivulet/feather/file_reader.py,sha256=SpgJDMw5lHFnQF2X0j1omuh02y1noYQPusgPsKTdlMA,4947
|
167
|
+
deltacat/experimental/storage/rivulet/feather/serializer.py,sha256=Jwcpnlo4fdtzwEdTBnzaOc210asi1wm1JppK8kUX0D8,1596
|
168
|
+
deltacat/experimental/storage/rivulet/fs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
169
|
+
deltacat/experimental/storage/rivulet/fs/file_provider.py,sha256=NPEdUnvQ6WDS3I8rWk5j4OKesIj4CR-an2BDDziwlHA,3863
|
170
|
+
deltacat/experimental/storage/rivulet/fs/file_store.py,sha256=OgAXGkSEKY0IR-Bk38tvbcDQ9g1ozGzQxNoCUDl8Yzk,5658
|
171
|
+
deltacat/experimental/storage/rivulet/fs/input_file.py,sha256=2SZdngGUlURg6ier3N-0D7zEySxgerg6EHuj3v0DXJM,1921
|
172
|
+
deltacat/experimental/storage/rivulet/fs/output_file.py,sha256=DRU36N3UpoPtu73RPkhnBBipUmUG6NYMd8lhRL_qCoY,2480
|
173
|
+
deltacat/experimental/storage/rivulet/metastore/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
174
|
+
deltacat/experimental/storage/rivulet/metastore/delta.py,sha256=EsaLbPvjlhn-UCwVjs-jIbCJF2qkrueeqZ0hAqgwN-A,6316
|
175
|
+
deltacat/experimental/storage/rivulet/metastore/json_sst.py,sha256=Gjw-R2U-v3q4gbjQ1vXqZjJHdE4y5qIvJNx202ivak4,3278
|
176
|
+
deltacat/experimental/storage/rivulet/metastore/sst.py,sha256=7mOn1OE0FNBSJ7C3CiLPqFXnHBb6LtJcAVgadVY7aHA,2432
|
177
|
+
deltacat/experimental/storage/rivulet/metastore/sst_interval_tree.py,sha256=3MQdduJF4TkSFJf9pMTTM8PofnpUpWkdlf0T1dMkzHY,10722
|
178
|
+
deltacat/experimental/storage/rivulet/mvp/Table.py,sha256=j82_-gF7byXx3a0yBCWVCtYH2H93qKPBD_1qp5GrlAA,3016
|
179
|
+
deltacat/experimental/storage/rivulet/mvp/__init__.py,sha256=28ULboqUHUGJuHjDWQgvG7cyR_ylV8fJ3B_wrsvI1f8,234
|
180
|
+
deltacat/experimental/storage/rivulet/parquet/__init__.py,sha256=1zx5IqSRMWsdW03kVmsphY28aWGzC9Tl-oGeoBUASgo,321
|
181
|
+
deltacat/experimental/storage/rivulet/parquet/data_reader.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
182
|
+
deltacat/experimental/storage/rivulet/parquet/file_reader.py,sha256=4Z0KTe14VquvqFSIRQlZ971L2RDqc_XkL8qO0OnjZEQ,4430
|
183
|
+
deltacat/experimental/storage/rivulet/parquet/serializer.py,sha256=SpILisUAg8Y--86aU4ZjQ24n_0kXOvEBHui2XC4R2sM,1575
|
184
|
+
deltacat/experimental/storage/rivulet/reader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
185
|
+
deltacat/experimental/storage/rivulet/reader/block_scanner.py,sha256=qVYCS_Cl7J9c-eQmv0QKxjkuMXL5oJoPAYnyEiFWPaU,15671
|
186
|
+
deltacat/experimental/storage/rivulet/reader/data_reader.py,sha256=kZTv2ErglqtILScvn2i7DDACW_2GcNPmnvuxmO0xdpA,4006
|
187
|
+
deltacat/experimental/storage/rivulet/reader/data_scan.py,sha256=-v4qrHQFHFg57yeweqfmhGuCjo539sQPVyNOvTSMUb4,2773
|
188
|
+
deltacat/experimental/storage/rivulet/reader/dataset_metastore.py,sha256=UupYrzzkv1eccGgLyU_zwnTAVs1J0eYx_GMfOWMeDaM,6249
|
189
|
+
deltacat/experimental/storage/rivulet/reader/dataset_reader.py,sha256=rRc0qB063mHToXZ4idA0lA01zL0xA3hrt-E7o8w699s,6236
|
190
|
+
deltacat/experimental/storage/rivulet/reader/pyarrow_data_reader.py,sha256=xjh0qAdpAuXpXIx7f7jfumofRXjQV0t_Gu61wtQ5u4E,4263
|
191
|
+
deltacat/experimental/storage/rivulet/reader/query_expression.py,sha256=S6EqP2Ru0liWbYws_ZKZz_aTCWJAHEGoZ2MUY5LZQLk,3121
|
192
|
+
deltacat/experimental/storage/rivulet/reader/reader_type_registrar.py,sha256=tIuYUOxOgFEBMNMKIXDqjHSR39IBTHyJpaz2FvwLl6Q,3083
|
193
|
+
deltacat/experimental/storage/rivulet/schema/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
194
|
+
deltacat/experimental/storage/rivulet/schema/datatype.py,sha256=QKlMwaEuALV6viNnNSRN1PNFUg7edzooEO3Jp901j1I,4370
|
195
|
+
deltacat/experimental/storage/rivulet/schema/schema.py,sha256=zjREiUpSXaRRcdCHstmdaAR2ykAo1uDTbrJcDqmHVSQ,9474
|
196
|
+
deltacat/experimental/storage/rivulet/shard/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
197
|
+
deltacat/experimental/storage/rivulet/shard/range_shard.py,sha256=uuulo-I9A2nXq2gY0qIMVC8yKf4xmVNhDheQ_SFBVFQ,4881
|
198
|
+
deltacat/experimental/storage/rivulet/writer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
199
|
+
deltacat/experimental/storage/rivulet/writer/dataset_writer.py,sha256=aLLwhMY9K5DENu_Rtqa3oWUmdPOYrP07BXntBW3pgdo,957
|
200
|
+
deltacat/experimental/storage/rivulet/writer/memtable_dataset_writer.py,sha256=jZJnxyB1PjoKgkoAfQzr-qXNISDcaSURLsbUVu6tImM,10164
|
156
201
|
deltacat/io/__init__.py,sha256=QgARmNDGbOKRV2g5HAG9GKL589zt4WUY3T6E6vWZRWI,382
|
157
202
|
deltacat/io/file_object_store.py,sha256=YoNL3Qla8uLOHaWnyBmIgotjSGAy3Td3Tumah0kk73Y,1868
|
158
203
|
deltacat/io/memcached_object_store.py,sha256=C96t77-4BQe0XZ4vC76Ygi2o1POUoMN4t4BiyPmulz0,10997
|
@@ -167,13 +212,9 @@ deltacat/io/datasink/deltacat_datasink.py,sha256=TcXi_jegA0eawUDq3s6nTm3IcAV3B8n
|
|
167
212
|
deltacat/io/datasource/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
168
213
|
deltacat/io/datasource/deltacat_datasource.py,sha256=G9casuqV8c5nYLelZrEX0tGHDYrzJtH2R1q6-z5zFLk,21741
|
169
214
|
deltacat/io/reader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
170
|
-
deltacat/io/reader/deltacat_read_api.py,sha256=
|
215
|
+
deltacat/io/reader/deltacat_read_api.py,sha256=OSQ8PTWvyyRq5UTSFBlkFgphkJEm0EB5oztr_iszzHc,8394
|
171
216
|
deltacat/storage/__init__.py,sha256=93XqQC1UVsFen-nduWpeQ_pqZu355qHryf7YlilmqMo,3696
|
172
217
|
deltacat/storage/interface.py,sha256=OkQr7WvsA_vPtr5OfG9lP8u4m5F8Y21Qa_CIi4LhrNo,25465
|
173
|
-
deltacat/storage/iceberg/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
174
|
-
deltacat/storage/iceberg/iceberg_scan_planner.py,sha256=TzHTFJidlOXEYCW1czMOOc6oCplDKIs8ooiPlIMXyB0,1069
|
175
|
-
deltacat/storage/iceberg/impl.py,sha256=SbAkGn4wp5UUlZcLtYQgoqDjl9Ak49eSObsp01ZZXlc,26313
|
176
|
-
deltacat/storage/iceberg/model.py,sha256=PbEnOEY5Ni7E5pJA2xL2d-rwLGOWw6eTARBAYFgCzm8,24288
|
177
218
|
deltacat/storage/main/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
178
219
|
deltacat/storage/main/impl.py,sha256=aVeo2t7m0_LumaDfH2CE_tB9-AXSYqlE-PXvprfTuOg,68493
|
179
220
|
deltacat/storage/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
@@ -186,7 +227,7 @@ deltacat/storage/model/metafile.py,sha256=l6XUBMMlHFaHgjttZfq4sEeeSYgH_8eTkOLAvY
|
|
186
227
|
deltacat/storage/model/namespace.py,sha256=gLli1V64O9RHIf-FesmqWA29Wi7P1kwt01uz5sDdJR0,2409
|
187
228
|
deltacat/storage/model/partition.py,sha256=qNCvc74o_4pmFVL-FCyKCZMH4lHSjRO560sb3vaF_H0,20759
|
188
229
|
deltacat/storage/model/schema.py,sha256=uDaOqm1m0DYu89d2vTTMV7hDMvtiVwHqI_HFhRRgAA0,33806
|
189
|
-
deltacat/storage/model/shard.py,sha256=
|
230
|
+
deltacat/storage/model/shard.py,sha256=boPOW45bwLwBazfXZpa3-C5SUSlgelpHf8Yl6357Bq0,1575
|
190
231
|
deltacat/storage/model/sort_key.py,sha256=HAZfDYqV24XClf1KslQw0duFVczvkglp0juENrU41nI,5905
|
191
232
|
deltacat/storage/model/stream.py,sha256=OwzdyCgBOgJ6T5tUASkEE1-Zt8c3psU2ebk8D5OSkTY,12280
|
192
233
|
deltacat/storage/model/table.py,sha256=5dm6ix9d5WEKRn-E-1l7DUEdtHgAOLfjlCRph4mD45Y,7276
|
@@ -201,48 +242,6 @@ deltacat/storage/model/scan/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMp
|
|
201
242
|
deltacat/storage/model/scan/push_down.py,sha256=SldUHgv-HRDpEqIcsD9HI_XuofFI8BI0yRONO0zCcPQ,1034
|
202
243
|
deltacat/storage/model/scan/scan_plan.py,sha256=P59yYbQWXbKZU068GzIh7ya_h2n7NgZJBY_eWVfGLAU,250
|
203
244
|
deltacat/storage/model/scan/scan_task.py,sha256=pOcGKMjnHrZwabtdk3rw5kbh_d_aJGqzgCj6c_im1vE,855
|
204
|
-
deltacat/storage/rivulet/__init__.py,sha256=OQcvRBY6cG-pvBQU8Wu-gncry8UIIS9ke1TY0v8yrAo,301
|
205
|
-
deltacat/storage/rivulet/dataset.py,sha256=WlTCWVS-GImBfEPtHxKsV4WyAJtWxOSBJyjC0_3dxiI,28605
|
206
|
-
deltacat/storage/rivulet/dataset_executor.py,sha256=NXTT8XJIK9ld9EM6bhy0sbxQNsZTdRncJ2uZI5UrRbU,3374
|
207
|
-
deltacat/storage/rivulet/logical_plan.py,sha256=UMW-T7ospMlci2l8suasZTnTUrw75rzTp8lo9lU1oyA,3279
|
208
|
-
deltacat/storage/rivulet/serializer.py,sha256=Xa6rrAy6QNKcFNh_WxkyP8MHrofGdFVRfxX_jp1_FIk,1677
|
209
|
-
deltacat/storage/rivulet/serializer_factory.py,sha256=zUoqew-cddK9o--UeIeDbjqVdSz09t-hzItouogkimo,1837
|
210
|
-
deltacat/storage/rivulet/arrow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
211
|
-
deltacat/storage/rivulet/arrow/serializer.py,sha256=MgODL6vBm8wyLgOH5fQ22OGrKwy-uO3OnJtM7FMomQY,2671
|
212
|
-
deltacat/storage/rivulet/feather/__init__.py,sha256=SsosQE9GeG-jWRy_t3mMBWWS8FLAALdIr-Jj3sXq-04,286
|
213
|
-
deltacat/storage/rivulet/feather/file_reader.py,sha256=riaJwxL0T2YxvPkB5uDtumIKw7eH76Ilx7YQkSCdze8,4873
|
214
|
-
deltacat/storage/rivulet/feather/serializer.py,sha256=0ESomIDsKRvn6OH2k_IG6zoxTlHdHnlK41yDSNWl9ac,1544
|
215
|
-
deltacat/storage/rivulet/fs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
216
|
-
deltacat/storage/rivulet/fs/file_provider.py,sha256=1bFiQcqJjyCMsZjhAicujwBLdo4P_moTY8BOYlbfvdE,3824
|
217
|
-
deltacat/storage/rivulet/fs/file_store.py,sha256=5qWZfWkI__9MNpycgFJVYBhnpbsO-qSbhreYtphqIcA,5632
|
218
|
-
deltacat/storage/rivulet/fs/input_file.py,sha256=2SZdngGUlURg6ier3N-0D7zEySxgerg6EHuj3v0DXJM,1921
|
219
|
-
deltacat/storage/rivulet/fs/output_file.py,sha256=kjJHDhwKxF3TQ81TBQGX2FGyE-Wk7oA8QVGLqTP0nhg,2467
|
220
|
-
deltacat/storage/rivulet/metastore/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
221
|
-
deltacat/storage/rivulet/metastore/delta.py,sha256=HWtDeKft6ExH29TndbrNQhTgUiUkJYmKr6_Tq23CwRs,6303
|
222
|
-
deltacat/storage/rivulet/metastore/json_sst.py,sha256=mckK9bzu1Tt1kDhZhzFFEbOlDMUqHTl9TeHd3r0AwaA,3239
|
223
|
-
deltacat/storage/rivulet/metastore/sst.py,sha256=1uHUHmh6q9hcGXhvrnQFQVLfKBKaBIay1gL5kfbd9nI,2406
|
224
|
-
deltacat/storage/rivulet/metastore/sst_interval_tree.py,sha256=j6BwNkpY18WYG3b4s0v_vt4fBKbrTGj5q_QsLPGU-Cg,10683
|
225
|
-
deltacat/storage/rivulet/mvp/Table.py,sha256=j82_-gF7byXx3a0yBCWVCtYH2H93qKPBD_1qp5GrlAA,3016
|
226
|
-
deltacat/storage/rivulet/mvp/__init__.py,sha256=28ULboqUHUGJuHjDWQgvG7cyR_ylV8fJ3B_wrsvI1f8,234
|
227
|
-
deltacat/storage/rivulet/parquet/__init__.py,sha256=n896g5SyboT2t_foaFnPXOdTgqZ9JarsFFzVrtnGeT0,286
|
228
|
-
deltacat/storage/rivulet/parquet/data_reader.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
229
|
-
deltacat/storage/rivulet/parquet/file_reader.py,sha256=fzDg2VN50GqljFQFFM_bytbOl2w7BP-FrlUThZIWrzQ,4356
|
230
|
-
deltacat/storage/rivulet/parquet/serializer.py,sha256=p9g2oskJTOgw7afiVdr9Vb-PTgzRV4R_9atQHlhnj58,1523
|
231
|
-
deltacat/storage/rivulet/reader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
232
|
-
deltacat/storage/rivulet/reader/block_scanner.py,sha256=QvaAAop6YunxrYsvTlFrAmgW_ZjmfPcj7ZpEAJ6cl5w,15505
|
233
|
-
deltacat/storage/rivulet/reader/data_reader.py,sha256=GojlClE3W7WwQnvohu3gJK0TEP05_ToGk_wLKtl3hi8,3967
|
234
|
-
deltacat/storage/rivulet/reader/data_scan.py,sha256=ada1INAyql42Q8Dw3l9tq1ZIlD2hllU2ivVc0sClI98,2725
|
235
|
-
deltacat/storage/rivulet/reader/dataset_metastore.py,sha256=E1BCBcBihPF9HLCy9vWaY54BRMTuHKz_yNlaXprxy10,6197
|
236
|
-
deltacat/storage/rivulet/reader/dataset_reader.py,sha256=wjdioXZA3IoLfXKPjjsnkUE1kKGmvy7vEegJNUKSl-4,6149
|
237
|
-
deltacat/storage/rivulet/reader/pyarrow_data_reader.py,sha256=QdtrAJ4d42lVxl7ZGE_lBDb_H6keZlcI2DkNkWcyCrU,4237
|
238
|
-
deltacat/storage/rivulet/reader/query_expression.py,sha256=S6EqP2Ru0liWbYws_ZKZz_aTCWJAHEGoZ2MUY5LZQLk,3121
|
239
|
-
deltacat/storage/rivulet/reader/reader_type_registrar.py,sha256=NFSSBHUopQDQtnadSm0W4HXGhEvPH5sKAXdZG7yNx34,3031
|
240
|
-
deltacat/storage/rivulet/schema/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
241
|
-
deltacat/storage/rivulet/schema/datatype.py,sha256=QKlMwaEuALV6viNnNSRN1PNFUg7edzooEO3Jp901j1I,4370
|
242
|
-
deltacat/storage/rivulet/schema/schema.py,sha256=t-y4BHSQkZ2bI8WxQfMsXoSYUe4sOLOA75u-BGKXaZM,9461
|
243
|
-
deltacat/storage/rivulet/writer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
244
|
-
deltacat/storage/rivulet/writer/dataset_writer.py,sha256=aLLwhMY9K5DENu_Rtqa3oWUmdPOYrP07BXntBW3pgdo,957
|
245
|
-
deltacat/storage/rivulet/writer/memtable_dataset_writer.py,sha256=YxB6q-FjD0h3VUbLabjRTiE_MVzj1JCDPM8HsOzgGU8,10012
|
246
245
|
deltacat/storage/util/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
247
246
|
deltacat/storage/util/scan_planner.py,sha256=yxvExB_rbTc5ijLdnSjJv0BAFGsNRqegzzDBUYzXZ-k,820
|
248
247
|
deltacat/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
@@ -262,8 +261,14 @@ deltacat/tests/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuF
|
|
262
261
|
deltacat/tests/aws/test_clients.py,sha256=23GMWfz27WWBDXSqphG9mfputsyS7j3I5P_HRk4YoKE,3790
|
263
262
|
deltacat/tests/aws/test_s3u.py,sha256=FsYCH8K8DsDRPOtTp-w1Nu3ATqt4p1mqDo6aVJV-SbU,7918
|
264
263
|
deltacat/tests/catalog/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
265
|
-
deltacat/tests/catalog/test_catalogs.py,sha256=
|
266
|
-
deltacat/tests/catalog/test_default_catalog_impl.py,sha256=
|
264
|
+
deltacat/tests/catalog/test_catalogs.py,sha256=hxmjV7tdtcqFdDnY-bqlIruxEt6x1bjJAVmou763jMM,10183
|
265
|
+
deltacat/tests/catalog/test_default_catalog_impl.py,sha256=meqLi9G1ToVQ8ZztgUSh-d1WhFaAlT3mXlugV7ioTDE,3289
|
266
|
+
deltacat/tests/catalog/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
267
|
+
deltacat/tests/catalog/main/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
268
|
+
deltacat/tests/catalog/main/test_catalog_impl_namespace_operations.py,sha256=XvMoW1yjjoIs4-0A8_dqeY7ArysN6HJkSSHk7JnHeUI,4313
|
269
|
+
deltacat/tests/catalog/main/test_catalog_impl_table_operations.py,sha256=74_c77MMQu4W1s2ildOOs2HO4PDwXUDergkPg6eSus0,15352
|
270
|
+
deltacat/tests/catalog/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
271
|
+
deltacat/tests/catalog/model/test_table_definition.py,sha256=R-_FGV-52o5fShItXrbcqt2-2ECjrDQmcOHWKBCU-HQ,574
|
267
272
|
deltacat/tests/compute/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
268
273
|
deltacat/tests/compute/compact_partition_multiple_rounds_test_cases.py,sha256=g4DKJEn7BtLtuDIxYVq9ZkK1S91o4ftRB7-3jhDCJ4s,37694
|
269
274
|
deltacat/tests/compute/compact_partition_rebase_test_cases.py,sha256=IwA0w5gURCFlUCFxXmnTSIPh5mkiGKRMw1UB9yU9ESQ,16821
|
@@ -297,6 +302,35 @@ deltacat/tests/compute/resource_estimation/__init__.py,sha256=47DEQpj8HBSa-_TImW
|
|
297
302
|
deltacat/tests/compute/resource_estimation/test_delta.py,sha256=5GyaYObGa-PjTS01d3uRyTLoVAvgoanMQn0EOFhNFjI,25159
|
298
303
|
deltacat/tests/compute/resource_estimation/test_manifest.py,sha256=yrMvqDjolExdRf6Vtg5XaKDuaKz9ok15PCZ7_aJOYrI,32893
|
299
304
|
deltacat/tests/compute/resource_estimation/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
305
|
+
deltacat/tests/daft/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
306
|
+
deltacat/tests/daft/test_model.py,sha256=nqKZV1lUqvFzfyjrV1OxtJi3Fj33fcYIl9I0eguxAnQ,3953
|
307
|
+
deltacat/tests/experimental/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
308
|
+
deltacat/tests/experimental/catalog/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
309
|
+
deltacat/tests/experimental/catalog/iceberg/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
310
|
+
deltacat/tests/experimental/catalog/iceberg/test_iceberg_catalog.py,sha256=K3YwWX8g_Tf7SQq-ruKj3_fAdPYvHdFYDB57byGhOGc,2029
|
311
|
+
deltacat/tests/experimental/daft/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
312
|
+
deltacat/tests/experimental/daft/test_deltacat_daft_integration.py,sha256=Vx-5MSPuvaUMHUBQ4cGhob26JWSxNqiHUYVMwvqfS2c,5102
|
313
|
+
deltacat/tests/experimental/storage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
314
|
+
deltacat/tests/experimental/storage/rivulet/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
315
|
+
deltacat/tests/experimental/storage/rivulet/conftest.py,sha256=rtaU6iaCBlCbUiIc8UcVo5m4ADv8jN9N0FVi-36pS9o,3698
|
316
|
+
deltacat/tests/experimental/storage/rivulet/test_dataset.py,sha256=yD0c5r8a5IogAA_rvv3vM_waXQylA3oyUOHmvM_vcBI,13577
|
317
|
+
deltacat/tests/experimental/storage/rivulet/test_manifest.py,sha256=RS-cZQb7lkwpFFHNnU5varlVp_3njrjBw2B-84tEOPA,1943
|
318
|
+
deltacat/tests/experimental/storage/rivulet/test_sst_interval_tree.py,sha256=eBOFI4Z9GXxRwVTbnfhum-WNTTULyOV53gcvKQ8rfos,6866
|
319
|
+
deltacat/tests/experimental/storage/rivulet/test_utils.py,sha256=d9igdUG7BSoYVZN0-skxmKRPJ4HmDJE-P6wjh2qqk4Q,4469
|
320
|
+
deltacat/tests/experimental/storage/rivulet/fs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
321
|
+
deltacat/tests/experimental/storage/rivulet/fs/test_file_location_provider.py,sha256=TlIEWRZjmMSaCn5NoCDsis1lc4lNxD74tXUzm8rERO0,2842
|
322
|
+
deltacat/tests/experimental/storage/rivulet/reader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
323
|
+
deltacat/tests/experimental/storage/rivulet/reader/query_expression.py,sha256=6z3x5jEX3L0cUtvd-bx4HYWcMrxWf20PQ0USSxq6nTM,2312
|
324
|
+
deltacat/tests/experimental/storage/rivulet/reader/test_data_scan.py,sha256=NMEkkhJvT8Kkc7xXrb5lpN1nup964Qd3vwuo5FbvAE8,3105
|
325
|
+
deltacat/tests/experimental/storage/rivulet/reader/test_dataset_metastore.py,sha256=-OLXFgNHlqVw7ezm51f-x6YMZoBciXoR22BKw5LRH6E,2311
|
326
|
+
deltacat/tests/experimental/storage/rivulet/schema/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
327
|
+
deltacat/tests/experimental/storage/rivulet/schema/test_schema.py,sha256=aefVhuRnpp60ddw3wiDVYcgl2WJFWhb1AFDHgBzvD0Y,7254
|
328
|
+
deltacat/tests/experimental/storage/rivulet/shard/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
329
|
+
deltacat/tests/experimental/storage/rivulet/shard/test_range_shard.py,sha256=aUKD1ASNqUP0a6BOnnU-uVwkGsxtv6PXVGHyW6BeFxQ,4687
|
330
|
+
deltacat/tests/experimental/storage/rivulet/writer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
331
|
+
deltacat/tests/experimental/storage/rivulet/writer/test_dataset_write_then_read.py,sha256=-7bA8eS_vzDfwMJZjfnhg7ogp7jnMSZNEuxqthI9g-k,12299
|
332
|
+
deltacat/tests/experimental/storage/rivulet/writer/test_dataset_writer.py,sha256=1bqq9_yGSJD8sXEVDrzNsSA9nsMJ04k7PbxGpUZQ0a4,2884
|
333
|
+
deltacat/tests/experimental/storage/rivulet/writer/test_memtable_dataset_writer.py,sha256=www9mD_vugITO42ojHX8HmNiZxvveB0qYg-1MXwl2AA,2579
|
300
334
|
deltacat/tests/local_deltacat_storage/__init__.py,sha256=kM-g8zLwXmGnCw6fUzc99v_id5lnalsg81u9D8nkvdQ,38771
|
301
335
|
deltacat/tests/local_deltacat_storage/exceptions.py,sha256=oxZ0psmrEO0M6P2r8gHQ2E8E-Y8UBfUCBUIwfuHcx38,251
|
302
336
|
deltacat/tests/storage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
@@ -309,23 +343,9 @@ deltacat/tests/storage/model/test_expression.py,sha256=eySWacUo8EK1NwxMQnNqpXAVb
|
|
309
343
|
deltacat/tests/storage/model/test_manifest.py,sha256=udp9YUNvIBpnT-NutjMaF25abEQOXEcPkQm8Aay_UCs,3733
|
310
344
|
deltacat/tests/storage/model/test_metafile_io.py,sha256=116U9aNJPzR0JS6iadJyyx0_4KyAi3D47WCNbndag6o,101639
|
311
345
|
deltacat/tests/storage/model/test_schema.py,sha256=5m4BscbxbbOiry-lDI8j4vQcnvkG2Y-f0ZfshncPiSI,9599
|
312
|
-
deltacat/tests/storage/model/test_shard.py,sha256=
|
346
|
+
deltacat/tests/storage/model/test_shard.py,sha256=gP8w1KMq73M4uMyQKsV6w1FFTSx0vdJ_ybA-Erqk2UY,785
|
313
347
|
deltacat/tests/storage/model/test_table_version.py,sha256=CtLiOe1EI6Ao9MkxyMWoxTWuqQevupVUEQVU5ZIEB34,3306
|
314
348
|
deltacat/tests/storage/model/test_transaction.py,sha256=Y5JMaYz6mf4DbruxMmz9hWXGX30MjKjRAvxlIf0MnaY,14458
|
315
|
-
deltacat/tests/storage/rivulet/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
316
|
-
deltacat/tests/storage/rivulet/conftest.py,sha256=1Ansaxs4WGVKhZ2iSGrFlz1Mjib6BLmZG98eSi9_EiU,3659
|
317
|
-
deltacat/tests/storage/rivulet/test_dataset.py,sha256=4V6FFxz66TNwl6Vviw-zfu_K2mBTRDT-zcaarcPVCn4,13529
|
318
|
-
deltacat/tests/storage/rivulet/test_manifest.py,sha256=mx3jlE0opEKla4i12V2THoM0f8-aWdPEJyJUrzi3lzg,1878
|
319
|
-
deltacat/tests/storage/rivulet/test_sst_interval_tree.py,sha256=VK8lh5ZtnIsnuyphsnXdnD9OQhAlyOLKJ1M0l4ui6i8,6801
|
320
|
-
deltacat/tests/storage/rivulet/test_utils.py,sha256=NvW_zA5CEEkooWDlc0aBCb_0pceCRb8WFY-HQ78mheM,4395
|
321
|
-
deltacat/tests/storage/rivulet/fs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
322
|
-
deltacat/tests/storage/rivulet/fs/test_file_location_provider.py,sha256=vH3QPEnEFHXKQrnHPbJjVKRK3H6mOIWaNyqlHfCZqEE,2803
|
323
|
-
deltacat/tests/storage/rivulet/schema/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
324
|
-
deltacat/tests/storage/rivulet/schema/test_schema.py,sha256=LngLuRlVK_1fK-JA5rW2_7RBPmeci7R_FdbsWytUVEo,7241
|
325
|
-
deltacat/tests/storage/rivulet/writer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
326
|
-
deltacat/tests/storage/rivulet/writer/test_dataset_write_then_read.py,sha256=IZTzlWLoFvedCN60AOo9tCfOqkNTgGDckOELn3B9Uoo,12173
|
327
|
-
deltacat/tests/storage/rivulet/writer/test_dataset_writer.py,sha256=HW-E7RvuyxloCL4_z7EsAHOYvw0UPzupXrvv3L3s1vQ,2858
|
328
|
-
deltacat/tests/storage/rivulet/writer/test_memtable_dataset_writer.py,sha256=lefIWItakJIYOfL3O0jvA2bpcY7t5C1u_TF-PygkU28,2488
|
329
349
|
deltacat/tests/test_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
330
350
|
deltacat/tests/test_utils/constants.py,sha256=UYe--9T_clYjiOpv0M7TtAMGdpje_SMZ-w8n0IeCAjc,214
|
331
351
|
deltacat/tests/test_utils/filesystem.py,sha256=H4LhAZ4q1yC4mrAv5EQQ_NaBeYtuFXKxxAyEvirTt4w,221
|
@@ -346,15 +366,15 @@ deltacat/tests/utils/ray_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5
|
|
346
366
|
deltacat/tests/utils/ray_utils/test_concurrency.py,sha256=TjZpX0cjMDEIS79p_--j_BfT0zXKNkTLY1ZzNokBTs0,1211
|
347
367
|
deltacat/tests/utils/ray_utils/test_dataset.py,sha256=1hoOR_AIO4iJQ_lCjNsJfq7S-2ZyOKyMkKc4Tjt6cwg,2092
|
348
368
|
deltacat/types/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
349
|
-
deltacat/types/media.py,sha256=
|
369
|
+
deltacat/types/media.py,sha256=kwp7DtZaa7jfB_hXPHXdXcUQy_nwrakeIPRF7I8rdHs,5487
|
350
370
|
deltacat/types/partial_download.py,sha256=QIpNTSwaiZ4TVl4A1N4PtblevKT5GwdXtGrouQMQs1E,2510
|
351
371
|
deltacat/types/tables.py,sha256=zz29727VTgu4InLPwjOjmDX0SrCiDq-VI1S75EZfIr8,5594
|
352
372
|
deltacat/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
353
373
|
deltacat/utils/arguments.py,sha256=5y1Xz4HSAD8M8Jt83i6gOEKoYjy_fMQe1V43IhIE4hY,1191
|
354
374
|
deltacat/utils/cloudpickle.py,sha256=XE7YDmQe56ksfl3NdYZkzOAhbHSuhNcBZGOehQpgZr0,1187
|
355
375
|
deltacat/utils/common.py,sha256=RG_-enXNpLKaYrqyx1ne2lL10lxN9vK7F631oJP6SE8,1375
|
356
|
-
deltacat/utils/daft.py,sha256=
|
357
|
-
deltacat/utils/export.py,sha256=
|
376
|
+
deltacat/utils/daft.py,sha256=POX96a67wiOSonaAgA9lVQRZ0buNua0xhFx43e-lLT8,24032
|
377
|
+
deltacat/utils/export.py,sha256=aiebPfWxtc-o7gCqDilcyMxb0GdAByW47dygtQ2Wrzg,1974
|
358
378
|
deltacat/utils/filesystem.py,sha256=7cmEmzc2JPPqbW-zlXv_d28R-IUqX6bmH3DWJ4NHB8A,13213
|
359
379
|
deltacat/utils/metafile_locator.py,sha256=_3yEW9n49jiEBuXHZmUKsFdYx6RxWWuS-Mu2gs_a1bw,2933
|
360
380
|
deltacat/utils/metrics.py,sha256=HYKyZSrtVLu8gXezg_TMNUKJp4h1WWI0VEzn0Xlzf-I,10778
|
@@ -367,15 +387,15 @@ deltacat/utils/pyarrow.py,sha256=gOSCs9XVeLy2PAERXM7DCg--ywuoxHO9dtl3LeIcJQU,342
|
|
367
387
|
deltacat/utils/resources.py,sha256=Ax1OgLLbZI4oYpp4Ki27OLaST-7I-AJgZwU87FVfY8g,8253
|
368
388
|
deltacat/utils/s3fs.py,sha256=PmUJ5Fm1WmD-_zp_M6yd9VbXvIoJuBeK6ApOdJJApLE,662
|
369
389
|
deltacat/utils/schema.py,sha256=m4Wm4ZQcpttzOUxex4dVneGlHy1_E36HspTcjNYzvVM,1564
|
370
|
-
deltacat/utils/url.py,sha256=
|
390
|
+
deltacat/utils/url.py,sha256=oKBfOKwojtF-G8Ry_shsAJjqWdPsKn1m5IXBbyb4ffc,43822
|
371
391
|
deltacat/utils/ray_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
372
392
|
deltacat/utils/ray_utils/collections.py,sha256=hj20s4D2RF2jZETU_44r6mFbsczA0JI_I_4kWKTmqes,1951
|
373
393
|
deltacat/utils/ray_utils/concurrency.py,sha256=Ceui6nQYKHTUOTltHNQIdb0OWHFhD73o8DhSXP-DYRQ,5457
|
374
394
|
deltacat/utils/ray_utils/dataset.py,sha256=waHdtH0c835a-2t51HYRHnulfC0_zBxx8mFSAPvPSPM,3274
|
375
395
|
deltacat/utils/ray_utils/performance.py,sha256=d7JFM7vTXHzkGx9qNQcZzUWajnqINvYRwaM088_FpsE,464
|
376
396
|
deltacat/utils/ray_utils/runtime.py,sha256=cf5koY9q4TzRg--BjPtC6y0jztq45F39KcC4K6Wmg4w,6946
|
377
|
-
deltacat-2.0.
|
378
|
-
deltacat-2.0.
|
379
|
-
deltacat-2.0.
|
380
|
-
deltacat-2.0.
|
381
|
-
deltacat-2.0.
|
397
|
+
deltacat-2.0.0b11.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
398
|
+
deltacat-2.0.0b11.dist-info/METADATA,sha256=oJQ9Ap_-LkIWKeCArIXq7tf8-kIcD2SdfuanP1XPgCI,2868
|
399
|
+
deltacat-2.0.0b11.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
|
400
|
+
deltacat-2.0.0b11.dist-info/top_level.txt,sha256=RWdIcid4Bv2i2ozLVh-70kJpyB61xEKXod9XXGpiono,9
|
401
|
+
deltacat-2.0.0b11.dist-info/RECORD,,
|
deltacat/daft/daft_scan.py
DELETED
@@ -1,115 +0,0 @@
|
|
1
|
-
from typing import Iterator
|
2
|
-
|
3
|
-
from daft import Schema
|
4
|
-
from daft.daft import (
|
5
|
-
StorageConfig,
|
6
|
-
PartitionField,
|
7
|
-
Pushdowns,
|
8
|
-
ScanTask,
|
9
|
-
FileFormatConfig,
|
10
|
-
ParquetSourceConfig,
|
11
|
-
)
|
12
|
-
from daft.io.scan import ScanOperator, ScanPushdowns
|
13
|
-
|
14
|
-
from deltacat.catalog.model.table_definition import TableDefinition
|
15
|
-
from deltacat.daft.model import DaftPartitionKeyMapper
|
16
|
-
from deltacat.daft.translator import translate_pushdown
|
17
|
-
|
18
|
-
|
19
|
-
class DeltaCatScanOperator(ScanOperator):
|
20
|
-
def __init__(self, table: TableDefinition, storage_config: StorageConfig) -> None:
|
21
|
-
super().__init__()
|
22
|
-
self.table = table
|
23
|
-
self._schema = self._infer_schema()
|
24
|
-
self.partition_keys = self._infer_partition_keys()
|
25
|
-
self.storage_config = storage_config
|
26
|
-
|
27
|
-
def schema(self) -> Schema:
|
28
|
-
return self._schema
|
29
|
-
|
30
|
-
def name(self) -> str:
|
31
|
-
return "DeltaCatScanOperator"
|
32
|
-
|
33
|
-
def display_name(self) -> str:
|
34
|
-
return f"DeltaCATScanOperator({self.table.table.namespace}.{self.table.table.table_name})"
|
35
|
-
|
36
|
-
def partitioning_keys(self) -> list[PartitionField]:
|
37
|
-
return self.partition_keys
|
38
|
-
|
39
|
-
def multiline_display(self) -> list[str]:
|
40
|
-
return [
|
41
|
-
self.display_name(),
|
42
|
-
f"Schema = {self._schema}",
|
43
|
-
f"Partitioning keys = {self.partitioning_keys}",
|
44
|
-
f"Storage config = {self.storage_config}",
|
45
|
-
]
|
46
|
-
|
47
|
-
def to_scan_tasks(self, pushdowns: Pushdowns) -> Iterator[ScanTask]:
|
48
|
-
daft_pushdowns = ScanPushdowns._from_pypushdowns(
|
49
|
-
pushdowns, schema=self.schema()
|
50
|
-
)
|
51
|
-
dc_pushdown = translate_pushdown(daft_pushdowns)
|
52
|
-
dc_scan_plan = self.table.create_scan_plan(pushdown=dc_pushdown)
|
53
|
-
scan_tasks = []
|
54
|
-
file_format_config = FileFormatConfig.from_parquet_config(
|
55
|
-
# maybe this: ParquetSourceConfig(field_id_mapping=self._field_id_mapping)
|
56
|
-
ParquetSourceConfig()
|
57
|
-
)
|
58
|
-
for dc_scan_task in dc_scan_plan.scan_tasks:
|
59
|
-
for data_file in dc_scan_task.data_files():
|
60
|
-
st = ScanTask.catalog_scan_task(
|
61
|
-
file=data_file.file_path,
|
62
|
-
file_format=file_format_config,
|
63
|
-
schema=self._schema._schema,
|
64
|
-
storage_config=self.storage_config,
|
65
|
-
pushdowns=pushdowns,
|
66
|
-
)
|
67
|
-
scan_tasks.append(st)
|
68
|
-
return iter(scan_tasks)
|
69
|
-
|
70
|
-
def can_absorb_filter(self) -> bool:
|
71
|
-
return False
|
72
|
-
|
73
|
-
def can_absorb_limit(self) -> bool:
|
74
|
-
return False
|
75
|
-
|
76
|
-
def can_absorb_select(self) -> bool:
|
77
|
-
return True
|
78
|
-
|
79
|
-
def _infer_schema(self) -> Schema:
|
80
|
-
|
81
|
-
if not (
|
82
|
-
self.table and self.table.table_version and self.table.table_version.schema
|
83
|
-
):
|
84
|
-
raise RuntimeError(
|
85
|
-
f"Failed to infer schema for DeltaCAT Table "
|
86
|
-
f"{self.table.table.namespace}.{self.table.table.table_name}"
|
87
|
-
)
|
88
|
-
|
89
|
-
return Schema.from_pyarrow_schema(self.table.table_version.schema.arrow)
|
90
|
-
|
91
|
-
def _infer_partition_keys(self) -> list[PartitionField]:
|
92
|
-
if not (
|
93
|
-
self.table
|
94
|
-
and self.table.table_version
|
95
|
-
and self.table.table_version.partition_scheme
|
96
|
-
and self.table.table_version.schema
|
97
|
-
):
|
98
|
-
raise RuntimeError(
|
99
|
-
f"Failed to infer partition keys for DeltaCAT Table "
|
100
|
-
f"{self.table.table.namespace}.{self.table.table.table_name}"
|
101
|
-
)
|
102
|
-
|
103
|
-
schema = self.table.table_version.schema
|
104
|
-
partition_keys = self.table.table_version.partition_scheme.keys
|
105
|
-
if not partition_keys:
|
106
|
-
return []
|
107
|
-
|
108
|
-
partition_fields = []
|
109
|
-
for key in partition_keys:
|
110
|
-
field = DaftPartitionKeyMapper.unmap(key, schema)
|
111
|
-
# Assert that the returned value is not None.
|
112
|
-
assert field is not None, f"Unmapping failed for key {key}"
|
113
|
-
partition_fields.append(field)
|
114
|
-
|
115
|
-
return partition_fields
|