deltacat 2.0.0b10__py3-none-any.whl → 2.0.0b11__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. deltacat/__init__.py +19 -15
  2. deltacat/benchmarking/benchmark_engine.py +4 -2
  3. deltacat/benchmarking/test_benchmark_pipeline.py +6 -4
  4. deltacat/catalog/__init__.py +62 -5
  5. deltacat/catalog/main/impl.py +18 -8
  6. deltacat/catalog/model/catalog.py +111 -73
  7. deltacat/catalog/model/properties.py +25 -22
  8. deltacat/compute/jobs/client.py +7 -5
  9. deltacat/constants.py +1 -2
  10. deltacat/env.py +10 -0
  11. deltacat/examples/basic_logging.py +1 -3
  12. deltacat/examples/{iceberg → experimental/iceberg}/iceberg_bucket_writer.py +3 -5
  13. deltacat/examples/{iceberg → experimental/iceberg}/iceberg_reader.py +2 -4
  14. deltacat/examples/indexer/indexer.py +2 -2
  15. deltacat/examples/indexer/job_runner.py +1 -2
  16. deltacat/experimental/catalog/iceberg/__init__.py +6 -0
  17. deltacat/{catalog → experimental/catalog}/iceberg/iceberg_catalog_config.py +1 -1
  18. deltacat/{catalog → experimental/catalog}/iceberg/impl.py +27 -9
  19. deltacat/{storage → experimental/storage}/iceberg/iceberg_scan_planner.py +1 -1
  20. deltacat/{storage → experimental/storage}/iceberg/impl.py +1 -1
  21. deltacat/experimental/storage/rivulet/__init__.py +11 -0
  22. deltacat/{storage → experimental/storage}/rivulet/arrow/serializer.py +7 -4
  23. deltacat/{storage → experimental/storage}/rivulet/dataset.py +13 -9
  24. deltacat/{storage → experimental/storage}/rivulet/dataset_executor.py +12 -20
  25. deltacat/experimental/storage/rivulet/feather/__init__.py +7 -0
  26. deltacat/{storage → experimental/storage}/rivulet/feather/file_reader.py +7 -5
  27. deltacat/{storage → experimental/storage}/rivulet/feather/serializer.py +4 -4
  28. deltacat/{storage → experimental/storage}/rivulet/fs/file_provider.py +3 -3
  29. deltacat/{storage → experimental/storage}/rivulet/fs/file_store.py +2 -2
  30. deltacat/{storage → experimental/storage}/rivulet/fs/output_file.py +1 -1
  31. deltacat/{storage → experimental/storage}/rivulet/logical_plan.py +4 -4
  32. deltacat/{storage → experimental/storage}/rivulet/metastore/delta.py +1 -1
  33. deltacat/{storage → experimental/storage}/rivulet/metastore/json_sst.py +3 -3
  34. deltacat/{storage → experimental/storage}/rivulet/metastore/sst.py +2 -2
  35. deltacat/{storage → experimental/storage}/rivulet/metastore/sst_interval_tree.py +3 -3
  36. deltacat/experimental/storage/rivulet/parquet/__init__.py +7 -0
  37. deltacat/{storage → experimental/storage}/rivulet/parquet/file_reader.py +7 -5
  38. deltacat/{storage → experimental/storage}/rivulet/parquet/serializer.py +4 -4
  39. deltacat/{storage → experimental/storage}/rivulet/reader/block_scanner.py +20 -9
  40. deltacat/{storage → experimental/storage}/rivulet/reader/data_reader.py +3 -3
  41. deltacat/{storage → experimental/storage}/rivulet/reader/data_scan.py +5 -3
  42. deltacat/{storage → experimental/storage}/rivulet/reader/dataset_metastore.py +4 -4
  43. deltacat/{storage → experimental/storage}/rivulet/reader/dataset_reader.py +8 -6
  44. deltacat/{storage → experimental/storage}/rivulet/reader/pyarrow_data_reader.py +4 -1
  45. deltacat/{storage → experimental/storage}/rivulet/reader/reader_type_registrar.py +4 -4
  46. deltacat/{storage → experimental/storage}/rivulet/schema/schema.py +1 -1
  47. deltacat/{storage → experimental/storage}/rivulet/serializer.py +1 -1
  48. deltacat/{storage → experimental/storage}/rivulet/serializer_factory.py +9 -5
  49. deltacat/experimental/storage/rivulet/shard/range_shard.py +129 -0
  50. deltacat/{storage → experimental/storage}/rivulet/writer/memtable_dataset_writer.py +20 -9
  51. deltacat/io/reader/deltacat_read_api.py +1 -1
  52. deltacat/storage/model/shard.py +6 -2
  53. deltacat/tests/catalog/main/test_catalog_impl_namespace_operations.py +130 -0
  54. deltacat/tests/catalog/main/test_catalog_impl_table_operations.py +436 -0
  55. deltacat/tests/catalog/model/__init__.py +0 -0
  56. deltacat/tests/catalog/model/test_table_definition.py +16 -0
  57. deltacat/tests/catalog/test_catalogs.py +52 -98
  58. deltacat/tests/catalog/test_default_catalog_impl.py +1 -2
  59. deltacat/tests/daft/__init__.py +0 -0
  60. deltacat/tests/daft/test_model.py +97 -0
  61. deltacat/tests/experimental/__init__.py +0 -0
  62. deltacat/tests/experimental/catalog/__init__.py +0 -0
  63. deltacat/tests/experimental/catalog/iceberg/__init__.py +0 -0
  64. deltacat/tests/experimental/catalog/iceberg/test_iceberg_catalog.py +71 -0
  65. deltacat/tests/experimental/daft/__init__.py +0 -0
  66. deltacat/tests/experimental/daft/test_deltacat_daft_integration.py +136 -0
  67. deltacat/tests/experimental/storage/__init__.py +0 -0
  68. deltacat/tests/experimental/storage/rivulet/__init__.py +0 -0
  69. deltacat/tests/{storage → experimental/storage}/rivulet/conftest.py +3 -3
  70. deltacat/tests/experimental/storage/rivulet/fs/__init__.py +0 -0
  71. deltacat/tests/{storage → experimental/storage}/rivulet/fs/test_file_location_provider.py +3 -3
  72. deltacat/tests/experimental/storage/rivulet/reader/__init__.py +0 -0
  73. deltacat/tests/experimental/storage/rivulet/reader/query_expression.py +80 -0
  74. deltacat/tests/experimental/storage/rivulet/reader/test_data_scan.py +119 -0
  75. deltacat/tests/experimental/storage/rivulet/reader/test_dataset_metastore.py +71 -0
  76. deltacat/tests/experimental/storage/rivulet/schema/__init__.py +0 -0
  77. deltacat/tests/{storage → experimental/storage}/rivulet/schema/test_schema.py +1 -1
  78. deltacat/tests/experimental/storage/rivulet/shard/__init__.py +0 -0
  79. deltacat/tests/experimental/storage/rivulet/shard/test_range_shard.py +162 -0
  80. deltacat/tests/{storage → experimental/storage}/rivulet/test_dataset.py +5 -3
  81. deltacat/tests/{storage → experimental/storage}/rivulet/test_manifest.py +5 -5
  82. deltacat/tests/{storage → experimental/storage}/rivulet/test_sst_interval_tree.py +5 -5
  83. deltacat/tests/{storage → experimental/storage}/rivulet/test_utils.py +8 -6
  84. deltacat/tests/experimental/storage/rivulet/writer/__init__.py +0 -0
  85. deltacat/tests/{storage → experimental/storage}/rivulet/writer/test_dataset_write_then_read.py +11 -9
  86. deltacat/tests/{storage → experimental/storage}/rivulet/writer/test_dataset_writer.py +2 -2
  87. deltacat/tests/{storage → experimental/storage}/rivulet/writer/test_memtable_dataset_writer.py +7 -7
  88. deltacat/tests/storage/model/test_shard.py +3 -1
  89. deltacat/types/media.py +3 -3
  90. deltacat/utils/daft.py +530 -4
  91. deltacat/utils/export.py +3 -1
  92. deltacat/utils/url.py +1 -1
  93. {deltacat-2.0.0b10.dist-info → deltacat-2.0.0b11.dist-info}/METADATA +4 -5
  94. {deltacat-2.0.0b10.dist-info → deltacat-2.0.0b11.dist-info}/RECORD +120 -100
  95. deltacat/catalog/iceberg/__init__.py +0 -4
  96. deltacat/daft/daft_scan.py +0 -115
  97. deltacat/daft/model.py +0 -258
  98. deltacat/daft/translator.py +0 -126
  99. deltacat/examples/common/fixtures.py +0 -15
  100. deltacat/storage/rivulet/__init__.py +0 -11
  101. deltacat/storage/rivulet/feather/__init__.py +0 -5
  102. deltacat/storage/rivulet/parquet/__init__.py +0 -5
  103. /deltacat/{daft → examples/experimental}/__init__.py +0 -0
  104. /deltacat/examples/{common → experimental/iceberg}/__init__.py +0 -0
  105. /deltacat/{examples/iceberg → experimental/catalog}/__init__.py +0 -0
  106. /deltacat/{catalog → experimental/catalog}/iceberg/overrides.py +0 -0
  107. /deltacat/{storage/iceberg → experimental/storage}/__init__.py +0 -0
  108. /deltacat/{storage/rivulet/arrow → experimental/storage/iceberg}/__init__.py +0 -0
  109. /deltacat/{storage → experimental/storage}/iceberg/model.py +0 -0
  110. /deltacat/{storage/rivulet/fs → experimental/storage/rivulet/arrow}/__init__.py +0 -0
  111. /deltacat/{storage/rivulet/metastore → experimental/storage/rivulet/fs}/__init__.py +0 -0
  112. /deltacat/{storage → experimental/storage}/rivulet/fs/input_file.py +0 -0
  113. /deltacat/{storage/rivulet/reader → experimental/storage/rivulet/metastore}/__init__.py +0 -0
  114. /deltacat/{storage → experimental/storage}/rivulet/mvp/Table.py +0 -0
  115. /deltacat/{storage → experimental/storage}/rivulet/mvp/__init__.py +0 -0
  116. /deltacat/{storage → experimental/storage}/rivulet/parquet/data_reader.py +0 -0
  117. /deltacat/{storage/rivulet/schema → experimental/storage/rivulet/reader}/__init__.py +0 -0
  118. /deltacat/{storage → experimental/storage}/rivulet/reader/query_expression.py +0 -0
  119. /deltacat/{storage/rivulet/writer → experimental/storage/rivulet/schema}/__init__.py +0 -0
  120. /deltacat/{storage → experimental/storage}/rivulet/schema/datatype.py +0 -0
  121. /deltacat/{tests/storage/rivulet → experimental/storage/rivulet/shard}/__init__.py +0 -0
  122. /deltacat/{tests/storage/rivulet/fs → experimental/storage/rivulet/writer}/__init__.py +0 -0
  123. /deltacat/{storage → experimental/storage}/rivulet/writer/dataset_writer.py +0 -0
  124. /deltacat/tests/{storage/rivulet/schema → catalog/data}/__init__.py +0 -0
  125. /deltacat/tests/{storage/rivulet/writer → catalog/main}/__init__.py +0 -0
  126. {deltacat-2.0.0b10.dist-info → deltacat-2.0.0b11.dist-info}/LICENSE +0 -0
  127. {deltacat-2.0.0b10.dist-info → deltacat-2.0.0b11.dist-info}/WHEEL +0 -0
  128. {deltacat-2.0.0b10.dist-info → deltacat-2.0.0b11.dist-info}/top_level.txt +0 -0
@@ -1,8 +1,8 @@
1
- deltacat/__init__.py,sha256=Or3bYqhSPeNUOTbbDRsou9tcCw3JoOpswtS3a-EZeV8,2859
1
+ deltacat/__init__.py,sha256=_r2iNgTB-UbRKhQooHH8Q_Ul_sb6dScSkIHSYsrZ_7E,2835
2
2
  deltacat/annotations.py,sha256=9lBi34DpIV_RPjCCK2Aiz_6nMyd-e-_CfQ1XtdRQQlM,1196
3
3
  deltacat/api.py,sha256=8wXFzNWDUYqy40ZtRex4outDPiFy8K8KHF-PSxMsZsw,18396
4
- deltacat/constants.py,sha256=gMfNHQNh8FXXweiYz3yvE1Cc1S5FVkQLRZQKUcWypZg,3441
5
- deltacat/env.py,sha256=cDEdyNtT448fAnioU3VoavLyprWR8si0YRLhfviNZrY,1994
4
+ deltacat/constants.py,sha256=ggHddITClnrcrzeGQkSas81UeyXGrF1DmLZQG-oNpik,3395
5
+ deltacat/env.py,sha256=BJdTt8od3IVR4RMLjBxy4oRUHM7Lb16AzMOz8-hpwOI,2303
6
6
  deltacat/exceptions.py,sha256=K44xpmWNptIpM2gEWX6cy8Ko0vU1G7qDwwILDW8NWKM,13938
7
7
  deltacat/logs.py,sha256=vlKFsid9D6dpX9JlIP67fRZy18SjA08XMEfn0QWrxt0,9513
8
8
  deltacat/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -10,27 +10,23 @@ deltacat/aws/clients.py,sha256=4eQvpkV1PzFfxog7EriuglOGGwNFHR5hbGYpjsNNPxk,6949
10
10
  deltacat/aws/constants.py,sha256=hcYAUot4ahq9GXCMClQiuYCtiDs5XaOebdUoKg4V84k,1222
11
11
  deltacat/aws/s3u.py,sha256=OQYDh0MqwH6UPSU62fzy0g226OiPeFndjsnwFkd8GGc,28572
12
12
  deltacat/benchmarking/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
13
- deltacat/benchmarking/benchmark_engine.py,sha256=v3s5VcY8U7s_G6gaAiSbH0RP92dIcrfai0oOsEfW0Nk,3342
13
+ deltacat/benchmarking/benchmark_engine.py,sha256=cLoENLTJNAO01Xfsyf14B7xvE4KW1FCyDNcyZp3AUdE,3377
14
14
  deltacat/benchmarking/benchmark_parquet_reads.py,sha256=2BctkvXAYcAxokLwMSTu4TQ6-HGqzkgYcVEAzPN2QQo,1709
15
15
  deltacat/benchmarking/benchmark_report.py,sha256=Bm-7NQFiQM0f47tQ9jMjcaQ8GBNxZ-m8Z130wgwTldk,2719
16
16
  deltacat/benchmarking/benchmark_suite.py,sha256=RbZR1p0U0MU0HkGR06MosV6TKysooQD0HQne2rUAIds,316
17
17
  deltacat/benchmarking/conftest.py,sha256=WRfEz46G8PyLp9E3RNpv_jpys7AyGiC-GFpUYs08f9M,2414
18
- deltacat/benchmarking/test_benchmark_pipeline.py,sha256=XRqrShvtbCrPsPDRSQkn4FM6fTawwxsY2xJN9P6jiQI,3790
18
+ deltacat/benchmarking/test_benchmark_pipeline.py,sha256=MD6Af3EJ1V80l9xqCbl4N5uEjI8thMb2zXgJIrs2Ofg,3851
19
19
  deltacat/benchmarking/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
20
20
  deltacat/benchmarking/data/random_row_generator.py,sha256=-D1RFIQ6jE6KBmWGN6We8smhYxWQ19N2Pi8Z3iwBCiM,3227
21
21
  deltacat/benchmarking/data/row_generator.py,sha256=PqtYdbgznqKVfidnR40gwET7yHov2N7Esd8bEY90ALU,333
22
- deltacat/catalog/__init__.py,sha256=w18zlda3Xzk2mfk53VlLt00Ao8hlGkHrnVZJDgwrITU,372
22
+ deltacat/catalog/__init__.py,sha256=8tC0n1Y1LSs-tePt1TAPg9i_kNGxo0WgyFSlGFYxxhw,1500
23
23
  deltacat/catalog/delegate.py,sha256=x3jj_T61gyExuAnbDqhU6smbaAbIN4UxrVMZuBEOg0A,10447
24
24
  deltacat/catalog/interface.py,sha256=YB-qNBFsWupqyWJuHr7eQ-_MshhZZ5HpLphoZ64yn2g,12244
25
- deltacat/catalog/iceberg/__init__.py,sha256=LOENcLTQQlu_694MvRhMd2TQDLzwfg2vz0D8DuVO3M8,190
26
- deltacat/catalog/iceberg/iceberg_catalog_config.py,sha256=LfHxv8pk-YmTRQy5LvKFzwSqZ8ek2Y6v0KY7xihhIN0,786
27
- deltacat/catalog/iceberg/impl.py,sha256=c_ONnLLyh8Vyqo5PusQSHySQ92iM4Qgk-rucHMfdd7s,14288
28
- deltacat/catalog/iceberg/overrides.py,sha256=WmM2mxf7ihDl8anb5GzBxo5-sxBkot8ZSRTxDpaauRA,2687
29
25
  deltacat/catalog/main/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
30
- deltacat/catalog/main/impl.py,sha256=nPM7TbgF3OeGkgtP3iDEiTbS1DdoV3wTfZWgb1mSqvo,23148
26
+ deltacat/catalog/main/impl.py,sha256=-thwtB19f45Hh2zlenXMcRRrKDWvjbGU10OagiVpYsI,23681
31
27
  deltacat/catalog/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
32
- deltacat/catalog/model/catalog.py,sha256=ioDzHqWJ6XMcFPHWVV1bEu7ICPi1prOfbR_n4XSDuuM,10714
33
- deltacat/catalog/model/properties.py,sha256=4Y-DjpYa5LrwdRYWYgxkv_R4qg6dw7gXuU2Cqj_-EV0,4081
28
+ deltacat/catalog/model/catalog.py,sha256=qgIw5faIOdy1Tbn6JJv0J-3gXXGmVY7tTOp5IDvg5Og,11148
29
+ deltacat/catalog/model/properties.py,sha256=UraiKC99UlZh1ev4UbydQbBaa8-L5QhgSxiMaMbxSE0,3894
34
30
  deltacat/catalog/model/table_definition.py,sha256=mKmwFd2qKlj7ZH4AdZwU3bEcFtzpGvl0mXuWjqLUgBw,1842
35
31
  deltacat/compute/__init__.py,sha256=Yopyz3mokk-HtV9CDjQFCluvHbI7UiOPZYVtapUu13A,247
36
32
  deltacat/compute/compactor/__init__.py,sha256=ivpOPve1yKi3Vz3tVgp-eeFMNEeUSf-dlRJNSCM85sE,1022
@@ -113,7 +109,7 @@ deltacat/compute/converter/utils/iceberg_columns.py,sha256=k7ng1LOCU8VUfEHMj6z0l
113
109
  deltacat/compute/converter/utils/io.py,sha256=8ymAFaciRpIhFbe_bl6D-Ozn5HehO5sCKV24DeenNbQ,4218
114
110
  deltacat/compute/converter/utils/s3u.py,sha256=Du5UKB0hdaQmT-yGaFtUVCXB2KTBVXWmKRL2er49KKg,4434
115
111
  deltacat/compute/jobs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
116
- deltacat/compute/jobs/client.py,sha256=Gli733BnvG0-o3gvH2AtJkYvbE08JbjzCx2jpgaPRWo,14737
112
+ deltacat/compute/jobs/client.py,sha256=xWhZUpBNoSFa7s0MxTbqJ0OKHHiB1O3SIbM8gVJZaq8,14750
117
113
  deltacat/compute/merge_on_read/__init__.py,sha256=ckbgngmqPjYBYz_NySsR1vNTOb_hNpeL1sYkZKvBI9M,214
118
114
  deltacat/compute/merge_on_read/daft.py,sha256=1oC38u5ig_aTrq7EzyWBo8Ui54rb6yERYMk-vEFbpxM,1400
119
115
  deltacat/compute/merge_on_read/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -133,26 +129,75 @@ deltacat/compute/stats/models/delta_stats.py,sha256=hBith8_hbF9TVr6HocLAt6RJ_kZZ
133
129
  deltacat/compute/stats/models/delta_stats_cache_result.py,sha256=mbJYxpZd5jaER_BWrCD2hROFy3p1nNdBrj66nUpc6io,1624
134
130
  deltacat/compute/stats/models/manifest_entry_stats.py,sha256=NCDAe2nPDEI4kOkuwNkRFgGPS-rqQaQqLuaLoKk20KQ,2419
135
131
  deltacat/compute/stats/models/stats_result.py,sha256=XQAlmzhUqRmg4jzEMUAOqcYn1HUOBTMryBH1CCVlet8,3820
136
- deltacat/daft/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
137
- deltacat/daft/daft_scan.py,sha256=30S6OgKx3eBovpAVomHcr8xjjJVf_vwhJ9FA6dKQKYc,3981
138
- deltacat/daft/model.py,sha256=6NaKkp9R0ruE0K2x-moyARNrQisswUl6TjMeA6YHtBM,9078
139
- deltacat/daft/translator.py,sha256=Tm2K1Zcik2TjV_vKCd8Jc2IEA5MvSKLz1FvGODKrWR8,3771
140
132
  deltacat/examples/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
141
- deltacat/examples/basic_logging.py,sha256=Ia4eUyvSINjIGEOMnVMyo6a1iIxEdqlUtC_FLoBtQ5c,2872
133
+ deltacat/examples/basic_logging.py,sha256=Umrum-gvY3gJjDNJ4hOMslMMq9bzeTM-s_DO4dGqJiw,2833
142
134
  deltacat/examples/hello_world.py,sha256=FvxkEDB1qVPJv55Fe1I7Coy0VLYJIisU7ZFYYkw9U2g,525
143
- deltacat/examples/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
144
- deltacat/examples/common/fixtures.py,sha256=MS0Hz1c__f9Axm3JgTajfWuMVeDAQmFmZ7KB7vz_1q4,430
145
- deltacat/examples/iceberg/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
146
- deltacat/examples/iceberg/iceberg_bucket_writer.py,sha256=PdJG3jXcgPVds4UanfyNWB1egv-Os7LnZCPhdgv9Yyk,6586
147
- deltacat/examples/iceberg/iceberg_reader.py,sha256=mlF-277vT04at-2jibAjgRJG6Y-zle_NNy1-pXwS2YQ,5023
135
+ deltacat/examples/experimental/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
136
+ deltacat/examples/experimental/iceberg/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
137
+ deltacat/examples/experimental/iceberg/iceberg_bucket_writer.py,sha256=7v4y_qWLpjB1oT_ZfUXORTF7kWZygUmxmKJoGakLsio,6573
138
+ deltacat/examples/experimental/iceberg/iceberg_reader.py,sha256=ygnfXiZkVI4MGnbkrAFfHl6eECjVUW2jqz6N5tjJ4Bk,4997
148
139
  deltacat/examples/indexer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
149
- deltacat/examples/indexer/indexer.py,sha256=jjJAwyL0AJ6FdZbbtCeHGSsWr09-7WaMEFg1GSr-4ew,6320
150
- deltacat/examples/indexer/job_runner.py,sha256=YlT89YB5s5pZLNrUuK0-i600NDu4O0Zbb5agAk1XDg8,5940
140
+ deltacat/examples/indexer/indexer.py,sha256=A5gfWomVtfu9uRrAB6UbitQ158ZFkviUscSveri1ggs,6334
141
+ deltacat/examples/indexer/job_runner.py,sha256=M3ZKTcPL5gy1u_E3aZPal2u5INWZlBGDrsRhb5-F7n4,5921
151
142
  deltacat/examples/indexer/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
152
143
  deltacat/examples/indexer/gcp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
153
144
  deltacat/experimental/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
145
+ deltacat/experimental/catalog/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
146
+ deltacat/experimental/catalog/iceberg/__init__.py,sha256=rKZk4_5Yh4k50DVxOMb91VUqE18dn3k8XML80ZX_Z60,225
147
+ deltacat/experimental/catalog/iceberg/iceberg_catalog_config.py,sha256=W3916LCtSP3IPjOzyY-OKH5sSfqixVjVoANb6WVPhB4,799
148
+ deltacat/experimental/catalog/iceberg/impl.py,sha256=YDNRa6363XCPQXcPJkLmoq5cc-qIB1wh_BGou0y2t7c,14753
149
+ deltacat/experimental/catalog/iceberg/overrides.py,sha256=WmM2mxf7ihDl8anb5GzBxo5-sxBkot8ZSRTxDpaauRA,2687
154
150
  deltacat/experimental/daft/__init__.py,sha256=0d1SsgjbDher8TKgS0gSBBdy5TGi01fewiwpG0BMwck,108
155
151
  deltacat/experimental/daft/daft_catalog.py,sha256=112wDqqzdtxmtZVwiZW59MektbRsFMjSRgqYHrUOuok,8396
152
+ deltacat/experimental/storage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
153
+ deltacat/experimental/storage/iceberg/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
154
+ deltacat/experimental/storage/iceberg/iceberg_scan_planner.py,sha256=kraWLzN9zC0_7F7jgtWXo6utPDGaez4CR7JgOgmT-R8,1082
155
+ deltacat/experimental/storage/iceberg/impl.py,sha256=vF8n_W6fY8SNhYS3abj2k36k3dUruYZm8Ol0L-H2_sg,26326
156
+ deltacat/experimental/storage/iceberg/model.py,sha256=PbEnOEY5Ni7E5pJA2xL2d-rwLGOWw6eTARBAYFgCzm8,24288
157
+ deltacat/experimental/storage/rivulet/__init__.py,sha256=X7bYog7c4NFjoGw16AsCasBRNqKH9N7Z9Pc8aFUT2sA,353
158
+ deltacat/experimental/storage/rivulet/dataset.py,sha256=5eNkHZEK9f1t4R-IatUFtGD-x7jWiRE5aJKOKP6YA6g,28740
159
+ deltacat/experimental/storage/rivulet/dataset_executor.py,sha256=8KVkXLdnyDVfbtFFGw4x3o-1EHUQt8j8DZkfpI-0jaQ,2772
160
+ deltacat/experimental/storage/rivulet/logical_plan.py,sha256=jvF4fyrIWHuLnMX-_tBytiGekPRqW1dwB6pbL3agF8s,3316
161
+ deltacat/experimental/storage/rivulet/serializer.py,sha256=LeyPOKzSa0m5mI__i7OUaVvBTT9T78NQe1hmlgvij_8,1690
162
+ deltacat/experimental/storage/rivulet/serializer_factory.py,sha256=XDntdBRtHRK9T3lZ0KNVlb5hLudfBXJhgZyPM-4G6OY,1920
163
+ deltacat/experimental/storage/rivulet/arrow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
164
+ deltacat/experimental/storage/rivulet/arrow/serializer.py,sha256=HlMaNlshxYjCq5wgWVI5ETKtf6vQtl0fpv6Y7X3_hfo,2736
165
+ deltacat/experimental/storage/rivulet/feather/__init__.py,sha256=rR91gf2z-iux8WNV5equ9FEfenrl_eu3F0PJnxDCnKs,321
166
+ deltacat/experimental/storage/rivulet/feather/file_reader.py,sha256=SpgJDMw5lHFnQF2X0j1omuh02y1noYQPusgPsKTdlMA,4947
167
+ deltacat/experimental/storage/rivulet/feather/serializer.py,sha256=Jwcpnlo4fdtzwEdTBnzaOc210asi1wm1JppK8kUX0D8,1596
168
+ deltacat/experimental/storage/rivulet/fs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
169
+ deltacat/experimental/storage/rivulet/fs/file_provider.py,sha256=NPEdUnvQ6WDS3I8rWk5j4OKesIj4CR-an2BDDziwlHA,3863
170
+ deltacat/experimental/storage/rivulet/fs/file_store.py,sha256=OgAXGkSEKY0IR-Bk38tvbcDQ9g1ozGzQxNoCUDl8Yzk,5658
171
+ deltacat/experimental/storage/rivulet/fs/input_file.py,sha256=2SZdngGUlURg6ier3N-0D7zEySxgerg6EHuj3v0DXJM,1921
172
+ deltacat/experimental/storage/rivulet/fs/output_file.py,sha256=DRU36N3UpoPtu73RPkhnBBipUmUG6NYMd8lhRL_qCoY,2480
173
+ deltacat/experimental/storage/rivulet/metastore/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
174
+ deltacat/experimental/storage/rivulet/metastore/delta.py,sha256=EsaLbPvjlhn-UCwVjs-jIbCJF2qkrueeqZ0hAqgwN-A,6316
175
+ deltacat/experimental/storage/rivulet/metastore/json_sst.py,sha256=Gjw-R2U-v3q4gbjQ1vXqZjJHdE4y5qIvJNx202ivak4,3278
176
+ deltacat/experimental/storage/rivulet/metastore/sst.py,sha256=7mOn1OE0FNBSJ7C3CiLPqFXnHBb6LtJcAVgadVY7aHA,2432
177
+ deltacat/experimental/storage/rivulet/metastore/sst_interval_tree.py,sha256=3MQdduJF4TkSFJf9pMTTM8PofnpUpWkdlf0T1dMkzHY,10722
178
+ deltacat/experimental/storage/rivulet/mvp/Table.py,sha256=j82_-gF7byXx3a0yBCWVCtYH2H93qKPBD_1qp5GrlAA,3016
179
+ deltacat/experimental/storage/rivulet/mvp/__init__.py,sha256=28ULboqUHUGJuHjDWQgvG7cyR_ylV8fJ3B_wrsvI1f8,234
180
+ deltacat/experimental/storage/rivulet/parquet/__init__.py,sha256=1zx5IqSRMWsdW03kVmsphY28aWGzC9Tl-oGeoBUASgo,321
181
+ deltacat/experimental/storage/rivulet/parquet/data_reader.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
182
+ deltacat/experimental/storage/rivulet/parquet/file_reader.py,sha256=4Z0KTe14VquvqFSIRQlZ971L2RDqc_XkL8qO0OnjZEQ,4430
183
+ deltacat/experimental/storage/rivulet/parquet/serializer.py,sha256=SpILisUAg8Y--86aU4ZjQ24n_0kXOvEBHui2XC4R2sM,1575
184
+ deltacat/experimental/storage/rivulet/reader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
185
+ deltacat/experimental/storage/rivulet/reader/block_scanner.py,sha256=qVYCS_Cl7J9c-eQmv0QKxjkuMXL5oJoPAYnyEiFWPaU,15671
186
+ deltacat/experimental/storage/rivulet/reader/data_reader.py,sha256=kZTv2ErglqtILScvn2i7DDACW_2GcNPmnvuxmO0xdpA,4006
187
+ deltacat/experimental/storage/rivulet/reader/data_scan.py,sha256=-v4qrHQFHFg57yeweqfmhGuCjo539sQPVyNOvTSMUb4,2773
188
+ deltacat/experimental/storage/rivulet/reader/dataset_metastore.py,sha256=UupYrzzkv1eccGgLyU_zwnTAVs1J0eYx_GMfOWMeDaM,6249
189
+ deltacat/experimental/storage/rivulet/reader/dataset_reader.py,sha256=rRc0qB063mHToXZ4idA0lA01zL0xA3hrt-E7o8w699s,6236
190
+ deltacat/experimental/storage/rivulet/reader/pyarrow_data_reader.py,sha256=xjh0qAdpAuXpXIx7f7jfumofRXjQV0t_Gu61wtQ5u4E,4263
191
+ deltacat/experimental/storage/rivulet/reader/query_expression.py,sha256=S6EqP2Ru0liWbYws_ZKZz_aTCWJAHEGoZ2MUY5LZQLk,3121
192
+ deltacat/experimental/storage/rivulet/reader/reader_type_registrar.py,sha256=tIuYUOxOgFEBMNMKIXDqjHSR39IBTHyJpaz2FvwLl6Q,3083
193
+ deltacat/experimental/storage/rivulet/schema/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
194
+ deltacat/experimental/storage/rivulet/schema/datatype.py,sha256=QKlMwaEuALV6viNnNSRN1PNFUg7edzooEO3Jp901j1I,4370
195
+ deltacat/experimental/storage/rivulet/schema/schema.py,sha256=zjREiUpSXaRRcdCHstmdaAR2ykAo1uDTbrJcDqmHVSQ,9474
196
+ deltacat/experimental/storage/rivulet/shard/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
197
+ deltacat/experimental/storage/rivulet/shard/range_shard.py,sha256=uuulo-I9A2nXq2gY0qIMVC8yKf4xmVNhDheQ_SFBVFQ,4881
198
+ deltacat/experimental/storage/rivulet/writer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
199
+ deltacat/experimental/storage/rivulet/writer/dataset_writer.py,sha256=aLLwhMY9K5DENu_Rtqa3oWUmdPOYrP07BXntBW3pgdo,957
200
+ deltacat/experimental/storage/rivulet/writer/memtable_dataset_writer.py,sha256=jZJnxyB1PjoKgkoAfQzr-qXNISDcaSURLsbUVu6tImM,10164
156
201
  deltacat/io/__init__.py,sha256=QgARmNDGbOKRV2g5HAG9GKL589zt4WUY3T6E6vWZRWI,382
157
202
  deltacat/io/file_object_store.py,sha256=YoNL3Qla8uLOHaWnyBmIgotjSGAy3Td3Tumah0kk73Y,1868
158
203
  deltacat/io/memcached_object_store.py,sha256=C96t77-4BQe0XZ4vC76Ygi2o1POUoMN4t4BiyPmulz0,10997
@@ -167,13 +212,9 @@ deltacat/io/datasink/deltacat_datasink.py,sha256=TcXi_jegA0eawUDq3s6nTm3IcAV3B8n
167
212
  deltacat/io/datasource/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
168
213
  deltacat/io/datasource/deltacat_datasource.py,sha256=G9casuqV8c5nYLelZrEX0tGHDYrzJtH2R1q6-z5zFLk,21741
169
214
  deltacat/io/reader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
170
- deltacat/io/reader/deltacat_read_api.py,sha256=WqO5ZSGqlYPgmOBIjrmdQy0MaFw6epBwBUQ6tj10d2c,8393
215
+ deltacat/io/reader/deltacat_read_api.py,sha256=OSQ8PTWvyyRq5UTSFBlkFgphkJEm0EB5oztr_iszzHc,8394
171
216
  deltacat/storage/__init__.py,sha256=93XqQC1UVsFen-nduWpeQ_pqZu355qHryf7YlilmqMo,3696
172
217
  deltacat/storage/interface.py,sha256=OkQr7WvsA_vPtr5OfG9lP8u4m5F8Y21Qa_CIi4LhrNo,25465
173
- deltacat/storage/iceberg/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
174
- deltacat/storage/iceberg/iceberg_scan_planner.py,sha256=TzHTFJidlOXEYCW1czMOOc6oCplDKIs8ooiPlIMXyB0,1069
175
- deltacat/storage/iceberg/impl.py,sha256=SbAkGn4wp5UUlZcLtYQgoqDjl9Ak49eSObsp01ZZXlc,26313
176
- deltacat/storage/iceberg/model.py,sha256=PbEnOEY5Ni7E5pJA2xL2d-rwLGOWw6eTARBAYFgCzm8,24288
177
218
  deltacat/storage/main/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
178
219
  deltacat/storage/main/impl.py,sha256=aVeo2t7m0_LumaDfH2CE_tB9-AXSYqlE-PXvprfTuOg,68493
179
220
  deltacat/storage/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -186,7 +227,7 @@ deltacat/storage/model/metafile.py,sha256=l6XUBMMlHFaHgjttZfq4sEeeSYgH_8eTkOLAvY
186
227
  deltacat/storage/model/namespace.py,sha256=gLli1V64O9RHIf-FesmqWA29Wi7P1kwt01uz5sDdJR0,2409
187
228
  deltacat/storage/model/partition.py,sha256=qNCvc74o_4pmFVL-FCyKCZMH4lHSjRO560sb3vaF_H0,20759
188
229
  deltacat/storage/model/schema.py,sha256=uDaOqm1m0DYu89d2vTTMV7hDMvtiVwHqI_HFhRRgAA0,33806
189
- deltacat/storage/model/shard.py,sha256=v6Aj0dZcwqzuQGNbqqf7hGfOZBeeT8yVg1v-FJbFVm8,1507
230
+ deltacat/storage/model/shard.py,sha256=boPOW45bwLwBazfXZpa3-C5SUSlgelpHf8Yl6357Bq0,1575
190
231
  deltacat/storage/model/sort_key.py,sha256=HAZfDYqV24XClf1KslQw0duFVczvkglp0juENrU41nI,5905
191
232
  deltacat/storage/model/stream.py,sha256=OwzdyCgBOgJ6T5tUASkEE1-Zt8c3psU2ebk8D5OSkTY,12280
192
233
  deltacat/storage/model/table.py,sha256=5dm6ix9d5WEKRn-E-1l7DUEdtHgAOLfjlCRph4mD45Y,7276
@@ -201,48 +242,6 @@ deltacat/storage/model/scan/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMp
201
242
  deltacat/storage/model/scan/push_down.py,sha256=SldUHgv-HRDpEqIcsD9HI_XuofFI8BI0yRONO0zCcPQ,1034
202
243
  deltacat/storage/model/scan/scan_plan.py,sha256=P59yYbQWXbKZU068GzIh7ya_h2n7NgZJBY_eWVfGLAU,250
203
244
  deltacat/storage/model/scan/scan_task.py,sha256=pOcGKMjnHrZwabtdk3rw5kbh_d_aJGqzgCj6c_im1vE,855
204
- deltacat/storage/rivulet/__init__.py,sha256=OQcvRBY6cG-pvBQU8Wu-gncry8UIIS9ke1TY0v8yrAo,301
205
- deltacat/storage/rivulet/dataset.py,sha256=WlTCWVS-GImBfEPtHxKsV4WyAJtWxOSBJyjC0_3dxiI,28605
206
- deltacat/storage/rivulet/dataset_executor.py,sha256=NXTT8XJIK9ld9EM6bhy0sbxQNsZTdRncJ2uZI5UrRbU,3374
207
- deltacat/storage/rivulet/logical_plan.py,sha256=UMW-T7ospMlci2l8suasZTnTUrw75rzTp8lo9lU1oyA,3279
208
- deltacat/storage/rivulet/serializer.py,sha256=Xa6rrAy6QNKcFNh_WxkyP8MHrofGdFVRfxX_jp1_FIk,1677
209
- deltacat/storage/rivulet/serializer_factory.py,sha256=zUoqew-cddK9o--UeIeDbjqVdSz09t-hzItouogkimo,1837
210
- deltacat/storage/rivulet/arrow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
211
- deltacat/storage/rivulet/arrow/serializer.py,sha256=MgODL6vBm8wyLgOH5fQ22OGrKwy-uO3OnJtM7FMomQY,2671
212
- deltacat/storage/rivulet/feather/__init__.py,sha256=SsosQE9GeG-jWRy_t3mMBWWS8FLAALdIr-Jj3sXq-04,286
213
- deltacat/storage/rivulet/feather/file_reader.py,sha256=riaJwxL0T2YxvPkB5uDtumIKw7eH76Ilx7YQkSCdze8,4873
214
- deltacat/storage/rivulet/feather/serializer.py,sha256=0ESomIDsKRvn6OH2k_IG6zoxTlHdHnlK41yDSNWl9ac,1544
215
- deltacat/storage/rivulet/fs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
216
- deltacat/storage/rivulet/fs/file_provider.py,sha256=1bFiQcqJjyCMsZjhAicujwBLdo4P_moTY8BOYlbfvdE,3824
217
- deltacat/storage/rivulet/fs/file_store.py,sha256=5qWZfWkI__9MNpycgFJVYBhnpbsO-qSbhreYtphqIcA,5632
218
- deltacat/storage/rivulet/fs/input_file.py,sha256=2SZdngGUlURg6ier3N-0D7zEySxgerg6EHuj3v0DXJM,1921
219
- deltacat/storage/rivulet/fs/output_file.py,sha256=kjJHDhwKxF3TQ81TBQGX2FGyE-Wk7oA8QVGLqTP0nhg,2467
220
- deltacat/storage/rivulet/metastore/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
221
- deltacat/storage/rivulet/metastore/delta.py,sha256=HWtDeKft6ExH29TndbrNQhTgUiUkJYmKr6_Tq23CwRs,6303
222
- deltacat/storage/rivulet/metastore/json_sst.py,sha256=mckK9bzu1Tt1kDhZhzFFEbOlDMUqHTl9TeHd3r0AwaA,3239
223
- deltacat/storage/rivulet/metastore/sst.py,sha256=1uHUHmh6q9hcGXhvrnQFQVLfKBKaBIay1gL5kfbd9nI,2406
224
- deltacat/storage/rivulet/metastore/sst_interval_tree.py,sha256=j6BwNkpY18WYG3b4s0v_vt4fBKbrTGj5q_QsLPGU-Cg,10683
225
- deltacat/storage/rivulet/mvp/Table.py,sha256=j82_-gF7byXx3a0yBCWVCtYH2H93qKPBD_1qp5GrlAA,3016
226
- deltacat/storage/rivulet/mvp/__init__.py,sha256=28ULboqUHUGJuHjDWQgvG7cyR_ylV8fJ3B_wrsvI1f8,234
227
- deltacat/storage/rivulet/parquet/__init__.py,sha256=n896g5SyboT2t_foaFnPXOdTgqZ9JarsFFzVrtnGeT0,286
228
- deltacat/storage/rivulet/parquet/data_reader.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
229
- deltacat/storage/rivulet/parquet/file_reader.py,sha256=fzDg2VN50GqljFQFFM_bytbOl2w7BP-FrlUThZIWrzQ,4356
230
- deltacat/storage/rivulet/parquet/serializer.py,sha256=p9g2oskJTOgw7afiVdr9Vb-PTgzRV4R_9atQHlhnj58,1523
231
- deltacat/storage/rivulet/reader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
232
- deltacat/storage/rivulet/reader/block_scanner.py,sha256=QvaAAop6YunxrYsvTlFrAmgW_ZjmfPcj7ZpEAJ6cl5w,15505
233
- deltacat/storage/rivulet/reader/data_reader.py,sha256=GojlClE3W7WwQnvohu3gJK0TEP05_ToGk_wLKtl3hi8,3967
234
- deltacat/storage/rivulet/reader/data_scan.py,sha256=ada1INAyql42Q8Dw3l9tq1ZIlD2hllU2ivVc0sClI98,2725
235
- deltacat/storage/rivulet/reader/dataset_metastore.py,sha256=E1BCBcBihPF9HLCy9vWaY54BRMTuHKz_yNlaXprxy10,6197
236
- deltacat/storage/rivulet/reader/dataset_reader.py,sha256=wjdioXZA3IoLfXKPjjsnkUE1kKGmvy7vEegJNUKSl-4,6149
237
- deltacat/storage/rivulet/reader/pyarrow_data_reader.py,sha256=QdtrAJ4d42lVxl7ZGE_lBDb_H6keZlcI2DkNkWcyCrU,4237
238
- deltacat/storage/rivulet/reader/query_expression.py,sha256=S6EqP2Ru0liWbYws_ZKZz_aTCWJAHEGoZ2MUY5LZQLk,3121
239
- deltacat/storage/rivulet/reader/reader_type_registrar.py,sha256=NFSSBHUopQDQtnadSm0W4HXGhEvPH5sKAXdZG7yNx34,3031
240
- deltacat/storage/rivulet/schema/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
241
- deltacat/storage/rivulet/schema/datatype.py,sha256=QKlMwaEuALV6viNnNSRN1PNFUg7edzooEO3Jp901j1I,4370
242
- deltacat/storage/rivulet/schema/schema.py,sha256=t-y4BHSQkZ2bI8WxQfMsXoSYUe4sOLOA75u-BGKXaZM,9461
243
- deltacat/storage/rivulet/writer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
244
- deltacat/storage/rivulet/writer/dataset_writer.py,sha256=aLLwhMY9K5DENu_Rtqa3oWUmdPOYrP07BXntBW3pgdo,957
245
- deltacat/storage/rivulet/writer/memtable_dataset_writer.py,sha256=YxB6q-FjD0h3VUbLabjRTiE_MVzj1JCDPM8HsOzgGU8,10012
246
245
  deltacat/storage/util/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
247
246
  deltacat/storage/util/scan_planner.py,sha256=yxvExB_rbTc5ijLdnSjJv0BAFGsNRqegzzDBUYzXZ-k,820
248
247
  deltacat/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -262,8 +261,14 @@ deltacat/tests/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuF
262
261
  deltacat/tests/aws/test_clients.py,sha256=23GMWfz27WWBDXSqphG9mfputsyS7j3I5P_HRk4YoKE,3790
263
262
  deltacat/tests/aws/test_s3u.py,sha256=FsYCH8K8DsDRPOtTp-w1Nu3ATqt4p1mqDo6aVJV-SbU,7918
264
263
  deltacat/tests/catalog/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
265
- deltacat/tests/catalog/test_catalogs.py,sha256=qllEWqma3aowxdbrY5F7W8okXq7X5IseSEsuBDmv-c4,12295
266
- deltacat/tests/catalog/test_default_catalog_impl.py,sha256=Z4eWJkbEhPsQ5O68btJzHs-B4WE2io7agSGT3v7Q3-Q,3361
264
+ deltacat/tests/catalog/test_catalogs.py,sha256=hxmjV7tdtcqFdDnY-bqlIruxEt6x1bjJAVmou763jMM,10183
265
+ deltacat/tests/catalog/test_default_catalog_impl.py,sha256=meqLi9G1ToVQ8ZztgUSh-d1WhFaAlT3mXlugV7ioTDE,3289
266
+ deltacat/tests/catalog/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
267
+ deltacat/tests/catalog/main/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
268
+ deltacat/tests/catalog/main/test_catalog_impl_namespace_operations.py,sha256=XvMoW1yjjoIs4-0A8_dqeY7ArysN6HJkSSHk7JnHeUI,4313
269
+ deltacat/tests/catalog/main/test_catalog_impl_table_operations.py,sha256=74_c77MMQu4W1s2ildOOs2HO4PDwXUDergkPg6eSus0,15352
270
+ deltacat/tests/catalog/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
271
+ deltacat/tests/catalog/model/test_table_definition.py,sha256=R-_FGV-52o5fShItXrbcqt2-2ECjrDQmcOHWKBCU-HQ,574
267
272
  deltacat/tests/compute/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
268
273
  deltacat/tests/compute/compact_partition_multiple_rounds_test_cases.py,sha256=g4DKJEn7BtLtuDIxYVq9ZkK1S91o4ftRB7-3jhDCJ4s,37694
269
274
  deltacat/tests/compute/compact_partition_rebase_test_cases.py,sha256=IwA0w5gURCFlUCFxXmnTSIPh5mkiGKRMw1UB9yU9ESQ,16821
@@ -297,6 +302,35 @@ deltacat/tests/compute/resource_estimation/__init__.py,sha256=47DEQpj8HBSa-_TImW
297
302
  deltacat/tests/compute/resource_estimation/test_delta.py,sha256=5GyaYObGa-PjTS01d3uRyTLoVAvgoanMQn0EOFhNFjI,25159
298
303
  deltacat/tests/compute/resource_estimation/test_manifest.py,sha256=yrMvqDjolExdRf6Vtg5XaKDuaKz9ok15PCZ7_aJOYrI,32893
299
304
  deltacat/tests/compute/resource_estimation/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
305
+ deltacat/tests/daft/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
306
+ deltacat/tests/daft/test_model.py,sha256=nqKZV1lUqvFzfyjrV1OxtJi3Fj33fcYIl9I0eguxAnQ,3953
307
+ deltacat/tests/experimental/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
308
+ deltacat/tests/experimental/catalog/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
309
+ deltacat/tests/experimental/catalog/iceberg/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
310
+ deltacat/tests/experimental/catalog/iceberg/test_iceberg_catalog.py,sha256=K3YwWX8g_Tf7SQq-ruKj3_fAdPYvHdFYDB57byGhOGc,2029
311
+ deltacat/tests/experimental/daft/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
312
+ deltacat/tests/experimental/daft/test_deltacat_daft_integration.py,sha256=Vx-5MSPuvaUMHUBQ4cGhob26JWSxNqiHUYVMwvqfS2c,5102
313
+ deltacat/tests/experimental/storage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
314
+ deltacat/tests/experimental/storage/rivulet/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
315
+ deltacat/tests/experimental/storage/rivulet/conftest.py,sha256=rtaU6iaCBlCbUiIc8UcVo5m4ADv8jN9N0FVi-36pS9o,3698
316
+ deltacat/tests/experimental/storage/rivulet/test_dataset.py,sha256=yD0c5r8a5IogAA_rvv3vM_waXQylA3oyUOHmvM_vcBI,13577
317
+ deltacat/tests/experimental/storage/rivulet/test_manifest.py,sha256=RS-cZQb7lkwpFFHNnU5varlVp_3njrjBw2B-84tEOPA,1943
318
+ deltacat/tests/experimental/storage/rivulet/test_sst_interval_tree.py,sha256=eBOFI4Z9GXxRwVTbnfhum-WNTTULyOV53gcvKQ8rfos,6866
319
+ deltacat/tests/experimental/storage/rivulet/test_utils.py,sha256=d9igdUG7BSoYVZN0-skxmKRPJ4HmDJE-P6wjh2qqk4Q,4469
320
+ deltacat/tests/experimental/storage/rivulet/fs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
321
+ deltacat/tests/experimental/storage/rivulet/fs/test_file_location_provider.py,sha256=TlIEWRZjmMSaCn5NoCDsis1lc4lNxD74tXUzm8rERO0,2842
322
+ deltacat/tests/experimental/storage/rivulet/reader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
323
+ deltacat/tests/experimental/storage/rivulet/reader/query_expression.py,sha256=6z3x5jEX3L0cUtvd-bx4HYWcMrxWf20PQ0USSxq6nTM,2312
324
+ deltacat/tests/experimental/storage/rivulet/reader/test_data_scan.py,sha256=NMEkkhJvT8Kkc7xXrb5lpN1nup964Qd3vwuo5FbvAE8,3105
325
+ deltacat/tests/experimental/storage/rivulet/reader/test_dataset_metastore.py,sha256=-OLXFgNHlqVw7ezm51f-x6YMZoBciXoR22BKw5LRH6E,2311
326
+ deltacat/tests/experimental/storage/rivulet/schema/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
327
+ deltacat/tests/experimental/storage/rivulet/schema/test_schema.py,sha256=aefVhuRnpp60ddw3wiDVYcgl2WJFWhb1AFDHgBzvD0Y,7254
328
+ deltacat/tests/experimental/storage/rivulet/shard/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
329
+ deltacat/tests/experimental/storage/rivulet/shard/test_range_shard.py,sha256=aUKD1ASNqUP0a6BOnnU-uVwkGsxtv6PXVGHyW6BeFxQ,4687
330
+ deltacat/tests/experimental/storage/rivulet/writer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
331
+ deltacat/tests/experimental/storage/rivulet/writer/test_dataset_write_then_read.py,sha256=-7bA8eS_vzDfwMJZjfnhg7ogp7jnMSZNEuxqthI9g-k,12299
332
+ deltacat/tests/experimental/storage/rivulet/writer/test_dataset_writer.py,sha256=1bqq9_yGSJD8sXEVDrzNsSA9nsMJ04k7PbxGpUZQ0a4,2884
333
+ deltacat/tests/experimental/storage/rivulet/writer/test_memtable_dataset_writer.py,sha256=www9mD_vugITO42ojHX8HmNiZxvveB0qYg-1MXwl2AA,2579
300
334
  deltacat/tests/local_deltacat_storage/__init__.py,sha256=kM-g8zLwXmGnCw6fUzc99v_id5lnalsg81u9D8nkvdQ,38771
301
335
  deltacat/tests/local_deltacat_storage/exceptions.py,sha256=oxZ0psmrEO0M6P2r8gHQ2E8E-Y8UBfUCBUIwfuHcx38,251
302
336
  deltacat/tests/storage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -309,23 +343,9 @@ deltacat/tests/storage/model/test_expression.py,sha256=eySWacUo8EK1NwxMQnNqpXAVb
309
343
  deltacat/tests/storage/model/test_manifest.py,sha256=udp9YUNvIBpnT-NutjMaF25abEQOXEcPkQm8Aay_UCs,3733
310
344
  deltacat/tests/storage/model/test_metafile_io.py,sha256=116U9aNJPzR0JS6iadJyyx0_4KyAi3D47WCNbndag6o,101639
311
345
  deltacat/tests/storage/model/test_schema.py,sha256=5m4BscbxbbOiry-lDI8j4vQcnvkG2Y-f0ZfshncPiSI,9599
312
- deltacat/tests/storage/model/test_shard.py,sha256=6QBr-ws3zQkJjjGyB7QEOhtNC5ql0cdjOPB2wxGNW3Q,755
346
+ deltacat/tests/storage/model/test_shard.py,sha256=gP8w1KMq73M4uMyQKsV6w1FFTSx0vdJ_ybA-Erqk2UY,785
313
347
  deltacat/tests/storage/model/test_table_version.py,sha256=CtLiOe1EI6Ao9MkxyMWoxTWuqQevupVUEQVU5ZIEB34,3306
314
348
  deltacat/tests/storage/model/test_transaction.py,sha256=Y5JMaYz6mf4DbruxMmz9hWXGX30MjKjRAvxlIf0MnaY,14458
315
- deltacat/tests/storage/rivulet/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
316
- deltacat/tests/storage/rivulet/conftest.py,sha256=1Ansaxs4WGVKhZ2iSGrFlz1Mjib6BLmZG98eSi9_EiU,3659
317
- deltacat/tests/storage/rivulet/test_dataset.py,sha256=4V6FFxz66TNwl6Vviw-zfu_K2mBTRDT-zcaarcPVCn4,13529
318
- deltacat/tests/storage/rivulet/test_manifest.py,sha256=mx3jlE0opEKla4i12V2THoM0f8-aWdPEJyJUrzi3lzg,1878
319
- deltacat/tests/storage/rivulet/test_sst_interval_tree.py,sha256=VK8lh5ZtnIsnuyphsnXdnD9OQhAlyOLKJ1M0l4ui6i8,6801
320
- deltacat/tests/storage/rivulet/test_utils.py,sha256=NvW_zA5CEEkooWDlc0aBCb_0pceCRb8WFY-HQ78mheM,4395
321
- deltacat/tests/storage/rivulet/fs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
322
- deltacat/tests/storage/rivulet/fs/test_file_location_provider.py,sha256=vH3QPEnEFHXKQrnHPbJjVKRK3H6mOIWaNyqlHfCZqEE,2803
323
- deltacat/tests/storage/rivulet/schema/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
324
- deltacat/tests/storage/rivulet/schema/test_schema.py,sha256=LngLuRlVK_1fK-JA5rW2_7RBPmeci7R_FdbsWytUVEo,7241
325
- deltacat/tests/storage/rivulet/writer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
326
- deltacat/tests/storage/rivulet/writer/test_dataset_write_then_read.py,sha256=IZTzlWLoFvedCN60AOo9tCfOqkNTgGDckOELn3B9Uoo,12173
327
- deltacat/tests/storage/rivulet/writer/test_dataset_writer.py,sha256=HW-E7RvuyxloCL4_z7EsAHOYvw0UPzupXrvv3L3s1vQ,2858
328
- deltacat/tests/storage/rivulet/writer/test_memtable_dataset_writer.py,sha256=lefIWItakJIYOfL3O0jvA2bpcY7t5C1u_TF-PygkU28,2488
329
349
  deltacat/tests/test_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
330
350
  deltacat/tests/test_utils/constants.py,sha256=UYe--9T_clYjiOpv0M7TtAMGdpje_SMZ-w8n0IeCAjc,214
331
351
  deltacat/tests/test_utils/filesystem.py,sha256=H4LhAZ4q1yC4mrAv5EQQ_NaBeYtuFXKxxAyEvirTt4w,221
@@ -346,15 +366,15 @@ deltacat/tests/utils/ray_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5
346
366
  deltacat/tests/utils/ray_utils/test_concurrency.py,sha256=TjZpX0cjMDEIS79p_--j_BfT0zXKNkTLY1ZzNokBTs0,1211
347
367
  deltacat/tests/utils/ray_utils/test_dataset.py,sha256=1hoOR_AIO4iJQ_lCjNsJfq7S-2ZyOKyMkKc4Tjt6cwg,2092
348
368
  deltacat/types/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
349
- deltacat/types/media.py,sha256=tW8PebSmr2mMobHFjuWYhrsc8UjD9CN_gCIcXJSwob4,5488
369
+ deltacat/types/media.py,sha256=kwp7DtZaa7jfB_hXPHXdXcUQy_nwrakeIPRF7I8rdHs,5487
350
370
  deltacat/types/partial_download.py,sha256=QIpNTSwaiZ4TVl4A1N4PtblevKT5GwdXtGrouQMQs1E,2510
351
371
  deltacat/types/tables.py,sha256=zz29727VTgu4InLPwjOjmDX0SrCiDq-VI1S75EZfIr8,5594
352
372
  deltacat/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
353
373
  deltacat/utils/arguments.py,sha256=5y1Xz4HSAD8M8Jt83i6gOEKoYjy_fMQe1V43IhIE4hY,1191
354
374
  deltacat/utils/cloudpickle.py,sha256=XE7YDmQe56ksfl3NdYZkzOAhbHSuhNcBZGOehQpgZr0,1187
355
375
  deltacat/utils/common.py,sha256=RG_-enXNpLKaYrqyx1ne2lL10lxN9vK7F631oJP6SE8,1375
356
- deltacat/utils/daft.py,sha256=ujZarxTdSA2GY_pOxXhHbMUB-CdOvYKLIqoM00NLdCI,5754
357
- deltacat/utils/export.py,sha256=As5aiwOw9vLxtfolPLU0yak6W2RVR0rkuaYQ5YCy49U,1952
376
+ deltacat/utils/daft.py,sha256=POX96a67wiOSonaAgA9lVQRZ0buNua0xhFx43e-lLT8,24032
377
+ deltacat/utils/export.py,sha256=aiebPfWxtc-o7gCqDilcyMxb0GdAByW47dygtQ2Wrzg,1974
358
378
  deltacat/utils/filesystem.py,sha256=7cmEmzc2JPPqbW-zlXv_d28R-IUqX6bmH3DWJ4NHB8A,13213
359
379
  deltacat/utils/metafile_locator.py,sha256=_3yEW9n49jiEBuXHZmUKsFdYx6RxWWuS-Mu2gs_a1bw,2933
360
380
  deltacat/utils/metrics.py,sha256=HYKyZSrtVLu8gXezg_TMNUKJp4h1WWI0VEzn0Xlzf-I,10778
@@ -367,15 +387,15 @@ deltacat/utils/pyarrow.py,sha256=gOSCs9XVeLy2PAERXM7DCg--ywuoxHO9dtl3LeIcJQU,342
367
387
  deltacat/utils/resources.py,sha256=Ax1OgLLbZI4oYpp4Ki27OLaST-7I-AJgZwU87FVfY8g,8253
368
388
  deltacat/utils/s3fs.py,sha256=PmUJ5Fm1WmD-_zp_M6yd9VbXvIoJuBeK6ApOdJJApLE,662
369
389
  deltacat/utils/schema.py,sha256=m4Wm4ZQcpttzOUxex4dVneGlHy1_E36HspTcjNYzvVM,1564
370
- deltacat/utils/url.py,sha256=V3SWhxfIMJ5g4mvNXb29NrxOiOKnPp3VVWspVdYD34Y,43821
390
+ deltacat/utils/url.py,sha256=oKBfOKwojtF-G8Ry_shsAJjqWdPsKn1m5IXBbyb4ffc,43822
371
391
  deltacat/utils/ray_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
372
392
  deltacat/utils/ray_utils/collections.py,sha256=hj20s4D2RF2jZETU_44r6mFbsczA0JI_I_4kWKTmqes,1951
373
393
  deltacat/utils/ray_utils/concurrency.py,sha256=Ceui6nQYKHTUOTltHNQIdb0OWHFhD73o8DhSXP-DYRQ,5457
374
394
  deltacat/utils/ray_utils/dataset.py,sha256=waHdtH0c835a-2t51HYRHnulfC0_zBxx8mFSAPvPSPM,3274
375
395
  deltacat/utils/ray_utils/performance.py,sha256=d7JFM7vTXHzkGx9qNQcZzUWajnqINvYRwaM088_FpsE,464
376
396
  deltacat/utils/ray_utils/runtime.py,sha256=cf5koY9q4TzRg--BjPtC6y0jztq45F39KcC4K6Wmg4w,6946
377
- deltacat-2.0.0b10.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
378
- deltacat-2.0.0b10.dist-info/METADATA,sha256=ySDCisOen9HXsW7IxUwKTnGXhec0L2de3CxpQJ7gA8M,2923
379
- deltacat-2.0.0b10.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
380
- deltacat-2.0.0b10.dist-info/top_level.txt,sha256=RWdIcid4Bv2i2ozLVh-70kJpyB61xEKXod9XXGpiono,9
381
- deltacat-2.0.0b10.dist-info/RECORD,,
397
+ deltacat-2.0.0b11.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
398
+ deltacat-2.0.0b11.dist-info/METADATA,sha256=oJQ9Ap_-LkIWKeCArIXq7tf8-kIcD2SdfuanP1XPgCI,2868
399
+ deltacat-2.0.0b11.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
400
+ deltacat-2.0.0b11.dist-info/top_level.txt,sha256=RWdIcid4Bv2i2ozLVh-70kJpyB61xEKXod9XXGpiono,9
401
+ deltacat-2.0.0b11.dist-info/RECORD,,
@@ -1,4 +0,0 @@
1
- from deltacat.catalog.iceberg.iceberg_catalog_config import IcebergCatalogConfig
2
- import deltacat.catalog.iceberg.impl as IcebergCatalog
3
-
4
- __all__ = ["IcebergCatalogConfig", "IcebergCatalog"]
@@ -1,115 +0,0 @@
1
- from typing import Iterator
2
-
3
- from daft import Schema
4
- from daft.daft import (
5
- StorageConfig,
6
- PartitionField,
7
- Pushdowns,
8
- ScanTask,
9
- FileFormatConfig,
10
- ParquetSourceConfig,
11
- )
12
- from daft.io.scan import ScanOperator, ScanPushdowns
13
-
14
- from deltacat.catalog.model.table_definition import TableDefinition
15
- from deltacat.daft.model import DaftPartitionKeyMapper
16
- from deltacat.daft.translator import translate_pushdown
17
-
18
-
19
- class DeltaCatScanOperator(ScanOperator):
20
- def __init__(self, table: TableDefinition, storage_config: StorageConfig) -> None:
21
- super().__init__()
22
- self.table = table
23
- self._schema = self._infer_schema()
24
- self.partition_keys = self._infer_partition_keys()
25
- self.storage_config = storage_config
26
-
27
- def schema(self) -> Schema:
28
- return self._schema
29
-
30
- def name(self) -> str:
31
- return "DeltaCatScanOperator"
32
-
33
- def display_name(self) -> str:
34
- return f"DeltaCATScanOperator({self.table.table.namespace}.{self.table.table.table_name})"
35
-
36
- def partitioning_keys(self) -> list[PartitionField]:
37
- return self.partition_keys
38
-
39
- def multiline_display(self) -> list[str]:
40
- return [
41
- self.display_name(),
42
- f"Schema = {self._schema}",
43
- f"Partitioning keys = {self.partitioning_keys}",
44
- f"Storage config = {self.storage_config}",
45
- ]
46
-
47
- def to_scan_tasks(self, pushdowns: Pushdowns) -> Iterator[ScanTask]:
48
- daft_pushdowns = ScanPushdowns._from_pypushdowns(
49
- pushdowns, schema=self.schema()
50
- )
51
- dc_pushdown = translate_pushdown(daft_pushdowns)
52
- dc_scan_plan = self.table.create_scan_plan(pushdown=dc_pushdown)
53
- scan_tasks = []
54
- file_format_config = FileFormatConfig.from_parquet_config(
55
- # maybe this: ParquetSourceConfig(field_id_mapping=self._field_id_mapping)
56
- ParquetSourceConfig()
57
- )
58
- for dc_scan_task in dc_scan_plan.scan_tasks:
59
- for data_file in dc_scan_task.data_files():
60
- st = ScanTask.catalog_scan_task(
61
- file=data_file.file_path,
62
- file_format=file_format_config,
63
- schema=self._schema._schema,
64
- storage_config=self.storage_config,
65
- pushdowns=pushdowns,
66
- )
67
- scan_tasks.append(st)
68
- return iter(scan_tasks)
69
-
70
- def can_absorb_filter(self) -> bool:
71
- return False
72
-
73
- def can_absorb_limit(self) -> bool:
74
- return False
75
-
76
- def can_absorb_select(self) -> bool:
77
- return True
78
-
79
- def _infer_schema(self) -> Schema:
80
-
81
- if not (
82
- self.table and self.table.table_version and self.table.table_version.schema
83
- ):
84
- raise RuntimeError(
85
- f"Failed to infer schema for DeltaCAT Table "
86
- f"{self.table.table.namespace}.{self.table.table.table_name}"
87
- )
88
-
89
- return Schema.from_pyarrow_schema(self.table.table_version.schema.arrow)
90
-
91
- def _infer_partition_keys(self) -> list[PartitionField]:
92
- if not (
93
- self.table
94
- and self.table.table_version
95
- and self.table.table_version.partition_scheme
96
- and self.table.table_version.schema
97
- ):
98
- raise RuntimeError(
99
- f"Failed to infer partition keys for DeltaCAT Table "
100
- f"{self.table.table.namespace}.{self.table.table.table_name}"
101
- )
102
-
103
- schema = self.table.table_version.schema
104
- partition_keys = self.table.table_version.partition_scheme.keys
105
- if not partition_keys:
106
- return []
107
-
108
- partition_fields = []
109
- for key in partition_keys:
110
- field = DaftPartitionKeyMapper.unmap(key, schema)
111
- # Assert that the returned value is not None.
112
- assert field is not None, f"Unmapping failed for key {key}"
113
- partition_fields.append(field)
114
-
115
- return partition_fields