deltacat 2.0__py3-none-any.whl → 2.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (324) hide show
  1. deltacat/__init__.py +117 -18
  2. deltacat/api.py +536 -126
  3. deltacat/aws/constants.py +0 -23
  4. deltacat/aws/s3u.py +4 -631
  5. deltacat/benchmarking/benchmark_engine.py +4 -2
  6. deltacat/benchmarking/conftest.py +1 -19
  7. deltacat/benchmarking/test_benchmark_pipeline.py +6 -4
  8. deltacat/catalog/__init__.py +64 -5
  9. deltacat/catalog/delegate.py +445 -63
  10. deltacat/catalog/interface.py +188 -62
  11. deltacat/catalog/main/impl.py +2444 -282
  12. deltacat/catalog/model/catalog.py +208 -113
  13. deltacat/catalog/model/properties.py +63 -24
  14. deltacat/compute/__init__.py +14 -0
  15. deltacat/compute/compactor/compaction_session.py +97 -75
  16. deltacat/compute/compactor/model/compact_partition_params.py +75 -30
  17. deltacat/compute/compactor/model/compaction_session_audit_info.py +17 -0
  18. deltacat/compute/compactor/model/round_completion_info.py +16 -6
  19. deltacat/compute/compactor/repartition_session.py +8 -21
  20. deltacat/compute/compactor/steps/hash_bucket.py +5 -5
  21. deltacat/compute/compactor/steps/materialize.py +9 -7
  22. deltacat/compute/compactor/steps/repartition.py +12 -11
  23. deltacat/compute/compactor/utils/io.py +6 -5
  24. deltacat/compute/compactor/utils/round_completion_reader.py +117 -0
  25. deltacat/compute/compactor/utils/system_columns.py +3 -1
  26. deltacat/compute/compactor_v2/compaction_session.py +17 -14
  27. deltacat/compute/compactor_v2/constants.py +30 -1
  28. deltacat/compute/compactor_v2/model/evaluate_compaction_result.py +0 -1
  29. deltacat/compute/compactor_v2/model/hash_bucket_input.py +9 -3
  30. deltacat/compute/compactor_v2/model/merge_file_group.py +5 -2
  31. deltacat/compute/compactor_v2/model/merge_input.py +33 -8
  32. deltacat/compute/compactor_v2/private/compaction_utils.py +167 -68
  33. deltacat/compute/compactor_v2/steps/hash_bucket.py +5 -2
  34. deltacat/compute/compactor_v2/steps/merge.py +267 -55
  35. deltacat/compute/compactor_v2/utils/content_type_params.py +34 -6
  36. deltacat/compute/compactor_v2/utils/dedupe.py +1 -1
  37. deltacat/compute/compactor_v2/utils/delta.py +5 -3
  38. deltacat/compute/compactor_v2/utils/io.py +11 -4
  39. deltacat/compute/compactor_v2/utils/merge.py +15 -2
  40. deltacat/compute/compactor_v2/utils/primary_key_index.py +28 -4
  41. deltacat/compute/compactor_v2/utils/task_options.py +45 -33
  42. deltacat/compute/converter/constants.py +5 -0
  43. deltacat/compute/converter/converter_session.py +207 -52
  44. deltacat/compute/converter/model/convert_input.py +43 -16
  45. deltacat/compute/converter/model/convert_input_files.py +33 -16
  46. deltacat/compute/converter/model/convert_result.py +80 -0
  47. deltacat/compute/converter/model/converter_session_params.py +64 -19
  48. deltacat/compute/converter/pyiceberg/catalog.py +21 -18
  49. deltacat/compute/converter/pyiceberg/overrides.py +193 -65
  50. deltacat/compute/converter/pyiceberg/update_snapshot_overrides.py +148 -100
  51. deltacat/compute/converter/steps/convert.py +230 -75
  52. deltacat/compute/converter/steps/dedupe.py +46 -12
  53. deltacat/compute/converter/utils/convert_task_options.py +66 -22
  54. deltacat/compute/converter/utils/converter_session_utils.py +126 -60
  55. deltacat/compute/converter/utils/iceberg_columns.py +13 -8
  56. deltacat/compute/converter/utils/io.py +173 -13
  57. deltacat/compute/converter/utils/s3u.py +42 -27
  58. deltacat/compute/janitor.py +205 -0
  59. deltacat/compute/jobs/client.py +417 -0
  60. deltacat/compute/resource_estimation/delta.py +38 -6
  61. deltacat/compute/resource_estimation/model.py +8 -0
  62. deltacat/constants.py +49 -6
  63. deltacat/docs/autogen/schema/inference/generate_type_mappings.py +687 -0
  64. deltacat/docs/autogen/schema/inference/parse_json_type_mappings.py +673 -0
  65. deltacat/env.py +10 -0
  66. deltacat/examples/basic_logging.py +6 -6
  67. deltacat/examples/compactor/aws/__init__.py +1 -0
  68. deltacat/examples/compactor/bootstrap.py +863 -0
  69. deltacat/examples/compactor/compactor.py +373 -0
  70. deltacat/examples/compactor/explorer.py +473 -0
  71. deltacat/examples/compactor/gcp/__init__.py +1 -0
  72. deltacat/examples/compactor/job_runner.py +439 -0
  73. deltacat/examples/compactor/utils/__init__.py +1 -0
  74. deltacat/examples/compactor/utils/common.py +261 -0
  75. deltacat/examples/experimental/iceberg/converter/beam/app.py +226 -0
  76. deltacat/examples/experimental/iceberg/converter/beam/main.py +133 -0
  77. deltacat/examples/experimental/iceberg/converter/beam/test_workflow.py +113 -0
  78. deltacat/examples/experimental/iceberg/converter/beam/utils/__init__.py +3 -0
  79. deltacat/examples/experimental/iceberg/converter/beam/utils/common.py +174 -0
  80. deltacat/examples/experimental/iceberg/converter/beam/utils/spark.py +263 -0
  81. deltacat/examples/{iceberg → experimental/iceberg}/iceberg_bucket_writer.py +66 -21
  82. deltacat/examples/{iceberg → experimental/iceberg}/iceberg_reader.py +2 -4
  83. deltacat/examples/hello_world.py +4 -2
  84. deltacat/examples/indexer/indexer.py +163 -0
  85. deltacat/examples/indexer/job_runner.py +198 -0
  86. deltacat/exceptions.py +66 -4
  87. deltacat/experimental/catalog/iceberg/__init__.py +6 -0
  88. deltacat/{catalog → experimental/catalog}/iceberg/iceberg_catalog_config.py +1 -1
  89. deltacat/{catalog → experimental/catalog}/iceberg/impl.py +43 -12
  90. deltacat/{catalog → experimental/catalog}/iceberg/overrides.py +12 -14
  91. deltacat/experimental/compatibility/backfill_locator_to_id_mappings.py +201 -0
  92. deltacat/experimental/converter_agent/__init__.py +0 -0
  93. deltacat/experimental/converter_agent/beam/__init__.py +0 -0
  94. deltacat/experimental/converter_agent/beam/managed.py +173 -0
  95. deltacat/experimental/converter_agent/table_monitor.py +479 -0
  96. deltacat/experimental/daft/__init__.py +4 -0
  97. deltacat/experimental/daft/daft_catalog.py +229 -0
  98. deltacat/experimental/storage/__init__.py +0 -0
  99. deltacat/experimental/storage/iceberg/__init__.py +0 -0
  100. deltacat/experimental/storage/iceberg/iceberg_scan_planner.py +129 -0
  101. deltacat/{storage → experimental/storage}/iceberg/impl.py +6 -4
  102. deltacat/{storage → experimental/storage}/iceberg/model.py +7 -3
  103. deltacat/experimental/storage/iceberg/visitor.py +119 -0
  104. deltacat/experimental/storage/rivulet/__init__.py +11 -0
  105. deltacat/experimental/storage/rivulet/arrow/__init__.py +0 -0
  106. deltacat/{storage → experimental/storage}/rivulet/arrow/serializer.py +7 -4
  107. deltacat/{storage → experimental/storage}/rivulet/dataset.py +13 -12
  108. deltacat/{storage → experimental/storage}/rivulet/dataset_executor.py +12 -20
  109. deltacat/experimental/storage/rivulet/feather/__init__.py +7 -0
  110. deltacat/{storage → experimental/storage}/rivulet/feather/file_reader.py +7 -5
  111. deltacat/{storage → experimental/storage}/rivulet/feather/serializer.py +4 -4
  112. deltacat/experimental/storage/rivulet/fs/__init__.py +0 -0
  113. deltacat/{storage → experimental/storage}/rivulet/fs/file_provider.py +3 -3
  114. deltacat/{storage → experimental/storage}/rivulet/fs/file_store.py +2 -2
  115. deltacat/{storage → experimental/storage}/rivulet/fs/output_file.py +1 -1
  116. deltacat/{storage → experimental/storage}/rivulet/logical_plan.py +4 -4
  117. deltacat/experimental/storage/rivulet/metastore/__init__.py +0 -0
  118. deltacat/{storage → experimental/storage}/rivulet/metastore/delta.py +1 -3
  119. deltacat/{storage → experimental/storage}/rivulet/metastore/json_sst.py +3 -3
  120. deltacat/{storage → experimental/storage}/rivulet/metastore/sst.py +2 -2
  121. deltacat/{storage → experimental/storage}/rivulet/metastore/sst_interval_tree.py +3 -3
  122. deltacat/experimental/storage/rivulet/parquet/__init__.py +7 -0
  123. deltacat/experimental/storage/rivulet/parquet/data_reader.py +0 -0
  124. deltacat/{storage → experimental/storage}/rivulet/parquet/file_reader.py +7 -5
  125. deltacat/{storage → experimental/storage}/rivulet/parquet/serializer.py +4 -4
  126. deltacat/experimental/storage/rivulet/reader/__init__.py +0 -0
  127. deltacat/{storage → experimental/storage}/rivulet/reader/block_scanner.py +20 -9
  128. deltacat/{storage → experimental/storage}/rivulet/reader/data_reader.py +3 -3
  129. deltacat/{storage → experimental/storage}/rivulet/reader/data_scan.py +5 -3
  130. deltacat/{storage → experimental/storage}/rivulet/reader/dataset_metastore.py +7 -6
  131. deltacat/{storage → experimental/storage}/rivulet/reader/dataset_reader.py +8 -6
  132. deltacat/{storage → experimental/storage}/rivulet/reader/pyarrow_data_reader.py +4 -1
  133. deltacat/{storage → experimental/storage}/rivulet/reader/reader_type_registrar.py +4 -4
  134. deltacat/experimental/storage/rivulet/schema/__init__.py +0 -0
  135. deltacat/{storage → experimental/storage}/rivulet/schema/schema.py +1 -1
  136. deltacat/{storage → experimental/storage}/rivulet/serializer.py +1 -1
  137. deltacat/{storage → experimental/storage}/rivulet/serializer_factory.py +9 -5
  138. deltacat/experimental/storage/rivulet/shard/__init__.py +0 -0
  139. deltacat/experimental/storage/rivulet/shard/range_shard.py +129 -0
  140. deltacat/experimental/storage/rivulet/writer/__init__.py +0 -0
  141. deltacat/{storage → experimental/storage}/rivulet/writer/memtable_dataset_writer.py +20 -9
  142. deltacat/io/__init__.py +13 -0
  143. deltacat/io/dataset/__init__.py +0 -0
  144. deltacat/io/dataset/deltacat_dataset.py +91 -0
  145. deltacat/io/datasink/__init__.py +0 -0
  146. deltacat/io/datasink/deltacat_datasink.py +207 -0
  147. deltacat/io/datasource/__init__.py +0 -0
  148. deltacat/io/datasource/deltacat_datasource.py +579 -0
  149. deltacat/io/reader/__init__.py +0 -0
  150. deltacat/io/reader/deltacat_read_api.py +172 -0
  151. deltacat/storage/__init__.py +22 -2
  152. deltacat/storage/interface.py +54 -32
  153. deltacat/storage/main/impl.py +1494 -541
  154. deltacat/storage/model/delta.py +27 -3
  155. deltacat/storage/model/expression/__init__.py +47 -0
  156. deltacat/storage/model/expression/expression.py +656 -0
  157. deltacat/storage/model/expression/visitor.py +248 -0
  158. deltacat/storage/model/locator.py +6 -12
  159. deltacat/storage/model/manifest.py +231 -6
  160. deltacat/storage/model/metafile.py +224 -119
  161. deltacat/storage/model/namespace.py +8 -1
  162. deltacat/storage/model/partition.py +117 -42
  163. deltacat/storage/model/scan/push_down.py +32 -5
  164. deltacat/storage/model/schema.py +2427 -159
  165. deltacat/storage/model/shard.py +6 -2
  166. deltacat/storage/model/sort_key.py +40 -0
  167. deltacat/storage/model/stream.py +9 -2
  168. deltacat/storage/model/table.py +12 -1
  169. deltacat/storage/model/table_version.py +11 -0
  170. deltacat/storage/model/transaction.py +1184 -208
  171. deltacat/storage/model/transform.py +81 -2
  172. deltacat/storage/model/types.py +53 -29
  173. deltacat/storage/util/__init__.py +0 -0
  174. deltacat/storage/util/scan_planner.py +26 -0
  175. deltacat/tests/_io/reader/__init__.py +0 -0
  176. deltacat/tests/_io/reader/test_deltacat_read_api.py +0 -0
  177. deltacat/tests/_io/test_cloudpickle_bug_fix.py +8 -4
  178. deltacat/tests/aws/test_s3u.py +2 -31
  179. deltacat/tests/catalog/data/__init__.py +0 -0
  180. deltacat/tests/catalog/main/__init__.py +0 -0
  181. deltacat/tests/catalog/main/test_catalog_impl_namespace_operations.py +130 -0
  182. deltacat/tests/catalog/main/test_catalog_impl_table_operations.py +1972 -0
  183. deltacat/tests/catalog/model/__init__.py +0 -0
  184. deltacat/tests/catalog/model/test_table_definition.py +16 -0
  185. deltacat/tests/catalog/test_catalogs.py +103 -106
  186. deltacat/tests/catalog/test_default_catalog_impl.py +12152 -72
  187. deltacat/tests/compute/compact_partition_test_cases.py +35 -8
  188. deltacat/tests/compute/compactor/steps/test_repartition.py +12 -12
  189. deltacat/tests/compute/compactor/utils/test_io.py +124 -120
  190. deltacat/tests/compute/compactor/utils/test_round_completion_reader.py +254 -0
  191. deltacat/tests/compute/compactor_v2/test_compaction_session.py +423 -312
  192. deltacat/tests/compute/compactor_v2/utils/test_content_type_params.py +266 -0
  193. deltacat/tests/compute/compactor_v2/utils/test_primary_key_index.py +45 -0
  194. deltacat/tests/compute/compactor_v2/utils/test_task_options.py +270 -1
  195. deltacat/tests/compute/conftest.py +8 -44
  196. deltacat/tests/compute/converter/test_convert_session.py +697 -349
  197. deltacat/tests/compute/converter/utils.py +15 -6
  198. deltacat/tests/compute/resource_estimation/test_delta.py +145 -79
  199. deltacat/tests/compute/test_compact_partition_incremental.py +103 -70
  200. deltacat/tests/compute/test_compact_partition_multiple_rounds.py +89 -66
  201. deltacat/tests/compute/test_compact_partition_params.py +13 -8
  202. deltacat/tests/compute/test_compact_partition_rebase.py +77 -62
  203. deltacat/tests/compute/test_compact_partition_rebase_then_incremental.py +263 -193
  204. deltacat/tests/compute/test_janitor.py +236 -0
  205. deltacat/tests/compute/test_util_common.py +716 -43
  206. deltacat/tests/compute/test_util_constant.py +0 -1
  207. deltacat/tests/{storage/conftest.py → conftest.py} +1 -1
  208. deltacat/tests/daft/__init__.py +0 -0
  209. deltacat/tests/daft/test_model.py +97 -0
  210. deltacat/tests/experimental/__init__.py +1 -0
  211. deltacat/tests/experimental/catalog/__init__.py +0 -0
  212. deltacat/tests/experimental/catalog/iceberg/__init__.py +0 -0
  213. deltacat/tests/experimental/catalog/iceberg/test_iceberg_catalog.py +71 -0
  214. deltacat/tests/experimental/compatibility/__init__.py +1 -0
  215. deltacat/tests/experimental/compatibility/test_backfill_locator_to_id_mappings.py +582 -0
  216. deltacat/tests/experimental/daft/__init__.py +0 -0
  217. deltacat/tests/experimental/daft/test_deltacat_daft_integration.py +136 -0
  218. deltacat/tests/experimental/storage/__init__.py +0 -0
  219. deltacat/tests/experimental/storage/rivulet/__init__.py +0 -0
  220. deltacat/tests/{storage → experimental/storage}/rivulet/conftest.py +3 -3
  221. deltacat/tests/experimental/storage/rivulet/fs/__init__.py +0 -0
  222. deltacat/tests/{storage → experimental/storage}/rivulet/fs/test_file_location_provider.py +3 -2
  223. deltacat/tests/experimental/storage/rivulet/reader/__init__.py +0 -0
  224. deltacat/tests/experimental/storage/rivulet/reader/query_expression.py +80 -0
  225. deltacat/tests/experimental/storage/rivulet/reader/test_data_scan.py +119 -0
  226. deltacat/tests/experimental/storage/rivulet/reader/test_dataset_metastore.py +71 -0
  227. deltacat/tests/experimental/storage/rivulet/schema/__init__.py +0 -0
  228. deltacat/tests/{storage → experimental/storage}/rivulet/schema/test_schema.py +1 -1
  229. deltacat/tests/experimental/storage/rivulet/shard/__init__.py +0 -0
  230. deltacat/tests/experimental/storage/rivulet/shard/test_range_shard.py +162 -0
  231. deltacat/tests/{storage → experimental/storage}/rivulet/test_dataset.py +6 -4
  232. deltacat/tests/{storage → experimental/storage}/rivulet/test_manifest.py +5 -5
  233. deltacat/tests/{storage → experimental/storage}/rivulet/test_sst_interval_tree.py +5 -5
  234. deltacat/tests/{storage → experimental/storage}/rivulet/test_utils.py +8 -6
  235. deltacat/tests/experimental/storage/rivulet/writer/__init__.py +0 -0
  236. deltacat/tests/{storage → experimental/storage}/rivulet/writer/test_dataset_write_then_read.py +11 -9
  237. deltacat/tests/{storage → experimental/storage}/rivulet/writer/test_dataset_writer.py +2 -2
  238. deltacat/tests/{storage → experimental/storage}/rivulet/writer/test_memtable_dataset_writer.py +7 -7
  239. deltacat/tests/storage/main/test_main_storage.py +6900 -95
  240. deltacat/tests/storage/model/test_expression.py +327 -0
  241. deltacat/tests/storage/model/test_manifest.py +129 -0
  242. deltacat/tests/storage/model/test_metafile_io.py +78 -173
  243. deltacat/tests/storage/model/test_partition_scheme.py +85 -0
  244. deltacat/tests/storage/model/test_schema.py +171 -0
  245. deltacat/tests/storage/model/test_schema_update.py +1925 -0
  246. deltacat/tests/storage/model/test_shard.py +3 -1
  247. deltacat/tests/storage/model/test_sort_scheme.py +90 -0
  248. deltacat/tests/storage/model/test_transaction.py +393 -48
  249. deltacat/tests/storage/model/test_transaction_history.py +886 -0
  250. deltacat/tests/test_deltacat_api.py +1036 -11
  251. deltacat/tests/test_exceptions.py +9 -5
  252. deltacat/tests/test_utils/pyarrow.py +52 -21
  253. deltacat/tests/test_utils/storage.py +23 -34
  254. deltacat/tests/types/__init__.py +0 -0
  255. deltacat/tests/types/test_tables.py +104 -0
  256. deltacat/tests/utils/exceptions.py +22 -0
  257. deltacat/tests/utils/main_deltacat_storage_mock.py +31 -0
  258. deltacat/tests/utils/ray_utils/test_dataset.py +123 -5
  259. deltacat/tests/utils/test_daft.py +121 -31
  260. deltacat/tests/utils/test_numpy.py +1193 -0
  261. deltacat/tests/utils/test_pandas.py +1106 -0
  262. deltacat/tests/utils/test_polars.py +1040 -0
  263. deltacat/tests/utils/test_pyarrow.py +1370 -89
  264. deltacat/types/media.py +345 -37
  265. deltacat/types/tables.py +2344 -46
  266. deltacat/utils/arguments.py +33 -1
  267. deltacat/utils/daft.py +824 -40
  268. deltacat/utils/export.py +3 -1
  269. deltacat/utils/filesystem.py +139 -9
  270. deltacat/utils/metafile_locator.py +2 -1
  271. deltacat/utils/numpy.py +118 -26
  272. deltacat/utils/pandas.py +577 -48
  273. deltacat/utils/polars.py +759 -0
  274. deltacat/utils/pyarrow.py +1373 -192
  275. deltacat/utils/ray_utils/concurrency.py +1 -1
  276. deltacat/utils/ray_utils/dataset.py +101 -10
  277. deltacat/utils/ray_utils/runtime.py +56 -4
  278. deltacat/utils/reader_compatibility_mapping.py +3083 -0
  279. deltacat/utils/url.py +1325 -0
  280. deltacat-2.0.0.dist-info/METADATA +1163 -0
  281. deltacat-2.0.0.dist-info/RECORD +439 -0
  282. {deltacat-2.0.dist-info → deltacat-2.0.0.dist-info}/WHEEL +1 -1
  283. deltacat/catalog/iceberg/__init__.py +0 -4
  284. deltacat/compute/compactor/utils/round_completion_file.py +0 -97
  285. deltacat/compute/merge_on_read/__init__.py +0 -4
  286. deltacat/compute/merge_on_read/daft.py +0 -40
  287. deltacat/compute/merge_on_read/model/merge_on_read_params.py +0 -66
  288. deltacat/compute/merge_on_read/utils/delta.py +0 -42
  289. deltacat/examples/common/fixtures.py +0 -15
  290. deltacat/storage/iceberg/iceberg_scan_planner.py +0 -28
  291. deltacat/storage/rivulet/__init__.py +0 -11
  292. deltacat/storage/rivulet/feather/__init__.py +0 -5
  293. deltacat/storage/rivulet/parquet/__init__.py +0 -5
  294. deltacat/tests/compute/compactor/utils/test_round_completion_file.py +0 -231
  295. deltacat/tests/compute/test_util_create_table_deltas_repo.py +0 -388
  296. deltacat/tests/local_deltacat_storage/__init__.py +0 -1235
  297. deltacat/tests/local_deltacat_storage/exceptions.py +0 -10
  298. deltacat/utils/s3fs.py +0 -21
  299. deltacat-2.0.dist-info/METADATA +0 -65
  300. deltacat-2.0.dist-info/RECORD +0 -347
  301. /deltacat/compute/{merge_on_read/model → jobs}/__init__.py +0 -0
  302. /deltacat/{compute/merge_on_read/utils → docs}/__init__.py +0 -0
  303. /deltacat/{examples/common → docs/autogen}/__init__.py +0 -0
  304. /deltacat/{examples/iceberg → docs/autogen/schema}/__init__.py +0 -0
  305. /deltacat/{storage/iceberg → docs/autogen/schema/inference}/__init__.py +0 -0
  306. /deltacat/{storage/rivulet/arrow → examples/compactor}/__init__.py +0 -0
  307. /deltacat/{storage/rivulet/fs → examples/experimental}/__init__.py +0 -0
  308. /deltacat/{storage/rivulet/metastore → examples/experimental/iceberg}/__init__.py +0 -0
  309. /deltacat/{storage/rivulet/reader → examples/experimental/iceberg/converter}/__init__.py +0 -0
  310. /deltacat/{storage/rivulet/schema → examples/experimental/iceberg/converter/beam}/__init__.py +0 -0
  311. /deltacat/{storage/rivulet/writer → examples/indexer}/__init__.py +0 -0
  312. /deltacat/{tests/storage/rivulet → examples/indexer/aws}/__init__.py +0 -0
  313. /deltacat/{tests/storage/rivulet/fs → examples/indexer/gcp}/__init__.py +0 -0
  314. /deltacat/{tests/storage/rivulet/schema → experimental}/__init__.py +0 -0
  315. /deltacat/{tests/storage/rivulet/writer → experimental/catalog}/__init__.py +0 -0
  316. /deltacat/{storage/rivulet/parquet/data_reader.py → experimental/compatibility/__init__.py} +0 -0
  317. /deltacat/{storage → experimental/storage}/rivulet/fs/input_file.py +0 -0
  318. /deltacat/{storage → experimental/storage}/rivulet/mvp/Table.py +0 -0
  319. /deltacat/{storage → experimental/storage}/rivulet/mvp/__init__.py +0 -0
  320. /deltacat/{storage → experimental/storage}/rivulet/reader/query_expression.py +0 -0
  321. /deltacat/{storage → experimental/storage}/rivulet/schema/datatype.py +0 -0
  322. /deltacat/{storage → experimental/storage}/rivulet/writer/dataset_writer.py +0 -0
  323. {deltacat-2.0.dist-info → deltacat-2.0.0.dist-info/licenses}/LICENSE +0 -0
  324. {deltacat-2.0.dist-info → deltacat-2.0.0.dist-info}/top_level.txt +0 -0
@@ -2,13 +2,16 @@ from __future__ import annotations
2
2
 
3
3
  from typing import List, Callable, Any
4
4
 
5
- from deltacat.storage.rivulet.field_group import FieldGroup
6
- from deltacat.storage.rivulet.mvp.Table import MvpTable
7
- from deltacat.storage.rivulet import Schema
8
- from deltacat.storage.rivulet.reader.data_scan import DataScan
9
- from deltacat.storage.rivulet.reader.dataset_metastore import DatasetMetastore
10
- from deltacat.storage.rivulet.reader.dataset_reader import DatasetReader
11
- from deltacat.storage.rivulet.reader.query_expression import QueryExpression
5
+ from deltacat.experimental.storage.rivulet.mvp.Table import MvpTable
6
+ from deltacat.experimental.storage.rivulet import Schema
7
+ from deltacat.experimental.storage.rivulet.reader.data_scan import DataScan
8
+ from deltacat.experimental.storage.rivulet.reader.dataset_metastore import (
9
+ DatasetMetastore,
10
+ )
11
+ from deltacat.experimental.storage.rivulet.reader.dataset_reader import DatasetReader
12
+ from deltacat.experimental.storage.rivulet.reader.query_expression import (
13
+ QueryExpression,
14
+ )
12
15
 
13
16
 
14
17
  class DatasetExecutor:
@@ -22,12 +25,10 @@ class DatasetExecutor:
22
25
 
23
26
  def __init__(
24
27
  self,
25
- field_groups: List[FieldGroup],
26
28
  schema: Schema,
27
29
  metastore: DatasetMetastore,
28
30
  ):
29
31
  self.effective_schema: Schema = schema.__deepcopy__()
30
- self.field_groups = field_groups
31
32
  self.output: MvpTable | None = None
32
33
  self._metastore = metastore
33
34
 
@@ -64,18 +65,9 @@ class DatasetExecutor:
64
65
 
65
66
  TODO for now this is doing dumb in-memory implementation and later this is going to be replaced by rust library
66
67
  """
67
- if len(self.field_groups) == 1:
68
- return self._read_as_mvp_table(schema, self.field_groups[0])
69
- else:
70
- ds1 = self._read_as_mvp_table(schema, self.field_groups[0])
71
- ds2 = self._read_as_mvp_table(schema, self.field_groups[1])
72
- merged = MvpTable.merge(ds1, ds2, schema.primary_key.name)
73
- for i in range(2, len(self.field_groups)):
74
- ds_i = self._read_as_mvp_table(schema, self.field_groups[i])
75
- merged = MvpTable.merge(merged, ds_i, schema.primary_key.name)
76
- return merged
68
+ return self._read_as_mvp_table(schema)
77
69
 
78
- def _read_as_mvp_table(self, schema: Schema, field_group: FieldGroup):
70
+ def _read_as_mvp_table(self, schema: Schema):
79
71
  data = list(
80
72
  DataScan(
81
73
  schema, QueryExpression(), DatasetReader(self._metastore)
@@ -0,0 +1,7 @@
1
+ # TODO later on this will be moved to a dedicated package
2
+ from deltacat.experimental.storage.rivulet.feather.file_reader import FeatherFileReader
3
+ from deltacat.experimental.storage.rivulet.reader.reader_type_registrar import (
4
+ FileReaderRegistrar,
5
+ )
6
+
7
+ FileReaderRegistrar.register_reader("feather", FeatherFileReader)
@@ -5,15 +5,17 @@ from typing import Optional
5
5
  import pyarrow.ipc
6
6
  from pyarrow import RecordBatch, RecordBatchFileReader
7
7
 
8
- from deltacat.storage.rivulet.fs.file_provider import FileProvider
9
- from deltacat.storage.rivulet.metastore.sst import SSTableRow
10
- from deltacat.storage.rivulet.reader.data_reader import (
8
+ from deltacat.experimental.storage.rivulet.fs.file_provider import FileProvider
9
+ from deltacat.experimental.storage.rivulet.metastore.sst import SSTableRow
10
+ from deltacat.experimental.storage.rivulet.reader.data_reader import (
11
11
  RowAndKey,
12
12
  FileReader,
13
13
  FILE_FORMAT,
14
14
  )
15
- from deltacat.storage.rivulet.reader.pyarrow_data_reader import RecordBatchRowIndex
16
- from deltacat.storage.rivulet.schema.schema import Schema
15
+ from deltacat.experimental.storage.rivulet.reader.pyarrow_data_reader import (
16
+ RecordBatchRowIndex,
17
+ )
18
+ from deltacat.experimental.storage.rivulet.schema.schema import Schema
17
19
 
18
20
 
19
21
  class FeatherFileReader(FileReader[RecordBatchRowIndex]):
@@ -3,10 +3,10 @@ from typing import List
3
3
  import pyarrow as pa
4
4
  from pyarrow import feather
5
5
 
6
- from deltacat.storage.rivulet.metastore.sst import SSTableRow
7
- from deltacat.storage.rivulet import Schema
8
- from deltacat.storage.rivulet.arrow.serializer import ArrowSerializer
9
- from deltacat.storage.rivulet.fs.file_provider import FileProvider
6
+ from deltacat.experimental.storage.rivulet.metastore.sst import SSTableRow
7
+ from deltacat.experimental.storage.rivulet import Schema
8
+ from deltacat.experimental.storage.rivulet.arrow.serializer import ArrowSerializer
9
+ from deltacat.experimental.storage.rivulet.fs.file_provider import FileProvider
10
10
 
11
11
 
12
12
  class FeatherDataSerializer(ArrowSerializer):
File without changes
@@ -3,9 +3,9 @@ import time
3
3
  from typing import List, Generator
4
4
 
5
5
  from deltacat.storage.model.partition import PartitionLocator
6
- from deltacat.storage.rivulet.fs.file_store import FileStore
7
- from deltacat.storage.rivulet.fs.input_file import InputFile
8
- from deltacat.storage.rivulet.fs.output_file import OutputFile
6
+ from deltacat.experimental.storage.rivulet.fs.file_store import FileStore
7
+ from deltacat.experimental.storage.rivulet.fs.input_file import InputFile
8
+ from deltacat.experimental.storage.rivulet.fs.output_file import OutputFile
9
9
  from deltacat.utils.metafile_locator import _find_partition_path
10
10
 
11
11
 
@@ -4,8 +4,8 @@ from pyarrow.fs import FileSystem, FileType, FileSelector
4
4
  # TODO(deltacat): Rely on deltacat implementation to resolve path and filesystem.
5
5
  from ray.data.datasource.path_util import _resolve_paths_and_filesystem
6
6
 
7
- from deltacat.storage.rivulet.fs.input_file import FSInputFile
8
- from deltacat.storage.rivulet.fs.output_file import FSOutputFile
7
+ from deltacat.experimental.storage.rivulet.fs.input_file import FSInputFile
8
+ from deltacat.experimental.storage.rivulet.fs.output_file import FSOutputFile
9
9
 
10
10
 
11
11
  class FileStore:
@@ -5,7 +5,7 @@ from typing import Protocol
5
5
 
6
6
  from pyarrow.fs import FileSystem, FileType
7
7
 
8
- from deltacat.storage.rivulet.fs.input_file import FSInputFile, InputFile
8
+ from deltacat.experimental.storage.rivulet.fs.input_file import FSInputFile, InputFile
9
9
 
10
10
 
11
11
  class OutputStream(Protocol): # pragma: no cover
@@ -1,9 +1,9 @@
1
1
  from dataclasses import dataclass
2
2
  from typing import List, Callable, Any, Protocol
3
3
 
4
- from deltacat.storage.rivulet.dataset_executor import DatasetExecutor
5
- from deltacat.storage.rivulet.mvp.Table import MvpTable
6
- from deltacat.storage.rivulet import Schema
4
+ from deltacat.experimental.storage.rivulet.dataset_executor import DatasetExecutor
5
+ from deltacat.experimental.storage.rivulet import Schema
6
+ from deltacat.experimental.storage.rivulet.mvp.Table import MvpTable
7
7
 
8
8
 
9
9
  class DatasetOperation(Protocol):
@@ -99,7 +99,7 @@ class LogicalPlan:
99
99
  self.operations.append(CollectOperation())
100
100
  return self
101
101
 
102
- def execute(self, executor: DatasetExecutor) -> "MvpTable":
102
+ def execute(self, executor: DatasetExecutor) -> MvpTable:
103
103
  for operation in self.operations:
104
104
  operation.visit(executor)
105
105
  return executor.output
@@ -10,7 +10,6 @@ from deltacat.storage import (
10
10
  Delta,
11
11
  DeltaType,
12
12
  Transaction,
13
- TransactionType,
14
13
  TransactionOperation,
15
14
  TransactionOperationType,
16
15
  )
@@ -19,7 +18,7 @@ from deltacat.storage.model.partition import PartitionLocator
19
18
  from deltacat.storage.model.transaction import TransactionOperationList
20
19
 
21
20
  from deltacat.storage.model.types import StreamFormat
22
- from deltacat.storage.rivulet import Schema
21
+ from deltacat.experimental.storage.rivulet import Schema
23
22
 
24
23
  StreamPosition = int
25
24
  """The stream position for creating a consistent ordering of manifests."""
@@ -169,7 +168,6 @@ class DeltacatManifestIO(ManifestIO):
169
168
  delta["level"] = level
170
169
 
171
170
  tx_results = Transaction.of(
172
- txn_type=TransactionType.APPEND,
173
171
  txn_operations=TransactionOperationList.of(
174
172
  [
175
173
  TransactionOperation.of(
@@ -4,9 +4,9 @@ import json
4
4
  from itertools import zip_longest
5
5
  from typing import List
6
6
 
7
- from deltacat.storage.rivulet.fs.input_file import InputFile
8
- from deltacat.storage.rivulet.fs.output_file import OutputFile
9
- from deltacat.storage.rivulet.metastore.sst import (
7
+ from deltacat.experimental.storage.rivulet.fs.input_file import InputFile
8
+ from deltacat.experimental.storage.rivulet.fs.output_file import OutputFile
9
+ from deltacat.experimental.storage.rivulet.metastore.sst import (
10
10
  SSTWriter,
11
11
  SSTableRow,
12
12
  SSTReader,
@@ -1,8 +1,8 @@
1
1
  from dataclasses import dataclass
2
2
  from typing import Protocol, Any, List
3
3
 
4
- from deltacat.storage.rivulet.fs.input_file import InputFile
5
- from deltacat.storage.rivulet.fs.output_file import OutputFile
4
+ from deltacat.experimental.storage.rivulet.fs.input_file import InputFile
5
+ from deltacat.experimental.storage.rivulet.fs.output_file import OutputFile
6
6
 
7
7
 
8
8
  @dataclass(frozen=True)
@@ -8,9 +8,9 @@ from typing import Any, Dict, Set, List, FrozenSet, Iterable, TypeVar, NamedTupl
8
8
 
9
9
  from intervaltree import Interval, IntervalTree
10
10
 
11
- from deltacat.storage.rivulet.metastore.delta import DeltaContext
12
- from deltacat.storage.rivulet.metastore.sst import SSTable, SSTableRow
13
- from deltacat.storage.rivulet import Schema
11
+ from deltacat.experimental.storage.rivulet.metastore.delta import DeltaContext
12
+ from deltacat.experimental.storage.rivulet.metastore.sst import SSTable, SSTableRow
13
+ from deltacat.experimental.storage.rivulet import Schema
14
14
 
15
15
  T = TypeVar("T")
16
16
 
@@ -0,0 +1,7 @@
1
+ # TODO later on this will be moved to a dedicated package
2
+ from deltacat.experimental.storage.rivulet.parquet.file_reader import ParquetFileReader
3
+ from deltacat.experimental.storage.rivulet.reader.reader_type_registrar import (
4
+ FileReaderRegistrar,
5
+ )
6
+
7
+ FileReaderRegistrar.register_reader("parquet", ParquetFileReader)
@@ -4,15 +4,17 @@ from typing import Optional
4
4
 
5
5
  from pyarrow import RecordBatch
6
6
 
7
- from deltacat.storage.rivulet.fs.file_provider import FileProvider
8
- from deltacat.storage.rivulet.metastore.sst import SSTableRow
9
- from deltacat.storage.rivulet.reader.data_reader import (
7
+ from deltacat.experimental.storage.rivulet.fs.file_provider import FileProvider
8
+ from deltacat.experimental.storage.rivulet.metastore.sst import SSTableRow
9
+ from deltacat.experimental.storage.rivulet.reader.data_reader import (
10
10
  RowAndKey,
11
11
  FileReader,
12
12
  FILE_FORMAT,
13
13
  )
14
- from deltacat.storage.rivulet.reader.pyarrow_data_reader import RecordBatchRowIndex
15
- from deltacat.storage.rivulet.schema.schema import Schema
14
+ from deltacat.experimental.storage.rivulet.reader.pyarrow_data_reader import (
15
+ RecordBatchRowIndex,
16
+ )
17
+ from deltacat.experimental.storage.rivulet.schema.schema import Schema
16
18
  import pyarrow.parquet as pq
17
19
  import pyarrow as pa
18
20
 
@@ -3,11 +3,11 @@ from typing import List, Any
3
3
  import pyarrow as pa
4
4
  from pyarrow.parquet import FileMetaData
5
5
 
6
- from deltacat.storage.rivulet.metastore.sst import SSTableRow
7
- from deltacat.storage.rivulet import Schema
8
- from deltacat.storage.rivulet.arrow.serializer import ArrowSerializer
6
+ from deltacat.experimental.storage.rivulet.metastore.sst import SSTableRow
7
+ from deltacat.experimental.storage.rivulet import Schema
8
+ from deltacat.experimental.storage.rivulet.arrow.serializer import ArrowSerializer
9
9
 
10
- from deltacat.storage.rivulet.fs.file_provider import FileProvider
10
+ from deltacat.experimental.storage.rivulet.fs.file_provider import FileProvider
11
11
 
12
12
 
13
13
  class ParquetDataSerializer(ArrowSerializer):
@@ -15,19 +15,30 @@ from typing import (
15
15
  AbstractSet,
16
16
  )
17
17
 
18
- from deltacat.storage.rivulet.metastore.delta import DeltaContext
19
- from deltacat.storage.rivulet.metastore.sst import SSTableRow
20
- from deltacat.storage.rivulet.metastore.sst_interval_tree import (
18
+ from deltacat.experimental.storage.rivulet.metastore.delta import DeltaContext
19
+ from deltacat.experimental.storage.rivulet.metastore.sst import SSTableRow
20
+ from deltacat.experimental.storage.rivulet.metastore.sst_interval_tree import (
21
21
  OrderedBlockGroups,
22
22
  BlockGroup,
23
23
  Block,
24
24
  )
25
- from deltacat.storage.rivulet.reader.data_reader import RowAndKey, FileReader
26
- from deltacat.storage.rivulet.reader.dataset_metastore import DatasetMetastore
27
- from deltacat.storage.rivulet.reader.pyarrow_data_reader import ArrowDataReader
28
- from deltacat.storage.rivulet.reader.query_expression import QueryExpression
29
- from deltacat.storage.rivulet.reader.reader_type_registrar import FileReaderRegistrar
30
- from deltacat.storage.rivulet import Schema
25
+ from deltacat.experimental.storage.rivulet.reader.data_reader import (
26
+ RowAndKey,
27
+ FileReader,
28
+ )
29
+ from deltacat.experimental.storage.rivulet.reader.dataset_metastore import (
30
+ DatasetMetastore,
31
+ )
32
+ from deltacat.experimental.storage.rivulet.reader.pyarrow_data_reader import (
33
+ ArrowDataReader,
34
+ )
35
+ from deltacat.experimental.storage.rivulet.reader.query_expression import (
36
+ QueryExpression,
37
+ )
38
+ from deltacat.experimental.storage.rivulet.reader.reader_type_registrar import (
39
+ FileReaderRegistrar,
40
+ )
41
+ from deltacat.experimental.storage.rivulet import Schema
31
42
  from deltacat import logs
32
43
 
33
44
  logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
@@ -13,9 +13,9 @@ from typing import (
13
13
  Optional,
14
14
  )
15
15
 
16
- from deltacat.storage.rivulet.fs.file_provider import FileProvider
17
- from deltacat.storage.rivulet.metastore.sst import SSTableRow
18
- from deltacat.storage.rivulet.schema.schema import Schema
16
+ from deltacat.experimental.storage.rivulet.fs.file_provider import FileProvider
17
+ from deltacat.experimental.storage.rivulet.metastore.sst import SSTableRow
18
+ from deltacat.experimental.storage.rivulet.schema.schema import Schema
19
19
 
20
20
  FILE_FORMAT = TypeVar("FILE_FORMAT")
21
21
  MEMORY_FORMAT = TypeVar("MEMORY_FORMAT")
@@ -3,9 +3,11 @@ from typing import Generator, Dict, Optional
3
3
  import pyarrow as pa
4
4
 
5
5
  from deltacat.storage.model.shard import Shard
6
- from deltacat.storage.rivulet.reader.dataset_reader import DatasetReader
7
- from deltacat.storage.rivulet.reader.query_expression import QueryExpression
8
- from deltacat.storage.rivulet import Schema
6
+ from deltacat.experimental.storage.rivulet.reader.dataset_reader import DatasetReader
7
+ from deltacat.experimental.storage.rivulet.reader.query_expression import (
8
+ QueryExpression,
9
+ )
10
+ from deltacat.experimental.storage.rivulet import Schema
9
11
 
10
12
 
11
13
  class DataScan:
@@ -5,18 +5,19 @@ from typing import Generator, Optional
5
5
  import pyarrow
6
6
  import pyarrow.fs
7
7
 
8
+ from deltacat.constants import REV_DIR_NAME
8
9
  from deltacat.storage import Delta
9
10
  from deltacat.storage.model.partition import PartitionLocator
10
- from deltacat.storage.rivulet.fs.file_provider import FileProvider
11
+ from deltacat.experimental.storage.rivulet.fs.file_provider import FileProvider
11
12
  from deltacat.utils.filesystem import resolve_path_and_filesystem
12
- from deltacat.storage.rivulet.metastore.json_sst import JsonSstReader
13
- from deltacat.storage.rivulet.metastore.delta import (
13
+ from deltacat.experimental.storage.rivulet.metastore.json_sst import JsonSstReader
14
+ from deltacat.experimental.storage.rivulet.metastore.delta import (
14
15
  ManifestIO,
15
16
  DeltaContext,
16
17
  RivuletDelta,
17
18
  DeltacatManifestIO,
18
19
  )
19
- from deltacat.storage.rivulet.metastore.sst import SSTReader, SSTable
20
+ from deltacat.experimental.storage.rivulet.metastore.sst import SSTReader, SSTable
20
21
  from deltacat.utils.metafile_locator import _find_table_path
21
22
  from deltacat import logs
22
23
 
@@ -83,7 +84,7 @@ class DatasetMetastore:
83
84
  param: filesystem: The filesystem to search for the revisions.
84
85
  returns: The latest revision as a RivuletDelta.
85
86
  """
86
- rev_directory = posixpath.join(delta_dir, "rev")
87
+ rev_directory = posixpath.join(delta_dir, REV_DIR_NAME)
87
88
  revisions = filesystem.get_file_info(
88
89
  pyarrow.fs.FileSelector(rev_directory, allow_not_found=True)
89
90
  )
@@ -128,7 +129,7 @@ class DatasetMetastore:
128
129
  return
129
130
 
130
131
  # Locate "rev" directory inside the partition
131
- rev_directory = posixpath.join(partition_path, "rev")
132
+ rev_directory = posixpath.join(partition_path, REV_DIR_NAME)
132
133
  rev_info = filesystem.get_file_info(rev_directory)
133
134
 
134
135
  if rev_info.type != pyarrow.fs.FileType.Directory:
@@ -2,18 +2,20 @@ import logging
2
2
  from typing import Generator, Optional, Set, Type, TypeVar, Any
3
3
 
4
4
  from deltacat.storage.model.shard import Shard
5
- from deltacat.storage.rivulet.metastore.sst import SSTableRow, SSTable
6
- from deltacat.storage.rivulet.metastore.sst_interval_tree import (
5
+ from deltacat.experimental.storage.rivulet.metastore.sst import SSTableRow, SSTable
6
+ from deltacat.experimental.storage.rivulet.metastore.sst_interval_tree import (
7
7
  BlockIntervalTree,
8
8
  OrderedBlockGroups,
9
9
  )
10
- from deltacat.storage.rivulet.reader.block_scanner import BlockScanner
11
- from deltacat.storage.rivulet.reader.dataset_metastore import (
10
+ from deltacat.experimental.storage.rivulet.reader.block_scanner import BlockScanner
11
+ from deltacat.experimental.storage.rivulet.reader.dataset_metastore import (
12
12
  DatasetMetastore,
13
13
  ManifestAccessor,
14
14
  )
15
- from deltacat.storage.rivulet.reader.query_expression import QueryExpression
16
- from deltacat.storage.rivulet import Schema
15
+ from deltacat.experimental.storage.rivulet.reader.query_expression import (
16
+ QueryExpression,
17
+ )
18
+ from deltacat.experimental.storage.rivulet import Schema
17
19
 
18
20
  # The type of data returned to reader
19
21
  T = TypeVar("T")
@@ -4,7 +4,10 @@ from typing import Generator, Dict, Type, NamedTuple, List
4
4
 
5
5
  from pyarrow import RecordBatch
6
6
 
7
- from deltacat.storage.rivulet.reader.data_reader import DataReader, MEMORY_FORMAT
7
+ from deltacat.experimental.storage.rivulet.reader.data_reader import (
8
+ DataReader,
9
+ MEMORY_FORMAT,
10
+ )
8
11
  import pyarrow as pa
9
12
 
10
13
 
@@ -1,9 +1,9 @@
1
- from deltacat.storage.rivulet.fs.file_provider import FileProvider
2
- from deltacat.storage.rivulet.metastore.sst import SSTableRow
3
- from deltacat.storage.rivulet.reader.data_reader import FileReader
1
+ from deltacat.experimental.storage.rivulet.fs.file_provider import FileProvider
2
+ from deltacat.experimental.storage.rivulet.metastore.sst import SSTableRow
3
+ from deltacat.experimental.storage.rivulet.reader.data_reader import FileReader
4
4
  from typing import Type, Dict
5
5
 
6
- from deltacat.storage.rivulet.schema.schema import Schema
6
+ from deltacat.experimental.storage.rivulet.schema.schema import Schema
7
7
 
8
8
 
9
9
  class FileReaderRegistrar:
@@ -5,7 +5,7 @@ from typing import MutableMapping, Dict, Iterable, Tuple, Optional
5
5
 
6
6
  import pyarrow as pa
7
7
 
8
- from deltacat.storage.rivulet.schema.datatype import Datatype
8
+ from deltacat.experimental.storage.rivulet.schema.datatype import Datatype
9
9
 
10
10
 
11
11
  @dataclass(frozen=True)
@@ -1,6 +1,6 @@
1
1
  from typing import Protocol, Iterable, List, Union, Any, Dict
2
2
 
3
- from deltacat.storage.rivulet.metastore.sst import SSTableRow
3
+ from deltacat.experimental.storage.rivulet.metastore.sst import SSTableRow
4
4
  import pyarrow as pa
5
5
 
6
6
  MEMTABLE_DATA = Union[Iterable[Dict[str, Any]], pa.Table]
@@ -1,11 +1,15 @@
1
1
  from __future__ import annotations
2
2
 
3
- from deltacat.storage.rivulet.parquet.serializer import ParquetDataSerializer
4
- from deltacat.storage.rivulet import Schema
5
- from deltacat.storage.rivulet.serializer import DataSerializer
6
- from deltacat.storage.rivulet.fs.file_provider import FileProvider
3
+ from deltacat.experimental.storage.rivulet.parquet.serializer import (
4
+ ParquetDataSerializer,
5
+ )
6
+ from deltacat.experimental.storage.rivulet import Schema
7
+ from deltacat.experimental.storage.rivulet.serializer import DataSerializer
8
+ from deltacat.experimental.storage.rivulet.fs.file_provider import FileProvider
7
9
 
8
- from deltacat.storage.rivulet.feather.serializer import FeatherDataSerializer
10
+ from deltacat.experimental.storage.rivulet.feather.serializer import (
11
+ FeatherDataSerializer,
12
+ )
9
13
 
10
14
 
11
15
  class DataSerializerFactory:
@@ -0,0 +1,129 @@
1
+ from __future__ import annotations
2
+ from typing import Generic, List, Union, Iterable
3
+ from deltacat.storage.model.shard import T, Shard, ShardingStrategy
4
+ from deltacat.experimental.storage.rivulet.reader.dataset_metastore import (
5
+ DatasetMetastore,
6
+ )
7
+
8
+
9
+ class RangeShard(Shard, Generic[T]):
10
+ """
11
+ Represents a range-based shard with minimum and maximum keys.
12
+
13
+ param: min_key: The minimum key for the shard.
14
+ param: max_key: The maximum key for the shard.
15
+ """
16
+
17
+ def __init__(self, min_key: T, max_key: T):
18
+ self.min_key = min_key
19
+ self.max_key = max_key
20
+
21
+ def __repr__(self) -> str:
22
+ return f"Shard(type=range, min_key={self.min_key}, max_key={self.max_key})"
23
+
24
+ @staticmethod
25
+ def split(
26
+ global_min: Union[int, str], global_max: Union[int, str], num_shards: int
27
+ ) -> List[RangeShard]:
28
+ """
29
+ Splits a range into `num_shards` shards.
30
+ Currently supports splitting ranges of integers and strings.
31
+
32
+ Note: If global_min == global_max or num_shards <= 1, a single shard is returned,
33
+ num_shards is ignored.
34
+
35
+ :param global_min: The minimum key for the entire range (int or str).
36
+ :param global_max: The maximum key for the entire range (int or str).
37
+ :param num_shards: The number of shards to create.
38
+ :return: A list of RangeShard objects.
39
+ """
40
+ if global_min == global_max or num_shards <= 1:
41
+ return [RangeShard(global_min, global_max)]
42
+
43
+ # Determine which interpolation function to use based on the type of min/max
44
+ if isinstance(global_min, int) and isinstance(global_max, int):
45
+ interpolate = RangeShard._interpolate_numeric
46
+ elif isinstance(global_min, str) and isinstance(global_max, str):
47
+ interpolate = RangeShard._interpolate_str
48
+ else:
49
+ raise ValueError(
50
+ "Unsupported combination of types for global_min and global_max."
51
+ )
52
+
53
+ shards: List[RangeShard] = []
54
+ for i in range(num_shards):
55
+ start = interpolate(global_min, global_max, i, num_shards)
56
+ end = interpolate(global_min, global_max, i + 1, num_shards)
57
+
58
+ if i > 0:
59
+ if isinstance(start, int):
60
+ start = shards[-1].max_key + 1
61
+ elif isinstance(start, int):
62
+ char_list = list(start)
63
+ char_list[-1] = chr(ord(char_list[-1]) + 1)
64
+ start = "".join(char_list)
65
+
66
+ shards.append(RangeShard(start, end))
67
+
68
+ return shards
69
+
70
+ @staticmethod
71
+ def _interpolate_numeric(start: int, end: int, step: int, total_steps: int) -> int:
72
+ """
73
+ Integer interpolation using integer (floor) division.
74
+
75
+ param: start (int): The starting number.
76
+ param: end (int): The ending number.
77
+ param: step (int): The current step in the interpolation (0-based).
78
+ param: total_steps (int): The total number of interpolation steps.
79
+
80
+ returns: int: The interpolated integer.
81
+ """
82
+ return start + (end - start) * step // total_steps
83
+
84
+ @staticmethod
85
+ def _interpolate_str(start: str, end: str, step: int, total_steps: int) -> str:
86
+ """
87
+ Interpolates between two strings lexicographically.
88
+
89
+ param: start (str): The starting string.
90
+ param: end (str): The ending string.
91
+ param: step (int): The current step in the interpolation (0-based).
92
+ param: total_steps (int): The total number of interpolation steps.
93
+
94
+ returns: str: The interpolated string.
95
+ """
96
+ max_len = max(len(start), len(end))
97
+
98
+ # Pad strings to the same length with spaces (smallest lexicographical character).
99
+ start = start.ljust(max_len, " ")
100
+ end = end.ljust(max_len, " ")
101
+
102
+ # Interpolate character by character based on ordinal values.
103
+ interpolated_chars = [
104
+ chr(round(ord(s) + (ord(e) - ord(s)) * step / total_steps))
105
+ for s, e in zip(start, end)
106
+ ]
107
+
108
+ return "".join(interpolated_chars).rstrip()
109
+
110
+
111
+ class RangeShardingStrategy(ShardingStrategy, Generic[T]):
112
+ """
113
+ Implements a sharding strategy to divide a range of keys into shards.
114
+
115
+ method: shards: Generates a list of RangeShard objects based on the global range.
116
+ """
117
+
118
+ def shards(
119
+ self, num_shards: int, metastore: DatasetMetastore
120
+ ) -> Iterable[RangeShard[T]]:
121
+ """
122
+ Divides the global range of keys into evenly sized shards.
123
+
124
+ param: num_shards: The number of shards to divide the range into.
125
+ param: metastore: The dataset metastore providing access to manifests.
126
+ returns: A list of RangeShard objects representing the divided range.
127
+ """
128
+ min, max = metastore.get_min_max_keys()
129
+ return RangeShard.split(min, max, num_shards)