deltacat 2.0__py3-none-any.whl → 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- deltacat/__init__.py +117 -18
- deltacat/api.py +536 -126
- deltacat/aws/constants.py +0 -23
- deltacat/aws/s3u.py +4 -631
- deltacat/benchmarking/benchmark_engine.py +4 -2
- deltacat/benchmarking/conftest.py +1 -19
- deltacat/benchmarking/test_benchmark_pipeline.py +6 -4
- deltacat/catalog/__init__.py +64 -5
- deltacat/catalog/delegate.py +445 -63
- deltacat/catalog/interface.py +188 -62
- deltacat/catalog/main/impl.py +2444 -282
- deltacat/catalog/model/catalog.py +208 -113
- deltacat/catalog/model/properties.py +63 -24
- deltacat/compute/__init__.py +14 -0
- deltacat/compute/compactor/compaction_session.py +97 -75
- deltacat/compute/compactor/model/compact_partition_params.py +75 -30
- deltacat/compute/compactor/model/compaction_session_audit_info.py +17 -0
- deltacat/compute/compactor/model/round_completion_info.py +16 -6
- deltacat/compute/compactor/repartition_session.py +8 -21
- deltacat/compute/compactor/steps/hash_bucket.py +5 -5
- deltacat/compute/compactor/steps/materialize.py +9 -7
- deltacat/compute/compactor/steps/repartition.py +12 -11
- deltacat/compute/compactor/utils/io.py +6 -5
- deltacat/compute/compactor/utils/round_completion_reader.py +117 -0
- deltacat/compute/compactor/utils/system_columns.py +3 -1
- deltacat/compute/compactor_v2/compaction_session.py +17 -14
- deltacat/compute/compactor_v2/constants.py +30 -1
- deltacat/compute/compactor_v2/model/evaluate_compaction_result.py +0 -1
- deltacat/compute/compactor_v2/model/hash_bucket_input.py +9 -3
- deltacat/compute/compactor_v2/model/merge_file_group.py +5 -2
- deltacat/compute/compactor_v2/model/merge_input.py +33 -8
- deltacat/compute/compactor_v2/private/compaction_utils.py +167 -68
- deltacat/compute/compactor_v2/steps/hash_bucket.py +5 -2
- deltacat/compute/compactor_v2/steps/merge.py +267 -55
- deltacat/compute/compactor_v2/utils/content_type_params.py +34 -6
- deltacat/compute/compactor_v2/utils/dedupe.py +1 -1
- deltacat/compute/compactor_v2/utils/delta.py +5 -3
- deltacat/compute/compactor_v2/utils/io.py +11 -4
- deltacat/compute/compactor_v2/utils/merge.py +15 -2
- deltacat/compute/compactor_v2/utils/primary_key_index.py +28 -4
- deltacat/compute/compactor_v2/utils/task_options.py +45 -33
- deltacat/compute/converter/constants.py +5 -0
- deltacat/compute/converter/converter_session.py +207 -52
- deltacat/compute/converter/model/convert_input.py +43 -16
- deltacat/compute/converter/model/convert_input_files.py +33 -16
- deltacat/compute/converter/model/convert_result.py +80 -0
- deltacat/compute/converter/model/converter_session_params.py +64 -19
- deltacat/compute/converter/pyiceberg/catalog.py +21 -18
- deltacat/compute/converter/pyiceberg/overrides.py +193 -65
- deltacat/compute/converter/pyiceberg/update_snapshot_overrides.py +148 -100
- deltacat/compute/converter/steps/convert.py +230 -75
- deltacat/compute/converter/steps/dedupe.py +46 -12
- deltacat/compute/converter/utils/convert_task_options.py +66 -22
- deltacat/compute/converter/utils/converter_session_utils.py +126 -60
- deltacat/compute/converter/utils/iceberg_columns.py +13 -8
- deltacat/compute/converter/utils/io.py +173 -13
- deltacat/compute/converter/utils/s3u.py +42 -27
- deltacat/compute/janitor.py +205 -0
- deltacat/compute/jobs/client.py +417 -0
- deltacat/compute/resource_estimation/delta.py +38 -6
- deltacat/compute/resource_estimation/model.py +8 -0
- deltacat/constants.py +49 -6
- deltacat/docs/autogen/schema/inference/generate_type_mappings.py +687 -0
- deltacat/docs/autogen/schema/inference/parse_json_type_mappings.py +673 -0
- deltacat/env.py +10 -0
- deltacat/examples/basic_logging.py +6 -6
- deltacat/examples/compactor/aws/__init__.py +1 -0
- deltacat/examples/compactor/bootstrap.py +863 -0
- deltacat/examples/compactor/compactor.py +373 -0
- deltacat/examples/compactor/explorer.py +473 -0
- deltacat/examples/compactor/gcp/__init__.py +1 -0
- deltacat/examples/compactor/job_runner.py +439 -0
- deltacat/examples/compactor/utils/__init__.py +1 -0
- deltacat/examples/compactor/utils/common.py +261 -0
- deltacat/examples/experimental/iceberg/converter/beam/app.py +226 -0
- deltacat/examples/experimental/iceberg/converter/beam/main.py +133 -0
- deltacat/examples/experimental/iceberg/converter/beam/test_workflow.py +113 -0
- deltacat/examples/experimental/iceberg/converter/beam/utils/__init__.py +3 -0
- deltacat/examples/experimental/iceberg/converter/beam/utils/common.py +174 -0
- deltacat/examples/experimental/iceberg/converter/beam/utils/spark.py +263 -0
- deltacat/examples/{iceberg → experimental/iceberg}/iceberg_bucket_writer.py +66 -21
- deltacat/examples/{iceberg → experimental/iceberg}/iceberg_reader.py +2 -4
- deltacat/examples/hello_world.py +4 -2
- deltacat/examples/indexer/indexer.py +163 -0
- deltacat/examples/indexer/job_runner.py +198 -0
- deltacat/exceptions.py +66 -4
- deltacat/experimental/catalog/iceberg/__init__.py +6 -0
- deltacat/{catalog → experimental/catalog}/iceberg/iceberg_catalog_config.py +1 -1
- deltacat/{catalog → experimental/catalog}/iceberg/impl.py +43 -12
- deltacat/{catalog → experimental/catalog}/iceberg/overrides.py +12 -14
- deltacat/experimental/compatibility/backfill_locator_to_id_mappings.py +201 -0
- deltacat/experimental/converter_agent/__init__.py +0 -0
- deltacat/experimental/converter_agent/beam/__init__.py +0 -0
- deltacat/experimental/converter_agent/beam/managed.py +173 -0
- deltacat/experimental/converter_agent/table_monitor.py +479 -0
- deltacat/experimental/daft/__init__.py +4 -0
- deltacat/experimental/daft/daft_catalog.py +229 -0
- deltacat/experimental/storage/__init__.py +0 -0
- deltacat/experimental/storage/iceberg/__init__.py +0 -0
- deltacat/experimental/storage/iceberg/iceberg_scan_planner.py +129 -0
- deltacat/{storage → experimental/storage}/iceberg/impl.py +6 -4
- deltacat/{storage → experimental/storage}/iceberg/model.py +7 -3
- deltacat/experimental/storage/iceberg/visitor.py +119 -0
- deltacat/experimental/storage/rivulet/__init__.py +11 -0
- deltacat/experimental/storage/rivulet/arrow/__init__.py +0 -0
- deltacat/{storage → experimental/storage}/rivulet/arrow/serializer.py +7 -4
- deltacat/{storage → experimental/storage}/rivulet/dataset.py +13 -12
- deltacat/{storage → experimental/storage}/rivulet/dataset_executor.py +12 -20
- deltacat/experimental/storage/rivulet/feather/__init__.py +7 -0
- deltacat/{storage → experimental/storage}/rivulet/feather/file_reader.py +7 -5
- deltacat/{storage → experimental/storage}/rivulet/feather/serializer.py +4 -4
- deltacat/experimental/storage/rivulet/fs/__init__.py +0 -0
- deltacat/{storage → experimental/storage}/rivulet/fs/file_provider.py +3 -3
- deltacat/{storage → experimental/storage}/rivulet/fs/file_store.py +2 -2
- deltacat/{storage → experimental/storage}/rivulet/fs/output_file.py +1 -1
- deltacat/{storage → experimental/storage}/rivulet/logical_plan.py +4 -4
- deltacat/experimental/storage/rivulet/metastore/__init__.py +0 -0
- deltacat/{storage → experimental/storage}/rivulet/metastore/delta.py +1 -3
- deltacat/{storage → experimental/storage}/rivulet/metastore/json_sst.py +3 -3
- deltacat/{storage → experimental/storage}/rivulet/metastore/sst.py +2 -2
- deltacat/{storage → experimental/storage}/rivulet/metastore/sst_interval_tree.py +3 -3
- deltacat/experimental/storage/rivulet/parquet/__init__.py +7 -0
- deltacat/experimental/storage/rivulet/parquet/data_reader.py +0 -0
- deltacat/{storage → experimental/storage}/rivulet/parquet/file_reader.py +7 -5
- deltacat/{storage → experimental/storage}/rivulet/parquet/serializer.py +4 -4
- deltacat/experimental/storage/rivulet/reader/__init__.py +0 -0
- deltacat/{storage → experimental/storage}/rivulet/reader/block_scanner.py +20 -9
- deltacat/{storage → experimental/storage}/rivulet/reader/data_reader.py +3 -3
- deltacat/{storage → experimental/storage}/rivulet/reader/data_scan.py +5 -3
- deltacat/{storage → experimental/storage}/rivulet/reader/dataset_metastore.py +7 -6
- deltacat/{storage → experimental/storage}/rivulet/reader/dataset_reader.py +8 -6
- deltacat/{storage → experimental/storage}/rivulet/reader/pyarrow_data_reader.py +4 -1
- deltacat/{storage → experimental/storage}/rivulet/reader/reader_type_registrar.py +4 -4
- deltacat/experimental/storage/rivulet/schema/__init__.py +0 -0
- deltacat/{storage → experimental/storage}/rivulet/schema/schema.py +1 -1
- deltacat/{storage → experimental/storage}/rivulet/serializer.py +1 -1
- deltacat/{storage → experimental/storage}/rivulet/serializer_factory.py +9 -5
- deltacat/experimental/storage/rivulet/shard/__init__.py +0 -0
- deltacat/experimental/storage/rivulet/shard/range_shard.py +129 -0
- deltacat/experimental/storage/rivulet/writer/__init__.py +0 -0
- deltacat/{storage → experimental/storage}/rivulet/writer/memtable_dataset_writer.py +20 -9
- deltacat/io/__init__.py +13 -0
- deltacat/io/dataset/__init__.py +0 -0
- deltacat/io/dataset/deltacat_dataset.py +91 -0
- deltacat/io/datasink/__init__.py +0 -0
- deltacat/io/datasink/deltacat_datasink.py +207 -0
- deltacat/io/datasource/__init__.py +0 -0
- deltacat/io/datasource/deltacat_datasource.py +579 -0
- deltacat/io/reader/__init__.py +0 -0
- deltacat/io/reader/deltacat_read_api.py +172 -0
- deltacat/storage/__init__.py +22 -2
- deltacat/storage/interface.py +54 -32
- deltacat/storage/main/impl.py +1494 -541
- deltacat/storage/model/delta.py +27 -3
- deltacat/storage/model/expression/__init__.py +47 -0
- deltacat/storage/model/expression/expression.py +656 -0
- deltacat/storage/model/expression/visitor.py +248 -0
- deltacat/storage/model/locator.py +6 -12
- deltacat/storage/model/manifest.py +231 -6
- deltacat/storage/model/metafile.py +224 -119
- deltacat/storage/model/namespace.py +8 -1
- deltacat/storage/model/partition.py +117 -42
- deltacat/storage/model/scan/push_down.py +32 -5
- deltacat/storage/model/schema.py +2427 -159
- deltacat/storage/model/shard.py +6 -2
- deltacat/storage/model/sort_key.py +40 -0
- deltacat/storage/model/stream.py +9 -2
- deltacat/storage/model/table.py +12 -1
- deltacat/storage/model/table_version.py +11 -0
- deltacat/storage/model/transaction.py +1184 -208
- deltacat/storage/model/transform.py +81 -2
- deltacat/storage/model/types.py +53 -29
- deltacat/storage/util/__init__.py +0 -0
- deltacat/storage/util/scan_planner.py +26 -0
- deltacat/tests/_io/reader/__init__.py +0 -0
- deltacat/tests/_io/reader/test_deltacat_read_api.py +0 -0
- deltacat/tests/_io/test_cloudpickle_bug_fix.py +8 -4
- deltacat/tests/aws/test_s3u.py +2 -31
- deltacat/tests/catalog/data/__init__.py +0 -0
- deltacat/tests/catalog/main/__init__.py +0 -0
- deltacat/tests/catalog/main/test_catalog_impl_namespace_operations.py +130 -0
- deltacat/tests/catalog/main/test_catalog_impl_table_operations.py +1972 -0
- deltacat/tests/catalog/model/__init__.py +0 -0
- deltacat/tests/catalog/model/test_table_definition.py +16 -0
- deltacat/tests/catalog/test_catalogs.py +103 -106
- deltacat/tests/catalog/test_default_catalog_impl.py +12152 -72
- deltacat/tests/compute/compact_partition_test_cases.py +35 -8
- deltacat/tests/compute/compactor/steps/test_repartition.py +12 -12
- deltacat/tests/compute/compactor/utils/test_io.py +124 -120
- deltacat/tests/compute/compactor/utils/test_round_completion_reader.py +254 -0
- deltacat/tests/compute/compactor_v2/test_compaction_session.py +423 -312
- deltacat/tests/compute/compactor_v2/utils/test_content_type_params.py +266 -0
- deltacat/tests/compute/compactor_v2/utils/test_primary_key_index.py +45 -0
- deltacat/tests/compute/compactor_v2/utils/test_task_options.py +270 -1
- deltacat/tests/compute/conftest.py +8 -44
- deltacat/tests/compute/converter/test_convert_session.py +697 -349
- deltacat/tests/compute/converter/utils.py +15 -6
- deltacat/tests/compute/resource_estimation/test_delta.py +145 -79
- deltacat/tests/compute/test_compact_partition_incremental.py +103 -70
- deltacat/tests/compute/test_compact_partition_multiple_rounds.py +89 -66
- deltacat/tests/compute/test_compact_partition_params.py +13 -8
- deltacat/tests/compute/test_compact_partition_rebase.py +77 -62
- deltacat/tests/compute/test_compact_partition_rebase_then_incremental.py +263 -193
- deltacat/tests/compute/test_janitor.py +236 -0
- deltacat/tests/compute/test_util_common.py +716 -43
- deltacat/tests/compute/test_util_constant.py +0 -1
- deltacat/tests/{storage/conftest.py → conftest.py} +1 -1
- deltacat/tests/daft/__init__.py +0 -0
- deltacat/tests/daft/test_model.py +97 -0
- deltacat/tests/experimental/__init__.py +1 -0
- deltacat/tests/experimental/catalog/__init__.py +0 -0
- deltacat/tests/experimental/catalog/iceberg/__init__.py +0 -0
- deltacat/tests/experimental/catalog/iceberg/test_iceberg_catalog.py +71 -0
- deltacat/tests/experimental/compatibility/__init__.py +1 -0
- deltacat/tests/experimental/compatibility/test_backfill_locator_to_id_mappings.py +582 -0
- deltacat/tests/experimental/daft/__init__.py +0 -0
- deltacat/tests/experimental/daft/test_deltacat_daft_integration.py +136 -0
- deltacat/tests/experimental/storage/__init__.py +0 -0
- deltacat/tests/experimental/storage/rivulet/__init__.py +0 -0
- deltacat/tests/{storage → experimental/storage}/rivulet/conftest.py +3 -3
- deltacat/tests/experimental/storage/rivulet/fs/__init__.py +0 -0
- deltacat/tests/{storage → experimental/storage}/rivulet/fs/test_file_location_provider.py +3 -2
- deltacat/tests/experimental/storage/rivulet/reader/__init__.py +0 -0
- deltacat/tests/experimental/storage/rivulet/reader/query_expression.py +80 -0
- deltacat/tests/experimental/storage/rivulet/reader/test_data_scan.py +119 -0
- deltacat/tests/experimental/storage/rivulet/reader/test_dataset_metastore.py +71 -0
- deltacat/tests/experimental/storage/rivulet/schema/__init__.py +0 -0
- deltacat/tests/{storage → experimental/storage}/rivulet/schema/test_schema.py +1 -1
- deltacat/tests/experimental/storage/rivulet/shard/__init__.py +0 -0
- deltacat/tests/experimental/storage/rivulet/shard/test_range_shard.py +162 -0
- deltacat/tests/{storage → experimental/storage}/rivulet/test_dataset.py +6 -4
- deltacat/tests/{storage → experimental/storage}/rivulet/test_manifest.py +5 -5
- deltacat/tests/{storage → experimental/storage}/rivulet/test_sst_interval_tree.py +5 -5
- deltacat/tests/{storage → experimental/storage}/rivulet/test_utils.py +8 -6
- deltacat/tests/experimental/storage/rivulet/writer/__init__.py +0 -0
- deltacat/tests/{storage → experimental/storage}/rivulet/writer/test_dataset_write_then_read.py +11 -9
- deltacat/tests/{storage → experimental/storage}/rivulet/writer/test_dataset_writer.py +2 -2
- deltacat/tests/{storage → experimental/storage}/rivulet/writer/test_memtable_dataset_writer.py +7 -7
- deltacat/tests/storage/main/test_main_storage.py +6900 -95
- deltacat/tests/storage/model/test_expression.py +327 -0
- deltacat/tests/storage/model/test_manifest.py +129 -0
- deltacat/tests/storage/model/test_metafile_io.py +78 -173
- deltacat/tests/storage/model/test_partition_scheme.py +85 -0
- deltacat/tests/storage/model/test_schema.py +171 -0
- deltacat/tests/storage/model/test_schema_update.py +1925 -0
- deltacat/tests/storage/model/test_shard.py +3 -1
- deltacat/tests/storage/model/test_sort_scheme.py +90 -0
- deltacat/tests/storage/model/test_transaction.py +393 -48
- deltacat/tests/storage/model/test_transaction_history.py +886 -0
- deltacat/tests/test_deltacat_api.py +1036 -11
- deltacat/tests/test_exceptions.py +9 -5
- deltacat/tests/test_utils/pyarrow.py +52 -21
- deltacat/tests/test_utils/storage.py +23 -34
- deltacat/tests/types/__init__.py +0 -0
- deltacat/tests/types/test_tables.py +104 -0
- deltacat/tests/utils/exceptions.py +22 -0
- deltacat/tests/utils/main_deltacat_storage_mock.py +31 -0
- deltacat/tests/utils/ray_utils/test_dataset.py +123 -5
- deltacat/tests/utils/test_daft.py +121 -31
- deltacat/tests/utils/test_numpy.py +1193 -0
- deltacat/tests/utils/test_pandas.py +1106 -0
- deltacat/tests/utils/test_polars.py +1040 -0
- deltacat/tests/utils/test_pyarrow.py +1370 -89
- deltacat/types/media.py +345 -37
- deltacat/types/tables.py +2344 -46
- deltacat/utils/arguments.py +33 -1
- deltacat/utils/daft.py +824 -40
- deltacat/utils/export.py +3 -1
- deltacat/utils/filesystem.py +139 -9
- deltacat/utils/metafile_locator.py +2 -1
- deltacat/utils/numpy.py +118 -26
- deltacat/utils/pandas.py +577 -48
- deltacat/utils/polars.py +759 -0
- deltacat/utils/pyarrow.py +1373 -192
- deltacat/utils/ray_utils/concurrency.py +1 -1
- deltacat/utils/ray_utils/dataset.py +101 -10
- deltacat/utils/ray_utils/runtime.py +56 -4
- deltacat/utils/reader_compatibility_mapping.py +3083 -0
- deltacat/utils/url.py +1325 -0
- deltacat-2.0.0.dist-info/METADATA +1163 -0
- deltacat-2.0.0.dist-info/RECORD +439 -0
- {deltacat-2.0.dist-info → deltacat-2.0.0.dist-info}/WHEEL +1 -1
- deltacat/catalog/iceberg/__init__.py +0 -4
- deltacat/compute/compactor/utils/round_completion_file.py +0 -97
- deltacat/compute/merge_on_read/__init__.py +0 -4
- deltacat/compute/merge_on_read/daft.py +0 -40
- deltacat/compute/merge_on_read/model/merge_on_read_params.py +0 -66
- deltacat/compute/merge_on_read/utils/delta.py +0 -42
- deltacat/examples/common/fixtures.py +0 -15
- deltacat/storage/iceberg/iceberg_scan_planner.py +0 -28
- deltacat/storage/rivulet/__init__.py +0 -11
- deltacat/storage/rivulet/feather/__init__.py +0 -5
- deltacat/storage/rivulet/parquet/__init__.py +0 -5
- deltacat/tests/compute/compactor/utils/test_round_completion_file.py +0 -231
- deltacat/tests/compute/test_util_create_table_deltas_repo.py +0 -388
- deltacat/tests/local_deltacat_storage/__init__.py +0 -1235
- deltacat/tests/local_deltacat_storage/exceptions.py +0 -10
- deltacat/utils/s3fs.py +0 -21
- deltacat-2.0.dist-info/METADATA +0 -65
- deltacat-2.0.dist-info/RECORD +0 -347
- /deltacat/compute/{merge_on_read/model → jobs}/__init__.py +0 -0
- /deltacat/{compute/merge_on_read/utils → docs}/__init__.py +0 -0
- /deltacat/{examples/common → docs/autogen}/__init__.py +0 -0
- /deltacat/{examples/iceberg → docs/autogen/schema}/__init__.py +0 -0
- /deltacat/{storage/iceberg → docs/autogen/schema/inference}/__init__.py +0 -0
- /deltacat/{storage/rivulet/arrow → examples/compactor}/__init__.py +0 -0
- /deltacat/{storage/rivulet/fs → examples/experimental}/__init__.py +0 -0
- /deltacat/{storage/rivulet/metastore → examples/experimental/iceberg}/__init__.py +0 -0
- /deltacat/{storage/rivulet/reader → examples/experimental/iceberg/converter}/__init__.py +0 -0
- /deltacat/{storage/rivulet/schema → examples/experimental/iceberg/converter/beam}/__init__.py +0 -0
- /deltacat/{storage/rivulet/writer → examples/indexer}/__init__.py +0 -0
- /deltacat/{tests/storage/rivulet → examples/indexer/aws}/__init__.py +0 -0
- /deltacat/{tests/storage/rivulet/fs → examples/indexer/gcp}/__init__.py +0 -0
- /deltacat/{tests/storage/rivulet/schema → experimental}/__init__.py +0 -0
- /deltacat/{tests/storage/rivulet/writer → experimental/catalog}/__init__.py +0 -0
- /deltacat/{storage/rivulet/parquet/data_reader.py → experimental/compatibility/__init__.py} +0 -0
- /deltacat/{storage → experimental/storage}/rivulet/fs/input_file.py +0 -0
- /deltacat/{storage → experimental/storage}/rivulet/mvp/Table.py +0 -0
- /deltacat/{storage → experimental/storage}/rivulet/mvp/__init__.py +0 -0
- /deltacat/{storage → experimental/storage}/rivulet/reader/query_expression.py +0 -0
- /deltacat/{storage → experimental/storage}/rivulet/schema/datatype.py +0 -0
- /deltacat/{storage → experimental/storage}/rivulet/writer/dataset_writer.py +0 -0
- {deltacat-2.0.dist-info → deltacat-2.0.0.dist-info/licenses}/LICENSE +0 -0
- {deltacat-2.0.dist-info → deltacat-2.0.0.dist-info}/top_level.txt +0 -0
@@ -2,13 +2,16 @@ from __future__ import annotations
|
|
2
2
|
|
3
3
|
from typing import List, Callable, Any
|
4
4
|
|
5
|
-
from deltacat.storage.rivulet.
|
6
|
-
from deltacat.storage.rivulet
|
7
|
-
from deltacat.storage.rivulet import
|
8
|
-
from deltacat.storage.rivulet.reader.
|
9
|
-
|
10
|
-
|
11
|
-
from deltacat.storage.rivulet.reader.
|
5
|
+
from deltacat.experimental.storage.rivulet.mvp.Table import MvpTable
|
6
|
+
from deltacat.experimental.storage.rivulet import Schema
|
7
|
+
from deltacat.experimental.storage.rivulet.reader.data_scan import DataScan
|
8
|
+
from deltacat.experimental.storage.rivulet.reader.dataset_metastore import (
|
9
|
+
DatasetMetastore,
|
10
|
+
)
|
11
|
+
from deltacat.experimental.storage.rivulet.reader.dataset_reader import DatasetReader
|
12
|
+
from deltacat.experimental.storage.rivulet.reader.query_expression import (
|
13
|
+
QueryExpression,
|
14
|
+
)
|
12
15
|
|
13
16
|
|
14
17
|
class DatasetExecutor:
|
@@ -22,12 +25,10 @@ class DatasetExecutor:
|
|
22
25
|
|
23
26
|
def __init__(
|
24
27
|
self,
|
25
|
-
field_groups: List[FieldGroup],
|
26
28
|
schema: Schema,
|
27
29
|
metastore: DatasetMetastore,
|
28
30
|
):
|
29
31
|
self.effective_schema: Schema = schema.__deepcopy__()
|
30
|
-
self.field_groups = field_groups
|
31
32
|
self.output: MvpTable | None = None
|
32
33
|
self._metastore = metastore
|
33
34
|
|
@@ -64,18 +65,9 @@ class DatasetExecutor:
|
|
64
65
|
|
65
66
|
TODO for now this is doing dumb in-memory implementation and later this is going to be replaced by rust library
|
66
67
|
"""
|
67
|
-
|
68
|
-
return self._read_as_mvp_table(schema, self.field_groups[0])
|
69
|
-
else:
|
70
|
-
ds1 = self._read_as_mvp_table(schema, self.field_groups[0])
|
71
|
-
ds2 = self._read_as_mvp_table(schema, self.field_groups[1])
|
72
|
-
merged = MvpTable.merge(ds1, ds2, schema.primary_key.name)
|
73
|
-
for i in range(2, len(self.field_groups)):
|
74
|
-
ds_i = self._read_as_mvp_table(schema, self.field_groups[i])
|
75
|
-
merged = MvpTable.merge(merged, ds_i, schema.primary_key.name)
|
76
|
-
return merged
|
68
|
+
return self._read_as_mvp_table(schema)
|
77
69
|
|
78
|
-
def _read_as_mvp_table(self, schema: Schema
|
70
|
+
def _read_as_mvp_table(self, schema: Schema):
|
79
71
|
data = list(
|
80
72
|
DataScan(
|
81
73
|
schema, QueryExpression(), DatasetReader(self._metastore)
|
@@ -0,0 +1,7 @@
|
|
1
|
+
# TODO later on this will be moved to a dedicated package
|
2
|
+
from deltacat.experimental.storage.rivulet.feather.file_reader import FeatherFileReader
|
3
|
+
from deltacat.experimental.storage.rivulet.reader.reader_type_registrar import (
|
4
|
+
FileReaderRegistrar,
|
5
|
+
)
|
6
|
+
|
7
|
+
FileReaderRegistrar.register_reader("feather", FeatherFileReader)
|
@@ -5,15 +5,17 @@ from typing import Optional
|
|
5
5
|
import pyarrow.ipc
|
6
6
|
from pyarrow import RecordBatch, RecordBatchFileReader
|
7
7
|
|
8
|
-
from deltacat.storage.rivulet.fs.file_provider import FileProvider
|
9
|
-
from deltacat.storage.rivulet.metastore.sst import SSTableRow
|
10
|
-
from deltacat.storage.rivulet.reader.data_reader import (
|
8
|
+
from deltacat.experimental.storage.rivulet.fs.file_provider import FileProvider
|
9
|
+
from deltacat.experimental.storage.rivulet.metastore.sst import SSTableRow
|
10
|
+
from deltacat.experimental.storage.rivulet.reader.data_reader import (
|
11
11
|
RowAndKey,
|
12
12
|
FileReader,
|
13
13
|
FILE_FORMAT,
|
14
14
|
)
|
15
|
-
from deltacat.storage.rivulet.reader.pyarrow_data_reader import
|
16
|
-
|
15
|
+
from deltacat.experimental.storage.rivulet.reader.pyarrow_data_reader import (
|
16
|
+
RecordBatchRowIndex,
|
17
|
+
)
|
18
|
+
from deltacat.experimental.storage.rivulet.schema.schema import Schema
|
17
19
|
|
18
20
|
|
19
21
|
class FeatherFileReader(FileReader[RecordBatchRowIndex]):
|
@@ -3,10 +3,10 @@ from typing import List
|
|
3
3
|
import pyarrow as pa
|
4
4
|
from pyarrow import feather
|
5
5
|
|
6
|
-
from deltacat.storage.rivulet.metastore.sst import SSTableRow
|
7
|
-
from deltacat.storage.rivulet import Schema
|
8
|
-
from deltacat.storage.rivulet.arrow.serializer import ArrowSerializer
|
9
|
-
from deltacat.storage.rivulet.fs.file_provider import FileProvider
|
6
|
+
from deltacat.experimental.storage.rivulet.metastore.sst import SSTableRow
|
7
|
+
from deltacat.experimental.storage.rivulet import Schema
|
8
|
+
from deltacat.experimental.storage.rivulet.arrow.serializer import ArrowSerializer
|
9
|
+
from deltacat.experimental.storage.rivulet.fs.file_provider import FileProvider
|
10
10
|
|
11
11
|
|
12
12
|
class FeatherDataSerializer(ArrowSerializer):
|
File without changes
|
@@ -3,9 +3,9 @@ import time
|
|
3
3
|
from typing import List, Generator
|
4
4
|
|
5
5
|
from deltacat.storage.model.partition import PartitionLocator
|
6
|
-
from deltacat.storage.rivulet.fs.file_store import FileStore
|
7
|
-
from deltacat.storage.rivulet.fs.input_file import InputFile
|
8
|
-
from deltacat.storage.rivulet.fs.output_file import OutputFile
|
6
|
+
from deltacat.experimental.storage.rivulet.fs.file_store import FileStore
|
7
|
+
from deltacat.experimental.storage.rivulet.fs.input_file import InputFile
|
8
|
+
from deltacat.experimental.storage.rivulet.fs.output_file import OutputFile
|
9
9
|
from deltacat.utils.metafile_locator import _find_partition_path
|
10
10
|
|
11
11
|
|
@@ -4,8 +4,8 @@ from pyarrow.fs import FileSystem, FileType, FileSelector
|
|
4
4
|
# TODO(deltacat): Rely on deltacat implementation to resolve path and filesystem.
|
5
5
|
from ray.data.datasource.path_util import _resolve_paths_and_filesystem
|
6
6
|
|
7
|
-
from deltacat.storage.rivulet.fs.input_file import FSInputFile
|
8
|
-
from deltacat.storage.rivulet.fs.output_file import FSOutputFile
|
7
|
+
from deltacat.experimental.storage.rivulet.fs.input_file import FSInputFile
|
8
|
+
from deltacat.experimental.storage.rivulet.fs.output_file import FSOutputFile
|
9
9
|
|
10
10
|
|
11
11
|
class FileStore:
|
@@ -5,7 +5,7 @@ from typing import Protocol
|
|
5
5
|
|
6
6
|
from pyarrow.fs import FileSystem, FileType
|
7
7
|
|
8
|
-
from deltacat.storage.rivulet.fs.input_file import FSInputFile, InputFile
|
8
|
+
from deltacat.experimental.storage.rivulet.fs.input_file import FSInputFile, InputFile
|
9
9
|
|
10
10
|
|
11
11
|
class OutputStream(Protocol): # pragma: no cover
|
@@ -1,9 +1,9 @@
|
|
1
1
|
from dataclasses import dataclass
|
2
2
|
from typing import List, Callable, Any, Protocol
|
3
3
|
|
4
|
-
from deltacat.storage.rivulet.dataset_executor import DatasetExecutor
|
5
|
-
from deltacat.storage.rivulet
|
6
|
-
from deltacat.storage.rivulet import
|
4
|
+
from deltacat.experimental.storage.rivulet.dataset_executor import DatasetExecutor
|
5
|
+
from deltacat.experimental.storage.rivulet import Schema
|
6
|
+
from deltacat.experimental.storage.rivulet.mvp.Table import MvpTable
|
7
7
|
|
8
8
|
|
9
9
|
class DatasetOperation(Protocol):
|
@@ -99,7 +99,7 @@ class LogicalPlan:
|
|
99
99
|
self.operations.append(CollectOperation())
|
100
100
|
return self
|
101
101
|
|
102
|
-
def execute(self, executor: DatasetExecutor) ->
|
102
|
+
def execute(self, executor: DatasetExecutor) -> MvpTable:
|
103
103
|
for operation in self.operations:
|
104
104
|
operation.visit(executor)
|
105
105
|
return executor.output
|
File without changes
|
@@ -10,7 +10,6 @@ from deltacat.storage import (
|
|
10
10
|
Delta,
|
11
11
|
DeltaType,
|
12
12
|
Transaction,
|
13
|
-
TransactionType,
|
14
13
|
TransactionOperation,
|
15
14
|
TransactionOperationType,
|
16
15
|
)
|
@@ -19,7 +18,7 @@ from deltacat.storage.model.partition import PartitionLocator
|
|
19
18
|
from deltacat.storage.model.transaction import TransactionOperationList
|
20
19
|
|
21
20
|
from deltacat.storage.model.types import StreamFormat
|
22
|
-
from deltacat.storage.rivulet import Schema
|
21
|
+
from deltacat.experimental.storage.rivulet import Schema
|
23
22
|
|
24
23
|
StreamPosition = int
|
25
24
|
"""The stream position for creating a consistent ordering of manifests."""
|
@@ -169,7 +168,6 @@ class DeltacatManifestIO(ManifestIO):
|
|
169
168
|
delta["level"] = level
|
170
169
|
|
171
170
|
tx_results = Transaction.of(
|
172
|
-
txn_type=TransactionType.APPEND,
|
173
171
|
txn_operations=TransactionOperationList.of(
|
174
172
|
[
|
175
173
|
TransactionOperation.of(
|
@@ -4,9 +4,9 @@ import json
|
|
4
4
|
from itertools import zip_longest
|
5
5
|
from typing import List
|
6
6
|
|
7
|
-
from deltacat.storage.rivulet.fs.input_file import InputFile
|
8
|
-
from deltacat.storage.rivulet.fs.output_file import OutputFile
|
9
|
-
from deltacat.storage.rivulet.metastore.sst import (
|
7
|
+
from deltacat.experimental.storage.rivulet.fs.input_file import InputFile
|
8
|
+
from deltacat.experimental.storage.rivulet.fs.output_file import OutputFile
|
9
|
+
from deltacat.experimental.storage.rivulet.metastore.sst import (
|
10
10
|
SSTWriter,
|
11
11
|
SSTableRow,
|
12
12
|
SSTReader,
|
@@ -1,8 +1,8 @@
|
|
1
1
|
from dataclasses import dataclass
|
2
2
|
from typing import Protocol, Any, List
|
3
3
|
|
4
|
-
from deltacat.storage.rivulet.fs.input_file import InputFile
|
5
|
-
from deltacat.storage.rivulet.fs.output_file import OutputFile
|
4
|
+
from deltacat.experimental.storage.rivulet.fs.input_file import InputFile
|
5
|
+
from deltacat.experimental.storage.rivulet.fs.output_file import OutputFile
|
6
6
|
|
7
7
|
|
8
8
|
@dataclass(frozen=True)
|
@@ -8,9 +8,9 @@ from typing import Any, Dict, Set, List, FrozenSet, Iterable, TypeVar, NamedTupl
|
|
8
8
|
|
9
9
|
from intervaltree import Interval, IntervalTree
|
10
10
|
|
11
|
-
from deltacat.storage.rivulet.metastore.delta import DeltaContext
|
12
|
-
from deltacat.storage.rivulet.metastore.sst import SSTable, SSTableRow
|
13
|
-
from deltacat.storage.rivulet import Schema
|
11
|
+
from deltacat.experimental.storage.rivulet.metastore.delta import DeltaContext
|
12
|
+
from deltacat.experimental.storage.rivulet.metastore.sst import SSTable, SSTableRow
|
13
|
+
from deltacat.experimental.storage.rivulet import Schema
|
14
14
|
|
15
15
|
T = TypeVar("T")
|
16
16
|
|
@@ -0,0 +1,7 @@
|
|
1
|
+
# TODO later on this will be moved to a dedicated package
|
2
|
+
from deltacat.experimental.storage.rivulet.parquet.file_reader import ParquetFileReader
|
3
|
+
from deltacat.experimental.storage.rivulet.reader.reader_type_registrar import (
|
4
|
+
FileReaderRegistrar,
|
5
|
+
)
|
6
|
+
|
7
|
+
FileReaderRegistrar.register_reader("parquet", ParquetFileReader)
|
File without changes
|
@@ -4,15 +4,17 @@ from typing import Optional
|
|
4
4
|
|
5
5
|
from pyarrow import RecordBatch
|
6
6
|
|
7
|
-
from deltacat.storage.rivulet.fs.file_provider import FileProvider
|
8
|
-
from deltacat.storage.rivulet.metastore.sst import SSTableRow
|
9
|
-
from deltacat.storage.rivulet.reader.data_reader import (
|
7
|
+
from deltacat.experimental.storage.rivulet.fs.file_provider import FileProvider
|
8
|
+
from deltacat.experimental.storage.rivulet.metastore.sst import SSTableRow
|
9
|
+
from deltacat.experimental.storage.rivulet.reader.data_reader import (
|
10
10
|
RowAndKey,
|
11
11
|
FileReader,
|
12
12
|
FILE_FORMAT,
|
13
13
|
)
|
14
|
-
from deltacat.storage.rivulet.reader.pyarrow_data_reader import
|
15
|
-
|
14
|
+
from deltacat.experimental.storage.rivulet.reader.pyarrow_data_reader import (
|
15
|
+
RecordBatchRowIndex,
|
16
|
+
)
|
17
|
+
from deltacat.experimental.storage.rivulet.schema.schema import Schema
|
16
18
|
import pyarrow.parquet as pq
|
17
19
|
import pyarrow as pa
|
18
20
|
|
@@ -3,11 +3,11 @@ from typing import List, Any
|
|
3
3
|
import pyarrow as pa
|
4
4
|
from pyarrow.parquet import FileMetaData
|
5
5
|
|
6
|
-
from deltacat.storage.rivulet.metastore.sst import SSTableRow
|
7
|
-
from deltacat.storage.rivulet import Schema
|
8
|
-
from deltacat.storage.rivulet.arrow.serializer import ArrowSerializer
|
6
|
+
from deltacat.experimental.storage.rivulet.metastore.sst import SSTableRow
|
7
|
+
from deltacat.experimental.storage.rivulet import Schema
|
8
|
+
from deltacat.experimental.storage.rivulet.arrow.serializer import ArrowSerializer
|
9
9
|
|
10
|
-
from deltacat.storage.rivulet.fs.file_provider import FileProvider
|
10
|
+
from deltacat.experimental.storage.rivulet.fs.file_provider import FileProvider
|
11
11
|
|
12
12
|
|
13
13
|
class ParquetDataSerializer(ArrowSerializer):
|
File without changes
|
@@ -15,19 +15,30 @@ from typing import (
|
|
15
15
|
AbstractSet,
|
16
16
|
)
|
17
17
|
|
18
|
-
from deltacat.storage.rivulet.metastore.delta import DeltaContext
|
19
|
-
from deltacat.storage.rivulet.metastore.sst import SSTableRow
|
20
|
-
from deltacat.storage.rivulet.metastore.sst_interval_tree import (
|
18
|
+
from deltacat.experimental.storage.rivulet.metastore.delta import DeltaContext
|
19
|
+
from deltacat.experimental.storage.rivulet.metastore.sst import SSTableRow
|
20
|
+
from deltacat.experimental.storage.rivulet.metastore.sst_interval_tree import (
|
21
21
|
OrderedBlockGroups,
|
22
22
|
BlockGroup,
|
23
23
|
Block,
|
24
24
|
)
|
25
|
-
from deltacat.storage.rivulet.reader.data_reader import
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
from deltacat.storage.rivulet.reader.
|
30
|
-
|
25
|
+
from deltacat.experimental.storage.rivulet.reader.data_reader import (
|
26
|
+
RowAndKey,
|
27
|
+
FileReader,
|
28
|
+
)
|
29
|
+
from deltacat.experimental.storage.rivulet.reader.dataset_metastore import (
|
30
|
+
DatasetMetastore,
|
31
|
+
)
|
32
|
+
from deltacat.experimental.storage.rivulet.reader.pyarrow_data_reader import (
|
33
|
+
ArrowDataReader,
|
34
|
+
)
|
35
|
+
from deltacat.experimental.storage.rivulet.reader.query_expression import (
|
36
|
+
QueryExpression,
|
37
|
+
)
|
38
|
+
from deltacat.experimental.storage.rivulet.reader.reader_type_registrar import (
|
39
|
+
FileReaderRegistrar,
|
40
|
+
)
|
41
|
+
from deltacat.experimental.storage.rivulet import Schema
|
31
42
|
from deltacat import logs
|
32
43
|
|
33
44
|
logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
|
@@ -13,9 +13,9 @@ from typing import (
|
|
13
13
|
Optional,
|
14
14
|
)
|
15
15
|
|
16
|
-
from deltacat.storage.rivulet.fs.file_provider import FileProvider
|
17
|
-
from deltacat.storage.rivulet.metastore.sst import SSTableRow
|
18
|
-
from deltacat.storage.rivulet.schema.schema import Schema
|
16
|
+
from deltacat.experimental.storage.rivulet.fs.file_provider import FileProvider
|
17
|
+
from deltacat.experimental.storage.rivulet.metastore.sst import SSTableRow
|
18
|
+
from deltacat.experimental.storage.rivulet.schema.schema import Schema
|
19
19
|
|
20
20
|
FILE_FORMAT = TypeVar("FILE_FORMAT")
|
21
21
|
MEMORY_FORMAT = TypeVar("MEMORY_FORMAT")
|
@@ -3,9 +3,11 @@ from typing import Generator, Dict, Optional
|
|
3
3
|
import pyarrow as pa
|
4
4
|
|
5
5
|
from deltacat.storage.model.shard import Shard
|
6
|
-
from deltacat.storage.rivulet.reader.dataset_reader import DatasetReader
|
7
|
-
from deltacat.storage.rivulet.reader.query_expression import
|
8
|
-
|
6
|
+
from deltacat.experimental.storage.rivulet.reader.dataset_reader import DatasetReader
|
7
|
+
from deltacat.experimental.storage.rivulet.reader.query_expression import (
|
8
|
+
QueryExpression,
|
9
|
+
)
|
10
|
+
from deltacat.experimental.storage.rivulet import Schema
|
9
11
|
|
10
12
|
|
11
13
|
class DataScan:
|
@@ -5,18 +5,19 @@ from typing import Generator, Optional
|
|
5
5
|
import pyarrow
|
6
6
|
import pyarrow.fs
|
7
7
|
|
8
|
+
from deltacat.constants import REV_DIR_NAME
|
8
9
|
from deltacat.storage import Delta
|
9
10
|
from deltacat.storage.model.partition import PartitionLocator
|
10
|
-
from deltacat.storage.rivulet.fs.file_provider import FileProvider
|
11
|
+
from deltacat.experimental.storage.rivulet.fs.file_provider import FileProvider
|
11
12
|
from deltacat.utils.filesystem import resolve_path_and_filesystem
|
12
|
-
from deltacat.storage.rivulet.metastore.json_sst import JsonSstReader
|
13
|
-
from deltacat.storage.rivulet.metastore.delta import (
|
13
|
+
from deltacat.experimental.storage.rivulet.metastore.json_sst import JsonSstReader
|
14
|
+
from deltacat.experimental.storage.rivulet.metastore.delta import (
|
14
15
|
ManifestIO,
|
15
16
|
DeltaContext,
|
16
17
|
RivuletDelta,
|
17
18
|
DeltacatManifestIO,
|
18
19
|
)
|
19
|
-
from deltacat.storage.rivulet.metastore.sst import SSTReader, SSTable
|
20
|
+
from deltacat.experimental.storage.rivulet.metastore.sst import SSTReader, SSTable
|
20
21
|
from deltacat.utils.metafile_locator import _find_table_path
|
21
22
|
from deltacat import logs
|
22
23
|
|
@@ -83,7 +84,7 @@ class DatasetMetastore:
|
|
83
84
|
param: filesystem: The filesystem to search for the revisions.
|
84
85
|
returns: The latest revision as a RivuletDelta.
|
85
86
|
"""
|
86
|
-
rev_directory = posixpath.join(delta_dir,
|
87
|
+
rev_directory = posixpath.join(delta_dir, REV_DIR_NAME)
|
87
88
|
revisions = filesystem.get_file_info(
|
88
89
|
pyarrow.fs.FileSelector(rev_directory, allow_not_found=True)
|
89
90
|
)
|
@@ -128,7 +129,7 @@ class DatasetMetastore:
|
|
128
129
|
return
|
129
130
|
|
130
131
|
# Locate "rev" directory inside the partition
|
131
|
-
rev_directory = posixpath.join(partition_path,
|
132
|
+
rev_directory = posixpath.join(partition_path, REV_DIR_NAME)
|
132
133
|
rev_info = filesystem.get_file_info(rev_directory)
|
133
134
|
|
134
135
|
if rev_info.type != pyarrow.fs.FileType.Directory:
|
@@ -2,18 +2,20 @@ import logging
|
|
2
2
|
from typing import Generator, Optional, Set, Type, TypeVar, Any
|
3
3
|
|
4
4
|
from deltacat.storage.model.shard import Shard
|
5
|
-
from deltacat.storage.rivulet.metastore.sst import SSTableRow, SSTable
|
6
|
-
from deltacat.storage.rivulet.metastore.sst_interval_tree import (
|
5
|
+
from deltacat.experimental.storage.rivulet.metastore.sst import SSTableRow, SSTable
|
6
|
+
from deltacat.experimental.storage.rivulet.metastore.sst_interval_tree import (
|
7
7
|
BlockIntervalTree,
|
8
8
|
OrderedBlockGroups,
|
9
9
|
)
|
10
|
-
from deltacat.storage.rivulet.reader.block_scanner import BlockScanner
|
11
|
-
from deltacat.storage.rivulet.reader.dataset_metastore import (
|
10
|
+
from deltacat.experimental.storage.rivulet.reader.block_scanner import BlockScanner
|
11
|
+
from deltacat.experimental.storage.rivulet.reader.dataset_metastore import (
|
12
12
|
DatasetMetastore,
|
13
13
|
ManifestAccessor,
|
14
14
|
)
|
15
|
-
from deltacat.storage.rivulet.reader.query_expression import
|
16
|
-
|
15
|
+
from deltacat.experimental.storage.rivulet.reader.query_expression import (
|
16
|
+
QueryExpression,
|
17
|
+
)
|
18
|
+
from deltacat.experimental.storage.rivulet import Schema
|
17
19
|
|
18
20
|
# The type of data returned to reader
|
19
21
|
T = TypeVar("T")
|
@@ -4,7 +4,10 @@ from typing import Generator, Dict, Type, NamedTuple, List
|
|
4
4
|
|
5
5
|
from pyarrow import RecordBatch
|
6
6
|
|
7
|
-
from deltacat.storage.rivulet.reader.data_reader import
|
7
|
+
from deltacat.experimental.storage.rivulet.reader.data_reader import (
|
8
|
+
DataReader,
|
9
|
+
MEMORY_FORMAT,
|
10
|
+
)
|
8
11
|
import pyarrow as pa
|
9
12
|
|
10
13
|
|
@@ -1,9 +1,9 @@
|
|
1
|
-
from deltacat.storage.rivulet.fs.file_provider import FileProvider
|
2
|
-
from deltacat.storage.rivulet.metastore.sst import SSTableRow
|
3
|
-
from deltacat.storage.rivulet.reader.data_reader import FileReader
|
1
|
+
from deltacat.experimental.storage.rivulet.fs.file_provider import FileProvider
|
2
|
+
from deltacat.experimental.storage.rivulet.metastore.sst import SSTableRow
|
3
|
+
from deltacat.experimental.storage.rivulet.reader.data_reader import FileReader
|
4
4
|
from typing import Type, Dict
|
5
5
|
|
6
|
-
from deltacat.storage.rivulet.schema.schema import Schema
|
6
|
+
from deltacat.experimental.storage.rivulet.schema.schema import Schema
|
7
7
|
|
8
8
|
|
9
9
|
class FileReaderRegistrar:
|
File without changes
|
@@ -5,7 +5,7 @@ from typing import MutableMapping, Dict, Iterable, Tuple, Optional
|
|
5
5
|
|
6
6
|
import pyarrow as pa
|
7
7
|
|
8
|
-
from deltacat.storage.rivulet.schema.datatype import Datatype
|
8
|
+
from deltacat.experimental.storage.rivulet.schema.datatype import Datatype
|
9
9
|
|
10
10
|
|
11
11
|
@dataclass(frozen=True)
|
@@ -1,6 +1,6 @@
|
|
1
1
|
from typing import Protocol, Iterable, List, Union, Any, Dict
|
2
2
|
|
3
|
-
from deltacat.storage.rivulet.metastore.sst import SSTableRow
|
3
|
+
from deltacat.experimental.storage.rivulet.metastore.sst import SSTableRow
|
4
4
|
import pyarrow as pa
|
5
5
|
|
6
6
|
MEMTABLE_DATA = Union[Iterable[Dict[str, Any]], pa.Table]
|
@@ -1,11 +1,15 @@
|
|
1
1
|
from __future__ import annotations
|
2
2
|
|
3
|
-
from deltacat.storage.rivulet.parquet.serializer import
|
4
|
-
|
5
|
-
|
6
|
-
from deltacat.storage.rivulet
|
3
|
+
from deltacat.experimental.storage.rivulet.parquet.serializer import (
|
4
|
+
ParquetDataSerializer,
|
5
|
+
)
|
6
|
+
from deltacat.experimental.storage.rivulet import Schema
|
7
|
+
from deltacat.experimental.storage.rivulet.serializer import DataSerializer
|
8
|
+
from deltacat.experimental.storage.rivulet.fs.file_provider import FileProvider
|
7
9
|
|
8
|
-
from deltacat.storage.rivulet.feather.serializer import
|
10
|
+
from deltacat.experimental.storage.rivulet.feather.serializer import (
|
11
|
+
FeatherDataSerializer,
|
12
|
+
)
|
9
13
|
|
10
14
|
|
11
15
|
class DataSerializerFactory:
|
File without changes
|
@@ -0,0 +1,129 @@
|
|
1
|
+
from __future__ import annotations
|
2
|
+
from typing import Generic, List, Union, Iterable
|
3
|
+
from deltacat.storage.model.shard import T, Shard, ShardingStrategy
|
4
|
+
from deltacat.experimental.storage.rivulet.reader.dataset_metastore import (
|
5
|
+
DatasetMetastore,
|
6
|
+
)
|
7
|
+
|
8
|
+
|
9
|
+
class RangeShard(Shard, Generic[T]):
|
10
|
+
"""
|
11
|
+
Represents a range-based shard with minimum and maximum keys.
|
12
|
+
|
13
|
+
param: min_key: The minimum key for the shard.
|
14
|
+
param: max_key: The maximum key for the shard.
|
15
|
+
"""
|
16
|
+
|
17
|
+
def __init__(self, min_key: T, max_key: T):
|
18
|
+
self.min_key = min_key
|
19
|
+
self.max_key = max_key
|
20
|
+
|
21
|
+
def __repr__(self) -> str:
|
22
|
+
return f"Shard(type=range, min_key={self.min_key}, max_key={self.max_key})"
|
23
|
+
|
24
|
+
@staticmethod
|
25
|
+
def split(
|
26
|
+
global_min: Union[int, str], global_max: Union[int, str], num_shards: int
|
27
|
+
) -> List[RangeShard]:
|
28
|
+
"""
|
29
|
+
Splits a range into `num_shards` shards.
|
30
|
+
Currently supports splitting ranges of integers and strings.
|
31
|
+
|
32
|
+
Note: If global_min == global_max or num_shards <= 1, a single shard is returned,
|
33
|
+
num_shards is ignored.
|
34
|
+
|
35
|
+
:param global_min: The minimum key for the entire range (int or str).
|
36
|
+
:param global_max: The maximum key for the entire range (int or str).
|
37
|
+
:param num_shards: The number of shards to create.
|
38
|
+
:return: A list of RangeShard objects.
|
39
|
+
"""
|
40
|
+
if global_min == global_max or num_shards <= 1:
|
41
|
+
return [RangeShard(global_min, global_max)]
|
42
|
+
|
43
|
+
# Determine which interpolation function to use based on the type of min/max
|
44
|
+
if isinstance(global_min, int) and isinstance(global_max, int):
|
45
|
+
interpolate = RangeShard._interpolate_numeric
|
46
|
+
elif isinstance(global_min, str) and isinstance(global_max, str):
|
47
|
+
interpolate = RangeShard._interpolate_str
|
48
|
+
else:
|
49
|
+
raise ValueError(
|
50
|
+
"Unsupported combination of types for global_min and global_max."
|
51
|
+
)
|
52
|
+
|
53
|
+
shards: List[RangeShard] = []
|
54
|
+
for i in range(num_shards):
|
55
|
+
start = interpolate(global_min, global_max, i, num_shards)
|
56
|
+
end = interpolate(global_min, global_max, i + 1, num_shards)
|
57
|
+
|
58
|
+
if i > 0:
|
59
|
+
if isinstance(start, int):
|
60
|
+
start = shards[-1].max_key + 1
|
61
|
+
elif isinstance(start, int):
|
62
|
+
char_list = list(start)
|
63
|
+
char_list[-1] = chr(ord(char_list[-1]) + 1)
|
64
|
+
start = "".join(char_list)
|
65
|
+
|
66
|
+
shards.append(RangeShard(start, end))
|
67
|
+
|
68
|
+
return shards
|
69
|
+
|
70
|
+
@staticmethod
|
71
|
+
def _interpolate_numeric(start: int, end: int, step: int, total_steps: int) -> int:
|
72
|
+
"""
|
73
|
+
Integer interpolation using integer (floor) division.
|
74
|
+
|
75
|
+
param: start (int): The starting number.
|
76
|
+
param: end (int): The ending number.
|
77
|
+
param: step (int): The current step in the interpolation (0-based).
|
78
|
+
param: total_steps (int): The total number of interpolation steps.
|
79
|
+
|
80
|
+
returns: int: The interpolated integer.
|
81
|
+
"""
|
82
|
+
return start + (end - start) * step // total_steps
|
83
|
+
|
84
|
+
@staticmethod
|
85
|
+
def _interpolate_str(start: str, end: str, step: int, total_steps: int) -> str:
|
86
|
+
"""
|
87
|
+
Interpolates between two strings lexicographically.
|
88
|
+
|
89
|
+
param: start (str): The starting string.
|
90
|
+
param: end (str): The ending string.
|
91
|
+
param: step (int): The current step in the interpolation (0-based).
|
92
|
+
param: total_steps (int): The total number of interpolation steps.
|
93
|
+
|
94
|
+
returns: str: The interpolated string.
|
95
|
+
"""
|
96
|
+
max_len = max(len(start), len(end))
|
97
|
+
|
98
|
+
# Pad strings to the same length with spaces (smallest lexicographical character).
|
99
|
+
start = start.ljust(max_len, " ")
|
100
|
+
end = end.ljust(max_len, " ")
|
101
|
+
|
102
|
+
# Interpolate character by character based on ordinal values.
|
103
|
+
interpolated_chars = [
|
104
|
+
chr(round(ord(s) + (ord(e) - ord(s)) * step / total_steps))
|
105
|
+
for s, e in zip(start, end)
|
106
|
+
]
|
107
|
+
|
108
|
+
return "".join(interpolated_chars).rstrip()
|
109
|
+
|
110
|
+
|
111
|
+
class RangeShardingStrategy(ShardingStrategy, Generic[T]):
|
112
|
+
"""
|
113
|
+
Implements a sharding strategy to divide a range of keys into shards.
|
114
|
+
|
115
|
+
method: shards: Generates a list of RangeShard objects based on the global range.
|
116
|
+
"""
|
117
|
+
|
118
|
+
def shards(
|
119
|
+
self, num_shards: int, metastore: DatasetMetastore
|
120
|
+
) -> Iterable[RangeShard[T]]:
|
121
|
+
"""
|
122
|
+
Divides the global range of keys into evenly sized shards.
|
123
|
+
|
124
|
+
param: num_shards: The number of shards to divide the range into.
|
125
|
+
param: metastore: The dataset metastore providing access to manifests.
|
126
|
+
returns: A list of RangeShard objects representing the divided range.
|
127
|
+
"""
|
128
|
+
min, max = metastore.get_min_max_keys()
|
129
|
+
return RangeShard.split(min, max, num_shards)
|
File without changes
|