deltacat 2.0.0b10__py3-none-any.whl → 2.0.0b12__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- deltacat/__init__.py +96 -17
- deltacat/api.py +122 -67
- deltacat/aws/constants.py +0 -23
- deltacat/aws/s3u.py +4 -631
- deltacat/benchmarking/benchmark_engine.py +4 -2
- deltacat/benchmarking/conftest.py +0 -18
- deltacat/benchmarking/test_benchmark_pipeline.py +6 -4
- deltacat/catalog/__init__.py +64 -5
- deltacat/catalog/delegate.py +445 -63
- deltacat/catalog/interface.py +188 -62
- deltacat/catalog/main/impl.py +2435 -279
- deltacat/catalog/model/catalog.py +154 -77
- deltacat/catalog/model/properties.py +63 -22
- deltacat/compute/compactor/compaction_session.py +97 -75
- deltacat/compute/compactor/model/compact_partition_params.py +75 -30
- deltacat/compute/compactor/model/compaction_session_audit_info.py +17 -0
- deltacat/compute/compactor/model/round_completion_info.py +16 -6
- deltacat/compute/compactor/repartition_session.py +8 -21
- deltacat/compute/compactor/steps/hash_bucket.py +5 -5
- deltacat/compute/compactor/steps/materialize.py +9 -7
- deltacat/compute/compactor/steps/repartition.py +12 -11
- deltacat/compute/compactor/utils/io.py +6 -5
- deltacat/compute/compactor/utils/round_completion_reader.py +117 -0
- deltacat/compute/compactor/utils/system_columns.py +3 -1
- deltacat/compute/compactor_v2/compaction_session.py +17 -14
- deltacat/compute/compactor_v2/constants.py +30 -1
- deltacat/compute/compactor_v2/model/evaluate_compaction_result.py +0 -1
- deltacat/compute/compactor_v2/model/hash_bucket_input.py +9 -3
- deltacat/compute/compactor_v2/model/merge_file_group.py +5 -2
- deltacat/compute/compactor_v2/model/merge_input.py +33 -8
- deltacat/compute/compactor_v2/private/compaction_utils.py +167 -68
- deltacat/compute/compactor_v2/steps/hash_bucket.py +5 -2
- deltacat/compute/compactor_v2/steps/merge.py +267 -55
- deltacat/compute/compactor_v2/utils/content_type_params.py +34 -6
- deltacat/compute/compactor_v2/utils/dedupe.py +1 -1
- deltacat/compute/compactor_v2/utils/delta.py +5 -3
- deltacat/compute/compactor_v2/utils/io.py +11 -4
- deltacat/compute/compactor_v2/utils/merge.py +15 -2
- deltacat/compute/compactor_v2/utils/primary_key_index.py +28 -4
- deltacat/compute/compactor_v2/utils/task_options.py +45 -33
- deltacat/compute/converter/converter_session.py +145 -32
- deltacat/compute/converter/model/convert_input.py +26 -19
- deltacat/compute/converter/model/convert_input_files.py +33 -16
- deltacat/compute/converter/model/convert_result.py +35 -16
- deltacat/compute/converter/model/converter_session_params.py +24 -21
- deltacat/compute/converter/pyiceberg/catalog.py +21 -18
- deltacat/compute/converter/pyiceberg/overrides.py +18 -9
- deltacat/compute/converter/pyiceberg/update_snapshot_overrides.py +148 -100
- deltacat/compute/converter/steps/convert.py +157 -50
- deltacat/compute/converter/steps/dedupe.py +24 -11
- deltacat/compute/converter/utils/convert_task_options.py +27 -12
- deltacat/compute/converter/utils/converter_session_utils.py +126 -60
- deltacat/compute/converter/utils/iceberg_columns.py +8 -8
- deltacat/compute/converter/utils/io.py +101 -12
- deltacat/compute/converter/utils/s3u.py +33 -27
- deltacat/compute/janitor.py +205 -0
- deltacat/compute/jobs/client.py +25 -12
- deltacat/compute/resource_estimation/delta.py +38 -6
- deltacat/compute/resource_estimation/model.py +8 -0
- deltacat/constants.py +45 -2
- deltacat/docs/autogen/schema/inference/generate_type_mappings.py +687 -0
- deltacat/docs/autogen/schema/inference/parse_json_type_mappings.py +673 -0
- deltacat/env.py +10 -0
- deltacat/examples/basic_logging.py +1 -3
- deltacat/examples/compactor/aws/__init__.py +1 -0
- deltacat/examples/compactor/bootstrap.py +863 -0
- deltacat/examples/compactor/compactor.py +373 -0
- deltacat/examples/compactor/explorer.py +473 -0
- deltacat/examples/compactor/gcp/__init__.py +1 -0
- deltacat/examples/compactor/job_runner.py +439 -0
- deltacat/examples/compactor/utils/__init__.py +1 -0
- deltacat/examples/compactor/utils/common.py +261 -0
- deltacat/examples/experimental/iceberg/converter/beam/app.py +226 -0
- deltacat/examples/experimental/iceberg/converter/beam/main.py +133 -0
- deltacat/examples/experimental/iceberg/converter/beam/test_workflow.py +113 -0
- deltacat/examples/experimental/iceberg/converter/beam/utils/__init__.py +3 -0
- deltacat/examples/experimental/iceberg/converter/beam/utils/common.py +174 -0
- deltacat/examples/experimental/iceberg/converter/beam/utils/spark.py +263 -0
- deltacat/examples/{iceberg → experimental/iceberg}/iceberg_bucket_writer.py +3 -5
- deltacat/examples/{iceberg → experimental/iceberg}/iceberg_reader.py +2 -4
- deltacat/examples/indexer/indexer.py +2 -2
- deltacat/examples/indexer/job_runner.py +1 -2
- deltacat/exceptions.py +66 -4
- deltacat/experimental/catalog/iceberg/__init__.py +6 -0
- deltacat/{catalog → experimental/catalog}/iceberg/iceberg_catalog_config.py +1 -1
- deltacat/{catalog → experimental/catalog}/iceberg/impl.py +29 -11
- deltacat/experimental/compatibility/backfill_locator_to_id_mappings.py +201 -0
- deltacat/experimental/converter_agent/beam/managed.py +173 -0
- deltacat/experimental/converter_agent/table_monitor.py +479 -0
- deltacat/experimental/storage/iceberg/iceberg_scan_planner.py +129 -0
- deltacat/{storage → experimental/storage}/iceberg/impl.py +6 -4
- deltacat/{storage → experimental/storage}/iceberg/model.py +7 -3
- deltacat/experimental/storage/iceberg/visitor.py +119 -0
- deltacat/experimental/storage/rivulet/__init__.py +11 -0
- deltacat/{storage → experimental/storage}/rivulet/arrow/serializer.py +7 -4
- deltacat/{storage → experimental/storage}/rivulet/dataset.py +13 -12
- deltacat/{storage → experimental/storage}/rivulet/dataset_executor.py +12 -20
- deltacat/experimental/storage/rivulet/feather/__init__.py +7 -0
- deltacat/{storage → experimental/storage}/rivulet/feather/file_reader.py +7 -5
- deltacat/{storage → experimental/storage}/rivulet/feather/serializer.py +4 -4
- deltacat/{storage → experimental/storage}/rivulet/fs/file_provider.py +3 -3
- deltacat/{storage → experimental/storage}/rivulet/fs/file_store.py +2 -2
- deltacat/{storage → experimental/storage}/rivulet/fs/output_file.py +1 -1
- deltacat/{storage → experimental/storage}/rivulet/logical_plan.py +4 -4
- deltacat/experimental/storage/rivulet/metastore/__init__.py +0 -0
- deltacat/{storage → experimental/storage}/rivulet/metastore/delta.py +1 -3
- deltacat/{storage → experimental/storage}/rivulet/metastore/json_sst.py +3 -3
- deltacat/{storage → experimental/storage}/rivulet/metastore/sst.py +2 -2
- deltacat/{storage → experimental/storage}/rivulet/metastore/sst_interval_tree.py +3 -3
- deltacat/experimental/storage/rivulet/parquet/__init__.py +7 -0
- deltacat/experimental/storage/rivulet/parquet/data_reader.py +0 -0
- deltacat/{storage → experimental/storage}/rivulet/parquet/file_reader.py +7 -5
- deltacat/{storage → experimental/storage}/rivulet/parquet/serializer.py +4 -4
- deltacat/experimental/storage/rivulet/reader/__init__.py +0 -0
- deltacat/{storage → experimental/storage}/rivulet/reader/block_scanner.py +20 -9
- deltacat/{storage → experimental/storage}/rivulet/reader/data_reader.py +3 -3
- deltacat/{storage → experimental/storage}/rivulet/reader/data_scan.py +5 -3
- deltacat/{storage → experimental/storage}/rivulet/reader/dataset_metastore.py +7 -6
- deltacat/{storage → experimental/storage}/rivulet/reader/dataset_reader.py +8 -6
- deltacat/{storage → experimental/storage}/rivulet/reader/pyarrow_data_reader.py +4 -1
- deltacat/{storage → experimental/storage}/rivulet/reader/reader_type_registrar.py +4 -4
- deltacat/experimental/storage/rivulet/schema/__init__.py +0 -0
- deltacat/{storage → experimental/storage}/rivulet/schema/schema.py +1 -1
- deltacat/{storage → experimental/storage}/rivulet/serializer.py +1 -1
- deltacat/{storage → experimental/storage}/rivulet/serializer_factory.py +9 -5
- deltacat/experimental/storage/rivulet/shard/__init__.py +0 -0
- deltacat/experimental/storage/rivulet/shard/range_shard.py +129 -0
- deltacat/experimental/storage/rivulet/writer/__init__.py +0 -0
- deltacat/{storage → experimental/storage}/rivulet/writer/memtable_dataset_writer.py +20 -9
- deltacat/io/datasource/deltacat_datasource.py +0 -1
- deltacat/io/reader/deltacat_read_api.py +1 -1
- deltacat/storage/__init__.py +20 -2
- deltacat/storage/interface.py +54 -32
- deltacat/storage/main/impl.py +1494 -541
- deltacat/storage/model/delta.py +27 -3
- deltacat/storage/model/locator.py +6 -12
- deltacat/storage/model/manifest.py +182 -6
- deltacat/storage/model/metafile.py +151 -78
- deltacat/storage/model/namespace.py +8 -1
- deltacat/storage/model/partition.py +117 -42
- deltacat/storage/model/schema.py +2427 -159
- deltacat/storage/model/shard.py +6 -2
- deltacat/storage/model/sort_key.py +40 -0
- deltacat/storage/model/stream.py +9 -2
- deltacat/storage/model/table.py +12 -1
- deltacat/storage/model/table_version.py +11 -0
- deltacat/storage/model/transaction.py +1184 -208
- deltacat/storage/model/transform.py +81 -2
- deltacat/storage/model/types.py +48 -26
- deltacat/tests/_io/test_cloudpickle_bug_fix.py +8 -4
- deltacat/tests/aws/test_s3u.py +2 -31
- deltacat/tests/catalog/data/__init__.py +0 -0
- deltacat/tests/catalog/main/__init__.py +0 -0
- deltacat/tests/catalog/main/test_catalog_impl_namespace_operations.py +130 -0
- deltacat/tests/catalog/main/test_catalog_impl_table_operations.py +1972 -0
- deltacat/tests/catalog/model/__init__.py +0 -0
- deltacat/tests/catalog/model/test_table_definition.py +16 -0
- deltacat/tests/catalog/test_catalogs.py +103 -106
- deltacat/tests/catalog/test_default_catalog_impl.py +12152 -72
- deltacat/tests/compute/compact_partition_test_cases.py +35 -8
- deltacat/tests/compute/compactor/steps/test_repartition.py +12 -12
- deltacat/tests/compute/compactor/utils/test_io.py +124 -120
- deltacat/tests/compute/compactor/utils/test_round_completion_reader.py +254 -0
- deltacat/tests/compute/compactor_v2/test_compaction_session.py +423 -312
- deltacat/tests/compute/compactor_v2/utils/test_content_type_params.py +266 -0
- deltacat/tests/compute/compactor_v2/utils/test_primary_key_index.py +45 -0
- deltacat/tests/compute/compactor_v2/utils/test_task_options.py +270 -1
- deltacat/tests/compute/conftest.py +8 -44
- deltacat/tests/compute/converter/test_convert_session.py +675 -490
- deltacat/tests/compute/converter/utils.py +15 -6
- deltacat/tests/compute/resource_estimation/test_delta.py +145 -79
- deltacat/tests/compute/test_compact_partition_incremental.py +103 -70
- deltacat/tests/compute/test_compact_partition_multiple_rounds.py +89 -66
- deltacat/tests/compute/test_compact_partition_params.py +13 -8
- deltacat/tests/compute/test_compact_partition_rebase.py +77 -62
- deltacat/tests/compute/test_compact_partition_rebase_then_incremental.py +263 -193
- deltacat/tests/compute/test_janitor.py +236 -0
- deltacat/tests/compute/test_util_common.py +716 -43
- deltacat/tests/compute/test_util_constant.py +0 -1
- deltacat/tests/{storage/conftest.py → conftest.py} +1 -1
- deltacat/tests/daft/__init__.py +0 -0
- deltacat/tests/daft/test_model.py +97 -0
- deltacat/tests/experimental/__init__.py +1 -0
- deltacat/tests/experimental/catalog/__init__.py +0 -0
- deltacat/tests/experimental/catalog/iceberg/__init__.py +0 -0
- deltacat/tests/experimental/catalog/iceberg/test_iceberg_catalog.py +71 -0
- deltacat/tests/experimental/compatibility/__init__.py +1 -0
- deltacat/tests/experimental/compatibility/test_backfill_locator_to_id_mappings.py +582 -0
- deltacat/tests/experimental/daft/__init__.py +0 -0
- deltacat/tests/experimental/daft/test_deltacat_daft_integration.py +136 -0
- deltacat/tests/experimental/storage/__init__.py +0 -0
- deltacat/tests/experimental/storage/rivulet/__init__.py +0 -0
- deltacat/tests/{storage → experimental/storage}/rivulet/conftest.py +3 -3
- deltacat/tests/experimental/storage/rivulet/fs/__init__.py +0 -0
- deltacat/tests/{storage → experimental/storage}/rivulet/fs/test_file_location_provider.py +3 -3
- deltacat/tests/experimental/storage/rivulet/reader/__init__.py +0 -0
- deltacat/tests/experimental/storage/rivulet/reader/query_expression.py +80 -0
- deltacat/tests/experimental/storage/rivulet/reader/test_data_scan.py +119 -0
- deltacat/tests/experimental/storage/rivulet/reader/test_dataset_metastore.py +71 -0
- deltacat/tests/experimental/storage/rivulet/schema/__init__.py +0 -0
- deltacat/tests/{storage → experimental/storage}/rivulet/schema/test_schema.py +1 -1
- deltacat/tests/experimental/storage/rivulet/shard/__init__.py +0 -0
- deltacat/tests/experimental/storage/rivulet/shard/test_range_shard.py +162 -0
- deltacat/tests/{storage → experimental/storage}/rivulet/test_dataset.py +5 -3
- deltacat/tests/{storage → experimental/storage}/rivulet/test_manifest.py +5 -5
- deltacat/tests/{storage → experimental/storage}/rivulet/test_sst_interval_tree.py +5 -5
- deltacat/tests/{storage → experimental/storage}/rivulet/test_utils.py +8 -6
- deltacat/tests/experimental/storage/rivulet/writer/__init__.py +0 -0
- deltacat/tests/{storage → experimental/storage}/rivulet/writer/test_dataset_write_then_read.py +11 -9
- deltacat/tests/{storage → experimental/storage}/rivulet/writer/test_dataset_writer.py +2 -2
- deltacat/tests/{storage → experimental/storage}/rivulet/writer/test_memtable_dataset_writer.py +7 -7
- deltacat/tests/storage/main/test_main_storage.py +6900 -95
- deltacat/tests/storage/model/test_metafile_io.py +78 -173
- deltacat/tests/storage/model/test_partition_scheme.py +85 -0
- deltacat/tests/storage/model/test_schema.py +171 -0
- deltacat/tests/storage/model/test_schema_update.py +1925 -0
- deltacat/tests/storage/model/test_shard.py +3 -1
- deltacat/tests/storage/model/test_sort_scheme.py +90 -0
- deltacat/tests/storage/model/test_transaction.py +393 -48
- deltacat/tests/storage/model/test_transaction_history.py +886 -0
- deltacat/tests/test_deltacat_api.py +988 -4
- deltacat/tests/test_exceptions.py +9 -5
- deltacat/tests/test_utils/pyarrow.py +52 -21
- deltacat/tests/test_utils/storage.py +23 -34
- deltacat/tests/types/__init__.py +0 -0
- deltacat/tests/types/test_tables.py +104 -0
- deltacat/tests/utils/exceptions.py +22 -0
- deltacat/tests/utils/main_deltacat_storage_mock.py +31 -0
- deltacat/tests/utils/ray_utils/test_dataset.py +123 -5
- deltacat/tests/utils/test_daft.py +121 -31
- deltacat/tests/utils/test_numpy.py +1193 -0
- deltacat/tests/utils/test_pandas.py +1106 -0
- deltacat/tests/utils/test_polars.py +1040 -0
- deltacat/tests/utils/test_pyarrow.py +1370 -89
- deltacat/types/media.py +224 -14
- deltacat/types/tables.py +2329 -59
- deltacat/utils/arguments.py +33 -1
- deltacat/utils/daft.py +823 -36
- deltacat/utils/export.py +3 -1
- deltacat/utils/filesystem.py +100 -0
- deltacat/utils/metafile_locator.py +2 -1
- deltacat/utils/numpy.py +118 -26
- deltacat/utils/pandas.py +577 -48
- deltacat/utils/polars.py +658 -27
- deltacat/utils/pyarrow.py +1258 -213
- deltacat/utils/ray_utils/dataset.py +101 -10
- deltacat/utils/reader_compatibility_mapping.py +3083 -0
- deltacat/utils/url.py +57 -16
- deltacat-2.0.0b12.dist-info/METADATA +1163 -0
- deltacat-2.0.0b12.dist-info/RECORD +439 -0
- {deltacat-2.0.0b10.dist-info → deltacat-2.0.0b12.dist-info}/WHEEL +1 -1
- deltacat/catalog/iceberg/__init__.py +0 -4
- deltacat/compute/compactor/utils/round_completion_file.py +0 -97
- deltacat/compute/merge_on_read/__init__.py +0 -4
- deltacat/compute/merge_on_read/daft.py +0 -40
- deltacat/compute/merge_on_read/model/merge_on_read_params.py +0 -66
- deltacat/compute/merge_on_read/utils/delta.py +0 -42
- deltacat/daft/daft_scan.py +0 -115
- deltacat/daft/model.py +0 -258
- deltacat/daft/translator.py +0 -126
- deltacat/examples/common/fixtures.py +0 -15
- deltacat/storage/iceberg/iceberg_scan_planner.py +0 -28
- deltacat/storage/rivulet/__init__.py +0 -11
- deltacat/storage/rivulet/feather/__init__.py +0 -5
- deltacat/storage/rivulet/parquet/__init__.py +0 -5
- deltacat/tests/compute/compactor/utils/test_round_completion_file.py +0 -231
- deltacat/tests/compute/test_util_create_table_deltas_repo.py +0 -388
- deltacat/tests/local_deltacat_storage/__init__.py +0 -1236
- deltacat/tests/local_deltacat_storage/exceptions.py +0 -10
- deltacat/utils/s3fs.py +0 -21
- deltacat-2.0.0b10.dist-info/METADATA +0 -68
- deltacat-2.0.0b10.dist-info/RECORD +0 -381
- /deltacat/{compute/merge_on_read/model → docs}/__init__.py +0 -0
- /deltacat/{compute/merge_on_read/utils → docs/autogen}/__init__.py +0 -0
- /deltacat/{daft → docs/autogen/schema}/__init__.py +0 -0
- /deltacat/{examples/common → docs/autogen/schema/inference}/__init__.py +0 -0
- /deltacat/examples/{iceberg → compactor}/__init__.py +0 -0
- /deltacat/{storage/iceberg → examples/experimental}/__init__.py +0 -0
- /deltacat/{storage/rivulet/arrow → examples/experimental/iceberg}/__init__.py +0 -0
- /deltacat/{storage/rivulet/fs → examples/experimental/iceberg/converter}/__init__.py +0 -0
- /deltacat/{storage/rivulet/metastore → examples/experimental/iceberg/converter/beam}/__init__.py +0 -0
- /deltacat/{storage/rivulet/reader → experimental/catalog}/__init__.py +0 -0
- /deltacat/{catalog → experimental/catalog}/iceberg/overrides.py +0 -0
- /deltacat/{storage/rivulet/schema → experimental/compatibility}/__init__.py +0 -0
- /deltacat/{storage/rivulet/writer → experimental/converter_agent}/__init__.py +0 -0
- /deltacat/{tests/storage/rivulet → experimental/converter_agent/beam}/__init__.py +0 -0
- /deltacat/{tests/storage/rivulet/fs → experimental/storage}/__init__.py +0 -0
- /deltacat/{tests/storage/rivulet/schema → experimental/storage/iceberg}/__init__.py +0 -0
- /deltacat/{tests/storage/rivulet/writer → experimental/storage/rivulet/arrow}/__init__.py +0 -0
- /deltacat/{storage/rivulet/parquet/data_reader.py → experimental/storage/rivulet/fs/__init__.py} +0 -0
- /deltacat/{storage → experimental/storage}/rivulet/fs/input_file.py +0 -0
- /deltacat/{storage → experimental/storage}/rivulet/mvp/Table.py +0 -0
- /deltacat/{storage → experimental/storage}/rivulet/mvp/__init__.py +0 -0
- /deltacat/{storage → experimental/storage}/rivulet/reader/query_expression.py +0 -0
- /deltacat/{storage → experimental/storage}/rivulet/schema/datatype.py +0 -0
- /deltacat/{storage → experimental/storage}/rivulet/writer/dataset_writer.py +0 -0
- {deltacat-2.0.0b10.dist-info → deltacat-2.0.0b12.dist-info/licenses}/LICENSE +0 -0
- {deltacat-2.0.0b10.dist-info → deltacat-2.0.0b12.dist-info}/top_level.txt +0 -0
@@ -20,6 +20,12 @@ from deltacat import (
|
|
20
20
|
NullOrder,
|
21
21
|
LifecycleState,
|
22
22
|
)
|
23
|
+
from deltacat.exceptions import (
|
24
|
+
ObjectAlreadyExistsError,
|
25
|
+
ConcurrentModificationError,
|
26
|
+
ObjectDeletedError,
|
27
|
+
ObjectNotFoundError,
|
28
|
+
)
|
23
29
|
from deltacat.storage import (
|
24
30
|
BucketTransform,
|
25
31
|
BucketTransformParameters,
|
@@ -47,7 +53,6 @@ from deltacat.storage import (
|
|
47
53
|
TableVersion,
|
48
54
|
Transaction,
|
49
55
|
TransactionOperation,
|
50
|
-
TransactionType,
|
51
56
|
TransactionOperationType,
|
52
57
|
TruncateTransform,
|
53
58
|
TruncateTransformParameters,
|
@@ -92,7 +97,6 @@ def _commit_single_delta_table(temp_dir: str) -> List[Tuple[Metafile, Metafile,
|
|
92
97
|
for meta in meta_to_create
|
93
98
|
]
|
94
99
|
transaction = Transaction.of(
|
95
|
-
txn_type=TransactionType.APPEND,
|
96
100
|
txn_operations=txn_operations,
|
97
101
|
)
|
98
102
|
write_paths, txn_log_path = transaction.commit(temp_dir)
|
@@ -116,7 +120,7 @@ def _commit_concurrent_transaction(
|
|
116
120
|
) -> None:
|
117
121
|
try:
|
118
122
|
return transaction.commit(catalog_root)
|
119
|
-
except (
|
123
|
+
except (ObjectAlreadyExistsError, ConcurrentModificationError) as e:
|
120
124
|
return e
|
121
125
|
|
122
126
|
|
@@ -133,7 +137,6 @@ class TestMetafileIO:
|
|
133
137
|
description="test table description",
|
134
138
|
)
|
135
139
|
transaction = Transaction.of(
|
136
|
-
txn_type=TransactionType.APPEND,
|
137
140
|
txn_operations=[
|
138
141
|
TransactionOperation.of(
|
139
142
|
operation_type=TransactionOperationType.CREATE,
|
@@ -158,9 +161,8 @@ class TestMetafileIO:
|
|
158
161
|
results = [future.get() for future in futures]
|
159
162
|
conflict_exception_count = 0
|
160
163
|
for result in results:
|
161
|
-
|
162
|
-
|
163
|
-
result, ValueError
|
164
|
+
if isinstance(result, ConcurrentModificationError) or isinstance(
|
165
|
+
result, ObjectAlreadyExistsError
|
164
166
|
):
|
165
167
|
conflict_exception_count += 1
|
166
168
|
else:
|
@@ -174,7 +176,6 @@ class TestMetafileIO:
|
|
174
176
|
namespace = Namespace.of(locator=namespace_locator)
|
175
177
|
# given a transaction that creates a single namespace
|
176
178
|
transaction = Transaction.of(
|
177
|
-
txn_type=TransactionType.APPEND,
|
178
179
|
txn_operations=[
|
179
180
|
TransactionOperation.of(
|
180
181
|
operation_type=TransactionOperationType.CREATE,
|
@@ -213,7 +214,6 @@ class TestMetafileIO:
|
|
213
214
|
)
|
214
215
|
]
|
215
216
|
transaction = Transaction.of(
|
216
|
-
txn_type=TransactionType.ALTER,
|
217
217
|
txn_operations=txn_operations,
|
218
218
|
)
|
219
219
|
# expect the bad timestamp to be detected and its commit to fail
|
@@ -274,7 +274,6 @@ class TestMetafileIO:
|
|
274
274
|
)
|
275
275
|
]
|
276
276
|
transaction = Transaction.of(
|
277
|
-
txn_type=TransactionType.ALTER,
|
278
277
|
txn_operations=txn_operations,
|
279
278
|
)
|
280
279
|
# expect the commit to fail due to a concurrent modification error
|
@@ -304,7 +303,6 @@ class TestMetafileIO:
|
|
304
303
|
original_delta = Delta.read(orig_delta_write_path)
|
305
304
|
new_delta = Delta.update_for(original_delta)
|
306
305
|
transaction = Transaction.of(
|
307
|
-
txn_type=TransactionType.ALTER,
|
308
306
|
txn_operations=[
|
309
307
|
TransactionOperation.of(
|
310
308
|
operation_type=TransactionOperationType.UPDATE,
|
@@ -314,10 +312,10 @@ class TestMetafileIO:
|
|
314
312
|
],
|
315
313
|
)
|
316
314
|
# expect the commit to fail due to a concurrent modification error
|
317
|
-
with pytest.raises(
|
315
|
+
with pytest.raises(ConcurrentModificationError):
|
318
316
|
transaction.commit(temp_dir)
|
319
317
|
# expect a commit retry to also fail
|
320
|
-
with pytest.raises(
|
318
|
+
with pytest.raises(ConcurrentModificationError):
|
321
319
|
transaction.commit(temp_dir)
|
322
320
|
|
323
321
|
def test_append_multiple_deltas(self, temp_dir):
|
@@ -342,7 +340,6 @@ class TestMetafileIO:
|
|
342
340
|
)
|
343
341
|
)
|
344
342
|
transaction = Transaction.of(
|
345
|
-
txn_type=TransactionType.APPEND,
|
346
343
|
txn_operations=txn_operations,
|
347
344
|
)
|
348
345
|
# when the transaction is committed
|
@@ -387,7 +384,6 @@ class TestMetafileIO:
|
|
387
384
|
)
|
388
385
|
]
|
389
386
|
transaction = Transaction.of(
|
390
|
-
txn_type=TransactionType.DELETE,
|
391
387
|
txn_operations=txn_operations,
|
392
388
|
)
|
393
389
|
# when the transaction is committed
|
@@ -415,10 +411,9 @@ class TestMetafileIO:
|
|
415
411
|
)
|
416
412
|
]
|
417
413
|
transaction = Transaction.of(
|
418
|
-
txn_type=TransactionType.OVERWRITE,
|
419
414
|
txn_operations=bad_txn_operations,
|
420
415
|
)
|
421
|
-
with pytest.raises(
|
416
|
+
with pytest.raises(ObjectDeletedError):
|
422
417
|
transaction.commit(temp_dir)
|
423
418
|
|
424
419
|
# expect subsequent deletes of the deleted delta to fail
|
@@ -429,10 +424,9 @@ class TestMetafileIO:
|
|
429
424
|
)
|
430
425
|
]
|
431
426
|
transaction = Transaction.of(
|
432
|
-
txn_type=TransactionType.DELETE,
|
433
427
|
txn_operations=bad_txn_operations,
|
434
428
|
)
|
435
|
-
with pytest.raises(
|
429
|
+
with pytest.raises(ObjectDeletedError):
|
436
430
|
transaction.commit(temp_dir)
|
437
431
|
|
438
432
|
def test_replace_delta(self, temp_dir):
|
@@ -452,13 +446,12 @@ class TestMetafileIO:
|
|
452
446
|
|
453
447
|
txn_operations = [
|
454
448
|
TransactionOperation.of(
|
455
|
-
operation_type=TransactionOperationType.
|
449
|
+
operation_type=TransactionOperationType.REPLACE,
|
456
450
|
dest_metafile=replacement_delta,
|
457
451
|
src_metafile=original_delta,
|
458
452
|
)
|
459
453
|
]
|
460
454
|
transaction = Transaction.of(
|
461
|
-
txn_type=TransactionType.OVERWRITE,
|
462
455
|
txn_operations=txn_operations,
|
463
456
|
)
|
464
457
|
# when the transaction is committed
|
@@ -483,16 +476,15 @@ class TestMetafileIO:
|
|
483
476
|
# expect a subsequent replace of the original delta to fail
|
484
477
|
bad_txn_operations = [
|
485
478
|
TransactionOperation.of(
|
486
|
-
operation_type=TransactionOperationType.
|
479
|
+
operation_type=TransactionOperationType.REPLACE,
|
487
480
|
dest_metafile=replacement_delta,
|
488
481
|
src_metafile=original_delta,
|
489
482
|
)
|
490
483
|
]
|
491
484
|
transaction = Transaction.of(
|
492
|
-
txn_type=TransactionType.OVERWRITE,
|
493
485
|
txn_operations=bad_txn_operations,
|
494
486
|
)
|
495
|
-
with pytest.raises(
|
487
|
+
with pytest.raises(ObjectDeletedError):
|
496
488
|
transaction.commit(temp_dir)
|
497
489
|
|
498
490
|
# expect deletes of the original delta to fail
|
@@ -503,10 +495,9 @@ class TestMetafileIO:
|
|
503
495
|
)
|
504
496
|
]
|
505
497
|
transaction = Transaction.of(
|
506
|
-
txn_type=TransactionType.DELETE,
|
507
498
|
txn_operations=bad_txn_operations,
|
508
499
|
)
|
509
|
-
with pytest.raises(
|
500
|
+
with pytest.raises(ObjectDeletedError):
|
510
501
|
transaction.commit(temp_dir)
|
511
502
|
|
512
503
|
def test_delete_partition(self, temp_dir):
|
@@ -522,7 +513,6 @@ class TestMetafileIO:
|
|
522
513
|
)
|
523
514
|
]
|
524
515
|
transaction = Transaction.of(
|
525
|
-
txn_type=TransactionType.DELETE,
|
526
516
|
txn_operations=txn_operations,
|
527
517
|
)
|
528
518
|
# when the transaction is committed
|
@@ -562,16 +552,15 @@ class TestMetafileIO:
|
|
562
552
|
)
|
563
553
|
bad_txn_operations = [
|
564
554
|
TransactionOperation.of(
|
565
|
-
operation_type=TransactionOperationType.
|
555
|
+
operation_type=TransactionOperationType.REPLACE,
|
566
556
|
dest_metafile=replacement_partition,
|
567
557
|
src_metafile=original_partition,
|
568
558
|
)
|
569
559
|
]
|
570
560
|
transaction = Transaction.of(
|
571
|
-
txn_type=TransactionType.OVERWRITE,
|
572
561
|
txn_operations=bad_txn_operations,
|
573
562
|
)
|
574
|
-
with pytest.raises(
|
563
|
+
with pytest.raises(ObjectDeletedError):
|
575
564
|
transaction.commit(temp_dir)
|
576
565
|
|
577
566
|
# expect subsequent deletes of the deleted partition to fail
|
@@ -582,10 +571,9 @@ class TestMetafileIO:
|
|
582
571
|
)
|
583
572
|
]
|
584
573
|
transaction = Transaction.of(
|
585
|
-
txn_type=TransactionType.DELETE,
|
586
574
|
txn_operations=bad_txn_operations,
|
587
575
|
)
|
588
|
-
with pytest.raises(
|
576
|
+
with pytest.raises(ObjectDeletedError):
|
589
577
|
transaction.commit(temp_dir)
|
590
578
|
|
591
579
|
# expect new child metafile creation under the deleted partition to fail
|
@@ -597,10 +585,9 @@ class TestMetafileIO:
|
|
597
585
|
)
|
598
586
|
]
|
599
587
|
transaction = Transaction.of(
|
600
|
-
txn_type=TransactionType.APPEND,
|
601
588
|
txn_operations=bad_txn_operations,
|
602
589
|
)
|
603
|
-
with pytest.raises(
|
590
|
+
with pytest.raises(ObjectAlreadyExistsError):
|
604
591
|
transaction.commit(temp_dir)
|
605
592
|
|
606
593
|
def test_replace_partition(self, temp_dir):
|
@@ -620,13 +607,12 @@ class TestMetafileIO:
|
|
620
607
|
|
621
608
|
txn_operations = [
|
622
609
|
TransactionOperation.of(
|
623
|
-
operation_type=TransactionOperationType.
|
610
|
+
operation_type=TransactionOperationType.REPLACE,
|
624
611
|
dest_metafile=replacement_partition,
|
625
612
|
src_metafile=original_partition,
|
626
613
|
)
|
627
614
|
]
|
628
615
|
transaction = Transaction.of(
|
629
|
-
txn_type=TransactionType.OVERWRITE,
|
630
616
|
txn_operations=txn_operations,
|
631
617
|
)
|
632
618
|
# when the transaction is committed
|
@@ -679,16 +665,15 @@ class TestMetafileIO:
|
|
679
665
|
# expect a subsequent replace of the original partition to fail
|
680
666
|
bad_txn_operations = [
|
681
667
|
TransactionOperation.of(
|
682
|
-
operation_type=TransactionOperationType.
|
668
|
+
operation_type=TransactionOperationType.REPLACE,
|
683
669
|
dest_metafile=replacement_partition,
|
684
670
|
src_metafile=original_partition,
|
685
671
|
)
|
686
672
|
]
|
687
673
|
transaction = Transaction.of(
|
688
|
-
txn_type=TransactionType.OVERWRITE,
|
689
674
|
txn_operations=bad_txn_operations,
|
690
675
|
)
|
691
|
-
with pytest.raises(
|
676
|
+
with pytest.raises(ObjectDeletedError):
|
692
677
|
transaction.commit(temp_dir)
|
693
678
|
|
694
679
|
# expect deletes of the original partition to fail
|
@@ -699,10 +684,9 @@ class TestMetafileIO:
|
|
699
684
|
)
|
700
685
|
]
|
701
686
|
transaction = Transaction.of(
|
702
|
-
txn_type=TransactionType.DELETE,
|
703
687
|
txn_operations=bad_txn_operations,
|
704
688
|
)
|
705
|
-
with pytest.raises(
|
689
|
+
with pytest.raises(ObjectDeletedError):
|
706
690
|
transaction.commit(temp_dir)
|
707
691
|
|
708
692
|
# expect new child metafile creation under the old partition to fail
|
@@ -714,10 +698,9 @@ class TestMetafileIO:
|
|
714
698
|
)
|
715
699
|
]
|
716
700
|
transaction = Transaction.of(
|
717
|
-
txn_type=TransactionType.APPEND,
|
718
701
|
txn_operations=bad_txn_operations,
|
719
702
|
)
|
720
|
-
with pytest.raises(
|
703
|
+
with pytest.raises(ObjectAlreadyExistsError):
|
721
704
|
transaction.commit(temp_dir)
|
722
705
|
|
723
706
|
def test_delete_stream(self, temp_dir):
|
@@ -733,7 +716,6 @@ class TestMetafileIO:
|
|
733
716
|
)
|
734
717
|
]
|
735
718
|
transaction = Transaction.of(
|
736
|
-
txn_type=TransactionType.DELETE,
|
737
719
|
txn_operations=txn_operations,
|
738
720
|
)
|
739
721
|
# when the transaction is committed
|
@@ -776,16 +758,15 @@ class TestMetafileIO:
|
|
776
758
|
)
|
777
759
|
bad_txn_operations = [
|
778
760
|
TransactionOperation.of(
|
779
|
-
operation_type=TransactionOperationType.
|
761
|
+
operation_type=TransactionOperationType.REPLACE,
|
780
762
|
dest_metafile=replacement_stream,
|
781
763
|
src_metafile=original_stream,
|
782
764
|
)
|
783
765
|
]
|
784
766
|
transaction = Transaction.of(
|
785
|
-
txn_type=TransactionType.OVERWRITE,
|
786
767
|
txn_operations=bad_txn_operations,
|
787
768
|
)
|
788
|
-
with pytest.raises(
|
769
|
+
with pytest.raises(ObjectDeletedError):
|
789
770
|
transaction.commit(temp_dir)
|
790
771
|
|
791
772
|
# expect subsequent deletes of the deleted stream to fail
|
@@ -796,10 +777,9 @@ class TestMetafileIO:
|
|
796
777
|
)
|
797
778
|
]
|
798
779
|
transaction = Transaction.of(
|
799
|
-
txn_type=TransactionType.DELETE,
|
800
780
|
txn_operations=bad_txn_operations,
|
801
781
|
)
|
802
|
-
with pytest.raises(
|
782
|
+
with pytest.raises(ObjectDeletedError):
|
803
783
|
transaction.commit(temp_dir)
|
804
784
|
|
805
785
|
# expect new child metafile creation under the deleted stream to fail
|
@@ -811,10 +791,9 @@ class TestMetafileIO:
|
|
811
791
|
)
|
812
792
|
]
|
813
793
|
transaction = Transaction.of(
|
814
|
-
txn_type=TransactionType.APPEND,
|
815
794
|
txn_operations=bad_txn_operations,
|
816
795
|
)
|
817
|
-
with pytest.raises(
|
796
|
+
with pytest.raises(ObjectAlreadyExistsError):
|
818
797
|
transaction.commit(temp_dir)
|
819
798
|
|
820
799
|
def test_replace_stream(self, temp_dir):
|
@@ -834,13 +813,12 @@ class TestMetafileIO:
|
|
834
813
|
|
835
814
|
txn_operations = [
|
836
815
|
TransactionOperation.of(
|
837
|
-
operation_type=TransactionOperationType.
|
816
|
+
operation_type=TransactionOperationType.REPLACE,
|
838
817
|
dest_metafile=replacement_stream,
|
839
818
|
src_metafile=original_stream,
|
840
819
|
)
|
841
820
|
]
|
842
821
|
transaction = Transaction.of(
|
843
|
-
txn_type=TransactionType.OVERWRITE,
|
844
822
|
txn_operations=txn_operations,
|
845
823
|
)
|
846
824
|
# when the transaction is committed
|
@@ -896,16 +874,15 @@ class TestMetafileIO:
|
|
896
874
|
# expect a subsequent replace of the original stream to fail
|
897
875
|
bad_txn_operations = [
|
898
876
|
TransactionOperation.of(
|
899
|
-
operation_type=TransactionOperationType.
|
877
|
+
operation_type=TransactionOperationType.REPLACE,
|
900
878
|
dest_metafile=replacement_stream,
|
901
879
|
src_metafile=original_stream,
|
902
880
|
)
|
903
881
|
]
|
904
882
|
transaction = Transaction.of(
|
905
|
-
txn_type=TransactionType.OVERWRITE,
|
906
883
|
txn_operations=bad_txn_operations,
|
907
884
|
)
|
908
|
-
with pytest.raises(
|
885
|
+
with pytest.raises(ObjectDeletedError):
|
909
886
|
transaction.commit(temp_dir)
|
910
887
|
|
911
888
|
# expect deletes of the original stream to fail
|
@@ -916,10 +893,9 @@ class TestMetafileIO:
|
|
916
893
|
)
|
917
894
|
]
|
918
895
|
transaction = Transaction.of(
|
919
|
-
txn_type=TransactionType.DELETE,
|
920
896
|
txn_operations=bad_txn_operations,
|
921
897
|
)
|
922
|
-
with pytest.raises(
|
898
|
+
with pytest.raises(ObjectDeletedError):
|
923
899
|
transaction.commit(temp_dir)
|
924
900
|
|
925
901
|
# expect new child metafile creation under the old stream to fail
|
@@ -931,10 +907,9 @@ class TestMetafileIO:
|
|
931
907
|
)
|
932
908
|
]
|
933
909
|
transaction = Transaction.of(
|
934
|
-
txn_type=TransactionType.APPEND,
|
935
910
|
txn_operations=bad_txn_operations,
|
936
911
|
)
|
937
|
-
with pytest.raises(
|
912
|
+
with pytest.raises(ObjectAlreadyExistsError):
|
938
913
|
transaction.commit(temp_dir)
|
939
914
|
|
940
915
|
def test_delete_table_version(self, temp_dir):
|
@@ -950,7 +925,6 @@ class TestMetafileIO:
|
|
950
925
|
)
|
951
926
|
]
|
952
927
|
transaction = Transaction.of(
|
953
|
-
txn_type=TransactionType.DELETE,
|
954
928
|
txn_operations=txn_operations,
|
955
929
|
)
|
956
930
|
# when the transaction is committed
|
@@ -996,16 +970,15 @@ class TestMetafileIO:
|
|
996
970
|
)
|
997
971
|
bad_txn_operations = [
|
998
972
|
TransactionOperation.of(
|
999
|
-
operation_type=TransactionOperationType.
|
973
|
+
operation_type=TransactionOperationType.REPLACE,
|
1000
974
|
dest_metafile=replacement_table_version,
|
1001
975
|
src_metafile=original_table_version,
|
1002
976
|
)
|
1003
977
|
]
|
1004
978
|
transaction = Transaction.of(
|
1005
|
-
txn_type=TransactionType.OVERWRITE,
|
1006
979
|
txn_operations=bad_txn_operations,
|
1007
980
|
)
|
1008
|
-
with pytest.raises(
|
981
|
+
with pytest.raises(ObjectDeletedError):
|
1009
982
|
transaction.commit(temp_dir)
|
1010
983
|
|
1011
984
|
# expect subsequent deletes of the deleted table version to fail
|
@@ -1016,10 +989,9 @@ class TestMetafileIO:
|
|
1016
989
|
)
|
1017
990
|
]
|
1018
991
|
transaction = Transaction.of(
|
1019
|
-
txn_type=TransactionType.DELETE,
|
1020
992
|
txn_operations=bad_txn_operations,
|
1021
993
|
)
|
1022
|
-
with pytest.raises(
|
994
|
+
with pytest.raises(ObjectDeletedError):
|
1023
995
|
transaction.commit(temp_dir)
|
1024
996
|
|
1025
997
|
# expect new child metafile creation under the deleted table version to fail
|
@@ -1031,10 +1003,9 @@ class TestMetafileIO:
|
|
1031
1003
|
)
|
1032
1004
|
]
|
1033
1005
|
transaction = Transaction.of(
|
1034
|
-
txn_type=TransactionType.APPEND,
|
1035
1006
|
txn_operations=bad_txn_operations,
|
1036
1007
|
)
|
1037
|
-
with pytest.raises(
|
1008
|
+
with pytest.raises(ObjectAlreadyExistsError):
|
1038
1009
|
transaction.commit(temp_dir)
|
1039
1010
|
|
1040
1011
|
def test_replace_table_version(self, temp_dir):
|
@@ -1054,13 +1025,12 @@ class TestMetafileIO:
|
|
1054
1025
|
|
1055
1026
|
txn_operations = [
|
1056
1027
|
TransactionOperation.of(
|
1057
|
-
operation_type=TransactionOperationType.
|
1028
|
+
operation_type=TransactionOperationType.REPLACE,
|
1058
1029
|
dest_metafile=replacement_table_version,
|
1059
1030
|
src_metafile=original_table_version,
|
1060
1031
|
)
|
1061
1032
|
]
|
1062
1033
|
transaction = Transaction.of(
|
1063
|
-
txn_type=TransactionType.OVERWRITE,
|
1064
1034
|
txn_operations=txn_operations,
|
1065
1035
|
)
|
1066
1036
|
# when the transaction is committed
|
@@ -1120,16 +1090,15 @@ class TestMetafileIO:
|
|
1120
1090
|
# expect a subsequent replace of the original table version to fail
|
1121
1091
|
bad_txn_operations = [
|
1122
1092
|
TransactionOperation.of(
|
1123
|
-
operation_type=TransactionOperationType.
|
1093
|
+
operation_type=TransactionOperationType.REPLACE,
|
1124
1094
|
dest_metafile=replacement_table_version,
|
1125
1095
|
src_metafile=original_table_version,
|
1126
1096
|
)
|
1127
1097
|
]
|
1128
1098
|
transaction = Transaction.of(
|
1129
|
-
txn_type=TransactionType.OVERWRITE,
|
1130
1099
|
txn_operations=bad_txn_operations,
|
1131
1100
|
)
|
1132
|
-
with pytest.raises(
|
1101
|
+
with pytest.raises(ObjectDeletedError):
|
1133
1102
|
transaction.commit(temp_dir)
|
1134
1103
|
|
1135
1104
|
# expect deletes of the original table version to fail
|
@@ -1140,10 +1109,9 @@ class TestMetafileIO:
|
|
1140
1109
|
)
|
1141
1110
|
]
|
1142
1111
|
transaction = Transaction.of(
|
1143
|
-
txn_type=TransactionType.DELETE,
|
1144
1112
|
txn_operations=bad_txn_operations,
|
1145
1113
|
)
|
1146
|
-
with pytest.raises(
|
1114
|
+
with pytest.raises(ObjectDeletedError):
|
1147
1115
|
transaction.commit(temp_dir)
|
1148
1116
|
|
1149
1117
|
# expect new child metafile creation under the old table version to fail
|
@@ -1155,10 +1123,9 @@ class TestMetafileIO:
|
|
1155
1123
|
)
|
1156
1124
|
]
|
1157
1125
|
transaction = Transaction.of(
|
1158
|
-
txn_type=TransactionType.APPEND,
|
1159
1126
|
txn_operations=bad_txn_operations,
|
1160
1127
|
)
|
1161
|
-
with pytest.raises(
|
1128
|
+
with pytest.raises(ObjectAlreadyExistsError):
|
1162
1129
|
transaction.commit(temp_dir)
|
1163
1130
|
|
1164
1131
|
def test_delete_table(self, temp_dir):
|
@@ -1174,7 +1141,6 @@ class TestMetafileIO:
|
|
1174
1141
|
)
|
1175
1142
|
]
|
1176
1143
|
transaction = Transaction.of(
|
1177
|
-
txn_type=TransactionType.DELETE,
|
1178
1144
|
txn_operations=txn_operations,
|
1179
1145
|
)
|
1180
1146
|
# when the transaction is committed
|
@@ -1220,16 +1186,15 @@ class TestMetafileIO:
|
|
1220
1186
|
replacement_table: Table = Table.based_on(original_table)
|
1221
1187
|
bad_txn_operations = [
|
1222
1188
|
TransactionOperation.of(
|
1223
|
-
operation_type=TransactionOperationType.
|
1189
|
+
operation_type=TransactionOperationType.REPLACE,
|
1224
1190
|
dest_metafile=replacement_table,
|
1225
1191
|
src_metafile=original_table,
|
1226
1192
|
)
|
1227
1193
|
]
|
1228
1194
|
transaction = Transaction.of(
|
1229
|
-
txn_type=TransactionType.OVERWRITE,
|
1230
1195
|
txn_operations=bad_txn_operations,
|
1231
1196
|
)
|
1232
|
-
with pytest.raises(
|
1197
|
+
with pytest.raises(ObjectDeletedError):
|
1233
1198
|
transaction.commit(temp_dir)
|
1234
1199
|
|
1235
1200
|
# expect subsequent deletes of the deleted table to fail
|
@@ -1240,10 +1205,9 @@ class TestMetafileIO:
|
|
1240
1205
|
)
|
1241
1206
|
]
|
1242
1207
|
transaction = Transaction.of(
|
1243
|
-
txn_type=TransactionType.DELETE,
|
1244
1208
|
txn_operations=bad_txn_operations,
|
1245
1209
|
)
|
1246
|
-
with pytest.raises(
|
1210
|
+
with pytest.raises(ObjectDeletedError):
|
1247
1211
|
transaction.commit(temp_dir)
|
1248
1212
|
|
1249
1213
|
# expect new child metafile creation under the deleted table to fail
|
@@ -1255,10 +1219,9 @@ class TestMetafileIO:
|
|
1255
1219
|
)
|
1256
1220
|
]
|
1257
1221
|
transaction = Transaction.of(
|
1258
|
-
txn_type=TransactionType.APPEND,
|
1259
1222
|
txn_operations=bad_txn_operations,
|
1260
1223
|
)
|
1261
|
-
with pytest.raises(
|
1224
|
+
with pytest.raises(ObjectAlreadyExistsError):
|
1262
1225
|
transaction.commit(temp_dir)
|
1263
1226
|
|
1264
1227
|
def test_replace_table(self, temp_dir):
|
@@ -1277,13 +1240,12 @@ class TestMetafileIO:
|
|
1277
1240
|
|
1278
1241
|
txn_operations = [
|
1279
1242
|
TransactionOperation.of(
|
1280
|
-
operation_type=TransactionOperationType.
|
1243
|
+
operation_type=TransactionOperationType.REPLACE,
|
1281
1244
|
dest_metafile=replacement_table,
|
1282
1245
|
src_metafile=original_table,
|
1283
1246
|
)
|
1284
1247
|
]
|
1285
1248
|
transaction = Transaction.of(
|
1286
|
-
txn_type=TransactionType.OVERWRITE,
|
1287
1249
|
txn_operations=txn_operations,
|
1288
1250
|
)
|
1289
1251
|
# when the transaction is committed
|
@@ -1345,16 +1307,15 @@ class TestMetafileIO:
|
|
1345
1307
|
# expect a subsequent table replace of the original table to fail
|
1346
1308
|
bad_txn_operations = [
|
1347
1309
|
TransactionOperation.of(
|
1348
|
-
operation_type=TransactionOperationType.
|
1310
|
+
operation_type=TransactionOperationType.REPLACE,
|
1349
1311
|
dest_metafile=replacement_table,
|
1350
1312
|
src_metafile=original_table,
|
1351
1313
|
)
|
1352
1314
|
]
|
1353
1315
|
transaction = Transaction.of(
|
1354
|
-
txn_type=TransactionType.OVERWRITE,
|
1355
1316
|
txn_operations=bad_txn_operations,
|
1356
1317
|
)
|
1357
|
-
with pytest.raises(
|
1318
|
+
with pytest.raises(ObjectDeletedError):
|
1358
1319
|
transaction.commit(temp_dir)
|
1359
1320
|
|
1360
1321
|
# expect table deletes of the original table to fail
|
@@ -1365,10 +1326,9 @@ class TestMetafileIO:
|
|
1365
1326
|
)
|
1366
1327
|
]
|
1367
1328
|
transaction = Transaction.of(
|
1368
|
-
txn_type=TransactionType.DELETE,
|
1369
1329
|
txn_operations=bad_txn_operations,
|
1370
1330
|
)
|
1371
|
-
with pytest.raises(
|
1331
|
+
with pytest.raises(ObjectDeletedError):
|
1372
1332
|
transaction.commit(temp_dir)
|
1373
1333
|
|
1374
1334
|
# expect new child metafile creation under the old table to fail
|
@@ -1380,10 +1340,9 @@ class TestMetafileIO:
|
|
1380
1340
|
)
|
1381
1341
|
]
|
1382
1342
|
transaction = Transaction.of(
|
1383
|
-
txn_type=TransactionType.APPEND,
|
1384
1343
|
txn_operations=bad_txn_operations,
|
1385
1344
|
)
|
1386
|
-
with pytest.raises(
|
1345
|
+
with pytest.raises(ObjectAlreadyExistsError):
|
1387
1346
|
transaction.commit(temp_dir)
|
1388
1347
|
|
1389
1348
|
def test_delete_namespace(self, temp_dir):
|
@@ -1399,7 +1358,6 @@ class TestMetafileIO:
|
|
1399
1358
|
)
|
1400
1359
|
]
|
1401
1360
|
transaction = Transaction.of(
|
1402
|
-
txn_type=TransactionType.DELETE,
|
1403
1361
|
txn_operations=txn_operations,
|
1404
1362
|
)
|
1405
1363
|
# when the transaction is committed
|
@@ -1448,16 +1406,15 @@ class TestMetafileIO:
|
|
1448
1406
|
replacement_namespace: Namespace = Namespace.based_on(original_namespace)
|
1449
1407
|
bad_txn_operations = [
|
1450
1408
|
TransactionOperation.of(
|
1451
|
-
operation_type=TransactionOperationType.
|
1409
|
+
operation_type=TransactionOperationType.REPLACE,
|
1452
1410
|
dest_metafile=replacement_namespace,
|
1453
1411
|
src_metafile=original_namespace,
|
1454
1412
|
)
|
1455
1413
|
]
|
1456
1414
|
transaction = Transaction.of(
|
1457
|
-
txn_type=TransactionType.OVERWRITE,
|
1458
1415
|
txn_operations=bad_txn_operations,
|
1459
1416
|
)
|
1460
|
-
with pytest.raises(
|
1417
|
+
with pytest.raises(ObjectDeletedError):
|
1461
1418
|
transaction.commit(temp_dir)
|
1462
1419
|
|
1463
1420
|
# expect subsequent deletes of the deleted namespace to fail
|
@@ -1468,10 +1425,9 @@ class TestMetafileIO:
|
|
1468
1425
|
)
|
1469
1426
|
]
|
1470
1427
|
transaction = Transaction.of(
|
1471
|
-
txn_type=TransactionType.DELETE,
|
1472
1428
|
txn_operations=bad_txn_operations,
|
1473
1429
|
)
|
1474
|
-
with pytest.raises(
|
1430
|
+
with pytest.raises(ObjectDeletedError):
|
1475
1431
|
transaction.commit(temp_dir)
|
1476
1432
|
|
1477
1433
|
# expect new child metafile creation under the deleted namespace to fail
|
@@ -1483,10 +1439,9 @@ class TestMetafileIO:
|
|
1483
1439
|
)
|
1484
1440
|
]
|
1485
1441
|
transaction = Transaction.of(
|
1486
|
-
txn_type=TransactionType.APPEND,
|
1487
1442
|
txn_operations=bad_txn_operations,
|
1488
1443
|
)
|
1489
|
-
with pytest.raises(
|
1444
|
+
with pytest.raises(ObjectAlreadyExistsError):
|
1490
1445
|
transaction.commit(temp_dir)
|
1491
1446
|
|
1492
1447
|
def test_replace_namespace(self, temp_dir):
|
@@ -1505,13 +1460,12 @@ class TestMetafileIO:
|
|
1505
1460
|
|
1506
1461
|
txn_operations = [
|
1507
1462
|
TransactionOperation.of(
|
1508
|
-
operation_type=TransactionOperationType.
|
1463
|
+
operation_type=TransactionOperationType.REPLACE,
|
1509
1464
|
dest_metafile=replacement_namespace,
|
1510
1465
|
src_metafile=original_namespace,
|
1511
1466
|
)
|
1512
1467
|
]
|
1513
1468
|
transaction = Transaction.of(
|
1514
|
-
txn_type=TransactionType.OVERWRITE,
|
1515
1469
|
txn_operations=txn_operations,
|
1516
1470
|
)
|
1517
1471
|
# when the transaction is committed
|
@@ -1576,16 +1530,15 @@ class TestMetafileIO:
|
|
1576
1530
|
# expect a subsequent namespace replace of the original namespace to fail
|
1577
1531
|
bad_txn_operations = [
|
1578
1532
|
TransactionOperation.of(
|
1579
|
-
operation_type=TransactionOperationType.
|
1533
|
+
operation_type=TransactionOperationType.REPLACE,
|
1580
1534
|
dest_metafile=replacement_namespace,
|
1581
1535
|
src_metafile=original_namespace,
|
1582
1536
|
)
|
1583
1537
|
]
|
1584
1538
|
transaction = Transaction.of(
|
1585
|
-
txn_type=TransactionType.OVERWRITE,
|
1586
1539
|
txn_operations=bad_txn_operations,
|
1587
1540
|
)
|
1588
|
-
with pytest.raises(
|
1541
|
+
with pytest.raises(ObjectDeletedError):
|
1589
1542
|
transaction.commit(temp_dir)
|
1590
1543
|
|
1591
1544
|
# expect namespace deletes of the original namespace to fail
|
@@ -1596,10 +1549,9 @@ class TestMetafileIO:
|
|
1596
1549
|
)
|
1597
1550
|
]
|
1598
1551
|
transaction = Transaction.of(
|
1599
|
-
txn_type=TransactionType.DELETE,
|
1600
1552
|
txn_operations=bad_txn_operations,
|
1601
1553
|
)
|
1602
|
-
with pytest.raises(
|
1554
|
+
with pytest.raises(ObjectDeletedError):
|
1603
1555
|
transaction.commit(temp_dir)
|
1604
1556
|
|
1605
1557
|
# expect new child metafile creation under the old namespace to fail
|
@@ -1611,10 +1563,9 @@ class TestMetafileIO:
|
|
1611
1563
|
)
|
1612
1564
|
]
|
1613
1565
|
transaction = Transaction.of(
|
1614
|
-
txn_type=TransactionType.APPEND,
|
1615
1566
|
txn_operations=bad_txn_operations,
|
1616
1567
|
)
|
1617
|
-
with pytest.raises(
|
1568
|
+
with pytest.raises(ObjectAlreadyExistsError):
|
1618
1569
|
transaction.commit(temp_dir)
|
1619
1570
|
|
1620
1571
|
def test_create_stream_bad_order_txn_op_chaining(self, temp_dir):
|
@@ -1650,16 +1601,14 @@ class TestMetafileIO:
|
|
1650
1601
|
),
|
1651
1602
|
]
|
1652
1603
|
transaction = Transaction.of(
|
1653
|
-
txn_type=TransactionType.APPEND,
|
1654
1604
|
txn_operations=txn_operations,
|
1655
1605
|
)
|
1656
1606
|
# when the transaction is committed,
|
1657
1607
|
# expect stream creation to fail
|
1658
|
-
with pytest.raises(
|
1608
|
+
with pytest.raises(ObjectNotFoundError):
|
1659
1609
|
transaction.commit(temp_dir)
|
1660
1610
|
# when a transaction with the operations reversed is committed,
|
1661
1611
|
transaction = Transaction.of(
|
1662
|
-
txn_type=TransactionType.APPEND,
|
1663
1612
|
txn_operations=list(reversed(txn_operations)),
|
1664
1613
|
)
|
1665
1614
|
# expect table version and stream creation to succeed
|
@@ -1699,16 +1648,14 @@ class TestMetafileIO:
|
|
1699
1648
|
),
|
1700
1649
|
]
|
1701
1650
|
transaction = Transaction.of(
|
1702
|
-
txn_type=TransactionType.ALTER,
|
1703
1651
|
txn_operations=txn_operations,
|
1704
1652
|
)
|
1705
1653
|
# when the transaction is committed,
|
1706
1654
|
# expect the transaction to fail due to incorrect operation order
|
1707
|
-
with pytest.raises(
|
1655
|
+
with pytest.raises(ObjectNotFoundError):
|
1708
1656
|
transaction.commit(temp_dir)
|
1709
1657
|
# when a transaction with the operations reversed is committed,
|
1710
1658
|
transaction = Transaction.of(
|
1711
|
-
txn_type=TransactionType.ALTER,
|
1712
1659
|
txn_operations=list(reversed(txn_operations)),
|
1713
1660
|
)
|
1714
1661
|
# expect table and table version creation to succeed
|
@@ -1721,7 +1668,6 @@ class TestMetafileIO:
|
|
1721
1668
|
# given serial transaction that try to create two namespaces with
|
1722
1669
|
# the same name
|
1723
1670
|
transaction = Transaction.of(
|
1724
|
-
txn_type=TransactionType.APPEND,
|
1725
1671
|
txn_operations=[
|
1726
1672
|
TransactionOperation.of(
|
1727
1673
|
TransactionOperationType.CREATE,
|
@@ -1734,7 +1680,7 @@ class TestMetafileIO:
|
|
1734
1680
|
deserialized_namespace = Namespace.read(write_paths.pop())
|
1735
1681
|
assert namespace.equivalent_to(deserialized_namespace)
|
1736
1682
|
# but expect the second transaction to fail
|
1737
|
-
with pytest.raises(
|
1683
|
+
with pytest.raises(ObjectAlreadyExistsError):
|
1738
1684
|
transaction.commit(temp_dir)
|
1739
1685
|
|
1740
1686
|
def test_create_duplicate_namespace_txn_op_chaining(self, temp_dir):
|
@@ -1742,7 +1688,6 @@ class TestMetafileIO:
|
|
1742
1688
|
namespace = Namespace.of(locator=namespace_locator)
|
1743
1689
|
# given a transaction that tries to create the same namespace twice
|
1744
1690
|
transaction = Transaction.of(
|
1745
|
-
txn_type=TransactionType.APPEND,
|
1746
1691
|
txn_operations=[
|
1747
1692
|
TransactionOperation.of(
|
1748
1693
|
TransactionOperationType.CREATE,
|
@@ -1756,7 +1701,7 @@ class TestMetafileIO:
|
|
1756
1701
|
)
|
1757
1702
|
# when the transaction is committed,
|
1758
1703
|
# expect duplicate namespace creation to fail
|
1759
|
-
with pytest.raises(
|
1704
|
+
with pytest.raises(ObjectAlreadyExistsError):
|
1760
1705
|
transaction.commit(temp_dir)
|
1761
1706
|
|
1762
1707
|
def test_create_stream_in_missing_table_version(self, temp_dir):
|
@@ -1772,7 +1717,6 @@ class TestMetafileIO:
|
|
1772
1717
|
)
|
1773
1718
|
new_stream.table_version_locator.table_version = "missing_table_version.0"
|
1774
1719
|
transaction = Transaction.of(
|
1775
|
-
txn_type=TransactionType.APPEND,
|
1776
1720
|
txn_operations=[
|
1777
1721
|
TransactionOperation.of(
|
1778
1722
|
TransactionOperationType.CREATE,
|
@@ -1782,7 +1726,7 @@ class TestMetafileIO:
|
|
1782
1726
|
)
|
1783
1727
|
# when the transaction is committed,
|
1784
1728
|
# expect stream creation to fail
|
1785
|
-
with pytest.raises(
|
1729
|
+
with pytest.raises(ObjectNotFoundError):
|
1786
1730
|
transaction.commit(temp_dir)
|
1787
1731
|
|
1788
1732
|
def test_create_table_version_in_missing_namespace(self, temp_dir):
|
@@ -1798,7 +1742,6 @@ class TestMetafileIO:
|
|
1798
1742
|
)
|
1799
1743
|
new_table_version.namespace_locator.namespace = "missing_namespace"
|
1800
1744
|
transaction = Transaction.of(
|
1801
|
-
txn_type=TransactionType.APPEND,
|
1802
1745
|
txn_operations=[
|
1803
1746
|
TransactionOperation.of(
|
1804
1747
|
TransactionOperationType.CREATE,
|
@@ -1808,7 +1751,7 @@ class TestMetafileIO:
|
|
1808
1751
|
)
|
1809
1752
|
# when the transaction is committed,
|
1810
1753
|
# expect table version creation to fail
|
1811
|
-
with pytest.raises(
|
1754
|
+
with pytest.raises(ObjectNotFoundError):
|
1812
1755
|
transaction.commit(temp_dir)
|
1813
1756
|
|
1814
1757
|
def test_create_table_version_in_missing_table(self, temp_dir):
|
@@ -1824,7 +1767,6 @@ class TestMetafileIO:
|
|
1824
1767
|
)
|
1825
1768
|
new_table_version.table_locator.table_name = "missing_table"
|
1826
1769
|
transaction = Transaction.of(
|
1827
|
-
txn_type=TransactionType.APPEND,
|
1828
1770
|
txn_operations=[
|
1829
1771
|
TransactionOperation.of(
|
1830
1772
|
TransactionOperationType.CREATE,
|
@@ -1834,7 +1776,7 @@ class TestMetafileIO:
|
|
1834
1776
|
)
|
1835
1777
|
# when the transaction is committed,
|
1836
1778
|
# expect table version creation to fail
|
1837
|
-
with pytest.raises(
|
1779
|
+
with pytest.raises(ObjectNotFoundError):
|
1838
1780
|
transaction.commit(temp_dir)
|
1839
1781
|
|
1840
1782
|
def test_create_table_in_missing_namespace(self, temp_dir):
|
@@ -1849,7 +1791,6 @@ class TestMetafileIO:
|
|
1849
1791
|
# given a transaction that tries to create a single table in a
|
1850
1792
|
# namespace that doesn't exist
|
1851
1793
|
transaction = Transaction.of(
|
1852
|
-
txn_type=TransactionType.APPEND,
|
1853
1794
|
txn_operations=[
|
1854
1795
|
TransactionOperation.of(
|
1855
1796
|
TransactionOperationType.CREATE,
|
@@ -1859,7 +1800,7 @@ class TestMetafileIO:
|
|
1859
1800
|
)
|
1860
1801
|
# when the transaction is committed,
|
1861
1802
|
# expect table creation to fail
|
1862
|
-
with pytest.raises(
|
1803
|
+
with pytest.raises(ObjectNotFoundError):
|
1863
1804
|
transaction.commit(temp_dir)
|
1864
1805
|
|
1865
1806
|
def test_rename_table_txn_op_chaining(self, temp_dir):
|
@@ -1932,7 +1873,6 @@ class TestMetafileIO:
|
|
1932
1873
|
),
|
1933
1874
|
]
|
1934
1875
|
transaction = Transaction.of(
|
1935
|
-
txn_type=TransactionType.ALTER,
|
1936
1876
|
txn_operations=txn_operations,
|
1937
1877
|
)
|
1938
1878
|
# when the transaction is committed
|
@@ -1986,7 +1926,6 @@ class TestMetafileIO:
|
|
1986
1926
|
)
|
1987
1927
|
]
|
1988
1928
|
transaction = Transaction.of(
|
1989
|
-
txn_type=TransactionType.ALTER,
|
1990
1929
|
txn_operations=txn_operations,
|
1991
1930
|
)
|
1992
1931
|
# when the transaction is committed
|
@@ -2038,10 +1977,9 @@ class TestMetafileIO:
|
|
2038
1977
|
)
|
2039
1978
|
]
|
2040
1979
|
transaction = Transaction.of(
|
2041
|
-
txn_type=TransactionType.RESTATE,
|
2042
1980
|
txn_operations=bad_txn_operations,
|
2043
1981
|
)
|
2044
|
-
with pytest.raises(
|
1982
|
+
with pytest.raises(ObjectDeletedError):
|
2045
1983
|
transaction.commit(temp_dir)
|
2046
1984
|
|
2047
1985
|
# expect table deletes of the old table name fail
|
@@ -2052,10 +1990,9 @@ class TestMetafileIO:
|
|
2052
1990
|
)
|
2053
1991
|
]
|
2054
1992
|
transaction = Transaction.of(
|
2055
|
-
txn_type=TransactionType.DELETE,
|
2056
1993
|
txn_operations=bad_txn_operations,
|
2057
1994
|
)
|
2058
|
-
with pytest.raises(
|
1995
|
+
with pytest.raises(ObjectDeletedError):
|
2059
1996
|
transaction.commit(temp_dir)
|
2060
1997
|
|
2061
1998
|
# expect child metafile creation under the old table name to fail
|
@@ -2067,10 +2004,9 @@ class TestMetafileIO:
|
|
2067
2004
|
)
|
2068
2005
|
]
|
2069
2006
|
transaction = Transaction.of(
|
2070
|
-
txn_type=TransactionType.APPEND,
|
2071
2007
|
txn_operations=bad_txn_operations,
|
2072
2008
|
)
|
2073
|
-
with pytest.raises(
|
2009
|
+
with pytest.raises(ObjectAlreadyExistsError):
|
2074
2010
|
transaction.commit(temp_dir)
|
2075
2011
|
|
2076
2012
|
def test_rename_namespace(self, temp_dir):
|
@@ -2091,7 +2027,6 @@ class TestMetafileIO:
|
|
2091
2027
|
)
|
2092
2028
|
]
|
2093
2029
|
transaction = Transaction.of(
|
2094
|
-
txn_type=TransactionType.ALTER,
|
2095
2030
|
txn_operations=txn_operations,
|
2096
2031
|
)
|
2097
2032
|
# when the transaction is committed
|
@@ -2146,10 +2081,9 @@ class TestMetafileIO:
|
|
2146
2081
|
)
|
2147
2082
|
]
|
2148
2083
|
transaction = Transaction.of(
|
2149
|
-
txn_type=TransactionType.ALTER,
|
2150
2084
|
txn_operations=bad_txn_operations,
|
2151
2085
|
)
|
2152
|
-
with pytest.raises(
|
2086
|
+
with pytest.raises(ObjectDeletedError):
|
2153
2087
|
transaction.commit(temp_dir)
|
2154
2088
|
|
2155
2089
|
# expect namespace deletes of the old namespace name fail
|
@@ -2160,10 +2094,9 @@ class TestMetafileIO:
|
|
2160
2094
|
)
|
2161
2095
|
]
|
2162
2096
|
transaction = Transaction.of(
|
2163
|
-
txn_type=TransactionType.DELETE,
|
2164
2097
|
txn_operations=bad_txn_operations,
|
2165
2098
|
)
|
2166
|
-
with pytest.raises(
|
2099
|
+
with pytest.raises(ObjectDeletedError):
|
2167
2100
|
transaction.commit(temp_dir)
|
2168
2101
|
|
2169
2102
|
# expect child metafile creation under the old namespace to fail
|
@@ -2175,10 +2108,9 @@ class TestMetafileIO:
|
|
2175
2108
|
)
|
2176
2109
|
]
|
2177
2110
|
transaction = Transaction.of(
|
2178
|
-
txn_type=TransactionType.APPEND,
|
2179
2111
|
txn_operations=bad_txn_operations,
|
2180
2112
|
)
|
2181
|
-
with pytest.raises(
|
2113
|
+
with pytest.raises(ObjectAlreadyExistsError):
|
2182
2114
|
transaction.commit(temp_dir)
|
2183
2115
|
|
2184
2116
|
def test_e2e_serde(self, temp_dir):
|
@@ -2195,7 +2127,6 @@ class TestMetafileIO:
|
|
2195
2127
|
namespace = Namespace.of(locator=namespace_locator)
|
2196
2128
|
# given a transaction that creates a single namespace
|
2197
2129
|
write_paths, txn_log_path = Transaction.of(
|
2198
|
-
txn_type=TransactionType.APPEND,
|
2199
2130
|
txn_operations=[
|
2200
2131
|
TransactionOperation.of(
|
2201
2132
|
operation_type=TransactionOperationType.CREATE,
|
@@ -2219,7 +2150,6 @@ class TestMetafileIO:
|
|
2219
2150
|
)
|
2220
2151
|
# given a transaction that creates a single table
|
2221
2152
|
write_paths, txn_log_path = Transaction.of(
|
2222
|
-
txn_type=TransactionType.APPEND,
|
2223
2153
|
txn_operations=[
|
2224
2154
|
TransactionOperation.of(
|
2225
2155
|
operation_type=TransactionOperationType.CREATE,
|
@@ -2267,7 +2197,7 @@ class TestMetafileIO:
|
|
2267
2197
|
PartitionKey.of(
|
2268
2198
|
key=["some_string", "some_int32"],
|
2269
2199
|
name="test_partition_key",
|
2270
|
-
field_id=
|
2200
|
+
field_id=1,
|
2271
2201
|
transform=bucket_transform,
|
2272
2202
|
)
|
2273
2203
|
]
|
@@ -2307,7 +2237,6 @@ class TestMetafileIO:
|
|
2307
2237
|
)
|
2308
2238
|
# given a transaction that creates a single table version
|
2309
2239
|
write_paths, txn_log_path = Transaction.of(
|
2310
|
-
txn_type=TransactionType.APPEND,
|
2311
2240
|
txn_operations=[
|
2312
2241
|
TransactionOperation.of(
|
2313
2242
|
operation_type=TransactionOperationType.CREATE,
|
@@ -2338,7 +2267,7 @@ class TestMetafileIO:
|
|
2338
2267
|
PartitionKey.of(
|
2339
2268
|
key=["some_string", "some_int32"],
|
2340
2269
|
name="test_partition_key",
|
2341
|
-
field_id=
|
2270
|
+
field_id=1,
|
2342
2271
|
transform=bucket_transform,
|
2343
2272
|
)
|
2344
2273
|
]
|
@@ -2356,7 +2285,6 @@ class TestMetafileIO:
|
|
2356
2285
|
)
|
2357
2286
|
# given a transaction that creates a single stream
|
2358
2287
|
write_paths, txn_log_path = Transaction.of(
|
2359
|
-
txn_type=TransactionType.APPEND,
|
2360
2288
|
txn_operations=[
|
2361
2289
|
TransactionOperation.of(
|
2362
2290
|
operation_type=TransactionOperationType.CREATE,
|
@@ -2379,28 +2307,8 @@ class TestMetafileIO:
|
|
2379
2307
|
partition_values=["a", 1],
|
2380
2308
|
partition_id="test_partition_id",
|
2381
2309
|
)
|
2382
|
-
schema = Schema.of(
|
2383
|
-
[
|
2384
|
-
Field.of(
|
2385
|
-
field=pa.field("some_string", pa.string(), nullable=False),
|
2386
|
-
field_id=1,
|
2387
|
-
is_merge_key=True,
|
2388
|
-
),
|
2389
|
-
Field.of(
|
2390
|
-
field=pa.field("some_int32", pa.int32(), nullable=False),
|
2391
|
-
field_id=2,
|
2392
|
-
is_merge_key=True,
|
2393
|
-
),
|
2394
|
-
Field.of(
|
2395
|
-
field=pa.field("some_float64", pa.float64()),
|
2396
|
-
field_id=3,
|
2397
|
-
is_merge_key=False,
|
2398
|
-
),
|
2399
|
-
]
|
2400
|
-
)
|
2401
2310
|
partition = Partition.of(
|
2402
2311
|
locator=partition_locator,
|
2403
|
-
schema=schema,
|
2404
2312
|
content_types=[ContentType.PARQUET],
|
2405
2313
|
state=CommitState.STAGED,
|
2406
2314
|
previous_stream_position=0,
|
@@ -2410,7 +2318,6 @@ class TestMetafileIO:
|
|
2410
2318
|
)
|
2411
2319
|
# given a transaction that creates a single partition
|
2412
2320
|
write_paths, txn_log_path = Transaction.of(
|
2413
|
-
txn_type=TransactionType.APPEND,
|
2414
2321
|
txn_operations=[
|
2415
2322
|
TransactionOperation.of(
|
2416
2323
|
operation_type=TransactionOperationType.CREATE,
|
@@ -2472,7 +2379,6 @@ class TestMetafileIO:
|
|
2472
2379
|
)
|
2473
2380
|
# given a transaction that creates a single delta
|
2474
2381
|
write_paths, txn_log_path = Transaction.of(
|
2475
|
-
txn_type=TransactionType.APPEND,
|
2476
2382
|
txn_operations=[
|
2477
2383
|
TransactionOperation.of(
|
2478
2384
|
operation_type=TransactionOperationType.CREATE,
|
@@ -2511,7 +2417,6 @@ class TestMetafileIO:
|
|
2511
2417
|
)
|
2512
2418
|
# when a transaction commits this table
|
2513
2419
|
write_paths, txn_log_path = Transaction.of(
|
2514
|
-
txn_type=TransactionType.APPEND,
|
2515
2420
|
txn_operations=[
|
2516
2421
|
TransactionOperation.of(
|
2517
2422
|
operation_type=TransactionOperationType.CREATE,
|