deltacat 2.0.0b10__py3-none-any.whl → 2.0.0b12__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (298) hide show
  1. deltacat/__init__.py +96 -17
  2. deltacat/api.py +122 -67
  3. deltacat/aws/constants.py +0 -23
  4. deltacat/aws/s3u.py +4 -631
  5. deltacat/benchmarking/benchmark_engine.py +4 -2
  6. deltacat/benchmarking/conftest.py +0 -18
  7. deltacat/benchmarking/test_benchmark_pipeline.py +6 -4
  8. deltacat/catalog/__init__.py +64 -5
  9. deltacat/catalog/delegate.py +445 -63
  10. deltacat/catalog/interface.py +188 -62
  11. deltacat/catalog/main/impl.py +2435 -279
  12. deltacat/catalog/model/catalog.py +154 -77
  13. deltacat/catalog/model/properties.py +63 -22
  14. deltacat/compute/compactor/compaction_session.py +97 -75
  15. deltacat/compute/compactor/model/compact_partition_params.py +75 -30
  16. deltacat/compute/compactor/model/compaction_session_audit_info.py +17 -0
  17. deltacat/compute/compactor/model/round_completion_info.py +16 -6
  18. deltacat/compute/compactor/repartition_session.py +8 -21
  19. deltacat/compute/compactor/steps/hash_bucket.py +5 -5
  20. deltacat/compute/compactor/steps/materialize.py +9 -7
  21. deltacat/compute/compactor/steps/repartition.py +12 -11
  22. deltacat/compute/compactor/utils/io.py +6 -5
  23. deltacat/compute/compactor/utils/round_completion_reader.py +117 -0
  24. deltacat/compute/compactor/utils/system_columns.py +3 -1
  25. deltacat/compute/compactor_v2/compaction_session.py +17 -14
  26. deltacat/compute/compactor_v2/constants.py +30 -1
  27. deltacat/compute/compactor_v2/model/evaluate_compaction_result.py +0 -1
  28. deltacat/compute/compactor_v2/model/hash_bucket_input.py +9 -3
  29. deltacat/compute/compactor_v2/model/merge_file_group.py +5 -2
  30. deltacat/compute/compactor_v2/model/merge_input.py +33 -8
  31. deltacat/compute/compactor_v2/private/compaction_utils.py +167 -68
  32. deltacat/compute/compactor_v2/steps/hash_bucket.py +5 -2
  33. deltacat/compute/compactor_v2/steps/merge.py +267 -55
  34. deltacat/compute/compactor_v2/utils/content_type_params.py +34 -6
  35. deltacat/compute/compactor_v2/utils/dedupe.py +1 -1
  36. deltacat/compute/compactor_v2/utils/delta.py +5 -3
  37. deltacat/compute/compactor_v2/utils/io.py +11 -4
  38. deltacat/compute/compactor_v2/utils/merge.py +15 -2
  39. deltacat/compute/compactor_v2/utils/primary_key_index.py +28 -4
  40. deltacat/compute/compactor_v2/utils/task_options.py +45 -33
  41. deltacat/compute/converter/converter_session.py +145 -32
  42. deltacat/compute/converter/model/convert_input.py +26 -19
  43. deltacat/compute/converter/model/convert_input_files.py +33 -16
  44. deltacat/compute/converter/model/convert_result.py +35 -16
  45. deltacat/compute/converter/model/converter_session_params.py +24 -21
  46. deltacat/compute/converter/pyiceberg/catalog.py +21 -18
  47. deltacat/compute/converter/pyiceberg/overrides.py +18 -9
  48. deltacat/compute/converter/pyiceberg/update_snapshot_overrides.py +148 -100
  49. deltacat/compute/converter/steps/convert.py +157 -50
  50. deltacat/compute/converter/steps/dedupe.py +24 -11
  51. deltacat/compute/converter/utils/convert_task_options.py +27 -12
  52. deltacat/compute/converter/utils/converter_session_utils.py +126 -60
  53. deltacat/compute/converter/utils/iceberg_columns.py +8 -8
  54. deltacat/compute/converter/utils/io.py +101 -12
  55. deltacat/compute/converter/utils/s3u.py +33 -27
  56. deltacat/compute/janitor.py +205 -0
  57. deltacat/compute/jobs/client.py +25 -12
  58. deltacat/compute/resource_estimation/delta.py +38 -6
  59. deltacat/compute/resource_estimation/model.py +8 -0
  60. deltacat/constants.py +45 -2
  61. deltacat/docs/autogen/schema/inference/generate_type_mappings.py +687 -0
  62. deltacat/docs/autogen/schema/inference/parse_json_type_mappings.py +673 -0
  63. deltacat/env.py +10 -0
  64. deltacat/examples/basic_logging.py +1 -3
  65. deltacat/examples/compactor/aws/__init__.py +1 -0
  66. deltacat/examples/compactor/bootstrap.py +863 -0
  67. deltacat/examples/compactor/compactor.py +373 -0
  68. deltacat/examples/compactor/explorer.py +473 -0
  69. deltacat/examples/compactor/gcp/__init__.py +1 -0
  70. deltacat/examples/compactor/job_runner.py +439 -0
  71. deltacat/examples/compactor/utils/__init__.py +1 -0
  72. deltacat/examples/compactor/utils/common.py +261 -0
  73. deltacat/examples/experimental/iceberg/converter/beam/app.py +226 -0
  74. deltacat/examples/experimental/iceberg/converter/beam/main.py +133 -0
  75. deltacat/examples/experimental/iceberg/converter/beam/test_workflow.py +113 -0
  76. deltacat/examples/experimental/iceberg/converter/beam/utils/__init__.py +3 -0
  77. deltacat/examples/experimental/iceberg/converter/beam/utils/common.py +174 -0
  78. deltacat/examples/experimental/iceberg/converter/beam/utils/spark.py +263 -0
  79. deltacat/examples/{iceberg → experimental/iceberg}/iceberg_bucket_writer.py +3 -5
  80. deltacat/examples/{iceberg → experimental/iceberg}/iceberg_reader.py +2 -4
  81. deltacat/examples/indexer/indexer.py +2 -2
  82. deltacat/examples/indexer/job_runner.py +1 -2
  83. deltacat/exceptions.py +66 -4
  84. deltacat/experimental/catalog/iceberg/__init__.py +6 -0
  85. deltacat/{catalog → experimental/catalog}/iceberg/iceberg_catalog_config.py +1 -1
  86. deltacat/{catalog → experimental/catalog}/iceberg/impl.py +29 -11
  87. deltacat/experimental/compatibility/backfill_locator_to_id_mappings.py +201 -0
  88. deltacat/experimental/converter_agent/beam/managed.py +173 -0
  89. deltacat/experimental/converter_agent/table_monitor.py +479 -0
  90. deltacat/experimental/storage/iceberg/iceberg_scan_planner.py +129 -0
  91. deltacat/{storage → experimental/storage}/iceberg/impl.py +6 -4
  92. deltacat/{storage → experimental/storage}/iceberg/model.py +7 -3
  93. deltacat/experimental/storage/iceberg/visitor.py +119 -0
  94. deltacat/experimental/storage/rivulet/__init__.py +11 -0
  95. deltacat/{storage → experimental/storage}/rivulet/arrow/serializer.py +7 -4
  96. deltacat/{storage → experimental/storage}/rivulet/dataset.py +13 -12
  97. deltacat/{storage → experimental/storage}/rivulet/dataset_executor.py +12 -20
  98. deltacat/experimental/storage/rivulet/feather/__init__.py +7 -0
  99. deltacat/{storage → experimental/storage}/rivulet/feather/file_reader.py +7 -5
  100. deltacat/{storage → experimental/storage}/rivulet/feather/serializer.py +4 -4
  101. deltacat/{storage → experimental/storage}/rivulet/fs/file_provider.py +3 -3
  102. deltacat/{storage → experimental/storage}/rivulet/fs/file_store.py +2 -2
  103. deltacat/{storage → experimental/storage}/rivulet/fs/output_file.py +1 -1
  104. deltacat/{storage → experimental/storage}/rivulet/logical_plan.py +4 -4
  105. deltacat/experimental/storage/rivulet/metastore/__init__.py +0 -0
  106. deltacat/{storage → experimental/storage}/rivulet/metastore/delta.py +1 -3
  107. deltacat/{storage → experimental/storage}/rivulet/metastore/json_sst.py +3 -3
  108. deltacat/{storage → experimental/storage}/rivulet/metastore/sst.py +2 -2
  109. deltacat/{storage → experimental/storage}/rivulet/metastore/sst_interval_tree.py +3 -3
  110. deltacat/experimental/storage/rivulet/parquet/__init__.py +7 -0
  111. deltacat/experimental/storage/rivulet/parquet/data_reader.py +0 -0
  112. deltacat/{storage → experimental/storage}/rivulet/parquet/file_reader.py +7 -5
  113. deltacat/{storage → experimental/storage}/rivulet/parquet/serializer.py +4 -4
  114. deltacat/experimental/storage/rivulet/reader/__init__.py +0 -0
  115. deltacat/{storage → experimental/storage}/rivulet/reader/block_scanner.py +20 -9
  116. deltacat/{storage → experimental/storage}/rivulet/reader/data_reader.py +3 -3
  117. deltacat/{storage → experimental/storage}/rivulet/reader/data_scan.py +5 -3
  118. deltacat/{storage → experimental/storage}/rivulet/reader/dataset_metastore.py +7 -6
  119. deltacat/{storage → experimental/storage}/rivulet/reader/dataset_reader.py +8 -6
  120. deltacat/{storage → experimental/storage}/rivulet/reader/pyarrow_data_reader.py +4 -1
  121. deltacat/{storage → experimental/storage}/rivulet/reader/reader_type_registrar.py +4 -4
  122. deltacat/experimental/storage/rivulet/schema/__init__.py +0 -0
  123. deltacat/{storage → experimental/storage}/rivulet/schema/schema.py +1 -1
  124. deltacat/{storage → experimental/storage}/rivulet/serializer.py +1 -1
  125. deltacat/{storage → experimental/storage}/rivulet/serializer_factory.py +9 -5
  126. deltacat/experimental/storage/rivulet/shard/__init__.py +0 -0
  127. deltacat/experimental/storage/rivulet/shard/range_shard.py +129 -0
  128. deltacat/experimental/storage/rivulet/writer/__init__.py +0 -0
  129. deltacat/{storage → experimental/storage}/rivulet/writer/memtable_dataset_writer.py +20 -9
  130. deltacat/io/datasource/deltacat_datasource.py +0 -1
  131. deltacat/io/reader/deltacat_read_api.py +1 -1
  132. deltacat/storage/__init__.py +20 -2
  133. deltacat/storage/interface.py +54 -32
  134. deltacat/storage/main/impl.py +1494 -541
  135. deltacat/storage/model/delta.py +27 -3
  136. deltacat/storage/model/locator.py +6 -12
  137. deltacat/storage/model/manifest.py +182 -6
  138. deltacat/storage/model/metafile.py +151 -78
  139. deltacat/storage/model/namespace.py +8 -1
  140. deltacat/storage/model/partition.py +117 -42
  141. deltacat/storage/model/schema.py +2427 -159
  142. deltacat/storage/model/shard.py +6 -2
  143. deltacat/storage/model/sort_key.py +40 -0
  144. deltacat/storage/model/stream.py +9 -2
  145. deltacat/storage/model/table.py +12 -1
  146. deltacat/storage/model/table_version.py +11 -0
  147. deltacat/storage/model/transaction.py +1184 -208
  148. deltacat/storage/model/transform.py +81 -2
  149. deltacat/storage/model/types.py +48 -26
  150. deltacat/tests/_io/test_cloudpickle_bug_fix.py +8 -4
  151. deltacat/tests/aws/test_s3u.py +2 -31
  152. deltacat/tests/catalog/data/__init__.py +0 -0
  153. deltacat/tests/catalog/main/__init__.py +0 -0
  154. deltacat/tests/catalog/main/test_catalog_impl_namespace_operations.py +130 -0
  155. deltacat/tests/catalog/main/test_catalog_impl_table_operations.py +1972 -0
  156. deltacat/tests/catalog/model/__init__.py +0 -0
  157. deltacat/tests/catalog/model/test_table_definition.py +16 -0
  158. deltacat/tests/catalog/test_catalogs.py +103 -106
  159. deltacat/tests/catalog/test_default_catalog_impl.py +12152 -72
  160. deltacat/tests/compute/compact_partition_test_cases.py +35 -8
  161. deltacat/tests/compute/compactor/steps/test_repartition.py +12 -12
  162. deltacat/tests/compute/compactor/utils/test_io.py +124 -120
  163. deltacat/tests/compute/compactor/utils/test_round_completion_reader.py +254 -0
  164. deltacat/tests/compute/compactor_v2/test_compaction_session.py +423 -312
  165. deltacat/tests/compute/compactor_v2/utils/test_content_type_params.py +266 -0
  166. deltacat/tests/compute/compactor_v2/utils/test_primary_key_index.py +45 -0
  167. deltacat/tests/compute/compactor_v2/utils/test_task_options.py +270 -1
  168. deltacat/tests/compute/conftest.py +8 -44
  169. deltacat/tests/compute/converter/test_convert_session.py +675 -490
  170. deltacat/tests/compute/converter/utils.py +15 -6
  171. deltacat/tests/compute/resource_estimation/test_delta.py +145 -79
  172. deltacat/tests/compute/test_compact_partition_incremental.py +103 -70
  173. deltacat/tests/compute/test_compact_partition_multiple_rounds.py +89 -66
  174. deltacat/tests/compute/test_compact_partition_params.py +13 -8
  175. deltacat/tests/compute/test_compact_partition_rebase.py +77 -62
  176. deltacat/tests/compute/test_compact_partition_rebase_then_incremental.py +263 -193
  177. deltacat/tests/compute/test_janitor.py +236 -0
  178. deltacat/tests/compute/test_util_common.py +716 -43
  179. deltacat/tests/compute/test_util_constant.py +0 -1
  180. deltacat/tests/{storage/conftest.py → conftest.py} +1 -1
  181. deltacat/tests/daft/__init__.py +0 -0
  182. deltacat/tests/daft/test_model.py +97 -0
  183. deltacat/tests/experimental/__init__.py +1 -0
  184. deltacat/tests/experimental/catalog/__init__.py +0 -0
  185. deltacat/tests/experimental/catalog/iceberg/__init__.py +0 -0
  186. deltacat/tests/experimental/catalog/iceberg/test_iceberg_catalog.py +71 -0
  187. deltacat/tests/experimental/compatibility/__init__.py +1 -0
  188. deltacat/tests/experimental/compatibility/test_backfill_locator_to_id_mappings.py +582 -0
  189. deltacat/tests/experimental/daft/__init__.py +0 -0
  190. deltacat/tests/experimental/daft/test_deltacat_daft_integration.py +136 -0
  191. deltacat/tests/experimental/storage/__init__.py +0 -0
  192. deltacat/tests/experimental/storage/rivulet/__init__.py +0 -0
  193. deltacat/tests/{storage → experimental/storage}/rivulet/conftest.py +3 -3
  194. deltacat/tests/experimental/storage/rivulet/fs/__init__.py +0 -0
  195. deltacat/tests/{storage → experimental/storage}/rivulet/fs/test_file_location_provider.py +3 -3
  196. deltacat/tests/experimental/storage/rivulet/reader/__init__.py +0 -0
  197. deltacat/tests/experimental/storage/rivulet/reader/query_expression.py +80 -0
  198. deltacat/tests/experimental/storage/rivulet/reader/test_data_scan.py +119 -0
  199. deltacat/tests/experimental/storage/rivulet/reader/test_dataset_metastore.py +71 -0
  200. deltacat/tests/experimental/storage/rivulet/schema/__init__.py +0 -0
  201. deltacat/tests/{storage → experimental/storage}/rivulet/schema/test_schema.py +1 -1
  202. deltacat/tests/experimental/storage/rivulet/shard/__init__.py +0 -0
  203. deltacat/tests/experimental/storage/rivulet/shard/test_range_shard.py +162 -0
  204. deltacat/tests/{storage → experimental/storage}/rivulet/test_dataset.py +5 -3
  205. deltacat/tests/{storage → experimental/storage}/rivulet/test_manifest.py +5 -5
  206. deltacat/tests/{storage → experimental/storage}/rivulet/test_sst_interval_tree.py +5 -5
  207. deltacat/tests/{storage → experimental/storage}/rivulet/test_utils.py +8 -6
  208. deltacat/tests/experimental/storage/rivulet/writer/__init__.py +0 -0
  209. deltacat/tests/{storage → experimental/storage}/rivulet/writer/test_dataset_write_then_read.py +11 -9
  210. deltacat/tests/{storage → experimental/storage}/rivulet/writer/test_dataset_writer.py +2 -2
  211. deltacat/tests/{storage → experimental/storage}/rivulet/writer/test_memtable_dataset_writer.py +7 -7
  212. deltacat/tests/storage/main/test_main_storage.py +6900 -95
  213. deltacat/tests/storage/model/test_metafile_io.py +78 -173
  214. deltacat/tests/storage/model/test_partition_scheme.py +85 -0
  215. deltacat/tests/storage/model/test_schema.py +171 -0
  216. deltacat/tests/storage/model/test_schema_update.py +1925 -0
  217. deltacat/tests/storage/model/test_shard.py +3 -1
  218. deltacat/tests/storage/model/test_sort_scheme.py +90 -0
  219. deltacat/tests/storage/model/test_transaction.py +393 -48
  220. deltacat/tests/storage/model/test_transaction_history.py +886 -0
  221. deltacat/tests/test_deltacat_api.py +988 -4
  222. deltacat/tests/test_exceptions.py +9 -5
  223. deltacat/tests/test_utils/pyarrow.py +52 -21
  224. deltacat/tests/test_utils/storage.py +23 -34
  225. deltacat/tests/types/__init__.py +0 -0
  226. deltacat/tests/types/test_tables.py +104 -0
  227. deltacat/tests/utils/exceptions.py +22 -0
  228. deltacat/tests/utils/main_deltacat_storage_mock.py +31 -0
  229. deltacat/tests/utils/ray_utils/test_dataset.py +123 -5
  230. deltacat/tests/utils/test_daft.py +121 -31
  231. deltacat/tests/utils/test_numpy.py +1193 -0
  232. deltacat/tests/utils/test_pandas.py +1106 -0
  233. deltacat/tests/utils/test_polars.py +1040 -0
  234. deltacat/tests/utils/test_pyarrow.py +1370 -89
  235. deltacat/types/media.py +224 -14
  236. deltacat/types/tables.py +2329 -59
  237. deltacat/utils/arguments.py +33 -1
  238. deltacat/utils/daft.py +823 -36
  239. deltacat/utils/export.py +3 -1
  240. deltacat/utils/filesystem.py +100 -0
  241. deltacat/utils/metafile_locator.py +2 -1
  242. deltacat/utils/numpy.py +118 -26
  243. deltacat/utils/pandas.py +577 -48
  244. deltacat/utils/polars.py +658 -27
  245. deltacat/utils/pyarrow.py +1258 -213
  246. deltacat/utils/ray_utils/dataset.py +101 -10
  247. deltacat/utils/reader_compatibility_mapping.py +3083 -0
  248. deltacat/utils/url.py +57 -16
  249. deltacat-2.0.0b12.dist-info/METADATA +1163 -0
  250. deltacat-2.0.0b12.dist-info/RECORD +439 -0
  251. {deltacat-2.0.0b10.dist-info → deltacat-2.0.0b12.dist-info}/WHEEL +1 -1
  252. deltacat/catalog/iceberg/__init__.py +0 -4
  253. deltacat/compute/compactor/utils/round_completion_file.py +0 -97
  254. deltacat/compute/merge_on_read/__init__.py +0 -4
  255. deltacat/compute/merge_on_read/daft.py +0 -40
  256. deltacat/compute/merge_on_read/model/merge_on_read_params.py +0 -66
  257. deltacat/compute/merge_on_read/utils/delta.py +0 -42
  258. deltacat/daft/daft_scan.py +0 -115
  259. deltacat/daft/model.py +0 -258
  260. deltacat/daft/translator.py +0 -126
  261. deltacat/examples/common/fixtures.py +0 -15
  262. deltacat/storage/iceberg/iceberg_scan_planner.py +0 -28
  263. deltacat/storage/rivulet/__init__.py +0 -11
  264. deltacat/storage/rivulet/feather/__init__.py +0 -5
  265. deltacat/storage/rivulet/parquet/__init__.py +0 -5
  266. deltacat/tests/compute/compactor/utils/test_round_completion_file.py +0 -231
  267. deltacat/tests/compute/test_util_create_table_deltas_repo.py +0 -388
  268. deltacat/tests/local_deltacat_storage/__init__.py +0 -1236
  269. deltacat/tests/local_deltacat_storage/exceptions.py +0 -10
  270. deltacat/utils/s3fs.py +0 -21
  271. deltacat-2.0.0b10.dist-info/METADATA +0 -68
  272. deltacat-2.0.0b10.dist-info/RECORD +0 -381
  273. /deltacat/{compute/merge_on_read/model → docs}/__init__.py +0 -0
  274. /deltacat/{compute/merge_on_read/utils → docs/autogen}/__init__.py +0 -0
  275. /deltacat/{daft → docs/autogen/schema}/__init__.py +0 -0
  276. /deltacat/{examples/common → docs/autogen/schema/inference}/__init__.py +0 -0
  277. /deltacat/examples/{iceberg → compactor}/__init__.py +0 -0
  278. /deltacat/{storage/iceberg → examples/experimental}/__init__.py +0 -0
  279. /deltacat/{storage/rivulet/arrow → examples/experimental/iceberg}/__init__.py +0 -0
  280. /deltacat/{storage/rivulet/fs → examples/experimental/iceberg/converter}/__init__.py +0 -0
  281. /deltacat/{storage/rivulet/metastore → examples/experimental/iceberg/converter/beam}/__init__.py +0 -0
  282. /deltacat/{storage/rivulet/reader → experimental/catalog}/__init__.py +0 -0
  283. /deltacat/{catalog → experimental/catalog}/iceberg/overrides.py +0 -0
  284. /deltacat/{storage/rivulet/schema → experimental/compatibility}/__init__.py +0 -0
  285. /deltacat/{storage/rivulet/writer → experimental/converter_agent}/__init__.py +0 -0
  286. /deltacat/{tests/storage/rivulet → experimental/converter_agent/beam}/__init__.py +0 -0
  287. /deltacat/{tests/storage/rivulet/fs → experimental/storage}/__init__.py +0 -0
  288. /deltacat/{tests/storage/rivulet/schema → experimental/storage/iceberg}/__init__.py +0 -0
  289. /deltacat/{tests/storage/rivulet/writer → experimental/storage/rivulet/arrow}/__init__.py +0 -0
  290. /deltacat/{storage/rivulet/parquet/data_reader.py → experimental/storage/rivulet/fs/__init__.py} +0 -0
  291. /deltacat/{storage → experimental/storage}/rivulet/fs/input_file.py +0 -0
  292. /deltacat/{storage → experimental/storage}/rivulet/mvp/Table.py +0 -0
  293. /deltacat/{storage → experimental/storage}/rivulet/mvp/__init__.py +0 -0
  294. /deltacat/{storage → experimental/storage}/rivulet/reader/query_expression.py +0 -0
  295. /deltacat/{storage → experimental/storage}/rivulet/schema/datatype.py +0 -0
  296. /deltacat/{storage → experimental/storage}/rivulet/writer/dataset_writer.py +0 -0
  297. {deltacat-2.0.0b10.dist-info → deltacat-2.0.0b12.dist-info/licenses}/LICENSE +0 -0
  298. {deltacat-2.0.0b10.dist-info → deltacat-2.0.0b12.dist-info}/top_level.txt +0 -0
@@ -20,6 +20,12 @@ from deltacat import (
20
20
  NullOrder,
21
21
  LifecycleState,
22
22
  )
23
+ from deltacat.exceptions import (
24
+ ObjectAlreadyExistsError,
25
+ ConcurrentModificationError,
26
+ ObjectDeletedError,
27
+ ObjectNotFoundError,
28
+ )
23
29
  from deltacat.storage import (
24
30
  BucketTransform,
25
31
  BucketTransformParameters,
@@ -47,7 +53,6 @@ from deltacat.storage import (
47
53
  TableVersion,
48
54
  Transaction,
49
55
  TransactionOperation,
50
- TransactionType,
51
56
  TransactionOperationType,
52
57
  TruncateTransform,
53
58
  TruncateTransformParameters,
@@ -92,7 +97,6 @@ def _commit_single_delta_table(temp_dir: str) -> List[Tuple[Metafile, Metafile,
92
97
  for meta in meta_to_create
93
98
  ]
94
99
  transaction = Transaction.of(
95
- txn_type=TransactionType.APPEND,
96
100
  txn_operations=txn_operations,
97
101
  )
98
102
  write_paths, txn_log_path = transaction.commit(temp_dir)
@@ -116,7 +120,7 @@ def _commit_concurrent_transaction(
116
120
  ) -> None:
117
121
  try:
118
122
  return transaction.commit(catalog_root)
119
- except (RuntimeError, ValueError) as e:
123
+ except (ObjectAlreadyExistsError, ConcurrentModificationError) as e:
120
124
  return e
121
125
 
122
126
 
@@ -133,7 +137,6 @@ class TestMetafileIO:
133
137
  description="test table description",
134
138
  )
135
139
  transaction = Transaction.of(
136
- txn_type=TransactionType.APPEND,
137
140
  txn_operations=[
138
141
  TransactionOperation.of(
139
142
  operation_type=TransactionOperationType.CREATE,
@@ -158,9 +161,8 @@ class TestMetafileIO:
158
161
  results = [future.get() for future in futures]
159
162
  conflict_exception_count = 0
160
163
  for result in results:
161
- # TODO(pdames): Add new concurrent conflict exception types.
162
- if isinstance(result, RuntimeError) or isinstance(
163
- result, ValueError
164
+ if isinstance(result, ConcurrentModificationError) or isinstance(
165
+ result, ObjectAlreadyExistsError
164
166
  ):
165
167
  conflict_exception_count += 1
166
168
  else:
@@ -174,7 +176,6 @@ class TestMetafileIO:
174
176
  namespace = Namespace.of(locator=namespace_locator)
175
177
  # given a transaction that creates a single namespace
176
178
  transaction = Transaction.of(
177
- txn_type=TransactionType.APPEND,
178
179
  txn_operations=[
179
180
  TransactionOperation.of(
180
181
  operation_type=TransactionOperationType.CREATE,
@@ -213,7 +214,6 @@ class TestMetafileIO:
213
214
  )
214
215
  ]
215
216
  transaction = Transaction.of(
216
- txn_type=TransactionType.ALTER,
217
217
  txn_operations=txn_operations,
218
218
  )
219
219
  # expect the bad timestamp to be detected and its commit to fail
@@ -274,7 +274,6 @@ class TestMetafileIO:
274
274
  )
275
275
  ]
276
276
  transaction = Transaction.of(
277
- txn_type=TransactionType.ALTER,
278
277
  txn_operations=txn_operations,
279
278
  )
280
279
  # expect the commit to fail due to a concurrent modification error
@@ -304,7 +303,6 @@ class TestMetafileIO:
304
303
  original_delta = Delta.read(orig_delta_write_path)
305
304
  new_delta = Delta.update_for(original_delta)
306
305
  transaction = Transaction.of(
307
- txn_type=TransactionType.ALTER,
308
306
  txn_operations=[
309
307
  TransactionOperation.of(
310
308
  operation_type=TransactionOperationType.UPDATE,
@@ -314,10 +312,10 @@ class TestMetafileIO:
314
312
  ],
315
313
  )
316
314
  # expect the commit to fail due to a concurrent modification error
317
- with pytest.raises(RuntimeError):
315
+ with pytest.raises(ConcurrentModificationError):
318
316
  transaction.commit(temp_dir)
319
317
  # expect a commit retry to also fail
320
- with pytest.raises(RuntimeError):
318
+ with pytest.raises(ConcurrentModificationError):
321
319
  transaction.commit(temp_dir)
322
320
 
323
321
  def test_append_multiple_deltas(self, temp_dir):
@@ -342,7 +340,6 @@ class TestMetafileIO:
342
340
  )
343
341
  )
344
342
  transaction = Transaction.of(
345
- txn_type=TransactionType.APPEND,
346
343
  txn_operations=txn_operations,
347
344
  )
348
345
  # when the transaction is committed
@@ -387,7 +384,6 @@ class TestMetafileIO:
387
384
  )
388
385
  ]
389
386
  transaction = Transaction.of(
390
- txn_type=TransactionType.DELETE,
391
387
  txn_operations=txn_operations,
392
388
  )
393
389
  # when the transaction is committed
@@ -415,10 +411,9 @@ class TestMetafileIO:
415
411
  )
416
412
  ]
417
413
  transaction = Transaction.of(
418
- txn_type=TransactionType.OVERWRITE,
419
414
  txn_operations=bad_txn_operations,
420
415
  )
421
- with pytest.raises(ValueError):
416
+ with pytest.raises(ObjectDeletedError):
422
417
  transaction.commit(temp_dir)
423
418
 
424
419
  # expect subsequent deletes of the deleted delta to fail
@@ -429,10 +424,9 @@ class TestMetafileIO:
429
424
  )
430
425
  ]
431
426
  transaction = Transaction.of(
432
- txn_type=TransactionType.DELETE,
433
427
  txn_operations=bad_txn_operations,
434
428
  )
435
- with pytest.raises(ValueError):
429
+ with pytest.raises(ObjectDeletedError):
436
430
  transaction.commit(temp_dir)
437
431
 
438
432
  def test_replace_delta(self, temp_dir):
@@ -452,13 +446,12 @@ class TestMetafileIO:
452
446
 
453
447
  txn_operations = [
454
448
  TransactionOperation.of(
455
- operation_type=TransactionOperationType.UPDATE,
449
+ operation_type=TransactionOperationType.REPLACE,
456
450
  dest_metafile=replacement_delta,
457
451
  src_metafile=original_delta,
458
452
  )
459
453
  ]
460
454
  transaction = Transaction.of(
461
- txn_type=TransactionType.OVERWRITE,
462
455
  txn_operations=txn_operations,
463
456
  )
464
457
  # when the transaction is committed
@@ -483,16 +476,15 @@ class TestMetafileIO:
483
476
  # expect a subsequent replace of the original delta to fail
484
477
  bad_txn_operations = [
485
478
  TransactionOperation.of(
486
- operation_type=TransactionOperationType.UPDATE,
479
+ operation_type=TransactionOperationType.REPLACE,
487
480
  dest_metafile=replacement_delta,
488
481
  src_metafile=original_delta,
489
482
  )
490
483
  ]
491
484
  transaction = Transaction.of(
492
- txn_type=TransactionType.OVERWRITE,
493
485
  txn_operations=bad_txn_operations,
494
486
  )
495
- with pytest.raises(ValueError):
487
+ with pytest.raises(ObjectDeletedError):
496
488
  transaction.commit(temp_dir)
497
489
 
498
490
  # expect deletes of the original delta to fail
@@ -503,10 +495,9 @@ class TestMetafileIO:
503
495
  )
504
496
  ]
505
497
  transaction = Transaction.of(
506
- txn_type=TransactionType.DELETE,
507
498
  txn_operations=bad_txn_operations,
508
499
  )
509
- with pytest.raises(ValueError):
500
+ with pytest.raises(ObjectDeletedError):
510
501
  transaction.commit(temp_dir)
511
502
 
512
503
  def test_delete_partition(self, temp_dir):
@@ -522,7 +513,6 @@ class TestMetafileIO:
522
513
  )
523
514
  ]
524
515
  transaction = Transaction.of(
525
- txn_type=TransactionType.DELETE,
526
516
  txn_operations=txn_operations,
527
517
  )
528
518
  # when the transaction is committed
@@ -562,16 +552,15 @@ class TestMetafileIO:
562
552
  )
563
553
  bad_txn_operations = [
564
554
  TransactionOperation.of(
565
- operation_type=TransactionOperationType.UPDATE,
555
+ operation_type=TransactionOperationType.REPLACE,
566
556
  dest_metafile=replacement_partition,
567
557
  src_metafile=original_partition,
568
558
  )
569
559
  ]
570
560
  transaction = Transaction.of(
571
- txn_type=TransactionType.OVERWRITE,
572
561
  txn_operations=bad_txn_operations,
573
562
  )
574
- with pytest.raises(ValueError):
563
+ with pytest.raises(ObjectDeletedError):
575
564
  transaction.commit(temp_dir)
576
565
 
577
566
  # expect subsequent deletes of the deleted partition to fail
@@ -582,10 +571,9 @@ class TestMetafileIO:
582
571
  )
583
572
  ]
584
573
  transaction = Transaction.of(
585
- txn_type=TransactionType.DELETE,
586
574
  txn_operations=bad_txn_operations,
587
575
  )
588
- with pytest.raises(ValueError):
576
+ with pytest.raises(ObjectDeletedError):
589
577
  transaction.commit(temp_dir)
590
578
 
591
579
  # expect new child metafile creation under the deleted partition to fail
@@ -597,10 +585,9 @@ class TestMetafileIO:
597
585
  )
598
586
  ]
599
587
  transaction = Transaction.of(
600
- txn_type=TransactionType.APPEND,
601
588
  txn_operations=bad_txn_operations,
602
589
  )
603
- with pytest.raises(ValueError):
590
+ with pytest.raises(ObjectAlreadyExistsError):
604
591
  transaction.commit(temp_dir)
605
592
 
606
593
  def test_replace_partition(self, temp_dir):
@@ -620,13 +607,12 @@ class TestMetafileIO:
620
607
 
621
608
  txn_operations = [
622
609
  TransactionOperation.of(
623
- operation_type=TransactionOperationType.UPDATE,
610
+ operation_type=TransactionOperationType.REPLACE,
624
611
  dest_metafile=replacement_partition,
625
612
  src_metafile=original_partition,
626
613
  )
627
614
  ]
628
615
  transaction = Transaction.of(
629
- txn_type=TransactionType.OVERWRITE,
630
616
  txn_operations=txn_operations,
631
617
  )
632
618
  # when the transaction is committed
@@ -679,16 +665,15 @@ class TestMetafileIO:
679
665
  # expect a subsequent replace of the original partition to fail
680
666
  bad_txn_operations = [
681
667
  TransactionOperation.of(
682
- operation_type=TransactionOperationType.UPDATE,
668
+ operation_type=TransactionOperationType.REPLACE,
683
669
  dest_metafile=replacement_partition,
684
670
  src_metafile=original_partition,
685
671
  )
686
672
  ]
687
673
  transaction = Transaction.of(
688
- txn_type=TransactionType.OVERWRITE,
689
674
  txn_operations=bad_txn_operations,
690
675
  )
691
- with pytest.raises(ValueError):
676
+ with pytest.raises(ObjectDeletedError):
692
677
  transaction.commit(temp_dir)
693
678
 
694
679
  # expect deletes of the original partition to fail
@@ -699,10 +684,9 @@ class TestMetafileIO:
699
684
  )
700
685
  ]
701
686
  transaction = Transaction.of(
702
- txn_type=TransactionType.DELETE,
703
687
  txn_operations=bad_txn_operations,
704
688
  )
705
- with pytest.raises(ValueError):
689
+ with pytest.raises(ObjectDeletedError):
706
690
  transaction.commit(temp_dir)
707
691
 
708
692
  # expect new child metafile creation under the old partition to fail
@@ -714,10 +698,9 @@ class TestMetafileIO:
714
698
  )
715
699
  ]
716
700
  transaction = Transaction.of(
717
- txn_type=TransactionType.APPEND,
718
701
  txn_operations=bad_txn_operations,
719
702
  )
720
- with pytest.raises(ValueError):
703
+ with pytest.raises(ObjectAlreadyExistsError):
721
704
  transaction.commit(temp_dir)
722
705
 
723
706
  def test_delete_stream(self, temp_dir):
@@ -733,7 +716,6 @@ class TestMetafileIO:
733
716
  )
734
717
  ]
735
718
  transaction = Transaction.of(
736
- txn_type=TransactionType.DELETE,
737
719
  txn_operations=txn_operations,
738
720
  )
739
721
  # when the transaction is committed
@@ -776,16 +758,15 @@ class TestMetafileIO:
776
758
  )
777
759
  bad_txn_operations = [
778
760
  TransactionOperation.of(
779
- operation_type=TransactionOperationType.UPDATE,
761
+ operation_type=TransactionOperationType.REPLACE,
780
762
  dest_metafile=replacement_stream,
781
763
  src_metafile=original_stream,
782
764
  )
783
765
  ]
784
766
  transaction = Transaction.of(
785
- txn_type=TransactionType.OVERWRITE,
786
767
  txn_operations=bad_txn_operations,
787
768
  )
788
- with pytest.raises(ValueError):
769
+ with pytest.raises(ObjectDeletedError):
789
770
  transaction.commit(temp_dir)
790
771
 
791
772
  # expect subsequent deletes of the deleted stream to fail
@@ -796,10 +777,9 @@ class TestMetafileIO:
796
777
  )
797
778
  ]
798
779
  transaction = Transaction.of(
799
- txn_type=TransactionType.DELETE,
800
780
  txn_operations=bad_txn_operations,
801
781
  )
802
- with pytest.raises(ValueError):
782
+ with pytest.raises(ObjectDeletedError):
803
783
  transaction.commit(temp_dir)
804
784
 
805
785
  # expect new child metafile creation under the deleted stream to fail
@@ -811,10 +791,9 @@ class TestMetafileIO:
811
791
  )
812
792
  ]
813
793
  transaction = Transaction.of(
814
- txn_type=TransactionType.APPEND,
815
794
  txn_operations=bad_txn_operations,
816
795
  )
817
- with pytest.raises(ValueError):
796
+ with pytest.raises(ObjectAlreadyExistsError):
818
797
  transaction.commit(temp_dir)
819
798
 
820
799
  def test_replace_stream(self, temp_dir):
@@ -834,13 +813,12 @@ class TestMetafileIO:
834
813
 
835
814
  txn_operations = [
836
815
  TransactionOperation.of(
837
- operation_type=TransactionOperationType.UPDATE,
816
+ operation_type=TransactionOperationType.REPLACE,
838
817
  dest_metafile=replacement_stream,
839
818
  src_metafile=original_stream,
840
819
  )
841
820
  ]
842
821
  transaction = Transaction.of(
843
- txn_type=TransactionType.OVERWRITE,
844
822
  txn_operations=txn_operations,
845
823
  )
846
824
  # when the transaction is committed
@@ -896,16 +874,15 @@ class TestMetafileIO:
896
874
  # expect a subsequent replace of the original stream to fail
897
875
  bad_txn_operations = [
898
876
  TransactionOperation.of(
899
- operation_type=TransactionOperationType.UPDATE,
877
+ operation_type=TransactionOperationType.REPLACE,
900
878
  dest_metafile=replacement_stream,
901
879
  src_metafile=original_stream,
902
880
  )
903
881
  ]
904
882
  transaction = Transaction.of(
905
- txn_type=TransactionType.OVERWRITE,
906
883
  txn_operations=bad_txn_operations,
907
884
  )
908
- with pytest.raises(ValueError):
885
+ with pytest.raises(ObjectDeletedError):
909
886
  transaction.commit(temp_dir)
910
887
 
911
888
  # expect deletes of the original stream to fail
@@ -916,10 +893,9 @@ class TestMetafileIO:
916
893
  )
917
894
  ]
918
895
  transaction = Transaction.of(
919
- txn_type=TransactionType.DELETE,
920
896
  txn_operations=bad_txn_operations,
921
897
  )
922
- with pytest.raises(ValueError):
898
+ with pytest.raises(ObjectDeletedError):
923
899
  transaction.commit(temp_dir)
924
900
 
925
901
  # expect new child metafile creation under the old stream to fail
@@ -931,10 +907,9 @@ class TestMetafileIO:
931
907
  )
932
908
  ]
933
909
  transaction = Transaction.of(
934
- txn_type=TransactionType.APPEND,
935
910
  txn_operations=bad_txn_operations,
936
911
  )
937
- with pytest.raises(ValueError):
912
+ with pytest.raises(ObjectAlreadyExistsError):
938
913
  transaction.commit(temp_dir)
939
914
 
940
915
  def test_delete_table_version(self, temp_dir):
@@ -950,7 +925,6 @@ class TestMetafileIO:
950
925
  )
951
926
  ]
952
927
  transaction = Transaction.of(
953
- txn_type=TransactionType.DELETE,
954
928
  txn_operations=txn_operations,
955
929
  )
956
930
  # when the transaction is committed
@@ -996,16 +970,15 @@ class TestMetafileIO:
996
970
  )
997
971
  bad_txn_operations = [
998
972
  TransactionOperation.of(
999
- operation_type=TransactionOperationType.UPDATE,
973
+ operation_type=TransactionOperationType.REPLACE,
1000
974
  dest_metafile=replacement_table_version,
1001
975
  src_metafile=original_table_version,
1002
976
  )
1003
977
  ]
1004
978
  transaction = Transaction.of(
1005
- txn_type=TransactionType.OVERWRITE,
1006
979
  txn_operations=bad_txn_operations,
1007
980
  )
1008
- with pytest.raises(ValueError):
981
+ with pytest.raises(ObjectDeletedError):
1009
982
  transaction.commit(temp_dir)
1010
983
 
1011
984
  # expect subsequent deletes of the deleted table version to fail
@@ -1016,10 +989,9 @@ class TestMetafileIO:
1016
989
  )
1017
990
  ]
1018
991
  transaction = Transaction.of(
1019
- txn_type=TransactionType.DELETE,
1020
992
  txn_operations=bad_txn_operations,
1021
993
  )
1022
- with pytest.raises(ValueError):
994
+ with pytest.raises(ObjectDeletedError):
1023
995
  transaction.commit(temp_dir)
1024
996
 
1025
997
  # expect new child metafile creation under the deleted table version to fail
@@ -1031,10 +1003,9 @@ class TestMetafileIO:
1031
1003
  )
1032
1004
  ]
1033
1005
  transaction = Transaction.of(
1034
- txn_type=TransactionType.APPEND,
1035
1006
  txn_operations=bad_txn_operations,
1036
1007
  )
1037
- with pytest.raises(ValueError):
1008
+ with pytest.raises(ObjectAlreadyExistsError):
1038
1009
  transaction.commit(temp_dir)
1039
1010
 
1040
1011
  def test_replace_table_version(self, temp_dir):
@@ -1054,13 +1025,12 @@ class TestMetafileIO:
1054
1025
 
1055
1026
  txn_operations = [
1056
1027
  TransactionOperation.of(
1057
- operation_type=TransactionOperationType.UPDATE,
1028
+ operation_type=TransactionOperationType.REPLACE,
1058
1029
  dest_metafile=replacement_table_version,
1059
1030
  src_metafile=original_table_version,
1060
1031
  )
1061
1032
  ]
1062
1033
  transaction = Transaction.of(
1063
- txn_type=TransactionType.OVERWRITE,
1064
1034
  txn_operations=txn_operations,
1065
1035
  )
1066
1036
  # when the transaction is committed
@@ -1120,16 +1090,15 @@ class TestMetafileIO:
1120
1090
  # expect a subsequent replace of the original table version to fail
1121
1091
  bad_txn_operations = [
1122
1092
  TransactionOperation.of(
1123
- operation_type=TransactionOperationType.UPDATE,
1093
+ operation_type=TransactionOperationType.REPLACE,
1124
1094
  dest_metafile=replacement_table_version,
1125
1095
  src_metafile=original_table_version,
1126
1096
  )
1127
1097
  ]
1128
1098
  transaction = Transaction.of(
1129
- txn_type=TransactionType.OVERWRITE,
1130
1099
  txn_operations=bad_txn_operations,
1131
1100
  )
1132
- with pytest.raises(ValueError):
1101
+ with pytest.raises(ObjectDeletedError):
1133
1102
  transaction.commit(temp_dir)
1134
1103
 
1135
1104
  # expect deletes of the original table version to fail
@@ -1140,10 +1109,9 @@ class TestMetafileIO:
1140
1109
  )
1141
1110
  ]
1142
1111
  transaction = Transaction.of(
1143
- txn_type=TransactionType.DELETE,
1144
1112
  txn_operations=bad_txn_operations,
1145
1113
  )
1146
- with pytest.raises(ValueError):
1114
+ with pytest.raises(ObjectDeletedError):
1147
1115
  transaction.commit(temp_dir)
1148
1116
 
1149
1117
  # expect new child metafile creation under the old table version to fail
@@ -1155,10 +1123,9 @@ class TestMetafileIO:
1155
1123
  )
1156
1124
  ]
1157
1125
  transaction = Transaction.of(
1158
- txn_type=TransactionType.APPEND,
1159
1126
  txn_operations=bad_txn_operations,
1160
1127
  )
1161
- with pytest.raises(ValueError):
1128
+ with pytest.raises(ObjectAlreadyExistsError):
1162
1129
  transaction.commit(temp_dir)
1163
1130
 
1164
1131
  def test_delete_table(self, temp_dir):
@@ -1174,7 +1141,6 @@ class TestMetafileIO:
1174
1141
  )
1175
1142
  ]
1176
1143
  transaction = Transaction.of(
1177
- txn_type=TransactionType.DELETE,
1178
1144
  txn_operations=txn_operations,
1179
1145
  )
1180
1146
  # when the transaction is committed
@@ -1220,16 +1186,15 @@ class TestMetafileIO:
1220
1186
  replacement_table: Table = Table.based_on(original_table)
1221
1187
  bad_txn_operations = [
1222
1188
  TransactionOperation.of(
1223
- operation_type=TransactionOperationType.UPDATE,
1189
+ operation_type=TransactionOperationType.REPLACE,
1224
1190
  dest_metafile=replacement_table,
1225
1191
  src_metafile=original_table,
1226
1192
  )
1227
1193
  ]
1228
1194
  transaction = Transaction.of(
1229
- txn_type=TransactionType.OVERWRITE,
1230
1195
  txn_operations=bad_txn_operations,
1231
1196
  )
1232
- with pytest.raises(ValueError):
1197
+ with pytest.raises(ObjectDeletedError):
1233
1198
  transaction.commit(temp_dir)
1234
1199
 
1235
1200
  # expect subsequent deletes of the deleted table to fail
@@ -1240,10 +1205,9 @@ class TestMetafileIO:
1240
1205
  )
1241
1206
  ]
1242
1207
  transaction = Transaction.of(
1243
- txn_type=TransactionType.DELETE,
1244
1208
  txn_operations=bad_txn_operations,
1245
1209
  )
1246
- with pytest.raises(ValueError):
1210
+ with pytest.raises(ObjectDeletedError):
1247
1211
  transaction.commit(temp_dir)
1248
1212
 
1249
1213
  # expect new child metafile creation under the deleted table to fail
@@ -1255,10 +1219,9 @@ class TestMetafileIO:
1255
1219
  )
1256
1220
  ]
1257
1221
  transaction = Transaction.of(
1258
- txn_type=TransactionType.APPEND,
1259
1222
  txn_operations=bad_txn_operations,
1260
1223
  )
1261
- with pytest.raises(ValueError):
1224
+ with pytest.raises(ObjectAlreadyExistsError):
1262
1225
  transaction.commit(temp_dir)
1263
1226
 
1264
1227
  def test_replace_table(self, temp_dir):
@@ -1277,13 +1240,12 @@ class TestMetafileIO:
1277
1240
 
1278
1241
  txn_operations = [
1279
1242
  TransactionOperation.of(
1280
- operation_type=TransactionOperationType.UPDATE,
1243
+ operation_type=TransactionOperationType.REPLACE,
1281
1244
  dest_metafile=replacement_table,
1282
1245
  src_metafile=original_table,
1283
1246
  )
1284
1247
  ]
1285
1248
  transaction = Transaction.of(
1286
- txn_type=TransactionType.OVERWRITE,
1287
1249
  txn_operations=txn_operations,
1288
1250
  )
1289
1251
  # when the transaction is committed
@@ -1345,16 +1307,15 @@ class TestMetafileIO:
1345
1307
  # expect a subsequent table replace of the original table to fail
1346
1308
  bad_txn_operations = [
1347
1309
  TransactionOperation.of(
1348
- operation_type=TransactionOperationType.UPDATE,
1310
+ operation_type=TransactionOperationType.REPLACE,
1349
1311
  dest_metafile=replacement_table,
1350
1312
  src_metafile=original_table,
1351
1313
  )
1352
1314
  ]
1353
1315
  transaction = Transaction.of(
1354
- txn_type=TransactionType.OVERWRITE,
1355
1316
  txn_operations=bad_txn_operations,
1356
1317
  )
1357
- with pytest.raises(ValueError):
1318
+ with pytest.raises(ObjectDeletedError):
1358
1319
  transaction.commit(temp_dir)
1359
1320
 
1360
1321
  # expect table deletes of the original table to fail
@@ -1365,10 +1326,9 @@ class TestMetafileIO:
1365
1326
  )
1366
1327
  ]
1367
1328
  transaction = Transaction.of(
1368
- txn_type=TransactionType.DELETE,
1369
1329
  txn_operations=bad_txn_operations,
1370
1330
  )
1371
- with pytest.raises(ValueError):
1331
+ with pytest.raises(ObjectDeletedError):
1372
1332
  transaction.commit(temp_dir)
1373
1333
 
1374
1334
  # expect new child metafile creation under the old table to fail
@@ -1380,10 +1340,9 @@ class TestMetafileIO:
1380
1340
  )
1381
1341
  ]
1382
1342
  transaction = Transaction.of(
1383
- txn_type=TransactionType.APPEND,
1384
1343
  txn_operations=bad_txn_operations,
1385
1344
  )
1386
- with pytest.raises(ValueError):
1345
+ with pytest.raises(ObjectAlreadyExistsError):
1387
1346
  transaction.commit(temp_dir)
1388
1347
 
1389
1348
  def test_delete_namespace(self, temp_dir):
@@ -1399,7 +1358,6 @@ class TestMetafileIO:
1399
1358
  )
1400
1359
  ]
1401
1360
  transaction = Transaction.of(
1402
- txn_type=TransactionType.DELETE,
1403
1361
  txn_operations=txn_operations,
1404
1362
  )
1405
1363
  # when the transaction is committed
@@ -1448,16 +1406,15 @@ class TestMetafileIO:
1448
1406
  replacement_namespace: Namespace = Namespace.based_on(original_namespace)
1449
1407
  bad_txn_operations = [
1450
1408
  TransactionOperation.of(
1451
- operation_type=TransactionOperationType.UPDATE,
1409
+ operation_type=TransactionOperationType.REPLACE,
1452
1410
  dest_metafile=replacement_namespace,
1453
1411
  src_metafile=original_namespace,
1454
1412
  )
1455
1413
  ]
1456
1414
  transaction = Transaction.of(
1457
- txn_type=TransactionType.OVERWRITE,
1458
1415
  txn_operations=bad_txn_operations,
1459
1416
  )
1460
- with pytest.raises(ValueError):
1417
+ with pytest.raises(ObjectDeletedError):
1461
1418
  transaction.commit(temp_dir)
1462
1419
 
1463
1420
  # expect subsequent deletes of the deleted namespace to fail
@@ -1468,10 +1425,9 @@ class TestMetafileIO:
1468
1425
  )
1469
1426
  ]
1470
1427
  transaction = Transaction.of(
1471
- txn_type=TransactionType.DELETE,
1472
1428
  txn_operations=bad_txn_operations,
1473
1429
  )
1474
- with pytest.raises(ValueError):
1430
+ with pytest.raises(ObjectDeletedError):
1475
1431
  transaction.commit(temp_dir)
1476
1432
 
1477
1433
  # expect new child metafile creation under the deleted namespace to fail
@@ -1483,10 +1439,9 @@ class TestMetafileIO:
1483
1439
  )
1484
1440
  ]
1485
1441
  transaction = Transaction.of(
1486
- txn_type=TransactionType.APPEND,
1487
1442
  txn_operations=bad_txn_operations,
1488
1443
  )
1489
- with pytest.raises(ValueError):
1444
+ with pytest.raises(ObjectAlreadyExistsError):
1490
1445
  transaction.commit(temp_dir)
1491
1446
 
1492
1447
  def test_replace_namespace(self, temp_dir):
@@ -1505,13 +1460,12 @@ class TestMetafileIO:
1505
1460
 
1506
1461
  txn_operations = [
1507
1462
  TransactionOperation.of(
1508
- operation_type=TransactionOperationType.UPDATE,
1463
+ operation_type=TransactionOperationType.REPLACE,
1509
1464
  dest_metafile=replacement_namespace,
1510
1465
  src_metafile=original_namespace,
1511
1466
  )
1512
1467
  ]
1513
1468
  transaction = Transaction.of(
1514
- txn_type=TransactionType.OVERWRITE,
1515
1469
  txn_operations=txn_operations,
1516
1470
  )
1517
1471
  # when the transaction is committed
@@ -1576,16 +1530,15 @@ class TestMetafileIO:
1576
1530
  # expect a subsequent namespace replace of the original namespace to fail
1577
1531
  bad_txn_operations = [
1578
1532
  TransactionOperation.of(
1579
- operation_type=TransactionOperationType.UPDATE,
1533
+ operation_type=TransactionOperationType.REPLACE,
1580
1534
  dest_metafile=replacement_namespace,
1581
1535
  src_metafile=original_namespace,
1582
1536
  )
1583
1537
  ]
1584
1538
  transaction = Transaction.of(
1585
- txn_type=TransactionType.OVERWRITE,
1586
1539
  txn_operations=bad_txn_operations,
1587
1540
  )
1588
- with pytest.raises(ValueError):
1541
+ with pytest.raises(ObjectDeletedError):
1589
1542
  transaction.commit(temp_dir)
1590
1543
 
1591
1544
  # expect namespace deletes of the original namespace to fail
@@ -1596,10 +1549,9 @@ class TestMetafileIO:
1596
1549
  )
1597
1550
  ]
1598
1551
  transaction = Transaction.of(
1599
- txn_type=TransactionType.DELETE,
1600
1552
  txn_operations=bad_txn_operations,
1601
1553
  )
1602
- with pytest.raises(ValueError):
1554
+ with pytest.raises(ObjectDeletedError):
1603
1555
  transaction.commit(temp_dir)
1604
1556
 
1605
1557
  # expect new child metafile creation under the old namespace to fail
@@ -1611,10 +1563,9 @@ class TestMetafileIO:
1611
1563
  )
1612
1564
  ]
1613
1565
  transaction = Transaction.of(
1614
- txn_type=TransactionType.APPEND,
1615
1566
  txn_operations=bad_txn_operations,
1616
1567
  )
1617
- with pytest.raises(ValueError):
1568
+ with pytest.raises(ObjectAlreadyExistsError):
1618
1569
  transaction.commit(temp_dir)
1619
1570
 
1620
1571
  def test_create_stream_bad_order_txn_op_chaining(self, temp_dir):
@@ -1650,16 +1601,14 @@ class TestMetafileIO:
1650
1601
  ),
1651
1602
  ]
1652
1603
  transaction = Transaction.of(
1653
- txn_type=TransactionType.APPEND,
1654
1604
  txn_operations=txn_operations,
1655
1605
  )
1656
1606
  # when the transaction is committed,
1657
1607
  # expect stream creation to fail
1658
- with pytest.raises(ValueError):
1608
+ with pytest.raises(ObjectNotFoundError):
1659
1609
  transaction.commit(temp_dir)
1660
1610
  # when a transaction with the operations reversed is committed,
1661
1611
  transaction = Transaction.of(
1662
- txn_type=TransactionType.APPEND,
1663
1612
  txn_operations=list(reversed(txn_operations)),
1664
1613
  )
1665
1614
  # expect table version and stream creation to succeed
@@ -1699,16 +1648,14 @@ class TestMetafileIO:
1699
1648
  ),
1700
1649
  ]
1701
1650
  transaction = Transaction.of(
1702
- txn_type=TransactionType.ALTER,
1703
1651
  txn_operations=txn_operations,
1704
1652
  )
1705
1653
  # when the transaction is committed,
1706
1654
  # expect the transaction to fail due to incorrect operation order
1707
- with pytest.raises(ValueError):
1655
+ with pytest.raises(ObjectNotFoundError):
1708
1656
  transaction.commit(temp_dir)
1709
1657
  # when a transaction with the operations reversed is committed,
1710
1658
  transaction = Transaction.of(
1711
- txn_type=TransactionType.ALTER,
1712
1659
  txn_operations=list(reversed(txn_operations)),
1713
1660
  )
1714
1661
  # expect table and table version creation to succeed
@@ -1721,7 +1668,6 @@ class TestMetafileIO:
1721
1668
  # given serial transaction that try to create two namespaces with
1722
1669
  # the same name
1723
1670
  transaction = Transaction.of(
1724
- txn_type=TransactionType.APPEND,
1725
1671
  txn_operations=[
1726
1672
  TransactionOperation.of(
1727
1673
  TransactionOperationType.CREATE,
@@ -1734,7 +1680,7 @@ class TestMetafileIO:
1734
1680
  deserialized_namespace = Namespace.read(write_paths.pop())
1735
1681
  assert namespace.equivalent_to(deserialized_namespace)
1736
1682
  # but expect the second transaction to fail
1737
- with pytest.raises(ValueError):
1683
+ with pytest.raises(ObjectAlreadyExistsError):
1738
1684
  transaction.commit(temp_dir)
1739
1685
 
1740
1686
  def test_create_duplicate_namespace_txn_op_chaining(self, temp_dir):
@@ -1742,7 +1688,6 @@ class TestMetafileIO:
1742
1688
  namespace = Namespace.of(locator=namespace_locator)
1743
1689
  # given a transaction that tries to create the same namespace twice
1744
1690
  transaction = Transaction.of(
1745
- txn_type=TransactionType.APPEND,
1746
1691
  txn_operations=[
1747
1692
  TransactionOperation.of(
1748
1693
  TransactionOperationType.CREATE,
@@ -1756,7 +1701,7 @@ class TestMetafileIO:
1756
1701
  )
1757
1702
  # when the transaction is committed,
1758
1703
  # expect duplicate namespace creation to fail
1759
- with pytest.raises(ValueError):
1704
+ with pytest.raises(ObjectAlreadyExistsError):
1760
1705
  transaction.commit(temp_dir)
1761
1706
 
1762
1707
  def test_create_stream_in_missing_table_version(self, temp_dir):
@@ -1772,7 +1717,6 @@ class TestMetafileIO:
1772
1717
  )
1773
1718
  new_stream.table_version_locator.table_version = "missing_table_version.0"
1774
1719
  transaction = Transaction.of(
1775
- txn_type=TransactionType.APPEND,
1776
1720
  txn_operations=[
1777
1721
  TransactionOperation.of(
1778
1722
  TransactionOperationType.CREATE,
@@ -1782,7 +1726,7 @@ class TestMetafileIO:
1782
1726
  )
1783
1727
  # when the transaction is committed,
1784
1728
  # expect stream creation to fail
1785
- with pytest.raises(ValueError):
1729
+ with pytest.raises(ObjectNotFoundError):
1786
1730
  transaction.commit(temp_dir)
1787
1731
 
1788
1732
  def test_create_table_version_in_missing_namespace(self, temp_dir):
@@ -1798,7 +1742,6 @@ class TestMetafileIO:
1798
1742
  )
1799
1743
  new_table_version.namespace_locator.namespace = "missing_namespace"
1800
1744
  transaction = Transaction.of(
1801
- txn_type=TransactionType.APPEND,
1802
1745
  txn_operations=[
1803
1746
  TransactionOperation.of(
1804
1747
  TransactionOperationType.CREATE,
@@ -1808,7 +1751,7 @@ class TestMetafileIO:
1808
1751
  )
1809
1752
  # when the transaction is committed,
1810
1753
  # expect table version creation to fail
1811
- with pytest.raises(ValueError):
1754
+ with pytest.raises(ObjectNotFoundError):
1812
1755
  transaction.commit(temp_dir)
1813
1756
 
1814
1757
  def test_create_table_version_in_missing_table(self, temp_dir):
@@ -1824,7 +1767,6 @@ class TestMetafileIO:
1824
1767
  )
1825
1768
  new_table_version.table_locator.table_name = "missing_table"
1826
1769
  transaction = Transaction.of(
1827
- txn_type=TransactionType.APPEND,
1828
1770
  txn_operations=[
1829
1771
  TransactionOperation.of(
1830
1772
  TransactionOperationType.CREATE,
@@ -1834,7 +1776,7 @@ class TestMetafileIO:
1834
1776
  )
1835
1777
  # when the transaction is committed,
1836
1778
  # expect table version creation to fail
1837
- with pytest.raises(ValueError):
1779
+ with pytest.raises(ObjectNotFoundError):
1838
1780
  transaction.commit(temp_dir)
1839
1781
 
1840
1782
  def test_create_table_in_missing_namespace(self, temp_dir):
@@ -1849,7 +1791,6 @@ class TestMetafileIO:
1849
1791
  # given a transaction that tries to create a single table in a
1850
1792
  # namespace that doesn't exist
1851
1793
  transaction = Transaction.of(
1852
- txn_type=TransactionType.APPEND,
1853
1794
  txn_operations=[
1854
1795
  TransactionOperation.of(
1855
1796
  TransactionOperationType.CREATE,
@@ -1859,7 +1800,7 @@ class TestMetafileIO:
1859
1800
  )
1860
1801
  # when the transaction is committed,
1861
1802
  # expect table creation to fail
1862
- with pytest.raises(ValueError):
1803
+ with pytest.raises(ObjectNotFoundError):
1863
1804
  transaction.commit(temp_dir)
1864
1805
 
1865
1806
  def test_rename_table_txn_op_chaining(self, temp_dir):
@@ -1932,7 +1873,6 @@ class TestMetafileIO:
1932
1873
  ),
1933
1874
  ]
1934
1875
  transaction = Transaction.of(
1935
- txn_type=TransactionType.ALTER,
1936
1876
  txn_operations=txn_operations,
1937
1877
  )
1938
1878
  # when the transaction is committed
@@ -1986,7 +1926,6 @@ class TestMetafileIO:
1986
1926
  )
1987
1927
  ]
1988
1928
  transaction = Transaction.of(
1989
- txn_type=TransactionType.ALTER,
1990
1929
  txn_operations=txn_operations,
1991
1930
  )
1992
1931
  # when the transaction is committed
@@ -2038,10 +1977,9 @@ class TestMetafileIO:
2038
1977
  )
2039
1978
  ]
2040
1979
  transaction = Transaction.of(
2041
- txn_type=TransactionType.RESTATE,
2042
1980
  txn_operations=bad_txn_operations,
2043
1981
  )
2044
- with pytest.raises(ValueError):
1982
+ with pytest.raises(ObjectDeletedError):
2045
1983
  transaction.commit(temp_dir)
2046
1984
 
2047
1985
  # expect table deletes of the old table name fail
@@ -2052,10 +1990,9 @@ class TestMetafileIO:
2052
1990
  )
2053
1991
  ]
2054
1992
  transaction = Transaction.of(
2055
- txn_type=TransactionType.DELETE,
2056
1993
  txn_operations=bad_txn_operations,
2057
1994
  )
2058
- with pytest.raises(ValueError):
1995
+ with pytest.raises(ObjectDeletedError):
2059
1996
  transaction.commit(temp_dir)
2060
1997
 
2061
1998
  # expect child metafile creation under the old table name to fail
@@ -2067,10 +2004,9 @@ class TestMetafileIO:
2067
2004
  )
2068
2005
  ]
2069
2006
  transaction = Transaction.of(
2070
- txn_type=TransactionType.APPEND,
2071
2007
  txn_operations=bad_txn_operations,
2072
2008
  )
2073
- with pytest.raises(ValueError):
2009
+ with pytest.raises(ObjectAlreadyExistsError):
2074
2010
  transaction.commit(temp_dir)
2075
2011
 
2076
2012
  def test_rename_namespace(self, temp_dir):
@@ -2091,7 +2027,6 @@ class TestMetafileIO:
2091
2027
  )
2092
2028
  ]
2093
2029
  transaction = Transaction.of(
2094
- txn_type=TransactionType.ALTER,
2095
2030
  txn_operations=txn_operations,
2096
2031
  )
2097
2032
  # when the transaction is committed
@@ -2146,10 +2081,9 @@ class TestMetafileIO:
2146
2081
  )
2147
2082
  ]
2148
2083
  transaction = Transaction.of(
2149
- txn_type=TransactionType.ALTER,
2150
2084
  txn_operations=bad_txn_operations,
2151
2085
  )
2152
- with pytest.raises(ValueError):
2086
+ with pytest.raises(ObjectDeletedError):
2153
2087
  transaction.commit(temp_dir)
2154
2088
 
2155
2089
  # expect namespace deletes of the old namespace name fail
@@ -2160,10 +2094,9 @@ class TestMetafileIO:
2160
2094
  )
2161
2095
  ]
2162
2096
  transaction = Transaction.of(
2163
- txn_type=TransactionType.DELETE,
2164
2097
  txn_operations=bad_txn_operations,
2165
2098
  )
2166
- with pytest.raises(ValueError):
2099
+ with pytest.raises(ObjectDeletedError):
2167
2100
  transaction.commit(temp_dir)
2168
2101
 
2169
2102
  # expect child metafile creation under the old namespace to fail
@@ -2175,10 +2108,9 @@ class TestMetafileIO:
2175
2108
  )
2176
2109
  ]
2177
2110
  transaction = Transaction.of(
2178
- txn_type=TransactionType.APPEND,
2179
2111
  txn_operations=bad_txn_operations,
2180
2112
  )
2181
- with pytest.raises(ValueError):
2113
+ with pytest.raises(ObjectAlreadyExistsError):
2182
2114
  transaction.commit(temp_dir)
2183
2115
 
2184
2116
  def test_e2e_serde(self, temp_dir):
@@ -2195,7 +2127,6 @@ class TestMetafileIO:
2195
2127
  namespace = Namespace.of(locator=namespace_locator)
2196
2128
  # given a transaction that creates a single namespace
2197
2129
  write_paths, txn_log_path = Transaction.of(
2198
- txn_type=TransactionType.APPEND,
2199
2130
  txn_operations=[
2200
2131
  TransactionOperation.of(
2201
2132
  operation_type=TransactionOperationType.CREATE,
@@ -2219,7 +2150,6 @@ class TestMetafileIO:
2219
2150
  )
2220
2151
  # given a transaction that creates a single table
2221
2152
  write_paths, txn_log_path = Transaction.of(
2222
- txn_type=TransactionType.APPEND,
2223
2153
  txn_operations=[
2224
2154
  TransactionOperation.of(
2225
2155
  operation_type=TransactionOperationType.CREATE,
@@ -2267,7 +2197,7 @@ class TestMetafileIO:
2267
2197
  PartitionKey.of(
2268
2198
  key=["some_string", "some_int32"],
2269
2199
  name="test_partition_key",
2270
- field_id="test_field_id",
2200
+ field_id=1,
2271
2201
  transform=bucket_transform,
2272
2202
  )
2273
2203
  ]
@@ -2307,7 +2237,6 @@ class TestMetafileIO:
2307
2237
  )
2308
2238
  # given a transaction that creates a single table version
2309
2239
  write_paths, txn_log_path = Transaction.of(
2310
- txn_type=TransactionType.APPEND,
2311
2240
  txn_operations=[
2312
2241
  TransactionOperation.of(
2313
2242
  operation_type=TransactionOperationType.CREATE,
@@ -2338,7 +2267,7 @@ class TestMetafileIO:
2338
2267
  PartitionKey.of(
2339
2268
  key=["some_string", "some_int32"],
2340
2269
  name="test_partition_key",
2341
- field_id="test_field_id",
2270
+ field_id=1,
2342
2271
  transform=bucket_transform,
2343
2272
  )
2344
2273
  ]
@@ -2356,7 +2285,6 @@ class TestMetafileIO:
2356
2285
  )
2357
2286
  # given a transaction that creates a single stream
2358
2287
  write_paths, txn_log_path = Transaction.of(
2359
- txn_type=TransactionType.APPEND,
2360
2288
  txn_operations=[
2361
2289
  TransactionOperation.of(
2362
2290
  operation_type=TransactionOperationType.CREATE,
@@ -2379,28 +2307,8 @@ class TestMetafileIO:
2379
2307
  partition_values=["a", 1],
2380
2308
  partition_id="test_partition_id",
2381
2309
  )
2382
- schema = Schema.of(
2383
- [
2384
- Field.of(
2385
- field=pa.field("some_string", pa.string(), nullable=False),
2386
- field_id=1,
2387
- is_merge_key=True,
2388
- ),
2389
- Field.of(
2390
- field=pa.field("some_int32", pa.int32(), nullable=False),
2391
- field_id=2,
2392
- is_merge_key=True,
2393
- ),
2394
- Field.of(
2395
- field=pa.field("some_float64", pa.float64()),
2396
- field_id=3,
2397
- is_merge_key=False,
2398
- ),
2399
- ]
2400
- )
2401
2310
  partition = Partition.of(
2402
2311
  locator=partition_locator,
2403
- schema=schema,
2404
2312
  content_types=[ContentType.PARQUET],
2405
2313
  state=CommitState.STAGED,
2406
2314
  previous_stream_position=0,
@@ -2410,7 +2318,6 @@ class TestMetafileIO:
2410
2318
  )
2411
2319
  # given a transaction that creates a single partition
2412
2320
  write_paths, txn_log_path = Transaction.of(
2413
- txn_type=TransactionType.APPEND,
2414
2321
  txn_operations=[
2415
2322
  TransactionOperation.of(
2416
2323
  operation_type=TransactionOperationType.CREATE,
@@ -2472,7 +2379,6 @@ class TestMetafileIO:
2472
2379
  )
2473
2380
  # given a transaction that creates a single delta
2474
2381
  write_paths, txn_log_path = Transaction.of(
2475
- txn_type=TransactionType.APPEND,
2476
2382
  txn_operations=[
2477
2383
  TransactionOperation.of(
2478
2384
  operation_type=TransactionOperationType.CREATE,
@@ -2511,7 +2417,6 @@ class TestMetafileIO:
2511
2417
  )
2512
2418
  # when a transaction commits this table
2513
2419
  write_paths, txn_log_path = Transaction.of(
2514
- txn_type=TransactionType.APPEND,
2515
2420
  txn_operations=[
2516
2421
  TransactionOperation.of(
2517
2422
  operation_type=TransactionOperationType.CREATE,