deltacat 1.1.13__py3-none-any.whl → 1.1.15__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- deltacat/__init__.py +1 -1
- deltacat/compute/compactor/compaction_session.py +3 -2
- deltacat/compute/compactor/model/compact_partition_params.py +11 -1
- deltacat/compute/compactor/model/compaction_session_audit_info.py +2 -2
- deltacat/compute/compactor/model/delta_annotated.py +2 -4
- deltacat/compute/compactor/steps/hash_bucket.py +2 -3
- deltacat/compute/compactor_v2/compaction_session.py +26 -27
- deltacat/compute/compactor_v2/constants.py +4 -0
- deltacat/compute/compactor_v2/private/__init__.py +0 -0
- deltacat/compute/compactor_v2/private/compaction_utils.py +753 -0
- deltacat/compute/compactor_v2/steps/merge.py +0 -3
- deltacat/compute/compactor_v2/utils/delta.py +2 -3
- deltacat/compute/compactor_v2/utils/io.py +0 -2
- deltacat/compute/compactor_v2/utils/merge.py +0 -1
- deltacat/tests/compute/compact_partition_multiple_rounds_test_cases.py +855 -0
- deltacat/tests/compute/compactor_v2/test_compaction_session.py +1 -1
- deltacat/tests/compute/test_compact_partition_multiple_rounds.py +330 -0
- deltacat/tests/compute/test_compact_partition_rebase.py +1 -1
- deltacat/tests/compute/test_util_create_table_deltas_repo.py +118 -0
- deltacat/tests/local_deltacat_storage/__init__.py +8 -5
- {deltacat-1.1.13.dist-info → deltacat-1.1.15.dist-info}/METADATA +1 -1
- {deltacat-1.1.13.dist-info → deltacat-1.1.15.dist-info}/RECORD +25 -21
- {deltacat-1.1.13.dist-info → deltacat-1.1.15.dist-info}/LICENSE +0 -0
- {deltacat-1.1.13.dist-info → deltacat-1.1.15.dist-info}/WHEEL +0 -0
- {deltacat-1.1.13.dist-info → deltacat-1.1.15.dist-info}/top_level.txt +0 -0
|
@@ -177,13 +177,10 @@ def _download_compacted_table(
|
|
|
177
177
|
|
|
178
178
|
if str(hb_index) not in hb_index_to_indices:
|
|
179
179
|
return None
|
|
180
|
-
|
|
181
180
|
indices = hb_index_to_indices[str(hb_index)]
|
|
182
|
-
|
|
183
181
|
assert (
|
|
184
182
|
indices is not None and len(indices) == 2
|
|
185
183
|
), "indices should not be none and contains exactly two elements"
|
|
186
|
-
|
|
187
184
|
for offset in range(indices[1] - indices[0]):
|
|
188
185
|
table = deltacat_storage.download_delta_manifest_entry(
|
|
189
186
|
rcf.compacted_delta_locator,
|
|
@@ -42,11 +42,10 @@ def read_delta_file_envelopes(
|
|
|
42
42
|
**deltacat_storage_kwargs,
|
|
43
43
|
)
|
|
44
44
|
annotations = annotated_delta.annotations
|
|
45
|
-
assert (
|
|
46
|
-
len(tables) == len(annotations),
|
|
45
|
+
assert len(tables) == len(annotations), (
|
|
47
46
|
f"Unexpected Error: Length of downloaded delta manifest tables "
|
|
48
47
|
f"({len(tables)}) doesn't match the length of delta manifest "
|
|
49
|
-
f"annotations ({len(annotations)})."
|
|
48
|
+
f"annotations ({len(annotations)})."
|
|
50
49
|
)
|
|
51
50
|
if not tables:
|
|
52
51
|
return None, 0, 0
|
|
@@ -61,7 +61,6 @@ def discover_deltas(
|
|
|
61
61
|
)
|
|
62
62
|
|
|
63
63
|
result.extend(delta_source_incremental_deltas)
|
|
64
|
-
|
|
65
64
|
logger.info(
|
|
66
65
|
f"Length of input deltas from delta source table is {len(delta_source_incremental_deltas)}"
|
|
67
66
|
f" from ({previous_compacted_high_watermark}, {last_stream_position_to_compact}]"
|
|
@@ -132,7 +131,6 @@ def create_uniform_input_deltas(
|
|
|
132
131
|
size_estimation_function = functools.partial(
|
|
133
132
|
estimate_manifest_entry_size_bytes, previous_inflation=previous_inflation
|
|
134
133
|
)
|
|
135
|
-
|
|
136
134
|
rebatched_da_list = DeltaAnnotated.rebatch(
|
|
137
135
|
input_da_list,
|
|
138
136
|
min_delta_bytes=min_delta_bytes,
|