deltacat 1.1.38__py3-none-any.whl → 2.0.0__py3-none-any.whl

deltacat/tests/experimental/storage/rivulet/test_dataset.py

@@ -0,0 +1,408 @@
+import posixpath
+from deltacat.utils.metafile_locator import _find_partition_path
+import pytest
+
+import pyarrow as pa
+from deltacat.experimental.storage.rivulet import Schema, Field, Datatype
+from deltacat.experimental.storage.rivulet.dataset import Dataset
+from deltacat.experimental.storage.rivulet.reader.query_expression import (
+    QueryExpression,
+)
+
+
+@pytest.fixture
+def sample_schema():
+    return Schema(
+        fields=[
+            Field("id", Datatype.int32(), is_merge_key=True),
+            Field("name", Datatype.string()),
+            Field("age", Datatype.int32()),
+        ]
+    )
+
+
+@pytest.fixture
+def sample_pydict():
+    return {"id": [1, 2, 3], "name": ["Alice", "Bob", "Charlie"], "age": [25, 30, 35]}
+
+
+@pytest.fixture
+def sample_parquet_data(tmp_path, sample_pydict):
+    parquet_path = tmp_path / "test.parquet"
+    table = pa.Table.from_pydict(sample_pydict)
+    pa.parquet.write_table(table, parquet_path)
+    return parquet_path
+
+
+# Updated Tests
+
+
+def test_dataset_creation_with_schema(tmp_path, sample_schema):
+    dataset = Dataset(dataset_name="test_dataset", schema=sample_schema)
+    assert len(dataset.fields) == 3
+    assert "id" in dataset.fields
+    assert dataset.fields["id"].is_merge_key
+
+
+def test_dataset_initialization_with_metadata(tmp_path):
+    dataset = Dataset(dataset_name="test_dataset", metadata_uri=str(tmp_path))
+    assert dataset.dataset_name == "test_dataset"
+    assert dataset._metadata_folder.startswith(".riv-meta")
+
+
+def test_invalid_dataset_initialization():
+    with pytest.raises(ValueError, match="Name must be a non-empty string"):
+        Dataset(dataset_name="")
+
+
+def test_dataset_creation_metadata_structure(tmp_path):
+    dataset = Dataset(dataset_name="test_dataset", metadata_uri=str(tmp_path))
+
+    assert dataset._metadata_folder.startswith(".riv-meta")
+    assert dataset._namespace == "default"
+    assert dataset.dataset_name == "test_dataset"
+    assert dataset._metadata_path == str(tmp_path / ".riv-meta-test_dataset")
+
+    locator = dataset._locator
+    root_uri = dataset._metadata_path
+
+    partition_path = _find_partition_path(root_uri, locator)
+
+    # Ensures that directory structure for namespace -> table -> table_version -> stream_id -> partition_id exists
+    assert posixpath.exists(partition_path)
+
+
+def test_fields_accessor_add_field(tmp_path, sample_schema):
+    dataset = Dataset(dataset_name="test_dataset", schema=sample_schema)
+    dataset.fields.add("new_field", Datatype.float())
+    assert "new_field" in dataset.fields
+    assert dataset.fields["new_field"].datatype == Datatype.float()
+
+    dataset.fields["new_field2"] = Field("new_field2", Datatype.int32())
+    assert "new_field2" in dataset.fields
+    assert "new_field2" in dataset.schemas["all"]
+    with pytest.raises(TypeError):
+        dataset.fields["new_field3"] = 2
+
+
+def test_field_removal(tmp_path, sample_schema):
+    dataset = Dataset(dataset_name="test_dataset", schema=sample_schema)
+    del dataset.fields["age"]
+    assert "age" not in dataset.fields
+    with pytest.raises(ValueError):
+        del dataset.fields["age"]
+    with pytest.raises(KeyError):
+        _ = dataset.fields["age"]
+
+
+def test_fields_accessor_repr(tmp_path, sample_schema):
+    dataset = Dataset(dataset_name="test_dataset", schema=sample_schema)
+    repr_output = repr(dataset.fields)
+    for field_name in ["id", "name", "age"]:
+        assert field_name in repr_output, f"Field '{field_name}' missing in repr output"
+
+
+def test_schemas_accessor_add_group(tmp_path, sample_schema):
+    dataset = Dataset(dataset_name="test_dataset", schema=sample_schema)
+    dataset.schemas["analytics"] = ["id", "name"]
+    assert "analytics" in dataset.schemas
+    assert len(dataset.schemas["analytics"]) == 2
+
+
+def test_schema_removal(tmp_path, sample_schema):
+    dataset = Dataset(dataset_name="test_dataset", schema=sample_schema)
+    with pytest.raises(ValueError):
+        del dataset.schemas["all"]
+    with pytest.raises(ValueError):
+        del dataset.schemas["does_not_exist"]
+    dataset.schemas["new"] = ["id", "name"]
+    del dataset.schemas["new"]
+    with pytest.raises(KeyError):
+        _ = dataset.schemas["new"]
+
+
+def test_dataset_from_parquet(tmp_path, sample_parquet_data):
+    dataset = Dataset.from_parquet(
+        name="test_dataset",
+        file_uri=str(sample_parquet_data),
+        metadata_uri=str(tmp_path),
+        merge_keys="id",
+    )
+    assert len(dataset.fields) == 3
+    assert "id" in dataset.fields
+    assert dataset.fields["id"].is_merge_key
+
+
+def test_parquet_schema_modes(tmp_path, sample_pydict):
+    # Create two parquet files with overlapping and unique schemas
+    data_1 = {"id": [1, 2, 3], "name": ["Alice", "Bob", "Charlie"]}
+    data_2 = {"id": [4, 5, 6], "age": [25, 30, 35]}
+
+    path_1 = tmp_path / "data1.parquet"
+    path_2 = tmp_path / "data2.parquet"
+    pa.parquet.write_table(pa.Table.from_pydict(data_1), path_1)
+    pa.parquet.write_table(pa.Table.from_pydict(data_2), path_2)
+
+    dataset_union = Dataset.from_parquet(
+        name="test_dataset_union",
+        file_uri=str(tmp_path),
+        merge_keys="id",
+        schema_mode="union",
+    )
+    assert len(dataset_union.fields) == 3  # id, name, age
+
+    dataset_intersect = Dataset.from_parquet(
+        name="test_dataset_intersect",
+        file_uri=str(tmp_path),
+        merge_keys="id",
+        schema_mode="intersect",
+    )
+    assert len(dataset_intersect.fields) == 1  # Only id
+
+
+def test_merge_all_schemas():
+    schema1 = Schema(
+        fields=[
+            Field("id", Datatype.int32(), is_merge_key=True),
+            Field("name", Datatype.string()),
+        ]
+    )
+    schema2 = Schema(
+        fields=[
+            Field("id", Datatype.int32(), is_merge_key=True),
+            Field("age", Datatype.int32()),
+        ]
+    )
+    merged_schema = Schema.merge_all([schema1, schema2])
+    assert len(merged_schema) == 3
+    assert "id" in merged_schema
+    assert "name" in merged_schema
+    assert "age" in merged_schema
+
+
+def test_writer_creation_with_custom_format(tmp_path, sample_schema):
+    dataset = Dataset(dataset_name="test_dataset", schema=sample_schema)
+    writer = dataset.writer(file_format="feather")
+    assert writer is not None
+
+
+def test_scan_with_query(tmp_path, sample_schema):
+    dataset = Dataset(dataset_name="test_dataset", schema=sample_schema)
+    query = QueryExpression()  # Placeholder query
+    scan = dataset.scan(query)
+    assert scan is not None
+
+
+def test_add_schema_to_new_schemas(tmp_path):
+    """Test adding a schema to a new field group."""
+    base_uri = str(tmp_path / "test_dataset")
+    dataset = Dataset(dataset_name=base_uri)
+
+    schema = Schema(
+        [
+            ("id", Datatype.int32()),
+            ("name", Datatype.string()),
+            ("age", Datatype.int32()),
+        ],
+        merge_keys=["id"],
+    )
+
+    dataset.add_schema(schema, schema_name="new_group")
+
+    # Verify the field group is added
+    assert "new_group" in dataset.schemas
+    assert len(dataset.schemas["new_group"]) == 3
+    assert dataset.schemas["new_group"]["id"].datatype == Datatype.int32()
+    assert dataset.schemas["new_group"]["name"].datatype == Datatype.string()
+    assert dataset.schemas["new_group"]["age"].datatype == Datatype.int32()
+
+
+def test_add_schema_to_existing_schemas(tmp_path):
+    """Test merging a schema into an existing field group."""
+    base_uri = str(tmp_path / "test_dataset")
+    dataset = Dataset(dataset_name=base_uri)
+
+    schema_1 = Schema(
+        [
+            ("id", Datatype.int32()),
+            ("name", Datatype.string()),
+        ],
+        merge_keys=["id"],
+    )
+
+    dataset.add_schema(schema_1, schema_name="existing_group")
+
+    schema_2 = Schema(
+        [
+            ("age", Datatype.int32()),
+            ("email", Datatype.string()),
+        ],
+        merge_keys=["id"],
+    )
+
+    dataset.add_schema(schema_2, schema_name="existing_group")
+
+    # Verify the merged schema
+    assert "existing_group" in dataset.schemas
+    assert len(dataset.schemas["existing_group"]) == 4
+    assert dataset.schemas["existing_group"]["id"].datatype == Datatype.int32()
+    assert dataset.schemas["existing_group"]["name"].datatype == Datatype.string()
+    assert dataset.schemas["existing_group"]["age"].datatype == Datatype.int32()
+    assert dataset.schemas["existing_group"]["email"].datatype == Datatype.string()
+
+
+def test_add_schema_conflicting_fields(tmp_path):
+    """Test adding a schema with conflicting fields."""
+    base_uri = str(tmp_path / "test_dataset")
+    dataset = Dataset(dataset_name=base_uri)
+
+    schema_1 = Schema(
+        [
+            ("id", Datatype.int32()),
+            ("name", Datatype.string()),
+        ],
+        merge_keys=["id"],
+    )
+
+    dataset.add_schema(schema_1, schema_name="conflicting_group")
+
+    schema_2 = Schema(
+        [
+            ("id", Datatype.string()),  # Conflict: datatype mismatch
+            ("age", Datatype.int32()),
+        ],
+        merge_keys=["id"],
+    )
+
+    with pytest.raises(ValueError, match="already exists"):
+        dataset.add_schema(schema_2, schema_name="conflicting_group")
+
+    schema_3 = Schema(
+        [
+            ("id", Datatype.int32()),  # Conflict: datatype mismatch
+            ("age", Datatype.int32()),
+        ],
+        merge_keys=["id"],
+    )
+
+    dataset.add_schema(schema_3, schema_name="conflicting_group")
+    assert "conflicting_group" in dataset.schemas
+    assert len(dataset.schemas["conflicting_group"]) == 3
+    assert dataset.schemas["conflicting_group"]["id"].datatype == Datatype.int32()
+    assert dataset.schemas["conflicting_group"]["name"].datatype == Datatype.string()
+    assert dataset.schemas["conflicting_group"]["age"].datatype == Datatype.int32()
+
+
+def test_add_fields_with_merge_key_field(tmp_path):
+    base_uri = str(tmp_path / "test_dataset")
+    dataset = Dataset(dataset_name=base_uri)
+    dataset.add_fields([Field("my_merge_key", Datatype.string(), True)])
+    assert dataset.schemas["default"].get_merge_key() == "my_merge_key"
+
+
+def test_add_schema_to_nonexistent_schemas(tmp_path):
+    """Test adding a schema to a nonexistent field group."""
+    base_uri = str(tmp_path / "test_dataset")
+    dataset = Dataset(dataset_name=base_uri)
+
+    schema = Schema(
+        [
+            ("id", Datatype.int32()),
+            ("name", Datatype.string()),
+        ],
+        merge_keys=["id"],
+    )
+
+    # Add to a non-existent field group
+    dataset.add_schema(schema, schema_name="nonexistent_group")
+
+    # Verify the field group is created
+    assert "nonexistent_group" in dataset.schemas
+    assert len(dataset.schemas["nonexistent_group"]) == 2
+
+
+def test_add_missing_field_to_schema_raises_error(tmp_path, sample_schema):
+    """
+    Test that attempting to add a missing field to the 'all' schema raises a ValueError.
+    """
+    dataset = Dataset(dataset_name="test_dataset", schema=sample_schema)
+
+    # Attempt to add a non-existent field to the 'all' schema
+    with pytest.raises(
+        ValueError, match="Field 'missing_field' does not exist in the dataset."
+    ):
+        dataset.schemas["all"] = [
+            "missing_field"
+        ]  # Attempt to set a list with a missing field
+
+
+def test_schemas_accessor_methods(tmp_path, sample_schema):
+    """
+    Test the __iter__, __len__, and __repr__ methods of SchemasAccessor.
+    """
+    dataset = Dataset(
+        dataset_name="test_dataset", schema=sample_schema
+    )  # Default schema is defined automatically
+    dataset.schemas["schema_1"] = ["id", "name"]
+    dataset.schemas["schema_2"] = ["age"]
+
+    # Test __iter__
+    schema_names = list(iter(dataset.schemas))
+    assert set(schema_names) == {
+        "schema_1",
+        "schema_2",
+        "all",
+        "default",
+    }, "Schema names do not match expected values"
+
+    # Test __len__
+    assert len(dataset.schemas) == 4, "Length of schemas accessor is incorrect"
+
+    # Test __repr__
+    repr_output = repr(dataset.schemas)
+    for schema_name in ["schema_1", "schema_2", "all"]:
+        assert (
+            schema_name in repr_output
+        ), f"Schema '{schema_name}' missing in repr output"
+
+
+def test_get_merge_keys(tmp_path, sample_schema):
+    """
+    Test the get_merge_keys method to ensure it returns all merge keys in the dataset.
+    """
+    dataset = Dataset(dataset_name="test_dataset", schema=sample_schema)
+
+    # Add fields with additional merge key to the dataset
+    other_schema = Schema(
+        [("id2", Datatype.int32()), ("zip", Datatype.string())], merge_keys=["id2"]
+    )
+
+    dataset.add_schema(other_schema, "id2+zip")
+
+    # Call get_merge_keys and validate the result
+    merge_keys = dataset.get_merge_keys()
+    assert merge_keys == [
+        "id",
+        "id2",
+    ], f"Expected merge keys ['id', 'id2'], got {merge_keys}"
+
+
+def test_add_fields_no_fields_raises_error(tmp_path, sample_schema):
+    dataset = Dataset(dataset_name="test_dataset")
+    with pytest.raises(ValueError):
+        dataset.add_fields(fields=[])
+
+
+def test_add_fields_mismatched_merge_keys_raises_error(tmp_path, sample_schema):
+    dataset = Dataset(dataset_name="test_dataset")
+    with pytest.raises(
+        ValueError,
+        match="The following merge keys were not found in the provided fields: does_not_exist",
+    ):
+        dataset.add_fields(fields=sample_schema.values(), merge_keys=["does_not_exist"])
+
+    with pytest.raises(TypeError, match="Merge key status conflict"):
+        dataset.add_fields(
+            fields=[Field("id", Datatype.int32()), Field("name", Datatype.string())],
+            merge_keys=["id"],
+        )

deltacat/tests/experimental/storage/rivulet/test_manifest.py

@@ -0,0 +1,67 @@
+import os
+
+import pytest
+
+from deltacat.experimental.storage.rivulet.dataset import Dataset
+from deltacat.experimental.storage.rivulet.fs.file_store import FileStore
+from deltacat.experimental.storage.rivulet.schema.datatype import Datatype
+from deltacat.experimental.storage.rivulet.metastore.delta import DeltacatManifestIO
+from deltacat.experimental.storage.rivulet import Schema, Field
+import pyarrow as pa
+import pyarrow.parquet
+
+
+@pytest.fixture
+def sample_schema():
+    return Schema(
+        fields=[
+            Field("id", Datatype.int32(), is_merge_key=True),
+            Field("name", Datatype.string()),
+            Field("age", Datatype.int32()),
+        ]
+    )
+
+
+@pytest.fixture
+def sample_pydict():
+    return {"id": [1, 2, 3], "name": ["Alice", "Bob", "Charlie"], "age": [25, 30, 35]}
+
+
+@pytest.fixture
+def path(tmp_path):
+    return tmp_path
+
+
+@pytest.fixture
+def sample_parquet_data(path, sample_pydict):
+    parquet_path = path / "test.parquet"
+    table = pa.Table.from_pydict(sample_pydict)
+    pyarrow.parquet.write_table(table, parquet_path)
+    return parquet_path
+
+
+def test_write_manifest_round_trip(sample_parquet_data, sample_schema):
+    dataset = Dataset.from_parquet(
+        file_uri=sample_parquet_data, name="dataset", merge_keys="id"
+    )
+
+    path, filesystem = FileStore.filesystem(dataset._metadata_path)
+    file_store = FileStore(path, filesystem=filesystem)
+    manifest_io = DeltacatManifestIO(path, dataset._locator)
+
+    sst_files = ["sst1.sst", "sst2.sst"]
+    schema = Schema(
+        {("id", Datatype.int32()), ("name", Datatype.string())},
+        "id",
+    )
+    level = 2
+
+    uri = os.path.join(path, "manifest.json")
+
+    file_store.create_output_file(uri)
+    written = manifest_io.write(sst_files, schema, level)
+    manifest = manifest_io.read(written)
+
+    assert manifest.context.schema == schema
+    assert manifest.context.level == level
+    assert manifest.sst_files == sst_files

deltacat/tests/experimental/storage/rivulet/test_sst_interval_tree.py

@@ -0,0 +1,232 @@
+from typing import List, FrozenSet, Dict
+
+import pytest
+
+from deltacat.experimental.storage.rivulet.metastore.delta import DeltaContext
+from deltacat.experimental.storage.rivulet.metastore.sst import SSTable, SSTableRow
+from deltacat.experimental.storage.rivulet.metastore.sst_interval_tree import (
+    BlockIntervalTree,
+    BlockGroup,
+    OrderedBlockGroups,
+    Block,
+)
+from deltacat.experimental.storage.rivulet.schema.datatype import Datatype
+from deltacat.experimental.storage.rivulet import Schema
+
+
+@pytest.fixture
+def schema1() -> Schema:
+    return Schema(
+        {
+            ("id", Datatype.int32()),
+            ("name", Datatype.string()),
+            ("age", Datatype.int32()),
+        },
+        "id",
+    )
+
+
+@pytest.fixture
+def schema2() -> Schema:
+    return Schema(
+        {
+            ("id", Datatype.int32()),
+            ("address", Datatype.string()),
+            ("zip", Datatype.string()),
+        },
+        "id",
+    )
+
+
+@pytest.fixture
+def sst_row_list() -> List[SSTableRow]:
+    return [
+        SSTableRow(0, 100, "block1", 0, 1),
+        SSTableRow(3, 90, "block2", 0, 1),
+        SSTableRow(10, 95, "block3", 0, 1),
+        SSTableRow(0, 10, "block4", 0, 1),
+        SSTableRow(0, 100, "block5", 0, 1),
+    ]
+
+
+@pytest.fixture
+def sst1(sst_row_list) -> SSTable:
+    return SSTable(sst_row_list[0:3], 0, 100)
+
+
+@pytest.fixture
+def sst2(sst_row_list) -> SSTable:
+    return SSTable(sst_row_list[3:5], 0, 100)
+
+
+@pytest.fixture
+def manifest_context1(schema1) -> DeltaContext:
+    return DeltaContext(schema1, "manifest-001", 0)
+
+
+@pytest.fixture
+def manifest_context2(schema2) -> DeltaContext:
+    return DeltaContext(schema2, "manifest-002", 1)
+
+
+def with_field_group(
+    context: DeltaContext, rows: List[SSTableRow], indexes: List[int]
+) -> Dict[Schema, FrozenSet[Block]]:
+    """Construct a BlockGroup dict for a singular field group"""
+    schema = context.schema
+    return {schema: frozenset([Block(rows[i], context) for i in indexes])}
+
+
+@pytest.fixture
+def expected_block_groups(
+    manifest_context1, manifest_context2, sst_row_list
+) -> List[BlockGroup]:
+    return [
+        BlockGroup(
+            0,
+            3,
+            with_field_group(manifest_context1, sst_row_list, [0])
+            | with_field_group(manifest_context2, sst_row_list, [3, 4]),
+        ),
+        BlockGroup(
+            3,
+            10,
+            with_field_group(manifest_context1, sst_row_list, [0, 1])
+            | with_field_group(manifest_context2, sst_row_list, [3, 4]),
+        ),
+        BlockGroup(
+            10,
+            90,
+            with_field_group(manifest_context1, sst_row_list, [0, 1, 2])
+            | with_field_group(manifest_context2, sst_row_list, [3, 4]),
+        ),
+        BlockGroup(
+            90,
+            95,
+            with_field_group(manifest_context1, sst_row_list, [0, 1, 2])
+            | with_field_group(manifest_context2, sst_row_list, [4]),
+        ),
+        BlockGroup(
+            95,
+            100,
+            with_field_group(manifest_context1, sst_row_list, [0, 2])
+            | with_field_group(manifest_context2, sst_row_list, [4]),
+        ),
+    ]
+
+
+def test_build_sst(
+    sst1,
+    sst2,
+    manifest_context1,
+    manifest_context2,
+    sst_row_list,
+    expected_block_groups,
+):
+    t = BlockIntervalTree()
+    t.add_sst_table(sst1, manifest_context1)
+    t.add_sst_table(sst2, manifest_context2)
+
+    block_groups = t.get_sorted_block_groups()
+    expected = _build_ordered_block_groups(expected_block_groups)
+    assert expected == block_groups
+
+
+def test_build_sst_with_bounds(
+    sst1,
+    sst2,
+    manifest_context1,
+    manifest_context2,
+    sst_row_list,
+    expected_block_groups,
+):
+    t = BlockIntervalTree()
+    t.add_sst_table(sst1, manifest_context1)
+    t.add_sst_table(sst2, manifest_context2)
+
+    block_groups_filtered = t.get_sorted_block_groups(20, 100)
+    expected = _build_ordered_block_groups(expected_block_groups[2:])
+    assert expected == block_groups_filtered
+
+    block_groups_filtered = t.get_sorted_block_groups(96, 100)
+    expected = _build_ordered_block_groups(expected_block_groups[4:])
+    assert expected == block_groups_filtered
+
+    block_groups_filtered = t.get_sorted_block_groups(0, 10)
+    expected = _build_ordered_block_groups(expected_block_groups[0:3])
+    assert expected == block_groups_filtered
+
+    # Max key of 95 is inclusive of last range so it is included
+    block_groups_filtered = t.get_sorted_block_groups(None, 95)
+    expected = _build_ordered_block_groups(expected_block_groups)
+    assert expected == block_groups_filtered
+
+    block_groups_filtered = t.get_sorted_block_groups(None, 94)
+    expected = _build_ordered_block_groups(expected_block_groups[0:4])
+    assert expected == block_groups_filtered
+
+    block_groups_filtered = t.get_sorted_block_groups(0, 10)
+    expected = _build_ordered_block_groups(expected_block_groups[0:3])
+    assert expected == block_groups_filtered
+
+    block_groups_filtered = t.get_sorted_block_groups(0, 0)
+    expected = _build_ordered_block_groups(expected_block_groups[0:1])
+    assert expected == block_groups_filtered
+
+
+def test_build_sst_with_non_zero_min_key_matching_global_min_key(manifest_context1):
+    # Using a non-0 value since 0 evaluates to False
+    min_key = 1
+    max_key = 95
+
+    sst_row = SSTableRow(min_key, max_key, "row-with-non-zero-min-key", 0, 1)
+    t = BlockIntervalTree()
+    t.add_sst_table(SSTable([sst_row], min_key, max_key), manifest_context1)
+
+    block_groups_filtered = t.get_sorted_block_groups(min_key, min_key + 1)
+    expected = _build_ordered_block_groups(
+        [
+            BlockGroup(
+                min_key,
+                max_key,
+                {
+                    manifest_context1.schema: frozenset(
+                        [Block(sst_row, manifest_context1)]
+                    )
+                },
+            )
+        ]
+    )
+    assert expected == block_groups_filtered
+
+
+def test_build_sst_invalid_bounds(
+    sst1, sst2, schema1, schema2, sst_row_list, expected_block_groups
+):
+    t = BlockIntervalTree()
+
+    with pytest.raises(ValueError):
+        t.get_sorted_block_groups(10, 0)
+
+
+def _build_ordered_block_groups(block_groups: List[BlockGroup]) -> OrderedBlockGroups:
+    """
+    Helper method to build OrderedBlockGroups from a sorted list of block groups
+
+    """
+    ordered_groups = []
+    boundary_table = []
+    for i, bg in enumerate(block_groups):
+        boundary_table.append(bg.key_min)
+        is_last = i == len(block_groups) - 1
+        if is_last:
+            bg = BlockGroup(bg.key_min, bg.key_max, bg.field_group_to_blocks, True)
+            boundary_table.append(bg.key_max)
+        ordered_groups.append(bg)
+
+    return OrderedBlockGroups(
+        ordered_groups[0].key_min,
+        ordered_groups[-1].key_max,
+        ordered_groups,
+        boundary_table,
+    )