opteryx-catalog 0.4.4__py3-none-any.whl → 0.4.11__py3-none-any.whl

{opteryx_catalog-0.4.4.dist-info → opteryx_catalog-0.4.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: opteryx-catalog
-Version: 0.4.4
+Version: 0.4.11
 Summary: Opteryx Cloud Catalog
 Author-email: joocer <justin.joyce@joocer.com>
 License:                                  Apache License

opteryx_catalog-0.4.11.dist-info/RECORD

@@ -0,0 +1,25 @@
+opteryx_catalog/__init__.py,sha256=cqGY7bl6iMBIqY_x6VTc5fAFH23M3XQeJYrHPX6FglY,902
+opteryx_catalog/exceptions.py,sha256=ZEaXmrrn030V8pfy8YMaLwzBWFms9OgZG21zVRGKlxM,652
+opteryx_catalog/opteryx_catalog.py,sha256=sgcCiBbIv8mUZQbNb34JwEf6Wq0iByoEvpqkoD_F1bc,39111
+opteryx_catalog/catalog/__init__.py,sha256=yD7egf-dLd1z_CNXunz3ldLyLMMkSNbS3aKjGp3dKQY,119
+opteryx_catalog/catalog/compaction.py,sha256=HGkDnlVBv5GjRiZhdGubxCVxRLScL9N667a19U01g1I,19100
+opteryx_catalog/catalog/dataset.py,sha256=3Q_lLZ1Y0I1E_R47pMFgql81Y1dy955NKlsgk9edfJE,46796
+opteryx_catalog/catalog/manifest.py,sha256=xTV3u_i8s7jxulLvATyBoP9FHTdxOB8b0__SabqhH6g,17045
+opteryx_catalog/catalog/metadata.py,sha256=a4UFj5xUqjqtuLu2_mYQaBHRWtjjX3KU2Ufp63Uo2AM,2870
+opteryx_catalog/catalog/metastore.py,sha256=mS4qaaOMzcIu730Jm0K_Nq-4sNI8kIX3UscevYO5E08,1997
+opteryx_catalog/catalog/view.py,sha256=mUzfRGYqLRx_9BfZdGY5HNz6na9VMEPITrYKiI5m694,219
+opteryx_catalog/iops/__init__.py,sha256=_CxR-hg8XUD2cIFucb3aHyTFqwi41QmEDf9gXzXt3ZU,171
+opteryx_catalog/iops/base.py,sha256=1IW9qjDkQEMXvrA2J73VSBCdzkf2W5xVsWVnpNglL1U,1206
+opteryx_catalog/iops/fileio.py,sha256=cjBl9fN-vutvXskzZkwJjjbBcUlE0O1WrQe5Ryx7pIg,4315
+opteryx_catalog/iops/gcs.py,sha256=aB6hvSAQhbKTSyaLbAPgpXtSnvkI7fndXCRjaAZ1Dxo,8155
+opteryx_catalog-0.4.11.dist-info/licenses/LICENSE,sha256=mc5l20siqdcNQM54xALIWJhyaWsmQJ-NZt81UjgJejo,11351
+scripts/create_dataset.py,sha256=K8zmQo3xbwc_yz2BxNK0IKj-DkDt3pFf13ycI6rgTHo,7798
+scripts/read_dataset.py,sha256=hpBa8Qv1Oj6ffVIUmELGSri2eYHPpdqLnWFKgKpG-FM,9610
+tests/test_compaction.py,sha256=7MLnfbGi3j17ZON8Qi9oq4i1UWkW0JigX46BBFWecMk,7871
+tests/test_dataset_metadata.py,sha256=bMzX2HiUnzFTyU3VkFuW5xjmFEP8cJSYPt1XF6IS0Qk,1019
+tests/test_import.py,sha256=ZvoHW-rmcYqkW6TJKD_brgeePqHHbz2iTyRWKIBHGHk,137
+tests/test_pyproject.py,sha256=o3rS_GOems1oYQDH3UATfqc6XUwDTKZF2Q4cspU-NYc,206
+opteryx_catalog-0.4.11.dist-info/METADATA,sha256=BV0mk_GugipH7BAhKWIJtJq_55ML4kipL8RbF-Cm7t4,22384
+opteryx_catalog-0.4.11.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+opteryx_catalog-0.4.11.dist-info/top_level.txt,sha256=HWATr4Wgxbg3c1X3EcsJ6cnHoR6ZAdTe1LQ2VssIBUo,30
+opteryx_catalog-0.4.11.dist-info/RECORD,,

scripts/create_dataset.py

@@ -34,7 +34,7 @@ def create_parquet_only_tables(count: int = 2) -> list:
 
     for i in range(count):
         table_name = f"test_table_{i}_{int(time.time())}"
-        location = f"gs://{os.environ.get('GCS_BUCKET')}/{workspace}/{collection}/{table_name}"
+        # location = f"gs://{os.environ.get('GCS_BUCKET')}/{workspace}/{collection}/{table_name}"
 
         # Use fixed test author
         author = "me"

scripts/read_dataset.py

@@ -44,7 +44,7 @@ if not datasets:
 datasets = sorted(datasets)
 table = datasets[-1]
 
-table = "test_table_1_1767106177"
+table = "test_table_0_1767300842"
 
 
 # opteryx.register_store(

tests/test_compaction.py

@@ -0,0 +1,233 @@
+"""
+Test script for compaction functionality.
+
+This tests the DatasetCompactor class with both brute and performance strategies.
+"""
+
+from unittest.mock import Mock
+
+import pyarrow as pa
+
+from opteryx_catalog.catalog.compaction import DatasetCompactor
+from opteryx_catalog.catalog.metadata import DatasetMetadata, Snapshot
+
+
+def create_test_table(num_rows: int, value_range: tuple = (0, 100)) -> pa.Table:
+    """Create a simple test table with a timestamp column for sorting."""
+    import random
+
+    timestamps = sorted([random.randint(value_range[0], value_range[1]) for _ in range(num_rows)])
+    values = [f"value_{i}" for i in range(num_rows)]
+
+    return pa.table({"timestamp": timestamps, "value": values})
+
+
+def test_brute_compaction():
+    """Test brute force compaction strategy."""
+    print("Testing brute force compaction...")
+
+    # Create mock dataset
+    dataset = Mock()
+    dataset.metadata = DatasetMetadata(
+        dataset_identifier="test_dataset",
+        location="/tmp/test_data",
+    )
+    dataset.metadata.sort_orders = []  # No sort order for brute
+    dataset.metadata.snapshots = []
+    dataset.metadata.current_snapshot = None
+
+    # Create mock entries - small files that should be combined
+    mock_entries = [
+        {
+            "file_path": "/tmp/file1.parquet",
+            "file_size_in_bytes": 30 * 1024 * 1024,  # 30MB compressed
+            "uncompressed_size_in_bytes": 40 * 1024 * 1024,  # 40MB uncompressed
+            "record_count": 1000,
+        },
+        {
+            "file_path": "/tmp/file2.parquet",
+            "file_size_in_bytes": 35 * 1024 * 1024,  # 35MB compressed
+            "uncompressed_size_in_bytes": 50 * 1024 * 1024,  # 50MB uncompressed
+            "record_count": 1200,
+        },
+        {
+            "file_path": "/tmp/file3.parquet",
+            "file_size_in_bytes": 110 * 1024 * 1024,  # 110MB compressed (acceptable)
+            "uncompressed_size_in_bytes": 130 * 1024 * 1024,  # 130MB uncompressed
+            "record_count": 3000,
+        },
+    ]
+
+    # Create current snapshot with manifest
+    dataset.metadata.current_snapshot = Snapshot(
+        snapshot_id=1000,
+        timestamp_ms=1000,
+        manifest_list="/tmp/manifest.parquet",
+    )
+
+    # Mock IO and catalog
+    dataset.io = Mock()
+    dataset.catalog = Mock()
+
+    # Create compactor
+    compactor = DatasetCompactor(dataset, strategy="brute", author="test", agent="test-agent")
+
+    # Verify strategy selection
+    assert compactor.strategy == "brute", "Strategy should be brute"
+    assert compactor.decision == "user", "Decision should be user"
+
+    # Test selection logic directly
+    plan = compactor._select_brute_compaction(mock_entries)
+
+    assert plan is not None, "Should find files to compact"
+    assert plan["type"] == "combine", "Should plan to combine small files"
+    assert len(plan["files"]) == 2, "Should select 2 small files"
+
+    print("✓ Brute force compaction test passed")
+
+
+def test_performance_compaction():
+    """Test performance compaction strategy."""
+    print("Testing performance compaction...")
+
+    # Create mock dataset with sort order
+    dataset = Mock()
+    dataset.metadata = DatasetMetadata(
+        dataset_identifier="test_dataset",
+        location="/tmp/test_data",
+    )
+    dataset.metadata.sort_orders = [0]  # Sort by first column
+    dataset.metadata.schema = Mock()
+    dataset.metadata.schema.fields = [Mock(name="timestamp")]
+    dataset.metadata.snapshots = []
+    dataset.metadata.current_snapshot = None
+
+    # Create mock entries with overlapping ranges
+    mock_entries = [
+        {
+            "file_path": "/tmp/file1.parquet",
+            "file_size_in_bytes": 30 * 1024 * 1024,
+            "uncompressed_size_in_bytes": 40 * 1024 * 1024,
+            "record_count": 1000,
+            "lower_bounds": {"timestamp": 1},
+            "upper_bounds": {"timestamp": 100},
+        },
+        {
+            "file_path": "/tmp/file2.parquet",
+            "file_size_in_bytes": 35 * 1024 * 1024,
+            "uncompressed_size_in_bytes": 50 * 1024 * 1024,
+            "record_count": 1200,
+            "lower_bounds": {"timestamp": 50},  # Overlaps with file1
+            "upper_bounds": {"timestamp": 150},
+        },
+        {
+            "file_path": "/tmp/file3.parquet",
+            "file_size_in_bytes": 110 * 1024 * 1024,
+            "uncompressed_size_in_bytes": 130 * 1024 * 1024,
+            "record_count": 3000,
+            "lower_bounds": {"timestamp": 200},  # No overlap
+            "upper_bounds": {"timestamp": 300},
+        },
+    ]
+
+    dataset.metadata.current_snapshot = Snapshot(
+        snapshot_id=1000,
+        timestamp_ms=1000,
+        manifest_list="/tmp/manifest.parquet",
+    )
+
+    # Mock IO and catalog
+    dataset.io = Mock()
+    dataset.catalog = Mock()
+
+    # Create compactor (auto-detect should choose performance)
+    compactor = DatasetCompactor(dataset, strategy=None, author="test", agent="test-agent")
+
+    # Verify strategy selection
+    assert compactor.strategy == "performance", "Should auto-select performance strategy"
+    assert compactor.decision == "auto", "Decision should be auto"
+
+    # Test selection logic directly
+    plan = compactor._select_performance_compaction(mock_entries)
+
+    assert plan is not None, "Should find overlapping files"
+    assert plan["type"] == "combine-split", "Should plan to combine and split"
+    assert len(plan["files"]) == 2, "Should select 2 overlapping files"
+    assert plan["sort_column"] == "timestamp", "Should identify sort column"
+
+    print("✓ Performance compaction test passed")
+
+
+def test_large_file_splitting():
+    """Test that large files are identified for splitting."""
+    print("Testing large file splitting...")
+
+    dataset = Mock()
+    dataset.metadata = DatasetMetadata(
+        dataset_identifier="test_dataset",
+        location="/tmp/test_data",
+    )
+    dataset.metadata.sort_orders = []
+
+    # Create entry for a large file
+    mock_entries = [
+        {
+            "file_path": "/tmp/large_file.parquet",
+            "file_size_in_bytes": 180 * 1024 * 1024,
+            "uncompressed_size_in_bytes": 200 * 1024 * 1024,  # 200MB > 196MB threshold
+            "record_count": 5000,
+        }
+    ]
+
+    compactor = DatasetCompactor(dataset, strategy="brute")
+    plan = compactor._select_brute_compaction(mock_entries)
+
+    assert plan is not None, "Should identify large file"
+    assert plan["type"] == "split", "Should plan to split"
+    assert plan["reason"] == "file-too-large", "Reason should be file too large"
+
+    print("✓ Large file splitting test passed")
+
+
+def test_no_compaction_needed():
+    """Test when no compaction is needed."""
+    print("Testing no compaction scenario...")
+
+    dataset = Mock()
+    dataset.metadata = DatasetMetadata(
+        dataset_identifier="test_dataset",
+        location="/tmp/test_data",
+    )
+    dataset.metadata.sort_orders = []
+
+    # All files are in acceptable range
+    mock_entries = [
+        {
+            "file_path": "/tmp/file1.parquet",
+            "file_size_in_bytes": 100 * 1024 * 1024,
+            "uncompressed_size_in_bytes": 110 * 1024 * 1024,
+            "record_count": 2000,
+        },
+        {
+            "file_path": "/tmp/file2.parquet",
+            "file_size_in_bytes": 120 * 1024 * 1024,
+            "uncompressed_size_in_bytes": 135 * 1024 * 1024,
+            "record_count": 2500,
+        },
+    ]
+
+    compactor = DatasetCompactor(dataset, strategy="brute")
+    plan = compactor._select_brute_compaction(mock_entries)
+
+    assert plan is None, "Should not find anything to compact"
+
+    print("✓ No compaction test passed")
+
+
+if __name__ == "__main__":
+    print("Running compaction tests...\n")
+    test_brute_compaction()
+    test_performance_compaction()
+    test_large_file_splitting()
+    test_no_compaction_needed()
+    print("\n✅ All tests passed!")

tests/test_dataset_metadata.py

@@ -13,3 +13,17 @@ def test_dataset_metadata_and_simpledataset():
     assert ds.metadata.dataset_identifier == "tests_temp.test"
     assert ds.snapshot() is None
     assert list(ds.snapshots()) == []
+
+
+def test_sequence_number_requires_history():
+    """Test that _next_sequence_number works with empty snapshots."""
+    meta = DatasetMetadata(
+        dataset_identifier="tests_temp.test",
+        location="gs://bucket/ws/tests_temp/test",
+        schema=None,
+        properties={},
+    )
+    ds = SimpleDataset(identifier="tests_temp.test", _metadata=meta)
+
+    # Should return 1 when no snapshots are loaded (first snapshot)
+    assert ds._next_sequence_number() == 1

opteryx_catalog-0.4.4.dist-info/RECORD

@@ -1,23 +0,0 @@
-opteryx_catalog/__init__.py,sha256=0fxsE-hZ6hcw0Fs4LEIys1CQwCL4W7iLSpp0M0v1f08,900
-opteryx_catalog/exceptions.py,sha256=NKAwocRPYo1KbzFce56Dn8rQH0Jc8mfxES5qPDp-OO4,650
-opteryx_catalog/opteryx_catalog.py,sha256=roKcwY2N4AoyujH-pIZo9L_EKy5gyUSGCYIaim9wqWU,38530
-opteryx_catalog/catalog/__init__.py,sha256=0I67nvttc_sO3ntZp8rk6agMl1LXBd67j6ZsPGonKtM,58
-opteryx_catalog/catalog/dataset.py,sha256=NWG2z4PSmLXr9Mwweq_bGt1fCczHJVk19aTZM1c7ucg,48738
-opteryx_catalog/catalog/manifest.py,sha256=psnigON41ZV1hQ1CshGB_rBqpbUPG-JJeOtjQwxokaE,552
-opteryx_catalog/catalog/metadata.py,sha256=i6AS-4M1OvzAKJ0XQx_gy5bqKKv98W2vWsDi8JXTIso,2869
-opteryx_catalog/catalog/metastore.py,sha256=T34_ki5FXoREprCDoO1uBXcgH7yY9YslhTirbPAkPoM,1993
-opteryx_catalog/catalog/view.py,sha256=mUzfRGYqLRx_9BfZdGY5HNz6na9VMEPITrYKiI5m694,219
-opteryx_catalog/iops/__init__.py,sha256=_CxR-hg8XUD2cIFucb3aHyTFqwi41QmEDf9gXzXt3ZU,171
-opteryx_catalog/iops/base.py,sha256=1IW9qjDkQEMXvrA2J73VSBCdzkf2W5xVsWVnpNglL1U,1206
-opteryx_catalog/iops/fileio.py,sha256=cjBl9fN-vutvXskzZkwJjjbBcUlE0O1WrQe5Ryx7pIg,4315
-opteryx_catalog/iops/gcs.py,sha256=SyXjJptrtgTuBfAJLnzl4efLK4TKAGhllGKxJKDhx7Q,7275
-opteryx_catalog-0.4.4.dist-info/licenses/LICENSE,sha256=mc5l20siqdcNQM54xALIWJhyaWsmQJ-NZt81UjgJejo,11351
-scripts/create_dataset.py,sha256=8C4I6bKk2ZiYypO9nM8OTKlVTqpxW3JgLifgklsmAAo,7796
-scripts/read_dataset.py,sha256=HwBy-kgTtcNq_fjkDS2WrM5OTg5O7IcOKQf7RxfxUJA,9610
-tests/test_dataset_metadata.py,sha256=KE7jqam-IoJYXklmGaxuO47nvdoiNn_RW4VuUtkilIY,546
-tests/test_import.py,sha256=ZvoHW-rmcYqkW6TJKD_brgeePqHHbz2iTyRWKIBHGHk,137
-tests/test_pyproject.py,sha256=o3rS_GOems1oYQDH3UATfqc6XUwDTKZF2Q4cspU-NYc,206
-opteryx_catalog-0.4.4.dist-info/METADATA,sha256=ZUqsh7gmOoNAlegsAimP30u6VVz57zwzAcGuXpy2rmM,22383
-opteryx_catalog-0.4.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-opteryx_catalog-0.4.4.dist-info/top_level.txt,sha256=HWATr4Wgxbg3c1X3EcsJ6cnHoR6ZAdTe1LQ2VssIBUo,30
-opteryx_catalog-0.4.4.dist-info/RECORD,,