opteryx-catalog 0.4.13__py3-none-any.whl

tests/test_compaction.py

@@ -0,0 +1,233 @@
+"""
+Test script for compaction functionality.
+
+This tests the DatasetCompactor class with both brute and performance strategies.
+"""
+
+from unittest.mock import Mock
+
+import pyarrow as pa
+
+from opteryx_catalog.catalog.compaction import DatasetCompactor
+from opteryx_catalog.catalog.metadata import DatasetMetadata, Snapshot
+
+
+def create_test_table(num_rows: int, value_range: tuple = (0, 100)) -> pa.Table:
+    """Create a simple test table with a timestamp column for sorting."""
+    import random
+
+    timestamps = sorted([random.randint(value_range[0], value_range[1]) for _ in range(num_rows)])
+    values = [f"value_{i}" for i in range(num_rows)]
+
+    return pa.table({"timestamp": timestamps, "value": values})
+
+
+def test_brute_compaction():
+    """Test brute force compaction strategy."""
+    print("Testing brute force compaction...")
+
+    # Create mock dataset
+    dataset = Mock()
+    dataset.metadata = DatasetMetadata(
+        dataset_identifier="test_dataset",
+        location="/tmp/test_data",
+    )
+    dataset.metadata.sort_orders = []  # No sort order for brute
+    dataset.metadata.snapshots = []
+    dataset.metadata.current_snapshot = None
+
+    # Create mock entries - small files that should be combined
+    mock_entries = [
+        {
+            "file_path": "/tmp/file1.parquet",
+            "file_size_in_bytes": 30 * 1024 * 1024,  # 30MB compressed
+            "uncompressed_size_in_bytes": 40 * 1024 * 1024,  # 40MB uncompressed
+            "record_count": 1000,
+        },
+        {
+            "file_path": "/tmp/file2.parquet",
+            "file_size_in_bytes": 35 * 1024 * 1024,  # 35MB compressed
+            "uncompressed_size_in_bytes": 50 * 1024 * 1024,  # 50MB uncompressed
+            "record_count": 1200,
+        },
+        {
+            "file_path": "/tmp/file3.parquet",
+            "file_size_in_bytes": 110 * 1024 * 1024,  # 110MB compressed (acceptable)
+            "uncompressed_size_in_bytes": 130 * 1024 * 1024,  # 130MB uncompressed
+            "record_count": 3000,
+        },
+    ]
+
+    # Create current snapshot with manifest
+    dataset.metadata.current_snapshot = Snapshot(
+        snapshot_id=1000,
+        timestamp_ms=1000,
+        manifest_list="/tmp/manifest.parquet",
+    )
+
+    # Mock IO and catalog
+    dataset.io = Mock()
+    dataset.catalog = Mock()
+
+    # Create compactor
+    compactor = DatasetCompactor(dataset, strategy="brute", author="test", agent="test-agent")
+
+    # Verify strategy selection
+    assert compactor.strategy == "brute", "Strategy should be brute"
+    assert compactor.decision == "user", "Decision should be user"
+
+    # Test selection logic directly
+    plan = compactor._select_brute_compaction(mock_entries)
+
+    assert plan is not None, "Should find files to compact"
+    assert plan["type"] == "combine", "Should plan to combine small files"
+    assert len(plan["files"]) == 2, "Should select 2 small files"
+
+    print("✓ Brute force compaction test passed")
+
+
+def test_performance_compaction():
+    """Test performance compaction strategy."""
+    print("Testing performance compaction...")
+
+    # Create mock dataset with sort order
+    dataset = Mock()
+    dataset.metadata = DatasetMetadata(
+        dataset_identifier="test_dataset",
+        location="/tmp/test_data",
+    )
+    dataset.metadata.sort_orders = [0]  # Sort by first column
+    dataset.metadata.schema = Mock()
+    dataset.metadata.schema.fields = [Mock(name="timestamp")]
+    dataset.metadata.snapshots = []
+    dataset.metadata.current_snapshot = None
+
+    # Create mock entries with overlapping ranges
+    mock_entries = [
+        {
+            "file_path": "/tmp/file1.parquet",
+            "file_size_in_bytes": 30 * 1024 * 1024,
+            "uncompressed_size_in_bytes": 40 * 1024 * 1024,
+            "record_count": 1000,
+            "lower_bounds": {"timestamp": 1},
+            "upper_bounds": {"timestamp": 100},
+        },
+        {
+            "file_path": "/tmp/file2.parquet",
+            "file_size_in_bytes": 35 * 1024 * 1024,
+            "uncompressed_size_in_bytes": 50 * 1024 * 1024,
+            "record_count": 1200,
+            "lower_bounds": {"timestamp": 50},  # Overlaps with file1
+            "upper_bounds": {"timestamp": 150},
+        },
+        {
+            "file_path": "/tmp/file3.parquet",
+            "file_size_in_bytes": 110 * 1024 * 1024,
+            "uncompressed_size_in_bytes": 130 * 1024 * 1024,
+            "record_count": 3000,
+            "lower_bounds": {"timestamp": 200},  # No overlap
+            "upper_bounds": {"timestamp": 300},
+        },
+    ]
+
+    dataset.metadata.current_snapshot = Snapshot(
+        snapshot_id=1000,
+        timestamp_ms=1000,
+        manifest_list="/tmp/manifest.parquet",
+    )
+
+    # Mock IO and catalog
+    dataset.io = Mock()
+    dataset.catalog = Mock()
+
+    # Create compactor (auto-detect should choose performance)
+    compactor = DatasetCompactor(dataset, strategy=None, author="test", agent="test-agent")
+
+    # Verify strategy selection
+    assert compactor.strategy == "performance", "Should auto-select performance strategy"
+    assert compactor.decision == "auto", "Decision should be auto"
+
+    # Test selection logic directly
+    plan = compactor._select_performance_compaction(mock_entries)
+
+    assert plan is not None, "Should find overlapping files"
+    assert plan["type"] == "combine-split", "Should plan to combine and split"
+    assert len(plan["files"]) == 2, "Should select 2 overlapping files"
+    assert plan["sort_column"] == "timestamp", "Should identify sort column"
+
+    print("✓ Performance compaction test passed")
+
+
+def test_large_file_splitting():
+    """Test that large files are identified for splitting."""
+    print("Testing large file splitting...")
+
+    dataset = Mock()
+    dataset.metadata = DatasetMetadata(
+        dataset_identifier="test_dataset",
+        location="/tmp/test_data",
+    )
+    dataset.metadata.sort_orders = []
+
+    # Create entry for a large file
+    mock_entries = [
+        {
+            "file_path": "/tmp/large_file.parquet",
+            "file_size_in_bytes": 180 * 1024 * 1024,
+            "uncompressed_size_in_bytes": 200 * 1024 * 1024,  # 200MB > 196MB threshold
+            "record_count": 5000,
+        }
+    ]
+
+    compactor = DatasetCompactor(dataset, strategy="brute")
+    plan = compactor._select_brute_compaction(mock_entries)
+
+    assert plan is not None, "Should identify large file"
+    assert plan["type"] == "split", "Should plan to split"
+    assert plan["reason"] == "file-too-large", "Reason should be file too large"
+
+    print("✓ Large file splitting test passed")
+
+
+def test_no_compaction_needed():
+    """Test when no compaction is needed."""
+    print("Testing no compaction scenario...")
+
+    dataset = Mock()
+    dataset.metadata = DatasetMetadata(
+        dataset_identifier="test_dataset",
+        location="/tmp/test_data",
+    )
+    dataset.metadata.sort_orders = []
+
+    # All files are in acceptable range
+    mock_entries = [
+        {
+            "file_path": "/tmp/file1.parquet",
+            "file_size_in_bytes": 100 * 1024 * 1024,
+            "uncompressed_size_in_bytes": 110 * 1024 * 1024,
+            "record_count": 2000,
+        },
+        {
+            "file_path": "/tmp/file2.parquet",
+            "file_size_in_bytes": 120 * 1024 * 1024,
+            "uncompressed_size_in_bytes": 135 * 1024 * 1024,
+            "record_count": 2500,
+        },
+    ]
+
+    compactor = DatasetCompactor(dataset, strategy="brute")
+    plan = compactor._select_brute_compaction(mock_entries)
+
+    assert plan is None, "Should not find anything to compact"
+
+    print("✓ No compaction test passed")
+
+
+if __name__ == "__main__":
+    print("Running compaction tests...\n")
+    test_brute_compaction()
+    test_performance_compaction()
+    test_large_file_splitting()
+    test_no_compaction_needed()
+    print("\n✅ All tests passed!")

tests/test_dataset_metadata.py

@@ -0,0 +1,29 @@
+from opteryx_catalog.catalog.metadata import DatasetMetadata
+from opteryx_catalog.catalog.dataset import SimpleDataset
+
+
+def test_dataset_metadata_and_simpledataset():
+    meta = DatasetMetadata(
+        dataset_identifier="tests_temp.test",
+        location="gs://bucket/ws/tests_temp/test",
+        schema=None,
+        properties={},
+    )
+    ds = SimpleDataset(identifier="tests_temp.test", _metadata=meta)
+    assert ds.metadata.dataset_identifier == "tests_temp.test"
+    assert ds.snapshot() is None
+    assert list(ds.snapshots()) == []
+
+
+def test_sequence_number_requires_history():
+    """Test that _next_sequence_number works with empty snapshots."""
+    meta = DatasetMetadata(
+        dataset_identifier="tests_temp.test",
+        location="gs://bucket/ws/tests_temp/test",
+        schema=None,
+        properties={},
+    )
+    ds = SimpleDataset(identifier="tests_temp.test", _metadata=meta)
+
+    # Should return 1 when no snapshots are loaded (first snapshot)
+    assert ds._next_sequence_number() == 1

tests/test_import.py

@@ -0,0 +1,5 @@
+def test_import_opteryx_catalog():
+    import importlib
+
+    mod = importlib.import_module("opteryx_catalog")
+    assert mod is not None

tests/test_pyproject.py

@@ -0,0 +1,8 @@
+import pathlib
+import tomllib
+
+
+def test_pyproject_name():
+    p = pathlib.Path("pyproject.toml")
+    data = tomllib.loads(p.read_text())
+    assert data.get("project", {}).get("name") == "opteryx-catalog"

tests/test_webhooks.py

@@ -0,0 +1,177 @@
+"""Tests for the webhook system."""
+
+import os
+from unittest.mock import MagicMock
+from unittest.mock import patch
+
+import pytest
+
+
+def test_webhook_manager_disabled_without_domain():
+    """Test that webhook manager is disabled when no domain is configured."""
+    from opteryx_catalog.webhooks import WebhookManager
+
+    # Clear any existing env vars
+    os.environ.pop("OPTERYX_WEBHOOK_DOMAIN", None)
+    os.environ.pop("OPTERYX_WEBHOOK_QUEUE", None)
+
+    manager = WebhookManager()
+    assert not manager.enabled
+
+    # Should return False without making any HTTP calls
+    result = manager.send(
+        action="create",
+        workspace="test",
+        collection="test",
+        resource_type="dataset",
+        resource_name="test",
+    )
+    assert result is False
+
+
+def test_webhook_manager_direct_http():
+    """Test that webhooks are sent via direct HTTP when queue is not configured."""
+    from opteryx_catalog.webhooks import WebhookManager
+
+    with patch("opteryx_catalog.webhooks.requests.post") as mock_post:
+        mock_response = MagicMock()
+        mock_response.status_code = 200
+        mock_post.return_value = mock_response
+
+        manager = WebhookManager(domain="router.example.com", queue_path=None)
+        assert manager.enabled
+        assert manager._tasks_client is None
+
+        result = manager.send(
+            action="create",
+            workspace="test-workspace",
+            collection="test-collection",
+            resource_type="dataset",
+            resource_name="test-dataset",
+            payload={"location": "gs://bucket/path"},
+        )
+
+        assert result is True
+        mock_post.assert_called_once()
+
+        # Verify the call arguments
+        call_args = mock_post.call_args
+        assert call_args.args[0] == "https://router.example.com/event"
+        assert call_args.kwargs["json"]["event"]["action"] == "create"
+        assert call_args.kwargs["json"]["event"]["resource_type"] == "dataset"
+        assert call_args.kwargs["json"]["event"]["resource_name"] == "test-dataset"
+        assert call_args.kwargs["json"]["data"]["location"] == "gs://bucket/path"
+
+
+def test_webhook_manager_payload_building():
+    """Test that webhook payloads are built correctly."""
+    from opteryx_catalog.webhooks import WebhookManager
+
+    manager = WebhookManager(domain="hook.example.com")
+
+    payload = manager._build_payload(
+        action="update",
+        workspace="ws",
+        collection="col",
+        resource_type="dataset",
+        resource_name="ds",
+        additional={"description": "New description"},
+    )
+
+    assert payload["event"]["action"] == "update"
+    assert payload["event"]["workspace"] == "ws"
+    assert payload["event"]["collection"] == "col"
+    assert payload["event"]["resource_type"] == "dataset"
+    assert payload["event"]["resource_name"] == "ds"
+    assert "timestamp" in payload["event"]
+    assert payload["data"]["description"] == "New description"
+
+
+def test_webhook_http_failure_returns_false():
+    """Test that HTTP failures return False without raising exceptions."""
+    from opteryx_catalog.webhooks import WebhookManager
+
+    with patch("opteryx_catalog.webhooks.requests.post") as mock_post:
+        # Simulate HTTP error
+        mock_post.side_effect = Exception("Connection failed")
+
+        manager = WebhookManager(domain="router.example.com")
+        result = manager.send(
+            action="create",
+            workspace="test",
+            collection="test",
+            resource_type="dataset",
+            resource_name="test",
+        )
+
+        assert result is False
+
+
+def test_send_webhook_convenience_function():
+    """Test the convenience send_webhook function."""
+    from opteryx_catalog.webhooks import send_webhook
+
+    with patch("opteryx_catalog.webhooks.requests.post") as mock_post:
+        mock_response = MagicMock()
+        mock_response.status_code = 200
+        mock_post.return_value = mock_response
+
+        os.environ["OPTERYX_WEBHOOK_DOMAIN"] = "router.example.com"
+        os.environ.pop("OPTERYX_WEBHOOK_QUEUE", None)
+
+        # Reset the global manager to pick up new env vars
+        import opteryx_catalog.webhooks as webhook_module
+
+        webhook_module._webhook_manager = None
+
+        result = send_webhook(
+            action="create",
+            workspace="test",
+            collection="test",
+            resource_type="dataset",
+            resource_name="test",
+            payload={"snapshot_id": 123},
+        )
+
+        assert result is True
+        mock_post.assert_called_once()
+
+        # Clean up
+        os.environ.pop("OPTERYX_WEBHOOK_DOMAIN", None)
+
+
+def test_event_payload_builders():
+    """Test the event payload builder functions."""
+    from opteryx_catalog.webhooks.events import dataset_commit_payload
+    from opteryx_catalog.webhooks.events import dataset_created_payload
+    from opteryx_catalog.webhooks.events import view_created_payload
+    from opteryx_catalog.webhooks.events import view_executed_payload
+
+    # Test dataset created
+    payload = dataset_created_payload(
+        schema=None, location="gs://bucket/path", properties={"key": "value"}
+    )
+    assert payload["location"] == "gs://bucket/path"
+    assert payload["properties"]["key"] == "value"
+
+    # Test dataset commit
+    payload = dataset_commit_payload(
+        snapshot_id=123, sequence_number=5, record_count=1000, file_count=2
+    )
+    assert payload["snapshot_id"] == 123
+    assert payload["sequence_number"] == 5
+    assert payload["record_count"] == 1000
+    assert payload["file_count"] == 2
+
+    # Test view created
+    payload = view_created_payload(definition="SELECT * FROM table", properties={})
+    assert payload["definition"] == "SELECT * FROM table"
+
+    # Test view executed
+    payload = view_executed_payload(execution_time_ms=1500, row_count=100)
+    assert payload["execution_time_ms"] == 1500
+    assert payload["row_count"] == 100
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])