earthcatalog 0.2.0__py3-none-any.whl

earthcatalog/tests/test_storage_backends.py

@@ -0,0 +1,236 @@
+# test_storage_backends.py
+"""Tests for storage backend enhancements.
+
+This module tests the new storage backend methods:
+- get_etag: Get file checksum/ETag for change detection
+- rmtree: Recursive directory deletion
+- list_files: List files in directory with pattern matching
+"""
+
+import tempfile
+from pathlib import Path
+
+import pytest
+
+from earthcatalog.storage_backends import LocalStorage, get_storage_backend
+
+
+class TestLocalStorageEnhancements:
+    """Test LocalStorage new methods."""
+
+    @pytest.fixture
+    def temp_dir(self):
+        """Create a temporary directory for testing."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            yield tmpdir
+
+    @pytest.fixture
+    def local_storage(self, temp_dir):
+        """Create a LocalStorage instance."""
+        return LocalStorage(temp_dir)
+
+    def test_get_etag_returns_hash_for_existing_file(self, local_storage, temp_dir):
+        """get_etag should return a hash string for existing files."""
+        # Create a test file
+        test_file = Path(temp_dir) / "test.txt"
+        test_file.write_text("hello world")
+
+        etag = local_storage.get_etag(str(test_file))
+
+        assert etag is not None
+        assert isinstance(etag, str)
+        assert len(etag) > 0
+
+    def test_get_etag_returns_none_for_missing_file(self, local_storage, temp_dir):
+        """get_etag should return None for non-existent files."""
+        missing_file = Path(temp_dir) / "nonexistent.txt"
+
+        etag = local_storage.get_etag(str(missing_file))
+
+        assert etag is None
+
+    def test_get_etag_different_content_different_hash(self, local_storage, temp_dir):
+        """Different file contents should produce different ETags."""
+        file1 = Path(temp_dir) / "file1.txt"
+        file2 = Path(temp_dir) / "file2.txt"
+        file1.write_text("content one")
+        file2.write_text("content two")
+
+        etag1 = local_storage.get_etag(str(file1))
+        etag2 = local_storage.get_etag(str(file2))
+
+        assert etag1 != etag2
+
+    def test_get_etag_same_content_same_hash(self, local_storage, temp_dir):
+        """Same file contents should produce same ETags."""
+        file1 = Path(temp_dir) / "file1.txt"
+        file2 = Path(temp_dir) / "file2.txt"
+        file1.write_text("identical content")
+        file2.write_text("identical content")
+
+        etag1 = local_storage.get_etag(str(file1))
+        etag2 = local_storage.get_etag(str(file2))
+
+        assert etag1 == etag2
+
+    def test_rmtree_removes_directory_recursively(self, local_storage, temp_dir):
+        """rmtree should remove directory and all contents."""
+        # Create nested directory structure
+        nested_dir = Path(temp_dir) / "parent" / "child"
+        nested_dir.mkdir(parents=True)
+        (nested_dir / "file.txt").write_text("content")
+        (Path(temp_dir) / "parent" / "sibling.txt").write_text("other")
+
+        parent_path = str(Path(temp_dir) / "parent")
+        local_storage.rmtree(parent_path)
+
+        assert not Path(parent_path).exists()
+
+    def test_rmtree_handles_missing_directory(self, local_storage, temp_dir):
+        """rmtree should not raise error for non-existent directory."""
+        missing_dir = Path(temp_dir) / "does_not_exist"
+
+        # Should not raise
+        local_storage.rmtree(str(missing_dir))
+
+    def test_rmtree_handles_empty_directory(self, local_storage, temp_dir):
+        """rmtree should handle empty directories."""
+        empty_dir = Path(temp_dir) / "empty"
+        empty_dir.mkdir()
+
+        local_storage.rmtree(str(empty_dir))
+
+        assert not empty_dir.exists()
+
+    def test_list_files_returns_matching_files(self, local_storage, temp_dir):
+        """list_files should return files matching the pattern."""
+        # Create test files
+        (Path(temp_dir) / "data1.parquet").write_text("data")
+        (Path(temp_dir) / "data2.parquet").write_text("data")
+        (Path(temp_dir) / "readme.txt").write_text("text")
+
+        files = local_storage.list_files(temp_dir, "*.parquet")
+
+        assert len(files) == 2
+        assert all(".parquet" in f for f in files)
+
+    def test_list_files_returns_empty_for_no_matches(self, local_storage, temp_dir):
+        """list_files should return empty list when no files match."""
+        (Path(temp_dir) / "data.txt").write_text("data")
+
+        files = local_storage.list_files(temp_dir, "*.parquet")
+
+        assert files == []
+
+    def test_list_files_handles_missing_directory(self, local_storage, temp_dir):
+        """list_files should return empty list for non-existent directory."""
+        missing_dir = Path(temp_dir) / "nonexistent"
+
+        files = local_storage.list_files(str(missing_dir), "*")
+
+        assert files == []
+
+    def test_list_files_recursive_with_glob_pattern(self, local_storage, temp_dir):
+        """list_files with ** should match files recursively."""
+        # Create nested structure
+        subdir = Path(temp_dir) / "subdir"
+        subdir.mkdir()
+        (Path(temp_dir) / "root.parquet").write_text("data")
+        (subdir / "nested.parquet").write_text("data")
+
+        files = local_storage.list_files(temp_dir, "**/*.parquet")
+
+        # Should find both files
+        assert len(files) >= 1  # At least the nested one with **
+
+    def test_list_files_default_pattern_matches_all(self, local_storage, temp_dir):
+        """list_files with default pattern should match all files."""
+        (Path(temp_dir) / "file1.txt").write_text("data")
+        (Path(temp_dir) / "file2.json").write_text("data")
+
+        files = local_storage.list_files(temp_dir)
+
+        assert len(files) == 2
+
+    def test_list_dirs_returns_subdirectories(self, local_storage, temp_dir):
+        """list_dirs should return immediate subdirectories."""
+        # Create subdirectories
+        (Path(temp_dir) / "subdir1").mkdir()
+        (Path(temp_dir) / "subdir2").mkdir()
+        # Create a file (should not be included)
+        (Path(temp_dir) / "file.txt").write_text("data")
+
+        dirs = local_storage.list_dirs(temp_dir)
+
+        assert len(dirs) == 2
+        dir_names = [Path(d).name for d in dirs]
+        assert "subdir1" in dir_names
+        assert "subdir2" in dir_names
+
+    def test_list_dirs_returns_empty_for_no_subdirs(self, local_storage, temp_dir):
+        """list_dirs should return empty list when no subdirectories exist."""
+        # Create only files
+        (Path(temp_dir) / "file1.txt").write_text("data")
+        (Path(temp_dir) / "file2.txt").write_text("data")
+
+        dirs = local_storage.list_dirs(temp_dir)
+
+        assert dirs == []
+
+    def test_list_dirs_handles_missing_directory(self, local_storage, temp_dir):
+        """list_dirs should return empty list for non-existent directory."""
+        missing_dir = Path(temp_dir) / "nonexistent"
+
+        dirs = local_storage.list_dirs(str(missing_dir))
+
+        assert dirs == []
+
+    def test_list_dirs_does_not_recurse(self, local_storage, temp_dir):
+        """list_dirs should only return immediate children, not nested."""
+        # Create nested structure
+        (Path(temp_dir) / "parent").mkdir()
+        (Path(temp_dir) / "parent" / "child").mkdir()
+
+        dirs = local_storage.list_dirs(temp_dir)
+
+        assert len(dirs) == 1
+        assert Path(dirs[0]).name == "parent"
+
+
+class TestGetStorageBackend:
+    """Test the storage backend factory function."""
+
+    def test_local_path_returns_local_storage(self):
+        """Local paths should return LocalStorage."""
+        storage = get_storage_backend("/tmp/test")
+        assert isinstance(storage, LocalStorage)
+
+    def test_relative_path_returns_local_storage(self):
+        """Relative paths should return LocalStorage."""
+        storage = get_storage_backend("./catalog")
+        assert isinstance(storage, LocalStorage)
+
+
+# S3 tests would require moto or similar mocking
+class TestS3StorageEnhancements:
+    """Test S3Storage new methods (requires mocking or live S3)."""
+
+    @pytest.mark.skip(reason="Requires S3 credentials or mocking")
+    def test_get_etag_returns_s3_etag(self):
+        """S3 get_etag should return the S3 ETag header value."""
+        pass
+
+    @pytest.mark.skip(reason="Requires S3 credentials or mocking")
+    def test_rmtree_deletes_s3_prefix(self):
+        """S3 rmtree should delete all objects with the given prefix."""
+        pass
+
+    @pytest.mark.skip(reason="Requires S3 credentials or mocking")
+    def test_list_files_lists_s3_objects(self):
+        """S3 list_files should list objects matching pattern."""
+        pass
+
+    @pytest.mark.skip(reason="Requires S3 credentials or mocking")
+    def test_list_dirs_lists_s3_prefixes(self):
+        """S3 list_dirs should list directory-like prefixes."""
+        pass

earthcatalog/tests/test_validation.py

@@ -0,0 +1,435 @@
+"""Tests for STAC GeoParquet validation module."""
+
+import geopandas as gpd
+import pytest
+from shapely.geometry import Point
+
+from earthcatalog.validation import (
+    CatalogValidationResult,
+    ValidationIssue,
+    ValidationResult,
+    get_geoparquet_metadata,
+    validate_catalog,
+    validate_geoparquet_file,
+    validate_stac_item,
+    validate_stac_items_batch,
+)
+
+
+class TestValidationResult:
+    """Tests for ValidationResult data class."""
+
+    def test_validation_result_defaults(self):
+        """Test that ValidationResult has correct defaults."""
+        result = ValidationResult(is_valid=True)
+        assert result.is_valid is True
+        assert result.issues == []
+        assert result.metadata == {}
+        assert result.warnings == []
+        assert result.errors == []
+
+    def test_add_warning(self):
+        """Test adding warnings to result."""
+        result = ValidationResult(is_valid=True)
+        result.add_warning("TEST_WARNING", "Test message", key="value")
+
+        assert len(result.warnings) == 1
+        assert result.warnings[0].code == "TEST_WARNING"
+        assert result.warnings[0].message == "Test message"
+        assert result.warnings[0].context == {"key": "value"}
+        assert result.is_valid is True  # Warnings don't invalidate
+
+    def test_add_error(self):
+        """Test adding errors to result."""
+        result = ValidationResult(is_valid=True)
+        result.add_error("TEST_ERROR", "Test error message")
+
+        assert len(result.errors) == 1
+        assert result.errors[0].code == "TEST_ERROR"
+        assert result.is_valid is False  # Errors invalidate
+
+    def test_merge_results(self):
+        """Test merging two validation results."""
+        result1 = ValidationResult(is_valid=True)
+        result1.add_warning("WARN1", "Warning 1")
+
+        result2 = ValidationResult(is_valid=True)
+        result2.add_error("ERR1", "Error 1")
+
+        merged = result1.merge(result2)
+
+        assert len(merged.issues) == 2
+        assert merged.is_valid is False
+        assert len(merged.warnings) == 1
+        assert len(merged.errors) == 1
+
+    def test_summary(self):
+        """Test summary generation."""
+        result = ValidationResult(is_valid=True)
+        result.add_warning("WARN", "A warning")
+        result.add_error("ERR", "An error")
+
+        summary = result.summary()
+        assert "Valid: False" in summary
+        assert "Warnings: 1" in summary
+        assert "Errors: 1" in summary
+
+
+class TestValidationIssue:
+    """Tests for ValidationIssue data class."""
+
+    def test_str_representation(self):
+        """Test string representation of issue."""
+        issue = ValidationIssue(
+            level="warning",
+            code="TEST_CODE",
+            message="Test message",
+            context={"key": "value"},
+        )
+        assert "[WARNING] TEST_CODE: Test message" == str(issue)
+
+
+class TestSTACItemValidation:
+    """Tests for STAC item validation."""
+
+    @pytest.fixture
+    def valid_stac_item(self):
+        """Create a valid STAC item for testing."""
+        return {
+            "type": "Feature",
+            "stac_version": "1.0.0",
+            "id": "test-item-001",
+            "geometry": {
+                "type": "Polygon",
+                "coordinates": [[[-10, -10], [10, -10], [10, 10], [-10, 10], [-10, -10]]],
+            },
+            "bbox": [-10, -10, 10, 10],
+            "properties": {
+                "datetime": "2024-01-15T10:00:00Z",
+            },
+            "links": [],
+            "assets": {},
+        }
+
+    def test_validate_valid_item(self, valid_stac_item):
+        """Test validation of a valid STAC item."""
+        result, corrected = validate_stac_item(valid_stac_item)
+
+        assert result.is_valid is True
+        assert len(result.errors) == 0
+        assert corrected is not None
+
+    def test_validate_missing_required_fields(self):
+        """Test validation catches missing required fields."""
+        item = {
+            "properties": {"datetime": "2024-01-15T10:00:00Z"},
+        }
+        result, corrected = validate_stac_item(item)
+
+        # Should have warnings for missing id, type, geometry
+        assert len(result.warnings) >= 2
+        warning_codes = [w.code for w in result.warnings]
+        assert "MISSING_FIELD" in warning_codes or "NULL_GEOMETRY" in warning_codes
+
+    def test_validate_invalid_type(self):
+        """Test validation warns on wrong type."""
+        item = {
+            "type": "Collection",  # Wrong type
+            "id": "test",
+            "geometry": {"type": "Point", "coordinates": [0, 0]},
+            "properties": {"datetime": "2024-01-15T10:00:00Z"},
+        }
+        result, _ = validate_stac_item(item)
+
+        warning_codes = [w.code for w in result.warnings]
+        assert "INVALID_TYPE" in warning_codes
+
+    def test_validate_null_geometry(self):
+        """Test validation handles null geometry."""
+        item = {
+            "type": "Feature",
+            "id": "test",
+            "geometry": None,
+            "properties": {"datetime": "2024-01-15T10:00:00Z"},
+        }
+        result, _ = validate_stac_item(item)
+
+        warning_codes = [w.code for w in result.warnings]
+        assert "NULL_GEOMETRY" in warning_codes
+
+    def test_validate_invalid_geometry_fixed(self):
+        """Test that invalid geometry can be fixed."""
+        # Self-intersecting polygon (bowtie)
+        item = {
+            "type": "Feature",
+            "id": "test",
+            "geometry": {
+                "type": "Polygon",
+                "coordinates": [[[0, 0], [10, 10], [10, 0], [0, 10], [0, 0]]],
+            },
+            "properties": {"datetime": "2024-01-15T10:00:00Z"},
+        }
+        result, corrected = validate_stac_item(item, fix_geometry=True)
+
+        # Should have warning about invalid geometry
+        warning_codes = [w.code for w in result.warnings]
+        assert "INVALID_GEOMETRY" in warning_codes
+
+        # Should have been fixed
+        if result.metadata.get("geometry_fixed"):
+            from shapely.geometry import shape
+
+            fixed_geom = shape(corrected["geometry"])
+            assert fixed_geom.is_valid
+
+    def test_validate_bbox_mismatch(self, valid_stac_item):
+        """Test validation catches bbox mismatch."""
+        # Set incorrect bbox
+        valid_stac_item["bbox"] = [0, 0, 5, 5]  # Doesn't match geometry
+
+        result, corrected = validate_stac_item(valid_stac_item)
+
+        warning_codes = [w.code for w in result.warnings]
+        assert "BBOX_MISMATCH" in warning_codes
+
+        # Should have corrected bbox
+        if result.metadata.get("bbox_corrected"):
+            assert corrected["bbox"] == [-10, -10, 10, 10]
+
+    def test_validate_missing_datetime(self):
+        """Test validation warns on missing datetime."""
+        item = {
+            "type": "Feature",
+            "id": "test",
+            "geometry": {"type": "Point", "coordinates": [0, 0]},
+            "properties": {},  # No datetime
+        }
+        result, _ = validate_stac_item(item)
+
+        warning_codes = [w.code for w in result.warnings]
+        assert "MISSING_DATETIME" in warning_codes
+
+    def test_validate_datetime_range_valid(self):
+        """Test validation accepts datetime range."""
+        item = {
+            "type": "Feature",
+            "id": "test",
+            "geometry": {"type": "Point", "coordinates": [0, 0]},
+            "properties": {
+                "datetime": None,
+                "start_datetime": "2024-01-01T00:00:00Z",
+                "end_datetime": "2024-01-31T23:59:59Z",
+            },
+        }
+        result, _ = validate_stac_item(item)
+
+        # Should NOT have MISSING_DATETIME warning
+        warning_codes = [w.code for w in result.warnings]
+        assert "MISSING_DATETIME" not in warning_codes
+
+
+class TestBatchValidation:
+    """Tests for batch STAC item validation."""
+
+    def test_validate_batch(self):
+        """Test batch validation of multiple items."""
+        items = [
+            {
+                "type": "Feature",
+                "id": f"item-{i}",
+                "geometry": {"type": "Point", "coordinates": [i, i]},
+                "properties": {"datetime": "2024-01-15T10:00:00Z"},
+            }
+            for i in range(5)
+        ]
+
+        results, corrected = validate_stac_items_batch(items)
+
+        assert len(results) == 5
+        assert len(corrected) == 5
+        assert all(r.is_valid for r in results)
+
+
+class TestGeoParquetValidation:
+    """Tests for GeoParquet file validation."""
+
+    @pytest.fixture
+    def valid_geoparquet(self, tmp_path):
+        """Create a valid GeoParquet file for testing."""
+        # Create a GeoDataFrame
+        gdf = gpd.GeoDataFrame(
+            {
+                "id": ["item-1", "item-2", "item-3"],
+                "datetime": ["2024-01-01", "2024-01-02", "2024-01-03"],
+                "geometry": [
+                    Point(0, 0),
+                    Point(1, 1),
+                    Point(2, 2),
+                ],
+            },
+            crs="EPSG:4326",
+        )
+
+        file_path = tmp_path / "test.parquet"
+        gdf.to_parquet(file_path)
+        return file_path
+
+    def test_validate_valid_geoparquet(self, valid_geoparquet):
+        """Test validation of a valid GeoParquet file."""
+        result = validate_geoparquet_file(valid_geoparquet)
+
+        assert result.is_valid is True
+        assert "num_rows" in result.metadata
+        assert result.metadata["num_rows"] == 3
+
+    def test_validate_nonexistent_file(self, tmp_path):
+        """Test validation of nonexistent file."""
+        result = validate_geoparquet_file(tmp_path / "nonexistent.parquet")
+
+        assert result.is_valid is False
+        error_codes = [e.code for e in result.errors]
+        assert "FILE_NOT_FOUND" in error_codes
+
+    def test_validate_geoparquet_has_geo_metadata(self, valid_geoparquet):
+        """Test that geo metadata is validated."""
+        result = validate_geoparquet_file(valid_geoparquet)
+
+        # GeoPandas creates proper geo metadata
+        assert "primary_column" in result.metadata
+        assert result.metadata["primary_column"] == "geometry"
+
+    def test_get_geoparquet_metadata(self, valid_geoparquet):
+        """Test extracting geo metadata from file."""
+        metadata = get_geoparquet_metadata(valid_geoparquet)
+
+        assert "primary_column" in metadata
+        assert "columns" in metadata
+        assert "geometry" in metadata["columns"]
+
+
+class TestCatalogValidation:
+    """Tests for catalog-level validation."""
+
+    @pytest.fixture
+    def catalog_with_files(self, tmp_path):
+        """Create a catalog directory with multiple GeoParquet files."""
+        # Create subdirectory structure
+        partition1 = tmp_path / "partition1"
+        partition2 = tmp_path / "partition2"
+        partition1.mkdir()
+        partition2.mkdir()
+
+        # Create GeoDataFrames
+        for i, partition_dir in enumerate([partition1, partition2]):
+            gdf = gpd.GeoDataFrame(
+                {
+                    "id": [f"item-{i}-{j}" for j in range(3)],
+                    "geometry": [Point(j + i * 10, j) for j in range(3)],
+                },
+                crs="EPSG:4326",
+            )
+            gdf.to_parquet(partition_dir / "data.parquet")
+
+        return tmp_path
+
+    def test_validate_catalog(self, catalog_with_files):
+        """Test validating an entire catalog."""
+        result = validate_catalog(catalog_with_files)
+
+        assert result.total_files == 2
+        assert result.valid_files >= 0  # May have warnings but should parse
+        assert result.is_valid or result.warnings_count > 0
+
+    def test_validate_catalog_summary(self, catalog_with_files):
+        """Test catalog validation summary."""
+        result = validate_catalog(catalog_with_files)
+        summary = result.summary()
+
+        assert "Total files: 2" in summary
+
+    def test_validate_single_file_as_catalog(self, tmp_path):
+        """Test validating a single file through catalog interface."""
+        gdf = gpd.GeoDataFrame(
+            {"id": ["item-1"], "geometry": [Point(0, 0)]},
+            crs="EPSG:4326",
+        )
+        file_path = tmp_path / "single.parquet"
+        gdf.to_parquet(file_path)
+
+        result = validate_catalog(file_path)
+
+        assert result.total_files == 1
+
+
+class TestCatalogValidationResult:
+    """Tests for CatalogValidationResult data class."""
+
+    def test_add_file_result(self):
+        """Test adding file results."""
+        catalog_result = CatalogValidationResult()
+
+        file_result1 = ValidationResult(is_valid=True)
+        file_result1.add_warning("WARN", "warning")
+
+        file_result2 = ValidationResult(is_valid=False)
+        file_result2.add_error("ERR", "error")
+
+        catalog_result.add_file_result("file1.parquet", file_result1)
+        catalog_result.add_file_result("file2.parquet", file_result2)
+
+        assert catalog_result.total_files == 2
+        assert catalog_result.valid_files == 1
+        assert catalog_result.invalid_files == 1
+        assert catalog_result.warnings_count == 1
+        assert catalog_result.errors_count == 1
+        assert catalog_result.is_valid is False
+
+    def test_summary(self):
+        """Test catalog summary generation."""
+        catalog_result = CatalogValidationResult()
+        file_result = ValidationResult(is_valid=False)
+        file_result.add_error("ERR", "error message")
+        catalog_result.add_file_result("bad.parquet", file_result)
+
+        summary = catalog_result.summary()
+        assert "Invalid files: 1" in summary
+        assert "bad.parquet" in summary
+
+
+class TestIntegrationWithPipeline:
+    """Integration tests for validation with ingestion pipeline."""
+
+    def test_processing_config_has_validation_options(self):
+        """Test that ProcessingConfig has validation options."""
+        from earthcatalog.ingestion_pipeline import ProcessingConfig
+
+        config = ProcessingConfig(
+            input_file="test.parquet",
+            output_catalog="./output",
+            scratch_location="./scratch",
+            enable_validation=True,
+            fix_invalid_geometry=True,
+            fix_bbox_mismatch=True,
+            bbox_tolerance=1e-6,
+            log_validation_warnings=True,
+        )
+
+        assert config.enable_validation is True
+        assert config.fix_invalid_geometry is True
+        assert config.fix_bbox_mismatch is True
+        assert config.bbox_tolerance == 1e-6
+        assert config.log_validation_warnings is True
+
+    def test_processing_config_validation_defaults(self):
+        """Test that ProcessingConfig has sensible validation defaults."""
+        from earthcatalog.ingestion_pipeline import ProcessingConfig
+
+        config = ProcessingConfig(
+            input_file="test.parquet",
+            output_catalog="./output",
+            scratch_location="./scratch",
+        )
+
+        # Validation should be enabled by default
+        assert config.enable_validation is True
+        assert config.fix_invalid_geometry is True