earthcatalog 0.2.0__py3-none-any.whl

earthcatalog/tests/test_e2e_synthetic.py

@@ -0,0 +1,1624 @@
+"""End-to-end tests using synthetic STAC items.
+
+This module provides comprehensive end-to-end testing of the EarthCatalog ingestion
+pipeline using synthetically generated STAC items. These tests are designed to verify
+the complete workflow from URL ingestion through consolidation with realistic,
+parameterizable data.
+
+These tests are NOT run as part of the regular test suite. They are intended to be
+run on-demand for integration testing and performance benchmarking.
+
+Run with:
+    pytest earthcatalog/tests/test_e2e_synthetic.py -v -m e2e
+
+Or for specific test configurations:
+    pytest earthcatalog/tests/test_e2e_synthetic.py -v -k "test_e2e_pipeline" --e2e-items=1000
+"""
+
+import json
+import random
+import shutil
+import tempfile
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+from http.server import HTTPServer, SimpleHTTPRequestHandler
+from pathlib import Path
+from threading import Thread
+from typing import Any
+
+import pandas as pd
+import pyarrow as pa
+import pyarrow.parquet as pq
+import pytest
+from jinja2 import Environment, FileSystemLoader
+
+from earthcatalog.ingestion_pipeline import (
+    LocalProcessor,
+    ProcessingConfig,
+    STACIngestionPipeline,
+)
+
+# Mark entire module for e2e testing - skipped by default
+pytestmark = pytest.mark.e2e
+
+
+@dataclass
+class GeometrySpec:
+    """Specification for geometry size/shape."""
+
+    width_deg: float  # Width in degrees longitude
+    height_deg: float  # Height in degrees latitude
+    name: str  # Descriptive name
+
+
+# Standard Landsat scene size (approximately 185km x 180km, ~1.7deg x 1.6deg at equator)
+LANDSAT_SCENE = GeometrySpec(width_deg=1.7, height_deg=1.6, name="landsat")
+
+# Sentinel-2 tile (100km x 100km, ~0.9deg x 0.9deg at equator)
+SENTINEL2_TILE = GeometrySpec(width_deg=0.9, height_deg=0.9, name="sentinel2")
+
+# Very small geometry (outlier - too small, like a single point observation)
+TINY_GEOMETRY = GeometrySpec(width_deg=0.001, height_deg=0.001, name="tiny")
+
+# Very large geometry (outlier - spans many grid cells, like MODIS swath)
+HUGE_GEOMETRY = GeometrySpec(width_deg=20.0, height_deg=15.0, name="huge")
+
+# Continental scale (extreme outlier - should go to global partition)
+CONTINENTAL = GeometrySpec(width_deg=50.0, height_deg=30.0, name="continental")
+
+
+class SyntheticSTACGenerator:
+    """Generator for synthetic STAC items using Jinja templates.
+
+    This class creates realistic STAC items with configurable parameters for
+    testing the ingestion pipeline with various geometry sizes, temporal
+    distributions, and outlier conditions.
+
+    Attributes:
+        template_dir: Directory containing Jinja templates
+        output_dir: Directory for generated STAC JSON files
+        base_geometry: Default geometry specification for items
+        outlier_percent: Percentage of items with outlier geometries
+        seed: Random seed for reproducibility
+    """
+
+    def __init__(
+        self,
+        output_dir: Path,
+        base_geometry: GeometrySpec = LANDSAT_SCENE,
+        outlier_tiny_percent: float = 5.0,
+        outlier_huge_percent: float = 5.0,
+        seed: int | None = None,
+    ):
+        """Initialize the synthetic STAC generator.
+
+        Args:
+            output_dir: Directory to write generated STAC JSON files
+            base_geometry: Default geometry size specification
+            outlier_tiny_percent: Percentage of items with tiny geometries (0-100)
+            outlier_huge_percent: Percentage of items with huge geometries (0-100)
+            seed: Random seed for reproducibility
+        """
+        self.output_dir = Path(output_dir)
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+
+        self.base_geometry = base_geometry
+        self.outlier_tiny_percent = outlier_tiny_percent
+        self.outlier_huge_percent = outlier_huge_percent
+
+        # Set random seed for reproducibility
+        if seed is not None:
+            random.seed(seed)
+
+        # Initialize Jinja environment
+        template_dir = Path(__file__).parent / "fixtures"
+        self.env = Environment(loader=FileSystemLoader(template_dir))
+        self.template = self.env.get_template("stac_item_template.jinja2")
+
+        # Track generated items
+        self.generated_items: list[dict[str, Any]] = []
+        self.generated_files: list[Path] = []
+
+    def _select_geometry_spec(self) -> GeometrySpec:
+        """Select geometry specification based on outlier percentages."""
+        roll = random.uniform(0, 100)
+
+        if roll < self.outlier_tiny_percent:
+            return TINY_GEOMETRY
+        elif roll < self.outlier_tiny_percent + self.outlier_huge_percent:
+            # Randomly choose between huge and continental for large outliers
+            return random.choice([HUGE_GEOMETRY, CONTINENTAL])
+        else:
+            return self.base_geometry
+
+    def _generate_random_bbox(self, geom_spec: GeometrySpec) -> tuple[float, float, float, float]:
+        """Generate a random bounding box within valid global bounds.
+
+        Returns:
+            Tuple of (min_lon, min_lat, max_lon, max_lat)
+        """
+        # Ensure geometry fits within global bounds
+        max_start_lon = 180.0 - geom_spec.width_deg
+        max_start_lat = 90.0 - geom_spec.height_deg
+
+        min_lon = random.uniform(-180.0, max_start_lon)
+        min_lat = random.uniform(-90.0, max_start_lat)
+
+        max_lon = min_lon + geom_spec.width_deg
+        max_lat = min_lat + geom_spec.height_deg
+
+        return (min_lon, min_lat, max_lon, max_lat)
+
+    def _generate_random_datetime(
+        self,
+        start_date: datetime = datetime(2020, 1, 1),
+        end_date: datetime = datetime(2024, 12, 31),
+    ) -> datetime:
+        """Generate a random datetime within the specified range."""
+        delta = end_date - start_date
+        random_days = random.randint(0, delta.days)
+        random_seconds = random.randint(0, 86400)
+        return start_date + timedelta(days=random_days, seconds=random_seconds)
+
+    def generate_item(
+        self,
+        item_index: int,
+        dataset_id: str = "landsat_c2l2",
+        collection: str = "landsat-c2-l2",
+    ) -> dict[str, Any]:
+        """Generate a single synthetic STAC item.
+
+        Args:
+            item_index: Index of this item (for unique ID generation)
+            dataset_id: Dataset identifier for partitioning
+            collection: STAC collection name
+
+        Returns:
+            Dictionary containing the generated STAC item
+        """
+        # Select geometry specification (may be outlier)
+        geom_spec = self._select_geometry_spec()
+
+        # Generate random bbox
+        min_lon, min_lat, max_lon, max_lat = self._generate_random_bbox(geom_spec)
+
+        # Generate random datetime
+        dt = self._generate_random_datetime()
+        dt_str = dt.strftime("%Y-%m-%dT%H:%M:%SZ")
+        created_str = datetime.now().strftime("%Y-%m-%dT%H:%M:%SZ")
+
+        # Generate unique identifiers
+        item_id = f"LC08_L2SP_{random.randint(1, 999):03d}{random.randint(1, 999):03d}_{dt.strftime('%Y%m%d')}_{item_index:06d}"
+        scene_id = f"LC08_{random.randint(1, 999):03d}{random.randint(1, 999):03d}_{dt.strftime('%Y%m%d')}_02_T1"
+
+        # WRS path/row (Landsat grid)
+        wrs_path = random.randint(1, 233)
+        wrs_row = random.randint(1, 248)
+
+        # Base URL for assets
+        base_url = f"https://example.com/data/{collection}/{item_id}"
+
+        # Template context
+        context = {
+            # Identification
+            "item_id": item_id,
+            "scene_id": scene_id,
+            # Geometry
+            "min_lon": round(min_lon, 6),
+            "min_lat": round(min_lat, 6),
+            "max_lon": round(max_lon, 6),
+            "max_lat": round(max_lat, 6),
+            # Temporal
+            "datetime": dt_str,
+            "created": created_str,
+            "updated": created_str,
+            # Platform info
+            "platform": "landsat-8",
+            "instruments": ["oli", "tirs"],
+            "constellation": "landsat",
+            "mission": "landsat-8",
+            "gsd": 30,
+            # Collection info
+            "collection": collection,
+            "dataset_id": dataset_id,
+            # Scene properties
+            "cloud_cover": round(random.uniform(0, 100), 2),
+            "off_nadir": round(random.uniform(0, 15), 2),
+            "sun_azimuth": round(random.uniform(0, 360), 2),
+            "sun_elevation": round(random.uniform(10, 80), 2),
+            # Projection
+            "epsg": 32600 + random.randint(1, 60),  # UTM zones
+            "proj_shape": [7611, 7531],  # Standard Landsat dimensions
+            "proj_transform": [30.0, 0.0, min_lon * 111000, 0.0, -30.0, max_lat * 111000],
+            # Landsat-specific
+            "wrs_path": wrs_path,
+            "wrs_row": wrs_row,
+            "collection_category": "T1",
+            "collection_number": "02",
+            # Links
+            "self_href": f"{base_url}/{item_id}.json",
+            "parent_href": f"https://example.com/collections/{collection}",
+            "collection_href": f"https://example.com/collections/{collection}",
+            "root_href": "https://example.com",
+            # Asset URLs
+            "thumbnail_href": f"{base_url}/{item_id}_thumb_small.jpeg",
+            "browse_href": f"{base_url}/{item_id}_thumb_large.jpeg",
+            "mtl_href": f"{base_url}/{item_id}_MTL.json",
+            "mtl_txt_href": f"{base_url}/{item_id}_MTL.txt",
+            "mtl_xml_href": f"{base_url}/{item_id}_MTL.xml",
+            "ang_href": f"{base_url}/{item_id}_ANG.txt",
+            "qa_pixel_href": f"{base_url}/{item_id}_QA_PIXEL.TIF",
+            "qa_radsat_href": f"{base_url}/{item_id}_QA_RADSAT.TIF",
+            "coastal_href": f"{base_url}/{item_id}_SR_B1.TIF",
+            "blue_href": f"{base_url}/{item_id}_SR_B2.TIF",
+            "green_href": f"{base_url}/{item_id}_SR_B3.TIF",
+            "red_href": f"{base_url}/{item_id}_SR_B4.TIF",
+            "nir08_href": f"{base_url}/{item_id}_SR_B5.TIF",
+            "swir16_href": f"{base_url}/{item_id}_SR_B6.TIF",
+            "swir22_href": f"{base_url}/{item_id}_SR_B7.TIF",
+            "lwir11_href": f"{base_url}/{item_id}_ST_B10.TIF",
+        }
+
+        # Render template
+        rendered = self.template.render(**context)
+        item = json.loads(rendered)
+
+        # Store metadata about geometry type for analysis
+        item["_synthetic_metadata"] = {
+            "geometry_type": geom_spec.name,
+            "index": item_index,
+        }
+
+        return item
+
+    def generate_items(
+        self,
+        n_items: int,
+        dataset_ids: list[str] | None = None,
+        shuffle: bool = True,
+    ) -> list[dict[str, Any]]:
+        """Generate multiple synthetic STAC items.
+
+        Args:
+            n_items: Number of items to generate
+            dataset_ids: List of dataset IDs to distribute items across
+            shuffle: Whether to shuffle the items (simulates unordered real-world data)
+
+        Returns:
+            List of generated STAC items
+        """
+        if dataset_ids is None:
+            dataset_ids = ["landsat_c2l2", "sentinel2_l2a", "modis_mcd43a4"]
+
+        items = []
+        for i in range(n_items):
+            dataset_id = random.choice(dataset_ids)
+            collection = dataset_id.replace("_", "-")
+            item = self.generate_item(i, dataset_id=dataset_id, collection=collection)
+            items.append(item)
+
+        if shuffle:
+            random.shuffle(items)
+
+        self.generated_items = items
+        return items
+
+    def write_items_to_files(self, items: list[dict[str, Any]] | None = None) -> list[Path]:
+        """Write STAC items to individual JSON files.
+
+        Args:
+            items: List of items to write. Uses self.generated_items if None.
+
+        Returns:
+            List of paths to generated files
+        """
+        if items is None:
+            items = self.generated_items
+
+        files = []
+        for item in items:
+            # Remove synthetic metadata before writing
+            item_clean = {k: v for k, v in item.items() if not k.startswith("_")}
+            item_id = item_clean["id"]
+            file_path = self.output_dir / f"{item_id}.json"
+
+            with open(file_path, "w") as f:
+                json.dump(item_clean, f, indent=2)
+
+            files.append(file_path)
+
+        self.generated_files = files
+        return files
+
+    def create_url_parquet(
+        self,
+        base_url: str,
+        output_path: Path,
+        shuffle: bool = True,
+    ) -> Path:
+        """Create a Parquet file with URLs pointing to generated STAC items.
+
+        Args:
+            base_url: Base URL where items will be served (e.g., http://localhost:8000)
+            output_path: Path for the output Parquet file
+            shuffle: Whether to shuffle URLs (simulates unordered real-world data)
+
+        Returns:
+            Path to the created Parquet file
+        """
+        urls = [f"{base_url}/{f.name}" for f in self.generated_files]
+
+        if shuffle:
+            random.shuffle(urls)
+
+        # Create DataFrame and write to Parquet
+        df = pd.DataFrame({"url": urls})
+        table = pa.Table.from_pandas(df)
+        pq.write_table(table, output_path)
+
+        return output_path
+
+    def get_geometry_distribution(self) -> dict[str, int]:
+        """Get distribution of geometry types in generated items."""
+        distribution: dict[str, int] = {}
+        for item in self.generated_items:
+            geom_type = item.get("_synthetic_metadata", {}).get("geometry_type", "unknown")
+            distribution[geom_type] = distribution.get(geom_type, 0) + 1
+        return distribution
+
+
+class SimpleSTACServer:
+    """Simple HTTP server for serving STAC items during tests."""
+
+    def __init__(self, directory: Path, port: int = 0):
+        """Initialize the server.
+
+        Args:
+            directory: Directory containing STAC JSON files to serve
+            port: Port to listen on (0 for auto-assign)
+        """
+        self.directory = directory
+        self.port = port
+        self.server: HTTPServer | None = None
+        self.thread: Thread | None = None
+
+    def start(self) -> str:
+        """Start the HTTP server in a background thread.
+
+        Returns:
+            Base URL of the server
+        """
+        import functools
+
+        # Create handler that serves from our directory
+        handler = functools.partial(SimpleHTTPRequestHandler, directory=str(self.directory))
+
+        self.server = HTTPServer(("localhost", self.port), handler)
+        self.port = self.server.server_port
+
+        self.thread = Thread(target=self.server.serve_forever, daemon=True)
+        self.thread.start()
+
+        return f"http://localhost:{self.port}"
+
+    def stop(self):
+        """Stop the HTTP server."""
+        if self.server:
+            self.server.shutdown()
+            self.server = None
+        if self.thread:
+            self.thread.join(timeout=5)
+            self.thread = None
+
+
+# Pytest fixtures
+# Note: pytest_addoption, pytest_configure, and pytest_collection_modifyitems
+# are defined in conftest.py to be picked up by pytest
+
+
+@pytest.fixture(scope="module")
+def e2e_config(request):
+    """Get e2e test configuration from command line options."""
+
+    class E2EConfig:
+        n_items = int(request.config.getoption("--e2e-items", 100))
+        outlier_tiny_percent = float(request.config.getoption("--e2e-outlier-tiny", 5))
+        outlier_huge_percent = float(request.config.getoption("--e2e-outlier-huge", 5))
+        seed = request.config.getoption("--e2e-seed", None)
+        if seed is not None:
+            seed = int(seed)
+
+    return E2EConfig()
+
+
+@pytest.fixture(scope="module")
+def temp_test_dir():
+    """Create a temporary directory for test files."""
+    temp_dir = Path(tempfile.mkdtemp(prefix="earthcatalog_e2e_"))
+    yield temp_dir
+    # Cleanup after all tests in module
+    shutil.rmtree(temp_dir, ignore_errors=True)
+
+
+@pytest.fixture(scope="module")
+def synthetic_generator(temp_test_dir, e2e_config):
+    """Create and configure a synthetic STAC generator."""
+    items_dir = temp_test_dir / "stac_items"
+    items_dir.mkdir(parents=True, exist_ok=True)
+
+    generator = SyntheticSTACGenerator(
+        output_dir=items_dir,
+        base_geometry=LANDSAT_SCENE,
+        outlier_tiny_percent=e2e_config.outlier_tiny_percent,
+        outlier_huge_percent=e2e_config.outlier_huge_percent,
+        seed=e2e_config.seed,
+    )
+
+    return generator
+
+
+@pytest.fixture(scope="module")
+def generated_items(synthetic_generator, e2e_config):
+    """Generate synthetic STAC items for testing."""
+    items = synthetic_generator.generate_items(
+        n_items=e2e_config.n_items,
+        shuffle=True,
+    )
+    synthetic_generator.write_items_to_files(items)
+    return items
+
+
+@pytest.fixture(scope="module")
+def stac_server(synthetic_generator, generated_items):
+    """Start HTTP server to serve STAC items."""
+    server = SimpleSTACServer(synthetic_generator.output_dir)
+    base_url = server.start()
+    yield base_url
+    server.stop()
+
+
+@pytest.fixture(scope="module")
+def urls_parquet(synthetic_generator, stac_server, temp_test_dir):
+    """Create Parquet file with URLs pointing to served STAC items."""
+    parquet_path = temp_test_dir / "stac_urls.parquet"
+    synthetic_generator.create_url_parquet(stac_server, parquet_path, shuffle=True)
+    return parquet_path
+
+
+class TestE2ESyntheticPipeline:
+    """End-to-end tests using synthetic STAC items and the full ingestion pipeline."""
+
+    @pytest.mark.e2e
+    def test_e2e_pipeline_basic(self, urls_parquet, temp_test_dir, e2e_config):
+        """Test basic end-to-end pipeline with synthetic data.
+
+        This test verifies:
+        1. Pipeline can ingest URLs from Parquet file
+        2. Items are correctly partitioned spatially and temporally
+        3. Consolidation produces valid GeoParquet output
+        4. Schema generation works correctly
+        """
+        output_catalog = temp_test_dir / "catalog_basic"
+        scratch_location = temp_test_dir / "scratch_basic"
+
+        config = ProcessingConfig(
+            input_file=str(urls_parquet),
+            output_catalog=str(output_catalog),
+            scratch_location=str(scratch_location),
+            grid_system="h3",
+            grid_resolution=2,
+            temporal_bin="month",
+            generate_schema=True,
+            enable_concurrent_http=True,
+            concurrent_requests=10,
+            batch_size=50,
+            max_workers=2,
+            items_per_shard=100,
+        )
+
+        processor = LocalProcessor(n_workers=config.max_workers)
+        pipeline = STACIngestionPipeline(config, processor)
+
+        try:
+            stats = pipeline.run()
+
+            # Verify output
+            assert len(stats) > 0, "Pipeline should produce at least one partition"
+
+            total_items = sum(s["total_items"] for s in stats.values())
+            assert total_items > 0, "Pipeline should ingest at least some items"
+
+            # Check catalog directory structure
+            assert output_catalog.exists(), "Output catalog should exist"
+
+            # Check schema file was created
+            schema_file = output_catalog / "catalog_schema.json"
+            assert schema_file.exists(), "Schema file should be generated"
+
+            with open(schema_file) as f:
+                schema = json.load(f)
+                assert "spatial_partitioning" in schema
+                assert schema["spatial_partitioning"]["grid_system"] == "h3"
+
+            # Verify Hive-style directory structure
+            parquet_files = list(output_catalog.rglob("items.parquet"))
+            assert len(parquet_files) > 0, "Should produce at least one items.parquet file"
+
+            # Check for Hive-style directories
+            for pf in parquet_files:
+                path_str = str(pf)
+                assert "partition=h3" in path_str or "partition=geojson" in path_str, (
+                    f"Path should have Hive-style partition: {path_str}"
+                )
+
+        finally:
+            processor.close()
+
+    @pytest.mark.e2e
+    def test_e2e_pipeline_with_global_partition(self, urls_parquet, temp_test_dir, e2e_config, synthetic_generator):
+        """Test that large geometries are routed to global partition.
+
+        This test verifies:
+        1. Items with geometries spanning many cells go to global partition
+        2. Regular items go to spatial partitions
+        3. Global partition threshold is respected
+        """
+        # Ensure we have some large geometries
+        geom_dist = synthetic_generator.get_geometry_distribution()
+        large_count = geom_dist.get("huge", 0) + geom_dist.get("continental", 0)
+
+        if large_count == 0:
+            pytest.skip("No large geometry outliers in generated data")
+
+        output_catalog = temp_test_dir / "catalog_global"
+        scratch_location = temp_test_dir / "scratch_global"
+
+        config = ProcessingConfig(
+            input_file=str(urls_parquet),
+            output_catalog=str(output_catalog),
+            scratch_location=str(scratch_location),
+            grid_system="h3",
+            grid_resolution=2,
+            temporal_bin="month",
+            enable_global_partitioning=True,
+            global_partition_threshold=5,  # Low threshold to trigger global routing
+            generate_schema=True,
+            enable_concurrent_http=True,
+            concurrent_requests=10,
+            batch_size=50,
+            max_workers=2,
+        )
+
+        processor = LocalProcessor(n_workers=config.max_workers)
+        pipeline = STACIngestionPipeline(config, processor)
+
+        try:
+            stats = pipeline.run()
+
+            # Check if any partitions have "global" in their path
+            _global_partitions = [k for k in stats.keys() if "global" in k.lower()]
+
+            # Note: Global partition may or may not exist depending on geometry distribution
+            # The test mainly verifies the pipeline handles this correctly without errors
+
+            total_items = sum(s["total_items"] for s in stats.values())
+            assert total_items > 0, "Pipeline should ingest items"
+
+        finally:
+            processor.close()
+
+    @pytest.mark.e2e
+    def test_e2e_pipeline_temporal_partitions(self, urls_parquet, temp_test_dir, e2e_config):
+        """Test temporal partitioning creates correct Hive-style directories.
+
+        This test verifies:
+        1. Year/month directories are created with Hive naming
+        2. Items are correctly distributed across temporal partitions
+        """
+        output_catalog = temp_test_dir / "catalog_temporal"
+        scratch_location = temp_test_dir / "scratch_temporal"
+
+        config = ProcessingConfig(
+            input_file=str(urls_parquet),
+            output_catalog=str(output_catalog),
+            scratch_location=str(scratch_location),
+            grid_system="h3",
+            grid_resolution=2,
+            temporal_bin="month",
+            generate_schema=True,
+            enable_concurrent_http=True,
+            concurrent_requests=10,
+            batch_size=50,
+            max_workers=2,
+        )
+
+        processor = LocalProcessor(n_workers=config.max_workers)
+        pipeline = STACIngestionPipeline(config, processor)
+
+        try:
+            _stats = pipeline.run()
+
+            # Check for Hive-style temporal directories
+            parquet_files = list(output_catalog.rglob("items.parquet"))
+            assert len(parquet_files) > 0
+
+            year_dirs = set()
+            month_dirs = set()
+
+            for pf in parquet_files:
+                path_parts = str(pf).split("/")
+                for part in path_parts:
+                    if part.startswith("year="):
+                        year_dirs.add(part)
+                    elif part.startswith("month="):
+                        month_dirs.add(part)
+
+            # Should have at least one year and month partition
+            assert len(year_dirs) > 0, "Should have year= directories"
+            assert len(month_dirs) > 0, "Should have month= directories"
+
+        finally:
+            processor.close()
+
+    @pytest.mark.e2e
+    def test_e2e_pipeline_multiple_datasets(self, temp_test_dir, e2e_config):
+        """Test pipeline with items from multiple datasets.
+
+        This test verifies:
+        1. Items are partitioned by dataset/mission
+        2. Multiple dataset directories are created
+        """
+        # Generate items with multiple datasets
+        items_dir = temp_test_dir / "multi_dataset_items"
+        items_dir.mkdir(parents=True, exist_ok=True)
+
+        generator = SyntheticSTACGenerator(
+            output_dir=items_dir,
+            base_geometry=LANDSAT_SCENE,
+            outlier_tiny_percent=0,  # No outliers for this test
+            outlier_huge_percent=0,
+            seed=42,
+        )
+
+        # Generate items for 3 different datasets
+        dataset_ids = ["landsat_c2l2", "sentinel2_l2a", "modis_mcd43a4"]
+        items = generator.generate_items(
+            n_items=min(e2e_config.n_items, 60),  # At least 20 per dataset
+            dataset_ids=dataset_ids,
+            shuffle=True,
+        )
+        generator.write_items_to_files(items)
+
+        # Start server and create URL parquet
+        server = SimpleSTACServer(items_dir)
+        base_url = server.start()
+
+        try:
+            parquet_path = temp_test_dir / "multi_dataset_urls.parquet"
+            generator.create_url_parquet(base_url, parquet_path, shuffle=True)
+
+            output_catalog = temp_test_dir / "catalog_multi_dataset"
+            scratch_location = temp_test_dir / "scratch_multi_dataset"
+
+            config = ProcessingConfig(
+                input_file=str(parquet_path),
+                output_catalog=str(output_catalog),
+                scratch_location=str(scratch_location),
+                grid_system="h3",
+                grid_resolution=2,
+                temporal_bin="month",
+                generate_schema=True,
+                enable_concurrent_http=True,
+                concurrent_requests=10,
+                batch_size=50,
+                max_workers=2,
+            )
+
+            processor = LocalProcessor(n_workers=config.max_workers)
+            pipeline = STACIngestionPipeline(config, processor)
+
+            try:
+                _stats = pipeline.run()
+
+                # Check for multiple dataset directories
+                assert output_catalog.exists()
+
+                # Get top-level directories (should be dataset names)
+                top_dirs = [d.name for d in output_catalog.iterdir() if d.is_dir()]
+
+                # Should have directories for multiple datasets
+                # (some items may fail to download, so just check we have at least 1)
+                assert len(top_dirs) >= 1, f"Should have dataset directories, found: {top_dirs}"
+
+            finally:
+                processor.close()
+
+        finally:
+            server.stop()
+
+    @pytest.mark.e2e
+    def test_e2e_consolidation_deduplication(self, temp_test_dir, e2e_config):
+        """Test that consolidation properly deduplicates items.
+
+        This test verifies:
+        1. Running pipeline twice with overlapping data deduplicates correctly
+        2. Newer items override older ones (keep='last' behavior)
+        """
+        items_dir = temp_test_dir / "dedup_items"
+        items_dir.mkdir(parents=True, exist_ok=True)
+
+        generator = SyntheticSTACGenerator(
+            output_dir=items_dir,
+            base_geometry=LANDSAT_SCENE,
+            outlier_tiny_percent=0,
+            outlier_huge_percent=0,
+            seed=42,
+        )
+
+        # Generate initial set of items
+        items = generator.generate_items(n_items=20, shuffle=False)
+        generator.write_items_to_files(items)
+
+        server = SimpleSTACServer(items_dir)
+        base_url = server.start()
+
+        try:
+            parquet_path = temp_test_dir / "dedup_urls.parquet"
+            generator.create_url_parquet(base_url, parquet_path, shuffle=True)
+
+            output_catalog = temp_test_dir / "catalog_dedup"
+            scratch_location = temp_test_dir / "scratch_dedup"
+
+            config = ProcessingConfig(
+                input_file=str(parquet_path),
+                output_catalog=str(output_catalog),
+                scratch_location=str(scratch_location),
+                grid_system="h3",
+                grid_resolution=2,
+                temporal_bin="month",
+                enable_concurrent_http=True,
+                concurrent_requests=5,
+                batch_size=20,
+                max_workers=1,
+            )
+
+            processor = LocalProcessor(n_workers=1)
+            pipeline = STACIngestionPipeline(config, processor)
+
+            try:
+                # First run
+                stats1 = pipeline.run()
+                total1 = sum(s["total_items"] for s in stats1.values())
+
+                # Second run with same data (should deduplicate)
+                stats2 = pipeline.run()
+                total2 = sum(s["total_items"] for s in stats2.values())
+
+                # Should have same total (deduplication working)
+                assert total2 == total1, f"Second run should have same total due to deduplication: {total2} vs {total1}"
+
+            finally:
+                processor.close()
+
+        finally:
+            server.stop()
+
+
+class TestSyntheticGenerator:
+    """Tests for the synthetic STAC generator itself."""
+
+    @pytest.mark.e2e
+    def test_generator_produces_valid_items(self, synthetic_generator, e2e_config):
+        """Test that generated items have valid STAC structure."""
+        items = synthetic_generator.generate_items(n_items=10, shuffle=False)
+
+        for item in items:
+            # Check required STAC fields
+            assert "type" in item and item["type"] == "Feature"
+            assert "stac_version" in item
+            assert "id" in item
+            assert "geometry" in item
+            assert "bbox" in item
+            assert "properties" in item
+            assert "assets" in item
+            assert "links" in item
+
+            # Check geometry structure
+            geom = item["geometry"]
+            assert geom["type"] == "Polygon"
+            assert "coordinates" in geom
+            assert len(geom["coordinates"][0]) == 5  # Closed polygon
+
+            # Check bbox consistency with geometry
+            bbox = item["bbox"]
+            coords = geom["coordinates"][0]
+            min_lon = min(c[0] for c in coords)
+            max_lon = max(c[0] for c in coords)
+            min_lat = min(c[1] for c in coords)
+            max_lat = max(c[1] for c in coords)
+
+            assert abs(bbox[0] - min_lon) < 0.0001
+            assert abs(bbox[1] - min_lat) < 0.0001
+            assert abs(bbox[2] - max_lon) < 0.0001
+            assert abs(bbox[3] - max_lat) < 0.0001
+
+    @pytest.mark.e2e
+    def test_generator_outlier_distribution(self, temp_test_dir):
+        """Test that outlier percentages are respected."""
+        items_dir = temp_test_dir / "outlier_test"
+        items_dir.mkdir(parents=True, exist_ok=True)
+
+        generator = SyntheticSTACGenerator(
+            output_dir=items_dir,
+            outlier_tiny_percent=20.0,
+            outlier_huge_percent=10.0,
+            seed=42,
+        )
+
+        # Generate many items to get statistical significance
+        items = generator.generate_items(n_items=1000, shuffle=False)
+        dist = generator.get_geometry_distribution()
+
+        # Check distribution is roughly correct (within tolerance)
+        total = len(items)
+        tiny_pct = (dist.get("tiny", 0) / total) * 100
+        large_pct = ((dist.get("huge", 0) + dist.get("continental", 0)) / total) * 100
+
+        # Allow 50% tolerance due to randomness
+        assert 10 < tiny_pct < 30, f"Tiny outlier percentage {tiny_pct:.1f}% not in expected range"
+        assert 5 < large_pct < 20, f"Large outlier percentage {large_pct:.1f}% not in expected range"
+
+    @pytest.mark.e2e
+    def test_generator_shuffle(self, temp_test_dir):
+        """Test that shuffle parameter works correctly."""
+        items_dir = temp_test_dir / "shuffle_test"
+        items_dir.mkdir(parents=True, exist_ok=True)
+
+        generator = SyntheticSTACGenerator(output_dir=items_dir, seed=42)
+
+        # Generate unshuffled
+        items_unshuffled = generator.generate_items(n_items=50, shuffle=False)
+        ids_unshuffled = [item["id"] for item in items_unshuffled]
+
+        # Reset and generate shuffled (same seed means same items, different order)
+        generator2 = SyntheticSTACGenerator(output_dir=items_dir, seed=42)
+        items_shuffled = generator2.generate_items(n_items=50, shuffle=True)
+        ids_shuffled = [item["id"] for item in items_shuffled]
+
+        # Should have same IDs but different order
+        assert set(ids_unshuffled) == set(ids_shuffled)
+        assert ids_unshuffled != ids_shuffled, "Shuffled order should differ from unshuffled"
+
+
+# =============================================================================
+# Query Performance Profiling Tests
+# =============================================================================
+
+
+@dataclass
+class QueryProfileResult:
+    """Result of a single query profile run."""
+
+    engine: str
+    query_type: str
+    iteration: int
+    duration_ms: float
+    rows_returned: int
+    partitions_scanned: int
+    error: str | None = None
+
+
+@dataclass
+class ProfileSummary:
+    """Summary statistics for a profiling run."""
+
+    engine: str
+    query_type: str
+    iterations: int
+    min_ms: float
+    max_ms: float
+    mean_ms: float
+    median_ms: float
+    std_ms: float
+    total_rows: int
+    partitions_scanned: int
+
+
+class QueryProfiler:
+    """Profiler for measuring query performance across different engines.
+
+    Supports profiling queries using:
+    - DuckDB (with spatial extension)
+    - rustac (Rust-based STAC querying)
+    - Spatial resolver (partition resolution only)
+    """
+
+    def __init__(
+        self,
+        catalog_path: Path,
+        schema_path: Path,
+        iterations: int = 10,
+    ):
+        """Initialize the query profiler.
+
+        Args:
+            catalog_path: Path to the catalog directory
+            schema_path: Path to the catalog schema JSON file
+            iterations: Number of iterations per query for averaging
+        """
+        self.catalog_path = catalog_path
+        self.schema_path = schema_path
+        self.iterations = iterations
+        self.results: list[QueryProfileResult] = []
+
+    def _get_test_geometries(self) -> dict[str, Any]:
+        """Generate test geometries of different sizes for profiling."""
+        from shapely.geometry import box
+
+        return {
+            "point_query": box(-105.0, 40.0, -104.99, 40.01),  # ~1km bbox
+            "small_region": box(-106.0, 39.0, -104.0, 41.0),  # ~200km bbox
+            "medium_region": box(-110.0, 35.0, -100.0, 45.0),  # ~1000km bbox
+            "large_region": box(-125.0, 25.0, -65.0, 50.0),  # Continental US
+        }
+
+    def profile_spatial_resolver(self, geometry, query_name: str) -> list[QueryProfileResult]:
+        """Profile spatial resolver partition resolution performance."""
+        import time
+
+        from earthcatalog.spatial_resolver import spatial_resolver
+
+        results = []
+
+        for i in range(self.iterations):
+            try:
+                start = time.perf_counter()
+
+                resolver = spatial_resolver(str(self.schema_path), str(self.catalog_path))
+                partitions = resolver.resolve_partitions(geometry, overlap=True)
+                _query_paths = resolver.generate_query_paths(partitions, temporal_filter=None)
+
+                duration_ms = (time.perf_counter() - start) * 1000
+
+                results.append(
+                    QueryProfileResult(
+                        engine="spatial_resolver",
+                        query_type=query_name,
+                        iteration=i,
+                        duration_ms=duration_ms,
+                        rows_returned=0,
+                        partitions_scanned=len(partitions),
+                    )
+                )
+
+            except (ValueError, TypeError, AttributeError, OSError) as e:
+                results.append(
+                    QueryProfileResult(
+                        engine="spatial_resolver",
+                        query_type=query_name,
+                        iteration=i,
+                        duration_ms=0,
+                        rows_returned=0,
+                        partitions_scanned=0,
+                        error=str(e),
+                    )
+                )
+
+        return results
+
+    def profile_duckdb(self, geometry, query_name: str, temporal_filter: str | None = None) -> list[QueryProfileResult]:
+        """Profile DuckDB query performance."""
+        import time
+
+        try:
+            import duckdb
+        except ImportError:
+            return [
+                QueryProfileResult(
+                    engine="duckdb",
+                    query_type=query_name,
+                    iteration=0,
+                    duration_ms=0,
+                    rows_returned=0,
+                    partitions_scanned=0,
+                    error="duckdb not installed",
+                )
+            ]
+
+        from earthcatalog.spatial_resolver import spatial_resolver
+
+        results = []
+
+        for i in range(self.iterations):
+            try:
+                # Get partition paths
+                resolver = spatial_resolver(str(self.schema_path), str(self.catalog_path))
+                partitions = resolver.resolve_partitions(geometry, overlap=True)
+                query_paths = resolver.generate_query_paths(partitions, temporal_filter)
+
+                if not query_paths:
+                    results.append(
+                        QueryProfileResult(
+                            engine="duckdb",
+                            query_type=query_name,
+                            iteration=i,
+                            duration_ms=0,
+                            rows_returned=0,
+                            partitions_scanned=0,
+                            error="no partitions found",
+                        )
+                    )
+                    continue
+
+                # Build DuckDB query
+                # Handle glob patterns in paths
+                file_patterns = []
+                for path in query_paths:
+                    if "*" in path:
+                        # Use glob for pattern matching
+                        from pathlib import Path as P
+
+                        matching_files = list(P(self.catalog_path).glob(path.replace(str(self.catalog_path) + "/", "")))
+                        file_patterns.extend([str(f) for f in matching_files])
+                    else:
+                        file_patterns.append(path)
+
+                if not file_patterns:
+                    results.append(
+                        QueryProfileResult(
+                            engine="duckdb",
+                            query_type=query_name,
+                            iteration=i,
+                            duration_ms=0,
+                            rows_returned=0,
+                            partitions_scanned=len(partitions),
+                            error="no matching files",
+                        )
+                    )
+                    continue
+
+                # Create file list for DuckDB
+                files_str = ", ".join([f"'{f}'" for f in file_patterns])
+                query = f"SELECT COUNT(*) as cnt FROM read_parquet([{files_str}])"
+
+                start = time.perf_counter()
+                result = duckdb.sql(query).fetchone()
+                duration_ms = (time.perf_counter() - start) * 1000
+
+                row_count = result[0] if result else 0
+
+                results.append(
+                    QueryProfileResult(
+                        engine="duckdb",
+                        query_type=query_name,
+                        iteration=i,
+                        duration_ms=duration_ms,
+                        rows_returned=row_count,
+                        partitions_scanned=len(partitions),
+                    )
+                )
+
+            except (ValueError, TypeError, AttributeError, OSError) as e:
+                results.append(
+                    QueryProfileResult(
+                        engine="duckdb",
+                        query_type=query_name,
+                        iteration=i,
+                        duration_ms=0,
+                        rows_returned=0,
+                        partitions_scanned=0,
+                        error=str(e),
+                    )
+                )
+
+        return results
+
+    def profile_rustac(self, geometry, query_name: str, temporal_filter: str | None = None) -> list[QueryProfileResult]:
+        """Profile rustac query performance."""
+        import time
+
+        try:
+            import rustac
+        except ImportError:
+            return [
+                QueryProfileResult(
+                    engine="rustac",
+                    query_type=query_name,
+                    iteration=0,
+                    duration_ms=0,
+                    rows_returned=0,
+                    partitions_scanned=0,
+                    error="rustac not installed",
+                )
+            ]
+
+        from earthcatalog.spatial_resolver import spatial_resolver
+
+        results = []
+
+        for i in range(self.iterations):
+            try:
+                # Get partition paths
+                resolver = spatial_resolver(str(self.schema_path), str(self.catalog_path))
+                partitions = resolver.resolve_partitions(geometry, overlap=True)
+                query_paths = resolver.generate_query_paths(partitions, temporal_filter)
+
+                if not query_paths:
+                    results.append(
+                        QueryProfileResult(
+                            engine="rustac",
+                            query_type=query_name,
+                            iteration=i,
+                            duration_ms=0,
+                            rows_returned=0,
+                            partitions_scanned=0,
+                            error="no partitions found",
+                        )
+                    )
+                    continue
+
+                # Resolve glob patterns to actual files
+                file_patterns = []
+                for path in query_paths:
+                    if "*" in path:
+                        from pathlib import Path as P
+
+                        matching_files = list(P(self.catalog_path).glob(path.replace(str(self.catalog_path) + "/", "")))
+                        file_patterns.extend([str(f) for f in matching_files])
+                    else:
+                        file_patterns.append(path)
+
+                if not file_patterns:
+                    results.append(
+                        QueryProfileResult(
+                            engine="rustac",
+                            query_type=query_name,
+                            iteration=i,
+                            duration_ms=0,
+                            rows_returned=0,
+                            partitions_scanned=len(partitions),
+                            error="no matching files",
+                        )
+                    )
+                    continue
+
+                start = time.perf_counter()
+
+                # Use rustac to read items (rustac.read is async)
+                import asyncio
+
+                async def count_items_async(paths: list[str]) -> int:
+                    count = 0
+                    for file_path in paths:
+                        if Path(file_path).exists():
+                            items = await rustac.read(file_path)
+                            count += len(items)
+                    return count
+
+                total_items = asyncio.run(count_items_async(file_patterns))
+
+                duration_ms = (time.perf_counter() - start) * 1000
+
+                results.append(
+                    QueryProfileResult(
+                        engine="rustac",
+                        query_type=query_name,
+                        iteration=i,
+                        duration_ms=duration_ms,
+                        rows_returned=total_items,
+                        partitions_scanned=len(partitions),
+                    )
+                )
+
+            except (ValueError, TypeError, AttributeError, OSError) as e:
+                results.append(
+                    QueryProfileResult(
+                        engine="rustac",
+                        query_type=query_name,
+                        iteration=i,
+                        duration_ms=0,
+                        rows_returned=0,
+                        partitions_scanned=0,
+                        error=str(e),
+                    )
+                )
+
+        return results
+
+    def run_all_profiles(self, engines: list[str] | None = None) -> list[QueryProfileResult]:
+        """Run all profiling queries across specified engines.
+
+        Args:
+            engines: List of engines to profile. Options: 'spatial_resolver', 'duckdb', 'rustac'
+        """
+        if engines is None:
+            engines = ["spatial_resolver", "duckdb", "rustac"]
+
+        all_results = []
+        test_geometries = self._get_test_geometries()
+
+        for query_name, geometry in test_geometries.items():
+            for engine in engines:
+                if engine == "spatial_resolver":
+                    results = self.profile_spatial_resolver(geometry, query_name)
+                elif engine == "duckdb":
+                    results = self.profile_duckdb(geometry, query_name)
+                elif engine == "rustac":
+                    results = self.profile_rustac(geometry, query_name)
+                else:
+                    continue
+
+                all_results.extend(results)
+
+        self.results = all_results
+        return all_results
+
+    def get_summary(self) -> list[ProfileSummary]:
+        """Calculate summary statistics for profiling results."""
+        import statistics
+        from collections import defaultdict
+
+        # Group by engine and query type
+        groups: dict[tuple[str, str], list[QueryProfileResult]] = defaultdict(list)
+        for r in self.results:
+            if r.error is None:
+                groups[(r.engine, r.query_type)].append(r)
+
+        summaries = []
+        for (engine, query_type), results in groups.items():
+            if not results:
+                continue
+
+            durations = [r.duration_ms for r in results]
+            summaries.append(
+                ProfileSummary(
+                    engine=engine,
+                    query_type=query_type,
+                    iterations=len(results),
+                    min_ms=min(durations),
+                    max_ms=max(durations),
+                    mean_ms=statistics.mean(durations),
+                    median_ms=statistics.median(durations),
+                    std_ms=statistics.stdev(durations) if len(durations) > 1 else 0,
+                    total_rows=sum(r.rows_returned for r in results),
+                    partitions_scanned=results[0].partitions_scanned if results else 0,
+                )
+            )
+
+        return summaries
+
+    def print_report(self):
+        """Print a formatted profiling report."""
+        summaries = self.get_summary()
+
+        print("\n" + "=" * 80)
+        print("QUERY PERFORMANCE PROFILING REPORT")
+        print("=" * 80)
+
+        # Group by query type
+        from collections import defaultdict
+
+        by_query: dict[str, list[ProfileSummary]] = defaultdict(list)
+        for s in summaries:
+            by_query[s.query_type].append(s)
+
+        for query_type, engine_summaries in sorted(by_query.items()):
+            print(f"\n{query_type}:")
+            print("-" * 60)
+            print(f"{'Engine':<20} {'Mean (ms)':<12} {'Median (ms)':<12} {'Std (ms)':<10} {'Partitions':<10}")
+            print("-" * 60)
+
+            for s in sorted(engine_summaries, key=lambda x: x.mean_ms):
+                print(
+                    f"{s.engine:<20} {s.mean_ms:<12.2f} {s.median_ms:<12.2f} {s.std_ms:<10.2f} {s.partitions_scanned:<10}"
+                )
+
+        print("\n" + "=" * 80)
+
+
+@pytest.fixture(scope="module")
+def grid_config(request):
+    """Get grid configuration from command line options."""
+
+    class GridConfig:
+        grid_system = request.config.getoption("--e2e-grid", "h3")
+        grid_level = int(request.config.getoption("--e2e-grid-level", 2))
+        temporal_bin = request.config.getoption("--e2e-temporal", "month")
+
+    return GridConfig()
+
+
+@pytest.fixture(scope="module")
+def profile_config(request):
+    """Get profiling configuration from command line options."""
+
+    class ProfileConfig:
+        enabled = request.config.getoption("--e2e-profile-queries", False)
+        iterations = int(request.config.getoption("--e2e-query-iterations", 10))
+        engines = request.config.getoption("--e2e-query-engines", "duckdb,rustac").split(",")
+
+    return ProfileConfig()
+
+
+class TestQueryPerformance:
+    """Tests for query performance profiling with configurable grid systems."""
+
+    @pytest.mark.e2e
+    def test_configurable_grid_pipeline(self, temp_test_dir, e2e_config, grid_config):
+        """Test pipeline with configurable grid system and resolution.
+
+        Run with:
+            pytest -m e2e -k "test_configurable_grid" --e2e-grid=h3 --e2e-grid-level=4 --e2e-temporal=year
+        """
+        items_dir = temp_test_dir / "grid_config_items"
+        items_dir.mkdir(parents=True, exist_ok=True)
+
+        generator = SyntheticSTACGenerator(
+            output_dir=items_dir,
+            base_geometry=LANDSAT_SCENE,
+            outlier_tiny_percent=e2e_config.outlier_tiny_percent,
+            outlier_huge_percent=e2e_config.outlier_huge_percent,
+            seed=e2e_config.seed,
+        )
+
+        items = generator.generate_items(n_items=e2e_config.n_items, shuffle=True)
+        generator.write_items_to_files(items)
+
+        server = SimpleSTACServer(items_dir)
+        base_url = server.start()
+
+        try:
+            parquet_path = temp_test_dir / "grid_config_urls.parquet"
+            generator.create_url_parquet(base_url, parquet_path, shuffle=True)
+
+            output_catalog = temp_test_dir / "catalog_grid_config"
+            scratch_location = temp_test_dir / "scratch_grid_config"
+
+            config = ProcessingConfig(
+                input_file=str(parquet_path),
+                output_catalog=str(output_catalog),
+                scratch_location=str(scratch_location),
+                grid_system=grid_config.grid_system,
+                grid_resolution=grid_config.grid_level,
+                temporal_bin=grid_config.temporal_bin,
+                generate_schema=True,
+                enable_concurrent_http=True,
+                concurrent_requests=10,
+                batch_size=50,
+                max_workers=2,
+            )
+
+            processor = LocalProcessor(n_workers=config.max_workers)
+            pipeline = STACIngestionPipeline(config, processor)
+
+            try:
+                stats = pipeline.run()
+
+                assert len(stats) > 0, "Pipeline should produce partitions"
+                total_items = sum(s["total_items"] for s in stats.values())
+                assert total_items > 0, "Pipeline should ingest items"
+
+                # Verify schema reflects configuration
+                schema_file = output_catalog / "catalog_schema.json"
+                assert schema_file.exists()
+
+                with open(schema_file) as f:
+                    schema = json.load(f)
+                    assert schema["spatial_partitioning"]["grid_system"] == grid_config.grid_system
+                    assert schema["temporal_partitioning"]["temporal_bin"] == grid_config.temporal_bin
+
+                print("\nGrid Config Test Results:")
+                print(f"  Grid System: {grid_config.grid_system}")
+                print(f"  Grid Level: {grid_config.grid_level}")
+                print(f"  Temporal Bin: {grid_config.temporal_bin}")
+                print(f"  Partitions Created: {len(stats)}")
+                print(f"  Items Ingested: {total_items}")
+
+            finally:
+                processor.close()
+
+        finally:
+            server.stop()
+
+    @pytest.mark.e2e
+    def test_query_performance_profiling(self, temp_test_dir, e2e_config, grid_config, profile_config):
+        """Profile query performance across different engines.
+
+        Run with:
+            pytest -m e2e -k "test_query_performance" \\
+                --e2e-items=500 \\
+                --e2e-grid=h3 --e2e-grid-level=2 \\
+                --e2e-profile-queries \\
+                --e2e-query-iterations=5 \\
+                --e2e-query-engines=duckdb,rustac
+        """
+        if not profile_config.enabled:
+            pytest.skip("Query profiling not enabled. Use --e2e-profile-queries to enable.")
+
+        # First, create a catalog with enough items for meaningful profiling
+        items_dir = temp_test_dir / "profile_items"
+        items_dir.mkdir(parents=True, exist_ok=True)
+
+        generator = SyntheticSTACGenerator(
+            output_dir=items_dir,
+            base_geometry=LANDSAT_SCENE,
+            outlier_tiny_percent=e2e_config.outlier_tiny_percent,
+            outlier_huge_percent=e2e_config.outlier_huge_percent,
+            seed=e2e_config.seed,
+        )
+
+        items = generator.generate_items(n_items=e2e_config.n_items, shuffle=True)
+        generator.write_items_to_files(items)
+
+        server = SimpleSTACServer(items_dir)
+        base_url = server.start()
+
+        try:
+            parquet_path = temp_test_dir / "profile_urls.parquet"
+            generator.create_url_parquet(base_url, parquet_path, shuffle=True)
+
+            output_catalog = temp_test_dir / "catalog_profile"
+            scratch_location = temp_test_dir / "scratch_profile"
+
+            config = ProcessingConfig(
+                input_file=str(parquet_path),
+                output_catalog=str(output_catalog),
+                scratch_location=str(scratch_location),
+                grid_system=grid_config.grid_system,
+                grid_resolution=grid_config.grid_level,
+                temporal_bin=grid_config.temporal_bin,
+                generate_schema=True,
+                enable_concurrent_http=True,
+                concurrent_requests=20,
+                batch_size=100,
+                max_workers=4,
+            )
+
+            processor = LocalProcessor(n_workers=config.max_workers)
+            pipeline = STACIngestionPipeline(config, processor)
+
+            try:
+                print(f"\n{'=' * 60}")
+                print("INGESTING CATALOG FOR PROFILING")
+                print(f"{'=' * 60}")
+                stats = pipeline.run()
+
+                total_items = sum(s["total_items"] for s in stats.values())
+                print(f"\nCatalog created: {len(stats)} partitions, {total_items} items")
+
+            finally:
+                processor.close()
+
+            # Run query profiling
+            schema_path = output_catalog / "catalog_schema.json"
+
+            profiler = QueryProfiler(
+                catalog_path=output_catalog,
+                schema_path=schema_path,
+                iterations=profile_config.iterations,
+            )
+
+            print(f"\n{'=' * 60}")
+            print("RUNNING QUERY PERFORMANCE PROFILING")
+            print(f"{'=' * 60}")
+            print(f"Engines: {profile_config.engines}")
+            print(f"Iterations per query: {profile_config.iterations}")
+
+            # Add spatial_resolver to engines for comparison
+            engines = ["spatial_resolver"] + profile_config.engines
+            profiler.run_all_profiles(engines=engines)
+
+            # Print report
+            profiler.print_report()
+
+            # Verify we got results
+            assert len(profiler.results) > 0, "Should have profiling results"
+
+            # Check for errors
+            errors = [r for r in profiler.results if r.error is not None]
+            if errors:
+                print(f"\nWarning: {len(errors)} queries had errors:")
+                for e in errors[:5]:  # Show first 5 errors
+                    print(f"  {e.engine}/{e.query_type}: {e.error}")
+
+        finally:
+            server.stop()
+
+    @pytest.mark.e2e
+    def test_temporal_query_performance(self, temp_test_dir, e2e_config, grid_config, profile_config):
+        """Profile query performance with temporal filtering.
+
+        Run with:
+            pytest -m e2e -k "test_temporal_query" \\
+                --e2e-items=200 \\
+                --e2e-temporal=month \\
+                --e2e-profile-queries
+        """
+        if not profile_config.enabled:
+            pytest.skip("Query profiling not enabled. Use --e2e-profile-queries to enable.")
+
+        items_dir = temp_test_dir / "temporal_profile_items"
+        items_dir.mkdir(parents=True, exist_ok=True)
+
+        generator = SyntheticSTACGenerator(
+            output_dir=items_dir,
+            base_geometry=LANDSAT_SCENE,
+            seed=e2e_config.seed,
+        )
+
+        items = generator.generate_items(n_items=e2e_config.n_items, shuffle=True)
+        generator.write_items_to_files(items)
+
+        server = SimpleSTACServer(items_dir)
+        base_url = server.start()
+
+        try:
+            parquet_path = temp_test_dir / "temporal_profile_urls.parquet"
+            generator.create_url_parquet(base_url, parquet_path, shuffle=True)
+
+            output_catalog = temp_test_dir / "catalog_temporal_profile"
+            scratch_location = temp_test_dir / "scratch_temporal_profile"
+
+            config = ProcessingConfig(
+                input_file=str(parquet_path),
+                output_catalog=str(output_catalog),
+                scratch_location=str(scratch_location),
+                grid_system=grid_config.grid_system,
+                grid_resolution=grid_config.grid_level,
+                temporal_bin=grid_config.temporal_bin,
+                generate_schema=True,
+                enable_concurrent_http=True,
+                max_workers=2,
+            )
+
+            processor = LocalProcessor(n_workers=config.max_workers)
+            pipeline = STACIngestionPipeline(config, processor)
+
+            try:
+                stats = pipeline.run()
+                total_items = sum(s["total_items"] for s in stats.values())
+                print(f"\nCatalog created: {len(stats)} partitions, {total_items} items")
+            finally:
+                processor.close()
+
+            # Profile with different temporal filters
+            from shapely.geometry import box
+
+            test_geometry = box(-110.0, 35.0, -100.0, 45.0)
+            schema_path = output_catalog / "catalog_schema.json"
+
+            from earthcatalog.spatial_resolver import spatial_resolver
+
+            resolver = spatial_resolver(str(schema_path), str(output_catalog))
+
+            temporal_filters = [
+                None,  # No filter
+                "year=2024/*",  # Specific year
+                "year=202*/*",  # Multiple years
+                "year=2024/month=0*/*",  # First half of year
+            ]
+
+            print(f"\n{'=' * 60}")
+            print("TEMPORAL FILTERING PERFORMANCE")
+            print(f"{'=' * 60}")
+
+            import time
+
+            for tf in temporal_filters:
+                start = time.perf_counter()
+                partitions = resolver.resolve_partitions(test_geometry, overlap=True)
+                paths = resolver.generate_query_paths(partitions, tf)
+                duration_ms = (time.perf_counter() - start) * 1000
+
+                print(f"\nFilter: {tf or 'None'}")
+                print(f"  Partitions: {len(partitions)}")
+                print(f"  Paths generated: {len(paths)}")
+                print(f"  Duration: {duration_ms:.2f} ms")
+
+        finally:
+            server.stop()
+
+
+if __name__ == "__main__":
+    # Allow running tests directly
+    pytest.main([__file__, "-v", "-m", "e2e"])