earthcatalog 0.2.0__py3-none-any.whl

earthcatalog/ingestion_pipeline.py

@@ -0,0 +1,2281 @@
+# ingestion_pipeline.py
+"""High-performance distributed STAC ingestion pipeline for massive geospatial dataset processing.
+
+This module provides EarthCatalog's core ingestion capabilities, transforming URL lists
+into spatially-partitioned, query-optimized GeoParquet catalogs. Designed for processing
+100M+ STAC items efficiently across distributed computing environments with intelligent
+partitioning, concurrent HTTP processing, and adaptive resource management.
+
+Architecture Overview:
+    The pipeline follows a multi-stage processing architecture:
+    1. URL Reading: Flexible input from Parquet, CSV, or TSV files
+    2. Batch Processing: URLs chunked into worker-manageable batches
+    3. Concurrent Download: High-performance HTTP with connection pooling
+    4. Spatial Partitioning: Grid-based organization for efficient queries
+    5. Temporal Binning: Time-based organization (year/month/day)
+    6. Consolidated Output: Optimized GeoParquet files with spatial indexing
+
+Key Components:
+    ProcessingConfig: Comprehensive configuration with production defaults
+    STACIngestionPipeline: Main pipeline orchestrating the complete workflow
+    LocalProcessor: Single-machine processing with async HTTP capabilities
+    DaskDistributedProcessor: Cluster-based processing for massive scale
+    DistributedProcessor: Abstract base for pluggable processing backends
+
+Performance Features:
+    - Async HTTP processing: 3-6x faster than sequential downloads
+    - Concurrent request processing: 50-100+ requests/second per worker
+    - Memory-efficient batching: Handles unlimited URL counts
+    - Adaptive resource management: Scales from laptops to clusters
+    - Intelligent retry strategies: Robust error handling and recovery
+    - Connection pooling: Optimized network resource usage
+
+Scalability:
+    Tested Performance Metrics:
+    - 100M URLs: ~7-14 hours with 32 workers (vs 71 hours sequential)
+    - Memory usage: <20% per worker with 16GB RAM
+    - Throughput: 50-100 STAC items/second per worker
+    - Storage: Linear scaling with intelligent partitioning
+
+Grid System Integration:
+    Supports all major spatial partitioning systems:
+    - H3: Recommended for most global applications
+    - S2: Optimal for polar regions and spherical accuracy
+    - UTM: High precision for regional datasets
+    - MGRS: Standard for government and defense applications
+    - Custom GeoJSON: Maximum flexibility for special use cases
+
+Storage Backend Support:
+    - Local filesystem: Development and small-scale production
+    - S3: AWS cloud storage with optimized multipart uploads
+    - GCS: Google Cloud Storage via fsspec integration
+    - Azure: Azure Blob Storage support
+    - Custom backends: Extensible storage abstraction
+
+Temporal Organization:
+    Flexible time-based partitioning:
+    - Year-level: Long-term climate datasets
+    - Month-level: Recommended for most time-series data
+    - Day-level: High-frequency monitoring applications
+    - Custom binning: Configurable temporal windows
+
+Processing Modes:
+    LocalProcessor:
+        - Single machine processing with async HTTP
+        - Ideal for development and small-to-medium datasets
+        - Memory efficient with configurable batch sizes
+        - Built-in progress tracking and error reporting
+
+    DaskDistributedProcessor:
+        - Cluster-based processing for massive scale
+        - Automatic task distribution and load balancing
+        - Fault tolerance and automatic recovery
+        - Resource monitoring and adaptive scheduling
+
+Example Usage:
+    >>> # Basic local processing
+    >>> config = ProcessingConfig(
+    ...     input_file='urls.parquet',
+    ...     output_catalog='./catalog',
+    ...     scratch_location='./scratch'
+    ... )
+    >>> pipeline = STACIngestionPipeline(config, LocalProcessor())
+    >>> pipeline.run()
+    >>>
+    >>> # High-performance distributed processing
+    >>> config = ProcessingConfig(
+    ...     input_file='s3://bucket/urls.parquet',
+    ...     output_catalog='s3://bucket/catalog',
+    ...     scratch_location='s3://bucket/scratch',
+    ...     concurrent_requests=100,  # High concurrency
+    ...     max_workers=32           # Multiple workers
+    ... )
+    >>> processor = DaskDistributedProcessor('scheduler-address:8786')
+    >>> pipeline = STACIngestionPipeline(config, processor)
+    >>> pipeline.run()
+
+Configuration Best Practices:
+    - Use H3 grid system for global datasets
+    - Set concurrent_requests=50-100 for cloud storage
+    - Configure batch_size=1000-5000 based on available memory
+    - Use month-level temporal binning for time-series data
+    - Enable global partitioning for datasets with large geometries
+
+Error Handling:
+    - Comprehensive retry strategies with exponential backoff
+    - Failed URL logging with detailed error categorization
+    - Graceful degradation for network and server issues
+    - Progress tracking with automatic resumption capabilities
+    - Memory pressure monitoring and adaptive batch sizing
+"""
+
+import asyncio
+import glob as glob_module
+import json
+import logging
+import tempfile
+import time
+import uuid
+from abc import ABC, abstractmethod
+from collections import defaultdict
+from collections.abc import Callable
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, cast
+
+import fsspec
+import geopandas as gpd
+import pandas as pd
+import requests
+from tqdm import tqdm
+
+# Import STAC engine abstraction
+from .engines import STACEngine, get_engine
+
+# Conditional async HTTP imports
+try:
+    from .async_http_client import HAS_ASYNC_HTTP, download_stac_items_async
+except ImportError:
+    HAS_ASYNC_HTTP = False
+
+    # Create a dummy async function for type checking
+    async def download_stac_items_async(*args, **kwargs):  # type: ignore
+        """Dummy async function for type checking when async HTTP is not available."""
+        raise ImportError("Async HTTP client not available")
+
+
+# Import from package modules
+from . import grid_systems, input_readers, schema_generator, storage_backends
+from .job_tracking import JobLogger, JobManifest, JobStatus
+from .statistics import IngestionStatistics
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class ProcessingConfig:
+    """Comprehensive configuration for EarthCatalog's STAC ingestion pipeline with production defaults.
+
+    This dataclass provides all configuration options for controlling pipeline behavior,
+    from basic input/output paths to advanced performance tuning. Designed with sensible
+    defaults that work well for most use cases while allowing fine-grained control for
+    specialized requirements and performance optimization.
+
+    Configuration Categories:
+        Input/Output: File paths and formats for data sources and destinations
+        Spatial: Grid system selection and partitioning parameters
+        Temporal: Time-based binning and organization
+        Async HTTP: High-performance concurrent request processing (3-6x speedup)
+        Performance: Concurrency, memory management, and optimization
+        Processing: Worker configuration and distributed computing
+        Storage: Backend selection and cloud storage settings
+
+    Async HTTP Performance Configuration:
+        EarthCatalog includes built-in async HTTP capabilities that provide 3-6x performance
+        improvements over sequential processing. Key async parameters:
+
+        - enable_concurrent_http (bool, default=True): Enable async HTTP processing
+        - concurrent_requests (int, default=50): Simultaneous HTTP requests per worker
+        - batch_size (int, default=1000): URLs processed in each batch
+        - connection_pool_size (int, default=100): HTTP connection pool size
+        - request_timeout (int, default=30): Individual request timeout in seconds
+        - retry_attempts (int, default=3): Maximum retry attempts per request
+        - retry_delay (float, default=1.0): Base delay between retries in seconds
+
+    Performance Tuning Guidelines:
+        Development/Testing:
+            - concurrent_requests=10-25, batch_size=100-500
+            - Lower resource usage, easier debugging
+
+        Production/Fast Networks:
+            - concurrent_requests=50-100, batch_size=1000-2000
+            - Maximum throughput for well-provisioned systems
+
+        Unreliable Networks:
+            - concurrent_requests=10-25, request_timeout=60-120
+            - Higher reliability with longer timeouts
+
+        Memory Constrained:
+            - batch_size=100-500, connection_pool_size=25-50
+            - Reduced memory footprint
+
+    Cloud Storage Optimization:
+        Special considerations for cloud deployments:
+        - Use s3_multipart_threshold_mb for large file handling
+        - Configure temp_dir_location for local staging
+        - Set appropriate timeouts for network latency
+        - Enable streaming_merge for memory efficiency
+        - Higher concurrent_requests work well with cloud storage
+
+    Configuration Examples:
+
+        Basic Configuration (3-6x speedup with defaults):
+            >>> config = ProcessingConfig(
+            ...     input_file='urls.parquet',
+            ...     output_catalog='./catalog',
+            ...     scratch_location='./scratch'
+            ...     # async HTTP enabled by default
+            ... )
+
+        High-Performance Cloud Configuration:
+            >>> config = ProcessingConfig(
+            ...     input_file='s3://bucket/urls.parquet',
+            ...     output_catalog='s3://bucket/catalog',
+            ...     scratch_location='s3://bucket/scratch',
+            ...     # Async HTTP tuning for cloud scale
+            ...     enable_concurrent_http=True,
+            ...     concurrent_requests=100,
+            ...     batch_size=2000,
+            ...     connection_pool_size=200,
+            ...     request_timeout=30,
+            ...     # Worker and processing settings
+            ...     max_workers=32,
+            ...     items_per_shard=20000
+            ... )
+
+        Conservative/Reliable Configuration:
+            >>> config = ProcessingConfig(
+            ...     input_file='urls.parquet',
+            ...     output_catalog='./catalog',
+            ...     scratch_location='./scratch',
+            ...     # Conservative async settings
+            ...     concurrent_requests=25,
+            ...     request_timeout=60,
+            ...     retry_attempts=5,
+            ...     retry_delay=2.0,
+            ...     batch_size=500
+            ... )
+
+        Development/Debug Configuration:
+            >>> config = ProcessingConfig(
+            ...     input_file='sample_urls.parquet',
+            ...     output_catalog='./test_catalog',
+            ...     scratch_location='./test_scratch',
+            ...     # Disable async for easier debugging
+            ...     enable_concurrent_http=False,
+            ...     max_workers=1,
+            ...     verbose=True
+            ... )
+
+    Performance Monitoring:
+        Use these settings to monitor and optimize performance:
+        >>> config = ProcessingConfig(
+        ...     stats_file='processing_stats.json',  # Performance metrics
+        ...     verbose=True,                        # Detailed logging
+        ...     progress_interval=100               # Progress updates
+        ... )
+
+    Validation:
+        Configuration validation ensures proper setup:
+        >>> config.validate()  # Raises ValueError for invalid settings
+        >>> # Automatically validates paths, dependencies, and parameter ranges
+        >>> config.validate()  # Raises exceptions for invalid settings
+    """
+
+    input_file: str
+    output_catalog: str
+    scratch_location: str
+    input_format: str = "auto"  # auto, parquet, csv, tsv, ndjson, jsonl
+    url_column: str = "url"  # Column name containing URLs
+    # Multi-file input support (glob patterns)
+    # If specified, input_file is treated as a base path and input_pattern is used for file discovery
+    # Example: "s3://bucket/bulk/2020_*.ndjson" will discover all matching files
+    input_pattern: str = ""  # Glob pattern for multi-file input
+    grid_system: str = "h3"  # h3 only for simplicity
+    grid_resolution: int = 2  # H3 resolution (default level 2)
+    temporal_bin: str = "month"  # year, month, day
+
+    # Output format options
+    output_format: str = "geoparquet"  # "geoparquet" or "ndjson"
+    mission_field: str = "dataset_id"  # Field to extract mission from
+    sort_key: str = "datetime"
+    sort_ascending: bool = True
+    items_per_shard: int = 10000
+    max_workers: int = 8
+    # Global partitioning options
+    enable_global_partitioning: bool = True  # Route multi-cell items to /global/
+    global_partition_threshold: int = 50  # H3 resolution 6 threshold
+    # Schema generation options
+    generate_schema: bool = True  # Generate catalog schema metadata (default: enabled)
+    schema_filename: str = "catalog_schema.json"  # Schema output filename
+    geojson_path: str = ""  # Path to custom GeoJSON tiles (for geojson grid system)
+    # Consolidation options
+    max_memory_per_partition_mb: int = 1024  # Memory limit per partition
+    enable_streaming_merge: bool = True  # Use streaming for large files
+    s3_multipart_threshold_mb: int = 100  # When to use multipart upload
+    temp_dir_location: str = tempfile.gettempdir()  # Local temp space for staging
+
+    # Async HTTP Configuration
+    enable_concurrent_http: bool = True  # Enable async HTTP processing
+    concurrent_requests: int = 50  # Concurrent requests per worker
+    connection_pool_size: int = 100  # HTTP connection pool size
+    request_timeout: int = 30  # Request timeout in seconds
+    retry_attempts: int = 3  # Max retry attempts
+    retry_delay: float = 1.0  # Base retry delay in seconds
+    batch_size: int = 1000  # URLs processed per async batch
+
+    # Validation Configuration
+    enable_validation: bool = True  # Enable STAC item validation on ingest
+    fix_invalid_geometry: bool = True  # Attempt to fix invalid geometries
+    fix_bbox_mismatch: bool = True  # Correct bbox when it doesn't match geometry
+    bbox_tolerance: float = 1e-6  # Tolerance for bbox comparison (degrees)
+    log_validation_warnings: bool = True  # Log validation warnings
+
+    # STAC Engine Configuration
+    stac_engine: str = "rustac"  # "rustac", "stac-geoparquet", or "auto"
+
+    # STAC Fetch Hook Configuration
+    # Allows custom STAC item generation when URLs don't point to STAC JSON
+    # Supported formats:
+    #   - "default": Standard STAC JSON fetch (default behavior)
+    #   - "module:package.module:function": Python function import path
+    #   - "script:/path/to/script": External executable
+    #   - "script:python:/path/to/script.py": Script with interpreter
+    stac_hook: str = "default"
+
+    # Batch mode configuration
+    # Controls when to use simple local processing vs full distributed processing
+    batch_threshold: int = 10000  # Below this, use simple local processing
+    distributed: bool | None = None  # True=force distributed, False=force local, None=auto
+    large_batch_confirm_threshold: int = 20000  # Prompt user if local mode exceeds this
+
+    # Dask distributed configuration
+    dask_scheduler_address: str = (
+        ""  # Dask scheduler address (e.g., 'tcp://localhost:8786'). Empty = create local cluster
+    )
+
+    # Checkpoint configuration for distributed processing
+    # Use time-based checkpointing for better performance with many partitions
+    checkpoint_interval_seconds: int = 30  # Save checkpoint every N seconds (0 to disable)
+    checkpoint_partition_count: int = 0  # Save checkpoint every N partitions (0 to disable)
+    # Note: If both are 0, defaults to time-based (30 seconds)
+
+    def validate(self):
+        """Validate configuration before processing."""
+        # Validate input source - either input_file or input_pattern
+        if not self.input_pattern:
+            # Traditional single file mode - validate file exists
+            if not Path(self.input_file).exists() and not self.input_file.startswith("s3://"):
+                raise FileNotFoundError(f"Input file not found: {self.input_file}")
+        # If input_pattern is provided, file existence will be validated during pattern resolution
+        if self.grid_resolution < 0 or self.grid_resolution > 15:
+            raise ValueError("H3 resolution must be 0-15")
+        if self.temporal_bin not in ["year", "month", "day"]:
+            raise ValueError("temporal_bin must be 'year', 'month', or 'day'")
+        if self.output_format not in ["geoparquet", "ndjson"]:
+            raise ValueError("output_format must be 'geoparquet' or 'ndjson'")
+        if self.items_per_shard <= 0:
+            raise ValueError("items_per_shard must be positive")
+        if self.max_workers <= 0:
+            raise ValueError("max_workers must be positive")
+        if self.max_memory_per_partition_mb <= 0:
+            raise ValueError("max_memory_per_partition_mb must be positive")
+        if self.s3_multipart_threshold_mb <= 0:
+            raise ValueError("s3_multipart_threshold_mb must be positive")
+
+        # Async HTTP validation
+        if self.concurrent_requests <= 0:
+            raise ValueError("concurrent_requests must be positive")
+        if self.connection_pool_size <= 0:
+            raise ValueError("connection_pool_size must be positive")
+        if self.request_timeout <= 0:
+            raise ValueError("request_timeout must be positive")
+        if self.retry_attempts < 0:
+            raise ValueError("retry_attempts must be non-negative")
+        if self.retry_delay < 0:
+            raise ValueError("retry_delay must be non-negative")
+        if self.batch_size <= 0:
+            raise ValueError("batch_size must be positive")
+
+        # STAC engine validation
+        valid_engines = ("rustac", "stac-geoparquet", "auto")
+        if self.stac_engine not in valid_engines:
+            raise ValueError(f"stac_engine must be one of {valid_engines}, got: {self.stac_engine}")
+
+        # STAC hook validation
+        valid_hook_prefixes = ("default", "passthrough", "module:", "script:")
+        if not any(self.stac_hook.startswith(prefix) for prefix in valid_hook_prefixes):
+            raise ValueError(
+                f"stac_hook must be 'default', 'passthrough', 'module:path:func', or 'script:/path', got: {self.stac_hook}"
+            )
+
+        # Dask scheduler validation - warn if remote scheduler with local storage
+        if self.dask_scheduler_address and not self.dask_scheduler_address.startswith("local"):
+            # Remote scheduler detected
+            local_paths = []
+            if not self.scratch_location.startswith(("s3://", "gs://", "az://")):
+                local_paths.append("scratch_location")
+            if not self.output_catalog.startswith(("s3://", "gs://", "az://")):
+                local_paths.append("output_catalog")
+
+            if local_paths:
+                logger.warning(
+                    f"Using remote Dask scheduler ({self.dask_scheduler_address}) with local storage paths: "
+                    f"{', '.join(local_paths)}. "
+                    f"Remote workers may not have access to local paths. "
+                    f"Consider using cloud storage (s3://, gs://) for {', '.join(local_paths)}."
+                )
+
+        # Batch mode validation
+        if self.batch_threshold <= 0:
+            raise ValueError("batch_threshold must be positive")
+        if self.large_batch_confirm_threshold <= 0:
+            raise ValueError("large_batch_confirm_threshold must be positive")
+
+    def __repr__(self) -> str:
+        """Return concise string representation with key configuration."""
+        return (
+            f"ProcessingConfig("
+            f"input='{self.input_file}', "
+            f"output='{self.output_catalog}', "
+            f"grid={self.grid_system}@{self.grid_resolution}, "
+            f"workers={self.max_workers})"
+        )
+
+    def __bool__(self) -> bool:
+        """Return True if configuration is valid."""
+        try:
+            self.validate()
+            return True
+        except (ValueError, FileNotFoundError):
+            return False
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serialize configuration to dictionary for storage or transmission.
+
+        All fields are included, enabling complete reconstruction via from_dict().
+        Useful for:
+        - Storing config in job manifests for recovery
+        - Passing config to remote Dask workers
+        - Caching/resumption of processing jobs
+
+        Returns:
+            Dictionary with all configuration fields.
+        """
+        return {
+            "input_file": self.input_file,
+            "output_catalog": self.output_catalog,
+            "scratch_location": self.scratch_location,
+            "input_format": self.input_format,
+            "url_column": self.url_column,
+            "input_pattern": self.input_pattern,
+            "grid_system": self.grid_system,
+            "grid_resolution": self.grid_resolution,
+            "temporal_bin": self.temporal_bin,
+            "output_format": self.output_format,
+            "mission_field": self.mission_field,
+            "sort_key": self.sort_key,
+            "sort_ascending": self.sort_ascending,
+            "items_per_shard": self.items_per_shard,
+            "max_workers": self.max_workers,
+            "enable_global_partitioning": self.enable_global_partitioning,
+            "global_partition_threshold": self.global_partition_threshold,
+            "generate_schema": self.generate_schema,
+            "schema_filename": self.schema_filename,
+            "geojson_path": self.geojson_path,
+            "max_memory_per_partition_mb": self.max_memory_per_partition_mb,
+            "enable_streaming_merge": self.enable_streaming_merge,
+            "s3_multipart_threshold_mb": self.s3_multipart_threshold_mb,
+            "temp_dir_location": self.temp_dir_location,
+            "enable_concurrent_http": self.enable_concurrent_http,
+            "concurrent_requests": self.concurrent_requests,
+            "connection_pool_size": self.connection_pool_size,
+            "request_timeout": self.request_timeout,
+            "retry_attempts": self.retry_attempts,
+            "retry_delay": self.retry_delay,
+            "batch_size": self.batch_size,
+            "enable_validation": self.enable_validation,
+            "fix_invalid_geometry": self.fix_invalid_geometry,
+            "fix_bbox_mismatch": self.fix_bbox_mismatch,
+            "bbox_tolerance": self.bbox_tolerance,
+            "log_validation_warnings": self.log_validation_warnings,
+            "stac_engine": self.stac_engine,
+            "stac_hook": self.stac_hook,
+            "batch_threshold": self.batch_threshold,
+            "distributed": self.distributed,
+            "large_batch_confirm_threshold": self.large_batch_confirm_threshold,
+            "dask_scheduler_address": self.dask_scheduler_address,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "ProcessingConfig":
+        """Reconstruct configuration from dictionary.
+
+        Only includes fields that are present in the dictionary, allowing
+        for backward compatibility when new fields are added.
+
+        Args:
+            data: Dictionary with configuration fields.
+
+        Returns:
+            ProcessingConfig instance.
+        """
+        # Required fields
+        config = cls(
+            input_file=data["input_file"],
+            output_catalog=data["output_catalog"],
+            scratch_location=data["scratch_location"],
+        )
+
+        # Optional fields - update only if present in data
+        optional_fields = [
+            "input_format",
+            "url_column",
+            "input_pattern",
+            "grid_system",
+            "grid_resolution",
+            "temporal_bin",
+            "output_format",
+            "mission_field",
+            "sort_key",
+            "sort_ascending",
+            "items_per_shard",
+            "max_workers",
+            "enable_global_partitioning",
+            "global_partition_threshold",
+            "generate_schema",
+            "schema_filename",
+            "geojson_path",
+            "max_memory_per_partition_mb",
+            "enable_streaming_merge",
+            "s3_multipart_threshold_mb",
+            "temp_dir_location",
+            "enable_concurrent_http",
+            "concurrent_requests",
+            "connection_pool_size",
+            "request_timeout",
+            "retry_attempts",
+            "retry_delay",
+            "batch_size",
+            "enable_validation",
+            "fix_invalid_geometry",
+            "fix_bbox_mismatch",
+            "bbox_tolerance",
+            "log_validation_warnings",
+            "stac_engine",
+            "stac_hook",
+            "batch_threshold",
+            "distributed",
+            "large_batch_confirm_threshold",
+            "dask_scheduler_address",
+        ]
+
+        for field in optional_fields:
+            if field in data:
+                setattr(config, field, data[field])
+
+        return config
+
+    def config_hash(self) -> str:
+        """Generate a hash of configuration for idempotency checking.
+
+        The hash covers settings that affect output content, excluding
+        paths and runtime-only settings. Two configs with the same hash
+        would produce identical output given the same input.
+
+        Returns:
+            Hex digest of the configuration hash.
+        """
+        import hashlib
+
+        # Fields that affect output content (not paths or runtime settings)
+        content_affecting = {
+            "grid_system": self.grid_system,
+            "grid_resolution": self.grid_resolution,
+            "temporal_bin": self.temporal_bin,
+            "output_format": self.output_format,
+            "mission_field": self.mission_field,
+            "sort_key": self.sort_key,
+            "sort_ascending": self.sort_ascending,
+            "items_per_shard": self.items_per_shard,
+            "enable_global_partitioning": self.enable_global_partitioning,
+            "global_partition_threshold": self.global_partition_threshold,
+            "enable_validation": self.enable_validation,
+            "fix_invalid_geometry": self.fix_invalid_geometry,
+            "fix_bbox_mismatch": self.fix_bbox_mismatch,
+        }
+
+        content_str = json.dumps(content_affecting, sort_keys=True)
+        return hashlib.sha256(content_str.encode()).hexdigest()[:16]
+
+
+class DistributedProcessor(ABC):
+    """Abstract base for different parallel processing backends."""
+
+    @abstractmethod
+    def process_urls(self, url_chunks: list[list[str]], process_fn: Callable, **kwargs) -> list[Any]:
+        """Process URL chunks in parallel."""
+        pass
+
+    @abstractmethod
+    def consolidate_shards(self, partition_items: list[tuple], consolidate_fn: Callable, **kwargs) -> list[Any]:
+        """Consolidate shards in parallel."""
+        pass
+
+    @abstractmethod
+    def close(self):
+        """Clean up resources."""
+        pass
+
+
+class DaskDistributedProcessor(DistributedProcessor):
+    """Dask-based distributed processing using serializable worker functions.
+
+    This processor uses the module-level functions from workers.py which can be
+    pickled and sent to remote Dask workers. Unlike LocalProcessor, it cannot
+    use closures that capture instance state.
+
+    Supports two modes:
+    1. Local cluster: Creates a local Dask cluster (default)
+    2. Remote cluster: Connects to an existing Dask scheduler via scheduler_address
+    """
+
+    def __init__(self, n_workers: int = 8, threads_per_worker: int = 1, scheduler_address: str | None = None):
+        """Initialize Dask distributed processor with specified worker configuration.
+
+        Args:
+            n_workers: Number of workers (only used when creating local cluster)
+            threads_per_worker: Threads per worker (only used when creating local cluster)
+            scheduler_address: Optional Dask scheduler address (e.g., 'tcp://localhost:8786').
+                             If provided, connects to existing cluster instead of creating local one.
+        """
+        self.n_workers = n_workers
+        self.threads_per_worker = threads_per_worker
+        self.scheduler_address = scheduler_address
+        try:
+            import dask.distributed as dd
+
+            if scheduler_address:
+                # Connect to existing Dask cluster
+                self.client = dd.Client(scheduler_address)
+                logger.info(f"Connected to Dask scheduler at {scheduler_address}")
+                logger.info(f"Dask dashboard: {self.client.dashboard_link}")
+            else:
+                # Create local Dask cluster
+                self.client = dd.Client(n_workers=n_workers, threads_per_worker=threads_per_worker, memory_limit="4GB")
+                logger.info(f"Created local Dask cluster with {n_workers} workers")
+                logger.info(f"Dask dashboard: {self.client.dashboard_link}")
+        except ImportError:
+            raise ImportError("Dask distributed required: pip install dask distributed") from None
+
+    def __repr__(self) -> str:
+        """Return string representation."""
+        if self.scheduler_address:
+            return f"DaskDistributedProcessor(scheduler_address='{self.scheduler_address}')"
+        return f"DaskDistributedProcessor(n_workers={self.n_workers}, threads_per_worker={self.threads_per_worker})"
+
+    def close(self):
+        """Close the Dask client connection."""
+        if hasattr(self, "client") and self.client:
+            self.client.close()
+            logger.info("Dask client closed")
+
+    def process_urls(self, url_chunks: list[list[str]], process_fn: Callable, **kwargs) -> list[Any]:
+        """Process URL chunks using Dask distributed workers.
+
+        Note: For Dask, we use the serializable workers.process_url_batch function
+        instead of the provided process_fn (which may capture non-serializable state).
+        The process_fn is ignored and workers.process_url_batch is used instead.
+        """
+        # Import here to avoid circular imports
+        from .workers import process_url_batch
+
+        # Extract required kwargs
+        config_dict = kwargs.get("config_dict")
+        job_id = kwargs.get("job_id", "unknown")
+
+        if config_dict is None:
+            # Fall back to old behavior if config_dict not provided
+            futures = [self.client.submit(process_fn, chunk, idx, **kwargs) for idx, chunk in enumerate(url_chunks)]
+            return cast(list[Any], self.client.gather(futures))
+
+        # Use serializable worker function
+        futures = [
+            self.client.submit(
+                process_url_batch,
+                urls=chunk,
+                worker_id=idx,
+                config_dict=config_dict,
+                job_id=job_id,
+                batch_idx=idx,
+            )
+            for idx, chunk in enumerate(url_chunks)
+        ]
+        return cast(list[Any], self.client.gather(futures))
+
+    def consolidate_shards(self, partition_items: list[tuple], consolidate_fn: Callable, **kwargs) -> list[Any]:
+        """Consolidate shards using Dask distributed workers.
+
+        Note: For Dask, we use the serializable workers.consolidate_partition function
+        instead of the provided consolidate_fn.
+        """
+        from .workers import consolidate_partition
+
+        config_dict = kwargs.get("config_dict")
+
+        if config_dict is None:
+            # Fall back to old behavior
+            futures = [
+                self.client.submit(consolidate_fn, partition_key, shard_paths, **kwargs)
+                for partition_key, shard_paths in partition_items
+            ]
+            return cast(list[Any], self.client.gather(futures))
+
+        # Use serializable worker function
+        futures = [
+            self.client.submit(
+                consolidate_partition,
+                partition_key=partition_key,
+                shard_paths=shard_paths,
+                config_dict=config_dict,
+            )
+            for partition_key, shard_paths in partition_items
+        ]
+        return cast(list[Any], self.client.gather(futures))
+
+
+class LocalProcessor(DistributedProcessor):
+    """Local multi-threading processor using concurrent.futures."""
+
+    def __init__(self, n_workers: int = 8):
+        from concurrent.futures import ThreadPoolExecutor
+
+        self.n_workers = n_workers
+        self.executor = ThreadPoolExecutor(max_workers=n_workers)
+        logger.info(f"Local processor started with {n_workers} workers")
+
+    def __repr__(self) -> str:
+        """Return string representation."""
+        return f"LocalProcessor(n_workers={self.n_workers})"
+
+    def process_urls(self, url_chunks: list[list[str]], process_fn: Callable, **kwargs) -> list[Any]:
+        futures = [self.executor.submit(process_fn, chunk, idx, **kwargs) for idx, chunk in enumerate(url_chunks)]
+        return [f.result() for f in futures]
+
+    def consolidate_shards(self, partition_items: list[tuple], consolidate_fn: Callable, **kwargs) -> list[Any]:
+        futures = [
+            self.executor.submit(consolidate_fn, partition_key, shard_paths, **kwargs)
+            for partition_key, shard_paths in partition_items
+        ]
+        return [f.result() for f in futures]
+
+    def close(self):
+        """Shutdown the executor."""
+        self.executor.shutdown(wait=True)
+
+
+class STACIngestionPipeline:
+    """Enterprise-grade STAC ingestion pipeline for massive geospatial dataset processing.
+
+    This is the main orchestrator class that coordinates the complete STAC ingestion
+    workflow, from URL reading through spatial partitioning to optimized catalog
+    generation. Designed for production environments processing 100M+ STAC items
+    with high performance, reliability, and scalability.
+
+    Pipeline Architecture:
+        The pipeline implements a sophisticated multi-stage processing architecture:
+        1. Input Reading: Flexible support for Parquet, CSV, TSV with auto-detection
+        2. URL Chunking: Intelligent batching based on memory and processing constraints
+        3. Distributed Processing: Pluggable processors for local or cluster execution
+        4. Concurrent Download: High-performance async HTTP with connection pooling
+        5. Spatial Partitioning: Grid-based organization optimized for spatial queries
+        6. Temporal Binning: Time-based organization for efficient time-series queries
+        7. Consolidation: Memory-efficient merging with streaming for large datasets
+        8. Output Generation: Optimized GeoParquet with spatial indexing
+
+    Performance Features:
+        - Async HTTP: 3-6x faster downloads with 50-100+ concurrent requests
+        - Memory Management: Constant memory usage regardless of dataset size
+        - Incremental Processing: Resume interrupted jobs and update existing catalogs
+        - Error Recovery: Comprehensive retry strategies and graceful degradation
+        - Progress Tracking: Real-time monitoring with detailed logging
+        - Resource Optimization: Adaptive batching and memory pressure handling
+
+    Storage Integration:
+        - Local Filesystem: Development and small-scale production
+        - S3 Compatible: AWS S3, MinIO, with optimized multipart uploads
+        - Google Cloud Storage: Native integration via fsspec
+        - Azure Blob Storage: Full support for Azure cloud deployments
+        - Custom Storage: Extensible backend system for specialized requirements
+
+    Incremental Updates:
+        Advanced incremental processing capabilities:
+        - Automatic detection of existing partitions
+        - Efficient merging of new data with existing catalogs
+        - Temporal range updates without full reprocessing
+        - Safe atomic operations preventing catalog corruption
+        - Rollback capabilities for failed updates
+
+    Processing Modes:
+        LocalProcessor:
+            - Single-machine processing optimized for async HTTP
+            - Memory-efficient for datasets up to tens of millions of items
+            - Ideal for development, testing, and medium-scale production
+
+        DaskDistributedProcessor:
+            - Cluster-based processing for massive scale
+            - Fault tolerance with automatic task recovery
+            - Dynamic scaling based on cluster resources
+            - Optimized for 100M+ item datasets
+
+    Quality Assurance:
+        - Comprehensive input validation and error checking
+        - STAC specification compliance verification
+        - Spatial geometry validation and repair
+        - Temporal data consistency checking
+        - Output format validation and optimization
+
+    Example:
+        >>> # Basic pipeline setup
+        >>> config = ProcessingConfig(
+        ...     input_file='stac_urls.parquet',
+        ...     output_catalog='./catalog',
+        ...     scratch_location='./scratch'
+        ... )
+        >>> pipeline = STACIngestionPipeline(config, LocalProcessor())
+        >>>
+        >>> # Execute complete pipeline
+        >>> pipeline.run()  # Processes all URLs and creates catalog
+        >>>
+        >>> # Advanced configuration for large-scale processing
+        >>> config = ProcessingConfig(
+        ...     input_file='s3://data/urls.parquet',
+        ...     output_catalog='s3://catalog/output',
+        ...     scratch_location='s3://catalog/scratch',
+        ...     grid_system='h3',
+        ...     grid_resolution=6,
+        ...     concurrent_requests=100,
+        ...     batch_size=5000,
+        ...     max_workers=32
+        ... )
+        >>> processor = DaskDistributedProcessor('scheduler:8786')
+        >>> pipeline = STACIngestionPipeline(config, processor)
+        >>> pipeline.run()
+
+    Thread Safety:
+        The pipeline is designed for single-threaded execution within each worker
+        process. Multiple pipeline instances can run concurrently in separate
+        processes for distributed processing scenarios.
+
+    Monitoring:
+        Comprehensive monitoring and observability:
+        - Progress bars with ETA and throughput metrics
+        - Detailed logging of all processing stages
+        - Error categorization and reporting
+        - Performance metrics and bottleneck identification
+        - Memory usage tracking and optimization recommendations
+    """
+
+    def __init__(self, config: ProcessingConfig, processor: DistributedProcessor):
+        self.config = config
+        self.processor = processor
+        self.storage = self._init_storage(config.output_catalog)
+        self.scratch_storage = self._init_storage(config.scratch_location)
+        self.grid = grid_systems.get_grid_system(config.grid_system, config.grid_resolution)
+
+        # Initialize STAC engine for item conversion
+        self.engine: STACEngine = get_engine(config.stac_engine)  # type: ignore[arg-type]
+        logger.info(f"Using STAC engine: {self.engine.name}")
+
+        # Initialize statistics collector
+        self.stats = IngestionStatistics()
+
+    def __repr__(self) -> str:
+        """Return string representation with key pipeline state."""
+        return (
+            f"STACIngestionPipeline("
+            f"input='{self.config.input_file}', "
+            f"output='{self.config.output_catalog}', "
+            f"grid={self.config.grid_system}@{self.config.grid_resolution}, "
+            f"processor={self.processor.__class__.__name__})"
+        )
+
+    def _init_storage(self, path: str) -> storage_backends.StorageBackend:
+        """Initialize storage backend."""
+        if path.startswith("s3://"):
+            return storage_backends.S3Storage(path)
+        else:
+            return storage_backends.LocalStorage(path)
+
+    def _should_use_distributed(self, url_count: int) -> bool:
+        """Determine whether to use distributed processing based on URL count and config.
+
+        Processing mode selection logic:
+        1. If config.distributed is True: always use distributed
+        2. If config.distributed is False: always use simple local
+        3. If config.distributed is None (auto): use URL count threshold
+
+        Args:
+            url_count: Number of URLs to process.
+
+        Returns:
+            True if distributed processing should be used, False for simple local.
+        """
+        if self.config.distributed is True:
+            logger.info("Using distributed processing (explicitly enabled)")
+            return True
+        if self.config.distributed is False:
+            logger.info("Using simple local processing (explicitly disabled distributed)")
+            return False
+        # Auto mode: decide based on threshold
+        use_distributed = url_count >= self.config.batch_threshold
+        if use_distributed:
+            logger.info(
+                f"Using distributed processing: {url_count:,} URLs >= threshold {self.config.batch_threshold:,}"
+            )
+        else:
+            logger.info(
+                f"Using simple local processing: {url_count:,} URLs < threshold {self.config.batch_threshold:,}"
+            )
+        return use_distributed
+
+    def run(self, job_id: str | None = None, resume: bool = False) -> dict[str, Any]:
+        """Execute the complete ingestion pipeline with job tracking.
+
+        Args:
+            job_id: Optional job ID to use. If None, generates a new UUID.
+            resume: If True, attempts to resume an incomplete job. Ignores job_id
+                   and searches for the most recent incomplete job.
+
+        Returns:
+            Dictionary mapping partition keys to consolidation stats.
+
+        Raises:
+            ValueError: If resume=True but no incomplete job found.
+        """
+        # Handle resume case
+        if resume:
+            manifest = JobManifest.find_incomplete(self.storage, self.config.output_catalog)
+            if manifest is None:
+                raise ValueError("No incomplete job found to resume")
+            job_id = manifest.job_id
+            logger.info(f"Resuming job {job_id} from {manifest.status.value} phase")
+        elif job_id is None:
+            job_id = str(uuid.uuid4())
+
+        # Initialize job logger
+        job_logger = JobLogger(self.storage, self.config.output_catalog, job_id)
+
+        logger.info("=" * 80)
+        logger.info("Starting STAC ingestion pipeline")
+        logger.info(f"Job ID: {job_id}")
+        logger.info(f"Input: {self.config.input_file}")
+        logger.info(f"Output: {self.config.output_catalog}")
+        logger.info(f"Grid: {self.config.grid_system} (resolution {self.config.grid_resolution})")
+        logger.info(f"Temporal binning: {self.config.temporal_bin}")
+        logger.info("=" * 80)
+
+        # Start statistics tracking
+        self.stats.start_processing()
+
+        # Phase 1: Read input and distribute work
+        urls = self._read_input_urls()
+        logger.info(f"Loaded {len(urls):,} URLs from input file")
+
+        # Check if we should use simple mode (for small batches)
+        # Note: Resume always uses distributed mode
+        if not resume and not self._should_use_distributed(len(urls)):
+            # Check if local mode with large batch needs confirmation
+            if len(urls) > self.config.large_batch_confirm_threshold:
+                logger.warning(
+                    f"Processing {len(urls):,} URLs in local mode. "
+                    f"Consider using --distributed for batches > {self.config.large_batch_confirm_threshold:,}"
+                )
+            # job_id is guaranteed to be set at this point (either from param or uuid)
+            assert job_id is not None
+            return self._run_simple(urls, job_id)
+
+        # Create or load job manifest
+        if resume:
+            manifest = JobManifest.load(self.storage, self.config.output_catalog, job_id)
+        else:
+            manifest = JobManifest(
+                job_id=job_id,
+                input_urls_count=len(urls),
+                config_hash=self.config.config_hash(),
+            )
+
+        # Track shard info for consolidation
+        shard_info: list[dict[str, Any]] = []
+
+        try:
+            # Phase 2: Parallel processing to scratch location
+            with tqdm(total=4, desc="Pipeline Progress", unit="phase") as pipeline_pbar:
+                # Skip download phase if already completed (resume case)
+                if manifest.download_phase.completed:
+                    logger.info("Download phase already completed, skipping")
+                    shard_info = [
+                        {"shard_path": path, "item_count": 0, "partition_key": ""}
+                        for path in manifest.download_phase.shards_written
+                    ]
+                    pipeline_pbar.update(1)
+                else:
+                    pipeline_pbar.set_description("Processing URLs")
+                    manifest.status = JobStatus.DOWNLOADING
+                    manifest.save(self.storage, self.config.output_catalog)
+                    job_logger.log_phase_start("download")
+
+                    shard_info = self._process_urls_distributed(urls, job_id=job_id)
+                    total_shards = len(shard_info)
+                    total_items = sum(s["item_count"] for s in shard_info)
+                    logger.info(f"Generated {total_shards:,} shards ({total_items:,} items) in scratch space")
+
+                    # Update manifest with download phase completion
+                    manifest.download_phase.completed = True
+                    manifest.download_phase.shards_written = [s["shard_path"] for s in shard_info]
+                    manifest.download_phase.urls_processed = len(urls)
+                    manifest.save(self.storage, self.config.output_catalog)
+                    job_logger.log_phase_complete("download", {"shards": total_shards, "items": total_items})
+                    pipeline_pbar.update(1)
+
+                # Phase 3: Consolidate shards into final catalog
+                # Skip if already completed (resume case)
+                if manifest.consolidation_phase.completed:
+                    logger.info("Consolidation phase already completed, skipping")
+                    pipeline_pbar.update(1)
+                    final_stats = {}  # Will be re-computed from catalog
+                else:
+                    pipeline_pbar.set_description("Consolidating shards")
+                    manifest.status = JobStatus.CONSOLIDATING
+                    manifest.save(self.storage, self.config.output_catalog)
+                    job_logger.log_phase_start("consolidation")
+
+                    # Filter out already-completed partitions for resume
+                    completed_partitions = set(manifest.consolidation_phase.completed_partitions)
+                    final_stats = self._consolidate_shards(
+                        shard_info,
+                        skip_partitions=completed_partitions,
+                        manifest=manifest,
+                    )
+                    logger.info(f"Consolidated into {len(final_stats):,} final partitions")
+
+                    manifest.consolidation_phase.completed = True
+                    manifest.save(self.storage, self.config.output_catalog)
+                    job_logger.log_phase_complete("consolidation", {"partitions": len(final_stats)})
+                    pipeline_pbar.update(1)
+
+                # Phase 4: Cleanup scratch space
+                pipeline_pbar.set_description("Cleaning up")
+                self._cleanup_scratch(shard_info)
+                logger.info("Scratch space cleanup completed")
+                pipeline_pbar.update(1)
+
+            # Mark job as completed
+            manifest.status = JobStatus.COMPLETED
+            manifest.save(self.storage, self.config.output_catalog)
+            job_logger.log("INFO", "Job completed successfully")
+
+        except Exception as e:
+            # Mark job as failed
+            manifest.status = JobStatus.FAILED
+            manifest.error = str(e)
+            manifest.save(self.storage, self.config.output_catalog)
+            job_logger.log_error(f"Job failed: {e}")
+            raise
+
+        # Finish statistics tracking
+        self.stats.finish_processing()
+
+        # Update statistics with consolidation results
+        for _partition_key, stats in final_stats.items():
+            self.stats.record_consolidation(
+                new_items=stats.get("new_items", 0),
+                existing_items=stats.get("existing_items", 0),
+                duplicates_removed=stats.get("duplicates_removed", 0),
+            )
+
+        # Generate schema if requested
+        if self.config.generate_schema:
+            logger.info("Generating catalog schema metadata...")
+            schema_gen = schema_generator.SchemaGenerator(self.config, self.grid, self.storage, self.stats)
+            schema_gen.generate_catalog_schema(final_stats, self.config.schema_filename)
+
+        # Summary with enhanced statistics
+        stats_summary = self.stats.get_summary()
+        logger.info("=" * 80)
+        logger.info("PIPELINE COMPLETED SUCCESSFULLY")
+        logger.info(f"Job ID: {job_id}")
+        logger.info(f"Total partitions: {len(final_stats)}")
+        logger.info(f"Unique granules: {stats_summary['unique_granules']:,}")
+        logger.info(f"Stored references: {stats_summary['stored_references']:,}")
+        logger.info(
+            f"Overhead: {stats_summary['overhead']['overhead_percentage']:.1f}% "
+            f"({stats_summary['overhead']['spanning_items']:,} spanning items)"
+        )
+        total_new = sum(s["new_items"] for s in final_stats.values())
+        logger.info(f"New items: {total_new:,}")
+        if self.config.generate_schema:
+            logger.info(f"Schema metadata: {self.config.output_catalog}/{self.config.schema_filename}")
+        logger.info("=" * 80)
+
+        return final_stats
+
+    def _run_simple(self, urls: list[str], job_id: str) -> dict[str, Any]:
+        """Execute simplified pipeline for small batches without full distributed overhead.
+
+        This method provides a streamlined processing path for datasets below the
+        batch_threshold. It still:
+        - Uses async HTTP for performance
+        - Creates a job manifest for observability
+        - Uses the same partitioning and consolidation logic
+
+        But it skips:
+        - Resume/checkpoint capability (not needed for small batches)
+        - Complex worker distribution logic
+        - Multi-phase progress tracking
+
+        Args:
+            urls: List of URLs to process.
+            job_id: Job identifier for logging.
+
+        Returns:
+            Dictionary mapping partition keys to consolidation stats.
+        """
+        logger.info("=" * 80)
+        logger.info("Running simplified pipeline (small batch mode)")
+        logger.info(f"Job ID: {job_id}")
+        logger.info(f"URLs: {len(urls):,}")
+        logger.info("=" * 80)
+
+        # Start statistics tracking
+        self.stats.start_processing()
+
+        # Create minimal job manifest for observability
+        manifest = JobManifest(
+            job_id=job_id,
+            input_urls_count=len(urls),
+            config_hash=self.config.config_hash(),
+        )
+        manifest.status = JobStatus.DOWNLOADING
+        manifest.save(self.storage, self.config.output_catalog)
+
+        # Process all URLs in a single batch using async HTTP
+        worker_tag = f"simple-{uuid.uuid4().hex[:8]}"
+        all_items: list[dict[str, Any]] = []
+
+        # Use async HTTP if available and enabled
+        if self.config.enable_concurrent_http and HAS_ASYNC_HTTP:
+            logger.info(f"Downloading {len(urls):,} items with async HTTP...")
+            items = self._download_stac_items_batch_async(urls, worker_id=0)
+            for item in items:
+                if item:
+                    all_items.append(item)
+                    self.stats.record_url_processed(success=True)
+                else:
+                    self.stats.record_url_processed(success=False)
+        else:
+            # Fallback to sync processing
+            logger.info(f"Downloading {len(urls):,} items with sync HTTP...")
+            with tqdm(total=len(urls), desc="Downloading", unit="items") as pbar:
+                for url in urls:
+                    fetched_item = self._download_stac_item(url)
+                    if fetched_item:
+                        all_items.append(fetched_item)
+                        self.stats.record_url_processed(success=True)
+                    else:
+                        self.stats.record_url_processed(success=False)
+                    pbar.update(1)
+
+        logger.info(f"Downloaded {len(all_items):,} items successfully")
+
+        if not all_items:
+            logger.warning("No items downloaded, nothing to process")
+            manifest.status = JobStatus.COMPLETED
+            manifest.save(self.storage, self.config.output_catalog)
+            self.stats.finish_processing()
+            return {}
+
+        # Write partition shards
+        logger.info("Writing partition shards...")
+        shard_info = self._write_partition_shards(all_items, worker_tag, self.stats)
+
+        # Update manifest
+        manifest.download_phase.completed = True
+        manifest.download_phase.shards_written = [s["shard_path"] for s in shard_info]
+        manifest.download_phase.urls_processed = len(urls)
+        manifest.status = JobStatus.CONSOLIDATING
+        manifest.save(self.storage, self.config.output_catalog)
+
+        # Consolidate shards
+        logger.info("Consolidating partitions...")
+        final_stats = self._consolidate_shards(shard_info, manifest=manifest)
+
+        # Cleanup scratch
+        self._cleanup_scratch(shard_info)
+
+        # Mark complete
+        manifest.status = JobStatus.COMPLETED
+        manifest.consolidation_phase.completed = True
+        manifest.save(self.storage, self.config.output_catalog)
+
+        # Finish statistics
+        self.stats.finish_processing()
+
+        # Update statistics with consolidation results
+        for _partition_key, stats in final_stats.items():
+            self.stats.record_consolidation(
+                new_items=stats.get("new_items", 0),
+                existing_items=stats.get("existing_items", 0),
+                duplicates_removed=stats.get("duplicates_removed", 0),
+            )
+
+        # Generate schema if requested
+        if self.config.generate_schema:
+            logger.info("Generating catalog schema metadata...")
+            schema_gen = schema_generator.SchemaGenerator(self.config, self.grid, self.storage, self.stats)
+            schema_gen.generate_catalog_schema(final_stats, self.config.schema_filename)
+
+        # Summary
+        stats_summary = self.stats.get_summary()
+        logger.info("=" * 80)
+        logger.info("SIMPLE PIPELINE COMPLETED SUCCESSFULLY")
+        logger.info(f"Job ID: {job_id}")
+        logger.info(f"Total partitions: {len(final_stats)}")
+        logger.info(f"Unique granules: {stats_summary['unique_granules']:,}")
+        total_new = sum(s.get("new_items", 0) for s in final_stats.values())
+        logger.info(f"New items: {total_new:,}")
+        logger.info("=" * 80)
+
+        return final_stats
+
+    def _read_input_urls(self) -> list[str]:
+        """Read URLs from input file(s) using appropriate format reader.
+
+        If input_pattern is configured, discovers all matching files and reads URLs
+        from each one. Otherwise, reads from the single input_file.
+
+        Returns:
+            List of URLs from all discovered files.
+        """
+        # Check if we should use pattern-based file discovery
+        if self.config.input_pattern:
+            return self._read_urls_from_pattern()
+
+        # Traditional single file mode
+        return self._read_urls_from_file(self.config.input_file)
+
+    def _read_urls_from_file(self, file_path: str) -> list[str]:
+        """Read URLs from a single file using appropriate format reader.
+
+        Args:
+            file_path: Path to the input file (local or S3).
+
+        Returns:
+            List of URLs extracted from the file.
+        """
+        ReaderFactory = input_readers.ReaderFactory
+
+        # Determine format
+        if self.config.input_format == "auto":
+            format_name = ReaderFactory.auto_detect_format(file_path)
+        else:
+            format_name = self.config.input_format
+
+        # Get appropriate reader
+        reader = ReaderFactory.get_reader(format_name)
+
+        # Read URLs
+        return reader.read_urls(file_path, self.config.url_column)
+
+    def _read_urls_from_pattern(self) -> list[str]:
+        """Read URLs from multiple files matching a glob pattern.
+
+        Uses StorageBackend.list_files() to discover files matching the pattern,
+        then reads URLs from each discovered file.
+
+        Returns:
+            Concatenated list of URLs from all matching files.
+
+        Raises:
+            ValueError: If no files match the pattern.
+        """
+        pattern = self.config.input_pattern
+        all_urls: list[str] = []
+
+        logger.info(f"Discovering files matching pattern: {pattern}")
+
+        # Check if pattern is for S3 or local filesystem
+        if pattern.startswith("s3://"):
+            # Use fsspec for S3 pattern matching
+            all_urls = self._read_s3_pattern(pattern)
+        else:
+            # Use glob for local filesystem
+            matching_files = glob_module.glob(pattern, recursive=True)
+
+            if not matching_files:
+                raise ValueError(f"No files found matching pattern: {pattern}")
+
+            logger.info(f"Found {len(matching_files)} files matching pattern")
+
+            # Read URLs from each file
+            for file_path in matching_files:
+                logger.info(f"Reading URLs from: {file_path}")
+                urls = self._read_urls_from_file(file_path)
+                all_urls.extend(urls)
+
+        logger.info(f"Total URLs read from all files: {len(all_urls)}")
+        return all_urls
+
+    def _read_s3_pattern(self, pattern: str) -> list[str]:
+        """Read URLs from S3 files matching a glob pattern.
+
+        Args:
+            pattern: S3 path with glob pattern (e.g., s3://bucket/bulk/2020_*.ndjson).
+
+        Returns:
+            Concatenated list of URLs from all matching S3 files.
+
+        Raises:
+            ValueError: If no files match the pattern or fsspec not available.
+        """
+        if not pattern.startswith("s3://"):
+            raise ValueError(f"S3 pattern must start with 's3://', got: {pattern}")
+
+        # Use fsspec.glob() directly for pattern matching
+        matching_files = fsspec.glob(pattern)
+
+        if not matching_files:
+            raise ValueError(f"No S3 files found matching pattern: {pattern}")
+
+        logger.info(f"Found {len(matching_files)} S3 files matching pattern")
+
+        # Read URLs from each file
+        all_urls: list[str] = []
+        for file_path in matching_files:
+            logger.info(f"Reading URLs from S3: {file_path}")
+            urls = self._read_urls_from_file(file_path)
+            all_urls.extend(urls)
+
+        return all_urls
+
+    def _process_urls_distributed(self, urls: list[str], job_id: str | None = None) -> list[dict[str, Any]]:
+        """Process URLs in parallel and write to scratch location.
+
+        For distributed processing (Dask), uses serializable worker functions from
+        workers.py. For local processing, uses inline closures that capture self.
+
+        Worker stats are returned alongside shard info and merged into the
+        pipeline's main stats at the end.
+
+        Args:
+            urls: List of STAC item URLs to process.
+            job_id: Job ID for tracking. If None, generates a new UUID.
+
+        Returns:
+            List of shard info dictionaries.
+        """
+        # Use provided job_id or generate one
+        if job_id is None:
+            job_id = str(uuid.uuid4())
+        logger.info(f"Starting job {job_id[:8]}...")
+
+        # Serialize config for distributed workers
+        config_dict = self.config.to_dict()
+
+        def process_url_batch(
+            url_batch: list[str], worker_id: int, **kwargs: Any
+        ) -> tuple[list[dict[str, Any]], IngestionStatistics]:
+            """Process a batch of URLs on a single worker with async HTTP support.
+
+            This closure captures self and is used for LocalProcessor.
+            For DaskDistributedProcessor, the serializable workers.process_url_batch is used instead.
+
+            Returns:
+                Tuple of (shard_info_list, worker_statistics)
+            """
+            # Create worker-local statistics collector
+            worker_stats = IngestionStatistics()
+
+            # Choose processing method based on configuration
+            if self.config.enable_concurrent_http and HAS_ASYNC_HTTP and len(url_batch) >= self.config.batch_size:
+                # Use async HTTP processing for better performance
+                shards = self._process_batch_with_async_http(url_batch, worker_id, worker_stats)
+                return (shards, worker_stats)
+
+            # Traditional synchronous processing (original implementation)
+            batch_shards = []
+            current_shard_items = []
+            worker_tag = f"worker-{worker_id}-{uuid.uuid4().hex[:8]}"
+
+            # Add progress bar for this batch
+            with tqdm(total=len(url_batch), desc=f"Worker {worker_id}", unit="urls", leave=False) as pbar:
+                for _idx, url in enumerate(url_batch):
+                    try:
+                        # Download and parse STAC item
+                        item = self._download_stac_item(url)
+                        if not item:
+                            # Record failed URL
+                            worker_stats.record_url_processed(success=False)
+                            pbar.update(1)
+                            continue
+
+                        # Record successful URL
+                        worker_stats.record_url_processed(success=True)
+
+                        # Add to current shard
+                        current_shard_items.append(item)
+
+                        # Write shard when it reaches target size
+                        if len(current_shard_items) >= self.config.items_per_shard:
+                            # Use partition-aware sharding
+                            partition_shards = self._write_partition_shards(
+                                current_shard_items, worker_tag, worker_stats
+                            )
+                            batch_shards.extend(partition_shards)
+                            current_shard_items = []
+
+                    except (
+                        requests.exceptions.RequestException,
+                        json.JSONDecodeError,
+                        OSError,
+                        ValueError,
+                    ) as e:
+                        logger.error(f"Error processing URL {url}: {e}")
+                        worker_stats.record_url_processed(success=False)
+                        pbar.update(1)
+                        continue
+
+                    pbar.update(1)
+
+            # Write final shard for this batch
+            if current_shard_items:
+                # Use partition-aware sharding
+                partition_shards = self._write_partition_shards(current_shard_items, worker_tag, worker_stats)
+                batch_shards.extend(partition_shards)
+
+            total_items = sum(s["item_count"] for s in batch_shards)
+            logger.info(f"Worker {worker_id} completed: {len(batch_shards)} shards, {total_items} items")
+            return (batch_shards, worker_stats)
+
+        # Distribute URLs to workers
+        url_chunks = self._chunk_urls(urls, self.config.max_workers)
+
+        # Process in parallel - pass config_dict and job_id for Dask
+        all_results = self.processor.process_urls(
+            url_chunks,
+            process_url_batch,
+            config_dict=config_dict,
+            job_id=job_id,
+        )
+
+        # Merge worker statistics into main stats and flatten shard results
+        all_shards = []
+        for result in all_results:
+            # Handle both tuple format (LocalProcessor) and dict format (DaskDistributedProcessor)
+            if isinstance(result, tuple):
+                shards, worker_stats = result
+                self.stats.merge(worker_stats)
+                all_shards.extend(shards)
+            elif isinstance(result, dict):
+                # Dict format from workers.process_url_batch
+                shards = result.get("shards", [])
+                stats_dict = result.get("stats", {})
+                failed_urls = result.get("failed_urls", [])
+
+                # Reconstruct stats from dict
+                worker_stats = IngestionStatistics()
+                for _ in range(stats_dict.get("urls_processed", 0)):
+                    worker_stats.record_url_processed(success=True)
+                for _ in range(stats_dict.get("urls_failed", 0)):
+                    worker_stats.record_url_processed(success=False)
+
+                self.stats.merge(worker_stats)
+                all_shards.extend(shards)
+
+                if failed_urls:
+                    logger.warning(f"Failed URLs: {len(failed_urls)}")
+            else:
+                logger.error(f"Unexpected result type: {type(result)}")
+
+        return all_shards
+
+    def _download_stac_items_batch_async(self, urls: list[str], worker_id: int) -> list[dict[str, Any]]:
+        """Download STAC items using async HTTP client for improved performance.
+
+        This method provides a high-performance alternative to sequential downloads
+        by utilizing concurrent HTTP requests with connection pooling and rate limiting.
+
+        Args:
+            urls: List of URLs to download
+            worker_id: Worker identifier for logging and progress tracking
+
+        Returns:
+            List of successfully downloaded STAC item dictionaries
+        """
+        if not HAS_ASYNC_HTTP:
+            # Fallback to synchronous processing
+            logger.warning("Async HTTP not available, falling back to synchronous processing")
+            items = []
+            for url in urls:
+                item = self._download_stac_item(url)
+                if item is not None:
+                    items.append(item)
+            return items
+
+        def run_async_download() -> list[dict[str, Any]]:
+            """Run async download in a new event loop with proper context handling.
+
+            This nested function handles the complexities of running async code from
+            a synchronous context, particularly dealing with existing event loops
+            in environments like Jupyter notebooks or async web frameworks.
+
+            Returns:
+                List of successfully downloaded STAC item dictionaries.
+
+            Note:
+                Automatically detects running event loops and uses thread pool
+                execution when necessary to avoid "RuntimeError: cannot be called
+                from a running event loop" issues.
+            """
+            try:
+                # Check if we're already in an async context
+                asyncio.current_task()
+                # If we reach here, there's a running event loop
+                # We need to run in a thread to avoid issues
+                import concurrent.futures
+
+                with concurrent.futures.ThreadPoolExecutor() as executor:
+                    future = executor.submit(asyncio.run, self._async_download_worker(urls))
+                    return future.result()
+            except RuntimeError:
+                # No event loop exists or not in async context, create a new one
+                return asyncio.run(self._async_download_worker(urls))
+
+        try:
+            return run_async_download()
+        except (RuntimeError, asyncio.CancelledError, OSError) as e:
+            logger.error(f"Async download failed for worker {worker_id}, falling back to sync: {e}")
+            # Fallback to synchronous processing
+            items = []
+            for url in urls:
+                item = self._download_stac_item(url)
+                if item is not None:
+                    items.append(item)
+            return items
+
+    async def _async_download_worker(self, urls: list[str]) -> list[dict[str, Any]]:
+        """Internal async worker for downloading STAC items with high concurrency.
+
+        This method serves as the async wrapper that calls the async HTTP client
+        with the current pipeline configuration. It handles the async/await mechanics
+        while maintaining error handling compatibility.
+
+        Args:
+            urls: List of URLs to download concurrently.
+
+        Returns:
+            List of successfully downloaded and parsed STAC item dictionaries.
+            Failed downloads are filtered out and logged separately.
+
+        Raises:
+            RuntimeError: If async HTTP client is not available when called.
+
+        Example:
+            >>> urls = ["https://example.com/item1.json", "https://example.com/item2.json"]
+            >>> items = await self._async_download_worker(urls)
+            >>> print(f"Downloaded {len(items)} items")
+        """
+        if not HAS_ASYNC_HTTP or download_stac_items_async is None:
+            raise RuntimeError("Async HTTP client not available")
+
+        return await download_stac_items_async(
+            urls=urls,
+            concurrent_requests=self.config.concurrent_requests,
+            connection_pool_size=self.config.connection_pool_size,
+            request_timeout=self.config.request_timeout,
+            retry_attempts=self.config.retry_attempts,
+            retry_delay=self.config.retry_delay,
+            batch_size=self.config.batch_size,
+        )
+
+    def _process_batch_with_async_http(
+        self, url_batch: list[str], worker_id: int, stats: IngestionStatistics
+    ) -> list[dict[str, Any]]:
+        """Process URL batch using async HTTP with adaptive batching and memory management.
+
+        This method processes URLs in smaller concurrent batches while maintaining
+        the same shard writing patterns as the original synchronous version. It provides
+        the bridge between sync pipeline processing and async HTTP downloading.
+
+        The method adapts batch sizes based on configuration and processes URLs in
+        sub-batches to manage memory usage while maximizing concurrency within each batch.
+
+        Args:
+            url_batch: List of URLs to process in this batch.
+            worker_id: Unique identifier for this worker process, used for logging
+                     and temporary file naming to avoid conflicts.
+
+        Returns:
+            List of successfully downloaded and processed STAC item dictionaries.
+            Items are filtered, validated, and ready for catalog ingestion.
+
+        Note:
+            This method maintains the same error handling semantics as the original
+            synchronous version - failed downloads return empty lists and errors are logged.
+
+        Example:
+            >>> batch = ["https://example.com/item1.json", "https://example.com/item2.json"]
+            >>> items = pipeline._process_batch_with_async_http(batch, worker_id=1)
+            >>> print(f"Processed {len(items)} items from batch")
+        """
+        batch_shards = []
+        worker_tag = f"worker-{worker_id}-{uuid.uuid4().hex[:8]}"
+
+        # Process URLs in smaller async batches for memory management
+        async_batch_size = min(self.config.batch_size, len(url_batch))
+
+        with tqdm(total=len(url_batch), desc=f"Worker {worker_id} (Async)", unit="urls", leave=False) as pbar:
+            for i in range(0, len(url_batch), async_batch_size):
+                batch_urls = url_batch[i : i + async_batch_size]
+
+                # Download batch concurrently
+                items = self._download_stac_items_batch_async(batch_urls, worker_id)
+
+                # Record URL processing stats (async batch) - use passed stats
+                success_count = len(items)
+                failed_count = len(batch_urls) - success_count
+                for _ in range(success_count):
+                    stats.record_url_processed(success=True)
+                for _ in range(failed_count):
+                    stats.record_url_processed(success=False)
+
+                # Process items into shards
+                if items:
+                    partition_shards = self._write_partition_shards(items, worker_tag, stats)
+                    batch_shards.extend(partition_shards)
+
+                pbar.update(len(batch_urls))
+
+        # Log completion stats
+        total_items = sum(shard.get("item_count", 0) for shard in batch_shards)
+        logger.info(f"Worker {worker_id} completed (async): {len(batch_shards)} shards, {total_items} items")
+
+        return batch_shards
+
+    def _validate_and_fix_items(self, items: list[dict[str, Any]]) -> list[dict[str, Any]]:
+        """Validate STAC items and optionally fix geometry/bbox issues.
+
+        Performs validation on each item including:
+        - Geometry validity check (self-intersection, ring orientation)
+        - Bbox-geometry consistency check
+        - Optionally fixes invalid geometries and mismatched bboxes
+
+        Args:
+            items: List of STAC item dictionaries
+
+        Returns:
+            List of validated (and optionally corrected) STAC items
+        """
+        from .validation import validate_stac_item
+
+        validated_items = []
+        validation_stats = {"total": len(items), "warnings": 0, "fixed_geometry": 0, "fixed_bbox": 0}
+
+        for item in items:
+            result, corrected_item = validate_stac_item(
+                item,
+                fix_geometry=self.config.fix_invalid_geometry,
+                bbox_tolerance=self.config.bbox_tolerance,
+            )
+
+            # Track statistics
+            if result.warnings:
+                validation_stats["warnings"] += len(result.warnings)
+                if self.config.log_validation_warnings:
+                    for warning in result.warnings:
+                        logger.debug(f"Validation warning for item {item.get('id')}: {warning}")
+
+            if result.metadata.get("geometry_fixed"):
+                validation_stats["fixed_geometry"] += 1
+            if result.metadata.get("bbox_corrected"):
+                validation_stats["fixed_bbox"] += 1
+
+            # Use corrected item if available, otherwise use original
+            if corrected_item is not None:
+                validated_items.append(corrected_item)
+            else:
+                validated_items.append(item)
+
+        # Log summary
+        if validation_stats["warnings"] > 0 or validation_stats["fixed_geometry"] > 0:
+            logger.debug(
+                f"Validation: {validation_stats['total']} items, "
+                f"{validation_stats['warnings']} warnings, "
+                f"{validation_stats['fixed_geometry']} geometries fixed, "
+                f"{validation_stats['fixed_bbox']} bboxes corrected"
+            )
+
+        return validated_items
+
+    def _write_partition_shards(
+        self,
+        items: list[dict[str, Any]],
+        worker_id: str,
+        stats: IngestionStatistics | None = None,
+    ) -> list[dict[str, Any]]:
+        """Group items by partition and write partition-aware shards.
+
+        Also records comprehensive statistics for each item including:
+        - Unique item tracking via HyperLogLog
+        - Spanning item detection and tile counts
+        - Spatial/temporal distribution
+        - Data quality metrics
+
+        Args:
+            items: List of STAC items to process
+            worker_id: Unique identifier for this worker
+            stats: Optional IngestionStatistics to record to. If None, uses self.stats.
+        """
+        # Use provided stats or fall back to pipeline's stats
+        target_stats = stats if stats is not None else self.stats
+
+        # Validate and optionally fix items if validation is enabled
+        if self.config.enable_validation:
+            items = self._validate_and_fix_items(items)
+
+        # Group items by partition key while recording statistics
+        partition_groups: dict[str, list[dict[str, Any]]] = {}
+
+        for item in items:
+            # Get geometry and compute tiles with spanning detection
+            geom = item.get("geometry")
+            mission = self._extract_mission(item)
+
+            if not geom:
+                tiles: list[str] = []
+                is_spanning = False
+                routed_to_global = False
+            else:
+                tiles, is_spanning = self.grid.tiles_for_geometry_with_spanning_detection(geom)
+                threshold = self.config.global_partition_threshold
+                routed_to_global = self.config.enable_global_partitioning and len(tiles) > threshold
+
+            # Record item statistics
+            target_stats.record_item(
+                item=item,
+                tiles=tiles,
+                is_spanning=is_spanning,
+                routed_to_global=routed_to_global,
+                mission=mission,
+            )
+
+            # Compute partition key and group
+            partition_key = self._compute_partition_key(item)
+            if partition_key not in partition_groups:
+                partition_groups[partition_key] = []
+            partition_groups[partition_key].append(item)
+
+        # Write shard for each partition
+        shard_info = []
+        for partition_key, partition_items in partition_groups.items():
+            if not partition_items:
+                continue
+
+            # Convert to GeoDataFrame using engine
+            gdf = self.engine.items_to_geodataframe(partition_items)
+
+            # Sort items within shard
+            if self.config.sort_key in gdf.columns:
+                gdf = gdf.sort_values(self.config.sort_key, ascending=self.config.sort_ascending)
+
+            # Determine file extension based on output format
+            file_ext = ".ndjson" if self.config.output_format == "ndjson" else ".parquet"
+            shard_path = f"{self.config.scratch_location}/shards/{partition_key}/{worker_id}{file_ext}"
+
+            self.scratch_storage.makedirs(Path(shard_path).parent)
+            self._write_partition_shard(gdf, shard_path)
+
+            shard_info.append(
+                {
+                    "shard_path": shard_path,
+                    "partition_key": partition_key,
+                    "item_count": len(partition_items),
+                    "worker_id": worker_id,
+                }
+            )
+
+        return shard_info
+
+    def _write_partition_shard(self, gdf, shard_path: str):
+        """Write partition shard in configured format."""
+
+        if self.config.output_format == "ndjson":
+            # Convert to NDJSON format
+            self._write_ndjson_shard(gdf, shard_path)
+        else:
+            # Default GeoParquet format
+            with self.scratch_storage.open(shard_path, "wb") as f:
+                gdf.to_parquet(f, index=False, compression="snappy")
+
+    def _write_ndjson_shard(self, gdf, shard_path: str):
+        """Write GeoDataFrame as NDJSON format."""
+        import orjson
+
+        try:
+            # Use engine to convert GeoDataFrame back to STAC items
+            features = self.engine.geodataframe_to_items(gdf)
+
+            with self.scratch_storage.open(shard_path, "wb") as f:
+                for item in features:
+                    f.write(orjson.dumps(item) + b"\n")
+
+        except (TypeError, ValueError, OSError) as e:
+            logger.warning(f"Failed to convert to STAC format, using raw GeoJSON: {e}")
+
+            # Fallback: write as GeoJSON features
+            with self.scratch_storage.open(shard_path, "wb") as f:
+                for _, row in gdf.iterrows():
+                    # Convert each row to a GeoJSON-like feature
+                    geom = row.get("geometry")
+                    if geom is not None and hasattr(geom, "__geo_interface__"):
+                        geometry_dict = geom.__geo_interface__
+                    else:
+                        geometry_dict = None
+
+                    feature = {
+                        "type": "Feature",
+                        "geometry": geometry_dict,
+                        "properties": {k: v for k, v in row.items() if k != "geometry"},
+                    }
+                    f.write(orjson.dumps(feature) + b"\n")
+
+    def _consolidate_shards(
+        self,
+        shard_info: list[dict[str, Any]],
+        skip_partitions: set[str] | None = None,
+        manifest: JobManifest | None = None,
+    ) -> dict[str, dict[str, int]]:
+        """Consolidate worker shards into final partitioned catalog.
+
+        Args:
+            shard_info: List of shard info dictionaries.
+            skip_partitions: Set of partition keys to skip (for resume).
+            manifest: Job manifest to update as partitions complete.
+
+        Returns:
+            Dictionary mapping partition keys to consolidation stats.
+        """
+        if skip_partitions is None:
+            skip_partitions = set()
+
+        def consolidate_partition_shards(partition_key: str, shard_paths: list[str], **kwargs: Any) -> dict[str, Any]:
+            """Consolidate all shards for a single partition, merging with existing data."""
+            return _consolidate_partition(partition_key, shard_paths)
+
+        def _consolidate_partition(partition_key: str, shard_paths: list[str]) -> dict[str, Any]:
+            """Efficient consolidation approach using local staging for atomic S3 operations."""
+            import io
+            import tempfile
+            from pathlib import Path
+
+            import pyarrow.parquet as pq
+
+            final_path = self._get_final_partition_path(partition_key)
+            existing_count = 0
+            new_count = 0
+
+            # Use temporary directory for local staging
+            with tempfile.TemporaryDirectory(dir=self.config.temp_dir_location) as temp_dir:
+                temp_existing_path = Path(temp_dir) / "existing.parquet"
+                temp_merged_path = Path(temp_dir) / "merged.parquet"
+
+                all_items = []
+
+                # Step 1: Download existing partition to local temp (if exists)
+                if self.storage.exists(final_path):
+                    try:
+                        logger.info(f"Partition {partition_key}: downloading existing data to local staging")
+
+                        # Download to temporary file
+                        with self.storage.open(final_path, "rb") as remote_f:
+                            with open(temp_existing_path, "wb") as local_f:
+                                # Stream in chunks to avoid memory issues
+                                chunk_size = self.config.max_memory_per_partition_mb * 1024 * 1024 // 10
+                                while True:
+                                    chunk = remote_f.read(chunk_size)
+                                    if not chunk:
+                                        break
+                                    local_f.write(chunk)
+
+                        # Read existing data from temp file
+                        table = pq.read_table(temp_existing_path)
+                        df = table.to_pandas()
+                        existing_gdf = gpd.GeoDataFrame(df)
+                        existing_count = len(existing_gdf)
+                        all_items.append(existing_gdf)
+
+                    except (OSError, ValueError, RuntimeError) as e:
+                        logger.error(f"Error downloading existing partition {final_path}: {e}")
+
+                # Step 2: Read all new shards for this partition (using streaming)
+                for shard_path in shard_paths:
+                    try:
+                        with self.scratch_storage.open(shard_path, "rb") as f:
+                            binary_data = f.read()
+                            table = pq.read_table(io.BytesIO(binary_data))
+                            df = table.to_pandas()
+                            gdf = gpd.GeoDataFrame(df)
+                            all_items.append(gdf)
+
+                    except (OSError, ValueError, RuntimeError) as e:
+                        logger.error(f"Error reading shard {shard_path}: {e}")
+                        continue
+
+                if not all_items:
+                    return {"partition": partition_key, "item_count": 0, "existing_count": 0, "new_count": 0}
+
+                # Step 3: Merge data using chunked processing if enabled
+                if self.config.enable_streaming_merge and len(all_items) > 1:
+                    # Process in batches to manage memory
+                    batch_size = max(1, self.config.max_memory_per_partition_mb // 100)  # Conservative estimate
+                    merged = pd.DataFrame()
+
+                    for i in range(0, len(all_items), batch_size):
+                        batch = all_items[i : i + batch_size]
+                        batch_merged = pd.concat(batch, ignore_index=True)
+
+                        if len(merged) == 0:
+                            merged = batch_merged
+                        else:
+                            # Merge with existing and deduplicate incrementally
+                            merged = pd.concat([merged, batch_merged], ignore_index=True)
+                            merged = merged.drop_duplicates(subset=["id"], keep="last")
+                else:
+                    # Standard merge for smaller datasets
+                    merged = pd.concat(all_items, ignore_index=True)
+
+                # Step 4: Deduplicate (keep="last" to prefer new data over existing)
+                original_count = len(merged)
+                merged = merged.drop_duplicates(subset=["id"], keep="last")
+                duplicates_removed = original_count - len(merged)
+
+                # Step 5: Final sort
+                if self.config.sort_key in merged.columns:
+                    merged = merged.sort_values(self.config.sort_key, ascending=self.config.sort_ascending)
+
+                # Step 6: Write merged data to local temp file
+                merged_gdf = gpd.GeoDataFrame(merged)
+                merged_gdf.to_parquet(temp_merged_path, index=False, compression="snappy")
+
+                # Step 7: Atomic upload to final location
+                if final_path.startswith("s3://"):
+                    # For S3, use storage.upload which should handle large files appropriately
+                    self.storage.upload(str(temp_merged_path), final_path)
+                else:
+                    # Standard atomic write
+                    self._write_final_partition(merged_gdf, final_path)
+
+                new_count = len(merged) - existing_count
+                logger.info(
+                    f"Partition {partition_key}: {existing_count} existing + "
+                    f"{new_count} new = {len(merged)} total "
+                    f"({duplicates_removed} duplicates removed) [efficient]"
+                )
+
+                return {
+                    "partition": partition_key,
+                    "item_count": len(merged),
+                    "existing_count": existing_count,
+                    "new_count": new_count,
+                    "duplicates_removed": duplicates_removed,
+                    "final_path": final_path,
+                }
+
+        # Group shards by their target partition
+        partition_shards = self._group_shards_by_partition(shard_info)
+        logger.info(f"Grouped shards into {len(partition_shards)} partitions")
+
+        # Filter out already-completed partitions (for resume)
+        if skip_partitions:
+            original_count = len(partition_shards)
+            partition_shards = {k: v for k, v in partition_shards.items() if k not in skip_partitions}
+            skipped_count = original_count - len(partition_shards)
+            if skipped_count > 0:
+                logger.info(f"Skipping {skipped_count} already-completed partitions")
+
+        # Consolidate partitions in parallel - pass config_dict for Dask
+        config_dict = self.config.to_dict()
+        consolidation_results = self.processor.consolidate_shards(
+            list(partition_shards.items()),
+            consolidate_partition_shards,
+            config_dict=config_dict,
+        )
+
+        # Determine checkpoint strategy
+        use_time_checkpoint = self.config.checkpoint_interval_seconds > 0
+        use_count_checkpoint = self.config.checkpoint_partition_count > 0
+        # Default to time-based (30s) if neither is set
+        if not use_time_checkpoint and not use_count_checkpoint:
+            use_time_checkpoint = True
+            checkpoint_interval = 30
+        else:
+            checkpoint_interval = self.config.checkpoint_interval_seconds
+
+        last_checkpoint_time = time.time()
+
+        # Convert to stats - handle both dict formats
+        final_stats = {}
+        for result in consolidation_results:
+            if result.get("item_count", 0) > 0:
+                # Handle both "partition" key (from local) and "partition_key" (from workers.py)
+                partition = result.get("partition") or result.get("partition_key", "unknown")
+                final_stats[partition] = {
+                    "total_items": result.get("item_count", 0),
+                    "existing_items": result.get("existing_count", 0),
+                    "new_items": result.get("new_count", 0),
+                    "duplicates_removed": result.get("duplicates_removed", 0),
+                }
+
+                # Update manifest if provided (for checkpointing)
+                if manifest is not None:
+                    manifest.consolidation_phase.completed_partitions.append(partition)
+                    manifest.consolidation_phase.partitions_completed += 1
+
+                    # Determine if we should checkpoint
+                    should_checkpoint = False
+                    current_time = time.time()
+
+                    if use_time_checkpoint and (current_time - last_checkpoint_time) >= checkpoint_interval:
+                        should_checkpoint = True
+                        last_checkpoint_time = current_time
+                    elif (
+                        use_count_checkpoint
+                        and manifest.consolidation_phase.partitions_completed % self.config.checkpoint_partition_count
+                        == 0
+                    ):
+                        should_checkpoint = True
+
+                    if should_checkpoint:
+                        manifest.save(self.storage, self.config.output_catalog)
+
+        return final_stats
+
+    def _group_shards_by_partition(self, shard_info: list[dict[str, Any]]) -> dict[str, list[str]]:
+        """Group shard paths by their target partition."""
+        partition_shards: dict[str, list[str]] = defaultdict(list)
+
+        for info in shard_info:
+            shard_path = info["shard_path"]
+
+            if "partition_key" in info:
+                # Use partition key from shard info
+                partition_key = info["partition_key"]
+                partition_shards[partition_key].append(shard_path)
+            else:
+                # Fallback: read shard to determine partitions (for backward compatibility)
+                try:
+                    with self.scratch_storage.open(shard_path, "rb") as f:
+                        import io
+
+                        import pyarrow.parquet as pq
+
+                        binary_data = f.read()
+                        table = pq.read_table(io.BytesIO(binary_data))
+                        df = table.to_pandas()
+                        gdf = gpd.GeoDataFrame(df)
+
+                    if len(gdf) == 0:
+                        continue
+
+                    # Group items within shard by partition
+                    for _idx, row in gdf.iterrows():
+                        item = row.to_dict()
+                        partition_key = self._compute_partition_key(item)
+                        partition_shards[partition_key].append(shard_path)
+
+                    # Remove duplicates (same shard might have items for same partition)
+                    partition_shards = cast(
+                        dict[str, list[str]], {k: list(set(v)) for k, v in partition_shards.items()}
+                    )
+
+                except (OSError, ValueError, RuntimeError, AttributeError, TypeError) as e:
+                    logger.error(f"Error processing shard {shard_path}: {e}")
+                    continue
+
+        return dict(partition_shards)
+
+    def _extract_mission(self, item: dict[str, Any]) -> str:
+        """Extract mission/dataset identifier from STAC item."""
+        props = item.get("properties", {})
+
+        # Primary: Extract from dataset_id field
+        dataset_id = props.get(self.config.mission_field, "")
+        if dataset_id:
+            return self._sanitize_mission_name(str(dataset_id))
+
+        # Fallback to collection if dataset_id not found
+        collection = props.get("collection", "")
+        if collection:
+            return self._sanitize_mission_name(str(collection))
+
+        # Final fallback
+        return "unknown_mission"
+
+    def _sanitize_mission_name(self, name: str) -> str:
+        """Sanitize mission name for filesystem compatibility."""
+        import re
+
+        # Replace invalid filesystem chars with underscores, convert to lowercase
+        sanitized = re.sub(r"[^\w]", "_", name.lower().strip())
+        # Remove consecutive underscores
+        sanitized = re.sub(r"_+", "_", sanitized)
+        # Remove leading/trailing underscores
+        return sanitized.strip("_") or "unnamed"
+
+    def _compute_partition_key(self, item: dict[str, Any]) -> str:
+        """Compute partition key with Hive-style temporal partitioning.
+
+        Format: {dataset_id}/partition=h3/level={resolution}/{h3_cell_id}/year=YYYY/month=MM[/day=DD]
+
+        The temporal partition depth is controlled by config.temporal_bin:
+        - year: year=YYYY
+        - month: year=YYYY/month=MM
+        - day: year=YYYY/month=MM/day=DD
+
+        This Hive-style naming enables directory-level pruning in DuckDB, Athena,
+        Spark, and other query engines for optimal query performance.
+        """
+        # Extract mission (dataset identifier)
+        mission = self._extract_mission(item)
+
+        # Extract spatial information
+        geom = item.get("geometry")
+        if not geom:
+            h3_cell = "unknown"
+        else:
+            # Use the new method with spanning detection
+            tiles, is_spanning = self.grid.tiles_for_geometry_with_spanning_detection(geom)
+
+            # Get resolution-specific threshold
+            threshold = self.config.global_partition_threshold
+
+            # Apply global partitioning logic with resolution-specific threshold
+            if self.config.enable_global_partitioning and len(tiles) > threshold:
+                # Multi-cell geometry - route to global partition
+                h3_cell = "global"
+            else:
+                # Single-cell geometry - use specific tile
+                h3_cell = tiles[0] if tiles else "unknown"
+
+        # Extract temporal information using Hive-style partitioning
+        temporal_parts = self._extract_temporal_hive_parts(item)
+
+        # Build partition path
+        grid_type = self.config.grid_system  # Should be 'h3'
+        resolution = self.config.grid_resolution
+
+        return f"{mission}/partition={grid_type}/level={resolution}/{h3_cell}/{temporal_parts}"
+
+    def _extract_temporal_hive_parts(self, item: dict[str, Any]) -> str:
+        """Extract Hive-style temporal partition parts from STAC item.
+
+        Generates directory path fragments using Hive partition naming convention
+        for optimal query pruning in DuckDB, Athena, Spark, and other query engines.
+
+        Returns:
+            Path fragment based on temporal_bin configuration:
+            - year: "year=2024"
+            - month: "year=2024/month=01"
+            - day: "year=2024/month=01/day=15"
+            - unknown: "unknown" (for missing/invalid datetimes)
+        """
+        props = item.get("properties", {})
+        dt_str = props.get("datetime")
+
+        if not dt_str:
+            return "unknown"
+
+        try:
+            dt = pd.to_datetime(dt_str)
+            if self.config.temporal_bin == "year":
+                return f"year={dt.year}"
+            elif self.config.temporal_bin == "month":
+                return f"year={dt.year}/month={dt.month:02d}"
+            elif self.config.temporal_bin == "day":
+                return f"year={dt.year}/month={dt.month:02d}/day={dt.day:02d}"
+            else:
+                return "unknown"
+        except (ValueError, TypeError):
+            return "unknown"
+
+    def _get_final_partition_path(self, partition_key: str) -> str:
+        """Get final path for partition structure with Hive-style naming.
+
+        The partition_key already contains Hive-style temporal directories
+        (e.g., mission/partition=h3/level=2/cell/year=2024/month=01).
+        This method appends the final filename (items.parquet or items.ndjson).
+        """
+        # Determine file extension based on output format
+        if self.config.output_format == "ndjson":
+            file_extension = ".ndjson"
+        else:
+            file_extension = ".parquet"  # Default to .parquet for geoparquet
+
+        return f"{self.config.output_catalog}/{partition_key}/items{file_extension}"
+
+    def _write_final_partition(self, gdf: gpd.GeoDataFrame, path: str):
+        """Write final partition with atomic safety."""
+        import tempfile
+
+        if path.startswith("s3://"):
+            # For S3, write to temporary local file then upload
+            with tempfile.NamedTemporaryFile(suffix=".parquet", delete=False) as tmp:
+                tmp_path = tmp.name
+                gdf.to_parquet(tmp_path, index=False, compression="snappy")
+                self.storage.upload(tmp_path, path)
+                Path(tmp_path).unlink()
+        else:
+            # Local filesystem - atomic write
+            tmp_path = f"{path}.tmp-{uuid.uuid4().hex}"
+            self.storage.makedirs(Path(path).parent)
+
+            with self.storage.open(tmp_path, "wb") as f:
+                gdf.to_parquet(f, index=False, compression="snappy")
+
+            self.storage.rename(tmp_path, path)
+
+    def _download_stac_item(self, url: str) -> dict[str, Any] | None:
+        """Download and parse a STAC item from URL."""
+        try:
+            if url.startswith("s3://"):
+                fs = fsspec.filesystem("s3")
+                with fs.open(url, "r") as f:
+                    return cast(dict[str, Any], json.load(f))
+            else:
+                import requests
+
+                response = requests.get(url, timeout=30)
+                response.raise_for_status()
+                return cast(dict[str, Any], response.json())
+        except (
+            requests.exceptions.RequestException,
+            json.JSONDecodeError,
+            OSError,
+            ValueError,
+        ) as e:
+            logger.error(f"Failed to download STAC item from {url}: {e}")
+            return None
+
+    def _chunk_urls(self, urls: list[str], n_chunks: int) -> list[list[str]]:
+        """Split URLs into chunks for parallel processing."""
+        chunk_size = max(1, len(urls) // n_chunks)
+        chunks = [urls[i : i + chunk_size] for i in range(0, len(urls), chunk_size)]
+        # Ensure we have exactly n_chunks by combining last small chunk if needed
+        if len(chunks) > n_chunks:
+            chunks[-2].extend(chunks[-1])
+            chunks = chunks[:-1]
+        return chunks
+
+    def _cleanup_scratch(self, shard_info: list[dict[str, Any]]):
+        """Clean up scratch space after successful processing."""
+        logger.info("Cleaning up scratch space...")
+        failed = 0
+        for info in shard_info:
+            try:
+                self.scratch_storage.remove(info["shard_path"])
+            except OSError as e:
+                failed += 1
+                logger.debug(f"Failed to cleanup shard {info['shard_path']}: {e}")
+
+        if failed > 0:
+            logger.warning(f"Failed to cleanup {failed} shards")