earthcatalog 0.2.0__py3-none-any.whl

earthcatalog/schema_generator.py

@@ -0,0 +1,911 @@
+"""Comprehensive catalog schema generator for EarthCatalog metadata and documentation.
+
+This module provides intelligent schema generation capabilities that create detailed
+metadata describing catalog structure, partitioning strategies, performance characteristics,
+and usage patterns. The generated schemas enable efficient catalog discovery, validation,
+and integration with spatial query tools.
+
+Schema Generation Features:
+    - Complete catalog metadata with processing configuration
+    - Spatial partitioning documentation with grid system details
+    - Temporal organization structure and binning strategies
+    - Statistical summaries and performance metrics
+    - Usage examples and query optimization guidance
+    - Version tracking and compatibility information
+
+Generated Schema Components:
+    Catalog Info: Basic metadata about source data and processing configuration
+    Spatial Partitioning: Detailed grid system parameters and spatial organization
+    Temporal Partitioning: Time-based binning configuration and structure
+    Partition Structure: Directory layout and file organization patterns
+    Global Partitioning: Large geometry handling and optimization strategies
+    Statistics: Performance metrics, item counts, and processing summaries
+    Usage Guidelines: Query examples and optimization recommendations
+
+Integration Benefits:
+    - Enables spatial_resolver to automatically configure partition resolution
+    - Provides documentation for catalog users and developers
+    - Supports catalog validation and integrity checking
+    - Facilitates integration with external tools and frameworks
+    - Enables performance monitoring and optimization tracking
+
+Performance Tracking:
+    The schema includes detailed performance metrics that help users understand:
+    - Processing throughput and timing information
+    - Memory usage patterns and optimization opportunities
+    - Error rates and reliability statistics
+    - Partition distribution and load balancing effectiveness
+
+Example Generated Schema Structure:
+    {
+        "earthcatalog_version": "1.0.0",
+        "generated_at": "2024-12-04T10:30:00Z",
+        "spatial_partitioning": {
+            "grid_system": "h3",
+            "resolution": 6,
+            "cell_area_km2": 36.1,
+            "description": "H3 hexagonal grid level 6..."
+        },
+        "temporal_partitioning": {
+            "bin_size": "month",
+            "pattern": "YYYY-MM",
+            "description": "Monthly temporal bins..."
+        },
+        "statistics": {
+            "total_items": 1000000,
+            "processing_time_seconds": 3600,
+            "partitions_created": 450,
+            "average_items_per_partition": 2222
+        }
+    }
+
+Usage Patterns:
+    >>> # Generate schema during pipeline execution
+    >>> generator = SchemaGenerator(config, grid, storage)
+    >>> schema = generator.generate_catalog_schema(stats)
+    >>>
+    >>> # Load existing schema for analysis
+    >>> with open('catalog_schema.json') as f:
+    ...     schema = json.load(f)
+    >>> print(f"Catalog uses {schema['spatial_partitioning']['grid_system']} grid")
+"""
+
+import json
+import logging
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from . import grid_systems
+from .statistics import IngestionStatistics
+
+logger = logging.getLogger(__name__)
+
+
+class SchemaGenerator:
+    """Intelligent catalog schema generator producing comprehensive metadata for EarthCatalog outputs.
+
+    This class generates rich metadata schemas that document every aspect of catalog
+    creation, from spatial partitioning strategies to performance characteristics.
+    The schemas serve as both technical documentation and enable automated tooling
+    for catalog discovery, validation, and query optimization.
+
+    Schema Generation Philosophy:
+        The generator follows a comprehensive documentation approach that captures
+        not just configuration parameters, but also derived information, performance
+        metrics, and usage guidance. This enables users to understand catalog
+        characteristics without examining the underlying data structure.
+
+    Key Features:
+        - Automatic grid system parameter calculation and documentation
+        - Comprehensive performance metrics and statistical summaries
+        - Usage examples and query optimization recommendations
+        - Version tracking for catalog format evolution
+        - Integration metadata for external tool compatibility
+
+    Generated Metadata Categories:
+        Configuration: Complete processing configuration and parameters
+        Spatial: Grid system details, resolution characteristics, and spatial organization
+        Temporal: Time-based binning strategies and partition patterns
+        Structure: Directory layout, file naming conventions, and organization
+        Performance: Processing metrics, throughput statistics, and optimization data
+        Usage: Query examples, best practices, and integration guidance
+
+    Performance Documentation:
+        The generator captures detailed performance characteristics including:
+        - Processing throughput and timing breakdowns
+        - Memory usage patterns and resource requirements
+        - Error rates and reliability metrics
+        - Partition distribution and load balancing statistics
+        - Query performance optimization recommendations
+
+    Integration Support:
+        Generated schemas enable seamless integration with:
+        - spatial_resolver for automatic partition resolution
+        - DuckDB and other query engines via documented structure
+        - Monitoring and observability tools via performance metrics
+        - Catalog validation and integrity checking systems
+        - External geospatial analysis frameworks
+
+    Thread Safety:
+        This class is thread-safe for read operations after initialization.
+        Schema generation methods can be called concurrently, though each
+        instance should be used for a single catalog generation workflow.
+
+    Example:
+        >>> # Initialize with pipeline components
+        >>> generator = SchemaGenerator(config, grid_system, storage_backend)
+        >>>
+        >>> # Generate complete schema with processing statistics
+        >>> schema = generator.generate_catalog_schema(
+        ...     partition_stats={'total_items': 1000000, 'processing_time': 3600}
+        ... )
+        >>>
+        >>> # Schema automatically written to catalog directory
+        >>> print(f"Generated schema for {schema['statistics']['total_items']} items")
+    """
+
+    def __init__(
+        self,
+        config: Any,
+        grid: grid_systems.GridSystem,
+        storage: Any,
+        stats: IngestionStatistics | None = None,
+    ):
+        """Initialize schema generator with pipeline configuration.
+
+        Args:
+            config: ProcessingConfig with all pipeline settings
+            grid: Grid system instance used for partitioning
+            storage: Storage backend for the output catalog
+            stats: Optional IngestionStatistics with comprehensive metrics
+        """
+        self.config = config
+        self.grid = grid
+        self.storage = storage
+        self.stats = stats
+
+    def generate_catalog_schema(
+        self, partition_stats: dict[str, Any], output_filename: str = "catalog_schema.json"
+    ) -> dict[str, Any]:
+        """Generate complete catalog schema metadata.
+
+        Args:
+            partition_stats: Statistics from pipeline execution
+            output_filename: Name of schema file to write
+
+        Returns:
+            Dictionary containing the complete schema
+        """
+        logger.info("Generating catalog schema metadata...")
+
+        # Build the schema
+        schema = {
+            "earthcatalog_version": "1.0.0",  # TODO: Get from package version
+            "schema_version": "1.0.0",
+            "generated_at": datetime.now(UTC).isoformat().replace("+00:00", "Z"),
+            "catalog_info": self._get_catalog_info(),
+            "spatial_partitioning": self._get_spatial_partitioning_info(),
+            "temporal_partitioning": self._get_temporal_partitioning_info(),
+            "partition_structure": self._get_partition_structure(partition_stats),
+            "global_partitioning": self._get_global_partitioning_info(),
+            "statistics": self._get_catalog_statistics(partition_stats),
+            "usage": self._get_usage_info(),
+        }
+
+        # Write schema to output location
+        self._write_schema(schema, output_filename)
+
+        return schema
+
+    def _get_catalog_info(self) -> dict[str, Any]:
+        """Get basic catalog information."""
+        config_dict = {
+            "sort_key": self.config.sort_key,
+            "sort_ascending": self.config.sort_ascending,
+            "items_per_shard": self.config.items_per_shard,
+            "max_workers": self.config.max_workers,
+        }
+
+        # Add standard configuration
+        config_dict["output_format"] = getattr(self.config, "output_format", "geoparquet")
+        config_dict["mission_field"] = getattr(self.config, "mission_field", "dataset_id")
+
+        return {
+            "output_path": self.config.output_catalog,
+            "input_source": self.config.input_file,
+            "processing_config": config_dict,
+            "directory_structure": "{mission}/partition={grid_type}/level={resolution}/{h3_cell}/{temporal_bin}.{ext}",
+            "mission_extraction": f"Extracts mission from '{config_dict['mission_field']}' field, falls back to 'collection'",
+            "output_format": config_dict["output_format"],
+        }
+
+    def _get_spatial_partitioning_info(self) -> dict[str, Any]:
+        """Get spatial partitioning configuration."""
+        grid_info = {
+            "grid_system": self.config.grid_system,
+            "coordinate_system": "EPSG:4326",  # All grids use WGS84
+            "description": self._get_grid_description(),
+        }
+
+        # Add grid-specific parameters
+        if self.config.grid_system == "h3":
+            grid_info.update(
+                {
+                    "resolution": self.config.grid_resolution,
+                    "resolution_description": self._get_h3_resolution_description(self.config.grid_resolution),
+                    "cell_area_km2": self._get_h3_average_area(self.config.grid_resolution),
+                    "cell_edge_length_km": self._get_h3_average_edge_length(self.config.grid_resolution),
+                }
+            )
+        elif self.config.grid_system == "s2":
+            grid_info.update(
+                {
+                    "level": self.config.grid_resolution,
+                    "level_description": self._get_s2_level_description(self.config.grid_resolution),
+                    "average_cell_area_km2": self._get_s2_average_area(self.config.grid_resolution),
+                }
+            )
+        elif self.config.grid_system == "mgrs":
+            grid_info.update(
+                {
+                    "precision": self.config.grid_resolution,
+                    "precision_description": self._get_mgrs_precision_description(self.config.grid_resolution),
+                }
+            )
+        elif self.config.grid_system == "utm":
+            grid_info.update(
+                {
+                    "precision": self.config.grid_resolution,
+                    "precision_description": self._get_utm_precision_description(self.config.grid_resolution),
+                }
+            )
+        elif self.config.grid_system == "latlon":
+            grid_info.update(
+                {
+                    "cell_size_degrees": self.config.grid_resolution,
+                    "cell_size_description": f"Each cell is {self.config.grid_resolution}° x {self.config.grid_resolution}°",
+                }
+            )
+        elif self.config.grid_system == "itslive":
+            grid_info.update(
+                {
+                    "cell_size_degrees": 10,
+                    "cell_size_description": "Fixed 10° x 10° cells with center-based naming",
+                    "naming_convention": "{N|S}{lat:02d}{E|W}{lon:03d}",
+                    "example_cell_name": "N60W040",
+                }
+            )
+        elif self.config.grid_system == "geojson":
+            grid_info.update(
+                {
+                    "custom_grid": True,
+                    "geojson_source": getattr(self.config, "geojson_path", "unknown"),
+                    "custom_tiles": self._get_custom_tiles_info(),
+                }
+            )
+
+        return grid_info
+
+    def _get_temporal_partitioning_info(self) -> dict[str, Any]:
+        """Get temporal partitioning configuration."""
+        return {
+            "temporal_bin": self.config.temporal_bin,
+            "temporal_bin_description": {
+                "year": "Items partitioned by year using Hive-style directories (year=YYYY/)",
+                "month": "Items partitioned by year-month using Hive-style directories (year=YYYY/month=MM/)",
+                "day": "Items partitioned by year-month-day using Hive-style directories (year=YYYY/month=MM/day=DD/)",
+            }[self.config.temporal_bin],
+            "datetime_field": self.config.sort_key,
+            "hive_path_examples": {
+                "year": "year=2024/items.parquet",
+                "month": "year=2024/month=01/items.parquet",
+                "day": "year=2024/month=01/day=15/items.parquet",
+            }[self.config.temporal_bin],
+            "pruning_benefit": "Directory-level pruning in DuckDB, Athena, Spark, and Trino",
+        }
+
+    def _get_partition_structure(self, partition_stats: dict[str, Any]) -> dict[str, Any]:
+        """Get information about the actual partitions created."""
+        partitions = []
+        spatial_partitions = set()
+        temporal_partitions = set()
+        missions = set()
+
+        for partition_key, stats in partition_stats.items():
+            # Parse partition key with Hive-style temporal parts:
+            # "mission/partition=grid_type/level=resolution/spatial_id/year=YYYY/month=MM[/day=DD]"
+            parts = partition_key.split("/")
+            if len(parts) >= 5:
+                mission = parts[0]
+                # Skip partition=grid_type and level=resolution parts
+                spatial_id = parts[3] if len(parts) > 3 else "unknown"
+
+                # Extract Hive-style temporal parts (year=YYYY, month=MM, day=DD)
+                temporal_parts = []
+                for part in parts[4:]:
+                    if part.startswith(("year=", "month=", "day=")):
+                        temporal_parts.append(part)
+
+                # Convert to readable format (e.g., "year=2024/month=01" -> "2024-01")
+                temporal_bin = self._hive_parts_to_temporal_bin(temporal_parts)
+
+                missions.add(mission)
+                spatial_partitions.add(spatial_id)
+                temporal_partitions.add(temporal_bin)
+
+                partitions.append(
+                    {
+                        "partition_key": partition_key,
+                        "mission": mission,
+                        "spatial_id": spatial_id,
+                        "temporal_bin": temporal_bin,
+                        "total_items": stats.get("total_items", 0),
+                        "new_items": stats.get("new_items", 0),
+                        "existing_items": stats.get("existing_items", 0),
+                    }
+                )
+
+        result = {
+            "total_partitions": len(partitions),
+            "spatial_partitions_count": len(spatial_partitions),
+            "temporal_partitions_count": len(temporal_partitions),
+            "missions_count": len(missions),
+            "spatial_partitions": sorted(spatial_partitions),
+            "temporal_partitions": sorted(temporal_partitions),
+            "missions": sorted(missions),
+            "partitions": partitions,
+        }
+
+        return result
+
+    def _hive_parts_to_temporal_bin(self, temporal_parts: list[str]) -> str:
+        """Convert Hive-style temporal parts to readable temporal bin format.
+
+        Args:
+            temporal_parts: List like ["year=2024", "month=01", "day=15"]
+
+        Returns:
+            Formatted string like "2024-01-15" or "2024-01" or "2024"
+        """
+        if not temporal_parts:
+            return "unknown"
+
+        year = month = day = None
+        for part in temporal_parts:
+            if part.startswith("year="):
+                year = part.split("=")[1]
+            elif part.startswith("month="):
+                month = part.split("=")[1]
+            elif part.startswith("day="):
+                day = part.split("=")[1]
+
+        if year and month and day:
+            return f"{year}-{month}-{day}"
+        elif year and month:
+            return f"{year}-{month}"
+        elif year:
+            return year
+        else:
+            return "unknown"
+
+    def _get_global_partitioning_info(self) -> dict[str, Any]:
+        """Get global partitioning configuration."""
+        return {
+            "enabled": self.config.enable_global_partitioning,
+            "threshold": self.config.global_partition_threshold,
+            "description": (
+                (
+                    "Items spanning more than the threshold number of spatial cells "
+                    "are placed in the 'global' partition instead of individual cells"
+                )
+                if self.config.enable_global_partitioning
+                else "Global partitioning disabled"
+            ),
+        }
+
+    def _get_catalog_statistics(self, partition_stats: dict[str, Any]) -> dict[str, Any]:
+        """Get overall catalog statistics.
+
+        If IngestionStatistics is available, returns comprehensive metrics including:
+        - Unique granule counts (via HyperLogLog)
+        - Overhead/duplication metrics
+        - Spatial and temporal distribution
+        - Data quality metrics
+        - Processing performance
+
+        Falls back to basic partition-derived stats if IngestionStatistics not provided.
+        """
+        # If we have comprehensive statistics from ingestion, use them
+        if self.stats is not None:
+            return self.stats.get_summary()
+
+        # Fallback: derive basic stats from partition_stats
+        total_items = sum(s.get("total_items", 0) for s in partition_stats.values())
+        new_items = sum(s.get("new_items", 0) for s in partition_stats.values())
+        existing_items = sum(s.get("existing_items", 0) for s in partition_stats.values())
+
+        return {
+            "unique_granules": total_items,  # Best approximation without HyperLogLog
+            "stored_references": total_items,
+            "total_partitions": len(partition_stats),
+            "total_files": len(partition_stats),
+            "overhead": {
+                "spanning_items": 0,
+                "spanning_percentage": 0.0,
+                "duplication_ratio": 1.0,
+                "overhead_percentage": 0.0,
+                "avg_tiles_per_spanning_item": 0.0,
+                "max_tiles_per_item": 0,
+                "tiles_distribution": {},
+            },
+            "global_partition": {
+                "items_routed_to_global": 0,
+                "percentage_global": 0.0,
+            },
+            "spatial": {
+                "bbox": None,
+                "cells_with_data": len(partition_stats),
+                "items_per_cell": None,
+                "hotspot_cells": [],
+            },
+            "temporal": {
+                "earliest": None,
+                "latest": None,
+                "years_with_data": [],
+                "distribution": {},
+            },
+            "quality": {
+                "null_geometries": 0,
+                "missing_datetime": 0,
+                "geometry_types": {},
+            },
+            "missions": {},
+            "files": {
+                "total_count": len(partition_stats),
+                "total_size_bytes": None,
+                "size_stats": None,
+                "items_per_file": None,
+            },
+            "processing": {
+                "run_timestamp": None,
+                "duration_seconds": 0,
+                "urls_processed": 0,
+                "urls_failed": 0,
+                "success_rate": 100.0,
+                "items_per_second": 0.0,
+                "new_items": new_items,
+                "existing_items": existing_items,
+                "duplicates_removed": 0,
+            },
+        }
+
+    def _get_usage_info(self) -> dict[str, Any]:
+        """Get usage examples and recommendations."""
+        return {
+            "file_structure": self._get_file_structure_info(),
+            "spatial_partition_resolution": {
+                "description": "Automatically resolve spatial partitions that intersect with your area of interest",
+                "python_example": """
+from earthcatalog.spatial_resolver import spatial_resolver
+from shapely.geometry import box
+import duckdb
+
+# Load resolver from this schema
+resolver = spatial_resolver('catalog_schema.json')
+
+# For remote schemas (requires fsspec):
+# resolver = spatial_resolver('s3://bucket/catalog_schema.json', 's3://bucket/catalog/')
+# resolver = spatial_resolver('https://example.com/schema.json', './catalog/')
+
+# Define area of interest (example: San Francisco Bay Area)
+aoi = box(-122.5, 37.7, -122.0, 38.0)
+
+# Resolve intersecting partitions dynamically
+partition_ids = resolver.resolve_partitions(aoi, overlap=True, buffer_cells=1)
+
+# Generate query paths with Hive-style temporal filtering
+# '2024-01' becomes 'year=2024/month=01/items.parquet'
+query_paths = resolver.generate_query_paths(partition_ids, '2024-01')
+
+# Query only relevant data - DuckDB skips non-matching temporal directories
+if query_paths:
+    result = duckdb.sql(f"SELECT * FROM read_parquet({query_paths})").df()
+    print(f"Queried {len(partition_ids)} partitions, found {len(result)} items")
+""",
+                "grid_specific_notes": self._get_grid_specific_notes(),
+            },
+            "partition_pruning": {
+                "description": "Use Hive-style directory structure for automatic partition pruning",
+                "recommended_approach": "Use SpatialPartitionResolver for automatic spatial and temporal filtering",
+                "manual_spatial_filter_example": self._get_spatial_filter_example(),
+                "temporal_filter_note": "Temporal filtering uses Hive-style directories (year=YYYY/month=MM/) for directory-level pruning",
+                "duckdb_examples": self._get_duckdb_examples(),
+            },
+            "recommended_tools": [
+                {
+                    "tool": "EarthCatalog SpatialPartitionResolver",
+                    "use_case": "Automatic spatial partition resolution from geometry",
+                    "example": "resolver.resolve_partitions(your_geometry) -> ['partition1', 'partition2']",
+                },
+                {
+                    "tool": "DuckDB",
+                    "use_case": "Fast analytical queries with automatic partition pruning",
+                    "example": "SELECT * FROM read_parquet(['catalog/**/year=2024/month=01/items.parquet'])",
+                },
+                {
+                    "tool": "Apache Arrow/Parquet",
+                    "use_case": "Column-oriented analysis and filtering",
+                    "example": "Use spatial_id and temporal_bin columns for efficient filtering",
+                },
+            ],
+        }
+
+    def _get_grid_description(self) -> str:
+        """Get human-readable description of the grid system."""
+        descriptions = {
+            "h3": "Uber H3 hexagonal hierarchical spatial index",
+            "s2": "Google S2 spherical geometry library cells",
+            "mgrs": "Military Grid Reference System (MGRS) grid",
+            "utm": "Universal Transverse Mercator (UTM) grid zones",
+            "latlon": "Simple latitude-longitude rectangular grid",
+            "itslive": "ITS_LIVE center-based 10-degree grid for glacier/ice datasets",
+            "geojson": "Custom grid defined by GeoJSON features",
+        }
+        return descriptions.get(self.config.grid_system, f"Unknown grid system: {self.config.grid_system}")
+
+    def _get_h3_resolution_description(self, resolution: int) -> str:
+        """Get description of H3 resolution level."""
+        descriptions = {
+            0: "Very coarse - continents/countries",
+            1: "Large countries/regions",
+            2: "Countries/large states",
+            3: "States/provinces",
+            4: "Large counties/regions",
+            5: "Counties/metropolitan areas",
+            6: "Cities/large municipalities",
+            7: "City districts/neighborhoods",
+            8: "Neighborhoods/census blocks",
+            9: "City blocks/large buildings",
+            10: "Buildings/small areas",
+            11: "Building parts/rooms",
+            12: "Very fine - room-level detail",
+            13: "Sub-room level",
+            14: "Very high precision",
+            15: "Highest precision",
+        }
+        return descriptions.get(resolution, f"Resolution {resolution}")
+
+    def _get_h3_average_area(self, resolution: int) -> float | None:
+        """Get average H3 cell area in km²."""
+        # Approximate areas from H3 documentation
+        areas = {
+            0: 4250546.848,
+            1: 607220.9782,
+            2: 86745.85403,
+            3: 12392.26486,
+            4: 1770.323552,
+            5: 252.9033645,
+            6: 36.1290521,
+            7: 5.1612932,
+            8: 0.7373276,
+            9: 0.1053325,
+            10: 0.0150475,
+            11: 0.0021496,
+            12: 0.0003071,
+            13: 0.0000439,
+            14: 0.0000063,
+            15: 0.0000009,
+        }
+        return areas.get(resolution)
+
+    def _get_h3_average_edge_length(self, resolution: int) -> float | None:
+        """Get average H3 cell edge length in km."""
+        # Approximate edge lengths from H3 documentation
+        edges = {
+            0: 1107.712591,
+            1: 418.6760055,
+            2: 158.2446558,
+            3: 59.81085794,
+            4: 22.6063794,
+            5: 8.544408276,
+            6: 3.229953667,
+            7: 1.220629759,
+            8: 0.461354684,
+            9: 0.174375668,
+            10: 0.065907807,
+            11: 0.024910561,
+            12: 0.009415526,
+            13: 0.003559893,
+            14: 0.001348575,
+            15: 0.000509713,
+        }
+        return edges.get(resolution)
+
+    def _get_s2_level_description(self, level: int) -> str:
+        """Get description of S2 level."""
+        if level <= 3:
+            return f"Very coarse - level {level}"
+        elif level <= 6:
+            return f"Coarse - level {level}"
+        elif level <= 10:
+            return f"Medium - level {level}"
+        elif level <= 15:
+            return f"Fine - level {level}"
+        else:
+            return f"Very fine - level {level}"
+
+    def _get_s2_average_area(self, level: int) -> float | None:
+        """Get approximate S2 cell area in km²."""
+        # S2 cells get 4x smaller each level
+        base_area = 85011012.19  # Area at level 0 in km²
+        return float(base_area / (4**level))
+
+    def _get_mgrs_precision_description(self, precision: int) -> str:
+        """Get MGRS precision description."""
+        descriptions = {
+            1: "100km x 100km grid squares",
+            2: "10km x 10km grid squares",
+            3: "1km x 1km grid squares",
+            4: "100m x 100m grid squares",
+            5: "10m x 10m grid squares",
+            6: "1m x 1m grid squares",
+        }
+        return descriptions.get(precision, f"Precision {precision}")
+
+    def _get_utm_precision_description(self, precision: int) -> str:
+        """Get UTM precision description."""
+        descriptions = {
+            1: "UTM zones (6° wide)",
+            2: "100km x 100km squares",
+            3: "10km x 10km squares",
+            4: "1km x 1km squares",
+            5: "100m x 100m squares",
+        }
+        return descriptions.get(precision, f"Precision {precision}")
+
+    def _get_custom_tiles_info(self) -> dict[str, Any] | None:
+        """Get information about custom tiles if using GeoJSON grid."""
+        if not hasattr(self.config, "geojson_path") or not self.config.geojson_path:
+            return None
+
+        try:
+            # Try to read the GeoJSON file to get tile information
+            if self.config.geojson_path.startswith("s3://"):
+                # Would need to implement S3 reading for geojson
+                return {"note": "Custom GeoJSON tiles (S3 path cannot be analyzed)"}
+            else:
+                with open(self.config.geojson_path) as f:
+                    geojson_data = json.load(f)
+
+                features = geojson_data.get("features", [])
+                tile_ids = [f.get("properties", {}).get("id") for f in features]
+                tile_ids = [tid for tid in tile_ids if tid]  # Filter None values
+
+                return {"total_tiles": len(features), "tile_ids": tile_ids, "source_file": self.config.geojson_path}
+        except (OSError, json.JSONDecodeError, ValueError, TypeError) as e:
+            logger.warning(f"Could not read custom GeoJSON file: {e}")
+            return {"note": "Custom GeoJSON tiles (file could not be read)"}
+
+    def _get_spatial_filter_example(self) -> str:
+        """Get example of spatial filtering based on grid system."""
+        if self.config.grid_system == "h3":
+            return "spatial_id IN ('8a2a1072b59ffff', '8a2a1072b5bffff', ...)"
+        elif self.config.grid_system == "itslive":
+            return "spatial_id IN ('N60W040', 'N60W030', 'N70W040', ...)"
+        elif self.config.grid_system == "geojson":
+            return "spatial_id IN ('region_a', 'region_b', ...)"
+        else:
+            return f"spatial_id IN ('tile_1', 'tile_2', ...) -- {self.config.grid_system} tile IDs"
+
+    def _get_duckdb_examples(self) -> list[dict[str, str]]:
+        """Get DuckDB query examples for partition pruning."""
+        examples = []
+
+        # Dynamic spatial resolution example (recommended approach)
+        global_note = ""
+        if self.config.enable_global_partitioning:
+            global_note = f"""
+# NOTE: Large queries automatically include 'global' partition
+# Threshold: {self.config.global_partition_threshold} cells
+# Items spanning > threshold cells are stored in global/"""
+
+        examples.append(
+            {
+                "description": "Dynamic spatial partition resolution with global partition support (RECOMMENDED)",
+                "query": f"""
+# Python code using SpatialPartitionResolver
+from earthcatalog.spatial_resolver import spatial_resolver
+from shapely.geometry import box
+
+# Load resolver from schema
+resolver = spatial_resolver('catalog_schema.json')
+
+# Small area (city-scale) - no global partition needed
+small_aoi = box(-122.5, 37.7, -122.0, 38.0)  # San Francisco
+small_partitions = resolver.resolve_partitions(small_aoi, overlap=True)
+print(f"Small query: {{len(small_partitions)}} partitions, global: {{'global' in small_partitions}}")
+
+# Large area (state-scale) - automatically includes global partition!
+large_aoi = box(-124.0, 32.0, -114.0, 42.0)  # California
+large_partitions = resolver.resolve_partitions(large_aoi, overlap=True)
+print(f"Large query: {{len(large_partitions)}} partitions, global: {{'global' in large_partitions}}")
+
+# Generate Hive-style query paths with temporal filter
+# '2024-01' becomes 'year=2024/month=01/items.parquet'
+query_patterns = resolver.generate_query_paths(large_partitions, '2024-01')
+
+# Query captures both spatial cells AND global partition items
+# DuckDB skips non-matching temporal directories (directory-level pruning)
+import duckdb
+result = duckdb.sql(f"SELECT * FROM read_parquet({{query_patterns}})").df(){global_note}
+""".strip(),
+            }
+        )
+
+        # Manual spatial filter (fallback)
+        examples.append(
+            {
+                "description": "Manual spatial partition filter (if you know the partition IDs)",
+                "query": """
+SELECT * FROM read_parquet('catalog/**/items.parquet')
+WHERE spatial_id IN ('8a2a1072b59ffff', '8a2a1072b5bffff')
+""".strip(),
+            }
+        )
+
+        # Temporal filter using directory paths
+        examples.append(
+            {
+                "description": f"Filter by temporal range using Hive-style paths ({self.config.temporal_bin})",
+                "query": """
+-- Use glob patterns for directory-level temporal pruning (most efficient)
+SELECT * FROM read_parquet('catalog/**/year=2024/month=*/items.parquet')
+
+-- Or query all and filter by datetime column (less efficient, but more flexible)
+SELECT * FROM read_parquet('catalog/**/items.parquet')
+WHERE datetime >= '2024-01-01' AND datetime < '2025-01-01'
+""".strip(),
+            }
+        )
+
+        # Combined with spatial intersection
+        examples.append(
+            {
+                "description": "Combine partition pruning with geometric intersection",
+                "query": """
+-- First use SpatialPartitionResolver to get relevant partitions, then:
+-- The resolver returns Hive-style paths like 'catalog/.../year=2024/month=06/items.parquet'
+SELECT * FROM read_parquet('catalog/[resolved_partitions]/**/year=2024/month=06/items.parquet')
+WHERE ST_Intersects(geometry, ST_GeomFromText('POLYGON((-122.5 37.7, -122.0 37.7, -122.0 38.0, -122.5 38.0, -122.5 37.7))'))
+""".strip(),
+            }
+        )
+
+        # Global partition query
+        if self.config.enable_global_partitioning:
+            examples.append(
+                {
+                    "description": "Query items that span multiple spatial partitions",
+                    "query": """
+-- Query global partition with Hive-style temporal directory
+SELECT * FROM read_parquet('catalog/**/global/year=2024/month=01/items.parquet')
+""".strip(),
+                }
+            )
+
+        return examples
+
+    def _get_grid_specific_notes(self) -> dict[str, str]:
+        """Get grid-specific notes for spatial resolution."""
+        notes = {
+            "h3": "H3 cells use hexagonal geometry. Use overlap=True to include boundary cells.",
+            "s2": "S2 cells use spherical geometry. Higher levels provide finer resolution.",
+            "mgrs": "MGRS uses military grid system. Precision affects cell size.",
+            "utm": "UTM zones are 6° wide. Consider zone boundaries for large areas.",
+            "latlon": "Simple rectangular grid. Cell size is in degrees.",
+            "itslive": "ITS_LIVE uses fixed 10° cells with center-based naming. Optimized for glacier datasets.",
+            "geojson": "Custom geometry tiles. Intersection depends on your tile definitions.",
+        }
+        return {self.config.grid_system: notes.get(self.config.grid_system, "Grid-specific resolution")}
+
+    def _get_file_structure_info(self) -> dict[str, str | list[str]]:
+        """Get file structure description and examples."""
+        ext = getattr(self.config, "output_format", "geoparquet")
+        ext_suffix = "parquet" if ext == "geoparquet" else ext
+
+        # Describe Hive-style temporal partitioning
+        temporal_desc = {
+            "year": "year={YYYY}",
+            "month": "year={YYYY}/month={MM}",
+            "day": "year={YYYY}/month={MM}/day={DD}",
+        }.get(self.config.temporal_bin, "year={YYYY}/month={MM}")
+
+        description = (
+            f"Catalog uses Hive-style partitioning: "
+            f"{{mission}}/partition={{grid_type}}/level={{resolution}}/{{spatial_id}}/{temporal_desc}/items.{ext_suffix}"
+        )
+
+        grid_type = self.config.grid_system
+        resolution = self.config.grid_resolution
+
+        if self.config.grid_system == "h3":
+            if self.config.temporal_bin == "day":
+                example_paths = [
+                    f"sentinel2/partition={grid_type}/level={resolution}/8a2a1072b59ffff/year=2024/month=01/day=15/items.{ext_suffix}",
+                    f"landsat8/partition={grid_type}/level={resolution}/8a2a1072b5bffff/year=2024/month=01/day=15/items.{ext_suffix}",
+                ]
+            elif self.config.temporal_bin == "year":
+                example_paths = [
+                    f"sentinel2/partition={grid_type}/level={resolution}/8a2a1072b59ffff/year=2024/items.{ext_suffix}",
+                    f"landsat8/partition={grid_type}/level={resolution}/8a2a1072b5bffff/year=2024/items.{ext_suffix}",
+                ]
+            else:  # month (default)
+                example_paths = [
+                    f"sentinel2/partition={grid_type}/level={resolution}/8a2a1072b59ffff/year=2024/month=01/items.{ext_suffix}",
+                    f"landsat8/partition={grid_type}/level={resolution}/8a2a1072b5bffff/year=2024/month=01/items.{ext_suffix}",
+                ]
+        else:
+            if self.config.temporal_bin == "day":
+                example_paths = [
+                    f"mission_a/partition={grid_type}/level={resolution}/tile_001/year=2024/month=01/day=15/items.{ext_suffix}",
+                    f"mission_b/partition={grid_type}/level={resolution}/tile_002/year=2024/month=01/day=15/items.{ext_suffix}",
+                ]
+            elif self.config.temporal_bin == "year":
+                example_paths = [
+                    f"mission_a/partition={grid_type}/level={resolution}/tile_001/year=2024/items.{ext_suffix}",
+                    f"mission_b/partition={grid_type}/level={resolution}/tile_002/year=2024/items.{ext_suffix}",
+                ]
+            else:  # month (default)
+                example_paths = [
+                    f"mission_a/partition={grid_type}/level={resolution}/tile_001/year=2024/month=01/items.{ext_suffix}",
+                    f"mission_b/partition={grid_type}/level={resolution}/tile_002/year=2024/month=01/items.{ext_suffix}",
+                ]
+
+        if self.config.enable_global_partitioning:
+            if self.config.temporal_bin == "day":
+                example_paths.append(
+                    f"sentinel2/partition={grid_type}/level={resolution}/global/year=2024/month=01/day=15/items.{ext_suffix}"
+                )
+            elif self.config.temporal_bin == "year":
+                example_paths.append(
+                    f"sentinel2/partition={grid_type}/level={resolution}/global/year=2024/items.{ext_suffix}"
+                )
+            else:
+                example_paths.append(
+                    f"sentinel2/partition={grid_type}/level={resolution}/global/year=2024/month=01/items.{ext_suffix}"
+                )
+
+        return {
+            "description": description,
+            "example_paths": example_paths,
+            "temporal_partitioning": f"Hive-style ({self.config.temporal_bin} granularity)",
+            "pruning_benefit": "DuckDB/Athena/Spark skip entire directories during temporal filtering",
+        }
+
+    def _write_schema(self, schema: dict[str, Any], filename: str) -> None:
+        """Write schema to the output location."""
+        schema_content = json.dumps(schema, indent=2, ensure_ascii=False)
+
+        # Build full path to schema file
+        full_path = f"{self.config.output_catalog}/{filename}"
+
+        try:
+            # Use the storage backend to write the schema
+            self.storage.makedirs(Path(self.config.output_catalog))
+            with self.storage.open(full_path, "w") as f:
+                f.write(schema_content.encode("utf-8"))
+        except (OSError, ValueError, TypeError, RuntimeError) as e:
+            logger.error(f"Failed to write schema using storage backend: {e}")
+            # Fallback to local write if storage backend fails
+            try:
+                output_path = Path(self.config.output_catalog) / filename
+                output_path.parent.mkdir(parents=True, exist_ok=True)
+                with open(output_path, "w", encoding="utf-8") as f:
+                    f.write(schema_content)
+                logger.info(f"Schema written to local path: {output_path}")
+            except (OSError, ValueError, TypeError) as fallback_error:
+                logger.error(f"Failed to write schema to local path: {fallback_error}")
+                raise
+
+        logger.info(f"Schema written to: {full_path}")