earthcatalog 0.2.0__py3-none-any.whl

earthcatalog/__init__.py

@@ -0,0 +1,164 @@
+"""EarthCatalog - High-Performance STAC Ingestion with Async HTTP Processing.
+
+A scalable, production-ready library for processing STAC items into spatially partitioned
+GeoParquet catalogs with built-in 3-6x performance improvements through async HTTP processing.
+
+Key Features:
+    - **Async HTTP Processing**: 50-100+ concurrent requests per worker (3-6x speedup)
+    - **Spatial Partitioning**: H3, S2, MGRS, UTM, and custom grid systems
+    - **Distributed Processing**: Single-machine to cluster-scale processing
+    - **Cloud-Native**: Native S3, GCS, Azure integration
+    - **Memory Efficient**: Handles 100M+ URL datasets with constant memory usage
+    - **Production Ready**: Comprehensive error handling, monitoring, and reliability
+
+Performance:
+    - Sequential HTTP: ~16 requests/second
+    - Async HTTP: 50-100+ requests/second (3-6x improvement)
+    - Large datasets: 100M URLs processed in 7-14 hours vs 71 hours sequential
+    - Memory usage: Linear with worker count, not dataset size
+
+Quick Start:
+    >>> import earthcatalog
+    >>>
+    >>> # Basic configuration (async enabled by default)
+    >>> config = earthcatalog.ProcessingConfig(
+    ...     input_file='urls.parquet',
+    ...     output_catalog='./catalog',
+    ...     scratch_location='./scratch'
+    ... )
+    >>>
+    >>> # Process with local workers
+    >>> processor = earthcatalog.LocalProcessor(n_workers=4)
+    >>> pipeline = earthcatalog.STACIngestionPipeline(config, processor)
+    >>> pipeline.run()
+
+High-Performance Example:
+    >>> # Optimized for 100M+ URLs
+    >>> config = earthcatalog.ProcessingConfig(
+    ...     input_file='s3://bucket/urls.parquet',
+    ...     output_catalog='s3://bucket/catalog',
+    ...     scratch_location='s3://bucket/scratch',
+    ...     enable_concurrent_http=True,    # Default: True
+    ...     concurrent_requests=100,        # High concurrency
+    ...     batch_size=2000,               # Large batches
+    ...     max_workers=32                 # Multiple workers
+    ... )
+
+Async HTTP Configuration:
+    - enable_concurrent_http: Enable/disable async processing (default: True)
+    - concurrent_requests: Simultaneous requests per worker (default: 50)
+    - batch_size: URLs processed per batch (default: 1000)
+    - request_timeout: Request timeout in seconds (default: 30)
+    - retry_attempts: Maximum retry attempts (default: 3)
+
+Grid Systems with Async Optimization:
+    - H3GridSystem: Recommended for global datasets (optimal async performance)
+    - S2GridSystem: Excellent for polar regions (adaptive async batching)
+    - UTMGridSystem: High precision for regional data (zone-optimized async)
+    - MGRSGridSystem: Government/defense standard (structured async processing)
+
+Requirements:
+    - Python 3.11+
+    - aiohttp>=3.9.0 (for async HTTP - automatically installed)
+    - Optional: Dask for distributed processing
+    - Optional: S3FS for cloud storage
+"""
+
+__version__ = "0.2.0"
+__author__ = "betolink"
+__email__ = "betolin@gmail.com"
+
+# Import main classes for convenient access
+from .engines import (
+    EngineNotAvailableError,
+    STACEngine,
+    get_engine,
+)
+from .grid_systems import (
+    GridSystem,
+    H3GridSystem,
+    MGRSGridSystem,
+    S2GridSystem,
+    SimpleLatLonGrid,
+    UTMGridSystem,
+)
+from .ingestion_pipeline import (
+    DaskDistributedProcessor,
+    LocalProcessor,
+    ProcessingConfig,
+    STACIngestionPipeline,
+)
+from .input_readers import (
+    CSVReader,
+    InputReader,
+    ParquetReader,
+)
+from .schema_generator import (
+    SchemaGenerator,
+)
+from .spatial_resolver import (
+    SpatialPartitionResolver,
+    resolve_and_query,
+    spatial_resolver,
+)
+from .storage_backends import (
+    LocalStorage,
+    S3Storage,
+    StorageBackend,
+    get_storage_backend,
+)
+from .validation import (
+    CatalogValidationResult,
+    ValidationIssue,
+    ValidationResult,
+    validate_catalog,
+    validate_geoparquet_file,
+    validate_stac_item,
+    validate_stac_items_batch,
+)
+
+__all__ = [
+    # Version info
+    "__version__",
+    "__author__",
+    "__email__",
+    # Core pipeline
+    "STACIngestionPipeline",
+    "ProcessingConfig",
+    "LocalProcessor",
+    "DaskDistributedProcessor",
+    # Storage backends
+    "StorageBackend",
+    "LocalStorage",
+    "S3Storage",
+    "get_storage_backend",
+    # STAC Engines
+    "STACEngine",
+    "EngineNotAvailableError",
+    "get_engine",
+    # Grid systems
+    "GridSystem",
+    "H3GridSystem",
+    "S2GridSystem",
+    "MGRSGridSystem",
+    "UTMGridSystem",
+    "SimpleLatLonGrid",
+    # Input readers
+    "InputReader",
+    "ParquetReader",
+    "CSVReader",
+    # Schema generation
+    "SchemaGenerator",
+    # Spatial resolution
+    "SpatialPartitionResolver",
+    "spatial_resolver",
+    "resolve_and_query",
+    # Validation
+    "ValidationResult",
+    "ValidationIssue",
+    "CatalogValidationResult",
+    "validate_stac_item",
+    "validate_stac_items_batch",
+    "validate_geoparquet_file",
+    "validate_catalog",
+]