PyGeoFetch 1.0.7__tar.gz → 1.0.8__tar.gz

pygeofetch-1.0.8/CHANGELOG.md

@@ -0,0 +1,42 @@
+# Changelog
+
+All notable changes to PyGeoFetch are documented here.
+Format follows [Keep a Changelog](https://keepachangelog.com/en/1.0.8/).
+
+## [1.0.8] — 2026-07-12
+
+### Added
+- Federated search across 22+ satellite data providers
+- Sentinel-1C and Sentinel-1D constellation support (active since May/April 2026)
+- SLC product type search with automatic provider routing
+- Precise orbit file management (POEORB/RESORB) for InSAR workflows
+- 17 spectral indices: NDVI, EVI, SAVI, NDWI, MNDWI, NDBI, NDSI, NDMI,
+  NBR, dNBR, TCT, PCA, Texture, LST, Albedo, Band Math, Stack
+- SAR processing: despeckle, calibrate, flood mapping, coherence
+- 41-step chainable processing pipeline builder
+- YAML pipeline definitions with cron scheduling
+- Cloud Optimized GeoTIFF output
+- Clean search result tables with ANSI-aware column alignment
+- Live download progress with Jupyter notebook HTML widget support
+- Credential redaction in all log output
+
+### Fixed
+- AuthManager.add_credentials() now accepts dict form (BUG 1)
+- download() length/order contract guaranteed (BUG 2)
+- Partial downloads detected via file validation (BUG 3)
+- CRS identity transform detection after reprojection (BUG 4)
+- resolve_band_keys missing import in aws_earth provider
+- Band alias conflict between Sentinel-2 and Landsat naming conventions
+- Copernicus product_type OData filter now correctly applied
+- USGS M2M API authentication payload (authType, catalogId fields)
+- ZIP archive validation no longer attempts rasterio open on archives
+
+### Providers
+- Copernicus Dataspace (Sentinel-1/2/3/5P, SLC + GRD)
+- AWS Earth (Sentinel-2, Landsat via STAC)
+- Planetary Computer (Sentinel-2, Landsat, MODIS)
+- Element84 Earth Search (STAC)
+- USGS Earth Explorer (Landsat 1-9, ASTER, SRTM)
+- NASA Earthdata (MODIS, ICESat-2, GEDI)
+- Alaska SAR Facility (Sentinel-1 SLC, ALOS PALSAR)
+- Planet Labs, Sentinel Hub, Maxar, Airbus, OpenTopography and more

pygeofetch-1.0.8/MANIFEST.in

@@ -0,0 +1,11 @@
+include README.md
+include LICENSE
+include CHANGELOG.md
+include pyproject.toml
+recursive-include pygeofetch *.py *.yaml *.typed
+recursive-exclude * __pycache__
+recursive-exclude * *.pyc
+recursive-exclude * *.pyo
+exclude tests
+exclude notebooks
+exclude docs

{pygeofetch-1.0.7 → pygeofetch-1.0.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pygeofetch
-Version: 1.0.7
+Version: 1.0.8
 Summary: Universal satellite data pipeline — unified access to 22+ providers
 Author: PyGeoFetch Contributors
 License: MIT License

{pygeofetch-1.0.7 → pygeofetch-1.0.8}/pygeofetch/__init__.py

@@ -1,5 +1,5 @@
 """
-PyGeoFetch v1.0.7 — Universal Satellite Data Pipeline.
+PyGeoFetch v1.0.8 — Universal Satellite Data Pipeline.
 
 Unified access to 22+ satellite data providers with a consistent
 Python API and CLI. Supports authentication management, federated search,
@@ -30,7 +30,7 @@ CLI::
     pygeofetch doctor
 """
 
-__version__ = "1.0.7"
+__version__ = "1.0.8"
 __author__ = "PyGeoFetch Contributors"
 __license__ = "MIT"
 

{pygeofetch-1.0.7 → pygeofetch-1.0.8}/pygeofetch/cli/main.py

@@ -1,5 +1,5 @@
 """
-PyGeoFetch CLI v1.0.7 main entry point.
+PyGeoFetch CLI v1.0.8 main entry point.
 
 Full command reference:
   pygeofetch auth       — credential management
@@ -59,7 +59,7 @@ def cli(ctx: click.Context, log_level: str, log_file: str, log_format: str,
         config_file: str) -> None:
     """
     \b
-    PyGeoFetch v1.0.7 — Universal Satellite Data Pipeline
+    PyGeoFetch v1.0.8 — Universal Satellite Data Pipeline
     ===========================================================
     Unified access to 22+ satellite data providers.
 

{pygeofetch-1.0.7 → pygeofetch-1.0.8}/pygeofetch/core/downloader.py

@@ -536,55 +536,94 @@ class AdaptiveDownloader:
 
     def _validate_downloaded_file(self, path) -> tuple:
         """
-        Validate a downloaded raster file by reading a sample tile.
+        Validate a downloaded file.
 
-        For non-raster formats (JSON, XML, CSV), checks file size > 0.
+        Strategy by file type:
+          .zip / .tar.gz  — check it's a valid archive (not truncated)
+          .tif / .tiff    — open with rasterio and read one tile
+          .jp2            — open with rasterio (GDAL driver)
+          .nc             — check size > 0 (netCDF, GDAL can open but slow)
+          .json / .xml / .csv / .txt / .parquet — check size > 0
+          anything else   — check size > 0 (safe fallback, never fail good data)
 
         Returns:
             (is_valid: bool, error_message: str)
         """
         import os
-        path = str(path)
-        suffix = path.rsplit(".", 1)[-1].lower() if "." in path else ""
+        path_str = str(path)
+        path_obj = Path(path_str)
+        suffix   = path_obj.suffix.lower()
 
-        # Non-raster: just check readable and non-empty
-        NON_RASTER = {"json", "xml", "csv", "txt", "html", "md", "yaml", "yml"}
-        if suffix in NON_RASTER:
+        # ── 0. File must exist and be non-empty ───────────────────────────
+        try:
+            size = os.path.getsize(path_str)
+        except OSError as exc:
+            return False, f"Cannot stat file: {exc}"
+        if size == 0:
+            return False, "File is empty (0 bytes)"
+
+        # ── 1. ZIP / TAR archives — validate archive structure ─────────────
+        if suffix == ".zip" or path_str.endswith(".zip"):
             try:
-                size = os.path.getsize(path)
-                if size == 0:
-                    return False, "File is empty (0 bytes)"
+                import zipfile
+                with zipfile.ZipFile(path_str) as zf:
+                    bad = zf.testzip()
+                    if bad:
+                        return False, f"ZIP is corrupt — bad entry: {bad}"
+                return True, ""
+            except zipfile.BadZipFile as exc:
+                return False, f"Invalid ZIP file: {exc}"
+            except Exception as exc:
+                # If we can't import zipfile or something weird — just trust size
                 return True, ""
-            except OSError as exc:
-                return False, f"Cannot stat file: {exc}"
 
-        # Raster: try rasterio
-        try:
-            import rasterio
-            with rasterio.open(path) as src:
-                if src.count == 0:
-                    return False, "File has zero bands"
-                if src.width == 0 or src.height == 0:
-                    return False, f"File has zero dimensions: {src.width}x{src.height}"
-                windows = list(src.block_windows(1))
-                if not windows:
-                    return False, "File has no readable blocks"
-                _, window = windows[0]
-                sample = src.read(1, window=window)
-                if sample is None or sample.size == 0:
-                    return False, "First tile returned empty array"
-            return True, ""
-        except ImportError:
-            # rasterio not installed — fall back to size check
+        if suffix in (".gz", ".tgz") or path_str.endswith(".tar.gz"):
+            try:
+                import tarfile
+                with tarfile.open(path_str) as tf:
+                    tf.getmembers()
+                return True, ""
+            except tarfile.TarError as exc:
+                return False, f"Invalid TAR file: {exc}"
+            except Exception:
+                return True, ""
+
+        # ── 2. Non-raster text/data formats — size check only ─────────────
+        NON_RASTER = {
+            ".json", ".geojson", ".xml", ".csv", ".txt",
+            ".html", ".md", ".yaml", ".yml", ".parquet", ".nc",
+            ".EOF",   # orbit files
+        }
+        if suffix in NON_RASTER or not suffix:
+            return True, ""   # already passed size > 0 above
+
+        # ── 3. Raster formats — open with rasterio ────────────────────────
+        RASTER_SUFFIXES = {".tif", ".tiff", ".jp2", ".img", ".vrt",
+                           ".hdf", ".h4", ".h5", ".hdf5"}
+        if suffix in RASTER_SUFFIXES:
             try:
-                size = os.path.getsize(path)
-                if size == 0:
-                    return False, "File is empty (0 bytes)"
+                import rasterio
+                with rasterio.open(path_str) as src:
+                    if src.count == 0:
+                        return False, "Raster has zero bands"
+                    if src.width == 0 or src.height == 0:
+                        return False, f"Raster has zero dimensions: {src.width}x{src.height}"
+                    # Read one small tile to confirm data is accessible
+                    windows = list(src.block_windows(1))
+                    if windows:
+                        _, window = windows[0]
+                        sample = src.read(1, window=window)
+                        if sample is None or sample.size == 0:
+                            return False, "First tile read returned empty array"
                 return True, ""
-            except OSError as exc:
-                return False, f"Cannot stat file: {exc}"
-        except Exception as exc:
-            return False, f"File read validation failed: {exc}"
+            except ImportError:
+                # rasterio not installed — size check already passed
+                return True, ""
+            except Exception as exc:
+                return False, f"Raster read failed: {exc}"
+
+        # ── 4. Unknown extension — trust size check (already passed) ──────
+        return True, ""
 
     def _has_identity_transform(self, path) -> bool:
         """

pygeofetch-1.0.8/pygeofetch/meta.yaml

@@ -0,0 +1,62 @@
+{% set version = "1.0.8" %}
+
+package:
+  name: pygeofetch
+  version: {{ version }}
+
+source:
+  url: https://pypi.org/packages/source/p/pygeofetch/pygeofetch-{{ version }}.tar.gz
+  sha256: 0307001e4a585f035c709ccc61c7b23217280a7ddf2775229600340d878b5cf2
+
+build:
+  entry_points:
+    - pygeofetch = pygeofetch.cli.main:cli
+  noarch: python
+  script: {{ PYTHON }} -m pip install . -vv --no-deps --no-build-isolation
+  number: 0
+
+requirements:
+  host:
+    - python >=3.9
+    - setuptools >=68
+    - wheel
+    - pip
+  run:
+    - python >=3.9
+    - httpx >=0.27
+    - click >=8.1
+    - pydantic >=2.5
+    - pydantic-settings >=2.1
+    - rich >=13.7
+    - pyyaml >=6.0
+    - cryptography >=42.0
+    - keyring >=24.3
+    - aiofiles >=23.2
+    - tenacity >=8.2
+    - python-dateutil >=2.9
+    - tqdm >=4.66
+    - requests >=2.31
+    - anyio >=4.2
+    - boto3 >=1.34
+    - click-completion >=0.5
+
+test:
+  imports:
+    - pygeofetch
+  commands:
+    - pip check
+    - pygeofetch --help
+  requires:
+    - pip
+    - python
+
+about:
+  home: https://github.com/appiahkubis14/pygeofetch
+  summary: Universal satellite data pipeline — unified access to 22+ providers
+  doc_url: https://pygeofetch.readthedocs.io
+  license: MIT
+  license_file: LICENSE
+
+extra:
+  recipe-maintainers:
+    - appiahkubis14

pygeofetch-1.0.8/pygeofetch/processing/base.py

@@ -0,0 +1,315 @@
+"""Base classes, result types, and shared I/O helpers for the processing engine."""
+from __future__ import annotations
+
+import logging
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple, Union
+
+logger = logging.getLogger(__name__)
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+# Result type
+# ══════════════════════════════════════════════════════════════════════════════
+
+@dataclass
+class ProcessingResult:
+    """Outcome of a single processing step."""
+    success: bool
+    output_path: Optional[Path] = None
+    input_path:  Optional[Path] = None
+    operation:   str            = ""
+    duration_seconds: float     = 0.0
+    metadata: Dict[str, Any]    = field(default_factory=dict)
+    error: Optional[str]        = None
+
+    @property
+    def path(self) -> Optional[Path]:
+        return self.output_path
+
+    def __str__(self) -> str:
+        status = "✓" if self.success else "✗"
+        out    = self.output_path.name if self.output_path else "none"
+        return f"{status} {self.operation} → {out} ({self.duration_seconds:.2f}s)"
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+# Dependency guards
+# ══════════════════════════════════════════════════════════════════════════════
+
+def _require_rasterio():
+    try:
+        import rasterio
+        return rasterio
+    except ImportError:
+        raise ImportError(
+            "rasterio is required for raster processing.\n"
+            "Install with: pip install rasterio\n"
+            "Or:           pip install \"PyGeoFetch[geo]\""
+        )
+
+
+def _require_numpy():
+    try:
+        import numpy as np
+        return np
+    except ImportError:
+        raise ImportError("numpy is required: pip install numpy")
+
+
+def _require_scipy():
+    try:
+        from scipy import ndimage
+        return ndimage
+    except ImportError:
+        raise ImportError("scipy is required: pip install scipy")
+
+
+def _require_geopandas():
+    try:
+        import geopandas as gpd
+        return gpd
+    except ImportError:
+        raise ImportError(
+            "geopandas is required for vector operations.\n"
+            "Install with: pip install geopandas"
+        )
+
+
+def _require_shapely():
+    try:
+        import shapely
+        return shapely
+    except ImportError:
+        raise ImportError("shapely is required: pip install shapely")
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+# _timed decorator — catches exceptions → returns ProcessingResult(success=False)
+# ══════════════════════════════════════════════════════════════════════════════
+
+def _timed(func):
+    """
+    Measure execution time.
+
+    On success: sets result.duration_seconds and returns result.
+    On exception: logs the error and returns ProcessingResult(success=False, error=...)
+    so callers always get a result object, never a raw exception.
+    """
+    from functools import wraps
+
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        t0 = time.time()
+        try:
+            result = func(*args, **kwargs)
+            if isinstance(result, ProcessingResult):
+                result.duration_seconds = time.time() - t0
+            return result
+        except Exception as exc:
+            elapsed = time.time() - t0
+            op = func.__name__.replace("_cmd", "")
+            logger.error("%s failed after %.2fs: %s", op, elapsed, exc, exc_info=True)
+            return ProcessingResult(
+                success=False,
+                operation=op,
+                duration_seconds=elapsed,
+                error=str(exc),
+            )
+
+    return wrapper
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+# Output path resolution
+# ══════════════════════════════════════════════════════════════════════════════
+
+def _resolve_output(
+    input_path: Optional[Path],
+    output: Optional[str],
+    suffix: str,
+) -> Path:
+    """Build an output path from input + suffix when output is not specified."""
+    if output:
+        out = Path(output)
+        if out.is_dir() or str(output).endswith("/"):
+            out.mkdir(parents=True, exist_ok=True)
+            stem = input_path.stem if input_path else "output"
+            return out / f"{stem}_{suffix}.tif"
+        out.parent.mkdir(parents=True, exist_ok=True)
+        return out
+    p = input_path or Path("output.tif")
+    return p.parent / f"{p.stem}_{suffix}.tif"
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+# Shared raster I/O — robust to tiled / COG / compressed GeoTIFFs
+# ══════════════════════════════════════════════════════════════════════════════
+
+def _safe_read_band(
+    path: Union[str, Path],
+    band: int = 1,
+    out_shape: Optional[Tuple[int, int]] = None,
+) -> Tuple["Any", dict, Any]:
+    """
+    Read a single band from any GeoTIFF, including tiled and COG formats.
+
+    Strategy:
+    1. Try the fast path: ``src.read(band)`` — works for stripped TIFFs.
+    2. If that fails (tiled COG, DEFLATE-encoded, JPEG-compressed tiles),
+       fall back to block-by-block reading which rasterio handles tile by tile.
+    3. Apply optional resampling to ``out_shape`` after reading.
+
+    Args:
+        path:      Path to a raster file.
+        band:      1-based band index to read.
+        out_shape: Optional (height, width) to resample to after reading.
+
+    Returns:
+        (data, profile, nodata)
+        data    — float32 numpy array, NaN where nodata
+        profile — rasterio profile dict for writing output
+        nodata  — original nodata value (may be None)
+    """
+    rasterio = _require_rasterio()
+    np       = _require_numpy()
+
+    p = Path(path)
+    if not p.exists():
+        raise FileNotFoundError(f"Raster not found: {p}")
+
+    with rasterio.open(p) as src:
+        if band > src.count:
+            raise ValueError(
+                f"Band {band} requested but file has only {src.count} band(s): {p.name}"
+            )
+
+        nodata  = src.nodata
+        profile = src.profile.copy()
+        h, w    = src.height, src.width
+
+        # ── attempt 1: fast whole-array read ──────────────────────────────
+        try:
+            data = src.read(band).astype(np.float32)
+        except Exception as fast_exc:
+            logger.debug(
+                "Fast read failed for %s (band %d): %s — using block-by-block fallback",
+                p.name, band, fast_exc,
+            )
+            # ── attempt 2: block-by-block read ────────────────────────────
+            data = np.empty((h, w), dtype=np.float32)
+            data[:] = np.nan
+
+            try:
+                for _, window in src.block_windows(band):
+                    row_off = window.row_off
+                    col_off = window.col_off
+                    row_end = row_off + window.height
+                    col_end = col_off + window.width
+                    try:
+                        block = src.read(band, window=window).astype(np.float32)
+                        data[row_off:row_end, col_off:col_end] = block
+                    except Exception as block_exc:
+                        # Fill failed blocks with nodata/NaN — don't crash
+                        logger.warning(
+                            "Block read failed at (%d,%d) in %s: %s — filling NaN",
+                            row_off, col_off, p.name, block_exc,
+                        )
+                        fill = float(nodata) if nodata is not None else np.nan
+                        data[row_off:row_end, col_off:col_end] = fill
+
+            except Exception as block_exc2:
+                raise RuntimeError(
+                    f"Both fast-read and block-read failed for {p.name} band {band}.\n"
+                    f"Fast error: {fast_exc}\n"
+                    f"Block error: {block_exc2}\n"
+                    "The file may be corrupt. Try: gdalinfo -checksum <file>"
+                ) from block_exc2
+
+    # ── nodata → NaN ──────────────────────────────────────────────────────
+    if nodata is not None:
+        nd = float(nodata)
+        if np.isnan(nd):
+            # nodata is NaN — already NaN in float32, nothing to do
+            pass
+        else:
+            data = np.where(data == nd, np.nan, data)
+
+    # ── optional resampling ───────────────────────────────────────────────
+    if out_shape is not None and out_shape != (h, w):
+        from scipy.ndimage import zoom
+        zoom_factors = (out_shape[0] / h, out_shape[1] / w)
+        data = zoom(data, zoom_factors, order=1, mode="nearest").astype(np.float32)
+
+    return data, profile, nodata
+
+
+def _safe_write_band(
+    data: "Any",
+    profile: dict,
+    out_path: Path,
+    nodata: float = -9999.0,
+    compress: str = "deflate",
+    tiled: bool = True,
+    blocksize: int = 256,
+) -> None:
+    """
+    Write a float32 array to a GeoTIFF, always with valid compression settings.
+
+    Cleans the inherited profile to remove settings incompatible with float32
+    (e.g. JPEG compression, photometric=RGB, etc.).
+
+    Args:
+        data:      2-D or 3-D (bands, h, w) float32 numpy array.
+        profile:   Base profile (from source raster). Will be updated safely.
+        out_path:  Destination path.
+        nodata:    Nodata value for NaN pixels.
+        compress:  Compression: ``"deflate"`` (default), ``"lzw"``, ``"zstd"``, ``None``.
+        tiled:     Write as tiled (COG-ready) GeoTIFF.
+        blocksize: Tile size in pixels (must be power of 2).
+    """
+    np       = _require_numpy()
+    rasterio = _require_rasterio()
+
+    if data.ndim == 2:
+        data_3d = data[np.newaxis, :, :]
+        n_bands = 1
+    else:
+        data_3d = data
+        n_bands = data.shape[0]
+
+    h, w = data_3d.shape[1], data_3d.shape[2]
+
+    # Replace NaN with nodata value
+    data_3d = np.where(np.isnan(data_3d), nodata, data_3d).astype(np.float32)
+
+    # Build a clean profile — remove anything incompatible with float32
+    clean = {
+        "driver":    "GTiff",
+        "dtype":     "float32",
+        "count":     n_bands,
+        "height":    h,
+        "width":     w,
+        "nodata":    nodata,
+        "crs":       profile.get("crs"),
+        "transform": profile.get("transform"),
+    }
+
+    # Only add compression if specified
+    if compress:
+        clean["compress"] = compress
+        if compress in ("deflate", "zstd"):
+            clean["predictor"] = 2   # horizontal differencing for float data
+
+    # Tiling for COG-ready output
+    if tiled and blocksize:
+        clean["tiled"]     = True
+        clean["blockxsize"] = blocksize
+        clean["blockysize"] = blocksize
+
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+
+    with rasterio.open(out_path, "w", **clean) as dst:
+        dst.write(data_3d)