PyGeoFetch 1.0.6__tar.gz → 1.0.8__tar.gz

pygeofetch-1.0.8/CHANGELOG.md

@@ -0,0 +1,42 @@
+# Changelog
+
+All notable changes to PyGeoFetch are documented here.
+Format follows [Keep a Changelog](https://keepachangelog.com/en/1.0.8/).
+
+## [1.0.8] — 2026-07-12
+
+### Added
+- Federated search across 22+ satellite data providers
+- Sentinel-1C and Sentinel-1D constellation support (active since May/April 2026)
+- SLC product type search with automatic provider routing
+- Precise orbit file management (POEORB/RESORB) for InSAR workflows
+- 17 spectral indices: NDVI, EVI, SAVI, NDWI, MNDWI, NDBI, NDSI, NDMI,
+  NBR, dNBR, TCT, PCA, Texture, LST, Albedo, Band Math, Stack
+- SAR processing: despeckle, calibrate, flood mapping, coherence
+- 41-step chainable processing pipeline builder
+- YAML pipeline definitions with cron scheduling
+- Cloud Optimized GeoTIFF output
+- Clean search result tables with ANSI-aware column alignment
+- Live download progress with Jupyter notebook HTML widget support
+- Credential redaction in all log output
+
+### Fixed
+- AuthManager.add_credentials() now accepts dict form (BUG 1)
+- download() length/order contract guaranteed (BUG 2)
+- Partial downloads detected via file validation (BUG 3)
+- CRS identity transform detection after reprojection (BUG 4)
+- resolve_band_keys missing import in aws_earth provider
+- Band alias conflict between Sentinel-2 and Landsat naming conventions
+- Copernicus product_type OData filter now correctly applied
+- USGS M2M API authentication payload (authType, catalogId fields)
+- ZIP archive validation no longer attempts rasterio open on archives
+
+### Providers
+- Copernicus Dataspace (Sentinel-1/2/3/5P, SLC + GRD)
+- AWS Earth (Sentinel-2, Landsat via STAC)
+- Planetary Computer (Sentinel-2, Landsat, MODIS)
+- Element84 Earth Search (STAC)
+- USGS Earth Explorer (Landsat 1-9, ASTER, SRTM)
+- NASA Earthdata (MODIS, ICESat-2, GEDI)
+- Alaska SAR Facility (Sentinel-1 SLC, ALOS PALSAR)
+- Planet Labs, Sentinel Hub, Maxar, Airbus, OpenTopography and more

pygeofetch-1.0.8/MANIFEST.in

@@ -0,0 +1,11 @@
+include README.md
+include LICENSE
+include CHANGELOG.md
+include pyproject.toml
+recursive-include pygeofetch *.py *.yaml *.typed
+recursive-exclude * __pycache__
+recursive-exclude * *.pyc
+recursive-exclude * *.pyo
+exclude tests
+exclude notebooks
+exclude docs

{pygeofetch-1.0.6 → pygeofetch-1.0.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pygeofetch
-Version: 1.0.6
+Version: 1.0.8
 Summary: Universal satellite data pipeline — unified access to 22+ providers
 Author: PyGeoFetch Contributors
 License: MIT License

{pygeofetch-1.0.6 → pygeofetch-1.0.8}/pygeofetch/__init__.py

@@ -1,5 +1,5 @@
 """
-PyGeoFetch v1.0.6 — Universal Satellite Data Pipeline.
+PyGeoFetch v1.0.8 — Universal Satellite Data Pipeline.
 
 Unified access to 22+ satellite data providers with a consistent
 Python API and CLI. Supports authentication management, federated search,
@@ -30,7 +30,7 @@ CLI::
     pygeofetch doctor
 """
 
-__version__ = "1.0.6"
+__version__ = "1.0.8"
 __author__ = "PyGeoFetch Contributors"
 __license__ = "MIT"
 

{pygeofetch-1.0.6 → pygeofetch-1.0.8}/pygeofetch/cli/main.py

@@ -1,5 +1,5 @@
 """
-PyGeoFetch CLI v1.0.6 main entry point.
+PyGeoFetch CLI v1.0.8 main entry point.
 
 Full command reference:
   pygeofetch auth       — credential management
@@ -59,7 +59,7 @@ def cli(ctx: click.Context, log_level: str, log_file: str, log_format: str,
         config_file: str) -> None:
     """
     \b
-    PyGeoFetch v1.0.6 — Universal Satellite Data Pipeline
+    PyGeoFetch v1.0.8 — Universal Satellite Data Pipeline
     ===========================================================
     Unified access to 22+ satellite data providers.
 

{pygeofetch-1.0.6 → pygeofetch-1.0.8}/pygeofetch/core/downloader.py

@@ -30,13 +30,14 @@ from uuid import uuid4
 
 from pygeofetch.models.download_task import (
     DownloadOptions,
-    DownloadProgress,
+    DownloadProgress as _ModelDownloadProgress,
     DownloadResult,
     DownloadStatus,
     DownloadTask,
 )
 from pygeofetch.models.satellite_data import SatelliteData
-from pygeofetch.core.logging import get_logger, _render_progress_bar
+from pygeofetch.core.logging import (get_logger, _render_progress_bar,
+    DownloadProgress)
 
 logger = get_logger(__name__)
 
@@ -91,7 +92,12 @@ class AdaptiveDownloader:
         destination.mkdir(parents=True, exist_ok=True)
 
         provider = self._get_provider(data.provider)
-        logger.info(f"Downloading {data.id!r} from {data.provider!r} → {destination}")
+        _item_start = __import__("time").time()
+        _dl_progress = globals().get("_active_progress")
+        if _dl_progress:
+            _dl_progress.start_item(str(data.id))
+        else:
+            logger.info("Downloading %s", str(data.id)[:60])
 
         for attempt in range(options.retry_attempts + 1):
             try:
@@ -179,29 +185,43 @@ class AdaptiveDownloader:
         completed_count = 0
         total = len(data_list)
 
-        logger.info("Downloading %d scene(s) to: %s", total, str(destination))
+        dp = DownloadProgress(total=total, destination=str(destination))
+        dp.__enter__()
+
+        def _run_one(data: SatelliteData, idx: int) -> tuple:
+            """Download one item and return (idx, result, bytes, duration)."""
+            import time as _it
+            t0 = _it.time()
+            dp.start_item(str(data.id))
+            try:
+                result = self.download(data, destination / data.provider, options)
+                dur    = _it.time() - t0
+                dp.complete_item(
+                    success=result.success,
+                    bytes_total=result.bytes_downloaded or 0,
+                    duration=dur,
+                )
+                return idx, result
+            except Exception as exc:
+                dur    = _it.time() - t0
+                result = DownloadResult(
+                    status=DownloadStatus.FAILED,
+                    data_id=data.id,
+                    provider=data.provider,
+                    error=str(exc),
+                )
+                dp.complete_item(success=False, duration=dur)
+                return idx, result
 
         with ThreadPoolExecutor(max_workers=options.parallel) as executor:
-            future_to_idx = {
-                executor.submit(self.download, data, destination / data.provider, options): i
+            futures = {
+                executor.submit(_run_one, data, i): i
                 for i, data in enumerate(data_list)
             }
-            for future in as_completed(future_to_idx):
-                idx = future_to_idx[future]
-                try:
-                    result = future.result()
-                    results[idx] = result
-                except Exception as exc:
-                    result = DownloadResult(
-                        status=DownloadStatus.FAILED,
-                        data_id=data_list[idx].id,
-                        provider=data_list[idx].provider,
-                        error=str(exc),
-                    )
-                    results[idx] = result
-
+            for future in as_completed(futures):
+                idx, result = future.result()
+                results[idx] = result
                 completed_count += 1
-                self._emit_progress(data_list, results, options)
                 if item_done_callback:
                     item_done_callback(completed_count, total, result)
 
@@ -218,6 +238,7 @@ class AdaptiveDownloader:
 
         completed = sum(1 for r in results if r and r.success)
         failed    = sum(1 for r in results if r and not r.success)
+        dp.__exit__(None, None, None)
         assert len(results) == len(data_list), (
             f"BUG: result count {len(results)} != input count {len(data_list)}"
         )
@@ -225,11 +246,7 @@ class AdaptiveDownloader:
             r.bytes_downloaded / (1024*1024)
             for r in results if r and r.success and r.bytes_downloaded
         )
-        import time as _t
-        logger.info(
-            "Download complete: %d/%d succeeded  %.0f MB total",
-            completed, len(data_list), total_mb
-        )
+        # summary printed by DownloadProgress footer
         return results  # type: ignore — all slots filled, never None
 
     def _get_provider(self, provider_id: str) -> Any:
@@ -519,55 +536,94 @@ class AdaptiveDownloader:
 
     def _validate_downloaded_file(self, path) -> tuple:
         """
-        Validate a downloaded raster file by reading a sample tile.
+        Validate a downloaded file.
 
-        For non-raster formats (JSON, XML, CSV), checks file size > 0.
+        Strategy by file type:
+          .zip / .tar.gz  — check it's a valid archive (not truncated)
+          .tif / .tiff    — open with rasterio and read one tile
+          .jp2            — open with rasterio (GDAL driver)
+          .nc             — check size > 0 (netCDF, GDAL can open but slow)
+          .json / .xml / .csv / .txt / .parquet — check size > 0
+          anything else   — check size > 0 (safe fallback, never fail good data)
 
         Returns:
             (is_valid: bool, error_message: str)
         """
         import os
-        path = str(path)
-        suffix = path.rsplit(".", 1)[-1].lower() if "." in path else ""
+        path_str = str(path)
+        path_obj = Path(path_str)
+        suffix   = path_obj.suffix.lower()
 
-        # Non-raster: just check readable and non-empty
-        NON_RASTER = {"json", "xml", "csv", "txt", "html", "md", "yaml", "yml"}
-        if suffix in NON_RASTER:
+        # ── 0. File must exist and be non-empty ───────────────────────────
+        try:
+            size = os.path.getsize(path_str)
+        except OSError as exc:
+            return False, f"Cannot stat file: {exc}"
+        if size == 0:
+            return False, "File is empty (0 bytes)"
+
+        # ── 1. ZIP / TAR archives — validate archive structure ─────────────
+        if suffix == ".zip" or path_str.endswith(".zip"):
             try:
-                size = os.path.getsize(path)
-                if size == 0:
-                    return False, "File is empty (0 bytes)"
+                import zipfile
+                with zipfile.ZipFile(path_str) as zf:
+                    bad = zf.testzip()
+                    if bad:
+                        return False, f"ZIP is corrupt — bad entry: {bad}"
+                return True, ""
+            except zipfile.BadZipFile as exc:
+                return False, f"Invalid ZIP file: {exc}"
+            except Exception as exc:
+                # If we can't import zipfile or something weird — just trust size
                 return True, ""
-            except OSError as exc:
-                return False, f"Cannot stat file: {exc}"
 
-        # Raster: try rasterio
-        try:
-            import rasterio
-            with rasterio.open(path) as src:
-                if src.count == 0:
-                    return False, "File has zero bands"
-                if src.width == 0 or src.height == 0:
-                    return False, f"File has zero dimensions: {src.width}x{src.height}"
-                windows = list(src.block_windows(1))
-                if not windows:
-                    return False, "File has no readable blocks"
-                _, window = windows[0]
-                sample = src.read(1, window=window)
-                if sample is None or sample.size == 0:
-                    return False, "First tile returned empty array"
-            return True, ""
-        except ImportError:
-            # rasterio not installed — fall back to size check
+        if suffix in (".gz", ".tgz") or path_str.endswith(".tar.gz"):
+            try:
+                import tarfile
+                with tarfile.open(path_str) as tf:
+                    tf.getmembers()
+                return True, ""
+            except tarfile.TarError as exc:
+                return False, f"Invalid TAR file: {exc}"
+            except Exception:
+                return True, ""
+
+        # ── 2. Non-raster text/data formats — size check only ─────────────
+        NON_RASTER = {
+            ".json", ".geojson", ".xml", ".csv", ".txt",
+            ".html", ".md", ".yaml", ".yml", ".parquet", ".nc",
+            ".EOF",   # orbit files
+        }
+        if suffix in NON_RASTER or not suffix:
+            return True, ""   # already passed size > 0 above
+
+        # ── 3. Raster formats — open with rasterio ────────────────────────
+        RASTER_SUFFIXES = {".tif", ".tiff", ".jp2", ".img", ".vrt",
+                           ".hdf", ".h4", ".h5", ".hdf5"}
+        if suffix in RASTER_SUFFIXES:
             try:
-                size = os.path.getsize(path)
-                if size == 0:
-                    return False, "File is empty (0 bytes)"
+                import rasterio
+                with rasterio.open(path_str) as src:
+                    if src.count == 0:
+                        return False, "Raster has zero bands"
+                    if src.width == 0 or src.height == 0:
+                        return False, f"Raster has zero dimensions: {src.width}x{src.height}"
+                    # Read one small tile to confirm data is accessible
+                    windows = list(src.block_windows(1))
+                    if windows:
+                        _, window = windows[0]
+                        sample = src.read(1, window=window)
+                        if sample is None or sample.size == 0:
+                            return False, "First tile read returned empty array"
+                return True, ""
+            except ImportError:
+                # rasterio not installed — size check already passed
                 return True, ""
-            except OSError as exc:
-                return False, f"Cannot stat file: {exc}"
-        except Exception as exc:
-            return False, f"File read validation failed: {exc}"
+            except Exception as exc:
+                return False, f"Raster read failed: {exc}"
+
+        # ── 4. Unknown extension — trust size check (already passed) ──────
+        return True, ""
 
     def _has_identity_transform(self, path) -> bool:
         """
@@ -673,4 +729,4 @@ class AdaptiveDownloader:
     @property
     def _result_total_size_mb(self) -> float:
         """Property used by result objects."""
-        return 0.0
+        return 0.0