PyGeoFetch 1.0.2__py3-none-any.whl → 1.0.4__py3-none-any.whl

pygeofetch/__init__.py

@@ -1,5 +1,5 @@
 """
-PyGeoFetch v1.0.2 — Universal Satellite Data Pipeline.
+PyGeoFetch v1.0.4 — Universal Satellite Data Pipeline.
 
 Unified access to 22+ satellite data providers with a consistent
 Python API and CLI. Supports authentication management, federated search,
@@ -30,7 +30,7 @@ CLI::
     pygeofetch doctor
 """
 
-__version__ = "1.0.2"
+__version__ = "1.0.4"
 __author__ = "PyGeoFetch Contributors"
 __license__ = "MIT"
 

pygeofetch/cli/main.py

@@ -1,5 +1,5 @@
 """
-PyGeoFetch CLI v1.0.2 main entry point.
+PyGeoFetch CLI v1.0.4 main entry point.
 
 Full command reference:
   pygeofetch auth       — credential management
@@ -59,7 +59,7 @@ def cli(ctx: click.Context, log_level: str, log_file: str, log_format: str,
         config_file: str) -> None:
     """
     \b
-    PyGeoFetch v1.0.2 — Universal Satellite Data Pipeline
+    PyGeoFetch v1.0.4 — Universal Satellite Data Pipeline
     ===========================================================
     Unified access to 22+ satellite data providers.
 
@@ -74,8 +74,7 @@ def cli(ctx: click.Context, log_level: str, log_file: str, log_format: str,
     from pathlib import Path
     setup_logging(
         level=log_level,
-        log_file=Path(log_file) if log_file else None,
-        use_rich=(log_format == "console"),
+        log_file=str(Path(log_file)) if log_file else None,
     )
     ctx.ensure_object(dict)
     ctx.obj["log_level"] = log_level
@@ -777,4 +776,4 @@ def main() -> None:
 
 
 if __name__ == "__main__":
-    main()
+    main()

pygeofetch/core/authenticator.py

@@ -27,7 +27,7 @@ from pathlib import Path
 from typing import Any, Dict, List, Optional
 
 from pygeofetch.models.user_auth import AuthSession, AuthType, Credentials
-from pygeofetch.utils.logging_setup import get_logger
+from pygeofetch.core.logging import get_logger
 
 logger = get_logger(__name__)
 
@@ -295,6 +295,35 @@ class AuthManager:
         self._sessions.pop(provider, None)
         logger.info(f"Credentials saved for provider {provider!r}")
 
+
+    def add_credentials(
+        self,
+        provider: str,
+        creds: Dict[str, Any],
+    ) -> None:
+        """
+        Store credentials for the given provider.
+
+        Accepts a dict (not keyword arguments — the engine passes a dict).
+        Idempotent: calling twice with same provider updates, does not duplicate.
+        Not raise for unknown provider names.
+
+        Args:
+            provider: Provider identifier e.g. "usgs", "planetary_computer"
+            creds:    Dict with any of: username, password, api_key,
+                      client_id, client_secret, token, access_key, secret_key.
+                      Only non-None values are stored.
+        """
+        # Filter out None values and the redundant "provider" key
+        clean_creds = {k: v for k, v in creds.items()
+                       if k != "provider" and v is not None}
+        if not clean_creds:
+            logger.warning(f"No credentials provided for {provider!r} — skipping")
+            return
+        # Delegate to add() which handles storage, session invalidation, type inference
+        self.add(provider, **clean_creds)
+        logger.debug(f"Stored credentials for provider: {provider}")
+
     def get_credentials(self, provider: str) -> Optional[Credentials]:
         """
         Load stored credentials for a provider.

pygeofetch/core/downloader.py

@@ -36,7 +36,7 @@ from pygeofetch.models.download_task import (
     DownloadTask,
 )
 from pygeofetch.models.satellite_data import SatelliteData
-from pygeofetch.utils.logging_setup import get_logger
+from pygeofetch.core.logging import get_logger, _render_progress_bar
 
 logger = get_logger(__name__)
 
@@ -102,16 +102,30 @@ class AdaptiveDownloader:
                 if result.duration_seconds == 0.0 and elapsed > 0:
                     result = result.model_copy(update={"duration_seconds": elapsed})
                 if result.success:
-                    if options.verify_checksum:
-                        result = self._verify_checksums(result, data, options)
-                    if options.post_process:
-                        result = self._run_post_process(result, options)
+                    # BUG 3: validate file integrity before claiming success
+                    for out_p in (result.output_paths or [result.output_path]):
+                        if out_p and out_p.exists():
+                            is_valid, err_msg = self._validate_downloaded_file(out_p)
+                            if not is_valid:
+                                result = result.model_copy(update={
+                                    "status": DownloadStatus.FAILED,
+                                    "error": f"Download appeared complete but file validation failed: {err_msg}",
+                                    "output_path": None,
+                                })
+                                logger.warning(
+                                    f"File validation failed for {data.id!r}: {err_msg}"
+                                )
+                                break
+                    if result.success:
+                        if options.verify_checksum:
+                            result = self._verify_checksums(result, data, options)
+                        if options.post_process:
+                            result = self._run_post_process(result, options)
                     size_mb = result.bytes_downloaded / (1024 * 1024) if result.bytes_downloaded else 0
                     speed = (size_mb / result.duration_seconds) if result.duration_seconds > 0 else 0
                     logger.info(
-                        f"Downloaded {data.id!r}: {size_mb:.1f} MB "
-                        f"in {result.duration_seconds:.1f}s"
-                        + (f" ({speed:.1f} MB/s)" if speed > 0 else "")
+                        "  ✓ %-45s %6.0f MB  %5.1fs",
+                        str(data.id)[:45], size_mb, result.duration_seconds
                     )
                 return result
             except Exception as exc:
@@ -165,9 +179,7 @@ class AdaptiveDownloader:
         completed_count = 0
         total = len(data_list)
 
-        logger.info(
-            f"Downloading {total} items with {options.parallel} parallel workers..."
-        )
+        logger.info("Downloading %d scene(s) to: %s", total, str(destination))
 
         with ThreadPoolExecutor(max_workers=options.parallel) as executor:
             future_to_idx = {
@@ -193,25 +205,44 @@ class AdaptiveDownloader:
                 if item_done_callback:
                     item_done_callback(completed_count, total, result)
 
-        completed = sum(1 for r in results if r and r.success)
-        failed = sum(1 for r in results if r and not r.success)
-        logger.info(f"Downloads complete: {completed} succeeded, {failed} failed")
+        # BUG 2: guarantee length == len(data_list) and order is preserved
+        # Fill any None slots (unexpected future crashes) with failure results
+        for i, item in enumerate(data_list):
+            if results[i] is None:
+                results[i] = DownloadResult(
+                    status=DownloadStatus.FAILED,
+                    data_id=item.id,
+                    provider=item.provider,
+                    error="Download future returned no result (internal error)",
+                )
 
-        return results  # type: ignore
+        completed = sum(1 for r in results if r and r.success)
+        failed    = sum(1 for r in results if r and not r.success)
+        assert len(results) == len(data_list), (
+            f"BUG: result count {len(results)} != input count {len(data_list)}"
+        )
+        total_mb = sum(
+            r.bytes_downloaded / (1024*1024)
+            for r in results if r and r.success and r.bytes_downloaded
+        )
+        import time as _t
+        logger.info(
+            "Download complete: %d/%d succeeded  %.0f MB total",
+            completed, len(data_list), total_mb
+        )
+        return results  # type: ignore — all slots filled, never None
 
     def _get_provider(self, provider_id: str) -> Any:
-        """Get a provider instance with session from auth manager."""
-        if provider_id not in self._provider_cache:
-            from pygeofetch.providers import get_provider
-            prov = get_provider(provider_id)
-            if self.auth_manager and prov.REQUIRES_AUTH:
-                try:
-                    session = self.auth_manager.authenticate(provider_id)
-                    prov.set_session(session)
-                except Exception as exc:
-                    logger.warning(f"Auth for {provider_id} failed: {exc}")
-            self._provider_cache[provider_id] = prov
-        return self._provider_cache[provider_id]
+        """Get a provider instance with a fresh authenticated session."""
+        from pygeofetch.providers import get_provider
+        prov = get_provider(provider_id)
+        if self.auth_manager and prov.REQUIRES_AUTH:
+            try:
+                session = self.auth_manager.authenticate(provider_id)
+                prov.set_session(session)
+            except Exception as exc:
+                logger.warning(f"Auth for {provider_id} failed: {exc}")
+        return prov
 
     def _verify_checksums(
         self, result: DownloadResult, data: SatelliteData, options: DownloadOptions
@@ -285,23 +316,7 @@ class AdaptiveDownloader:
                 for path in result.output_paths:
                     if path.suffix.lower() in (".tif", ".tiff"):
                         out_path = path.with_stem(f"{path.stem}_{target_crs.replace(':', '_')}")
-                        with rasterio.open(path) as src:
-                            transform, width, height = calculate_default_transform(
-                                src.crs, target_crs, src.width, src.height, *src.bounds
-                            )
-                            meta = src.meta.copy()
-                            meta.update(crs=target_crs, transform=transform, width=width, height=height)
-                            with rasterio.open(out_path, "w", **meta) as dst:
-                                for i in range(1, src.count + 1):
-                                    reproject(
-                                        source=rasterio.band(src, i),
-                                        destination=rasterio.band(dst, i),
-                                        src_transform=src.transform,
-                                        src_crs=src.crs,
-                                        dst_transform=transform,
-                                        dst_crs=target_crs,
-                                        resampling=Resampling.nearest,
-                                    )
+                        self._reproject_with_validation(path, out_path, target_crs)
                         new_paths.append(out_path)
                     else:
                         new_paths.append(path)
@@ -310,10 +325,317 @@ class AdaptiveDownloader:
                 logger.warning("rasterio not installed; skipping reproject action")
 
         elif action.action == "compress":
-            logger.debug(f"Compress action {action.params} noted (requires rasterio)")
+            method = action.params.get("value", "lzw")
+            try:
+                import rasterio
+                new_paths = []
+                for path in result.output_paths:
+                    if path.suffix.lower() in (".tif", ".tiff"):
+                        out_path = path.with_stem(f"{path.stem}_{method}")
+                        with rasterio.open(path) as src:
+                            profile = src.profile.copy()
+                            profile.update(compress=method, tiled=True,
+                                           blockxsize=512, blockysize=512)
+                            with rasterio.open(out_path, "w", **profile) as dst:
+                                dst.write(src.read())
+                        new_paths.append(out_path)
+                    else:
+                        new_paths.append(path)
+                result.output_paths = new_paths
+                if new_paths:
+                    result.output_path = new_paths[0]
+            except ImportError:
+                logger.warning("rasterio not installed; skipping compress action")
+
+        elif action.action == "cog":
+            compress = action.params.get("value", "deflate")
+            try:
+                import rasterio, tempfile
+                from rasterio.enums import Resampling as RS
+                new_paths = []
+                for path in result.output_paths:
+                    if path.suffix.lower() in (".tif", ".tiff"):
+                        out_path = path.with_stem(f"{path.stem}_cog")
+                        with rasterio.open(path) as src:
+                            profile = src.profile.copy()
+                            profile.update(compress=compress, tiled=True,
+                                           blockxsize=512, blockysize=512,
+                                           interleave="band", driver="GTiff")
+                            data = src.read()
+                            with tempfile.NamedTemporaryFile(suffix=".tif", delete=False) as tmp:
+                                tmp_path = Path(tmp.name)
+                            with rasterio.open(tmp_path, "w", **profile) as tmp_dst:
+                                tmp_dst.write(data)
+                                tmp_dst.build_overviews([2, 4, 8, 16, 32], RS.average)
+                                tmp_dst.update_tags(ns="rio_overview", resampling="average")
+                            profile.update(copy_src_overviews=True)
+                            with rasterio.open(tmp_path) as tmp_src:
+                                with rasterio.open(out_path, "w", **profile) as cog_dst:
+                                    cog_dst.write(tmp_src.read())
+                            tmp_path.unlink(missing_ok=True)
+                        new_paths.append(out_path)
+                    else:
+                        new_paths.append(path)
+                result.output_paths = new_paths
+                if new_paths:
+                    result.output_path = new_paths[0]
+            except ImportError:
+                logger.warning("rasterio not installed; skipping cog action")
+
+        elif action.action == "clip":
+            import json
+            geom_src = action.params.get("value", "")
+            try:
+                import rasterio
+                from rasterio.mask import mask as rasterio_mask
+                shapes = []
+                if geom_src.endswith(".geojson") or geom_src.endswith(".json"):
+                    with open(geom_src) as f:
+                        gj = json.load(f)
+                    if gj.get("type") == "FeatureCollection":
+                        shapes = [feat["geometry"] for feat in gj["features"]]
+                    elif gj.get("type") == "Feature":
+                        shapes = [gj["geometry"]]
+                    else:
+                        shapes = [gj]
+                elif geom_src:
+                    # bbox string "minx,miny,maxx,maxy"
+                    try:
+                        parts = [float(x) for x in geom_src.split(",")]
+                        from shapely.geometry import box as sbox
+                        shapes = [sbox(*parts).__geo_interface__]
+                    except Exception:
+                        pass
+                if shapes:
+                    new_paths = []
+                    for path in result.output_paths:
+                        if path.suffix.lower() in (".tif", ".tiff"):
+                            out_path = path.with_stem(f"{path.stem}_clip")
+                            with rasterio.open(path) as src:
+                                out_img, out_transform = rasterio_mask(src, shapes, crop=True)
+                                profile = src.profile.copy()
+                                profile.update(height=out_img.shape[1], width=out_img.shape[2],
+                                               transform=out_transform)
+                                with rasterio.open(out_path, "w", **profile) as dst:
+                                    dst.write(out_img)
+                            new_paths.append(out_path)
+                        else:
+                            new_paths.append(path)
+                    result.output_paths = new_paths
+                    if new_paths:
+                        result.output_path = new_paths[0]
+            except ImportError:
+                logger.warning("rasterio/shapely not installed; skipping clip action")
+
+        elif action.action == "resample":
+            val = action.params.get("value", "")
+            try:
+                import rasterio
+                from rasterio.enums import Resampling as RS
+                new_paths = []
+                for path in result.output_paths:
+                    if path.suffix.lower() in (".tif", ".tiff"):
+                        with rasterio.open(path) as src:
+                            # value is target resolution in CRS units
+                            try:
+                                target_res = float(val)
+                                scale_x = src.res[0] / target_res
+                                scale_y = src.res[1] / target_res
+                            except (ValueError, ZeroDivisionError):
+                                scale_x = scale_y = 0.5
+                            new_h = max(1, int(src.height * scale_y))
+                            new_w = max(1, int(src.width  * scale_x))
+                            data = src.read(out_shape=(src.count, new_h, new_w),
+                                            resampling=RS.bilinear)
+                            transform = src.transform * src.transform.scale(
+                                src.width / new_w, src.height / new_h)
+                            profile = src.profile.copy()
+                            profile.update(height=new_h, width=new_w, transform=transform)
+                        out_path = path.with_stem(f"{path.stem}_resamp")
+                        with rasterio.open(out_path, "w", **profile) as dst:
+                            dst.write(data)
+                        new_paths.append(out_path)
+                    else:
+                        new_paths.append(path)
+                result.output_paths = new_paths
+                if new_paths:
+                    result.output_path = new_paths[0]
+            except ImportError:
+                logger.warning("rasterio not installed; skipping resample action")
+
+        elif action.action == "ndvi":
+            # Compute NDVI from downloaded bands — expects B04 (Red) and B08 (NIR)
+            import numpy as np
+            try:
+                import rasterio
+                red_path = next((p for p in result.output_paths if "B04" in p.name or "red" in p.name.lower()), None)
+                nir_path = next((p for p in result.output_paths if "B08" in p.name or "nir" in p.name.lower()), None)
+                if red_path and nir_path:
+                    with rasterio.open(red_path) as rs:
+                        red = rs.read(1).astype(np.float32)
+                        profile = rs.profile.copy()
+                    with rasterio.open(nir_path) as ns:
+                        nir = ns.read(1, out_shape=red.shape,
+                                      resampling=rasterio.enums.Resampling.bilinear).astype(np.float32)
+                    with np.errstate(divide="ignore", invalid="ignore"):
+                        ndvi = np.where(nir + red > 0, (nir - red) / (nir + red), -9999.0)
+                    ndvi_path = red_path.parent / "ndvi.tif"
+                    profile.update(count=1, dtype="float32", nodata=-9999.0)
+                    with rasterio.open(ndvi_path, "w", **profile) as dst:
+                        dst.write(ndvi[np.newaxis, :, :])
+                    result.output_paths.append(ndvi_path)
+                    result.output_path = ndvi_path
+                    logger.info(f"NDVI computed → {ndvi_path}")
+                else:
+                    logger.warning("ndvi action: could not find B04 (Red) and B08 (NIR) in output paths")
+            except ImportError:
+                logger.warning("rasterio/numpy not installed; skipping ndvi action")
+
+        elif action.action in ("pan-sharpen", "pan_sharpen"):
+            logger.info("pan-sharpen action: use PyGeoFetch preprocess pansharpen for this operation")
+
+        elif action.action == "merge":
+            try:
+                import rasterio
+                from rasterio.merge import merge
+                tifs = [p for p in result.output_paths if p.suffix.lower() in (".tif", ".tiff")]
+                if len(tifs) > 1:
+                    src_files = [rasterio.open(p) for p in tifs]
+                    mosaic, transform = merge(src_files)
+                    profile = src_files[0].profile.copy()
+                    profile.update(height=mosaic.shape[1], width=mosaic.shape[2],
+                                   transform=transform)
+                    merge_path = tifs[0].parent / "merged.tif"
+                    with rasterio.open(merge_path, "w", **profile) as dst:
+                        dst.write(mosaic)
+                    for s in src_files:
+                        s.close()
+                    result.output_paths = [merge_path]
+                    result.output_path = merge_path
+            except ImportError:
+                logger.warning("rasterio not installed; skipping merge action")
 
         return result
 
+    def _validate_downloaded_file(self, path) -> tuple:
+        """
+        Validate a downloaded raster file by reading a sample tile.
+
+        For non-raster formats (JSON, XML, CSV), checks file size > 0.
+
+        Returns:
+            (is_valid: bool, error_message: str)
+        """
+        import os
+        path = str(path)
+        suffix = path.rsplit(".", 1)[-1].lower() if "." in path else ""
+
+        # Non-raster: just check readable and non-empty
+        NON_RASTER = {"json", "xml", "csv", "txt", "html", "md", "yaml", "yml"}
+        if suffix in NON_RASTER:
+            try:
+                size = os.path.getsize(path)
+                if size == 0:
+                    return False, "File is empty (0 bytes)"
+                return True, ""
+            except OSError as exc:
+                return False, f"Cannot stat file: {exc}"
+
+        # Raster: try rasterio
+        try:
+            import rasterio
+            with rasterio.open(path) as src:
+                if src.count == 0:
+                    return False, "File has zero bands"
+                if src.width == 0 or src.height == 0:
+                    return False, f"File has zero dimensions: {src.width}x{src.height}"
+                windows = list(src.block_windows(1))
+                if not windows:
+                    return False, "File has no readable blocks"
+                _, window = windows[0]
+                sample = src.read(1, window=window)
+                if sample is None or sample.size == 0:
+                    return False, "First tile returned empty array"
+            return True, ""
+        except ImportError:
+            # rasterio not installed — fall back to size check
+            try:
+                size = os.path.getsize(path)
+                if size == 0:
+                    return False, "File is empty (0 bytes)"
+                return True, ""
+            except OSError as exc:
+                return False, f"Cannot stat file: {exc}"
+        except Exception as exc:
+            return False, f"File read validation failed: {exc}"
+
+    def _has_identity_transform(self, path) -> bool:
+        """
+        Detect a reprojection that produced an identity (garbage) transform.
+
+        Returns True if the file has pixel_width=1, pixel_height=1,
+        origin=(0,0) in a projected (metre-unit) CRS — which indicates
+        the warp produced no real geographic transformation.
+        """
+        try:
+            import rasterio
+            with rasterio.open(str(path)) as src:
+                t = src.transform
+                return (
+                    abs(t.a) == 1.0
+                    and abs(t.e) == 1.0
+                    and t.c == 0.0
+                    and t.f == 0.0
+                    and src.crs is not None
+                    and src.crs.is_projected
+                )
+        except Exception:
+            return False
+
+    def _reproject_with_validation(self, source_path, target_path, target_crs) -> None:
+        """
+        Reproject source to target CRS and validate the output transform.
+        Raises RuntimeError if reprojection produces an identity transform.
+        """
+        import rasterio
+        from rasterio.warp import calculate_default_transform, reproject, Resampling
+        from rasterio.enums import Resampling as RS
+
+        with rasterio.open(str(source_path)) as src:
+            src_crs       = src.crs
+            src_transform = src.transform
+            src_width     = src.width
+            src_height    = src.height
+
+            dst_transform, dst_width, dst_height = calculate_default_transform(
+                src_crs, target_crs, src_width, src_height, *src.bounds
+            )
+            kwargs = src.meta.copy()
+            kwargs.update({
+                "crs":       target_crs,
+                "transform": dst_transform,
+                "width":     dst_width,
+                "height":    dst_height,
+            })
+            with rasterio.open(str(target_path), "w", **kwargs) as dst:
+                for band_idx in range(1, src.count + 1):
+                    reproject(
+                        source=rasterio.band(src, band_idx),
+                        destination=rasterio.band(dst, band_idx),
+                        src_transform=src_transform,
+                        src_crs=src_crs,
+                        dst_transform=dst_transform,
+                        dst_crs=target_crs,
+                        resampling=RS.bilinear,
+                    )
+
+        if self._has_identity_transform(target_path):
+            raise RuntimeError(
+                f"Reprojection produced identity transform in {target_path}. "
+                f"Source: {source_path}, Target CRS: {target_crs}. "
+                "This is a known rasterio/GDAL edge case with certain input CRS."
+            )
+
     def _backoff(self, attempt: int, options: DownloadOptions) -> float:
         """Calculate retry delay based on strategy."""
         import random