ECOv003-L2T-STARS 1.0.1__py3-none-any.whl → 1.2.0__py3-none-any.whl

ECOv003_L2T_STARS/Project.toml

@@ -0,0 +1,14 @@
+[deps]
+Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
+DelimitedFiles = "8bb1440f-4735-579b-a4ab-409b98df4dab"
+Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b"
+Glob = "c27321d9-0574-5035-807b-f59d2c89b15c"
+HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
+JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
+LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"
+OpenSSL = "4d8831e6-92b7-49fb-bdf8-b643e874388c"
+Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+Rasters = "a3a2b9e3-a471-40c9-b274-f788e487c689"
+STARSDataFusion = "70ccc657-289f-4534-a407-e03a16fd1153"
+Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"

ECOv003_L2T_STARS/VIIRS/VIIRSDataPool.py

@@ -0,0 +1,294 @@
+import json
+import logging
+import posixpath
+from datetime import date
+from datetime import datetime
+from os import makedirs
+from os.path import basename, splitext, abspath, expanduser
+from os.path import join
+from typing import List, Union
+
+import colored_logging as cl
+import h5py
+import numpy as np
+import pandas as pd
+import rasters as rt
+
+from dateutil import parser
+from shapely.geometry import Point, Polygon
+
+from rasters import RasterGrid, Raster
+
+from modland import find_modland_tiles, generate_modland_grid, parsehv
+
+from ..daterange import date_range
+from ..LPDAAC import LPDAACDataPool
+
+
+logger = logging.getLogger(__name__)
+
+
+def parse_VIIRS_product(filename: str) -> str:
+    return str(basename(filename).split(".")[0])
+
+
+def parse_VIIRS_date(filename: str) -> date:
+    return datetime.strptime(basename(filename).split(".")[1][1:], "%Y%j").date()
+
+
+def parse_VIIRS_tile(filename: str) -> str:
+    return str(basename(filename).split(".")[2])
+
+
+def parse_VIIRS_build(filename: str) -> int:
+    return int(basename(filename).split(".")[3])
+
+
+DEFAULT_WORKING_DIRECTORY = "."
+DEFAULT_PRODUCTS_DIRECTORY = "VIIRS_products"
+
+
+class VIIRSGranule:
+    CLOUD_DATASET_NAME = "HDFEOS/GRIDS/VNP_Grid_1km_2D/Data Fields/SurfReflect_QF1_1"
+
+    def __init__(self, filename: str, working_directory: str = None, products_directory: str = None):
+        if working_directory is None:
+            working_directory = DEFAULT_WORKING_DIRECTORY
+
+        working_directory = abspath(expanduser(working_directory))
+
+        if products_directory is None:
+            products_directory = join(working_directory, DEFAULT_PRODUCTS_DIRECTORY)
+
+        products_directory = abspath(expanduser(products_directory))
+
+        self._filename = abspath(filename)
+        self._cloud_mask = None
+        self.working_directory = working_directory
+        self.products_directory = products_directory
+
+    def __repr__(self):
+        display_dict = {
+            "filename": self.filename,
+            "products_directory": self.products_directory
+        }
+
+        display_string = json.dumps(display_dict, indent=2)
+
+        return display_string
+
+    @property
+    def filename(self):
+        return self._filename
+
+    @property
+    def filename_base(self):
+        return basename(self.filename)
+
+    @property
+    def filename_stem(self) -> str:
+        return splitext(self.filename_base)[0]
+
+    @property
+    def tile(self):
+        return parse_VIIRS_tile(self.filename)
+
+    @property
+    def hv(self):
+        return parsehv(self.tile)
+
+    @property
+    def h(self):
+        return self.hv[0]
+
+    @property
+    def v(self):
+        return self.hv[1]
+
+    @property
+    def date_UTC(self):
+        return datetime.strptime(self.filename_base.split(".")[1][1:], "%Y%j")
+
+    @property
+    def grids(self) -> List[str]:
+        with h5py.File(self.filename, "r") as file:
+            return list(file["HDFEOS/GRIDS/"].keys())
+
+    def variables(self, grid: str) -> List[str]:
+        with h5py.File(self.filename, "r") as file:
+            return list(file[f"HDFEOS/GRIDS/{grid}/Data Fields/"].keys())
+
+    def dataset(
+            self,
+            filename: str,
+            dataset_name: str,
+            fill_value: int,
+            scale_factor: float) -> Raster:
+        tile = parse_VIIRS_tile(filename)
+        h, v = parsehv(tile)
+
+        with h5py.File(filename, "r") as f:
+            DN = np.array(f[dataset_name])
+            grid = generate_modland_grid(h, v, DN.shape[0])
+            logger.info(f"opening VIIRS file: {cl.file(self.filename)}")
+            logger.info(f"loading {cl.val(dataset_name)} at {cl.val(f'{grid.cell_size:0.2f} m')} resolution")
+            DN = Raster(DN, geometry=grid)
+
+        data = rt.where(DN == fill_value, np.nan, DN * scale_factor)
+
+        return data
+
+    def product_directory(self, product) -> Union[str, None]:
+        if self.products_directory is None:
+            return None
+        else:
+            return join(self.products_directory, product, f"{self.date_UTC:%Y.%m.%d}")
+
+    def product_filename(self, product: str) -> str:
+        if self.product_directory(product) is None:
+            raise ValueError("no product directory given")
+
+        return join(
+            self.product_directory(product),
+            f"{self.filename_stem}_{product}.tif"
+        )
+
+
+class VIIRSDataPool(LPDAACDataPool):
+    DEFAULT_WORKING_DIRECTORY = "."
+    DEFAULT_DOWNLOAD_DIRECTORY = "VIIRS_download"
+    DEFAULT_PRODUCTS_DIRECTORY = "VIIRS_products"
+    DEFAULT_MOSAIC_DIRECTORY = "VIIRS_mosaics"
+
+    def __init__(
+            self,
+            username: str = None,
+            password: str = None,
+            remote: str = None,
+            working_directory: str = None,
+            download_directory: str = None,
+            products_directory: str = None,
+            mosaic_directory: str = None,
+            *args,
+            **kwargs):
+        super(VIIRSDataPool, self).__init__(
+            username=username,
+            password=password,
+            remote=remote,
+            *args,
+            **kwargs
+        )
+
+        if working_directory is None:
+            working_directory = self.DEFAULT_WORKING_DIRECTORY
+
+        working_directory = abspath(expanduser(working_directory))
+
+        if download_directory is None:
+            download_directory = join(working_directory, self.DEFAULT_DOWNLOAD_DIRECTORY)
+
+        download_directory = abspath(expanduser(download_directory))
+
+        if products_directory is None:
+            products_directory = join(working_directory, self.DEFAULT_PRODUCTS_DIRECTORY)
+
+        products_directory = abspath(expanduser(products_directory))
+
+        if mosaic_directory is None:
+            mosaic_directory = join(working_directory, self.DEFAULT_MOSAIC_DIRECTORY)
+
+        mosaic_directory = abspath(expanduser(mosaic_directory))
+
+        self.working_directory = working_directory
+        self.download_directory = download_directory
+        self.products_directory = products_directory
+        self.mosaic_directory = mosaic_directory
+
+    def __repr__(self):
+        display_dict = {
+            "download_directory": self.download_directory,
+            "products_directory": self.products_directory,
+            "mosaic_directory": self.mosaic_directory
+        }
+
+        display_string = json.dumps(display_dict, indent=2)
+
+        return display_string
+
+    def search(
+            self,
+            product: str,
+            start_date: date or datetime or str,
+            end_date: date or datetime or str = None,
+            build: str = None,
+            tiles: List[str] or str = None,
+            target_geometry: Point or Polygon or RasterGrid = None,
+            *args,
+            **kwargs) -> pd.DataFrame:
+        if tiles is None and target_geometry is not None:
+            tiles = find_modland_tiles(target_geometry)
+
+        if isinstance(tiles, str):
+            tiles = [tiles]
+
+        if isinstance(start_date, str):
+            start_date = parser.parse(start_date).date()
+
+        if end_date is None:
+            end_date = start_date
+        elif isinstance(end_date, str):
+            end_date = parser.parse(end_date).date()
+
+        rows = []
+
+        for acquisition_date in date_range(start_date, end_date):
+            date_URL = self.date_URL(
+                "VIIRS",
+                product,
+                acquisition_date,
+                build
+            )
+
+            logger.info(f"scanning LP-DAAC: {date_URL}")
+
+            if tiles is None:
+                listing = self.get_HTTP_listing(date_URL, pattern="*.h5")
+            else:
+                listing = []
+
+                for tile in tiles:
+                    listing.extend(self.get_HTTP_listing(date_URL, pattern=f"*.{tile}.*.h5"))
+
+            URLs = sorted([
+                posixpath.join(date_URL, item)
+                for item
+                in listing
+            ])
+
+            for URL in URLs:
+                tile = parse_VIIRS_tile(posixpath.basename(URL))
+                rows.append([acquisition_date, tile, URL])
+
+        df = pd.DataFrame(rows, columns=["date", "tile", "URL"])
+
+        return df
+
+    def download_URL(self, URL: str, download_location: str = None) -> str:
+        if download_location is None:
+            acquisition_date = parse_VIIRS_date(posixpath.basename(URL))
+            product = parse_VIIRS_product(posixpath.basename(URL))
+
+            download_location = join(
+                self.download_directory,
+                product,
+                f"{acquisition_date:%Y.%m.%d}"
+            )
+
+            makedirs(download_location, exist_ok=True)
+
+        filename = super(VIIRSDataPool, self).download_URL(
+            URL=URL,
+            download_location=download_location
+        )
+
+        return filename

ECOv003_L2T_STARS/VIIRS/VIIRSDownloader.py

@@ -0,0 +1,26 @@
+
+from abc import ABC, abstractmethod
+from datetime import date
+
+from rasters import Raster, RasterGeometry
+
+class VIIRSDownloaderAlbedo(ABC):
+    @abstractmethod
+    def albedo(
+            self,
+            date_UTC: date or str,
+            geometry: RasterGeometry,
+            filename: str = None,
+            resampling: str = None) -> Raster:
+        pass
+
+
+class VIIRSDownloaderNDVI(ABC):
+    @abstractmethod
+    def NDVI(
+            self,
+            date_UTC: date or str,
+            geometry: RasterGeometry,
+            filename: str = None,
+            resampling: str = None) -> Raster:
+        pass

ECOv003_L2T_STARS/VIIRS/VIIRS_CMR_LOGIN.py

@@ -0,0 +1,36 @@
+import netrc
+import os
+
+import earthaccess
+
+_AUTH = None
+
+class CMRServerUnreachable(Exception):
+    pass
+
+def VIIRS_CMR_login() -> earthaccess.Auth:
+    """
+    Login to Earthdata using netrc credentials if available, falling back to environment variables.
+    """
+    # Only login to earthaccess once
+    global _AUTH
+    if _AUTH is not None:
+        return _AUTH
+
+    try:
+        # Attempt to use netrc for credentials
+        secrets = netrc.netrc()
+        auth = secrets.authenticators("urs.earthdata.nasa.gov")
+        if auth:
+            _AUTH = earthaccess.login(strategy="netrc")  # Use strategy="netrc"
+            return _AUTH
+
+        # Fallback to environment variables if netrc fails
+        if "EARTHDATA_USERNAME" in os.environ and "EARTHDATA_PASSWORD" in os.environ:
+            _AUTH = earthaccess.login(strategy="environment")
+            return _AUTH
+        else:
+            raise CMRServerUnreachable("Missing netrc credentials or environment variables 'EARTHDATA_USERNAME' and 'EARTHDATA_PASSWORD'")
+
+    except Exception as e:
+        raise CMRServerUnreachable(e)