ECOv003-L2T-STARS 1.2.0__py3-none-any.whl → 1.4.0__py3-none-any.whl

ECOv003_L2T_STARS/generate_STARS_inputs.py

@@ -1,6 +1,7 @@
 from typing import Union
 from datetime import date, datetime
 from dateutil.rrule import rrule, DAILY
+from os.path import exists
 import logging
 
 import colored_logging as cl
@@ -16,6 +17,7 @@ from .generate_NDVI_coarse_image import generate_NDVI_coarse_image
 from .generate_NDVI_fine_image import generate_NDVI_fine_image
 from .generate_albedo_coarse_image import generate_albedo_coarse_image
 from .generate_albedo_fine_image import generate_albedo_fine_image
+from .generate_downsampled_filename import generate_downsampled_filename
 from .calibrate_fine_to_coarse import calibrate_fine_to_coarse
 from .VIIRS.VIIRSDownloader import VIIRSDownloaderAlbedo, VIIRSDownloaderNDVI
 
@@ -33,11 +35,7 @@ def generate_STARS_inputs(
     target_resolution: int,
     NDVI_coarse_geometry: RasterGeometry,
     albedo_coarse_geometry: RasterGeometry,
-    working_directory: str,
-    NDVI_coarse_directory: str,
-    NDVI_fine_directory: str,
-    albedo_coarse_directory: str,
-    albedo_fine_directory: str,
+    downsampled_directory: str,
     HLS_connection: HLS2CMR,
     NDVI_VIIRS_connection: VIIRSDownloaderNDVI,
     albedo_VIIRS_connection: VIIRSDownloaderAlbedo,
@@ -80,68 +78,83 @@ def generate_STARS_inputs(
     """
     missing_coarse_dates = set()  # Track dates where coarse data could not be generated
 
+    logger.info(f"preparing coarse and fine images for STARS at {cl.place(tile)}")
+
     # Process each day within the VIIRS data fusion window
     for processing_date in [
         get_date(dt) for dt in rrule(DAILY, dtstart=VIIRS_start_date, until=VIIRS_end_date)
     ]:
-        logger.info(
-            f"Preparing coarse image for STARS NDVI at {cl.place(tile)} on {cl.time(processing_date)}"
+        NDVI_coarse_filename = generate_downsampled_filename(
+            directory=downsampled_directory,
+            variable="NDVI",
+            date_UTC=processing_date,
+            tile=tile,
+            cell_size=NDVI_resolution
+        )
+
+        NDVI_fine_filename = generate_downsampled_filename(
+            directory=downsampled_directory,
+            variable="NDVI",
+            date_UTC=processing_date,
+            tile=tile,
+            cell_size=target_resolution
+        )
+
+        albedo_coarse_filename = generate_downsampled_filename(
+            directory=downsampled_directory,
+            variable="albedo",
+            date_UTC=processing_date,
+            tile=tile,
+            cell_size=albedo_resolution
+        )
+
+        albedo_fine_filename = generate_downsampled_filename(
+            directory=downsampled_directory,
+            variable="albedo",
+            date_UTC=processing_date,
+            tile=tile,
+            cell_size=target_resolution
         )
 
         try:
-            # Generate coarse NDVI image
-            NDVI_coarse_image = generate_NDVI_coarse_image(
-                date_UTC=processing_date,
-                VIIRS_connection=NDVI_VIIRS_connection,
-                geometry=NDVI_coarse_geometry,
-            )
+            # Cache whether the NDVI coarse exists to avoid ToCToU
+            NDVI_coarse_exists = exists(NDVI_coarse_filename)
+            if not NDVI_coarse_exists:
+                logger.info(f"preparing coarse image for STARS NDVI at {cl.place(tile)} on {cl.time(processing_date)}")
+
+                NDVI_coarse_image = generate_NDVI_coarse_image(
+                    date_UTC=processing_date,
+                    VIIRS_connection=NDVI_VIIRS_connection,
+                    geometry=NDVI_coarse_geometry
+                )
 
-            # Define filename for coarse NDVI and save
-            NDVI_coarse_filename = generate_filename(
-                directory=NDVI_coarse_directory,
-                variable="NDVI",
-                date_UTC=processing_date,
-                tile=tile,
-                cell_size=NDVI_resolution,
-            )
-            logger.info(
-                f"Saving coarse image for STARS NDVI at {cl.place(tile)} on {cl.time(processing_date)}: {NDVI_coarse_filename}"
-            )
-            NDVI_coarse_image.to_geotiff(NDVI_coarse_filename)
+                logger.info(
+                    f"saving coarse image for STARS NDVI at {cl.place(tile)} on {cl.time(processing_date)}: {NDVI_coarse_filename}")
+                NDVI_coarse_image.to_geotiff(NDVI_coarse_filename)
 
-            # If the processing date is within the HLS range, generate fine NDVI
             if processing_date >= HLS_start_date:
-                logger.info(
-                    f"Preparing fine image for STARS NDVI at {cl.place(tile)} on {cl.time(processing_date)}"
-                )
                 try:
-                    NDVI_fine_image = generate_NDVI_fine_image(
-                        date_UTC=processing_date,
-                        tile=tile,
-                        HLS_connection=HLS_connection,
-                    )
-
-                    # Optionally calibrate the fine NDVI image to the coarse NDVI image
-                    if calibrate_fine:
+                    if not exists(NDVI_fine_filename):
                         logger.info(
-                            f"Calibrating fine image for STARS NDVI at {cl.place(tile)} on {cl.time(processing_date)}"
-                        )
-                        NDVI_fine_image = calibrate_fine_to_coarse(
-                            NDVI_fine_image, NDVI_coarse_image
+                            f"preparing fine image for STARS NDVI at {cl.place(tile)} on {cl.time(processing_date)}")
+
+                        NDVI_fine_image = generate_NDVI_fine_image(
+                            date_UTC=processing_date,
+                            tile=tile,
+                            HLS_connection=HLS_connection
                         )
 
-                    # Define filename for fine NDVI and save
-                    NDVI_fine_filename = generate_filename(
-                        directory=NDVI_fine_directory,
-                        variable="NDVI",
-                        date_UTC=processing_date,
-                        tile=tile,
-                        cell_size=target_resolution,
-                    )
-                    logger.info(
-                        f"Saving fine image for STARS NDVI at {cl.place(tile)} on {cl.time(processing_date)}: {NDVI_fine_filename}"
-                    )
-                    NDVI_fine_image.to_geotiff(NDVI_fine_filename)
+                        if calibrate_fine:
+                            # Ensure that the NDVI_coarse_image variable is set
+                            if NDVI_coarse_exists:
+                                NDVI_coarse_image = Raster.open(NDVI_coarse_filename)
+                            logger.info(
+                                f"calibrating fine image for STARS NDVI at {cl.place(tile)} on {cl.time(processing_date)}")
+                            NDVI_fine_image = calibrate_fine_to_coarse(NDVI_fine_image, NDVI_coarse_image)
+
+                        logger.info(
+                            f"saving fine image for STARS NDVI at {cl.place(tile)} on {cl.time(processing_date)}: {NDVI_fine_filename}")
+                        NDVI_fine_image.to_geotiff(NDVI_fine_filename)
                 except Exception:  # Catch any exception during HLS fine image generation
                     logger.info(f"HLS NDVI is not available on {processing_date}")
         except Exception as e:
@@ -151,63 +164,47 @@ def generate_STARS_inputs(
             )
             missing_coarse_dates.add(processing_date)  # Add date to missing set
 
-        logger.info(
-            f"Preparing coarse image for STARS albedo at {cl.place(tile)} on {cl.time(processing_date)}"
-        )
         try:
-            # Generate coarse albedo image
-            albedo_coarse_image = generate_albedo_coarse_image(
-                date_UTC=processing_date,
-                VIIRS_connection=albedo_VIIRS_connection,
-                geometry=albedo_coarse_geometry,
-            )
+            # Cache whether the albedo coarse exists to avoid ToCToU
+            albedo_coarse_exists = exists(albedo_coarse_filename)
+            if not albedo_coarse_exists:
+                logger.info(
+                    f"preparing coarse image for STARS albedo at {cl.place(tile)} on {cl.time(processing_date)}")
 
-            # Define filename for coarse albedo and save
-            albedo_coarse_filename = generate_filename(
-                directory=albedo_coarse_directory,
-                variable="albedo",
-                date_UTC=processing_date,
-                tile=tile,
-                cell_size=albedo_resolution,
-            )
-            logger.info(
-                f"Saving coarse image for STARS albedo at {cl.place(tile)} on {cl.time(processing_date)}: {albedo_coarse_filename}"
-            )
-            albedo_coarse_image.to_geotiff(albedo_coarse_filename)
+                albedo_coarse_image = generate_albedo_coarse_image(
+                    date_UTC=processing_date,
+                    VIIRS_connection=albedo_VIIRS_connection,
+                    geometry=albedo_coarse_geometry
+                )
 
-            # If the processing date is within the HLS range, generate fine albedo
-            if processing_date >= HLS_start_date:
                 logger.info(
-                    f"Preparing fine image for STARS albedo at {cl.place(tile)} on {cl.time(processing_date)}"
-                )
+                    f"saving coarse image for STARS albedo at {cl.place(tile)} on {cl.time(processing_date)}: {albedo_coarse_filename}")
+                albedo_coarse_image.to_geotiff(albedo_coarse_filename)
+
+            if processing_date >= HLS_start_date:
                 try:
-                    albedo_fine_image = generate_albedo_fine_image(
-                        date_UTC=processing_date,
-                        tile=tile,
-                        HLS_connection=HLS_connection,
-                    )
-
-                    # Optionally calibrate the fine albedo image to the coarse albedo image
-                    if calibrate_fine:
+                    if not exists(albedo_fine_filename):
                         logger.info(
-                            f"Calibrating fine image for STARS albedo at {cl.place(tile)} on {cl.time(processing_date)}"
-                        )
-                        albedo_fine_image = calibrate_fine_to_coarse(
-                            albedo_fine_image, albedo_coarse_image
+                            f"preparing fine image for STARS albedo at {cl.place(tile)} on {cl.time(processing_date)}")
+
+                        albedo_fine_image = generate_albedo_fine_image(
+                            date_UTC=processing_date,
+                            tile=tile,
+                            HLS_connection=HLS_connection
                         )
 
-                    # Define filename for fine albedo and save
-                    albedo_fine_filename = generate_filename(
-                        directory=albedo_fine_directory,
-                        variable="albedo",
-                        date_UTC=processing_date,
-                        tile=tile,
-                        cell_size=target_resolution,
-                    )
-                    logger.info(
-                        f"Saving fine image for STARS albedo at {cl.place(tile)} on {cl.time(processing_date)}: {albedo_fine_filename}"
-                    )
-                    albedo_fine_image.to_geotiff(albedo_fine_filename)
+                        if calibrate_fine:
+                            # Ensure that the albedo_coarse_image variable is set
+                            if albedo_coarse_exists:
+                                albedo_coarse_image = Raster.open(albedo_coarse_filename)
+
+                            logger.info(
+                                f"calibrating fine image for STARS albedo at {cl.place(tile)} on {cl.time(processing_date)}")
+                            albedo_fine_image = calibrate_fine_to_coarse(albedo_fine_image, albedo_coarse_image)
+
+                        logger.info(
+                            f"saving fine image for STARS albedo at {cl.place(tile)} on {cl.time(processing_date)}: {albedo_fine_filename}")
+                        albedo_fine_image.to_geotiff(albedo_fine_filename)
                 except Exception:  # Catch any exception during HLS fine image generation
                     logger.info(f"HLS albedo is not available on {processing_date}")
         except Exception as e:
@@ -217,7 +214,12 @@ def generate_STARS_inputs(
             )
             missing_coarse_dates.add(processing_date)  # Add date to missing set
 
-    # Check for missing coarse dates within the give-up window
+    # We need to deal with the possibility that VIIRS has not yet published their data yet.
+    #  VIIRS_GIVEUP_DAYS is the number of days before we assume that missing observations aren't coming.
+    #  If any missing days are closer to now than VIIRS_GIVEUP_DAYS, we want to retry this run later, when VIIRS
+    #  might have uploaded the missing observations. To cause this retry, we'll throw the `AncillaryLatency` exception.
+    #  L2T_STARS converts this exception to an exit code, and the orchestration system marks this run
+    #  as needing a retry at a later date.
     coarse_latency_dates = [
         d
         for d in missing_coarse_dates

ECOv003_L2T_STARS/generate_downsampled_filename.py

@@ -0,0 +1,20 @@
+from os import makedirs
+from os.path import join, dirname
+from dateutil import parser
+from datetime import date
+
+from typing import Union
+
+def generate_downsampled_filename(directory: str, variable: str, date_UTC: Union[date, str], tile: str, cell_size: int) -> str:
+    if isinstance(date_UTC, str):
+        date_UTC = parser.parse(date_UTC).date()
+
+    variable = str(variable)
+    year = str(date_UTC.year)
+    timestamp = date_UTC.strftime("%Y-%m-%d")
+    tile = str(tile)
+    cell_size = int(cell_size)
+    filename = join(directory, year, timestamp, tile, f"STARS_{variable}_{tile}_{cell_size}m.tif")
+    makedirs(dirname(filename), exist_ok=True)
+
+    return filename

ECOv003_L2T_STARS/{instantiate_STARS_jl.py → instantiate_STARSDataFusion_jl.py}

@@ -3,11 +3,11 @@ import logging
 
 logger = logging.getLogger(__name__)
 
-def instantiate_STARS_jl(package_location: str) -> subprocess.CompletedProcess:
+def instantiate_STARSDataFusion_jl(package_location: str) -> subprocess.CompletedProcess:
     """
     Activates a Julia project at a given location and instantiates its dependencies.
 
-    This is necessary to ensure all required Julia packages for STARS.jl are
+    This is necessary to ensure all required Julia packages for STARSDataFusion.jl are
     downloaded and ready for use within the specified project environment.
 
     Args:
@@ -30,7 +30,7 @@ def instantiate_STARS_jl(package_location: str) -> subprocess.CompletedProcess:
 
     if result.returncode == 0:
         logger.info(
-            f"STARS.jl instantiated successfully in directory '{package_location}'!"
+            f"STARSDataFusion.jl instantiated successfully in directory '{package_location}'!"
         )
     else:
         logger.error("Error instantiating STARS.jl:")

ECOv003_L2T_STARS/load_prior.py

@@ -98,7 +98,7 @@ def load_prior(
                 cell_size=target_resolution,
             )
             # Assuming L2T_STARS_prior_granule has a .NDVI_bias attribute
-            if hasattr(L2T_STARS_prior_granule, "NDVI_bias") and L2T_STARS_prior_granule.NDVI_bias is not None:
+            if L2T_STARS_prior_granule.NDVI_bias is not None:
                 L2T_STARS_prior_granule.NDVI_bias.to_geotiff(prior_NDVI_bias_filename)
             else:
                 prior_NDVI_bias_filename = None # Set to None if not available
@@ -111,7 +111,7 @@ def load_prior(
                 cell_size=target_resolution,
             )
             # Assuming L2T_STARS_prior_granule has a .NDVI_bias_UQ attribute
-            if hasattr(L2T_STARS_prior_granule, "NDVI_bias_UQ") and L2T_STARS_prior_granule.NDVI_bias_UQ is not None:
+            if L2T_STARS_prior_granule.NDVI_bias_UQ is not None:
                 L2T_STARS_prior_granule.NDVI_bias_UQ.to_geotiff(prior_NDVI_bias_UQ_filename)
             else:
                 prior_NDVI_bias_UQ_filename = None # Set to None if not available
@@ -143,7 +143,7 @@ def load_prior(
                 cell_size=target_resolution,
             )
             # Assuming L2T_STARS_prior_granule has a .albedo_bias attribute
-            if hasattr(L2T_STARS_prior_granule, "albedo_bias") and L2T_STARS_prior_granule.albedo_bias is not None:
+            if L2T_STARS_prior_granule.albedo_bias is not None:
                 L2T_STARS_prior_granule.albedo_bias.to_geotiff(prior_albedo_bias_filename)
             else:
                 prior_albedo_bias_filename = None # Set to None if not available
@@ -156,7 +156,7 @@ def load_prior(
                 cell_size=target_resolution,
             )
             # Assuming L2T_STARS_prior_granule has a .albedo_bias_UQ attribute
-            if hasattr(L2T_STARS_prior_granule, "albedo_bias_UQ") and L2T_STARS_prior_granule.albedo_bias_UQ is not None:
+            if L2T_STARS_prior_granule.albedo_bias_UQ is not None:
                 L2T_STARS_prior_granule.albedo_bias_UQ.to_geotiff(prior_albedo_bias_UQ_filename)
             else:
                 prior_albedo_bias_UQ_filename = None # Set to None if not available

ECOv003_L2T_STARS/login.py

@@ -0,0 +1,61 @@
+import logging
+import netrc
+import os
+
+import earthaccess
+
+from .exceptions import *
+
+__author__ = "Evan Davis"
+
+_AUTH = None
+
+def login() -> earthaccess.Auth:
+    """
+    Login to Earthdata using environment variables if available, falling back to netrc credentials, then interactive login.
+    """
+    # Only login to earthaccess once
+    global _AUTH
+    if _AUTH is not None:
+        return _AUTH
+
+    # Check if we're in a testing environment where authentication should be skipped
+    if os.environ.get("SKIP_EARTHDATA_LOGIN", "").lower() in ("true", "1", "yes"):
+        # Return a mock auth object for testing
+        class MockAuth:
+            def __init__(self):
+                self.authenticated = True
+        _AUTH = MockAuth()
+        return _AUTH
+
+    # Temporarily suppress INFO logs from earthaccess during login
+    earthaccess_logger = logging.getLogger('earthaccess')
+    original_level = earthaccess_logger.level
+    earthaccess_logger.setLevel(logging.WARNING)
+
+    try:
+        # First priority: environment variables
+        if "EARTHDATA_USERNAME" in os.environ and "EARTHDATA_PASSWORD" in os.environ:
+            _AUTH = earthaccess.login(strategy="environment")
+            return _AUTH
+
+        # Second priority: netrc credentials
+        try:
+            secrets = netrc.netrc()
+            auth = secrets.authenticators("urs.earthdata.nasa.gov")
+            if auth:
+                _AUTH = earthaccess.login(strategy="netrc")
+                return _AUTH
+        except (FileNotFoundError, netrc.NetrcParseError):
+            # .netrc file doesn't exist or is malformed, continue to interactive login
+            pass
+
+        # Last resort: interactive login
+        _AUTH = earthaccess.login(strategy="interactive")
+        return _AUTH
+
+    except Exception as e:
+        raise CMRServerUnreachable(e)
+    finally:
+        # Restore original logging level
+        earthaccess_logger.setLevel(original_level)

ECOv003_L2T_STARS/{ECOv003_L2T_STARS.py → main.py}

@@ -115,6 +115,12 @@ def main():
         default=True,
         help="Do NOT remove posterior intermediate files after product generation.",
     )
+    parser.add_argument(
+        "--initialize-julia",
+        action="store_true",
+        dest="initialize_julia",
+        help="Initialize a julia environment before running julia.",
+    )
     parser.add_argument(
         "--threads",
         type=str,
@@ -157,6 +163,7 @@ def main():
         remove_input_staging=args.remove_input_staging,
         remove_prior=args.remove_prior,
         remove_posterior=args.remove_posterior,
+        initialize_julia=args.initialize_julia,
         threads=args.threads,
         num_workers=args.num_workers,
         overwrite=args.overwrite, # Pass the new overwrite argument

ECOv003_L2T_STARS/process_ECOSTRESS_data_fusion_distributed_bias.jl

@@ -8,24 +8,8 @@ using STARSDataFusion.sentinel_tiles
 using STARSDataFusion.HLS
 using STARSDataFusion.VNP43
 using Logging
-using Pkg
 using Statistics
 using Distributed
-Pkg.add("OpenSSL")
-using HTTP
-using JSON
-
-function read_json(file::String)::Dict
-    open(file, "r") do f
-        return JSON.parse(f)
-    end
-end
-
-function write_json(file::String, data::Dict)
-    open(file, "w") do f
-        JSON.print(f, data)
-    end
-end
 
 @info "processing STARS data fusion"
 
@@ -78,10 +62,10 @@ HLS_start_date = Date(ARGS[7])
 @info "HLS start date: $(HLS_start_date)"
 HLS_end_date = Date(ARGS[8])
 @info "HLS end date: $(HLS_end_date)"
-coarse_directory = ARGS[9]
-@info "coarse inputs directory: $(coarse_directory)"
-fine_directory = ARGS[10]
-@info "fine inputs directory: $(fine_directory)"
+downsampled_directory = ARGS[9]
+@info "downsampled inputs directory: $(downsampled_directory)"
+product_name = ARGS[10]
+@info "Computing $(product_name) product"
 posterior_filename = ARGS[11]
 @info "posterior filename: $(posterior_filename)"
 posterior_UQ_filename = ARGS[12]
@@ -130,20 +114,29 @@ y_fine_size = size(y_fine)[1]
 @info "fine x size: $(x_fine_size)"
 @info "fine y size: $(y_fine_size)"
 
-coarse_image_filenames = sort(glob("*.tif", coarse_directory))
-coarse_dates_found = [Date(split(basename(filename), "_")[3]) for filename in coarse_image_filenames]
-
-fine_image_filenames = sort(glob("*.tif", fine_directory))
-fine_dates_found = [Date(split(basename(filename), "_")[3]) for filename in fine_image_filenames]
-
+# The range of dates to check for VIIRS files
 coarse_start_date = VIIRS_start_date
 coarse_end_date = VIIRS_end_date
 
+# Check each coarse date for a downsampled image
+# For each day we find, convert the date directory back into a date object
+coarse_dates = [coarse_start_date + Day(d - 1) for d in 1:((coarse_end_date - coarse_start_date).value + 1)]
+coarse_image_filenames = [joinpath("$(downsampled_directory)", "$(year(date))", "$(Dates.format(date, dateformat"yyyy-mm-dd"))", "$(tile)", "STARS_$(product_name)_$(tile)_$(coarse_cell_size)m.tif") for date in coarse_dates]
+coarse_image_filenames = [filename for filename in coarse_image_filenames if ispath(filename)]
+coarse_dates_found = [Date(basename(dirname(dirname(filename)))) for filename in coarse_image_filenames]
+
+# The range of dates to check for HLS files
 fine_flag_start_date = HLS_end_date - Day(7)
 fine_start_date = HLS_start_date
 fine_end_date = HLS_end_date
 
+# Check each fine date for a downsampled image
+# For each day we find, convert the date directory back into a date object
 dates = [fine_start_date + Day(d - 1) for d in 1:((fine_end_date - fine_start_date).value + 1)]
+fine_image_filenames = [joinpath("$(downsampled_directory)", "$(year(date))", "$(Dates.format(date, dateformat"yyyy-mm-dd"))", "$(tile)", "STARS_$(product_name)_$(tile)_$(fine_cell_size)m.tif") for date in dates]
+fine_image_filenames = [filename for filename in fine_image_filenames if ispath(filename)]
+fine_dates_found = [Date(basename(dirname(dirname(filename)))) for filename in fine_image_filenames]
+
 t = Ti(dates)
 coarse_dims = (x_coarse, y_coarse, t)
 fine_dims = (x_fine, y_fine, t)