ECOv003-L2T-STARS 1.3.0__tar.gz → 1.4.0__tar.gz

{ecov003_l2t_stars-1.3.0 → ecov003_l2t_stars-1.4.0}/ECOv003_L2T_STARS/L2T_STARS.py

@@ -35,10 +35,6 @@ from .VNP43NRT import VNP43NRT
 from .runconfig import ECOSTRESSRunConfig
 from .L2TSTARSConfig import L2TSTARSConfig
 from .load_prior import load_prior
-from .generate_NDVI_coarse_directory import generate_NDVI_coarse_directory
-from .generate_NDVI_fine_directory import generate_NDVI_fine_directory
-from .generate_albedo_coarse_directory import generate_albedo_coarse_directory
-from .generate_albedo_fine_directory import generate_albedo_fine_directory
 from .generate_STARS_inputs import generate_STARS_inputs
 from .process_STARS_product import process_STARS_product
 from .retrieve_STARS_sources import retrieve_STARS_sources
@@ -58,6 +54,7 @@ def L2T_STARS(
     remove_input_staging: bool = True,
     remove_prior: bool = True,
     remove_posterior: bool = True,
+    initialize_julia: bool = False,
     threads: Union[int, str] = "auto",
     num_workers: int = 4,
     overwrite: bool = False, # New parameter for overwriting existing files
@@ -240,6 +237,8 @@ def L2T_STARS(
         logger.info(f"VNP09GA products directory: {cl.dir(VNP09GA_products_directory)}")
         VNP43NRT_products_directory = join(sources_directory, DEFAULT_VNP43NRT_PRODUCTS_DIRECTORY)
         logger.info(f"VNP43NRT products directory: {cl.dir(VNP43NRT_products_directory)}")
+        DOWNSAMPLED_products_directory = join(sources_directory, DEFAULT_STARS_DOWNSAMPLED_DIRECTORY)
+        logger.info(f"DOWNSAMPLED products directory: {cl.dir(DOWNSAMPLED_products_directory)}")
 
         # Re-check for existing product (double-check in case another process created it) with overwrite option
         if not overwrite and exists(L2T_STARS_zip_filename):
@@ -283,6 +282,7 @@ def L2T_STARS(
                     GEOS5FP_products=GEOS5FP_products_directory,
                     VNP09GA_directory=VNP09GA_products_directory,
                     VNP43NRT_directory=VNP43NRT_products_directory,
+                    initialize_julia=initialize_julia,
                 )
 
                 albedo_VIIRS_connection = VNP43NRT(
@@ -293,6 +293,7 @@ def L2T_STARS(
                     GEOS5FP_products=GEOS5FP_products_directory,
                     VNP09GA_directory=VNP09GA_products_directory,
                     VNP43NRT_directory=VNP43NRT_products_directory,
+                    initialize_julia=initialize_julia,
                 )
             except CMRServerUnreachable as e:
                 logger.exception(e)
@@ -404,19 +405,6 @@ def L2T_STARS(
             NDVI_coarse_geometry = HLS_connection.grid(tile=tile, cell_size=NDVI_resolution)
             albedo_coarse_geometry = HLS_connection.grid(tile=tile, cell_size=albedo_resolution)
 
-            NDVI_coarse_directory = generate_NDVI_coarse_directory(
-                input_staging_directory=input_staging_directory, tile=tile
-            )
-            NDVI_fine_directory = generate_NDVI_fine_directory(
-                input_staging_directory=input_staging_directory, tile=tile
-            )
-            albedo_coarse_directory = generate_albedo_coarse_directory(
-                input_staging_directory=input_staging_directory, tile=tile
-            )
-            albedo_fine_directory = generate_albedo_fine_directory(
-                input_staging_directory=input_staging_directory, tile=tile
-            )
-
             generate_STARS_inputs(
                 tile=tile,
                 date_UTC=date_UTC,
@@ -429,11 +417,7 @@ def L2T_STARS(
                 target_resolution=target_resolution,
                 NDVI_coarse_geometry=NDVI_coarse_geometry,
                 albedo_coarse_geometry=albedo_coarse_geometry,
-                working_directory=working_directory,
-                NDVI_coarse_directory=NDVI_coarse_directory,
-                NDVI_fine_directory=NDVI_fine_directory,
-                albedo_coarse_directory=albedo_coarse_directory,
-                albedo_fine_directory=albedo_fine_directory,
+                downsampled_directory=DOWNSAMPLED_products_directory,
                 HLS_connection=HLS_connection,
                 NDVI_VIIRS_connection=NDVI_VIIRS_connection,
                 albedo_VIIRS_connection=albedo_VIIRS_connection,
@@ -454,7 +438,7 @@ def L2T_STARS(
                 NDVI_resolution=NDVI_resolution,
                 albedo_resolution=albedo_resolution,
                 target_resolution=target_resolution,
-                working_directory=working_directory,
+                downsampled_directory=DOWNSAMPLED_products_directory,
                 model_directory=model_directory,
                 input_staging_directory=input_staging_directory,
                 L2T_STARS_granule_directory=L2T_STARS_granule_directory,
@@ -470,6 +454,7 @@ def L2T_STARS(
                 remove_input_staging=remove_input_staging,
                 remove_prior=remove_prior,
                 remove_posterior=remove_posterior,
+                initialize_julia=initialize_julia,
                 threads=threads,
                 num_workers=num_workers,
             )

{ecov003_l2t_stars-1.3.0 → ecov003_l2t_stars-1.4.0}/ECOv003_L2T_STARS/Project.toml

@@ -3,12 +3,8 @@ Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 DelimitedFiles = "8bb1440f-4735-579b-a4ab-409b98df4dab"
 Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b"
 Glob = "c27321d9-0574-5035-807b-f59d2c89b15c"
-HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
-JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"
-OpenSSL = "4d8831e6-92b7-49fb-bdf8-b643e874388c"
-Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 Rasters = "a3a2b9e3-a471-40c9-b274-f788e487c689"
 STARSDataFusion = "70ccc657-289f-4534-a407-e03a16fd1153"
 Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"

{ecov003_l2t_stars-1.3.0 → ecov003_l2t_stars-1.4.0}/ECOv003_L2T_STARS/VIIRS/VNP09GA.py

@@ -1096,7 +1096,7 @@ class VNP09GA:
 
         self.resampling = resampling
 
-        self._granules = pd.DataFrame({"date_UTC": {}, "tile": {}, "granule": {}})
+        self._granules = pd.DataFrame(columns=["date_UTC", "tile", "granule"])
 
         if working_directory is None:
             working_directory = self.DEFAULT_WORKING_DIRECTORY
@@ -1233,7 +1233,7 @@ class VNP09GA:
         if "date_UTC" not in self._granules.columns:
             raise ValueError(f"date_UTC column not in granules table")
 
-        subset = self._granules[(self._granules.date_UTC == date_UTC) & (self._granules.tile == tile)]
+        subset = self._granules[(self._granules['date_UTC'] == date_UTC) & (self._granules['tile'] == tile)]
         if len(subset) > 0:
             return subset.iloc[0].granule
 

{ecov003_l2t_stars-1.3.0 → ecov003_l2t_stars-1.4.0}/ECOv003_L2T_STARS/VNP43NRT/VNP43NRT.py

@@ -106,12 +106,14 @@ def process_julia_BRDF(
         sensor_zenith_directory: str,
         relative_azimuth_directory: str,
         SZA_filename: str,
-        output_directory: str):
+        output_directory: str,
+        initialize_julia: bool):
     parent_directory = abspath(join(dirname(__file__), ".."))
     julia_source_directory = join(parent_directory, "VNP43NRT_jl")
     julia_script_filename = join(abspath(dirname(__file__)), "process_VNP43NRT.jl")
 
-    instantiate_VNP43NRT_jl(julia_source_directory)
+    if initialize_julia:
+        instantiate_VNP43NRT_jl(julia_source_directory)
 
     command = f'julia "{julia_script_filename}" "{band}" "{h}" "{v}" "{tile_width_cells}" "{start_date:%Y-%m-%d}" "{end_date:%Y-%m-%d}" "{reflectance_directory}" "{solar_zenith_directory}" "{sensor_zenith_directory}" "{relative_azimuth_directory}" "{SZA_filename}" "{output_directory}"'
     logger.info(command)
@@ -265,7 +267,8 @@ class VNP43NRT(VIIRSDownloaderAlbedo, VIIRSDownloaderNDVI):
             VNP43NRT_staging_directory: str = None,
             GEOS5FP_connection: GEOS5FP = None,
             GEOS5FP_download: str = None,
-            GEOS5FP_products: str = None):
+            GEOS5FP_products: str = None,
+            initialize_julia: bool = False):
         if working_directory is None:
             working_directory = VNP09GA.DEFAULT_WORKING_DIRECTORY
 
@@ -302,6 +305,7 @@ class VNP43NRT(VIIRSDownloaderAlbedo, VIIRSDownloaderNDVI):
         self.VNP43NRT_directory = VNP43NRT_directory
         self.GEOS5FP = GEOS5FP_connection
         self.VNP43NRT_staging_directory = VNP43NRT_staging_directory
+        self.initialize_julia = initialize_julia
 
     def __repr__(self):
         display_dict = {
@@ -493,7 +497,8 @@ class VNP43NRT(VIIRSDownloaderAlbedo, VIIRSDownloaderNDVI):
                 sensor_zenith_directory=sensor_zenith_directory,
                 relative_azimuth_directory=relative_azimuth_directory,
                 SZA_filename=SZA_filename,
-                output_directory=output_directory
+                output_directory=output_directory,
+                initialize_julia=self.initialize_julia,
             )
 
             WSA = Raster.open(join(output_directory, f"{date_UTC:%Y-%m-%d}_WSA.tif"))

{ecov003_l2t_stars-1.3.0 → ecov003_l2t_stars-1.4.0}/ECOv003_L2T_STARS/constants.py

@@ -22,6 +22,7 @@ DEFAULT_GEOS5FP_DOWNLOAD_DIRECTORY = "GEOS5FP_download"
 DEFAULT_GEOS5FP_PRODUCTS_DIRECTORY = "GEOS5FP_products"
 DEFAULT_VNP09GA_PRODUCTS_DIRECTORY = "VNP09GA_products"
 DEFAULT_VNP43NRT_PRODUCTS_DIRECTORY = "VNP43NRT_products"
+DEFAULT_STARS_DOWNSAMPLED_DIRECTORY = "DOWNSAMPLED_products"
 
 # Processing parameters
 VIIRS_GIVEUP_DAYS = 4  # Number of days to give up waiting for VIIRS data

ecov003_l2t_stars-1.4.0/ECOv003_L2T_STARS/generate_STARS_inputs.py

@@ -0,0 +1,233 @@
+from typing import Union
+from datetime import date, datetime
+from dateutil.rrule import rrule, DAILY
+from os.path import exists
+import logging
+
+import colored_logging as cl
+from rasters import Raster, RasterGeometry
+from harmonized_landsat_sentinel import HLS2CMR
+
+from ECOv003_exit_codes import AuxiliaryLatency
+
+from .constants import VIIRS_GIVEUP_DAYS
+from .generate_filename import generate_filename
+from .daterange import get_date
+from .generate_NDVI_coarse_image import generate_NDVI_coarse_image
+from .generate_NDVI_fine_image import generate_NDVI_fine_image
+from .generate_albedo_coarse_image import generate_albedo_coarse_image
+from .generate_albedo_fine_image import generate_albedo_fine_image
+from .generate_downsampled_filename import generate_downsampled_filename
+from .calibrate_fine_to_coarse import calibrate_fine_to_coarse
+from .VIIRS.VIIRSDownloader import VIIRSDownloaderAlbedo, VIIRSDownloaderNDVI
+
+logger = logging.getLogger(__name__)
+
+def generate_STARS_inputs(
+    tile: str,
+    date_UTC: date,
+    HLS_start_date: date,
+    HLS_end_date: date,
+    VIIRS_start_date: date,
+    VIIRS_end_date: date,
+    NDVI_resolution: int,
+    albedo_resolution: int,
+    target_resolution: int,
+    NDVI_coarse_geometry: RasterGeometry,
+    albedo_coarse_geometry: RasterGeometry,
+    downsampled_directory: str,
+    HLS_connection: HLS2CMR,
+    NDVI_VIIRS_connection: VIIRSDownloaderNDVI,
+    albedo_VIIRS_connection: VIIRSDownloaderAlbedo,
+    calibrate_fine: bool = False,
+):
+    """
+    Generates and stages the necessary coarse and fine resolution input images
+    for the STARS data fusion process.
+
+    This function iterates through the VIIRS date range, retrieving and saving
+    coarse NDVI and albedo images. For dates within the HLS range, it also
+    retrieves and saves fine NDVI and albedo images. It can optionally
+    calibrate the fine images to the coarse images.
+
+    Args:
+        tile (str): The HLS tile ID.
+        date_UTC (date): The target UTC date for the L2T_STARS product.
+        HLS_start_date (date): The start date for HLS data retrieval for the fusion period.
+        HLS_end_date (date): The end date for HLS data retrieval for the fusion period.
+        VIIRS_start_date (date): The start date for VIIRS data retrieval for the fusion period.
+        VIIRS_end_date (date): The end date for VIIRS data retrieval for the fusion period.
+        NDVI_resolution (int): The resolution of the coarse NDVI data.
+        albedo_resolution (int): The resolution of the coarse albedo data.
+        target_resolution (int): The desired output resolution of the fused product.
+        NDVI_coarse_geometry (RasterGeometry): The target geometry for coarse NDVI images.
+        albedo_coarse_geometry (RasterGeometry): The target geometry for coarse albedo images.
+        working_directory (str): The main working directory.
+        NDVI_coarse_directory (str): Directory for staging coarse NDVI images.
+        NDVI_fine_directory (str): Directory for staging fine NDVI images.
+        albedo_coarse_directory (str): Directory for staging coarse albedo images.
+        albedo_fine_directory (str): Directory for staging fine albedo images.
+        HLS_connection (HLS2CMR): An initialized HLS data connection object.
+        NDVI_VIIRS_connection (VIIRSDownloaderNDVI): An initialized VIIRS NDVI downloader.
+        albedo_VIIRS_connection (VIIRSDownloaderAlbedo): An initialized VIIRS albedo downloader.
+        calibrate_fine (bool, optional): If True, calibrate fine images to coarse images.
+                                         Defaults to False.
+
+    Raises:
+        AuxiliaryLatency: If coarse VIIRS data is missing within the VIIRS_GIVEUP_DAYS window.
+    """
+    missing_coarse_dates = set()  # Track dates where coarse data could not be generated
+
+    logger.info(f"preparing coarse and fine images for STARS at {cl.place(tile)}")
+
+    # Process each day within the VIIRS data fusion window
+    for processing_date in [
+        get_date(dt) for dt in rrule(DAILY, dtstart=VIIRS_start_date, until=VIIRS_end_date)
+    ]:
+        NDVI_coarse_filename = generate_downsampled_filename(
+            directory=downsampled_directory,
+            variable="NDVI",
+            date_UTC=processing_date,
+            tile=tile,
+            cell_size=NDVI_resolution
+        )
+
+        NDVI_fine_filename = generate_downsampled_filename(
+            directory=downsampled_directory,
+            variable="NDVI",
+            date_UTC=processing_date,
+            tile=tile,
+            cell_size=target_resolution
+        )
+
+        albedo_coarse_filename = generate_downsampled_filename(
+            directory=downsampled_directory,
+            variable="albedo",
+            date_UTC=processing_date,
+            tile=tile,
+            cell_size=albedo_resolution
+        )
+
+        albedo_fine_filename = generate_downsampled_filename(
+            directory=downsampled_directory,
+            variable="albedo",
+            date_UTC=processing_date,
+            tile=tile,
+            cell_size=target_resolution
+        )
+
+        try:
+            # Cache whether the NDVI coarse exists to avoid ToCToU
+            NDVI_coarse_exists = exists(NDVI_coarse_filename)
+            if not NDVI_coarse_exists:
+                logger.info(f"preparing coarse image for STARS NDVI at {cl.place(tile)} on {cl.time(processing_date)}")
+
+                NDVI_coarse_image = generate_NDVI_coarse_image(
+                    date_UTC=processing_date,
+                    VIIRS_connection=NDVI_VIIRS_connection,
+                    geometry=NDVI_coarse_geometry
+                )
+
+                logger.info(
+                    f"saving coarse image for STARS NDVI at {cl.place(tile)} on {cl.time(processing_date)}: {NDVI_coarse_filename}")
+                NDVI_coarse_image.to_geotiff(NDVI_coarse_filename)
+
+            if processing_date >= HLS_start_date:
+                try:
+                    if not exists(NDVI_fine_filename):
+                        logger.info(
+                            f"preparing fine image for STARS NDVI at {cl.place(tile)} on {cl.time(processing_date)}")
+
+                        NDVI_fine_image = generate_NDVI_fine_image(
+                            date_UTC=processing_date,
+                            tile=tile,
+                            HLS_connection=HLS_connection
+                        )
+
+                        if calibrate_fine:
+                            # Ensure that the NDVI_coarse_image variable is set
+                            if NDVI_coarse_exists:
+                                NDVI_coarse_image = Raster.open(NDVI_coarse_filename)
+                            logger.info(
+                                f"calibrating fine image for STARS NDVI at {cl.place(tile)} on {cl.time(processing_date)}")
+                            NDVI_fine_image = calibrate_fine_to_coarse(NDVI_fine_image, NDVI_coarse_image)
+
+                        logger.info(
+                            f"saving fine image for STARS NDVI at {cl.place(tile)} on {cl.time(processing_date)}: {NDVI_fine_filename}")
+                        NDVI_fine_image.to_geotiff(NDVI_fine_filename)
+                except Exception:  # Catch any exception during HLS fine image generation
+                    logger.info(f"HLS NDVI is not available on {processing_date}")
+        except Exception as e:
+            logger.exception(e)
+            logger.warning(
+                f"Unable to produce coarse NDVI for date {processing_date}"
+            )
+            missing_coarse_dates.add(processing_date)  # Add date to missing set
+
+        try:
+            # Cache whether the albedo coarse exists to avoid ToCToU
+            albedo_coarse_exists = exists(albedo_coarse_filename)
+            if not albedo_coarse_exists:
+                logger.info(
+                    f"preparing coarse image for STARS albedo at {cl.place(tile)} on {cl.time(processing_date)}")
+
+                albedo_coarse_image = generate_albedo_coarse_image(
+                    date_UTC=processing_date,
+                    VIIRS_connection=albedo_VIIRS_connection,
+                    geometry=albedo_coarse_geometry
+                )
+
+                logger.info(
+                    f"saving coarse image for STARS albedo at {cl.place(tile)} on {cl.time(processing_date)}: {albedo_coarse_filename}")
+                albedo_coarse_image.to_geotiff(albedo_coarse_filename)
+
+            if processing_date >= HLS_start_date:
+                try:
+                    if not exists(albedo_fine_filename):
+                        logger.info(
+                            f"preparing fine image for STARS albedo at {cl.place(tile)} on {cl.time(processing_date)}")
+
+                        albedo_fine_image = generate_albedo_fine_image(
+                            date_UTC=processing_date,
+                            tile=tile,
+                            HLS_connection=HLS_connection
+                        )
+
+                        if calibrate_fine:
+                            # Ensure that the albedo_coarse_image variable is set
+                            if albedo_coarse_exists:
+                                albedo_coarse_image = Raster.open(albedo_coarse_filename)
+
+                            logger.info(
+                                f"calibrating fine image for STARS albedo at {cl.place(tile)} on {cl.time(processing_date)}")
+                            albedo_fine_image = calibrate_fine_to_coarse(albedo_fine_image, albedo_coarse_image)
+
+                        logger.info(
+                            f"saving fine image for STARS albedo at {cl.place(tile)} on {cl.time(processing_date)}: {albedo_fine_filename}")
+                        albedo_fine_image.to_geotiff(albedo_fine_filename)
+                except Exception:  # Catch any exception during HLS fine image generation
+                    logger.info(f"HLS albedo is not available on {processing_date}")
+        except Exception as e:
+            logger.exception(e)
+            logger.warning(
+                f"Unable to produce coarse albedo for date {processing_date}"
+            )
+            missing_coarse_dates.add(processing_date)  # Add date to missing set
+
+    # We need to deal with the possibility that VIIRS has not yet published their data yet.
+    #  VIIRS_GIVEUP_DAYS is the number of days before we assume that missing observations aren't coming.
+    #  If any missing days are closer to now than VIIRS_GIVEUP_DAYS, we want to retry this run later, when VIIRS
+    #  might have uploaded the missing observations. To cause this retry, we'll throw the `AncillaryLatency` exception.
+    #  L2T_STARS converts this exception to an exit code, and the orchestration system marks this run
+    #  as needing a retry at a later date.
+    coarse_latency_dates = [
+        d
+        for d in missing_coarse_dates
+        if (datetime.utcnow().date() - d).days <= VIIRS_GIVEUP_DAYS
+    ]
+
+    if len(coarse_latency_dates) > 0:
+        raise AuxiliaryLatency(
+            f"Missing coarse dates within {VIIRS_GIVEUP_DAYS}-day window: "
+            f"{', '.join([str(d) for d in sorted(list(coarse_latency_dates))])}"
+        )

ecov003_l2t_stars-1.4.0/ECOv003_L2T_STARS/generate_downsampled_filename.py

@@ -0,0 +1,20 @@
+from os import makedirs
+from os.path import join, dirname
+from dateutil import parser
+from datetime import date
+
+from typing import Union
+
+def generate_downsampled_filename(directory: str, variable: str, date_UTC: Union[date, str], tile: str, cell_size: int) -> str:
+    if isinstance(date_UTC, str):
+        date_UTC = parser.parse(date_UTC).date()
+
+    variable = str(variable)
+    year = str(date_UTC.year)
+    timestamp = date_UTC.strftime("%Y-%m-%d")
+    tile = str(tile)
+    cell_size = int(cell_size)
+    filename = join(directory, year, timestamp, tile, f"STARS_{variable}_{tile}_{cell_size}m.tif")
+    makedirs(dirname(filename), exist_ok=True)
+
+    return filename

{ecov003_l2t_stars-1.3.0 → ecov003_l2t_stars-1.4.0}/ECOv003_L2T_STARS/load_prior.py

@@ -98,7 +98,7 @@ def load_prior(
                 cell_size=target_resolution,
             )
             # Assuming L2T_STARS_prior_granule has a .NDVI_bias attribute
-            if hasattr(L2T_STARS_prior_granule, "NDVI_bias") and L2T_STARS_prior_granule.NDVI_bias is not None:
+            if L2T_STARS_prior_granule.NDVI_bias is not None:
                 L2T_STARS_prior_granule.NDVI_bias.to_geotiff(prior_NDVI_bias_filename)
             else:
                 prior_NDVI_bias_filename = None # Set to None if not available
@@ -111,7 +111,7 @@ def load_prior(
                 cell_size=target_resolution,
             )
             # Assuming L2T_STARS_prior_granule has a .NDVI_bias_UQ attribute
-            if hasattr(L2T_STARS_prior_granule, "NDVI_bias_UQ") and L2T_STARS_prior_granule.NDVI_bias_UQ is not None:
+            if L2T_STARS_prior_granule.NDVI_bias_UQ is not None:
                 L2T_STARS_prior_granule.NDVI_bias_UQ.to_geotiff(prior_NDVI_bias_UQ_filename)
             else:
                 prior_NDVI_bias_UQ_filename = None # Set to None if not available
@@ -143,7 +143,7 @@ def load_prior(
                 cell_size=target_resolution,
             )
             # Assuming L2T_STARS_prior_granule has a .albedo_bias attribute
-            if hasattr(L2T_STARS_prior_granule, "albedo_bias") and L2T_STARS_prior_granule.albedo_bias is not None:
+            if L2T_STARS_prior_granule.albedo_bias is not None:
                 L2T_STARS_prior_granule.albedo_bias.to_geotiff(prior_albedo_bias_filename)
             else:
                 prior_albedo_bias_filename = None # Set to None if not available
@@ -156,7 +156,7 @@ def load_prior(
                 cell_size=target_resolution,
             )
             # Assuming L2T_STARS_prior_granule has a .albedo_bias_UQ attribute
-            if hasattr(L2T_STARS_prior_granule, "albedo_bias_UQ") and L2T_STARS_prior_granule.albedo_bias_UQ is not None:
+            if L2T_STARS_prior_granule.albedo_bias_UQ is not None:
                 L2T_STARS_prior_granule.albedo_bias_UQ.to_geotiff(prior_albedo_bias_UQ_filename)
             else:
                 prior_albedo_bias_UQ_filename = None # Set to None if not available

{ecov003_l2t_stars-1.3.0 → ecov003_l2t_stars-1.4.0}/ECOv003_L2T_STARS/main.py

@@ -115,6 +115,12 @@ def main():
         default=True,
         help="Do NOT remove posterior intermediate files after product generation.",
     )
+    parser.add_argument(
+        "--initialize-julia",
+        action="store_true",
+        dest="initialize_julia",
+        help="Initialize a julia environment before running julia.",
+    )
     parser.add_argument(
         "--threads",
         type=str,
@@ -157,6 +163,7 @@ def main():
         remove_input_staging=args.remove_input_staging,
         remove_prior=args.remove_prior,
         remove_posterior=args.remove_posterior,
+        initialize_julia=args.initialize_julia,
         threads=args.threads,
         num_workers=args.num_workers,
         overwrite=args.overwrite, # Pass the new overwrite argument

{ecov003_l2t_stars-1.3.0 → ecov003_l2t_stars-1.4.0}/ECOv003_L2T_STARS/process_ECOSTRESS_data_fusion_distributed_bias.jl

@@ -8,24 +8,8 @@ using STARSDataFusion.sentinel_tiles
 using STARSDataFusion.HLS
 using STARSDataFusion.VNP43
 using Logging
-using Pkg
 using Statistics
 using Distributed
-Pkg.add("OpenSSL")
-using HTTP
-using JSON
-
-function read_json(file::String)::Dict
-    open(file, "r") do f
-        return JSON.parse(f)
-    end
-end
-
-function write_json(file::String, data::Dict)
-    open(file, "w") do f
-        JSON.print(f, data)
-    end
-end
 
 @info "processing STARS data fusion"
 
@@ -78,10 +62,10 @@ HLS_start_date = Date(ARGS[7])
 @info "HLS start date: $(HLS_start_date)"
 HLS_end_date = Date(ARGS[8])
 @info "HLS end date: $(HLS_end_date)"
-coarse_directory = ARGS[9]
-@info "coarse inputs directory: $(coarse_directory)"
-fine_directory = ARGS[10]
-@info "fine inputs directory: $(fine_directory)"
+downsampled_directory = ARGS[9]
+@info "downsampled inputs directory: $(downsampled_directory)"
+product_name = ARGS[10]
+@info "Computing $(product_name) product"
 posterior_filename = ARGS[11]
 @info "posterior filename: $(posterior_filename)"
 posterior_UQ_filename = ARGS[12]
@@ -130,20 +114,29 @@ y_fine_size = size(y_fine)[1]
 @info "fine x size: $(x_fine_size)"
 @info "fine y size: $(y_fine_size)"
 
-coarse_image_filenames = sort(glob("*.tif", coarse_directory))
-coarse_dates_found = [Date(split(basename(filename), "_")[3]) for filename in coarse_image_filenames]
-
-fine_image_filenames = sort(glob("*.tif", fine_directory))
-fine_dates_found = [Date(split(basename(filename), "_")[3]) for filename in fine_image_filenames]
-
+# The range of dates to check for VIIRS files
 coarse_start_date = VIIRS_start_date
 coarse_end_date = VIIRS_end_date
 
+# Check each coarse date for a downsampled image
+# For each day we find, convert the date directory back into a date object
+coarse_dates = [coarse_start_date + Day(d - 1) for d in 1:((coarse_end_date - coarse_start_date).value + 1)]
+coarse_image_filenames = [joinpath("$(downsampled_directory)", "$(year(date))", "$(Dates.format(date, dateformat"yyyy-mm-dd"))", "$(tile)", "STARS_$(product_name)_$(tile)_$(coarse_cell_size)m.tif") for date in coarse_dates]
+coarse_image_filenames = [filename for filename in coarse_image_filenames if ispath(filename)]
+coarse_dates_found = [Date(basename(dirname(dirname(filename)))) for filename in coarse_image_filenames]
+
+# The range of dates to check for HLS files
 fine_flag_start_date = HLS_end_date - Day(7)
 fine_start_date = HLS_start_date
 fine_end_date = HLS_end_date
 
+# Check each fine date for a downsampled image
+# For each day we find, convert the date directory back into a date object
 dates = [fine_start_date + Day(d - 1) for d in 1:((fine_end_date - fine_start_date).value + 1)]
+fine_image_filenames = [joinpath("$(downsampled_directory)", "$(year(date))", "$(Dates.format(date, dateformat"yyyy-mm-dd"))", "$(tile)", "STARS_$(product_name)_$(tile)_$(fine_cell_size)m.tif") for date in dates]
+fine_image_filenames = [filename for filename in fine_image_filenames if ispath(filename)]
+fine_dates_found = [Date(basename(dirname(dirname(filename)))) for filename in fine_image_filenames]
+
 t = Ti(dates)
 coarse_dims = (x_coarse, y_coarse, t)
 fine_dims = (x_fine, y_fine, t)