ECOv003-L2T-STARS 1.1.0__tar.gz → 1.2.0__tar.gz

{ecov003_l2t_stars-1.1.0 → ecov003_l2t_stars-1.2.0}/.github/workflows/ci.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.10"]
+        python-version: ["3.10", "3.11"]
 
     steps:
       - name: Checkout repository

{ecov003_l2t_stars-1.1.0 → ecov003_l2t_stars-1.2.0}/ECOv003_L2T_STARS/ECOv003_DL.py

@@ -265,33 +265,33 @@ class ECOv003DLConfig(ECOSTRESSRunConfig):
             runconfig = read_runconfig(filename)
 
             # Validate and extract working directory
-            if "StaticAuxiliaryFileGroup" not in runconfig:
-                raise MissingRunConfigValue(f"missing StaticAuxiliaryFileGroup in ECOv003_DL run-config: {filename}")
-            if "ECOv003_DL_WORKING" not in runconfig["StaticAuxiliaryFileGroup"]:
+            if "StaticAncillaryFileGroup" not in runconfig:
+                raise MissingRunConfigValue(f"missing StaticAncillaryFileGroup in ECOv003_DL run-config: {filename}")
+            if "ECOv003_DL_WORKING" not in runconfig["StaticAncillaryFileGroup"]:
                 raise MissingRunConfigValue(
-                    f"missing StaticAuxiliaryFileGroup/ECOv003_DL_WORKING in ECOv003_DL run-config: {filename}")
-            working_directory = abspath(runconfig["StaticAuxiliaryFileGroup"]["ECOv003_DL_WORKING"])
+                    f"missing StaticAncillaryFileGroup/ECOv003_DL_WORKING in ECOv003_DL run-config: {filename}")
+            working_directory = abspath(runconfig["StaticAncillaryFileGroup"]["ECOv003_DL_WORKING"])
             logger.info(f"working directory: {cl.dir(working_directory)}")
 
             # Validate and extract L2T STARS sources directory
-            if "L2T_STARS_SOURCES" not in runconfig["StaticAuxiliaryFileGroup"]:
+            if "L2T_STARS_SOURCES" not in runconfig["StaticAncillaryFileGroup"]:
                 raise MissingRunConfigValue(
-                    f"missing StaticAuxiliaryFileGroup/L2T_STARS_SOURCES in ECOv003_DL run-config: {filename}")
-            L2T_STARS_sources_directory = abspath(runconfig["StaticAuxiliaryFileGroup"]["L2T_STARS_SOURCES"])
+                    f"missing StaticAncillaryFileGroup/L2T_STARS_SOURCES in ECOv003_DL run-config: {filename}")
+            L2T_STARS_sources_directory = abspath(runconfig["StaticAncillaryFileGroup"]["L2T_STARS_SOURCES"])
             logger.info(f"L2T STARS sources directory: {cl.dir(L2T_STARS_sources_directory)}")
 
             # Validate and extract L2T STARS indices directory
-            if "L2T_STARS_INDICES" not in runconfig["StaticAuxiliaryFileGroup"]:
+            if "L2T_STARS_INDICES" not in runconfig["StaticAncillaryFileGroup"]:
                 raise MissingRunConfigValue(
-                    f"missing StaticAuxiliaryFileGroup/L2T_STARS_INDICES in ECOv003_DL run-config: {filename}")
-            L2T_STARS_indices_directory = abspath(runconfig["StaticAuxiliaryFileGroup"]["L2T_STARS_INDICES"])
+                    f"missing StaticAncillaryFileGroup/L2T_STARS_INDICES in ECOv003_DL run-config: {filename}")
+            L2T_STARS_indices_directory = abspath(runconfig["StaticAncillaryFileGroup"]["L2T_STARS_INDICES"])
             logger.info(f"L2T STARS indices directory: {cl.dir(L2T_STARS_indices_directory)}")
 
             # Validate and extract L2T STARS model directory
-            if "L2T_STARS_MODEL" not in runconfig["StaticAuxiliaryFileGroup"]:
+            if "L2T_STARS_MODEL" not in runconfig["StaticAncillaryFileGroup"]:
                 raise MissingRunConfigValue(
-                    f"missing StaticAuxiliaryFileGroup/L2T_STARS_MODEL in ECOv003_DL run-config: {filename}")
-            L2T_STARS_model_directory = abspath(runconfig["StaticAuxiliaryFileGroup"]["L2T_STARS_MODEL"])
+                    f"missing StaticAncillaryFileGroup/L2T_STARS_MODEL in ECOv003_DL run-config: {filename}")
+            L2T_STARS_model_directory = abspath(runconfig["StaticAncillaryFileGroup"]["L2T_STARS_MODEL"])
             logger.info(f"L2T STARS model directory: {cl.dir(L2T_STARS_model_directory)}")
 
             # Validate ProductPathGroup

{ecov003_l2t_stars-1.1.0 → ecov003_l2t_stars-1.2.0}/ECOv003_L2T_STARS/ECOv003_DL.xml

@@ -28,7 +28,7 @@
             <element>L2T_LSTE_filename1</element>
         </vector>
     </group>
-    <group name="StaticAuxiliaryFileGroup">
+    <group name="StaticAncillaryFileGroup">
         <scalar name="ECOv003_DL_WORKING">working_directory</scalar>
         <scalar name="L2T_STARS_SOURCES">L2T_STARS_sources_directory</scalar>
         <scalar name="L2T_STARS_INDICES">L2T_STARS_indices_directory</scalar>

{ecov003_l2t_stars-1.1.0 → ecov003_l2t_stars-1.2.0}/ECOv003_L2T_STARS/ECOv003_L2T_STARS.py

@@ -26,6 +26,7 @@ def main():
                "  python {sys.argv[0]} --runconfig /path/to/RunConfig.xml\n"
                "  python {sys.argv[0]} --runconfig /path/to/RunConfig.xml --date 2023-01-15\n"
                "  python {sys.argv[0]} --runconfig /path/to/RunConfig.xml --sources-only\n"
+               "  python {sys.argv[0]} --runconfig /path/to/RunConfig.xml --overwrite\n" # Added example usage
     )
 
     # Positional argument for the runconfig file
@@ -128,6 +129,11 @@ def main():
         help=f"Number of Julia workers for distributed processing. Defaults to 4.",
         metavar="COUNT"
     )
+    parser.add_argument(
+        "--overwrite", # New argument for overwrite option
+        action="store_true",
+        help="Reproduce the output files even if they already exist.",
+    )
     parser.add_argument(
         "--version",
         action="version",
@@ -153,10 +159,11 @@ def main():
         remove_posterior=args.remove_posterior,
         threads=args.threads,
         num_workers=args.num_workers,
+        overwrite=args.overwrite, # Pass the new overwrite argument
     )
 
     sys.exit(exit_code)
 
 
 if __name__ == "__main__":
-    main()
+    main()

{ecov003_l2t_stars-1.1.0 → ecov003_l2t_stars-1.2.0}/ECOv003_L2T_STARS/ECOv003_L2T_STARS.xml

@@ -22,7 +22,7 @@
         <scalar name="L2T_LSTE">L2T_LSTE_filename</scalar>
         <scalar name="L2T_STARS_PRIOR">prior_L2T_STARS_filename</scalar>
     </group>
-    <group name="StaticAuxiliaryFileGroup">
+    <group name="StaticAncillaryFileGroup">
         <scalar name="L2T_STARS_MODEL">model_directory</scalar>
         <scalar name="L2T_STARS_INDICES">indices_directory</scalar>
         <scalar name="L2T_STARS_SOURCES">sources_directory</scalar>

{ecov003_l2t_stars-1.1.0 → ecov003_l2t_stars-1.2.0}/ECOv003_L2T_STARS/L2TSTARSConfig.py

@@ -37,48 +37,50 @@ class L2TSTARSConfig(ECOSTRESSRunConfig):
         # Read the run-config XML into a dictionary
         runconfig = self.read_runconfig(filename)
 
+        # reverting to StaticAncillaryFileGroup for now instead of StaticAuxiliaryFileGroup
+
         try:
-            # Validate and extract working directory from StaticAuxiliaryFileGroup
-            if "StaticAuxiliaryFileGroup" not in runconfig:
+            # Validate and extract working directory from StaticAncillaryFileGroup
+            if "StaticAncillaryFileGroup" not in runconfig:
                 raise MissingRunConfigValue(
-                    f"Missing StaticAuxiliaryFileGroup in L2T_STARS run-config: {filename}"
+                    f"Missing StaticAncillaryFileGroup in L2T_STARS run-config: {filename}"
                 )
-            if "L2T_STARS_WORKING" not in runconfig["StaticAuxiliaryFileGroup"]:
+            if "L2T_STARS_WORKING" not in runconfig["StaticAncillaryFileGroup"]:
                 raise MissingRunConfigValue(
-                    f"Missing StaticAuxiliaryFileGroup/L2T_STARS_WORKING in L2T_STARS run-config: {filename}"
+                    f"Missing StaticAncillaryFileGroup/L2T_STARS_WORKING in L2T_STARS run-config: {filename}"
                 )
             self.working_directory = abspath(
-                runconfig["StaticAuxiliaryFileGroup"]["L2T_STARS_WORKING"]
+                runconfig["StaticAncillaryFileGroup"]["L2T_STARS_WORKING"]
             )
             logger.info(f"Working directory: {cl.dir(self.working_directory)}")
 
             # Validate and extract sources directory
-            if "L2T_STARS_SOURCES" not in runconfig["StaticAuxiliaryFileGroup"]:
+            if "L2T_STARS_SOURCES" not in runconfig["StaticAncillaryFileGroup"]:
                 raise MissingRunConfigValue(
-                    f"Missing StaticAuxiliaryFileGroup/L2T_STARS_SOURCES in L2T_STARS run-config: {filename}"
+                    f"Missing StaticAncillaryFileGroup/L2T_STARS_SOURCES in L2T_STARS run-config: {filename}"
                 )
             self.sources_directory = abspath(
-                runconfig["StaticAuxiliaryFileGroup"]["L2T_STARS_SOURCES"]
+                runconfig["StaticAncillaryFileGroup"]["L2T_STARS_SOURCES"]
             )
             logger.info(f"Sources directory: {cl.dir(self.sources_directory)}")
 
             # Validate and extract indices directory
-            if "L2T_STARS_INDICES" not in runconfig["StaticAuxiliaryFileGroup"]:
+            if "L2T_STARS_INDICES" not in runconfig["StaticAncillaryFileGroup"]:
                 raise MissingRunConfigValue(
-                    f"Missing StaticAuxiliaryFileGroup/L2T_STARS_INDICES in L2T_STARS run-config: {filename}"
+                    f"Missing StaticAncillaryFileGroup/L2T_STARS_INDICES in L2T_STARS run-config: {filename}"
                 )
             self.indices_directory = abspath(
-                runconfig["StaticAuxiliaryFileGroup"]["L2T_STARS_INDICES"]
+                runconfig["StaticAncillaryFileGroup"]["L2T_STARS_INDICES"]
             )
             logger.info(f"Indices directory: {cl.dir(self.indices_directory)}")
 
             # Validate and extract model directory
-            if "L2T_STARS_MODEL" not in runconfig["StaticAuxiliaryFileGroup"]:
+            if "L2T_STARS_MODEL" not in runconfig["StaticAncillaryFileGroup"]:
                 raise MissingRunConfigValue(
-                    f"Missing StaticAuxiliaryFileGroup/L2T_STARS_MODEL in L2T_STARS run-config: {filename}"
+                    f"Missing StaticAncillaryFileGroup/L2T_STARS_MODEL in L2T_STARS run-config: {filename}"
                 )
             self.model_directory = abspath(
-                runconfig["StaticAuxiliaryFileGroup"]["L2T_STARS_MODEL"]
+                runconfig["StaticAncillaryFileGroup"]["L2T_STARS_MODEL"]
             )
             logger.info(f"Model directory: {cl.dir(self.model_directory)}")
 

{ecov003_l2t_stars-1.1.0 → ecov003_l2t_stars-1.2.0}/ECOv003_L2T_STARS/L2T_STARS.py

@@ -60,6 +60,7 @@ def L2T_STARS(
     remove_posterior: bool = True,
     threads: Union[int, str] = "auto",
     num_workers: int = 4,
+    overwrite: bool = False, # New parameter for overwriting existing files
 ) -> int:
     """
     ECOSTRESS Collection 3 L2T_STARS PGE (Product Generation Executive).
@@ -98,6 +99,8 @@ def L2T_STARS(
                                             Defaults to "auto".
         num_workers (int, optional): Number of Julia workers for distributed processing.
                                      Defaults to 4.
+        overwrite (bool, optional): If True, existing output files will be overwritten.
+                                    Defaults to False.
 
     Returns:
         int: An exit code indicating the success or failure of the PGE execution.
@@ -127,11 +130,17 @@ def L2T_STARS(
         L2T_STARS_browse_filename = runconfig.L2T_STARS_browse_filename
         logger.info(f"Browse filename: " + cl.file(L2T_STARS_browse_filename))
 
-        # Check if the final product already exists to avoid reprocessing
-        if exists(L2T_STARS_zip_filename) and exists(L2T_STARS_browse_filename):
+        # Check if the final product already exists and 'overwrite' is not enabled
+        if not overwrite and exists(L2T_STARS_zip_filename) and exists(L2T_STARS_browse_filename):
             logger.info(f"Found existing L2T STARS file: {L2T_STARS_zip_filename}")
             logger.info(f"Found existing L2T STARS preview: {L2T_STARS_browse_filename}")
+            logger.info("Overwrite option is not enabled, skipping reprocessing.")
             return SUCCESS_EXIT_CODE
+        elif overwrite and exists(L2T_STARS_zip_filename) and exists(L2T_STARS_browse_filename):
+            logger.info(f"Found existing L2T STARS file: {L2T_STARS_zip_filename}")
+            logger.info(f"Found existing L2T STARS preview: {L2T_STARS_browse_filename}")
+            logger.info("Overwrite option is enabled, proceeding with reprocessing.")
+
 
         logger.info(f"Working directory: {cl.dir(working_directory)}")
         logger.info(f"Log file: {cl.file(log_filename)}")
@@ -232,12 +241,17 @@ def L2T_STARS(
         VNP43NRT_products_directory = join(sources_directory, DEFAULT_VNP43NRT_PRODUCTS_DIRECTORY)
         logger.info(f"VNP43NRT products directory: {cl.dir(VNP43NRT_products_directory)}")
 
-        # Re-check for existing product (double-check in case another process created it)
-        if exists(L2T_STARS_zip_filename):
+        # Re-check for existing product (double-check in case another process created it) with overwrite option
+        if not overwrite and exists(L2T_STARS_zip_filename):
             logger.info(
-                f"Found L2T STARS product zip: {cl.file(L2T_STARS_zip_filename)}"
+                f"Found L2T STARS product zip: {cl.file(L2T_STARS_zip_filename)}. Overwrite is False, returning."
             )
             return exit_code
+        elif overwrite and exists(L2T_STARS_zip_filename):
+            logger.info(
+                f"Found L2T STARS product zip: {cl.file(L2T_STARS_zip_filename)}. Overwrite is True, proceeding."
+            )
+
 
         # Initialize HLS data connection
         logger.info(f"Connecting to CMR Search server: {CMR_SEARCH_URL}")

{ecov003_l2t_stars-1.1.0 → ecov003_l2t_stars-1.2.0}/ECOv003_L2T_STARS/LPDAAC/LPDAACDataPool.py

@@ -54,7 +54,7 @@ class LPDAACServerUnreachable(ConnectionError):
 class LPDAACDataPool:
     logger = logging.getLogger(__name__)
     DEFAULT_CHUNK_SIZE = 2 ** 20
-    DATE_REGEX = re.compile('^(19|20)\d\d[- /.](0[1-9]|1[012])[- /.](0[1-9]|[12][0-9]|3[01])$')
+    DATE_REGEX = re.compile(r'^(19|20)\d\d[- /.](0[1-9]|1[012])[- /.](0[1-9]|[12][0-9]|3[01])$')
     DEFAULT_REMOTE = DEFAULT_REMOTE
 
     def __init__(self, username: str = None, password: str = None, remote: str = None, offline_ok: bool = False):

{ecov003_l2t_stars-1.1.0 → ecov003_l2t_stars-1.2.0}/ECOv003_L2T_STARS/VIIRS/VNP09GA.py

@@ -1019,7 +1019,8 @@ def latest_datetime(date_in: Union[date, str]) -> datetime:
     return parser.parse(f"{date_string}T23:59:59Z")
 
 
-VIIRS_FILENAME_REGEX = re.compile("^VNP09GA\.[^.]+\.([^.]+)\.002\.\d+\.h5$")
+VIIRS_FILENAME_REGEX = re.compile(r"^VNP09GA\.[^.]+\.([^.]+)\.002\.\d+\.h5$")
+
 def modland_tile_from_filename(filename: str) -> str:
     match = VIIRS_FILENAME_REGEX.match(filename)
     if match is None:

{ecov003_l2t_stars-1.1.0 → ecov003_l2t_stars-1.2.0}/ECOv003_L2T_STARS/generate_L2T_STARS_runconfig.py

@@ -246,4 +246,4 @@ def generate_L2T_STARS_runconfig(
         f"Finished generating L2T_STARS run-config for orbit {cl.val(orbit)} scene {cl.val(scene)} ({timer.tocvalue():.2f} seconds)"
     )
 
-    return
+    return runconfig_filename

{ecov003_l2t_stars-1.1.0 → ecov003_l2t_stars-1.2.0}/ECOv003_L2T_STARS/load_prior.py

@@ -1,6 +1,8 @@
 from os.path import exists
 import logging
 
+import colored_logging as cl
+
 from ECOv003_granules import L2TSTARS
 
 from .prior import Prior
@@ -235,12 +237,12 @@ def load_prior(
         L2T_STARS_prior_filename=L2T_STARS_prior_filename,
         prior_NDVI_filename=prior_NDVI_filename,
         prior_NDVI_UQ_filename=prior_NDVI_UQ_filename,
-        prior_NDVI_flag_filename=prior_NDVI_flag_filename,
+        # prior_NDVI_flag_filename=prior_NDVI_flag_filename,
         prior_NDVI_bias_filename=prior_NDVI_bias_filename,
         prior_NDVI_bias_UQ_filename=prior_NDVI_bias_UQ_filename,
         prior_albedo_filename=prior_albedo_filename,
         prior_albedo_UQ_filename=prior_albedo_UQ_filename,
-        prior_albedo_flag_filename=prior_albedo_flag_filename,
+        # prior_albedo_flag_filename=prior_albedo_flag_filename,
         prior_albedo_bias_filename=prior_albedo_bias_filename,
         prior_albedo_bias_UQ_filename=prior_albedo_bias_UQ_filename,
     )

ecov003_l2t_stars-1.2.0/ECOv003_L2T_STARS/version.txt

@@ -0,0 +1 @@
+1.2.0

{ecov003_l2t_stars-1.1.0 → ecov003_l2t_stars-1.2.0}/ECOv003_L2T_STARS.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
-Name: ECOv003_L2T_STARS
-Version: 1.1.0
+Name: ECOv003-L2T-STARS
+Version: 1.2.0
 Summary: ECOSTRESS Collection 3 JPL STARS Data Fusion Product Generating Executable (PGE)
 Author-email: "Gregory H. Halverson" <gregory.h.halverson@jpl.nasa.gov>
 Project-URL: Homepage, https://github.com/ECOSTRESS-Collection-3/ECOv003-L2T-STARS

{ecov003_l2t_stars-1.1.0 → ecov003_l2t_stars-1.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
-Name: ECOv003_L2T_STARS
-Version: 1.1.0
+Name: ECOv003-L2T-STARS
+Version: 1.2.0
 Summary: ECOSTRESS Collection 3 JPL STARS Data Fusion Product Generating Executable (PGE)
 Author-email: "Gregory H. Halverson" <gregory.h.halverson@jpl.nasa.gov>
 Project-URL: Homepage, https://github.com/ECOSTRESS-Collection-3/ECOv003-L2T-STARS

{ecov003_l2t_stars-1.1.0 → ecov003_l2t_stars-1.2.0}/example_L2T_STARS_with_download.py

@@ -20,9 +20,6 @@ L2T_LSTE_granule = download_ECOSTRESS_granule(
     parent_directory=working_directory
 )
 
-# Preview surface temperature
-print("Surface temperature preview:", L2T_LSTE_granule.ST_C)
-
 # Retrieve L2T STARS granule from CMR API as prior
 L2T_STARS_granule = download_ECOSTRESS_granule(
     product="L2T_STARS", 
@@ -31,9 +28,6 @@ L2T_STARS_granule = download_ECOSTRESS_granule(
     parent_directory=working_directory
 )
 
-# Preview albedo
-print("Albedo preview:", L2T_STARS_granule.albedo)
-
 # Generate XML run-config file for L2T STARS PGE run
 runconfig_filename = generate_L2T_STARS_runconfig(
     L2T_LSTE_filename=L2T_LSTE_granule.product_filename,
@@ -43,9 +37,6 @@ runconfig_filename = generate_L2T_STARS_runconfig(
 
 print("Runconfig generated:", runconfig_filename)
 
-with open(runconfig_filename, "r") as f:
-    print(f.read())
-
 # Run L2T STARS PGE
 exit_code = L2T_STARS(
     runconfig_filename=runconfig_filename,
@@ -54,7 +45,8 @@ exit_code = L2T_STARS(
     num_workers=8,
     remove_input_staging=False,
     remove_prior=False,
-    remove_posterior=False
+    remove_posterior=False,
+    overwrite=True
 )
 
 print("L2T STARS PGE exit code:", exit_code)

{ecov003_l2t_stars-1.1.0 → ecov003_l2t_stars-1.2.0}/makefile

@@ -39,21 +39,13 @@ reinstall:
 	make install
 
 environment:
-	mamba create -y -n $(ENVIRONMENT_NAME) -c conda-forge python=3.10
+	mamba create -y -n $(ENVIRONMENT_NAME) -c conda-forge python=3.11
 
 remove-environment:
 	mamba env remove -y -n $(ENVIRONMENT_NAME)
 
 install-julia:
-	julia -e 'using Pkg; Pkg.add("Glob")'
-	julia -e 'using Pkg; Pkg.add("DimensionalData")'
-	julia -e 'using Pkg; Pkg.add("HTTP")'
-	julia -e 'using Pkg; Pkg.add("JSON")'
-	julia -e 'using Pkg; Pkg.add("ArchGDAL")'
-	julia -e 'using Pkg; Pkg.add("Rasters")'
-	# julia -e 'using Pkg; Pkg.add(Pkg.PackageSpec(name="Rasters", version="0.5.3"))'
-	julia -e 'using Pkg; Pkg.add("STARSDataFusion")'
-	julia -e 'using Pkg; Pkg.develop(path="ECOv003_L2T_STARS/VNP43NRT_jl")'
+    julia -e 'using Pkg; Pkg.add.(["Glob", "DimensionalData", "HTTP", "JSON", "ArchGDAL", "Rasters", "STARSDataFusion"]); Pkg.develop(path="ECOv003_L2T_STARS/VNP43NRT_jl")'
 
 colima-start:
 	colima start -m 16 -a x86_64 -d 100 

{ecov003_l2t_stars-1.1.0 → ecov003_l2t_stars-1.2.0}/pyproject.toml

@@ -3,8 +3,8 @@ requires = ["setuptools>=60", "setuptools-scm>=8.0", "wheel"]
 build-backend = "setuptools.build_meta"
 
 [project]
-name = "ECOv003_L2T_STARS"
-version = "1.1.0"
+name = "ECOv003-L2T-STARS"
+version = "1.2.0"
 description = "ECOSTRESS Collection 3 JPL STARS Data Fusion Product Generating Executable (PGE)"
 readme = "README.md"
 authors = [

ecov003_l2t_stars-1.1.0/ECOv003_L2T_STARS/version.txt

@@ -1 +0,0 @@
-1.1.0