ECOv003-L2T-STARS 1.0.1__py3-none-any.whl → 1.2.0__py3-none-any.whl

ECOv003_L2T_STARS/ECOv003_L2T_STARS.py

@@ -0,0 +1,169 @@
+import sys
+import argparse
+import logging
+
+from .version import __version__
+from .constants import *
+from .L2T_STARS import L2T_STARS
+
+# Initialize the logger for the module
+logger = logging.getLogger(__name__)
+
+
+def main():
+    """
+    Main function for parsing command-line arguments and running the L2T_STARS PGE.
+
+    This function uses `argparse` for robust command-line argument parsing,
+    providing a clear interface for users to specify the run-configuration file
+    and other optional parameters.
+    """
+    parser = argparse.ArgumentParser(
+        description="ECOSTRESS Collection 3 L2T_STARS PGE for generating Tiled Auxiliary NDVI and Albedo products.",
+        formatter_class=argparse.RawTextHelpFormatter, # Allows for more flexible help text formatting
+        epilog=f"L2T_STARS PGE Version: {__version__}\n\n"
+               "Example usage:\n"
+               "  python {sys.argv[0]} --runconfig /path/to/RunConfig.xml\n"
+               "  python {sys.argv[0]} --runconfig /path/to/RunConfig.xml --date 2023-01-15\n"
+               "  python {sys.argv[0]} --runconfig /path/to/RunConfig.xml --sources-only\n"
+               "  python {sys.argv[0]} --runconfig /path/to/RunConfig.xml --overwrite\n" # Added example usage
+    )
+
+    # Positional argument for the runconfig file
+    parser.add_argument(
+        "runconfig",
+        type=str,
+        help="Path to the XML run-configuration file.",
+    )
+
+    # Optional arguments
+    parser.add_argument(
+        "--date",
+        type=str,
+        help="Target UTC date for product generation (YYYY-MM-DD). Overrides date in runconfig.",
+        metavar="YYYY-MM-DD"
+    )
+    parser.add_argument(
+        "--spinup-days",
+        type=int,
+        default=DEFAULT_SPINUP_DAYS,
+        help=f"Number of days for the VIIRS time-series spin-up. Defaults to {DEFAULT_SPINUP_DAYS} days.",
+        metavar="DAYS"
+    )
+    parser.add_argument(
+        "--target-resolution",
+        type=int,
+        default=DEFAULT_TARGET_RESOLUTION,
+        help=f"Desired output product resolution in meters. Defaults to {DEFAULT_TARGET_RESOLUTION}m.",
+        metavar="METERS"
+    )
+    parser.add_argument(
+        "--ndvi-resolution",
+        type=int,
+        default=DEFAULT_NDVI_RESOLUTION,
+        help=f"Resolution of coarse NDVI data in meters. Defaults to {DEFAULT_NDVI_RESOLUTION}m.",
+        metavar="METERS"
+    )
+    parser.add_argument(
+        "--albedo-resolution",
+        type=int,
+        default=DEFAULT_ALBEDO_RESOLUTION,
+        help=f"Resolution of coarse albedo data in meters. Defaults to {DEFAULT_ALBEDO_RESOLUTION}m.",
+        metavar="METERS"
+    )
+    parser.add_argument(
+        "--use-vnp43nrt",
+        action="store_true",
+        default=DEFAULT_USE_VNP43NRT,
+        help=f"Use VNP43NRT for VIIRS products. Defaults to {'True' if DEFAULT_USE_VNP43NRT else 'False'}.",
+    )
+    parser.add_argument(
+        "--no-vnp43nrt",
+        action="store_false",
+        dest="use_vnp43nrt", # This argument sets use_vnp43nrt to False
+        help="Do NOT use VNP43NRT for VIIRS products. Use VNP43IA4/VNP43MA3 instead.",
+    )
+    parser.add_argument(
+        "--calibrate-fine",
+        action="store_true",
+        default=DEFAULT_CALIBRATE_FINE,
+        help=f"Calibrate fine resolution HLS data to coarse resolution VIIRS data. Defaults to {'True' if DEFAULT_CALIBRATE_FINE else 'False'}.",
+    )
+    parser.add_argument(
+        "--sources-only",
+        action="store_true",
+        help="Only retrieve and stage source data (HLS, VIIRS); do not perform data fusion or generate final product.",
+    )
+    parser.add_argument(
+        "--no-remove-input-staging",
+        action="store_false",
+        dest="remove_input_staging",
+        default=True,
+        help="Do NOT remove the input staging directory after processing.",
+    )
+    parser.add_argument(
+        "--no-remove-prior",
+        action="store_false",
+        dest="remove_prior",
+        default=True,
+        help="Do NOT remove prior intermediate files after use.",
+    )
+    parser.add_argument(
+        "--no-remove-posterior",
+        action="store_false",
+        dest="remove_posterior",
+        default=True,
+        help="Do NOT remove posterior intermediate files after product generation.",
+    )
+    parser.add_argument(
+        "--threads",
+        type=str,
+        default="auto",
+        help='Number of Julia threads to use, or "auto". Defaults to "auto".',
+        metavar="COUNT"
+    )
+    parser.add_argument(
+        "--num-workers",
+        type=int,
+        default=4,
+        help=f"Number of Julia workers for distributed processing. Defaults to 4.",
+        metavar="COUNT"
+    )
+    parser.add_argument(
+        "--overwrite", # New argument for overwrite option
+        action="store_true",
+        help="Reproduce the output files even if they already exist.",
+    )
+    parser.add_argument(
+        "--version",
+        action="version",
+        version=f"%(prog)s {__version__}",
+        help="Show program's version number and exit.",
+    )
+
+    args = parser.parse_args()
+
+    # Call the main L2T_STARS processing function with parsed arguments
+    exit_code = L2T_STARS(
+        runconfig_filename=args.runconfig,
+        date_UTC=args.date,
+        spinup_days=args.spinup_days,
+        target_resolution=args.target_resolution,
+        NDVI_resolution=args.ndvi_resolution,
+        albedo_resolution=args.albedo_resolution,
+        use_VNP43NRT=args.use_vnp43nrt,
+        calibrate_fine=args.calibrate_fine,
+        sources_only=args.sources_only,
+        remove_input_staging=args.remove_input_staging,
+        remove_prior=args.remove_prior,
+        remove_posterior=args.remove_posterior,
+        threads=args.threads,
+        num_workers=args.num_workers,
+        overwrite=args.overwrite, # Pass the new overwrite argument
+    )
+
+    sys.exit(exit_code)
+
+
+if __name__ == "__main__":
+    main()

ECOv003_L2T_STARS/ECOv003_L2T_STARS.xml

@@ -0,0 +1,47 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<input xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xmlns:xalan="http://xml.apache.org/xslt"
+       xmlns:cas="http://oodt.jpl.nasa.gov/1.0/cas"
+       xsi:noNamespaceSchemaLocation="">
+    <group name="LogMetadata">
+        <vector name="CommandLineParameters">
+            <element>executable_filename</element>
+            <element>runconfig_filename</element>
+            <element>log_filename</element>
+        </vector>
+    </group>
+    <group name="JobIdentification">
+        <scalar name="ProductionLocation">ECOSTRESS Science Computing Facility</scalar>
+        <scalar name="TaskId">urn:ecostress:L2GL2TLSTETask</scalar>
+        <scalar name="ProcessingNode">processing_node</scalar>
+        <scalar name="ProductionDateTime">production_datetime</scalar>
+        <scalar name="JobId">job_ID</scalar>
+        <scalar name="WorkflowInstanceId">instance_ID</scalar>
+    </group>
+    <group name="InputFileGroup">
+        <scalar name="L2T_LSTE">L2T_LSTE_filename</scalar>
+        <scalar name="L2T_STARS_PRIOR">prior_L2T_STARS_filename</scalar>
+    </group>
+    <group name="StaticAncillaryFileGroup">
+        <scalar name="L2T_STARS_MODEL">model_directory</scalar>
+        <scalar name="L2T_STARS_INDICES">indices_directory</scalar>
+        <scalar name="L2T_STARS_SOURCES">sources_directory</scalar>
+        <scalar name="L2T_STARS_WORKING">working_directory</scalar>
+    </group>
+    <group name="PrimaryExecutable">
+        <scalar name="BuildID">build_ID</scalar>
+        <scalar name="FullPathname">executable_filename</scalar>
+    </group>
+    <group name="Geometry">
+        <scalar name="TileId">tile_ID</scalar>
+        <scalar name="SceneId">scene_ID</scalar>
+        <scalar name="OrbitNumber">orbit_number</scalar>
+    </group>
+    <group name="ProductPathGroup">
+        <scalar name="ProductCounter">product_counter</scalar>
+        <scalar name="ProductPath">output_directory</scalar>
+    </group>
+    <group name="PGENameGroup">
+        <scalar name="PGEName">L2T_STARS_PGE</scalar>
+    </group>
+</input>

ECOv003_L2T_STARS/L2TSTARSConfig.py

@@ -0,0 +1,190 @@
+from os.path import exists
+import logging
+
+from dateutil import parser
+
+import colored_logging as cl
+
+from ECOv003_exit_codes import *
+from ECOv003_granules import L2TLSTE
+
+from .constants import *
+from .runconfig import ECOSTRESSRunConfig
+
+logger = logging.getLogger(__name__)
+
+class L2TSTARSConfig(ECOSTRESSRunConfig):
+    """
+    Parses and validates the L2T_STARS specific parameters from an XML run-configuration file.
+
+    This class extends the base ECOSTRESSRunConfig to extract paths, IDs,
+    and processing parameters relevant to the L2T_STARS product generation.
+    It performs validation to ensure all critical parameters are present.
+    """
+
+    def __init__(self, filename: str):
+        """
+        Initializes the L2TSTARSConfig by parsing the provided run-config XML file.
+
+        Args:
+            filename (str): The path to the L2T_STARS run-configuration XML file.
+
+        Raises:
+            MissingRunConfigValue: If a required value is missing from the run-config.
+            UnableToParseRunConfig: If the run-config file cannot be parsed due to other errors.
+        """
+        logger.info(f"Loading L2T_STARS run-config: {cl.file(filename)}")
+        # Read the run-config XML into a dictionary
+        runconfig = self.read_runconfig(filename)
+
+        # reverting to StaticAncillaryFileGroup for now instead of StaticAuxiliaryFileGroup
+
+        try:
+            # Validate and extract working directory from StaticAncillaryFileGroup
+            if "StaticAncillaryFileGroup" not in runconfig:
+                raise MissingRunConfigValue(
+                    f"Missing StaticAncillaryFileGroup in L2T_STARS run-config: {filename}"
+                )
+            if "L2T_STARS_WORKING" not in runconfig["StaticAncillaryFileGroup"]:
+                raise MissingRunConfigValue(
+                    f"Missing StaticAncillaryFileGroup/L2T_STARS_WORKING in L2T_STARS run-config: {filename}"
+                )
+            self.working_directory = abspath(
+                runconfig["StaticAncillaryFileGroup"]["L2T_STARS_WORKING"]
+            )
+            logger.info(f"Working directory: {cl.dir(self.working_directory)}")
+
+            # Validate and extract sources directory
+            if "L2T_STARS_SOURCES" not in runconfig["StaticAncillaryFileGroup"]:
+                raise MissingRunConfigValue(
+                    f"Missing StaticAncillaryFileGroup/L2T_STARS_SOURCES in L2T_STARS run-config: {filename}"
+                )
+            self.sources_directory = abspath(
+                runconfig["StaticAncillaryFileGroup"]["L2T_STARS_SOURCES"]
+            )
+            logger.info(f"Sources directory: {cl.dir(self.sources_directory)}")
+
+            # Validate and extract indices directory
+            if "L2T_STARS_INDICES" not in runconfig["StaticAncillaryFileGroup"]:
+                raise MissingRunConfigValue(
+                    f"Missing StaticAncillaryFileGroup/L2T_STARS_INDICES in L2T_STARS run-config: {filename}"
+                )
+            self.indices_directory = abspath(
+                runconfig["StaticAncillaryFileGroup"]["L2T_STARS_INDICES"]
+            )
+            logger.info(f"Indices directory: {cl.dir(self.indices_directory)}")
+
+            # Validate and extract model directory
+            if "L2T_STARS_MODEL" not in runconfig["StaticAncillaryFileGroup"]:
+                raise MissingRunConfigValue(
+                    f"Missing StaticAncillaryFileGroup/L2T_STARS_MODEL in L2T_STARS run-config: {filename}"
+                )
+            self.model_directory = abspath(
+                runconfig["StaticAncillaryFileGroup"]["L2T_STARS_MODEL"]
+            )
+            logger.info(f"Model directory: {cl.dir(self.model_directory)}")
+
+            # Validate and extract output directory from ProductPathGroup
+            if "ProductPathGroup" not in runconfig:
+                raise MissingRunConfigValue(
+                    f"Missing ProductPathGroup in L2T_STARS run-config: {filename}"
+                )
+            if "ProductPath" not in runconfig["ProductPathGroup"]:
+                raise MissingRunConfigValue(
+                    f"Missing ProductPathGroup/ProductPath in L2T_STARS run-config: {filename}"
+                )
+            self.output_directory = abspath(
+                runconfig["ProductPathGroup"]["ProductPath"]
+            )
+            logger.info(f"Output directory: {cl.dir(self.output_directory)}")
+
+            # Validate and extract input L2T_LSTE filename
+            if "InputFileGroup" not in runconfig:
+                raise MissingRunConfigValue(
+                    f"Missing InputFileGroup in L2G_L2T_LSTE run-config: {filename}"
+                )
+            if "L2T_LSTE" not in runconfig["InputFileGroup"]:
+                raise MissingRunConfigValue(
+                    f"Missing InputFileGroup/L2T_LSTE in L2T_STARS run-config: {filename}"
+                )
+            self.L2T_LSTE_filename = abspath(runconfig["InputFileGroup"]["L2T_LSTE"])
+            logger.info(f"L2T_LSTE file: {cl.file(self.L2T_LSTE_filename)}")
+
+            # Extract optional prior L2T_STARS filename
+            self.L2T_STARS_prior_filename = None
+            if "L2T_STARS_PRIOR" in runconfig["InputFileGroup"]:
+                prior_filename = runconfig["InputFileGroup"]["L2T_STARS_PRIOR"]
+                if prior_filename != "" and exists(prior_filename):
+                    self.L2T_STARS_prior_filename = abspath(prior_filename)
+                logger.info(
+                    f"L2T_STARS prior file: {cl.file(self.L2T_STARS_prior_filename)}"
+                )
+
+            # Extract geometry parameters (orbit, scene, tile)
+            self.orbit = int(runconfig["Geometry"]["OrbitNumber"])
+            logger.info(f"Orbit: {cl.val(self.orbit)}")
+            if "SceneId" not in runconfig["Geometry"]:
+                raise MissingRunConfigValue(
+                    f"Missing Geometry/SceneId in L2T_STARS run-config: {filename}"
+                )
+            self.scene = int(runconfig["Geometry"]["SceneId"])
+            logger.info(f"Scene: {cl.val(self.scene)}")
+            if "TileId" not in runconfig["Geometry"]:
+                raise MissingRunConfigValue(
+                    f"Missing Geometry/TileId in L2T_STARS run-config: {filename}"
+                )
+            self.tile = str(runconfig["Geometry"]["TileId"])
+            logger.info(f"Tile: {cl.val(self.tile)}")
+
+            # Extract production details
+            if "ProductionDateTime" not in runconfig["JobIdentification"]:
+                raise MissingRunConfigValue(
+                    f"Missing JobIdentification/ProductionDateTime in L2T_STARS run-config {filename}"
+                )
+            self.production_datetime = parser.parse(
+                runconfig["JobIdentification"]["ProductionDateTime"]
+            )
+            logger.info(f"Production time: {cl.time(self.production_datetime)}")
+
+            # Extract build ID
+            if "BuildID" not in runconfig["PrimaryExecutable"]:
+                raise MissingRunConfigValue(
+                    f"Missing PrimaryExecutable/BuildID in L2T_STARS run-config {filename}"
+                )
+            self.build = str(runconfig["PrimaryExecutable"]["BuildID"])
+
+            # Extract product counter
+            if "ProductCounter" not in runconfig["ProductPathGroup"]:
+                raise MissingRunConfigValue(
+                    f"Missing ProductPathGroup/ProductCounter in L2T_STARS run-config {filename}"
+                )
+            self.product_counter = int(runconfig["ProductPathGroup"]["ProductCounter"])
+
+            # Get UTC time from the L2T_LSTE granule itself
+            l2t_lste_granule_obj = L2TLSTE(self.L2T_LSTE_filename)
+            time_UTC = l2t_lste_granule_obj.time_UTC
+
+            # Construct the full granule ID and paths for the output product
+            granule_ID = (
+                f"ECOv003_L2T_STARS_{self.tile}_{time_UTC:%Y%m%d}_{self.build}_"
+                f"{self.product_counter:02d}"
+            )
+            self.granule_ID = granule_ID
+            self.L2T_STARS_granule_directory = join(self.output_directory, granule_ID)
+            self.L2T_STARS_zip_filename = f"{self.L2T_STARS_granule_directory}.zip"
+            self.L2T_STARS_browse_filename = (
+                f"{self.L2T_STARS_granule_directory}.png"
+            )
+
+        except MissingRunConfigValue as e:
+            # Re-raise specific missing value errors
+            raise e
+        except ECOSTRESSExitCodeException as e:
+            # Re-raise custom ECOSTRESS exit code exceptions
+            raise e
+        except Exception as e:
+            # Catch any other parsing errors and raise a generic UnableToParseRunConfig
+            logger.exception(e)
+            raise UnableToParseRunConfig(
+                f"Unable to parse run-config file: {filename}"
+            )