ECOv003-L2T-STARS 1.2.0__tar.gz → 1.4.0__tar.gz

{ecov003_l2t_stars-1.2.0 → ecov003_l2t_stars-1.4.0}/.github/workflows/ci.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.10", "3.11"]
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
 
     steps:
       - name: Checkout repository
@@ -32,3 +32,8 @@ jobs:
       - name: Run tests
         run: |
           pytest
+        env:
+          PYTHONPATH: .
+          EARTHDATA_USERNAME: ${{ secrets.EARTHDATA_USERNAME }}
+          EARTHDATA_PASSWORD: ${{ secrets.EARTHDATA_PASSWORD }}
+          SKIP_EARTHDATA_LOGIN: "true"

{ecov003_l2t_stars-1.2.0 → ecov003_l2t_stars-1.4.0}/ECOv003_L2T_STARS/L2T_STARS.py

@@ -35,10 +35,6 @@ from .VNP43NRT import VNP43NRT
 from .runconfig import ECOSTRESSRunConfig
 from .L2TSTARSConfig import L2TSTARSConfig
 from .load_prior import load_prior
-from .generate_NDVI_coarse_directory import generate_NDVI_coarse_directory
-from .generate_NDVI_fine_directory import generate_NDVI_fine_directory
-from .generate_albedo_coarse_directory import generate_albedo_coarse_directory
-from .generate_albedo_fine_directory import generate_albedo_fine_directory
 from .generate_STARS_inputs import generate_STARS_inputs
 from .process_STARS_product import process_STARS_product
 from .retrieve_STARS_sources import retrieve_STARS_sources
@@ -58,6 +54,7 @@ def L2T_STARS(
     remove_input_staging: bool = True,
     remove_prior: bool = True,
     remove_posterior: bool = True,
+    initialize_julia: bool = False,
     threads: Union[int, str] = "auto",
     num_workers: int = 4,
     overwrite: bool = False, # New parameter for overwriting existing files
@@ -240,6 +237,8 @@ def L2T_STARS(
         logger.info(f"VNP09GA products directory: {cl.dir(VNP09GA_products_directory)}")
         VNP43NRT_products_directory = join(sources_directory, DEFAULT_VNP43NRT_PRODUCTS_DIRECTORY)
         logger.info(f"VNP43NRT products directory: {cl.dir(VNP43NRT_products_directory)}")
+        DOWNSAMPLED_products_directory = join(sources_directory, DEFAULT_STARS_DOWNSAMPLED_DIRECTORY)
+        logger.info(f"DOWNSAMPLED products directory: {cl.dir(DOWNSAMPLED_products_directory)}")
 
         # Re-check for existing product (double-check in case another process created it) with overwrite option
         if not overwrite and exists(L2T_STARS_zip_filename):
@@ -259,7 +258,7 @@ def L2T_STARS(
             HLS_connection = HLS2CMR(
                 working_directory=working_directory,
                 download_directory=HLS_download_directory,
-                products_directory=HLS_products_directory,
+                # products_directory=HLS_products_directory,
                 target_resolution=target_resolution,
             )
         except CMRServerUnreachable as e:
@@ -283,6 +282,7 @@ def L2T_STARS(
                     GEOS5FP_products=GEOS5FP_products_directory,
                     VNP09GA_directory=VNP09GA_products_directory,
                     VNP43NRT_directory=VNP43NRT_products_directory,
+                    initialize_julia=initialize_julia,
                 )
 
                 albedo_VIIRS_connection = VNP43NRT(
@@ -293,6 +293,7 @@ def L2T_STARS(
                     GEOS5FP_products=GEOS5FP_products_directory,
                     VNP09GA_directory=VNP09GA_products_directory,
                     VNP43NRT_directory=VNP43NRT_products_directory,
+                    initialize_julia=initialize_julia,
                 )
             except CMRServerUnreachable as e:
                 logger.exception(e)
@@ -404,19 +405,6 @@ def L2T_STARS(
             NDVI_coarse_geometry = HLS_connection.grid(tile=tile, cell_size=NDVI_resolution)
             albedo_coarse_geometry = HLS_connection.grid(tile=tile, cell_size=albedo_resolution)
 
-            NDVI_coarse_directory = generate_NDVI_coarse_directory(
-                input_staging_directory=input_staging_directory, tile=tile
-            )
-            NDVI_fine_directory = generate_NDVI_fine_directory(
-                input_staging_directory=input_staging_directory, tile=tile
-            )
-            albedo_coarse_directory = generate_albedo_coarse_directory(
-                input_staging_directory=input_staging_directory, tile=tile
-            )
-            albedo_fine_directory = generate_albedo_fine_directory(
-                input_staging_directory=input_staging_directory, tile=tile
-            )
-
             generate_STARS_inputs(
                 tile=tile,
                 date_UTC=date_UTC,
@@ -429,11 +417,7 @@ def L2T_STARS(
                 target_resolution=target_resolution,
                 NDVI_coarse_geometry=NDVI_coarse_geometry,
                 albedo_coarse_geometry=albedo_coarse_geometry,
-                working_directory=working_directory,
-                NDVI_coarse_directory=NDVI_coarse_directory,
-                NDVI_fine_directory=NDVI_fine_directory,
-                albedo_coarse_directory=albedo_coarse_directory,
-                albedo_fine_directory=albedo_fine_directory,
+                downsampled_directory=DOWNSAMPLED_products_directory,
                 HLS_connection=HLS_connection,
                 NDVI_VIIRS_connection=NDVI_VIIRS_connection,
                 albedo_VIIRS_connection=albedo_VIIRS_connection,
@@ -454,7 +438,7 @@ def L2T_STARS(
                 NDVI_resolution=NDVI_resolution,
                 albedo_resolution=albedo_resolution,
                 target_resolution=target_resolution,
-                working_directory=working_directory,
+                downsampled_directory=DOWNSAMPLED_products_directory,
                 model_directory=model_directory,
                 input_staging_directory=input_staging_directory,
                 L2T_STARS_granule_directory=L2T_STARS_granule_directory,
@@ -470,6 +454,7 @@ def L2T_STARS(
                 remove_input_staging=remove_input_staging,
                 remove_prior=remove_prior,
                 remove_posterior=remove_posterior,
+                initialize_julia=initialize_julia,
                 threads=threads,
                 num_workers=num_workers,
             )

{ecov003_l2t_stars-1.2.0 → ecov003_l2t_stars-1.4.0}/ECOv003_L2T_STARS/LPDAAC/LPDAACDataPool.py

@@ -1,6 +1,5 @@
-import base64
+import netrc
 import hashlib
-import json
 import logging
 import os
 import posixpath
@@ -11,24 +10,27 @@ from datetime import date
 from fnmatch import fnmatch
 from http.cookiejar import CookieJar
 from os import makedirs, remove
-from os.path import abspath
 from os.path import dirname
 from os.path import exists
 from os.path import getsize
 from os.path import isdir
 from os.path import join
+from os.path import abspath
+from os.path import expanduser
 from time import sleep
 from typing import List, OrderedDict
-import netrc
+
 import requests
 import xmltodict
 from bs4 import BeautifulSoup
 from dateutil import parser
-from pycksum import cksum
+from ..cksum import cksum
 
 import colored_logging as cl
 
-from ECOv003_exit_codes import *
+
+class DownloadFailed(Exception):
+    pass
 
 CONNECTION_CLOSE = {
     "Connection": "close",
@@ -57,7 +59,7 @@ class LPDAACDataPool:
     DATE_REGEX = re.compile(r'^(19|20)\d\d[- /.](0[1-9]|1[012])[- /.](0[1-9]|[12][0-9]|3[01])$')
     DEFAULT_REMOTE = DEFAULT_REMOTE
 
-    def __init__(self, username: str = None, password: str = None, remote: str = None, offline_ok: bool = False):
+    def __init__(self, username: str = None, password: str = None, remote: str = None, offline_ok: bool = True):
         if remote is None:
             remote = DEFAULT_REMOTE
 
@@ -66,15 +68,14 @@ class LPDAACDataPool:
                 netrc_file = netrc.netrc()
                 username, _, password = netrc_file.authenticators("urs.earthdata.nasa.gov")
             except Exception as e:
-                logger.exception(e)
                 logger.warning("netrc credentials not found for urs.earthdata.nasa.gov")
 
         if username is None or password is None:
             if not "LPDAAC_USERNAME" in os.environ or not "LPDAAC_PASSWORD" in os.environ:
-                raise RuntimeError("Missing environment variable 'LPDAAC_USERNAME' or 'LPDAAC_PASSWORD'")
-
-            username = os.environ["LPDAAC_USERNAME"]
-            password = os.environ["LPDAAC_PASSWORD"]
+                logger.warning("missing environment variable 'LPDAAC_USERNAME' or 'LPDAAC_PASSWORD'")
+            else:
+                username = os.environ["LPDAAC_USERNAME"]
+                password = os.environ["LPDAAC_PASSWORD"]
 
         self._remote = remote
         self._username = username
@@ -86,14 +87,15 @@ class LPDAACDataPool:
 
         self._listings = {}
 
-        try:
-            self._authenticate()
-            self._check_remote()
-        except Exception as e:
-            if self.offline_ok:
-                logger.warning("unable to connect to LP-DAAC data pool")
-            else:
-                raise e
+        if not self.offline_ok:
+            try:
+                self._authenticate()
+                self._check_remote()
+            except Exception as e:
+                if self.offline_ok:
+                    logger.warning("unable to connect to LP-DAAC data pool")
+                else:
+                    raise e
 
     def _authenticate(self):
         try:
@@ -128,7 +130,7 @@ class LPDAACDataPool:
                 raise ConnectionError(message)
 
     def _check_remote(self):
-        logger.info(f"checking URL: {cl.URL(self.remote)}")
+        logger.debug(f"checking URL: {cl.URL(self.remote)}")
 
         try:
             response = requests.head(self.remote, headers=CONNECTION_CLOSE)
@@ -145,7 +147,7 @@ class LPDAACDataPool:
                 raise LPDAACServerUnreachable(message)
 
         if status == 200:
-            logger.info(
+            logger.debug(
                 "remote verified with status " + cl.val(200) +
                 " in " + cl.time(f"{duration:0.2f}") +
                 " seconds: " + cl.URL(self.remote))
@@ -307,7 +309,7 @@ class LPDAACDataPool:
         else:
             metadata_filename = f"{download_location}.xml"
 
-        makedirs(dirname(metadata_filename), exist_ok=True)
+        makedirs(abspath(dirname(expanduser(metadata_filename))), exist_ok=True)
 
         if XML_retries is None:
             XML_retries = XML_RETRIES
@@ -325,8 +327,8 @@ class LPDAACDataPool:
 
         while XML_retries > 0:
             XML_retries -= 1
-            command = f"wget -nc -c --user {self._username} --password {self._password} -O {metadata_filename} {metadata_URL}"
-            logger.info(command)
+            command = f"wget -nc -c --user {self._username} --password {self._password} -O {abspath(expanduser(metadata_filename))} {metadata_URL}"
+            # logger.info(command)
             os.system(command)
 
             if not exists(metadata_filename):
@@ -350,12 +352,12 @@ class LPDAACDataPool:
                 continue
 
             try:
-                with open(metadata_filename, "r") as file:
+                with open(abspath(expanduser(metadata_filename)), "r") as file:
                     metadata = xmltodict.parse(file.read())
             except Exception as e:
                 logger.warning(e)
                 logger.warning(f"unable to parse metadata file: {metadata_filename}")
-                os.remove(metadata_filename)
+                os.remove(abspath(expanduser(metadata_filename)))
                 logger.warning(f"waiting {XML_timeout_seconds} for retry")
                 sleep(XML_timeout_seconds)
                 continue
@@ -372,7 +374,7 @@ class LPDAACDataPool:
 
         logger.info(
             f"metadata retrieved {checksum_type} checksum: {cl.val(remote_checksum)} size: {cl.val(remote_filesize)} URL: {cl.URL(metadata_URL)}")
-        makedirs(dirname(filename), exist_ok=True)
+        makedirs(abspath(dirname(expanduser(filename))), exist_ok=True)
         logger.info(f"downloading {cl.URL(URL)} -> {cl.file(filename)}")
 
         # Use a temporary file for downloading
@@ -382,8 +384,8 @@ class LPDAACDataPool:
             download_retries -=1
 
             try:
-                if exists(temporary_filename):
-                    temporary_filesize = self.get_local_filesize(temporary_filename)
+                if exists(abspath(expanduser(temporary_filename))):
+                    temporary_filesize = self.get_local_filesize(abspath(expanduser(temporary_filename)))
 
                     if temporary_filesize > remote_filesize:
                         logger.warning(
@@ -391,11 +393,11 @@ class LPDAACDataPool:
                         remove(temporary_filename)
 
                     elif temporary_filesize == remote_filesize:
-                        local_checksum = self.get_local_checksum(temporary_filename, checksum_type=checksum_type)
+                        local_checksum = self.get_local_checksum(abspath(expanduser(temporary_filename)), checksum_type=checksum_type)
 
                         if local_checksum == remote_checksum:
                             try:
-                                shutil.move(temporary_filename, filename)
+                                shutil.move(abspath(expanduser(temporary_filename)), abspath(expanduser(filename)))
                             except Exception as e:
                                 if exists(filename):
                                     logger.warning(f"unable to move temporary file: {temporary_filename}")
@@ -408,27 +410,27 @@ class LPDAACDataPool:
                         else:
                             logger.warning(
                                 f"removing corrupted file with local checksum {local_checksum} and remote checksum {remote_checksum}: {temporary_filename}")
-                            remove(temporary_filename)
+                            remove(abspath(expanduser(temporary_filename)))
                     else:
                         logger.info(f"resuming incomplete download: {cl.file(temporary_filename)}")
 
-                command = f"wget -nc -c --user {self._username} --password {self._password} -O {temporary_filename} {URL}"
-                logger.info(command)
+                command = f"wget -nc -c --user {self._username} --password {self._password} -O {abspath(expanduser(temporary_filename))} {URL}"
+                # logger.info(command)
                 os.system(command)
 
-                if not exists(temporary_filename):
+                if not exists(abspath(expanduser(temporary_filename))):
                     raise ConnectionError(f"unable to download URL: {URL}")
 
-                local_filesize = self.get_local_filesize(temporary_filename)
-                local_checksum = self.get_local_checksum(temporary_filename, checksum_type=checksum_type)
+                local_filesize = self.get_local_filesize(abspath(expanduser(temporary_filename)))
+                local_checksum = self.get_local_checksum(abspath(expanduser(temporary_filename)), checksum_type=checksum_type)
 
                 if local_filesize != remote_filesize or local_checksum != remote_checksum:
-                    os.remove(temporary_filename)
+                    os.remove(abspath(expanduser(temporary_filename)))
                     raise ConnectionError(
                         f"removing corrupted file with local filesize {local_filesize} remote filesize {remote_filesize} local checksum {local_checksum} remote checksum {remote_checksum}: {temporary_filename}")
 
                 # Download successful, rename the temporary file to its proper name
-                shutil.move(temporary_filename, filename)
+                shutil.move(abspath(expanduser(temporary_filename)), abspath(expanduser(filename)))
 
                 logger.info(
                     f"successful download with filesize {cl.val(local_filesize)} checksum {cl.val(local_checksum)}: {cl.file(filename)}")

{ecov003_l2t_stars-1.2.0 → ecov003_l2t_stars-1.4.0}/ECOv003_L2T_STARS/Project.toml

@@ -3,12 +3,8 @@ Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 DelimitedFiles = "8bb1440f-4735-579b-a4ab-409b98df4dab"
 Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b"
 Glob = "c27321d9-0574-5035-807b-f59d2c89b15c"
-HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
-JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"
-OpenSSL = "4d8831e6-92b7-49fb-bdf8-b643e874388c"
-Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 Rasters = "a3a2b9e3-a471-40c9-b274-f788e487c689"
 STARSDataFusion = "70ccc657-289f-4534-a407-e03a16fd1153"
 Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"

{ecov003_l2t_stars-1.2.0 → ecov003_l2t_stars-1.4.0}/ECOv003_L2T_STARS/VIIRS/VNP09GA.py

@@ -24,10 +24,11 @@ from modland import generate_modland_grid
 
 from ECOv003_exit_codes import *
 
+from ..login import login
 from ..daterange import get_date
 from ..LPDAAC.LPDAACDataPool import RETRIES
 from .VIIRSDataPool import VIIRSGranule
-from .VIIRS_CMR_LOGIN import CMRServerUnreachable, VIIRS_CMR_login
+from ..exceptions import *
 
 NDVI_COLORMAP = LinearSegmentedColormap.from_list(
     name="NDVI",
@@ -1095,7 +1096,7 @@ class VNP09GA:
 
         self.resampling = resampling
 
-        self._granules = pd.DataFrame({"date_UTC": {}, "tile": {}, "granule": {}})
+        self._granules = pd.DataFrame(columns=["date_UTC", "tile", "granule"])
 
         if working_directory is None:
             working_directory = self.DEFAULT_WORKING_DIRECTORY
@@ -1122,7 +1123,7 @@ class VNP09GA:
         self.products_directory = products_directory
         self.mosaic_directory = mosaic_directory
 
-        self.auth = VIIRS_CMR_login()
+        self.auth = login()
 
     def add_granules(self, granules: List[earthaccess.search.DataGranule]):
         data = pd.DataFrame([
@@ -1232,7 +1233,7 @@ class VNP09GA:
         if "date_UTC" not in self._granules.columns:
             raise ValueError(f"date_UTC column not in granules table")
 
-        subset = self._granules[(self._granules.date_UTC == date_UTC) & (self._granules.tile == tile)]
+        subset = self._granules[(self._granules['date_UTC'] == date_UTC) & (self._granules['tile'] == tile)]
         if len(subset) > 0:
             return subset.iloc[0].granule
 

{ecov003_l2t_stars-1.2.0 → ecov003_l2t_stars-1.4.0}/ECOv003_L2T_STARS/VNP43NRT/VNP43NRT.py

@@ -106,12 +106,14 @@ def process_julia_BRDF(
         sensor_zenith_directory: str,
         relative_azimuth_directory: str,
         SZA_filename: str,
-        output_directory: str):
+        output_directory: str,
+        initialize_julia: bool):
     parent_directory = abspath(join(dirname(__file__), ".."))
     julia_source_directory = join(parent_directory, "VNP43NRT_jl")
     julia_script_filename = join(abspath(dirname(__file__)), "process_VNP43NRT.jl")
 
-    instantiate_VNP43NRT_jl(julia_source_directory)
+    if initialize_julia:
+        instantiate_VNP43NRT_jl(julia_source_directory)
 
     command = f'julia "{julia_script_filename}" "{band}" "{h}" "{v}" "{tile_width_cells}" "{start_date:%Y-%m-%d}" "{end_date:%Y-%m-%d}" "{reflectance_directory}" "{solar_zenith_directory}" "{sensor_zenith_directory}" "{relative_azimuth_directory}" "{SZA_filename}" "{output_directory}"'
     logger.info(command)
@@ -265,7 +267,8 @@ class VNP43NRT(VIIRSDownloaderAlbedo, VIIRSDownloaderNDVI):
             VNP43NRT_staging_directory: str = None,
             GEOS5FP_connection: GEOS5FP = None,
             GEOS5FP_download: str = None,
-            GEOS5FP_products: str = None):
+            GEOS5FP_products: str = None,
+            initialize_julia: bool = False):
         if working_directory is None:
             working_directory = VNP09GA.DEFAULT_WORKING_DIRECTORY
 
@@ -302,6 +305,7 @@ class VNP43NRT(VIIRSDownloaderAlbedo, VIIRSDownloaderNDVI):
         self.VNP43NRT_directory = VNP43NRT_directory
         self.GEOS5FP = GEOS5FP_connection
         self.VNP43NRT_staging_directory = VNP43NRT_staging_directory
+        self.initialize_julia = initialize_julia
 
     def __repr__(self):
         display_dict = {
@@ -493,7 +497,8 @@ class VNP43NRT(VIIRSDownloaderAlbedo, VIIRSDownloaderNDVI):
                 sensor_zenith_directory=sensor_zenith_directory,
                 relative_azimuth_directory=relative_azimuth_directory,
                 SZA_filename=SZA_filename,
-                output_directory=output_directory
+                output_directory=output_directory,
+                initialize_julia=self.initialize_julia,
             )
 
             WSA = Raster.open(join(output_directory, f"{date_UTC:%Y-%m-%d}_WSA.tif"))

{ecov003_l2t_stars-1.2.0 → ecov003_l2t_stars-1.4.0}/ECOv003_L2T_STARS/__init__.py

@@ -1,3 +1,3 @@
 from .version import __version__, __author__
-from .ECOv003_L2T_STARS import *
+from .main import *
 from .ECOv003_DL import *

ecov003_l2t_stars-1.4.0/ECOv003_L2T_STARS/cksum.py

@@ -0,0 +1,66 @@
+"""
+Pure Python implementation of POSIX cksum algorithm.
+
+This module provides a replacement for the pycksum package that is compatible
+with Python 3.12 and later versions.
+"""
+
+
+def cksum(data_or_file):
+    """
+    Calculate POSIX cksum checksum for data or file-like object.
+    
+    Args:
+        data_or_file: Either bytes data or a file-like object opened in binary mode
+        
+    Returns:
+        int: The POSIX cksum checksum value
+    """
+    # Handle file-like objects
+    if hasattr(data_or_file, 'read'):
+        data = data_or_file.read()
+    else:
+        data = data_or_file
+    
+    # Ensure we have bytes
+    if isinstance(data, str):
+        data = data.encode('utf-8')
+    
+    # Initialize CRC with 0
+    crc = 0
+    
+    # Process each byte of data
+    for byte in data:
+        # XOR the byte with the current CRC (shifted left 8 bits) 
+        crc ^= byte << 24
+        
+        # Process 8 bits
+        for _ in range(8):
+            if crc & 0x80000000:  # If MSB is set
+                crc = (crc << 1) ^ 0x04c11db7  # CRC-32 polynomial
+            else:
+                crc = crc << 1
+            crc &= 0xffffffff  # Keep it 32-bit
+    
+    # Append the length in bytes as a big-endian value
+    length = len(data)
+    length_bytes = []
+    while length > 0:
+        length_bytes.insert(0, length & 0xff)
+        length >>= 8
+    
+    # Process the length bytes
+    for byte in length_bytes:
+        crc ^= byte << 24
+        for _ in range(8):
+            if crc & 0x80000000:
+                crc = (crc << 1) ^ 0x04c11db7
+            else:
+                crc = crc << 1
+            crc &= 0xffffffff
+    
+    # Final XOR and return as signed 32-bit integer equivalent
+    result = crc ^ 0xffffffff
+    
+    # Convert to match expected return type (unsigned 32-bit integer)
+    return result

{ecov003_l2t_stars-1.2.0 → ecov003_l2t_stars-1.4.0}/ECOv003_L2T_STARS/constants.py

@@ -22,6 +22,7 @@ DEFAULT_GEOS5FP_DOWNLOAD_DIRECTORY = "GEOS5FP_download"
 DEFAULT_GEOS5FP_PRODUCTS_DIRECTORY = "GEOS5FP_products"
 DEFAULT_VNP09GA_PRODUCTS_DIRECTORY = "VNP09GA_products"
 DEFAULT_VNP43NRT_PRODUCTS_DIRECTORY = "VNP43NRT_products"
+DEFAULT_STARS_DOWNSAMPLED_DIRECTORY = "DOWNSAMPLED_products"
 
 # Processing parameters
 VIIRS_GIVEUP_DAYS = 4  # Number of days to give up waiting for VIIRS data

ecov003_l2t_stars-1.4.0/ECOv003_L2T_STARS/exceptions.py

@@ -0,0 +1,2 @@
+class CMRServerUnreachable(Exception):
+    pass