ECOv003-L2T-STARS 1.0.1__py3-none-any.whl → 1.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. ECOv003_L2T_STARS/BRDF/BRDF.py +57 -0
  2. ECOv003_L2T_STARS/BRDF/SZA.py +65 -0
  3. ECOv003_L2T_STARS/BRDF/__init__.py +1 -0
  4. ECOv003_L2T_STARS/BRDF/statistical_radiative_transport.txt +90 -0
  5. ECOv003_L2T_STARS/BRDF/version.txt +1 -0
  6. ECOv003_L2T_STARS/ECOv003_DL.py +527 -0
  7. ECOv003_L2T_STARS/ECOv003_DL.xml +47 -0
  8. ECOv003_L2T_STARS/ECOv003_L2T_STARS.py +162 -0
  9. ECOv003_L2T_STARS/ECOv003_L2T_STARS.xml +47 -0
  10. ECOv003_L2T_STARS/L2TSTARSConfig.py +188 -0
  11. ECOv003_L2T_STARS/L2T_STARS.py +489 -0
  12. ECOv003_L2T_STARS/LPDAAC/LPDAACDataPool.py +444 -0
  13. ECOv003_L2T_STARS/LPDAAC/__init__.py +9 -0
  14. ECOv003_L2T_STARS/LPDAAC/version.txt +1 -0
  15. ECOv003_L2T_STARS/Manifest.toml +2332 -0
  16. ECOv003_L2T_STARS/Project.toml +14 -0
  17. ECOv003_L2T_STARS/VIIRS/VIIRSDataPool.py +294 -0
  18. ECOv003_L2T_STARS/VIIRS/VIIRSDownloader.py +26 -0
  19. ECOv003_L2T_STARS/VIIRS/VIIRS_CMR_LOGIN.py +36 -0
  20. ECOv003_L2T_STARS/VIIRS/VNP09GA.py +1277 -0
  21. ECOv003_L2T_STARS/VIIRS/VNP43IA4.py +288 -0
  22. ECOv003_L2T_STARS/VIIRS/VNP43MA3.py +323 -0
  23. ECOv003_L2T_STARS/VIIRS/__init__.py +9 -0
  24. ECOv003_L2T_STARS/VIIRS/version.txt +1 -0
  25. ECOv003_L2T_STARS/VNP43NRT/VNP43NRT.py +863 -0
  26. ECOv003_L2T_STARS/VNP43NRT/__init__.py +1 -0
  27. ECOv003_L2T_STARS/VNP43NRT/process_VNP43NRT.jl +169 -0
  28. ECOv003_L2T_STARS/VNP43NRT/version.txt +1 -0
  29. ECOv003_L2T_STARS/VNP43NRT_jl/Manifest.toml +995 -0
  30. ECOv003_L2T_STARS/VNP43NRT_jl/Project.toml +15 -0
  31. ECOv003_L2T_STARS/VNP43NRT_jl/__init__.py +0 -0
  32. ECOv003_L2T_STARS/VNP43NRT_jl/instantiate.jl +25 -0
  33. ECOv003_L2T_STARS/VNP43NRT_jl/instantiate.py +13 -0
  34. ECOv003_L2T_STARS/VNP43NRT_jl/src/VNP43NRT.jl +411 -0
  35. ECOv003_L2T_STARS/VNP43NRT_jl/src/__init__.py +0 -0
  36. ECOv003_L2T_STARS/__init__.py +3 -0
  37. ECOv003_L2T_STARS/calibrate_fine_to_coarse.py +60 -0
  38. ECOv003_L2T_STARS/constants.py +38 -0
  39. ECOv003_L2T_STARS/daterange/__init__.py +1 -0
  40. ECOv003_L2T_STARS/daterange/daterange.py +35 -0
  41. ECOv003_L2T_STARS/generate_L2T_STARS_runconfig.py +249 -0
  42. ECOv003_L2T_STARS/generate_NDVI_coarse_directory.py +21 -0
  43. ECOv003_L2T_STARS/generate_NDVI_coarse_image.py +30 -0
  44. ECOv003_L2T_STARS/generate_NDVI_fine_directory.py +14 -0
  45. ECOv003_L2T_STARS/generate_NDVI_fine_image.py +28 -0
  46. ECOv003_L2T_STARS/generate_STARS_inputs.py +231 -0
  47. ECOv003_L2T_STARS/generate_albedo_coarse_directory.py +18 -0
  48. ECOv003_L2T_STARS/generate_albedo_coarse_image.py +30 -0
  49. ECOv003_L2T_STARS/generate_albedo_fine_directory.py +17 -0
  50. ECOv003_L2T_STARS/generate_albedo_fine_image.py +30 -0
  51. ECOv003_L2T_STARS/generate_filename.py +37 -0
  52. ECOv003_L2T_STARS/generate_input_staging_directory.py +23 -0
  53. ECOv003_L2T_STARS/generate_model_state_tile_date_directory.py +28 -0
  54. ECOv003_L2T_STARS/generate_output_directory.py +28 -0
  55. ECOv003_L2T_STARS/install_STARS_jl.py +43 -0
  56. ECOv003_L2T_STARS/instantiate_STARS_jl.py +38 -0
  57. ECOv003_L2T_STARS/load_prior.py +248 -0
  58. ECOv003_L2T_STARS/prior.py +56 -0
  59. ECOv003_L2T_STARS/process_ECOSTRESS_data_fusion_distributed_bias.jl +420 -0
  60. ECOv003_L2T_STARS/process_STARS_product.py +507 -0
  61. ECOv003_L2T_STARS/process_julia_data_fusion.py +110 -0
  62. ECOv003_L2T_STARS/retrieve_STARS_sources.py +101 -0
  63. ECOv003_L2T_STARS/runconfig.py +70 -0
  64. ECOv003_L2T_STARS/timer/__init__.py +1 -0
  65. ECOv003_L2T_STARS/timer/timer.py +77 -0
  66. ECOv003_L2T_STARS/version.py +8 -0
  67. ECOv003_L2T_STARS/version.txt +1 -0
  68. {ECOv003_L2T_STARS-1.0.1.dist-info → ecov003_l2t_stars-1.1.0.dist-info}/METADATA +30 -23
  69. ecov003_l2t_stars-1.1.0.dist-info/RECORD +73 -0
  70. {ECOv003_L2T_STARS-1.0.1.dist-info → ecov003_l2t_stars-1.1.0.dist-info}/WHEEL +1 -1
  71. ecov003_l2t_stars-1.1.0.dist-info/entry_points.txt +3 -0
  72. ecov003_l2t_stars-1.1.0.dist-info/top_level.txt +1 -0
  73. ECOv003_L2T_STARS-1.0.1.dist-info/RECORD +0 -5
  74. ECOv003_L2T_STARS-1.0.1.dist-info/top_level.txt +0 -1
  75. {ECOv003_L2T_STARS-1.0.1.dist-info → ecov003_l2t_stars-1.1.0.dist-info/licenses}/LICENSE +0 -0
@@ -0,0 +1,249 @@
1
+ from typing import Union
2
+ from datetime import datetime, timezone
3
+ from os import makedirs
4
+ from os.path import join, abspath, exists, expanduser, dirname
5
+ from glob import glob
6
+ import logging
7
+ from shutil import which
8
+ from uuid import uuid4
9
+ import socket
10
+
11
+ import colored_logging as cl
12
+ from pytictoc import TicToc # Import pytictoc
13
+
14
+ from ECOv003_granules import L2TLSTE
15
+
16
+ from .constants import *
17
+
18
+ logger = logging.getLogger(__name__)
19
+
20
+ def generate_L2T_STARS_runconfig(
21
+ L2T_LSTE_filename: str,
22
+ prior_L2T_STARS_filename: str = "",
23
+ orbit: int = None,
24
+ scene: int = None,
25
+ tile: str = None,
26
+ time_UTC: Union[datetime, str] = None,
27
+ working_directory: str = None,
28
+ sources_directory: str = None,
29
+ indices_directory: str = None,
30
+ model_directory: str = None,
31
+ executable_filename: str = None,
32
+ output_directory: str = None,
33
+ runconfig_filename: str = None,
34
+ log_filename: str = None,
35
+ build: str = None,
36
+ processing_node: str = None,
37
+ production_datetime: datetime = None,
38
+ job_ID: str = None,
39
+ instance_ID: str = None,
40
+ product_counter: int = None,
41
+ template_filename: str = None,
42
+ ) -> str:
43
+ """
44
+ Generates an XML run-configuration file for the L2T_STARS processing.
45
+
46
+ This function dynamically creates an XML run-config file by populating a template
47
+ with provided or default parameters. It also checks for and returns existing
48
+ run-configs to prevent redundant generation.
49
+
50
+ Args:
51
+ L2T_LSTE_filename (str): Path to the input ECOSTRESS L2T LSTE granule file.
52
+ prior_L2T_STARS_filename (str, optional): Path to a prior L2T_STARS product file.
53
+ Defaults to "".
54
+ orbit (int, optional): Orbit number. If None, derived from L2T_LSTE_filename.
55
+ scene (int, optional): Scene ID. If None, derived from L2T_LSTE_filename.
56
+ tile (str, optional): HLS tile ID (e.g., '11SPS'). If None, derived from L2T_LSTE_filename.
57
+ time_UTC (Union[datetime, str], optional): UTC time of the L2T_LSTE granule. If None,
58
+ derived from L2T_LSTE_filename.
59
+ working_directory (str, optional): Root directory for all processing outputs.
60
+ Defaults to ".".
61
+ sources_directory (str, optional): Directory for downloaded source data (HLS, VIIRS).
62
+ indices_directory (str, optional): Directory for intermediate index products.
63
+ model_directory (str, optional): Directory for model state files (priors, posteriors).
64
+ executable_filename (str, optional): Path to the L2T_STARS executable. If None,
65
+ 'L2T_STARS' is assumed to be in the system's PATH.
66
+ output_directory (str, optional): Directory for final L2T_STARS products.
67
+ runconfig_filename (str, optional): Specific filename for the generated run-config.
68
+ If None, a default name based on granule ID is used.
69
+ log_filename (str, optional): Specific filename for the log file. If None,
70
+ a default name based on granule ID is used.
71
+ build (str, optional): Build ID of the PGE. Defaults to DEFAULT_BUILD.
72
+ processing_node (str, optional): Name of the processing node. Defaults to system hostname.
73
+ production_datetime (datetime, optional): Production date and time. Defaults to now (UTC).
74
+ job_ID (str, optional): Job identifier. Defaults to a timestamp.
75
+ instance_ID (str, optional): Unique instance identifier. Defaults to a UUID.
76
+ product_counter (int, optional): Counter for product generation. Defaults to 1.
77
+ template_filename (str, optional): Path to the XML run-config template file.
78
+ Defaults to L2T_STARS_TEMPLATE.
79
+
80
+ Returns:
81
+ str: The absolute path to the generated or existing run-configuration file.
82
+ """
83
+ timer = TicToc() # Initialize pytictoc timer
84
+ timer.tic() # Start the timer
85
+
86
+ # Load the L2T_LSTE granule to extract necessary metadata if not provided
87
+ l2t_lste_granule = L2TLSTE(L2T_LSTE_filename)
88
+
89
+ # Use values from L2T_LSTE granule if not explicitly provided
90
+ if orbit is None:
91
+ orbit = l2t_lste_granule.orbit
92
+ if scene is None:
93
+ scene = l2t_lste_granule.scene
94
+ if tile is None:
95
+ tile = l2t_lste_granule.tile
96
+ if time_UTC is None:
97
+ time_UTC = l2t_lste_granule.time_UTC
98
+
99
+ # Set default values for other parameters if not provided
100
+ if build is None:
101
+ build = DEFAULT_BUILD
102
+ if working_directory is None:
103
+ working_directory = "."
104
+
105
+ date_UTC = time_UTC.date()
106
+
107
+ logger.info(
108
+ f"Started generating L2T_STARS run-config for tile {cl.val(tile)} on date {cl.time(date_UTC)}"
109
+ )
110
+
111
+ # Check for previous run-configs to avoid re-generating
112
+ pattern = join(
113
+ working_directory, "runconfig", f"ECOv003_L2T_STARS_{tile}_*_{build}_*.xml"
114
+ )
115
+ logger.info(f"Scanning for previous run-configs: {cl.val(pattern)}")
116
+ previous_runconfigs = glob(pattern)
117
+ previous_runconfig_count = len(previous_runconfigs)
118
+
119
+ if previous_runconfig_count > 0:
120
+ logger.info(f"Found {cl.val(previous_runconfig_count)} previous run-configs")
121
+ # Return the most recent run-config if found
122
+ previous_runconfig = sorted(previous_runconfigs)[-1]
123
+ logger.info(f"Previous run-config: {cl.file(previous_runconfig)}")
124
+ return previous_runconfig
125
+
126
+ # Resolve the path to the run-config template
127
+ if template_filename is None:
128
+ template_filename = L2T_STARS_TEMPLATE
129
+ template_filename = abspath(expanduser(template_filename))
130
+
131
+ # Set production datetime if not provided
132
+ if production_datetime is None:
133
+ production_datetime = datetime.now(timezone.utc)
134
+
135
+ # Set product counter if not provided
136
+ if product_counter is None:
137
+ product_counter = 1
138
+
139
+ # Format timestamp and generate granule ID
140
+ timestamp = f"{time_UTC:%Y%m%d}"
141
+ granule_ID = (
142
+ f"ECOv003_L2T_STARS_{tile}_{timestamp}_{build}_{product_counter:02d}"
143
+ )
144
+
145
+ # Define run-config filename and resolve absolute path
146
+ if runconfig_filename is None:
147
+ runconfig_filename = join(working_directory, "runconfig", f"{granule_ID}.xml")
148
+ runconfig_filename = abspath(expanduser(runconfig_filename))
149
+
150
+ # If the run-config file already exists, log and return its path
151
+ if exists(runconfig_filename):
152
+ logger.info(f"Run-config already exists {cl.file(runconfig_filename)}")
153
+ return runconfig_filename
154
+
155
+ # Resolve absolute paths for various directories if not already defined
156
+ working_directory = abspath(expanduser(working_directory))
157
+ if sources_directory is None:
158
+ sources_directory = join(working_directory, DEFAULT_STARS_SOURCES_DIRECTORY)
159
+ if indices_directory is None:
160
+ indices_directory = join(working_directory, DEFAULT_STARS_INDICES_DIRECTORY)
161
+ if model_directory is None:
162
+ model_directory = join(working_directory, DEFAULT_STARS_MODEL_DIRECTORY)
163
+
164
+ # Determine executable path; fall back to just the name if not found in PATH
165
+ if executable_filename is None:
166
+ executable_filename = which("L2T_STARS")
167
+ if executable_filename is None:
168
+ executable_filename = "L2T_STARS"
169
+
170
+ # Define output and log file paths
171
+ if output_directory is None:
172
+ output_directory = join(working_directory, DEFAULT_OUTPUT_DIRECTORY)
173
+ output_directory = abspath(expanduser(output_directory))
174
+ if log_filename is None:
175
+ log_filename = join(working_directory, "log", f"{granule_ID}.log")
176
+ log_filename = abspath(expanduser(log_filename))
177
+
178
+ # Get processing node hostname
179
+ if processing_node is None:
180
+ processing_node = socket.gethostname()
181
+
182
+ # Set Job ID and Instance ID
183
+ if job_ID is None:
184
+ job_ID = timestamp
185
+ if instance_ID is None:
186
+ instance_ID = str(uuid4()) # Generate a unique UUID for the instance
187
+
188
+ # Resolve absolute path for the input L2T_LSTE file
189
+ L2T_LSTE_filename = abspath(expanduser(L2T_LSTE_filename))
190
+
191
+ logger.info(f"Loading L2T_STARS template: {cl.file(template_filename)}")
192
+
193
+ # Read the XML template file content
194
+ with open(template_filename, "r") as file:
195
+ template = file.read()
196
+
197
+ # Replace placeholders in the template with actual values
198
+ logger.info(f"Orbit: {cl.val(orbit)}")
199
+ template = template.replace("orbit_number", f"{orbit:05d}")
200
+ logger.info(f"Scene: {cl.val(scene)}")
201
+ template = template.replace("scene_ID", f"{scene:03d}")
202
+ logger.info(f"Tile: {cl.val(tile)}")
203
+ template = template.replace("tile_ID", f"{tile}")
204
+ logger.info(f"L2T_LSTE file: {cl.file(L2T_LSTE_filename)}")
205
+ template = template.replace("L2T_LSTE_filename", L2T_LSTE_filename)
206
+ logger.info(f"Prior L2T_STARS file: {cl.file(prior_L2T_STARS_filename)}")
207
+ template = template.replace("prior_L2T_STARS_filename", prior_L2T_STARS_filename)
208
+ logger.info(f"Working directory: {cl.dir(working_directory)}")
209
+ template = template.replace("working_directory", working_directory)
210
+ logger.info(f"Sources directory: {cl.dir(sources_directory)}")
211
+ template = template.replace("sources_directory", sources_directory)
212
+ logger.info(f"Indices directory: {cl.dir(indices_directory)}")
213
+ template = template.replace("indices_directory", indices_directory)
214
+ logger.info(f"Model directory: {cl.dir(model_directory)}")
215
+ template = template.replace("model_directory", model_directory)
216
+ logger.info(f"Executable: {cl.file(executable_filename)}")
217
+ template = template.replace("executable_filename", executable_filename)
218
+ logger.info(f"Output directory: {cl.dir(output_directory)}")
219
+ template = template.replace("output_directory", output_directory)
220
+ logger.info(f"Run-config: {cl.file(runconfig_filename)}")
221
+ template = template.replace("runconfig_filename", runconfig_filename)
222
+ logger.info(f"Log: {cl.file(log_filename)}")
223
+ template = template.replace("log_filename", log_filename)
224
+ logger.info(f"Build: {cl.val(build)}")
225
+ template = template.replace("build_ID", build)
226
+ logger.info(f"Processing node: {cl.val(processing_node)}")
227
+ template = template.replace("processing_node", processing_node)
228
+ logger.info(f"Production date/time: {cl.time(production_datetime)}")
229
+ template = template.replace("production_datetime", f"{production_datetime:%Y-%m-%dT%H:%M:%SZ}")
230
+ logger.info(f"Job ID: {cl.val(job_ID)}")
231
+ template = template.replace("job_ID", job_ID)
232
+ logger.info(f"Instance ID: {cl.val(instance_ID)}")
233
+ template = template.replace("instance_ID", instance_ID)
234
+ logger.info(f"Product counter: {cl.val(product_counter)}")
235
+ template = template.replace("product_counter", f"{product_counter:02d}")
236
+
237
+ # Create the directory for the run-config file if it doesn't exist
238
+ makedirs(dirname(abspath(runconfig_filename)), exist_ok=True)
239
+ logger.info(f"Writing run-config file: {cl.file(runconfig_filename)}")
240
+
241
+ # Write the populated template to the run-config file
242
+ with open(runconfig_filename, "w") as file:
243
+ file.write(template)
244
+
245
+ logger.info(
246
+ f"Finished generating L2T_STARS run-config for orbit {cl.val(orbit)} scene {cl.val(scene)} ({timer.tocvalue():.2f} seconds)"
247
+ )
248
+
249
+ return
@@ -0,0 +1,21 @@
1
+ from .generate_input_staging_directory import generate_input_staging_directory
2
+
3
+ def generate_NDVI_coarse_directory(
4
+ input_staging_directory: str,
5
+ tile: str) -> str:
6
+ """
7
+ Generates the specific staging directory for coarse NDVI images.
8
+
9
+ Args:
10
+ input_staging_directory (str): The base input staging directory.
11
+ tile (str): The HLS tile ID.
12
+
13
+ Returns:
14
+ str: The full path to the coarse NDVI staging directory.
15
+ """
16
+ return generate_input_staging_directory(
17
+ input_staging_directory,
18
+ tile,
19
+ "NDVI_coarse"
20
+ )
21
+
@@ -0,0 +1,30 @@
1
+ from typing import Union
2
+ from datetime import date
3
+ import numpy as np
4
+
5
+ import rasters as rt
6
+ from rasters import Raster, RasterGeometry
7
+
8
+ from .VIIRS import VIIRSDownloaderAlbedo, VIIRSDownloaderNDVI
9
+
10
+ def generate_NDVI_coarse_image(
11
+ date_UTC: Union[date, str],
12
+ VIIRS_connection: VIIRSDownloaderNDVI,
13
+ geometry: RasterGeometry = None) -> Raster:
14
+ """
15
+ Generates a coarse-resolution NDVI image from VIIRS data.
16
+
17
+ Args:
18
+ date_UTC (Union[date, str]): The UTC date for which to retrieve NDVI data.
19
+ VIIRS_connection (VIIRSDownloaderNDVI): An initialized VIIRS NDVI downloader object.
20
+ geometry (RasterGeometry, optional): The target geometry for the VIIRS image.
21
+ If None, the native VIIRS geometry is used.
22
+
23
+ Returns:
24
+ Raster: A Raster object representing the coarse-resolution NDVI image.
25
+ Zero values are converted to NaN.
26
+ """
27
+ coarse_image = VIIRS_connection.NDVI(date_UTC=date_UTC, geometry=geometry)
28
+ # Convert zero values (often used as NoData in some datasets) to NaN for proper handling
29
+ coarse_image = rt.where(coarse_image == 0, np.nan, coarse_image)
30
+ return coarse_image
@@ -0,0 +1,14 @@
1
+ from .generate_input_staging_directory import generate_input_staging_directory
2
+
3
+ def generate_NDVI_fine_directory(input_staging_directory: str, tile: str) -> str:
4
+ """
5
+ Generates the specific staging directory for fine NDVI images.
6
+
7
+ Args:
8
+ input_staging_directory (str): The base input staging directory.
9
+ tile (str): The HLS tile ID.
10
+
11
+ Returns:
12
+ str: The full path to the fine NDVI staging directory.
13
+ """
14
+ return generate_input_staging_directory(input_staging_directory, tile, "NDVI_fine")
@@ -0,0 +1,28 @@
1
+ from typing import Union
2
+ from datetime import date
3
+ import numpy as np
4
+ import rasters as rt
5
+ from rasters import Raster
6
+
7
+ from harmonized_landsat_sentinel import HLS
8
+
9
+ def generate_NDVI_fine_image(
10
+ date_UTC: Union[date, str],
11
+ tile: str,
12
+ HLS_connection: HLS) -> Raster:
13
+ """
14
+ Generates a fine-resolution NDVI image from HLS data.
15
+
16
+ Args:
17
+ date_UTC (Union[date, str]): The UTC date for which to retrieve NDVI data.
18
+ tile (str): The HLS tile ID.
19
+ HLS_connection (HLS): An initialized HLS data connection object.
20
+
21
+ Returns:
22
+ Raster: A Raster object representing the fine-resolution NDVI image.
23
+ Zero values are converted to NaN.
24
+ """
25
+ fine_image = HLS_connection.NDVI(tile=tile, date_UTC=date_UTC)
26
+ # Convert zero values to NaN for consistency
27
+ fine_image = rt.where(fine_image == 0, np.nan, fine_image)
28
+ return fine_image
@@ -0,0 +1,231 @@
1
+ from typing import Union
2
+ from datetime import date, datetime
3
+ from dateutil.rrule import rrule, DAILY
4
+ import logging
5
+
6
+ import colored_logging as cl
7
+ from rasters import Raster, RasterGeometry
8
+ from harmonized_landsat_sentinel import HLS2CMR
9
+
10
+ from ECOv003_exit_codes import AuxiliaryLatency
11
+
12
+ from .constants import VIIRS_GIVEUP_DAYS
13
+ from .generate_filename import generate_filename
14
+ from .daterange import get_date
15
+ from .generate_NDVI_coarse_image import generate_NDVI_coarse_image
16
+ from .generate_NDVI_fine_image import generate_NDVI_fine_image
17
+ from .generate_albedo_coarse_image import generate_albedo_coarse_image
18
+ from .generate_albedo_fine_image import generate_albedo_fine_image
19
+ from .calibrate_fine_to_coarse import calibrate_fine_to_coarse
20
+ from .VIIRS.VIIRSDownloader import VIIRSDownloaderAlbedo, VIIRSDownloaderNDVI
21
+
22
+ logger = logging.getLogger(__name__)
23
+
24
+ def generate_STARS_inputs(
25
+ tile: str,
26
+ date_UTC: date,
27
+ HLS_start_date: date,
28
+ HLS_end_date: date,
29
+ VIIRS_start_date: date,
30
+ VIIRS_end_date: date,
31
+ NDVI_resolution: int,
32
+ albedo_resolution: int,
33
+ target_resolution: int,
34
+ NDVI_coarse_geometry: RasterGeometry,
35
+ albedo_coarse_geometry: RasterGeometry,
36
+ working_directory: str,
37
+ NDVI_coarse_directory: str,
38
+ NDVI_fine_directory: str,
39
+ albedo_coarse_directory: str,
40
+ albedo_fine_directory: str,
41
+ HLS_connection: HLS2CMR,
42
+ NDVI_VIIRS_connection: VIIRSDownloaderNDVI,
43
+ albedo_VIIRS_connection: VIIRSDownloaderAlbedo,
44
+ calibrate_fine: bool = False,
45
+ ):
46
+ """
47
+ Generates and stages the necessary coarse and fine resolution input images
48
+ for the STARS data fusion process.
49
+
50
+ This function iterates through the VIIRS date range, retrieving and saving
51
+ coarse NDVI and albedo images. For dates within the HLS range, it also
52
+ retrieves and saves fine NDVI and albedo images. It can optionally
53
+ calibrate the fine images to the coarse images.
54
+
55
+ Args:
56
+ tile (str): The HLS tile ID.
57
+ date_UTC (date): The target UTC date for the L2T_STARS product.
58
+ HLS_start_date (date): The start date for HLS data retrieval for the fusion period.
59
+ HLS_end_date (date): The end date for HLS data retrieval for the fusion period.
60
+ VIIRS_start_date (date): The start date for VIIRS data retrieval for the fusion period.
61
+ VIIRS_end_date (date): The end date for VIIRS data retrieval for the fusion period.
62
+ NDVI_resolution (int): The resolution of the coarse NDVI data.
63
+ albedo_resolution (int): The resolution of the coarse albedo data.
64
+ target_resolution (int): The desired output resolution of the fused product.
65
+ NDVI_coarse_geometry (RasterGeometry): The target geometry for coarse NDVI images.
66
+ albedo_coarse_geometry (RasterGeometry): The target geometry for coarse albedo images.
67
+ working_directory (str): The main working directory.
68
+ NDVI_coarse_directory (str): Directory for staging coarse NDVI images.
69
+ NDVI_fine_directory (str): Directory for staging fine NDVI images.
70
+ albedo_coarse_directory (str): Directory for staging coarse albedo images.
71
+ albedo_fine_directory (str): Directory for staging fine albedo images.
72
+ HLS_connection (HLS2CMR): An initialized HLS data connection object.
73
+ NDVI_VIIRS_connection (VIIRSDownloaderNDVI): An initialized VIIRS NDVI downloader.
74
+ albedo_VIIRS_connection (VIIRSDownloaderAlbedo): An initialized VIIRS albedo downloader.
75
+ calibrate_fine (bool, optional): If True, calibrate fine images to coarse images.
76
+ Defaults to False.
77
+
78
+ Raises:
79
+ AuxiliaryLatency: If coarse VIIRS data is missing within the VIIRS_GIVEUP_DAYS window.
80
+ """
81
+ missing_coarse_dates = set() # Track dates where coarse data could not be generated
82
+
83
+ # Process each day within the VIIRS data fusion window
84
+ for processing_date in [
85
+ get_date(dt) for dt in rrule(DAILY, dtstart=VIIRS_start_date, until=VIIRS_end_date)
86
+ ]:
87
+ logger.info(
88
+ f"Preparing coarse image for STARS NDVI at {cl.place(tile)} on {cl.time(processing_date)}"
89
+ )
90
+
91
+ try:
92
+ # Generate coarse NDVI image
93
+ NDVI_coarse_image = generate_NDVI_coarse_image(
94
+ date_UTC=processing_date,
95
+ VIIRS_connection=NDVI_VIIRS_connection,
96
+ geometry=NDVI_coarse_geometry,
97
+ )
98
+
99
+ # Define filename for coarse NDVI and save
100
+ NDVI_coarse_filename = generate_filename(
101
+ directory=NDVI_coarse_directory,
102
+ variable="NDVI",
103
+ date_UTC=processing_date,
104
+ tile=tile,
105
+ cell_size=NDVI_resolution,
106
+ )
107
+ logger.info(
108
+ f"Saving coarse image for STARS NDVI at {cl.place(tile)} on {cl.time(processing_date)}: {NDVI_coarse_filename}"
109
+ )
110
+ NDVI_coarse_image.to_geotiff(NDVI_coarse_filename)
111
+
112
+ # If the processing date is within the HLS range, generate fine NDVI
113
+ if processing_date >= HLS_start_date:
114
+ logger.info(
115
+ f"Preparing fine image for STARS NDVI at {cl.place(tile)} on {cl.time(processing_date)}"
116
+ )
117
+ try:
118
+ NDVI_fine_image = generate_NDVI_fine_image(
119
+ date_UTC=processing_date,
120
+ tile=tile,
121
+ HLS_connection=HLS_connection,
122
+ )
123
+
124
+ # Optionally calibrate the fine NDVI image to the coarse NDVI image
125
+ if calibrate_fine:
126
+ logger.info(
127
+ f"Calibrating fine image for STARS NDVI at {cl.place(tile)} on {cl.time(processing_date)}"
128
+ )
129
+ NDVI_fine_image = calibrate_fine_to_coarse(
130
+ NDVI_fine_image, NDVI_coarse_image
131
+ )
132
+
133
+ # Define filename for fine NDVI and save
134
+ NDVI_fine_filename = generate_filename(
135
+ directory=NDVI_fine_directory,
136
+ variable="NDVI",
137
+ date_UTC=processing_date,
138
+ tile=tile,
139
+ cell_size=target_resolution,
140
+ )
141
+ logger.info(
142
+ f"Saving fine image for STARS NDVI at {cl.place(tile)} on {cl.time(processing_date)}: {NDVI_fine_filename}"
143
+ )
144
+ NDVI_fine_image.to_geotiff(NDVI_fine_filename)
145
+ except Exception: # Catch any exception during HLS fine image generation
146
+ logger.info(f"HLS NDVI is not available on {processing_date}")
147
+ except Exception as e:
148
+ logger.exception(e)
149
+ logger.warning(
150
+ f"Unable to produce coarse NDVI for date {processing_date}"
151
+ )
152
+ missing_coarse_dates.add(processing_date) # Add date to missing set
153
+
154
+ logger.info(
155
+ f"Preparing coarse image for STARS albedo at {cl.place(tile)} on {cl.time(processing_date)}"
156
+ )
157
+ try:
158
+ # Generate coarse albedo image
159
+ albedo_coarse_image = generate_albedo_coarse_image(
160
+ date_UTC=processing_date,
161
+ VIIRS_connection=albedo_VIIRS_connection,
162
+ geometry=albedo_coarse_geometry,
163
+ )
164
+
165
+ # Define filename for coarse albedo and save
166
+ albedo_coarse_filename = generate_filename(
167
+ directory=albedo_coarse_directory,
168
+ variable="albedo",
169
+ date_UTC=processing_date,
170
+ tile=tile,
171
+ cell_size=albedo_resolution,
172
+ )
173
+ logger.info(
174
+ f"Saving coarse image for STARS albedo at {cl.place(tile)} on {cl.time(processing_date)}: {albedo_coarse_filename}"
175
+ )
176
+ albedo_coarse_image.to_geotiff(albedo_coarse_filename)
177
+
178
+ # If the processing date is within the HLS range, generate fine albedo
179
+ if processing_date >= HLS_start_date:
180
+ logger.info(
181
+ f"Preparing fine image for STARS albedo at {cl.place(tile)} on {cl.time(processing_date)}"
182
+ )
183
+ try:
184
+ albedo_fine_image = generate_albedo_fine_image(
185
+ date_UTC=processing_date,
186
+ tile=tile,
187
+ HLS_connection=HLS_connection,
188
+ )
189
+
190
+ # Optionally calibrate the fine albedo image to the coarse albedo image
191
+ if calibrate_fine:
192
+ logger.info(
193
+ f"Calibrating fine image for STARS albedo at {cl.place(tile)} on {cl.time(processing_date)}"
194
+ )
195
+ albedo_fine_image = calibrate_fine_to_coarse(
196
+ albedo_fine_image, albedo_coarse_image
197
+ )
198
+
199
+ # Define filename for fine albedo and save
200
+ albedo_fine_filename = generate_filename(
201
+ directory=albedo_fine_directory,
202
+ variable="albedo",
203
+ date_UTC=processing_date,
204
+ tile=tile,
205
+ cell_size=target_resolution,
206
+ )
207
+ logger.info(
208
+ f"Saving fine image for STARS albedo at {cl.place(tile)} on {cl.time(processing_date)}: {albedo_fine_filename}"
209
+ )
210
+ albedo_fine_image.to_geotiff(albedo_fine_filename)
211
+ except Exception: # Catch any exception during HLS fine image generation
212
+ logger.info(f"HLS albedo is not available on {processing_date}")
213
+ except Exception as e:
214
+ logger.exception(e)
215
+ logger.warning(
216
+ f"Unable to produce coarse albedo for date {processing_date}"
217
+ )
218
+ missing_coarse_dates.add(processing_date) # Add date to missing set
219
+
220
+ # Check for missing coarse dates within the give-up window
221
+ coarse_latency_dates = [
222
+ d
223
+ for d in missing_coarse_dates
224
+ if (datetime.utcnow().date() - d).days <= VIIRS_GIVEUP_DAYS
225
+ ]
226
+
227
+ if len(coarse_latency_dates) > 0:
228
+ raise AuxiliaryLatency(
229
+ f"Missing coarse dates within {VIIRS_GIVEUP_DAYS}-day window: "
230
+ f"{', '.join([str(d) for d in sorted(list(coarse_latency_dates))])}"
231
+ )
@@ -0,0 +1,18 @@
1
+
2
+ from .generate_input_staging_directory import generate_input_staging_directory
3
+
4
+
5
+ def generate_albedo_coarse_directory(input_staging_directory: str, tile: str) -> str:
6
+ """
7
+ Generates the specific staging directory for coarse albedo images.
8
+
9
+ Args:
10
+ input_staging_directory (str): The base input staging directory.
11
+ tile (str): The HLS tile ID.
12
+
13
+ Returns:
14
+ str: The full path to the coarse albedo staging directory.
15
+ """
16
+ return generate_input_staging_directory(
17
+ input_staging_directory, tile, "albedo_coarse"
18
+ )
@@ -0,0 +1,30 @@
1
+ from typing import Union
2
+ from datetime import date
3
+
4
+ import numpy as np
5
+
6
+ import rasters as rt
7
+ from rasters import Raster, RasterGeometry
8
+
9
+ from .VIIRS.VIIRSDownloader import VIIRSDownloaderAlbedo
10
+
11
+ def generate_albedo_coarse_image(
12
+ date_UTC: Union[date, str], VIIRS_connection: VIIRSDownloaderAlbedo, geometry: RasterGeometry = None
13
+ ) -> Raster:
14
+ """
15
+ Generates a coarse-resolution albedo image from VIIRS data.
16
+
17
+ Args:
18
+ date_UTC (Union[date, str]): The UTC date for which to retrieve albedo data.
19
+ VIIRS_connection (VIIRSDownloaderAlbedo): An initialized VIIRS albedo downloader object.
20
+ geometry (RasterGeometry, optional): The target geometry for the VIIRS image.
21
+ If None, the native VIIRS geometry is used.
22
+
23
+ Returns:
24
+ Raster: A Raster object representing the coarse-resolution albedo image.
25
+ Zero values are converted to NaN.
26
+ """
27
+ coarse_image = VIIRS_connection.albedo(date_UTC=date_UTC, geometry=geometry)
28
+ # Convert zero values to NaN for consistency
29
+ coarse_image = rt.where(coarse_image == 0, np.nan, coarse_image)
30
+ return coarse_image
@@ -0,0 +1,17 @@
1
+
2
+ from .generate_input_staging_directory import generate_input_staging_directory
3
+
4
+ def generate_albedo_fine_directory(input_staging_directory: str, tile: str) -> str:
5
+ """
6
+ Generates the specific staging directory for fine albedo images.
7
+
8
+ Args:
9
+ input_staging_directory (str): The base input staging directory.
10
+ tile (str): The HLS tile ID.
11
+
12
+ Returns:
13
+ str: The full path to the fine albedo staging directory.
14
+ """
15
+ return generate_input_staging_directory(
16
+ input_staging_directory, tile, "albedo_fine"
17
+ )
@@ -0,0 +1,30 @@
1
+ from typing import Union
2
+ from datetime import date
3
+
4
+ import numpy as np
5
+
6
+ import rasters as rt
7
+ from rasters import Raster
8
+
9
+ from harmonized_landsat_sentinel import HLS
10
+
11
+ def generate_albedo_fine_image(
12
+ date_UTC: Union[date, str],
13
+ tile: str,
14
+ HLS_connection: HLS) -> Raster:
15
+ """
16
+ Generates a fine-resolution albedo image from HLS data.
17
+
18
+ Args:
19
+ date_UTC (Union[date, str]): The UTC date for which to retrieve albedo data.
20
+ tile (str): The HLS tile ID.
21
+ HLS_connection (HLS): An initialized HLS data connection object.
22
+
23
+ Returns:
24
+ Raster: A Raster object representing the fine-resolution albedo image.
25
+ Zero values are converted to NaN.
26
+ """
27
+ fine_image = HLS_connection.albedo(tile=tile, date_UTC=date_UTC)
28
+ # Convert zero values to NaN for consistency
29
+ fine_image = rt.where(fine_image == 0, np.nan, fine_image)
30
+ return fine_image