ECOv003-L2T-STARS 1.0.0__py3-none-any.whl → 1.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. ECOv003_L2T_STARS/BRDF/BRDF.py +57 -0
  2. ECOv003_L2T_STARS/BRDF/SZA.py +65 -0
  3. ECOv003_L2T_STARS/BRDF/__init__.py +1 -0
  4. ECOv003_L2T_STARS/BRDF/statistical_radiative_transport.txt +90 -0
  5. ECOv003_L2T_STARS/BRDF/version.txt +1 -0
  6. ECOv003_L2T_STARS/ECOv003_DL.py +527 -0
  7. ECOv003_L2T_STARS/ECOv003_DL.xml +47 -0
  8. ECOv003_L2T_STARS/ECOv003_L2T_STARS.py +162 -0
  9. ECOv003_L2T_STARS/ECOv003_L2T_STARS.xml +47 -0
  10. ECOv003_L2T_STARS/L2TSTARSConfig.py +188 -0
  11. ECOv003_L2T_STARS/L2T_STARS.py +489 -0
  12. ECOv003_L2T_STARS/LPDAAC/LPDAACDataPool.py +444 -0
  13. ECOv003_L2T_STARS/LPDAAC/__init__.py +9 -0
  14. ECOv003_L2T_STARS/LPDAAC/version.txt +1 -0
  15. ECOv003_L2T_STARS/Manifest.toml +2332 -0
  16. ECOv003_L2T_STARS/Project.toml +14 -0
  17. ECOv003_L2T_STARS/VIIRS/VIIRSDataPool.py +294 -0
  18. ECOv003_L2T_STARS/VIIRS/VIIRSDownloader.py +26 -0
  19. ECOv003_L2T_STARS/VIIRS/VIIRS_CMR_LOGIN.py +36 -0
  20. ECOv003_L2T_STARS/VIIRS/VNP09GA.py +1277 -0
  21. ECOv003_L2T_STARS/VIIRS/VNP43IA4.py +288 -0
  22. ECOv003_L2T_STARS/VIIRS/VNP43MA3.py +323 -0
  23. ECOv003_L2T_STARS/VIIRS/__init__.py +9 -0
  24. ECOv003_L2T_STARS/VIIRS/version.txt +1 -0
  25. ECOv003_L2T_STARS/VNP43NRT/VNP43NRT.py +863 -0
  26. ECOv003_L2T_STARS/VNP43NRT/__init__.py +1 -0
  27. ECOv003_L2T_STARS/VNP43NRT/process_VNP43NRT.jl +169 -0
  28. ECOv003_L2T_STARS/VNP43NRT/version.txt +1 -0
  29. ECOv003_L2T_STARS/VNP43NRT_jl/Manifest.toml +995 -0
  30. ECOv003_L2T_STARS/VNP43NRT_jl/Project.toml +15 -0
  31. ECOv003_L2T_STARS/VNP43NRT_jl/__init__.py +0 -0
  32. ECOv003_L2T_STARS/VNP43NRT_jl/instantiate.jl +25 -0
  33. ECOv003_L2T_STARS/VNP43NRT_jl/instantiate.py +13 -0
  34. ECOv003_L2T_STARS/VNP43NRT_jl/src/VNP43NRT.jl +411 -0
  35. ECOv003_L2T_STARS/VNP43NRT_jl/src/__init__.py +0 -0
  36. ECOv003_L2T_STARS/__init__.py +3 -0
  37. ECOv003_L2T_STARS/calibrate_fine_to_coarse.py +60 -0
  38. ECOv003_L2T_STARS/constants.py +38 -0
  39. ECOv003_L2T_STARS/daterange/__init__.py +1 -0
  40. ECOv003_L2T_STARS/daterange/daterange.py +35 -0
  41. ECOv003_L2T_STARS/generate_L2T_STARS_runconfig.py +249 -0
  42. ECOv003_L2T_STARS/generate_NDVI_coarse_directory.py +21 -0
  43. ECOv003_L2T_STARS/generate_NDVI_coarse_image.py +30 -0
  44. ECOv003_L2T_STARS/generate_NDVI_fine_directory.py +14 -0
  45. ECOv003_L2T_STARS/generate_NDVI_fine_image.py +28 -0
  46. ECOv003_L2T_STARS/generate_STARS_inputs.py +231 -0
  47. ECOv003_L2T_STARS/generate_albedo_coarse_directory.py +18 -0
  48. ECOv003_L2T_STARS/generate_albedo_coarse_image.py +30 -0
  49. ECOv003_L2T_STARS/generate_albedo_fine_directory.py +17 -0
  50. ECOv003_L2T_STARS/generate_albedo_fine_image.py +30 -0
  51. ECOv003_L2T_STARS/generate_filename.py +37 -0
  52. ECOv003_L2T_STARS/generate_input_staging_directory.py +23 -0
  53. ECOv003_L2T_STARS/generate_model_state_tile_date_directory.py +28 -0
  54. ECOv003_L2T_STARS/generate_output_directory.py +28 -0
  55. ECOv003_L2T_STARS/install_STARS_jl.py +43 -0
  56. ECOv003_L2T_STARS/instantiate_STARS_jl.py +38 -0
  57. ECOv003_L2T_STARS/load_prior.py +248 -0
  58. ECOv003_L2T_STARS/prior.py +56 -0
  59. ECOv003_L2T_STARS/process_ECOSTRESS_data_fusion_distributed_bias.jl +420 -0
  60. ECOv003_L2T_STARS/process_STARS_product.py +507 -0
  61. ECOv003_L2T_STARS/process_julia_data_fusion.py +110 -0
  62. ECOv003_L2T_STARS/retrieve_STARS_sources.py +101 -0
  63. ECOv003_L2T_STARS/runconfig.py +70 -0
  64. ECOv003_L2T_STARS/timer/__init__.py +1 -0
  65. ECOv003_L2T_STARS/timer/timer.py +77 -0
  66. ECOv003_L2T_STARS/version.py +8 -0
  67. ECOv003_L2T_STARS/version.txt +1 -0
  68. {ECOv003_L2T_STARS-1.0.0.dist-info → ecov003_l2t_stars-1.1.0.dist-info}/METADATA +30 -23
  69. ecov003_l2t_stars-1.1.0.dist-info/RECORD +73 -0
  70. {ECOv003_L2T_STARS-1.0.0.dist-info → ecov003_l2t_stars-1.1.0.dist-info}/WHEEL +1 -1
  71. ecov003_l2t_stars-1.1.0.dist-info/entry_points.txt +3 -0
  72. ecov003_l2t_stars-1.1.0.dist-info/top_level.txt +1 -0
  73. ECOv003_L2T_STARS-1.0.0.dist-info/RECORD +0 -5
  74. ECOv003_L2T_STARS-1.0.0.dist-info/top_level.txt +0 -1
  75. {ECOv003_L2T_STARS-1.0.0.dist-info → ecov003_l2t_stars-1.1.0.dist-info/licenses}/LICENSE +0 -0
@@ -0,0 +1,489 @@
1
+ import argparse
2
+ import logging
3
+ import sys
4
+ from datetime import date, timedelta
5
+ from os import makedirs
6
+ from os.path import join, exists
7
+ from typing import Union
8
+ import logging
9
+ import colored_logging as cl
10
+ import pandas as pd
11
+ from dateutil import parser
12
+
13
+ # Custom modules for Harmonized Landsat Sentinel (HLS) and ECOSTRESS data
14
+ from harmonized_landsat_sentinel import (
15
+ CMRServerUnreachable,
16
+ HLS2CMR,
17
+ HLSTileNotAvailable,
18
+ HLSSentinelMissing,
19
+ HLSLandsatMissing,
20
+ HLSNotAvailable,
21
+ HLSBandNotAcquired,
22
+ CMR_SEARCH_URL
23
+ )
24
+
25
+ from ECOv003_exit_codes import *
26
+
27
+ from ECOv002_granules import L2TLSTE
28
+ import urllib
29
+
30
+ from .version import __version__
31
+ from .constants import *
32
+ from .VIIRS.VNP43IA4 import VNP43IA4
33
+ from .VIIRS.VNP43MA3 import VNP43MA3
34
+ from .VNP43NRT import VNP43NRT
35
+ from .runconfig import ECOSTRESSRunConfig
36
+ from .L2TSTARSConfig import L2TSTARSConfig
37
+ from .load_prior import load_prior
38
+ from .generate_NDVI_coarse_directory import generate_NDVI_coarse_directory
39
+ from .generate_NDVI_fine_directory import generate_NDVI_fine_directory
40
+ from .generate_albedo_coarse_directory import generate_albedo_coarse_directory
41
+ from .generate_albedo_fine_directory import generate_albedo_fine_directory
42
+ from .generate_STARS_inputs import generate_STARS_inputs
43
+ from .process_STARS_product import process_STARS_product
44
+ from .retrieve_STARS_sources import retrieve_STARS_sources
45
+
46
+ logger = logging.getLogger(__name__)
47
+
48
+ def L2T_STARS(
49
+ runconfig_filename: str,
50
+ date_UTC: Union[date, str] = None,
51
+ spinup_days: int = DEFAULT_SPINUP_DAYS,
52
+ target_resolution: int = DEFAULT_TARGET_RESOLUTION,
53
+ NDVI_resolution: int = DEFAULT_NDVI_RESOLUTION,
54
+ albedo_resolution: int = DEFAULT_ALBEDO_RESOLUTION,
55
+ use_VNP43NRT: bool = DEFAULT_USE_VNP43NRT,
56
+ calibrate_fine: bool = DEFAULT_CALIBRATE_FINE,
57
+ sources_only: bool = False,
58
+ remove_input_staging: bool = True,
59
+ remove_prior: bool = True,
60
+ remove_posterior: bool = True,
61
+ threads: Union[int, str] = "auto",
62
+ num_workers: int = 4,
63
+ ) -> int:
64
+ """
65
+ ECOSTRESS Collection 3 L2T_STARS PGE (Product Generation Executive).
66
+
67
+ This function serves as the main entry point for the L2T_STARS processing.
68
+ It orchestrates the entire workflow, including reading the run-config,
69
+ connecting to data servers, retrieving source data, performing data fusion
70
+ (via Julia subprocess), generating the final product, and handling cleanup.
71
+
72
+ Args:
73
+ runconfig_filename (str): Path to the XML run-configuration file.
74
+ date_UTC (Union[date, str], optional): The target UTC date for product generation.
75
+ If None, it's derived from the input L2T LSTE granule.
76
+ spinup_days (int, optional): Number of days for the VIIRS time-series spin-up.
77
+ Defaults to DEFAULT_SPINUP_DAYS (7).
78
+ target_resolution (int, optional): The desired output resolution in meters.
79
+ Defaults to DEFAULT_TARGET_RESOLUTION (70).
80
+ NDVI_resolution (int, optional): The resolution of the coarse NDVI data.
81
+ Defaults to DEFAULT_NDVI_RESOLUTION (490).
82
+ albedo_resolution (int, optional): The resolution of the coarse albedo data.
83
+ Defaults to DEFAULT_ALBEDO_RESOLUTION (980).
84
+ use_VNP43NRT (bool, optional): If True, use VNP43NRT for VIIRS products.
85
+ If False, use VNP43IA4 (NDVI) and VNP43MA3 (Albedo).
86
+ Defaults to DEFAULT_USE_VNP43NRT (True).
87
+ calibrate_fine (bool, optional): If True, calibrate fine resolution HLS data to
88
+ coarse resolution VIIRS data. Defaults to DEFAULT_CALIBRATE_FINE (False).
89
+ sources_only (bool, optional): If True, only retrieve source data and exit,
90
+ without performing data fusion. Defaults to False.
91
+ remove_input_staging (bool, optional): If True, remove the input staging directory
92
+ after processing. Defaults to True.
93
+ remove_prior (bool, optional): If True, remove prior intermediate files after use.
94
+ Defaults to True.
95
+ remove_posterior (bool, optional): If True, remove posterior intermediate files after
96
+ product generation. Defaults to True.
97
+ threads (Union[int, str], optional): Number of Julia threads to use, or "auto".
98
+ Defaults to "auto".
99
+ num_workers (int, optional): Number of Julia workers for distributed processing.
100
+ Defaults to 4.
101
+
102
+ Returns:
103
+ int: An exit code indicating the success or failure of the PGE execution.
104
+ (e.g., SUCCESS_EXIT_CODE, AUXILIARY_SERVER_UNREACHABLE, DOWNLOAD_FAILED, etc.)
105
+ """
106
+ exit_code = SUCCESS_EXIT_CODE # Initialize exit code to success
107
+
108
+ try:
109
+ # Load and parse the run-configuration file
110
+ runconfig = L2TSTARSConfig(runconfig_filename)
111
+
112
+ # Configure logging with the specified log filename from runconfig
113
+ working_directory = runconfig.working_directory
114
+ granule_ID = runconfig.granule_ID
115
+ log_filename = join(working_directory, "log", f"{granule_ID}.log")
116
+ cl.configure(filename=log_filename) # Reconfigure logger with the specific log file
117
+
118
+ logger.info(f"L2T_STARS PGE ({cl.val(__version__)})")
119
+ logger.info(f"L2T_STARS run-config: {cl.file(runconfig_filename)}")
120
+ logger.info(f"Granule ID: {cl.val(granule_ID)}")
121
+
122
+ # Extract paths from the run-config
123
+ L2T_STARS_granule_directory = runconfig.L2T_STARS_granule_directory
124
+ logger.info(f"Granule directory: {cl.dir(L2T_STARS_granule_directory)}")
125
+ L2T_STARS_zip_filename = runconfig.L2T_STARS_zip_filename
126
+ logger.info(f"Zip filename: {cl.file(L2T_STARS_zip_filename)}")
127
+ L2T_STARS_browse_filename = runconfig.L2T_STARS_browse_filename
128
+ logger.info(f"Browse filename: " + cl.file(L2T_STARS_browse_filename))
129
+
130
+ # Check if the final product already exists to avoid reprocessing
131
+ if exists(L2T_STARS_zip_filename) and exists(L2T_STARS_browse_filename):
132
+ logger.info(f"Found existing L2T STARS file: {L2T_STARS_zip_filename}")
133
+ logger.info(f"Found existing L2T STARS preview: {L2T_STARS_browse_filename}")
134
+ return SUCCESS_EXIT_CODE
135
+
136
+ logger.info(f"Working directory: {cl.dir(working_directory)}")
137
+ logger.info(f"Log file: {cl.file(log_filename)}")
138
+
139
+ input_staging_directory = join(working_directory, "input_staging")
140
+ logger.info(f"Input staging directory: {cl.dir(input_staging_directory)}")
141
+
142
+ sources_directory = runconfig.sources_directory
143
+ logger.info(f"Source directory: {cl.dir(sources_directory)}")
144
+ indices_directory = runconfig.indices_directory
145
+ logger.info(f"Indices directory: {cl.dir(indices_directory)}")
146
+ model_directory = runconfig.model_directory
147
+ logger.info(f"Model directory: {cl.dir(model_directory)}")
148
+ output_directory = runconfig.output_directory
149
+ logger.info(f"Output directory: {cl.dir(output_directory)}")
150
+ tile = runconfig.tile
151
+ logger.info(f"Tile: {cl.val(tile)}")
152
+ build = runconfig.build
153
+ logger.info(f"Build: {cl.val(build)}")
154
+ product_counter = runconfig.product_counter
155
+ logger.info(f"Product counter: {cl.val(product_counter)}")
156
+ L2T_LSTE_filename = runconfig.L2T_LSTE_filename
157
+ logger.info(f"Input L2T LSTE file: {cl.file(L2T_LSTE_filename)}")
158
+
159
+ # Validate existence of input L2T LSTE file
160
+ if not exists(L2T_LSTE_filename):
161
+ raise InputFilesInaccessible(
162
+ f"L2T LSTE file does not exist: {L2T_LSTE_filename}"
163
+ )
164
+
165
+ # Load the L2T_LSTE granule to get geometry and base metadata
166
+ l2t_granule = L2TLSTE(L2T_LSTE_filename)
167
+ geometry = l2t_granule.geometry
168
+ metadata = l2t_granule.metadata_dict
169
+ metadata["StandardMetadata"]["PGEName"] = "L2T_STARS"
170
+
171
+ # Update product names in metadata
172
+ short_name = L2T_STARS_SHORT_NAME
173
+ logger.info(f"L2T STARS short name: {cl.val(short_name)}")
174
+ metadata["StandardMetadata"]["ShortName"] = short_name
175
+
176
+ long_name = L2T_STARS_LONG_NAME
177
+ logger.info(f"L2T STARS long name: {cl.val(long_name)}")
178
+ metadata["StandardMetadata"]["LongName"] = long_name
179
+
180
+ # Update auxiliary input pointers in metadata and remove irrelevant sections
181
+ metadata["StandardMetadata"]["AuxiliaryInputPointer"] = "HLS,VIIRS"
182
+ if "ProductMetadata" in metadata:
183
+ metadata["ProductMetadata"].pop("AuxiliaryNWP", None) # Safe removal
184
+ metadata["ProductMetadata"].pop("NWPSource", None)
185
+
186
+ # Determine the target date for processing
187
+ time_UTC = l2t_granule.time_UTC
188
+ logger.info(f"ECOSTRESS overpass time: {cl.time(f'{time_UTC:%Y-%m-%d %H:%M:%S} UTC')}")
189
+
190
+ if date_UTC is None:
191
+ # Use date from L2T granule if not provided via command line
192
+ date_UTC = l2t_granule.date_UTC
193
+ logger.info(f"ECOSTRESS overpass date: {cl.time(f'{date_UTC:%Y-%m-%d} UTC')}")
194
+ else:
195
+ logger.warning(f"Over-riding target date from command line to: {date_UTC}")
196
+ if isinstance(date_UTC, str):
197
+ date_UTC = parser.parse(date_UTC).date()
198
+
199
+ # TODO: Add a check if the L2T LSTE granule is day-time and halt L2T STARS run if it's not.
200
+ # This is a critical step to ensure valid scientific output.
201
+
202
+ # Load prior data if specified in the run-config
203
+ L2T_STARS_prior_filename = runconfig.L2T_STARS_prior_filename
204
+ prior = load_prior(
205
+ tile=tile,
206
+ target_resolution=target_resolution,
207
+ model_directory=model_directory,
208
+ L2T_STARS_prior_filename=L2T_STARS_prior_filename,
209
+ )
210
+ using_prior = prior.using_prior
211
+ prior_date_UTC = prior.prior_date_UTC
212
+
213
+ # Define various product and download directories
214
+ products_directory = join(working_directory, DEFAULT_STARS_PRODUCTS_DIRECTORY)
215
+ logger.info(f"STARS products directory: {cl.dir(products_directory)}")
216
+ HLS_download_directory = join(sources_directory, DEFAULT_HLS_DOWNLOAD_DIRECTORY)
217
+ logger.info(f"HLS download directory: {cl.dir(HLS_download_directory)}")
218
+ HLS_products_directory = join(sources_directory, DEFAULT_HLS_PRODUCTS_DIRECTORY)
219
+ logger.info(f"HLS products directory: {cl.dir(HLS_products_directory)}")
220
+ VIIRS_download_directory = join(sources_directory, DEFAULT_VIIRS_DOWNLOAD_DIRECTORY)
221
+ logger.info(f"VIIRS download directory: {cl.dir(VIIRS_download_directory)}")
222
+ VIIRS_products_directory = join(sources_directory, DEFAULT_VIIRS_PRODUCTS_DIRECTORY)
223
+ logger.info(f"VIIRS products directory: {cl.dir(VIIRS_products_directory)}")
224
+ VIIRS_mosaic_directory = join(sources_directory, DEFAUL_VIIRS_MOSAIC_DIRECTORY)
225
+ logger.info(f"VIIRS mosaic directory: {cl.dir(VIIRS_mosaic_directory)}")
226
+ GEOS5FP_download_directory = join(sources_directory, DEFAULT_GEOS5FP_DOWNLOAD_DIRECTORY)
227
+ logger.info(f"GEOS-5 FP download directory: {cl.dir(GEOS5FP_download_directory)}")
228
+ GEOS5FP_products_directory = join(sources_directory, DEFAULT_GEOS5FP_PRODUCTS_DIRECTORY)
229
+ logger.info(f"GEOS-5 FP products directory: {cl.dir(GEOS5FP_products_directory)}")
230
+ VNP09GA_products_directory = join(sources_directory, DEFAULT_VNP09GA_PRODUCTS_DIRECTORY)
231
+ logger.info(f"VNP09GA products directory: {cl.dir(VNP09GA_products_directory)}")
232
+ VNP43NRT_products_directory = join(sources_directory, DEFAULT_VNP43NRT_PRODUCTS_DIRECTORY)
233
+ logger.info(f"VNP43NRT products directory: {cl.dir(VNP43NRT_products_directory)}")
234
+
235
+ # Re-check for existing product (double-check in case another process created it)
236
+ if exists(L2T_STARS_zip_filename):
237
+ logger.info(
238
+ f"Found L2T STARS product zip: {cl.file(L2T_STARS_zip_filename)}"
239
+ )
240
+ return exit_code
241
+
242
+ # Initialize HLS data connection
243
+ logger.info(f"Connecting to CMR Search server: {CMR_SEARCH_URL}")
244
+ try:
245
+ HLS_connection = HLS2CMR(
246
+ working_directory=working_directory,
247
+ download_directory=HLS_download_directory,
248
+ products_directory=HLS_products_directory,
249
+ target_resolution=target_resolution,
250
+ )
251
+ except CMRServerUnreachable as e:
252
+ logger.exception(e)
253
+ raise AuxiliaryServerUnreachable(
254
+ f"Unable to connect to CMR Search server: {CMR_SEARCH_URL}"
255
+ )
256
+
257
+ # Check if the tile is on land (HLS tiles cover land and ocean, STARS is for land)
258
+ if not HLS_connection.tile_grid.land(tile=tile):
259
+ raise LandFilter(f"Sentinel tile {tile} is not on land. Skipping processing.")
260
+
261
+ # Initialize VIIRS data connections based on 'use_VNP43NRT' flag
262
+ if use_VNP43NRT:
263
+ try:
264
+ NDVI_VIIRS_connection = VNP43NRT(
265
+ working_directory=working_directory,
266
+ download_directory=VIIRS_download_directory,
267
+ mosaic_directory=VIIRS_mosaic_directory,
268
+ GEOS5FP_download=GEOS5FP_download_directory,
269
+ GEOS5FP_products=GEOS5FP_products_directory,
270
+ VNP09GA_directory=VNP09GA_products_directory,
271
+ VNP43NRT_directory=VNP43NRT_products_directory,
272
+ )
273
+
274
+ albedo_VIIRS_connection = VNP43NRT(
275
+ working_directory=working_directory,
276
+ download_directory=VIIRS_download_directory,
277
+ mosaic_directory=VIIRS_mosaic_directory,
278
+ GEOS5FP_download=GEOS5FP_download_directory,
279
+ GEOS5FP_products=GEOS5FP_products_directory,
280
+ VNP09GA_directory=VNP09GA_products_directory,
281
+ VNP43NRT_directory=VNP43NRT_products_directory,
282
+ )
283
+ except CMRServerUnreachable as e:
284
+ logger.exception(e)
285
+ raise AuxiliaryServerUnreachable(f"Unable to connect to CMR search server for VNP43NRT.")
286
+ else:
287
+ try:
288
+ NDVI_VIIRS_connection = VNP43IA4(
289
+ working_directory=working_directory,
290
+ download_directory=VIIRS_download_directory,
291
+ products_directory=VIIRS_products_directory,
292
+ mosaic_directory=VIIRS_mosaic_directory,
293
+ )
294
+
295
+ albedo_VIIRS_connection = VNP43MA3(
296
+ working_directory=working_directory,
297
+ download_directory=VIIRS_download_directory,
298
+ products_directory=VIIRS_products_directory,
299
+ mosaic_directory=VIIRS_mosaic_directory,
300
+ )
301
+ except LPDAACServerUnreachable as e:
302
+ logger.exception(e)
303
+ raise AuxiliaryServerUnreachable(f"Unable to connect to VIIRS LPDAAC server.")
304
+
305
+ # Define date ranges for data retrieval and fusion
306
+ end_date = date_UTC
307
+ # The start date of the BRDF-corrected VIIRS coarse time-series is 'spinup_days' before the target date
308
+ VIIRS_start_date = end_date - timedelta(days=spinup_days)
309
+ # To produce that first BRDF-corrected image, VNP09GA (raw VIIRS) is needed starting 16 days prior to the first coarse date
310
+ VIIRS_download_start_date = VIIRS_start_date - timedelta(days=16)
311
+ VIIRS_end_date = end_date
312
+
313
+ # Define start date of HLS fine image input time-series
314
+ if using_prior and prior_date_UTC and prior_date_UTC >= VIIRS_start_date:
315
+ # If a valid prior is used and its date is within or before the VIIRS start date,
316
+ # HLS inputs begin the day after the prior
317
+ HLS_start_date = prior_date_UTC + timedelta(days=1)
318
+ else:
319
+ # If no prior or prior is too old, HLS inputs begin on the same day as the VIIRS inputs
320
+ HLS_start_date = VIIRS_start_date
321
+ HLS_end_date = end_date # HLS end date is always the same as the target date
322
+
323
+ logger.info(
324
+ f"Processing STARS HLS-VIIRS NDVI and albedo for tile {cl.place(tile)} from "
325
+ f"{cl.time(VIIRS_start_date)} to {cl.time(end_date)}"
326
+ )
327
+
328
+ # Get HLS listing to check for data availability
329
+ try:
330
+ HLS_listing = HLS_connection.listing(
331
+ tile=tile, start_UTC=HLS_start_date, end_UTC=HLS_end_date
332
+ )
333
+ except HLSTileNotAvailable as e:
334
+ logger.exception(e)
335
+ raise LandFilter(f"Sentinel tile {tile} cannot be processed due to HLS tile unavailability.")
336
+ except Exception as e:
337
+ logger.exception(e)
338
+ raise AuxiliaryServerUnreachable(
339
+ f"Unable to scan Harmonized Landsat Sentinel server: {HLS_connection.remote}"
340
+ )
341
+
342
+ # Check for missing HLS Sentinel data
343
+ missing_sentinel_dates = HLS_listing[HLS_listing.sentinel == "missing"].date_UTC
344
+ if len(missing_sentinel_dates) > 0:
345
+ raise AuxiliaryLatency(
346
+ f"HLS Sentinel is not yet available at tile {tile} for dates: "
347
+ f"{', '.join(missing_sentinel_dates.dt.strftime('%Y-%m-%d'))}"
348
+ )
349
+
350
+ # Log available HLS Sentinel data
351
+ sentinel_listing = HLS_listing[~pd.isna(HLS_listing.sentinel)][
352
+ ["date_UTC", "sentinel"]
353
+ ]
354
+ logger.info(f"HLS Sentinel is available on {cl.val(len(sentinel_listing))} dates:")
355
+ for i, (list_date_utc, sentinel_granule) in sentinel_listing.iterrows():
356
+ sentinel_filename = sentinel_granule["meta"]["native-id"]
357
+ logger.info(f"* {cl.time(list_date_utc)}: {cl.file(sentinel_filename)}")
358
+
359
+ # Check for missing HLS Landsat data
360
+ missing_landsat_dates = HLS_listing[HLS_listing.landsat == "missing"].date_UTC
361
+ if len(missing_landsat_dates) > 0:
362
+ raise AuxiliaryLatency(
363
+ f"HLS Landsat is not yet available at tile {tile} for dates: "
364
+ f"{', '.join(missing_landsat_dates.dt.strftime('%Y-%m-%d'))}"
365
+ )
366
+
367
+ # Log available HLS Landsat data
368
+ landsat_listing = HLS_listing[~pd.isna(HLS_listing.landsat)][
369
+ ["date_UTC", "landsat"]
370
+ ]
371
+ logger.info(f"HLS Landsat is available on {cl.val(len(landsat_listing))} dates:")
372
+ for i, (list_date_utc, landsat_granule) in landsat_listing.iterrows():
373
+ landsat_filename = landsat_granule["meta"]["native-id"]
374
+ logger.info(f"* {cl.time(list_date_utc)}: {cl.file(landsat_filename)}")
375
+
376
+ # If only sources are requested, retrieve them and exit
377
+ if sources_only:
378
+ logger.info("Sources only flag enabled. Retrieving source data.")
379
+ retrieve_STARS_sources(
380
+ tile=tile,
381
+ geometry=geometry,
382
+ HLS_start_date=HLS_start_date,
383
+ HLS_end_date=HLS_end_date,
384
+ VIIRS_start_date=VIIRS_download_start_date,
385
+ VIIRS_end_date=VIIRS_end_date,
386
+ HLS_connection=HLS_connection,
387
+ VIIRS_connection=NDVI_VIIRS_connection, # Use NDVI_VIIRS_connection as a general VIIRS connection
388
+ )
389
+ # Regenerate inputs to ensure all files are staged, even if not fused
390
+ NDVI_coarse_geometry = HLS_connection.grid(tile=tile, cell_size=NDVI_resolution)
391
+ albedo_coarse_geometry = HLS_connection.grid(tile=tile, cell_size=albedo_resolution)
392
+
393
+ NDVI_coarse_directory = generate_NDVI_coarse_directory(
394
+ input_staging_directory=input_staging_directory, tile=tile
395
+ )
396
+ NDVI_fine_directory = generate_NDVI_fine_directory(
397
+ input_staging_directory=input_staging_directory, tile=tile
398
+ )
399
+ albedo_coarse_directory = generate_albedo_coarse_directory(
400
+ input_staging_directory=input_staging_directory, tile=tile
401
+ )
402
+ albedo_fine_directory = generate_albedo_fine_directory(
403
+ input_staging_directory=input_staging_directory, tile=tile
404
+ )
405
+
406
+ generate_STARS_inputs(
407
+ tile=tile,
408
+ date_UTC=date_UTC,
409
+ HLS_start_date=HLS_start_date,
410
+ HLS_end_date=HLS_end_date,
411
+ VIIRS_start_date=VIIRS_start_date,
412
+ VIIRS_end_date=VIIRS_end_date,
413
+ NDVI_resolution=NDVI_resolution,
414
+ albedo_resolution=albedo_resolution,
415
+ target_resolution=target_resolution,
416
+ NDVI_coarse_geometry=NDVI_coarse_geometry,
417
+ albedo_coarse_geometry=albedo_coarse_geometry,
418
+ working_directory=working_directory,
419
+ NDVI_coarse_directory=NDVI_coarse_directory,
420
+ NDVI_fine_directory=NDVI_fine_directory,
421
+ albedo_coarse_directory=albedo_coarse_directory,
422
+ albedo_fine_directory=albedo_fine_directory,
423
+ HLS_connection=HLS_connection,
424
+ NDVI_VIIRS_connection=NDVI_VIIRS_connection,
425
+ albedo_VIIRS_connection=albedo_VIIRS_connection,
426
+ calibrate_fine=calibrate_fine,
427
+ )
428
+ else:
429
+ # Otherwise, proceed with full product processing
430
+ process_STARS_product(
431
+ tile=tile,
432
+ date_UTC=date_UTC,
433
+ time_UTC=time_UTC,
434
+ build=build,
435
+ product_counter=product_counter,
436
+ HLS_start_date=HLS_start_date,
437
+ HLS_end_date=HLS_end_date,
438
+ VIIRS_start_date=VIIRS_start_date,
439
+ VIIRS_end_date=VIIRS_end_date,
440
+ NDVI_resolution=NDVI_resolution,
441
+ albedo_resolution=albedo_resolution,
442
+ target_resolution=target_resolution,
443
+ working_directory=working_directory,
444
+ model_directory=model_directory,
445
+ input_staging_directory=input_staging_directory,
446
+ L2T_STARS_granule_directory=L2T_STARS_granule_directory,
447
+ L2T_STARS_zip_filename=L2T_STARS_zip_filename,
448
+ L2T_STARS_browse_filename=L2T_STARS_browse_filename,
449
+ metadata=metadata,
450
+ prior=prior,
451
+ HLS_connection=HLS_connection,
452
+ NDVI_VIIRS_connection=NDVI_VIIRS_connection,
453
+ albedo_VIIRS_connection=albedo_VIIRS_connection,
454
+ using_prior=using_prior,
455
+ calibrate_fine=calibrate_fine,
456
+ remove_input_staging=remove_input_staging,
457
+ remove_prior=remove_prior,
458
+ remove_posterior=remove_posterior,
459
+ threads=threads,
460
+ num_workers=num_workers,
461
+ )
462
+
463
+ # --- Exception Handling for PGE ---
464
+ except (ConnectionError, urllib.error.HTTPError, CMRServerUnreachable) as exception:
465
+ logger.exception(exception)
466
+ exit_code = AUXILIARY_SERVER_UNREACHABLE
467
+ except DownloadFailed as exception:
468
+ logger.exception(exception)
469
+ exit_code = DOWNLOAD_FAILED
470
+ except HLSBandNotAcquired as exception:
471
+ logger.exception(exception)
472
+ exit_code = DOWNLOAD_FAILED
473
+ except HLSNotAvailable as exception:
474
+ logger.exception(exception)
475
+ exit_code = LAND_FILTER # This might indicate no HLS data for the tile, similar to land filter
476
+ except (HLSSentinelMissing, HLSLandsatMissing) as exception:
477
+ logger.exception(exception)
478
+ exit_code = AUXILIARY_LATENCY
479
+ except ECOSTRESSExitCodeException as exception:
480
+ # Catch custom ECOSTRESS exceptions and use their defined exit code
481
+ logger.exception(exception)
482
+ exit_code = exception.exit_code
483
+ except Exception as exception:
484
+ # Catch any other unexpected exceptions
485
+ logger.exception(exception)
486
+ exit_code = UNCLASSIFIED_FAILURE_EXIT_CODE
487
+
488
+ logger.info(f"L2T_STARS exit code: {exit_code}")
489
+ return exit_code