ECOv003-L2T-STARS 1.0.1__py3-none-any.whl → 1.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. ECOv003_L2T_STARS/BRDF/BRDF.py +57 -0
  2. ECOv003_L2T_STARS/BRDF/SZA.py +65 -0
  3. ECOv003_L2T_STARS/BRDF/__init__.py +1 -0
  4. ECOv003_L2T_STARS/BRDF/statistical_radiative_transport.txt +90 -0
  5. ECOv003_L2T_STARS/BRDF/version.txt +1 -0
  6. ECOv003_L2T_STARS/ECOv003_DL.py +527 -0
  7. ECOv003_L2T_STARS/ECOv003_DL.xml +47 -0
  8. ECOv003_L2T_STARS/ECOv003_L2T_STARS.py +169 -0
  9. ECOv003_L2T_STARS/ECOv003_L2T_STARS.xml +47 -0
  10. ECOv003_L2T_STARS/L2TSTARSConfig.py +190 -0
  11. ECOv003_L2T_STARS/L2T_STARS.py +503 -0
  12. ECOv003_L2T_STARS/LPDAAC/LPDAACDataPool.py +444 -0
  13. ECOv003_L2T_STARS/LPDAAC/__init__.py +9 -0
  14. ECOv003_L2T_STARS/LPDAAC/version.txt +1 -0
  15. ECOv003_L2T_STARS/Manifest.toml +2332 -0
  16. ECOv003_L2T_STARS/Project.toml +14 -0
  17. ECOv003_L2T_STARS/VIIRS/VIIRSDataPool.py +294 -0
  18. ECOv003_L2T_STARS/VIIRS/VIIRSDownloader.py +26 -0
  19. ECOv003_L2T_STARS/VIIRS/VIIRS_CMR_LOGIN.py +36 -0
  20. ECOv003_L2T_STARS/VIIRS/VNP09GA.py +1278 -0
  21. ECOv003_L2T_STARS/VIIRS/VNP43IA4.py +288 -0
  22. ECOv003_L2T_STARS/VIIRS/VNP43MA3.py +323 -0
  23. ECOv003_L2T_STARS/VIIRS/__init__.py +9 -0
  24. ECOv003_L2T_STARS/VIIRS/version.txt +1 -0
  25. ECOv003_L2T_STARS/VNP43NRT/VNP43NRT.py +863 -0
  26. ECOv003_L2T_STARS/VNP43NRT/__init__.py +1 -0
  27. ECOv003_L2T_STARS/VNP43NRT/process_VNP43NRT.jl +169 -0
  28. ECOv003_L2T_STARS/VNP43NRT/version.txt +1 -0
  29. ECOv003_L2T_STARS/VNP43NRT_jl/Manifest.toml +995 -0
  30. ECOv003_L2T_STARS/VNP43NRT_jl/Project.toml +15 -0
  31. ECOv003_L2T_STARS/VNP43NRT_jl/__init__.py +0 -0
  32. ECOv003_L2T_STARS/VNP43NRT_jl/instantiate.jl +25 -0
  33. ECOv003_L2T_STARS/VNP43NRT_jl/instantiate.py +13 -0
  34. ECOv003_L2T_STARS/VNP43NRT_jl/src/VNP43NRT.jl +411 -0
  35. ECOv003_L2T_STARS/VNP43NRT_jl/src/__init__.py +0 -0
  36. ECOv003_L2T_STARS/__init__.py +3 -0
  37. ECOv003_L2T_STARS/calibrate_fine_to_coarse.py +60 -0
  38. ECOv003_L2T_STARS/constants.py +38 -0
  39. ECOv003_L2T_STARS/daterange/__init__.py +1 -0
  40. ECOv003_L2T_STARS/daterange/daterange.py +35 -0
  41. ECOv003_L2T_STARS/generate_L2T_STARS_runconfig.py +249 -0
  42. ECOv003_L2T_STARS/generate_NDVI_coarse_directory.py +21 -0
  43. ECOv003_L2T_STARS/generate_NDVI_coarse_image.py +30 -0
  44. ECOv003_L2T_STARS/generate_NDVI_fine_directory.py +14 -0
  45. ECOv003_L2T_STARS/generate_NDVI_fine_image.py +28 -0
  46. ECOv003_L2T_STARS/generate_STARS_inputs.py +231 -0
  47. ECOv003_L2T_STARS/generate_albedo_coarse_directory.py +18 -0
  48. ECOv003_L2T_STARS/generate_albedo_coarse_image.py +30 -0
  49. ECOv003_L2T_STARS/generate_albedo_fine_directory.py +17 -0
  50. ECOv003_L2T_STARS/generate_albedo_fine_image.py +30 -0
  51. ECOv003_L2T_STARS/generate_filename.py +37 -0
  52. ECOv003_L2T_STARS/generate_input_staging_directory.py +23 -0
  53. ECOv003_L2T_STARS/generate_model_state_tile_date_directory.py +28 -0
  54. ECOv003_L2T_STARS/generate_output_directory.py +28 -0
  55. ECOv003_L2T_STARS/install_STARS_jl.py +43 -0
  56. ECOv003_L2T_STARS/instantiate_STARS_jl.py +38 -0
  57. ECOv003_L2T_STARS/load_prior.py +250 -0
  58. ECOv003_L2T_STARS/prior.py +56 -0
  59. ECOv003_L2T_STARS/process_ECOSTRESS_data_fusion_distributed_bias.jl +420 -0
  60. ECOv003_L2T_STARS/process_STARS_product.py +507 -0
  61. ECOv003_L2T_STARS/process_julia_data_fusion.py +110 -0
  62. ECOv003_L2T_STARS/retrieve_STARS_sources.py +101 -0
  63. ECOv003_L2T_STARS/runconfig.py +70 -0
  64. ECOv003_L2T_STARS/timer/__init__.py +1 -0
  65. ECOv003_L2T_STARS/timer/timer.py +77 -0
  66. ECOv003_L2T_STARS/version.py +8 -0
  67. ECOv003_L2T_STARS/version.txt +1 -0
  68. {ECOv003_L2T_STARS-1.0.1.dist-info → ecov003_l2t_stars-1.2.0.dist-info}/METADATA +31 -24
  69. ecov003_l2t_stars-1.2.0.dist-info/RECORD +73 -0
  70. {ECOv003_L2T_STARS-1.0.1.dist-info → ecov003_l2t_stars-1.2.0.dist-info}/WHEEL +1 -1
  71. ecov003_l2t_stars-1.2.0.dist-info/entry_points.txt +3 -0
  72. ecov003_l2t_stars-1.2.0.dist-info/top_level.txt +1 -0
  73. ECOv003_L2T_STARS-1.0.1.dist-info/RECORD +0 -5
  74. ECOv003_L2T_STARS-1.0.1.dist-info/top_level.txt +0 -1
  75. {ECOv003_L2T_STARS-1.0.1.dist-info → ecov003_l2t_stars-1.2.0.dist-info/licenses}/LICENSE +0 -0
@@ -0,0 +1,503 @@
1
+ import argparse
2
+ import logging
3
+ import sys
4
+ from datetime import date, timedelta
5
+ from os import makedirs
6
+ from os.path import join, exists
7
+ from typing import Union
8
+ import logging
9
+ import colored_logging as cl
10
+ import pandas as pd
11
+ from dateutil import parser
12
+
13
+ # Custom modules for Harmonized Landsat Sentinel (HLS) and ECOSTRESS data
14
+ from harmonized_landsat_sentinel import (
15
+ CMRServerUnreachable,
16
+ HLS2CMR,
17
+ HLSTileNotAvailable,
18
+ HLSSentinelMissing,
19
+ HLSLandsatMissing,
20
+ HLSNotAvailable,
21
+ HLSBandNotAcquired,
22
+ CMR_SEARCH_URL
23
+ )
24
+
25
+ from ECOv003_exit_codes import *
26
+
27
+ from ECOv002_granules import L2TLSTE
28
+ import urllib
29
+
30
+ from .version import __version__
31
+ from .constants import *
32
+ from .VIIRS.VNP43IA4 import VNP43IA4
33
+ from .VIIRS.VNP43MA3 import VNP43MA3
34
+ from .VNP43NRT import VNP43NRT
35
+ from .runconfig import ECOSTRESSRunConfig
36
+ from .L2TSTARSConfig import L2TSTARSConfig
37
+ from .load_prior import load_prior
38
+ from .generate_NDVI_coarse_directory import generate_NDVI_coarse_directory
39
+ from .generate_NDVI_fine_directory import generate_NDVI_fine_directory
40
+ from .generate_albedo_coarse_directory import generate_albedo_coarse_directory
41
+ from .generate_albedo_fine_directory import generate_albedo_fine_directory
42
+ from .generate_STARS_inputs import generate_STARS_inputs
43
+ from .process_STARS_product import process_STARS_product
44
+ from .retrieve_STARS_sources import retrieve_STARS_sources
45
+
46
+ logger = logging.getLogger(__name__)
47
+
48
+ def L2T_STARS(
49
+ runconfig_filename: str,
50
+ date_UTC: Union[date, str] = None,
51
+ spinup_days: int = DEFAULT_SPINUP_DAYS,
52
+ target_resolution: int = DEFAULT_TARGET_RESOLUTION,
53
+ NDVI_resolution: int = DEFAULT_NDVI_RESOLUTION,
54
+ albedo_resolution: int = DEFAULT_ALBEDO_RESOLUTION,
55
+ use_VNP43NRT: bool = DEFAULT_USE_VNP43NRT,
56
+ calibrate_fine: bool = DEFAULT_CALIBRATE_FINE,
57
+ sources_only: bool = False,
58
+ remove_input_staging: bool = True,
59
+ remove_prior: bool = True,
60
+ remove_posterior: bool = True,
61
+ threads: Union[int, str] = "auto",
62
+ num_workers: int = 4,
63
+ overwrite: bool = False, # New parameter for overwriting existing files
64
+ ) -> int:
65
+ """
66
+ ECOSTRESS Collection 3 L2T_STARS PGE (Product Generation Executive).
67
+
68
+ This function serves as the main entry point for the L2T_STARS processing.
69
+ It orchestrates the entire workflow, including reading the run-config,
70
+ connecting to data servers, retrieving source data, performing data fusion
71
+ (via Julia subprocess), generating the final product, and handling cleanup.
72
+
73
+ Args:
74
+ runconfig_filename (str): Path to the XML run-configuration file.
75
+ date_UTC (Union[date, str], optional): The target UTC date for product generation.
76
+ If None, it's derived from the input L2T LSTE granule.
77
+ spinup_days (int, optional): Number of days for the VIIRS time-series spin-up.
78
+ Defaults to DEFAULT_SPINUP_DAYS (7).
79
+ target_resolution (int, optional): The desired output resolution in meters.
80
+ Defaults to DEFAULT_TARGET_RESOLUTION (70).
81
+ NDVI_resolution (int, optional): The resolution of the coarse NDVI data.
82
+ Defaults to DEFAULT_NDVI_RESOLUTION (490).
83
+ albedo_resolution (int, optional): The resolution of the coarse albedo data.
84
+ Defaults to DEFAULT_ALBEDO_RESOLUTION (980).
85
+ use_VNP43NRT (bool, optional): If True, use VNP43NRT for VIIRS products.
86
+ If False, use VNP43IA4 (NDVI) and VNP43MA3 (Albedo).
87
+ Defaults to DEFAULT_USE_VNP43NRT (True).
88
+ calibrate_fine (bool, optional): If True, calibrate fine resolution HLS data to
89
+ coarse resolution VIIRS data. Defaults to DEFAULT_CALIBRATE_FINE (False).
90
+ sources_only (bool, optional): If True, only retrieve source data and exit,
91
+ without performing data fusion. Defaults to False.
92
+ remove_input_staging (bool, optional): If True, remove the input staging directory
93
+ after processing. Defaults to True.
94
+ remove_prior (bool, optional): If True, remove prior intermediate files after use.
95
+ Defaults to True.
96
+ remove_posterior (bool, optional): If True, remove posterior intermediate files after
97
+ product generation. Defaults to True.
98
+ threads (Union[int, str], optional): Number of Julia threads to use, or "auto".
99
+ Defaults to "auto".
100
+ num_workers (int, optional): Number of Julia workers for distributed processing.
101
+ Defaults to 4.
102
+ overwrite (bool, optional): If True, existing output files will be overwritten.
103
+ Defaults to False.
104
+
105
+ Returns:
106
+ int: An exit code indicating the success or failure of the PGE execution.
107
+ (e.g., SUCCESS_EXIT_CODE, AUXILIARY_SERVER_UNREACHABLE, DOWNLOAD_FAILED, etc.)
108
+ """
109
+ exit_code = SUCCESS_EXIT_CODE # Initialize exit code to success
110
+
111
+ try:
112
+ # Load and parse the run-configuration file
113
+ runconfig = L2TSTARSConfig(runconfig_filename)
114
+
115
+ # Configure logging with the specified log filename from runconfig
116
+ working_directory = runconfig.working_directory
117
+ granule_ID = runconfig.granule_ID
118
+ log_filename = join(working_directory, "log", f"{granule_ID}.log")
119
+ cl.configure(filename=log_filename) # Reconfigure logger with the specific log file
120
+
121
+ logger.info(f"L2T_STARS PGE ({cl.val(__version__)})")
122
+ logger.info(f"L2T_STARS run-config: {cl.file(runconfig_filename)}")
123
+ logger.info(f"Granule ID: {cl.val(granule_ID)}")
124
+
125
+ # Extract paths from the run-config
126
+ L2T_STARS_granule_directory = runconfig.L2T_STARS_granule_directory
127
+ logger.info(f"Granule directory: {cl.dir(L2T_STARS_granule_directory)}")
128
+ L2T_STARS_zip_filename = runconfig.L2T_STARS_zip_filename
129
+ logger.info(f"Zip filename: {cl.file(L2T_STARS_zip_filename)}")
130
+ L2T_STARS_browse_filename = runconfig.L2T_STARS_browse_filename
131
+ logger.info(f"Browse filename: " + cl.file(L2T_STARS_browse_filename))
132
+
133
+ # Check if the final product already exists and 'overwrite' is not enabled
134
+ if not overwrite and exists(L2T_STARS_zip_filename) and exists(L2T_STARS_browse_filename):
135
+ logger.info(f"Found existing L2T STARS file: {L2T_STARS_zip_filename}")
136
+ logger.info(f"Found existing L2T STARS preview: {L2T_STARS_browse_filename}")
137
+ logger.info("Overwrite option is not enabled, skipping reprocessing.")
138
+ return SUCCESS_EXIT_CODE
139
+ elif overwrite and exists(L2T_STARS_zip_filename) and exists(L2T_STARS_browse_filename):
140
+ logger.info(f"Found existing L2T STARS file: {L2T_STARS_zip_filename}")
141
+ logger.info(f"Found existing L2T STARS preview: {L2T_STARS_browse_filename}")
142
+ logger.info("Overwrite option is enabled, proceeding with reprocessing.")
143
+
144
+
145
+ logger.info(f"Working directory: {cl.dir(working_directory)}")
146
+ logger.info(f"Log file: {cl.file(log_filename)}")
147
+
148
+ input_staging_directory = join(working_directory, "input_staging")
149
+ logger.info(f"Input staging directory: {cl.dir(input_staging_directory)}")
150
+
151
+ sources_directory = runconfig.sources_directory
152
+ logger.info(f"Source directory: {cl.dir(sources_directory)}")
153
+ indices_directory = runconfig.indices_directory
154
+ logger.info(f"Indices directory: {cl.dir(indices_directory)}")
155
+ model_directory = runconfig.model_directory
156
+ logger.info(f"Model directory: {cl.dir(model_directory)}")
157
+ output_directory = runconfig.output_directory
158
+ logger.info(f"Output directory: {cl.dir(output_directory)}")
159
+ tile = runconfig.tile
160
+ logger.info(f"Tile: {cl.val(tile)}")
161
+ build = runconfig.build
162
+ logger.info(f"Build: {cl.val(build)}")
163
+ product_counter = runconfig.product_counter
164
+ logger.info(f"Product counter: {cl.val(product_counter)}")
165
+ L2T_LSTE_filename = runconfig.L2T_LSTE_filename
166
+ logger.info(f"Input L2T LSTE file: {cl.file(L2T_LSTE_filename)}")
167
+
168
+ # Validate existence of input L2T LSTE file
169
+ if not exists(L2T_LSTE_filename):
170
+ raise InputFilesInaccessible(
171
+ f"L2T LSTE file does not exist: {L2T_LSTE_filename}"
172
+ )
173
+
174
+ # Load the L2T_LSTE granule to get geometry and base metadata
175
+ l2t_granule = L2TLSTE(L2T_LSTE_filename)
176
+ geometry = l2t_granule.geometry
177
+ metadata = l2t_granule.metadata_dict
178
+ metadata["StandardMetadata"]["PGEName"] = "L2T_STARS"
179
+
180
+ # Update product names in metadata
181
+ short_name = L2T_STARS_SHORT_NAME
182
+ logger.info(f"L2T STARS short name: {cl.val(short_name)}")
183
+ metadata["StandardMetadata"]["ShortName"] = short_name
184
+
185
+ long_name = L2T_STARS_LONG_NAME
186
+ logger.info(f"L2T STARS long name: {cl.val(long_name)}")
187
+ metadata["StandardMetadata"]["LongName"] = long_name
188
+
189
+ # Update auxiliary input pointers in metadata and remove irrelevant sections
190
+ metadata["StandardMetadata"]["AuxiliaryInputPointer"] = "HLS,VIIRS"
191
+ if "ProductMetadata" in metadata:
192
+ metadata["ProductMetadata"].pop("AuxiliaryNWP", None) # Safe removal
193
+ metadata["ProductMetadata"].pop("NWPSource", None)
194
+
195
+ # Determine the target date for processing
196
+ time_UTC = l2t_granule.time_UTC
197
+ logger.info(f"ECOSTRESS overpass time: {cl.time(f'{time_UTC:%Y-%m-%d %H:%M:%S} UTC')}")
198
+
199
+ if date_UTC is None:
200
+ # Use date from L2T granule if not provided via command line
201
+ date_UTC = l2t_granule.date_UTC
202
+ logger.info(f"ECOSTRESS overpass date: {cl.time(f'{date_UTC:%Y-%m-%d} UTC')}")
203
+ else:
204
+ logger.warning(f"Over-riding target date from command line to: {date_UTC}")
205
+ if isinstance(date_UTC, str):
206
+ date_UTC = parser.parse(date_UTC).date()
207
+
208
+ # TODO: Add a check if the L2T LSTE granule is day-time and halt L2T STARS run if it's not.
209
+ # This is a critical step to ensure valid scientific output.
210
+
211
+ # Load prior data if specified in the run-config
212
+ L2T_STARS_prior_filename = runconfig.L2T_STARS_prior_filename
213
+ prior = load_prior(
214
+ tile=tile,
215
+ target_resolution=target_resolution,
216
+ model_directory=model_directory,
217
+ L2T_STARS_prior_filename=L2T_STARS_prior_filename,
218
+ )
219
+ using_prior = prior.using_prior
220
+ prior_date_UTC = prior.prior_date_UTC
221
+
222
+ # Define various product and download directories
223
+ products_directory = join(working_directory, DEFAULT_STARS_PRODUCTS_DIRECTORY)
224
+ logger.info(f"STARS products directory: {cl.dir(products_directory)}")
225
+ HLS_download_directory = join(sources_directory, DEFAULT_HLS_DOWNLOAD_DIRECTORY)
226
+ logger.info(f"HLS download directory: {cl.dir(HLS_download_directory)}")
227
+ HLS_products_directory = join(sources_directory, DEFAULT_HLS_PRODUCTS_DIRECTORY)
228
+ logger.info(f"HLS products directory: {cl.dir(HLS_products_directory)}")
229
+ VIIRS_download_directory = join(sources_directory, DEFAULT_VIIRS_DOWNLOAD_DIRECTORY)
230
+ logger.info(f"VIIRS download directory: {cl.dir(VIIRS_download_directory)}")
231
+ VIIRS_products_directory = join(sources_directory, DEFAULT_VIIRS_PRODUCTS_DIRECTORY)
232
+ logger.info(f"VIIRS products directory: {cl.dir(VIIRS_products_directory)}")
233
+ VIIRS_mosaic_directory = join(sources_directory, DEFAUL_VIIRS_MOSAIC_DIRECTORY)
234
+ logger.info(f"VIIRS mosaic directory: {cl.dir(VIIRS_mosaic_directory)}")
235
+ GEOS5FP_download_directory = join(sources_directory, DEFAULT_GEOS5FP_DOWNLOAD_DIRECTORY)
236
+ logger.info(f"GEOS-5 FP download directory: {cl.dir(GEOS5FP_download_directory)}")
237
+ GEOS5FP_products_directory = join(sources_directory, DEFAULT_GEOS5FP_PRODUCTS_DIRECTORY)
238
+ logger.info(f"GEOS-5 FP products directory: {cl.dir(GEOS5FP_products_directory)}")
239
+ VNP09GA_products_directory = join(sources_directory, DEFAULT_VNP09GA_PRODUCTS_DIRECTORY)
240
+ logger.info(f"VNP09GA products directory: {cl.dir(VNP09GA_products_directory)}")
241
+ VNP43NRT_products_directory = join(sources_directory, DEFAULT_VNP43NRT_PRODUCTS_DIRECTORY)
242
+ logger.info(f"VNP43NRT products directory: {cl.dir(VNP43NRT_products_directory)}")
243
+
244
+ # Re-check for existing product (double-check in case another process created it) with overwrite option
245
+ if not overwrite and exists(L2T_STARS_zip_filename):
246
+ logger.info(
247
+ f"Found L2T STARS product zip: {cl.file(L2T_STARS_zip_filename)}. Overwrite is False, returning."
248
+ )
249
+ return exit_code
250
+ elif overwrite and exists(L2T_STARS_zip_filename):
251
+ logger.info(
252
+ f"Found L2T STARS product zip: {cl.file(L2T_STARS_zip_filename)}. Overwrite is True, proceeding."
253
+ )
254
+
255
+
256
+ # Initialize HLS data connection
257
+ logger.info(f"Connecting to CMR Search server: {CMR_SEARCH_URL}")
258
+ try:
259
+ HLS_connection = HLS2CMR(
260
+ working_directory=working_directory,
261
+ download_directory=HLS_download_directory,
262
+ products_directory=HLS_products_directory,
263
+ target_resolution=target_resolution,
264
+ )
265
+ except CMRServerUnreachable as e:
266
+ logger.exception(e)
267
+ raise AuxiliaryServerUnreachable(
268
+ f"Unable to connect to CMR Search server: {CMR_SEARCH_URL}"
269
+ )
270
+
271
+ # Check if the tile is on land (HLS tiles cover land and ocean, STARS is for land)
272
+ if not HLS_connection.tile_grid.land(tile=tile):
273
+ raise LandFilter(f"Sentinel tile {tile} is not on land. Skipping processing.")
274
+
275
+ # Initialize VIIRS data connections based on 'use_VNP43NRT' flag
276
+ if use_VNP43NRT:
277
+ try:
278
+ NDVI_VIIRS_connection = VNP43NRT(
279
+ working_directory=working_directory,
280
+ download_directory=VIIRS_download_directory,
281
+ mosaic_directory=VIIRS_mosaic_directory,
282
+ GEOS5FP_download=GEOS5FP_download_directory,
283
+ GEOS5FP_products=GEOS5FP_products_directory,
284
+ VNP09GA_directory=VNP09GA_products_directory,
285
+ VNP43NRT_directory=VNP43NRT_products_directory,
286
+ )
287
+
288
+ albedo_VIIRS_connection = VNP43NRT(
289
+ working_directory=working_directory,
290
+ download_directory=VIIRS_download_directory,
291
+ mosaic_directory=VIIRS_mosaic_directory,
292
+ GEOS5FP_download=GEOS5FP_download_directory,
293
+ GEOS5FP_products=GEOS5FP_products_directory,
294
+ VNP09GA_directory=VNP09GA_products_directory,
295
+ VNP43NRT_directory=VNP43NRT_products_directory,
296
+ )
297
+ except CMRServerUnreachable as e:
298
+ logger.exception(e)
299
+ raise AuxiliaryServerUnreachable(f"Unable to connect to CMR search server for VNP43NRT.")
300
+ else:
301
+ try:
302
+ NDVI_VIIRS_connection = VNP43IA4(
303
+ working_directory=working_directory,
304
+ download_directory=VIIRS_download_directory,
305
+ products_directory=VIIRS_products_directory,
306
+ mosaic_directory=VIIRS_mosaic_directory,
307
+ )
308
+
309
+ albedo_VIIRS_connection = VNP43MA3(
310
+ working_directory=working_directory,
311
+ download_directory=VIIRS_download_directory,
312
+ products_directory=VIIRS_products_directory,
313
+ mosaic_directory=VIIRS_mosaic_directory,
314
+ )
315
+ except LPDAACServerUnreachable as e:
316
+ logger.exception(e)
317
+ raise AuxiliaryServerUnreachable(f"Unable to connect to VIIRS LPDAAC server.")
318
+
319
+ # Define date ranges for data retrieval and fusion
320
+ end_date = date_UTC
321
+ # The start date of the BRDF-corrected VIIRS coarse time-series is 'spinup_days' before the target date
322
+ VIIRS_start_date = end_date - timedelta(days=spinup_days)
323
+ # To produce that first BRDF-corrected image, VNP09GA (raw VIIRS) is needed starting 16 days prior to the first coarse date
324
+ VIIRS_download_start_date = VIIRS_start_date - timedelta(days=16)
325
+ VIIRS_end_date = end_date
326
+
327
+ # Define start date of HLS fine image input time-series
328
+ if using_prior and prior_date_UTC and prior_date_UTC >= VIIRS_start_date:
329
+ # If a valid prior is used and its date is within or before the VIIRS start date,
330
+ # HLS inputs begin the day after the prior
331
+ HLS_start_date = prior_date_UTC + timedelta(days=1)
332
+ else:
333
+ # If no prior or prior is too old, HLS inputs begin on the same day as the VIIRS inputs
334
+ HLS_start_date = VIIRS_start_date
335
+ HLS_end_date = end_date # HLS end date is always the same as the target date
336
+
337
+ logger.info(
338
+ f"Processing STARS HLS-VIIRS NDVI and albedo for tile {cl.place(tile)} from "
339
+ f"{cl.time(VIIRS_start_date)} to {cl.time(end_date)}"
340
+ )
341
+
342
+ # Get HLS listing to check for data availability
343
+ try:
344
+ HLS_listing = HLS_connection.listing(
345
+ tile=tile, start_UTC=HLS_start_date, end_UTC=HLS_end_date
346
+ )
347
+ except HLSTileNotAvailable as e:
348
+ logger.exception(e)
349
+ raise LandFilter(f"Sentinel tile {tile} cannot be processed due to HLS tile unavailability.")
350
+ except Exception as e:
351
+ logger.exception(e)
352
+ raise AuxiliaryServerUnreachable(
353
+ f"Unable to scan Harmonized Landsat Sentinel server: {HLS_connection.remote}"
354
+ )
355
+
356
+ # Check for missing HLS Sentinel data
357
+ missing_sentinel_dates = HLS_listing[HLS_listing.sentinel == "missing"].date_UTC
358
+ if len(missing_sentinel_dates) > 0:
359
+ raise AuxiliaryLatency(
360
+ f"HLS Sentinel is not yet available at tile {tile} for dates: "
361
+ f"{', '.join(missing_sentinel_dates.dt.strftime('%Y-%m-%d'))}"
362
+ )
363
+
364
+ # Log available HLS Sentinel data
365
+ sentinel_listing = HLS_listing[~pd.isna(HLS_listing.sentinel)][
366
+ ["date_UTC", "sentinel"]
367
+ ]
368
+ logger.info(f"HLS Sentinel is available on {cl.val(len(sentinel_listing))} dates:")
369
+ for i, (list_date_utc, sentinel_granule) in sentinel_listing.iterrows():
370
+ sentinel_filename = sentinel_granule["meta"]["native-id"]
371
+ logger.info(f"* {cl.time(list_date_utc)}: {cl.file(sentinel_filename)}")
372
+
373
+ # Check for missing HLS Landsat data
374
+ missing_landsat_dates = HLS_listing[HLS_listing.landsat == "missing"].date_UTC
375
+ if len(missing_landsat_dates) > 0:
376
+ raise AuxiliaryLatency(
377
+ f"HLS Landsat is not yet available at tile {tile} for dates: "
378
+ f"{', '.join(missing_landsat_dates.dt.strftime('%Y-%m-%d'))}"
379
+ )
380
+
381
+ # Log available HLS Landsat data
382
+ landsat_listing = HLS_listing[~pd.isna(HLS_listing.landsat)][
383
+ ["date_UTC", "landsat"]
384
+ ]
385
+ logger.info(f"HLS Landsat is available on {cl.val(len(landsat_listing))} dates:")
386
+ for i, (list_date_utc, landsat_granule) in landsat_listing.iterrows():
387
+ landsat_filename = landsat_granule["meta"]["native-id"]
388
+ logger.info(f"* {cl.time(list_date_utc)}: {cl.file(landsat_filename)}")
389
+
390
+ # If only sources are requested, retrieve them and exit
391
+ if sources_only:
392
+ logger.info("Sources only flag enabled. Retrieving source data.")
393
+ retrieve_STARS_sources(
394
+ tile=tile,
395
+ geometry=geometry,
396
+ HLS_start_date=HLS_start_date,
397
+ HLS_end_date=HLS_end_date,
398
+ VIIRS_start_date=VIIRS_download_start_date,
399
+ VIIRS_end_date=VIIRS_end_date,
400
+ HLS_connection=HLS_connection,
401
+ VIIRS_connection=NDVI_VIIRS_connection, # Use NDVI_VIIRS_connection as a general VIIRS connection
402
+ )
403
+ # Regenerate inputs to ensure all files are staged, even if not fused
404
+ NDVI_coarse_geometry = HLS_connection.grid(tile=tile, cell_size=NDVI_resolution)
405
+ albedo_coarse_geometry = HLS_connection.grid(tile=tile, cell_size=albedo_resolution)
406
+
407
+ NDVI_coarse_directory = generate_NDVI_coarse_directory(
408
+ input_staging_directory=input_staging_directory, tile=tile
409
+ )
410
+ NDVI_fine_directory = generate_NDVI_fine_directory(
411
+ input_staging_directory=input_staging_directory, tile=tile
412
+ )
413
+ albedo_coarse_directory = generate_albedo_coarse_directory(
414
+ input_staging_directory=input_staging_directory, tile=tile
415
+ )
416
+ albedo_fine_directory = generate_albedo_fine_directory(
417
+ input_staging_directory=input_staging_directory, tile=tile
418
+ )
419
+
420
+ generate_STARS_inputs(
421
+ tile=tile,
422
+ date_UTC=date_UTC,
423
+ HLS_start_date=HLS_start_date,
424
+ HLS_end_date=HLS_end_date,
425
+ VIIRS_start_date=VIIRS_start_date,
426
+ VIIRS_end_date=VIIRS_end_date,
427
+ NDVI_resolution=NDVI_resolution,
428
+ albedo_resolution=albedo_resolution,
429
+ target_resolution=target_resolution,
430
+ NDVI_coarse_geometry=NDVI_coarse_geometry,
431
+ albedo_coarse_geometry=albedo_coarse_geometry,
432
+ working_directory=working_directory,
433
+ NDVI_coarse_directory=NDVI_coarse_directory,
434
+ NDVI_fine_directory=NDVI_fine_directory,
435
+ albedo_coarse_directory=albedo_coarse_directory,
436
+ albedo_fine_directory=albedo_fine_directory,
437
+ HLS_connection=HLS_connection,
438
+ NDVI_VIIRS_connection=NDVI_VIIRS_connection,
439
+ albedo_VIIRS_connection=albedo_VIIRS_connection,
440
+ calibrate_fine=calibrate_fine,
441
+ )
442
+ else:
443
+ # Otherwise, proceed with full product processing
444
+ process_STARS_product(
445
+ tile=tile,
446
+ date_UTC=date_UTC,
447
+ time_UTC=time_UTC,
448
+ build=build,
449
+ product_counter=product_counter,
450
+ HLS_start_date=HLS_start_date,
451
+ HLS_end_date=HLS_end_date,
452
+ VIIRS_start_date=VIIRS_start_date,
453
+ VIIRS_end_date=VIIRS_end_date,
454
+ NDVI_resolution=NDVI_resolution,
455
+ albedo_resolution=albedo_resolution,
456
+ target_resolution=target_resolution,
457
+ working_directory=working_directory,
458
+ model_directory=model_directory,
459
+ input_staging_directory=input_staging_directory,
460
+ L2T_STARS_granule_directory=L2T_STARS_granule_directory,
461
+ L2T_STARS_zip_filename=L2T_STARS_zip_filename,
462
+ L2T_STARS_browse_filename=L2T_STARS_browse_filename,
463
+ metadata=metadata,
464
+ prior=prior,
465
+ HLS_connection=HLS_connection,
466
+ NDVI_VIIRS_connection=NDVI_VIIRS_connection,
467
+ albedo_VIIRS_connection=albedo_VIIRS_connection,
468
+ using_prior=using_prior,
469
+ calibrate_fine=calibrate_fine,
470
+ remove_input_staging=remove_input_staging,
471
+ remove_prior=remove_prior,
472
+ remove_posterior=remove_posterior,
473
+ threads=threads,
474
+ num_workers=num_workers,
475
+ )
476
+
477
+ # --- Exception Handling for PGE ---
478
+ except (ConnectionError, urllib.error.HTTPError, CMRServerUnreachable) as exception:
479
+ logger.exception(exception)
480
+ exit_code = AUXILIARY_SERVER_UNREACHABLE
481
+ except DownloadFailed as exception:
482
+ logger.exception(exception)
483
+ exit_code = DOWNLOAD_FAILED
484
+ except HLSBandNotAcquired as exception:
485
+ logger.exception(exception)
486
+ exit_code = DOWNLOAD_FAILED
487
+ except HLSNotAvailable as exception:
488
+ logger.exception(exception)
489
+ exit_code = LAND_FILTER # This might indicate no HLS data for the tile, similar to land filter
490
+ except (HLSSentinelMissing, HLSLandsatMissing) as exception:
491
+ logger.exception(exception)
492
+ exit_code = AUXILIARY_LATENCY
493
+ except ECOSTRESSExitCodeException as exception:
494
+ # Catch custom ECOSTRESS exceptions and use their defined exit code
495
+ logger.exception(exception)
496
+ exit_code = exception.exit_code
497
+ except Exception as exception:
498
+ # Catch any other unexpected exceptions
499
+ logger.exception(exception)
500
+ exit_code = UNCLASSIFIED_FAILURE_EXIT_CODE
501
+
502
+ logger.info(f"L2T_STARS exit code: {exit_code}")
503
+ return exit_code