ECOv003-L2T-STARS 1.0.1__py3-none-any.whl → 1.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ECOv003_L2T_STARS/BRDF/BRDF.py +57 -0
- ECOv003_L2T_STARS/BRDF/SZA.py +65 -0
- ECOv003_L2T_STARS/BRDF/__init__.py +1 -0
- ECOv003_L2T_STARS/BRDF/statistical_radiative_transport.txt +90 -0
- ECOv003_L2T_STARS/BRDF/version.txt +1 -0
- ECOv003_L2T_STARS/ECOv003_DL.py +527 -0
- ECOv003_L2T_STARS/ECOv003_DL.xml +47 -0
- ECOv003_L2T_STARS/ECOv003_L2T_STARS.py +162 -0
- ECOv003_L2T_STARS/ECOv003_L2T_STARS.xml +47 -0
- ECOv003_L2T_STARS/L2TSTARSConfig.py +188 -0
- ECOv003_L2T_STARS/L2T_STARS.py +489 -0
- ECOv003_L2T_STARS/LPDAAC/LPDAACDataPool.py +444 -0
- ECOv003_L2T_STARS/LPDAAC/__init__.py +9 -0
- ECOv003_L2T_STARS/LPDAAC/version.txt +1 -0
- ECOv003_L2T_STARS/Manifest.toml +2332 -0
- ECOv003_L2T_STARS/Project.toml +14 -0
- ECOv003_L2T_STARS/VIIRS/VIIRSDataPool.py +294 -0
- ECOv003_L2T_STARS/VIIRS/VIIRSDownloader.py +26 -0
- ECOv003_L2T_STARS/VIIRS/VIIRS_CMR_LOGIN.py +36 -0
- ECOv003_L2T_STARS/VIIRS/VNP09GA.py +1277 -0
- ECOv003_L2T_STARS/VIIRS/VNP43IA4.py +288 -0
- ECOv003_L2T_STARS/VIIRS/VNP43MA3.py +323 -0
- ECOv003_L2T_STARS/VIIRS/__init__.py +9 -0
- ECOv003_L2T_STARS/VIIRS/version.txt +1 -0
- ECOv003_L2T_STARS/VNP43NRT/VNP43NRT.py +863 -0
- ECOv003_L2T_STARS/VNP43NRT/__init__.py +1 -0
- ECOv003_L2T_STARS/VNP43NRT/process_VNP43NRT.jl +169 -0
- ECOv003_L2T_STARS/VNP43NRT/version.txt +1 -0
- ECOv003_L2T_STARS/VNP43NRT_jl/Manifest.toml +995 -0
- ECOv003_L2T_STARS/VNP43NRT_jl/Project.toml +15 -0
- ECOv003_L2T_STARS/VNP43NRT_jl/__init__.py +0 -0
- ECOv003_L2T_STARS/VNP43NRT_jl/instantiate.jl +25 -0
- ECOv003_L2T_STARS/VNP43NRT_jl/instantiate.py +13 -0
- ECOv003_L2T_STARS/VNP43NRT_jl/src/VNP43NRT.jl +411 -0
- ECOv003_L2T_STARS/VNP43NRT_jl/src/__init__.py +0 -0
- ECOv003_L2T_STARS/__init__.py +3 -0
- ECOv003_L2T_STARS/calibrate_fine_to_coarse.py +60 -0
- ECOv003_L2T_STARS/constants.py +38 -0
- ECOv003_L2T_STARS/daterange/__init__.py +1 -0
- ECOv003_L2T_STARS/daterange/daterange.py +35 -0
- ECOv003_L2T_STARS/generate_L2T_STARS_runconfig.py +249 -0
- ECOv003_L2T_STARS/generate_NDVI_coarse_directory.py +21 -0
- ECOv003_L2T_STARS/generate_NDVI_coarse_image.py +30 -0
- ECOv003_L2T_STARS/generate_NDVI_fine_directory.py +14 -0
- ECOv003_L2T_STARS/generate_NDVI_fine_image.py +28 -0
- ECOv003_L2T_STARS/generate_STARS_inputs.py +231 -0
- ECOv003_L2T_STARS/generate_albedo_coarse_directory.py +18 -0
- ECOv003_L2T_STARS/generate_albedo_coarse_image.py +30 -0
- ECOv003_L2T_STARS/generate_albedo_fine_directory.py +17 -0
- ECOv003_L2T_STARS/generate_albedo_fine_image.py +30 -0
- ECOv003_L2T_STARS/generate_filename.py +37 -0
- ECOv003_L2T_STARS/generate_input_staging_directory.py +23 -0
- ECOv003_L2T_STARS/generate_model_state_tile_date_directory.py +28 -0
- ECOv003_L2T_STARS/generate_output_directory.py +28 -0
- ECOv003_L2T_STARS/install_STARS_jl.py +43 -0
- ECOv003_L2T_STARS/instantiate_STARS_jl.py +38 -0
- ECOv003_L2T_STARS/load_prior.py +248 -0
- ECOv003_L2T_STARS/prior.py +56 -0
- ECOv003_L2T_STARS/process_ECOSTRESS_data_fusion_distributed_bias.jl +420 -0
- ECOv003_L2T_STARS/process_STARS_product.py +507 -0
- ECOv003_L2T_STARS/process_julia_data_fusion.py +110 -0
- ECOv003_L2T_STARS/retrieve_STARS_sources.py +101 -0
- ECOv003_L2T_STARS/runconfig.py +70 -0
- ECOv003_L2T_STARS/timer/__init__.py +1 -0
- ECOv003_L2T_STARS/timer/timer.py +77 -0
- ECOv003_L2T_STARS/version.py +8 -0
- ECOv003_L2T_STARS/version.txt +1 -0
- {ECOv003_L2T_STARS-1.0.1.dist-info → ecov003_l2t_stars-1.1.0.dist-info}/METADATA +30 -23
- ecov003_l2t_stars-1.1.0.dist-info/RECORD +73 -0
- {ECOv003_L2T_STARS-1.0.1.dist-info → ecov003_l2t_stars-1.1.0.dist-info}/WHEEL +1 -1
- ecov003_l2t_stars-1.1.0.dist-info/entry_points.txt +3 -0
- ecov003_l2t_stars-1.1.0.dist-info/top_level.txt +1 -0
- ECOv003_L2T_STARS-1.0.1.dist-info/RECORD +0 -5
- ECOv003_L2T_STARS-1.0.1.dist-info/top_level.txt +0 -1
- {ECOv003_L2T_STARS-1.0.1.dist-info → ecov003_l2t_stars-1.1.0.dist-info/licenses}/LICENSE +0 -0
@@ -0,0 +1,489 @@
|
|
1
|
+
import argparse
|
2
|
+
import logging
|
3
|
+
import sys
|
4
|
+
from datetime import date, timedelta
|
5
|
+
from os import makedirs
|
6
|
+
from os.path import join, exists
|
7
|
+
from typing import Union
|
8
|
+
import logging
|
9
|
+
import colored_logging as cl
|
10
|
+
import pandas as pd
|
11
|
+
from dateutil import parser
|
12
|
+
|
13
|
+
# Custom modules for Harmonized Landsat Sentinel (HLS) and ECOSTRESS data
|
14
|
+
from harmonized_landsat_sentinel import (
|
15
|
+
CMRServerUnreachable,
|
16
|
+
HLS2CMR,
|
17
|
+
HLSTileNotAvailable,
|
18
|
+
HLSSentinelMissing,
|
19
|
+
HLSLandsatMissing,
|
20
|
+
HLSNotAvailable,
|
21
|
+
HLSBandNotAcquired,
|
22
|
+
CMR_SEARCH_URL
|
23
|
+
)
|
24
|
+
|
25
|
+
from ECOv003_exit_codes import *
|
26
|
+
|
27
|
+
from ECOv002_granules import L2TLSTE
|
28
|
+
import urllib
|
29
|
+
|
30
|
+
from .version import __version__
|
31
|
+
from .constants import *
|
32
|
+
from .VIIRS.VNP43IA4 import VNP43IA4
|
33
|
+
from .VIIRS.VNP43MA3 import VNP43MA3
|
34
|
+
from .VNP43NRT import VNP43NRT
|
35
|
+
from .runconfig import ECOSTRESSRunConfig
|
36
|
+
from .L2TSTARSConfig import L2TSTARSConfig
|
37
|
+
from .load_prior import load_prior
|
38
|
+
from .generate_NDVI_coarse_directory import generate_NDVI_coarse_directory
|
39
|
+
from .generate_NDVI_fine_directory import generate_NDVI_fine_directory
|
40
|
+
from .generate_albedo_coarse_directory import generate_albedo_coarse_directory
|
41
|
+
from .generate_albedo_fine_directory import generate_albedo_fine_directory
|
42
|
+
from .generate_STARS_inputs import generate_STARS_inputs
|
43
|
+
from .process_STARS_product import process_STARS_product
|
44
|
+
from .retrieve_STARS_sources import retrieve_STARS_sources
|
45
|
+
|
46
|
+
logger = logging.getLogger(__name__)
|
47
|
+
|
48
|
+
def L2T_STARS(
|
49
|
+
runconfig_filename: str,
|
50
|
+
date_UTC: Union[date, str] = None,
|
51
|
+
spinup_days: int = DEFAULT_SPINUP_DAYS,
|
52
|
+
target_resolution: int = DEFAULT_TARGET_RESOLUTION,
|
53
|
+
NDVI_resolution: int = DEFAULT_NDVI_RESOLUTION,
|
54
|
+
albedo_resolution: int = DEFAULT_ALBEDO_RESOLUTION,
|
55
|
+
use_VNP43NRT: bool = DEFAULT_USE_VNP43NRT,
|
56
|
+
calibrate_fine: bool = DEFAULT_CALIBRATE_FINE,
|
57
|
+
sources_only: bool = False,
|
58
|
+
remove_input_staging: bool = True,
|
59
|
+
remove_prior: bool = True,
|
60
|
+
remove_posterior: bool = True,
|
61
|
+
threads: Union[int, str] = "auto",
|
62
|
+
num_workers: int = 4,
|
63
|
+
) -> int:
|
64
|
+
"""
|
65
|
+
ECOSTRESS Collection 3 L2T_STARS PGE (Product Generation Executive).
|
66
|
+
|
67
|
+
This function serves as the main entry point for the L2T_STARS processing.
|
68
|
+
It orchestrates the entire workflow, including reading the run-config,
|
69
|
+
connecting to data servers, retrieving source data, performing data fusion
|
70
|
+
(via Julia subprocess), generating the final product, and handling cleanup.
|
71
|
+
|
72
|
+
Args:
|
73
|
+
runconfig_filename (str): Path to the XML run-configuration file.
|
74
|
+
date_UTC (Union[date, str], optional): The target UTC date for product generation.
|
75
|
+
If None, it's derived from the input L2T LSTE granule.
|
76
|
+
spinup_days (int, optional): Number of days for the VIIRS time-series spin-up.
|
77
|
+
Defaults to DEFAULT_SPINUP_DAYS (7).
|
78
|
+
target_resolution (int, optional): The desired output resolution in meters.
|
79
|
+
Defaults to DEFAULT_TARGET_RESOLUTION (70).
|
80
|
+
NDVI_resolution (int, optional): The resolution of the coarse NDVI data.
|
81
|
+
Defaults to DEFAULT_NDVI_RESOLUTION (490).
|
82
|
+
albedo_resolution (int, optional): The resolution of the coarse albedo data.
|
83
|
+
Defaults to DEFAULT_ALBEDO_RESOLUTION (980).
|
84
|
+
use_VNP43NRT (bool, optional): If True, use VNP43NRT for VIIRS products.
|
85
|
+
If False, use VNP43IA4 (NDVI) and VNP43MA3 (Albedo).
|
86
|
+
Defaults to DEFAULT_USE_VNP43NRT (True).
|
87
|
+
calibrate_fine (bool, optional): If True, calibrate fine resolution HLS data to
|
88
|
+
coarse resolution VIIRS data. Defaults to DEFAULT_CALIBRATE_FINE (False).
|
89
|
+
sources_only (bool, optional): If True, only retrieve source data and exit,
|
90
|
+
without performing data fusion. Defaults to False.
|
91
|
+
remove_input_staging (bool, optional): If True, remove the input staging directory
|
92
|
+
after processing. Defaults to True.
|
93
|
+
remove_prior (bool, optional): If True, remove prior intermediate files after use.
|
94
|
+
Defaults to True.
|
95
|
+
remove_posterior (bool, optional): If True, remove posterior intermediate files after
|
96
|
+
product generation. Defaults to True.
|
97
|
+
threads (Union[int, str], optional): Number of Julia threads to use, or "auto".
|
98
|
+
Defaults to "auto".
|
99
|
+
num_workers (int, optional): Number of Julia workers for distributed processing.
|
100
|
+
Defaults to 4.
|
101
|
+
|
102
|
+
Returns:
|
103
|
+
int: An exit code indicating the success or failure of the PGE execution.
|
104
|
+
(e.g., SUCCESS_EXIT_CODE, AUXILIARY_SERVER_UNREACHABLE, DOWNLOAD_FAILED, etc.)
|
105
|
+
"""
|
106
|
+
exit_code = SUCCESS_EXIT_CODE # Initialize exit code to success
|
107
|
+
|
108
|
+
try:
|
109
|
+
# Load and parse the run-configuration file
|
110
|
+
runconfig = L2TSTARSConfig(runconfig_filename)
|
111
|
+
|
112
|
+
# Configure logging with the specified log filename from runconfig
|
113
|
+
working_directory = runconfig.working_directory
|
114
|
+
granule_ID = runconfig.granule_ID
|
115
|
+
log_filename = join(working_directory, "log", f"{granule_ID}.log")
|
116
|
+
cl.configure(filename=log_filename) # Reconfigure logger with the specific log file
|
117
|
+
|
118
|
+
logger.info(f"L2T_STARS PGE ({cl.val(__version__)})")
|
119
|
+
logger.info(f"L2T_STARS run-config: {cl.file(runconfig_filename)}")
|
120
|
+
logger.info(f"Granule ID: {cl.val(granule_ID)}")
|
121
|
+
|
122
|
+
# Extract paths from the run-config
|
123
|
+
L2T_STARS_granule_directory = runconfig.L2T_STARS_granule_directory
|
124
|
+
logger.info(f"Granule directory: {cl.dir(L2T_STARS_granule_directory)}")
|
125
|
+
L2T_STARS_zip_filename = runconfig.L2T_STARS_zip_filename
|
126
|
+
logger.info(f"Zip filename: {cl.file(L2T_STARS_zip_filename)}")
|
127
|
+
L2T_STARS_browse_filename = runconfig.L2T_STARS_browse_filename
|
128
|
+
logger.info(f"Browse filename: " + cl.file(L2T_STARS_browse_filename))
|
129
|
+
|
130
|
+
# Check if the final product already exists to avoid reprocessing
|
131
|
+
if exists(L2T_STARS_zip_filename) and exists(L2T_STARS_browse_filename):
|
132
|
+
logger.info(f"Found existing L2T STARS file: {L2T_STARS_zip_filename}")
|
133
|
+
logger.info(f"Found existing L2T STARS preview: {L2T_STARS_browse_filename}")
|
134
|
+
return SUCCESS_EXIT_CODE
|
135
|
+
|
136
|
+
logger.info(f"Working directory: {cl.dir(working_directory)}")
|
137
|
+
logger.info(f"Log file: {cl.file(log_filename)}")
|
138
|
+
|
139
|
+
input_staging_directory = join(working_directory, "input_staging")
|
140
|
+
logger.info(f"Input staging directory: {cl.dir(input_staging_directory)}")
|
141
|
+
|
142
|
+
sources_directory = runconfig.sources_directory
|
143
|
+
logger.info(f"Source directory: {cl.dir(sources_directory)}")
|
144
|
+
indices_directory = runconfig.indices_directory
|
145
|
+
logger.info(f"Indices directory: {cl.dir(indices_directory)}")
|
146
|
+
model_directory = runconfig.model_directory
|
147
|
+
logger.info(f"Model directory: {cl.dir(model_directory)}")
|
148
|
+
output_directory = runconfig.output_directory
|
149
|
+
logger.info(f"Output directory: {cl.dir(output_directory)}")
|
150
|
+
tile = runconfig.tile
|
151
|
+
logger.info(f"Tile: {cl.val(tile)}")
|
152
|
+
build = runconfig.build
|
153
|
+
logger.info(f"Build: {cl.val(build)}")
|
154
|
+
product_counter = runconfig.product_counter
|
155
|
+
logger.info(f"Product counter: {cl.val(product_counter)}")
|
156
|
+
L2T_LSTE_filename = runconfig.L2T_LSTE_filename
|
157
|
+
logger.info(f"Input L2T LSTE file: {cl.file(L2T_LSTE_filename)}")
|
158
|
+
|
159
|
+
# Validate existence of input L2T LSTE file
|
160
|
+
if not exists(L2T_LSTE_filename):
|
161
|
+
raise InputFilesInaccessible(
|
162
|
+
f"L2T LSTE file does not exist: {L2T_LSTE_filename}"
|
163
|
+
)
|
164
|
+
|
165
|
+
# Load the L2T_LSTE granule to get geometry and base metadata
|
166
|
+
l2t_granule = L2TLSTE(L2T_LSTE_filename)
|
167
|
+
geometry = l2t_granule.geometry
|
168
|
+
metadata = l2t_granule.metadata_dict
|
169
|
+
metadata["StandardMetadata"]["PGEName"] = "L2T_STARS"
|
170
|
+
|
171
|
+
# Update product names in metadata
|
172
|
+
short_name = L2T_STARS_SHORT_NAME
|
173
|
+
logger.info(f"L2T STARS short name: {cl.val(short_name)}")
|
174
|
+
metadata["StandardMetadata"]["ShortName"] = short_name
|
175
|
+
|
176
|
+
long_name = L2T_STARS_LONG_NAME
|
177
|
+
logger.info(f"L2T STARS long name: {cl.val(long_name)}")
|
178
|
+
metadata["StandardMetadata"]["LongName"] = long_name
|
179
|
+
|
180
|
+
# Update auxiliary input pointers in metadata and remove irrelevant sections
|
181
|
+
metadata["StandardMetadata"]["AuxiliaryInputPointer"] = "HLS,VIIRS"
|
182
|
+
if "ProductMetadata" in metadata:
|
183
|
+
metadata["ProductMetadata"].pop("AuxiliaryNWP", None) # Safe removal
|
184
|
+
metadata["ProductMetadata"].pop("NWPSource", None)
|
185
|
+
|
186
|
+
# Determine the target date for processing
|
187
|
+
time_UTC = l2t_granule.time_UTC
|
188
|
+
logger.info(f"ECOSTRESS overpass time: {cl.time(f'{time_UTC:%Y-%m-%d %H:%M:%S} UTC')}")
|
189
|
+
|
190
|
+
if date_UTC is None:
|
191
|
+
# Use date from L2T granule if not provided via command line
|
192
|
+
date_UTC = l2t_granule.date_UTC
|
193
|
+
logger.info(f"ECOSTRESS overpass date: {cl.time(f'{date_UTC:%Y-%m-%d} UTC')}")
|
194
|
+
else:
|
195
|
+
logger.warning(f"Over-riding target date from command line to: {date_UTC}")
|
196
|
+
if isinstance(date_UTC, str):
|
197
|
+
date_UTC = parser.parse(date_UTC).date()
|
198
|
+
|
199
|
+
# TODO: Add a check if the L2T LSTE granule is day-time and halt L2T STARS run if it's not.
|
200
|
+
# This is a critical step to ensure valid scientific output.
|
201
|
+
|
202
|
+
# Load prior data if specified in the run-config
|
203
|
+
L2T_STARS_prior_filename = runconfig.L2T_STARS_prior_filename
|
204
|
+
prior = load_prior(
|
205
|
+
tile=tile,
|
206
|
+
target_resolution=target_resolution,
|
207
|
+
model_directory=model_directory,
|
208
|
+
L2T_STARS_prior_filename=L2T_STARS_prior_filename,
|
209
|
+
)
|
210
|
+
using_prior = prior.using_prior
|
211
|
+
prior_date_UTC = prior.prior_date_UTC
|
212
|
+
|
213
|
+
# Define various product and download directories
|
214
|
+
products_directory = join(working_directory, DEFAULT_STARS_PRODUCTS_DIRECTORY)
|
215
|
+
logger.info(f"STARS products directory: {cl.dir(products_directory)}")
|
216
|
+
HLS_download_directory = join(sources_directory, DEFAULT_HLS_DOWNLOAD_DIRECTORY)
|
217
|
+
logger.info(f"HLS download directory: {cl.dir(HLS_download_directory)}")
|
218
|
+
HLS_products_directory = join(sources_directory, DEFAULT_HLS_PRODUCTS_DIRECTORY)
|
219
|
+
logger.info(f"HLS products directory: {cl.dir(HLS_products_directory)}")
|
220
|
+
VIIRS_download_directory = join(sources_directory, DEFAULT_VIIRS_DOWNLOAD_DIRECTORY)
|
221
|
+
logger.info(f"VIIRS download directory: {cl.dir(VIIRS_download_directory)}")
|
222
|
+
VIIRS_products_directory = join(sources_directory, DEFAULT_VIIRS_PRODUCTS_DIRECTORY)
|
223
|
+
logger.info(f"VIIRS products directory: {cl.dir(VIIRS_products_directory)}")
|
224
|
+
VIIRS_mosaic_directory = join(sources_directory, DEFAUL_VIIRS_MOSAIC_DIRECTORY)
|
225
|
+
logger.info(f"VIIRS mosaic directory: {cl.dir(VIIRS_mosaic_directory)}")
|
226
|
+
GEOS5FP_download_directory = join(sources_directory, DEFAULT_GEOS5FP_DOWNLOAD_DIRECTORY)
|
227
|
+
logger.info(f"GEOS-5 FP download directory: {cl.dir(GEOS5FP_download_directory)}")
|
228
|
+
GEOS5FP_products_directory = join(sources_directory, DEFAULT_GEOS5FP_PRODUCTS_DIRECTORY)
|
229
|
+
logger.info(f"GEOS-5 FP products directory: {cl.dir(GEOS5FP_products_directory)}")
|
230
|
+
VNP09GA_products_directory = join(sources_directory, DEFAULT_VNP09GA_PRODUCTS_DIRECTORY)
|
231
|
+
logger.info(f"VNP09GA products directory: {cl.dir(VNP09GA_products_directory)}")
|
232
|
+
VNP43NRT_products_directory = join(sources_directory, DEFAULT_VNP43NRT_PRODUCTS_DIRECTORY)
|
233
|
+
logger.info(f"VNP43NRT products directory: {cl.dir(VNP43NRT_products_directory)}")
|
234
|
+
|
235
|
+
# Re-check for existing product (double-check in case another process created it)
|
236
|
+
if exists(L2T_STARS_zip_filename):
|
237
|
+
logger.info(
|
238
|
+
f"Found L2T STARS product zip: {cl.file(L2T_STARS_zip_filename)}"
|
239
|
+
)
|
240
|
+
return exit_code
|
241
|
+
|
242
|
+
# Initialize HLS data connection
|
243
|
+
logger.info(f"Connecting to CMR Search server: {CMR_SEARCH_URL}")
|
244
|
+
try:
|
245
|
+
HLS_connection = HLS2CMR(
|
246
|
+
working_directory=working_directory,
|
247
|
+
download_directory=HLS_download_directory,
|
248
|
+
products_directory=HLS_products_directory,
|
249
|
+
target_resolution=target_resolution,
|
250
|
+
)
|
251
|
+
except CMRServerUnreachable as e:
|
252
|
+
logger.exception(e)
|
253
|
+
raise AuxiliaryServerUnreachable(
|
254
|
+
f"Unable to connect to CMR Search server: {CMR_SEARCH_URL}"
|
255
|
+
)
|
256
|
+
|
257
|
+
# Check if the tile is on land (HLS tiles cover land and ocean, STARS is for land)
|
258
|
+
if not HLS_connection.tile_grid.land(tile=tile):
|
259
|
+
raise LandFilter(f"Sentinel tile {tile} is not on land. Skipping processing.")
|
260
|
+
|
261
|
+
# Initialize VIIRS data connections based on 'use_VNP43NRT' flag
|
262
|
+
if use_VNP43NRT:
|
263
|
+
try:
|
264
|
+
NDVI_VIIRS_connection = VNP43NRT(
|
265
|
+
working_directory=working_directory,
|
266
|
+
download_directory=VIIRS_download_directory,
|
267
|
+
mosaic_directory=VIIRS_mosaic_directory,
|
268
|
+
GEOS5FP_download=GEOS5FP_download_directory,
|
269
|
+
GEOS5FP_products=GEOS5FP_products_directory,
|
270
|
+
VNP09GA_directory=VNP09GA_products_directory,
|
271
|
+
VNP43NRT_directory=VNP43NRT_products_directory,
|
272
|
+
)
|
273
|
+
|
274
|
+
albedo_VIIRS_connection = VNP43NRT(
|
275
|
+
working_directory=working_directory,
|
276
|
+
download_directory=VIIRS_download_directory,
|
277
|
+
mosaic_directory=VIIRS_mosaic_directory,
|
278
|
+
GEOS5FP_download=GEOS5FP_download_directory,
|
279
|
+
GEOS5FP_products=GEOS5FP_products_directory,
|
280
|
+
VNP09GA_directory=VNP09GA_products_directory,
|
281
|
+
VNP43NRT_directory=VNP43NRT_products_directory,
|
282
|
+
)
|
283
|
+
except CMRServerUnreachable as e:
|
284
|
+
logger.exception(e)
|
285
|
+
raise AuxiliaryServerUnreachable(f"Unable to connect to CMR search server for VNP43NRT.")
|
286
|
+
else:
|
287
|
+
try:
|
288
|
+
NDVI_VIIRS_connection = VNP43IA4(
|
289
|
+
working_directory=working_directory,
|
290
|
+
download_directory=VIIRS_download_directory,
|
291
|
+
products_directory=VIIRS_products_directory,
|
292
|
+
mosaic_directory=VIIRS_mosaic_directory,
|
293
|
+
)
|
294
|
+
|
295
|
+
albedo_VIIRS_connection = VNP43MA3(
|
296
|
+
working_directory=working_directory,
|
297
|
+
download_directory=VIIRS_download_directory,
|
298
|
+
products_directory=VIIRS_products_directory,
|
299
|
+
mosaic_directory=VIIRS_mosaic_directory,
|
300
|
+
)
|
301
|
+
except LPDAACServerUnreachable as e:
|
302
|
+
logger.exception(e)
|
303
|
+
raise AuxiliaryServerUnreachable(f"Unable to connect to VIIRS LPDAAC server.")
|
304
|
+
|
305
|
+
# Define date ranges for data retrieval and fusion
|
306
|
+
end_date = date_UTC
|
307
|
+
# The start date of the BRDF-corrected VIIRS coarse time-series is 'spinup_days' before the target date
|
308
|
+
VIIRS_start_date = end_date - timedelta(days=spinup_days)
|
309
|
+
# To produce that first BRDF-corrected image, VNP09GA (raw VIIRS) is needed starting 16 days prior to the first coarse date
|
310
|
+
VIIRS_download_start_date = VIIRS_start_date - timedelta(days=16)
|
311
|
+
VIIRS_end_date = end_date
|
312
|
+
|
313
|
+
# Define start date of HLS fine image input time-series
|
314
|
+
if using_prior and prior_date_UTC and prior_date_UTC >= VIIRS_start_date:
|
315
|
+
# If a valid prior is used and its date is within or before the VIIRS start date,
|
316
|
+
# HLS inputs begin the day after the prior
|
317
|
+
HLS_start_date = prior_date_UTC + timedelta(days=1)
|
318
|
+
else:
|
319
|
+
# If no prior or prior is too old, HLS inputs begin on the same day as the VIIRS inputs
|
320
|
+
HLS_start_date = VIIRS_start_date
|
321
|
+
HLS_end_date = end_date # HLS end date is always the same as the target date
|
322
|
+
|
323
|
+
logger.info(
|
324
|
+
f"Processing STARS HLS-VIIRS NDVI and albedo for tile {cl.place(tile)} from "
|
325
|
+
f"{cl.time(VIIRS_start_date)} to {cl.time(end_date)}"
|
326
|
+
)
|
327
|
+
|
328
|
+
# Get HLS listing to check for data availability
|
329
|
+
try:
|
330
|
+
HLS_listing = HLS_connection.listing(
|
331
|
+
tile=tile, start_UTC=HLS_start_date, end_UTC=HLS_end_date
|
332
|
+
)
|
333
|
+
except HLSTileNotAvailable as e:
|
334
|
+
logger.exception(e)
|
335
|
+
raise LandFilter(f"Sentinel tile {tile} cannot be processed due to HLS tile unavailability.")
|
336
|
+
except Exception as e:
|
337
|
+
logger.exception(e)
|
338
|
+
raise AuxiliaryServerUnreachable(
|
339
|
+
f"Unable to scan Harmonized Landsat Sentinel server: {HLS_connection.remote}"
|
340
|
+
)
|
341
|
+
|
342
|
+
# Check for missing HLS Sentinel data
|
343
|
+
missing_sentinel_dates = HLS_listing[HLS_listing.sentinel == "missing"].date_UTC
|
344
|
+
if len(missing_sentinel_dates) > 0:
|
345
|
+
raise AuxiliaryLatency(
|
346
|
+
f"HLS Sentinel is not yet available at tile {tile} for dates: "
|
347
|
+
f"{', '.join(missing_sentinel_dates.dt.strftime('%Y-%m-%d'))}"
|
348
|
+
)
|
349
|
+
|
350
|
+
# Log available HLS Sentinel data
|
351
|
+
sentinel_listing = HLS_listing[~pd.isna(HLS_listing.sentinel)][
|
352
|
+
["date_UTC", "sentinel"]
|
353
|
+
]
|
354
|
+
logger.info(f"HLS Sentinel is available on {cl.val(len(sentinel_listing))} dates:")
|
355
|
+
for i, (list_date_utc, sentinel_granule) in sentinel_listing.iterrows():
|
356
|
+
sentinel_filename = sentinel_granule["meta"]["native-id"]
|
357
|
+
logger.info(f"* {cl.time(list_date_utc)}: {cl.file(sentinel_filename)}")
|
358
|
+
|
359
|
+
# Check for missing HLS Landsat data
|
360
|
+
missing_landsat_dates = HLS_listing[HLS_listing.landsat == "missing"].date_UTC
|
361
|
+
if len(missing_landsat_dates) > 0:
|
362
|
+
raise AuxiliaryLatency(
|
363
|
+
f"HLS Landsat is not yet available at tile {tile} for dates: "
|
364
|
+
f"{', '.join(missing_landsat_dates.dt.strftime('%Y-%m-%d'))}"
|
365
|
+
)
|
366
|
+
|
367
|
+
# Log available HLS Landsat data
|
368
|
+
landsat_listing = HLS_listing[~pd.isna(HLS_listing.landsat)][
|
369
|
+
["date_UTC", "landsat"]
|
370
|
+
]
|
371
|
+
logger.info(f"HLS Landsat is available on {cl.val(len(landsat_listing))} dates:")
|
372
|
+
for i, (list_date_utc, landsat_granule) in landsat_listing.iterrows():
|
373
|
+
landsat_filename = landsat_granule["meta"]["native-id"]
|
374
|
+
logger.info(f"* {cl.time(list_date_utc)}: {cl.file(landsat_filename)}")
|
375
|
+
|
376
|
+
# If only sources are requested, retrieve them and exit
|
377
|
+
if sources_only:
|
378
|
+
logger.info("Sources only flag enabled. Retrieving source data.")
|
379
|
+
retrieve_STARS_sources(
|
380
|
+
tile=tile,
|
381
|
+
geometry=geometry,
|
382
|
+
HLS_start_date=HLS_start_date,
|
383
|
+
HLS_end_date=HLS_end_date,
|
384
|
+
VIIRS_start_date=VIIRS_download_start_date,
|
385
|
+
VIIRS_end_date=VIIRS_end_date,
|
386
|
+
HLS_connection=HLS_connection,
|
387
|
+
VIIRS_connection=NDVI_VIIRS_connection, # Use NDVI_VIIRS_connection as a general VIIRS connection
|
388
|
+
)
|
389
|
+
# Regenerate inputs to ensure all files are staged, even if not fused
|
390
|
+
NDVI_coarse_geometry = HLS_connection.grid(tile=tile, cell_size=NDVI_resolution)
|
391
|
+
albedo_coarse_geometry = HLS_connection.grid(tile=tile, cell_size=albedo_resolution)
|
392
|
+
|
393
|
+
NDVI_coarse_directory = generate_NDVI_coarse_directory(
|
394
|
+
input_staging_directory=input_staging_directory, tile=tile
|
395
|
+
)
|
396
|
+
NDVI_fine_directory = generate_NDVI_fine_directory(
|
397
|
+
input_staging_directory=input_staging_directory, tile=tile
|
398
|
+
)
|
399
|
+
albedo_coarse_directory = generate_albedo_coarse_directory(
|
400
|
+
input_staging_directory=input_staging_directory, tile=tile
|
401
|
+
)
|
402
|
+
albedo_fine_directory = generate_albedo_fine_directory(
|
403
|
+
input_staging_directory=input_staging_directory, tile=tile
|
404
|
+
)
|
405
|
+
|
406
|
+
generate_STARS_inputs(
|
407
|
+
tile=tile,
|
408
|
+
date_UTC=date_UTC,
|
409
|
+
HLS_start_date=HLS_start_date,
|
410
|
+
HLS_end_date=HLS_end_date,
|
411
|
+
VIIRS_start_date=VIIRS_start_date,
|
412
|
+
VIIRS_end_date=VIIRS_end_date,
|
413
|
+
NDVI_resolution=NDVI_resolution,
|
414
|
+
albedo_resolution=albedo_resolution,
|
415
|
+
target_resolution=target_resolution,
|
416
|
+
NDVI_coarse_geometry=NDVI_coarse_geometry,
|
417
|
+
albedo_coarse_geometry=albedo_coarse_geometry,
|
418
|
+
working_directory=working_directory,
|
419
|
+
NDVI_coarse_directory=NDVI_coarse_directory,
|
420
|
+
NDVI_fine_directory=NDVI_fine_directory,
|
421
|
+
albedo_coarse_directory=albedo_coarse_directory,
|
422
|
+
albedo_fine_directory=albedo_fine_directory,
|
423
|
+
HLS_connection=HLS_connection,
|
424
|
+
NDVI_VIIRS_connection=NDVI_VIIRS_connection,
|
425
|
+
albedo_VIIRS_connection=albedo_VIIRS_connection,
|
426
|
+
calibrate_fine=calibrate_fine,
|
427
|
+
)
|
428
|
+
else:
|
429
|
+
# Otherwise, proceed with full product processing
|
430
|
+
process_STARS_product(
|
431
|
+
tile=tile,
|
432
|
+
date_UTC=date_UTC,
|
433
|
+
time_UTC=time_UTC,
|
434
|
+
build=build,
|
435
|
+
product_counter=product_counter,
|
436
|
+
HLS_start_date=HLS_start_date,
|
437
|
+
HLS_end_date=HLS_end_date,
|
438
|
+
VIIRS_start_date=VIIRS_start_date,
|
439
|
+
VIIRS_end_date=VIIRS_end_date,
|
440
|
+
NDVI_resolution=NDVI_resolution,
|
441
|
+
albedo_resolution=albedo_resolution,
|
442
|
+
target_resolution=target_resolution,
|
443
|
+
working_directory=working_directory,
|
444
|
+
model_directory=model_directory,
|
445
|
+
input_staging_directory=input_staging_directory,
|
446
|
+
L2T_STARS_granule_directory=L2T_STARS_granule_directory,
|
447
|
+
L2T_STARS_zip_filename=L2T_STARS_zip_filename,
|
448
|
+
L2T_STARS_browse_filename=L2T_STARS_browse_filename,
|
449
|
+
metadata=metadata,
|
450
|
+
prior=prior,
|
451
|
+
HLS_connection=HLS_connection,
|
452
|
+
NDVI_VIIRS_connection=NDVI_VIIRS_connection,
|
453
|
+
albedo_VIIRS_connection=albedo_VIIRS_connection,
|
454
|
+
using_prior=using_prior,
|
455
|
+
calibrate_fine=calibrate_fine,
|
456
|
+
remove_input_staging=remove_input_staging,
|
457
|
+
remove_prior=remove_prior,
|
458
|
+
remove_posterior=remove_posterior,
|
459
|
+
threads=threads,
|
460
|
+
num_workers=num_workers,
|
461
|
+
)
|
462
|
+
|
463
|
+
# --- Exception Handling for PGE ---
|
464
|
+
except (ConnectionError, urllib.error.HTTPError, CMRServerUnreachable) as exception:
|
465
|
+
logger.exception(exception)
|
466
|
+
exit_code = AUXILIARY_SERVER_UNREACHABLE
|
467
|
+
except DownloadFailed as exception:
|
468
|
+
logger.exception(exception)
|
469
|
+
exit_code = DOWNLOAD_FAILED
|
470
|
+
except HLSBandNotAcquired as exception:
|
471
|
+
logger.exception(exception)
|
472
|
+
exit_code = DOWNLOAD_FAILED
|
473
|
+
except HLSNotAvailable as exception:
|
474
|
+
logger.exception(exception)
|
475
|
+
exit_code = LAND_FILTER # This might indicate no HLS data for the tile, similar to land filter
|
476
|
+
except (HLSSentinelMissing, HLSLandsatMissing) as exception:
|
477
|
+
logger.exception(exception)
|
478
|
+
exit_code = AUXILIARY_LATENCY
|
479
|
+
except ECOSTRESSExitCodeException as exception:
|
480
|
+
# Catch custom ECOSTRESS exceptions and use their defined exit code
|
481
|
+
logger.exception(exception)
|
482
|
+
exit_code = exception.exit_code
|
483
|
+
except Exception as exception:
|
484
|
+
# Catch any other unexpected exceptions
|
485
|
+
logger.exception(exception)
|
486
|
+
exit_code = UNCLASSIFIED_FAILURE_EXIT_CODE
|
487
|
+
|
488
|
+
logger.info(f"L2T_STARS exit code: {exit_code}")
|
489
|
+
return exit_code
|