ECOv003-L2T-STARS 1.0.1__py3-none-any.whl → 1.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ECOv003_L2T_STARS/BRDF/BRDF.py +57 -0
- ECOv003_L2T_STARS/BRDF/SZA.py +65 -0
- ECOv003_L2T_STARS/BRDF/__init__.py +1 -0
- ECOv003_L2T_STARS/BRDF/statistical_radiative_transport.txt +90 -0
- ECOv003_L2T_STARS/BRDF/version.txt +1 -0
- ECOv003_L2T_STARS/ECOv003_DL.py +527 -0
- ECOv003_L2T_STARS/ECOv003_DL.xml +47 -0
- ECOv003_L2T_STARS/ECOv003_L2T_STARS.py +162 -0
- ECOv003_L2T_STARS/ECOv003_L2T_STARS.xml +47 -0
- ECOv003_L2T_STARS/L2TSTARSConfig.py +188 -0
- ECOv003_L2T_STARS/L2T_STARS.py +489 -0
- ECOv003_L2T_STARS/LPDAAC/LPDAACDataPool.py +444 -0
- ECOv003_L2T_STARS/LPDAAC/__init__.py +9 -0
- ECOv003_L2T_STARS/LPDAAC/version.txt +1 -0
- ECOv003_L2T_STARS/Manifest.toml +2332 -0
- ECOv003_L2T_STARS/Project.toml +14 -0
- ECOv003_L2T_STARS/VIIRS/VIIRSDataPool.py +294 -0
- ECOv003_L2T_STARS/VIIRS/VIIRSDownloader.py +26 -0
- ECOv003_L2T_STARS/VIIRS/VIIRS_CMR_LOGIN.py +36 -0
- ECOv003_L2T_STARS/VIIRS/VNP09GA.py +1277 -0
- ECOv003_L2T_STARS/VIIRS/VNP43IA4.py +288 -0
- ECOv003_L2T_STARS/VIIRS/VNP43MA3.py +323 -0
- ECOv003_L2T_STARS/VIIRS/__init__.py +9 -0
- ECOv003_L2T_STARS/VIIRS/version.txt +1 -0
- ECOv003_L2T_STARS/VNP43NRT/VNP43NRT.py +863 -0
- ECOv003_L2T_STARS/VNP43NRT/__init__.py +1 -0
- ECOv003_L2T_STARS/VNP43NRT/process_VNP43NRT.jl +169 -0
- ECOv003_L2T_STARS/VNP43NRT/version.txt +1 -0
- ECOv003_L2T_STARS/VNP43NRT_jl/Manifest.toml +995 -0
- ECOv003_L2T_STARS/VNP43NRT_jl/Project.toml +15 -0
- ECOv003_L2T_STARS/VNP43NRT_jl/__init__.py +0 -0
- ECOv003_L2T_STARS/VNP43NRT_jl/instantiate.jl +25 -0
- ECOv003_L2T_STARS/VNP43NRT_jl/instantiate.py +13 -0
- ECOv003_L2T_STARS/VNP43NRT_jl/src/VNP43NRT.jl +411 -0
- ECOv003_L2T_STARS/VNP43NRT_jl/src/__init__.py +0 -0
- ECOv003_L2T_STARS/__init__.py +3 -0
- ECOv003_L2T_STARS/calibrate_fine_to_coarse.py +60 -0
- ECOv003_L2T_STARS/constants.py +38 -0
- ECOv003_L2T_STARS/daterange/__init__.py +1 -0
- ECOv003_L2T_STARS/daterange/daterange.py +35 -0
- ECOv003_L2T_STARS/generate_L2T_STARS_runconfig.py +249 -0
- ECOv003_L2T_STARS/generate_NDVI_coarse_directory.py +21 -0
- ECOv003_L2T_STARS/generate_NDVI_coarse_image.py +30 -0
- ECOv003_L2T_STARS/generate_NDVI_fine_directory.py +14 -0
- ECOv003_L2T_STARS/generate_NDVI_fine_image.py +28 -0
- ECOv003_L2T_STARS/generate_STARS_inputs.py +231 -0
- ECOv003_L2T_STARS/generate_albedo_coarse_directory.py +18 -0
- ECOv003_L2T_STARS/generate_albedo_coarse_image.py +30 -0
- ECOv003_L2T_STARS/generate_albedo_fine_directory.py +17 -0
- ECOv003_L2T_STARS/generate_albedo_fine_image.py +30 -0
- ECOv003_L2T_STARS/generate_filename.py +37 -0
- ECOv003_L2T_STARS/generate_input_staging_directory.py +23 -0
- ECOv003_L2T_STARS/generate_model_state_tile_date_directory.py +28 -0
- ECOv003_L2T_STARS/generate_output_directory.py +28 -0
- ECOv003_L2T_STARS/install_STARS_jl.py +43 -0
- ECOv003_L2T_STARS/instantiate_STARS_jl.py +38 -0
- ECOv003_L2T_STARS/load_prior.py +248 -0
- ECOv003_L2T_STARS/prior.py +56 -0
- ECOv003_L2T_STARS/process_ECOSTRESS_data_fusion_distributed_bias.jl +420 -0
- ECOv003_L2T_STARS/process_STARS_product.py +507 -0
- ECOv003_L2T_STARS/process_julia_data_fusion.py +110 -0
- ECOv003_L2T_STARS/retrieve_STARS_sources.py +101 -0
- ECOv003_L2T_STARS/runconfig.py +70 -0
- ECOv003_L2T_STARS/timer/__init__.py +1 -0
- ECOv003_L2T_STARS/timer/timer.py +77 -0
- ECOv003_L2T_STARS/version.py +8 -0
- ECOv003_L2T_STARS/version.txt +1 -0
- {ECOv003_L2T_STARS-1.0.1.dist-info → ecov003_l2t_stars-1.1.0.dist-info}/METADATA +30 -23
- ecov003_l2t_stars-1.1.0.dist-info/RECORD +73 -0
- {ECOv003_L2T_STARS-1.0.1.dist-info → ecov003_l2t_stars-1.1.0.dist-info}/WHEEL +1 -1
- ecov003_l2t_stars-1.1.0.dist-info/entry_points.txt +3 -0
- ecov003_l2t_stars-1.1.0.dist-info/top_level.txt +1 -0
- ECOv003_L2T_STARS-1.0.1.dist-info/RECORD +0 -5
- ECOv003_L2T_STARS-1.0.1.dist-info/top_level.txt +0 -1
- {ECOv003_L2T_STARS-1.0.1.dist-info → ecov003_l2t_stars-1.1.0.dist-info/licenses}/LICENSE +0 -0
@@ -0,0 +1,249 @@
|
|
1
|
+
from typing import Union
|
2
|
+
from datetime import datetime, timezone
|
3
|
+
from os import makedirs
|
4
|
+
from os.path import join, abspath, exists, expanduser, dirname
|
5
|
+
from glob import glob
|
6
|
+
import logging
|
7
|
+
from shutil import which
|
8
|
+
from uuid import uuid4
|
9
|
+
import socket
|
10
|
+
|
11
|
+
import colored_logging as cl
|
12
|
+
from pytictoc import TicToc # Import pytictoc
|
13
|
+
|
14
|
+
from ECOv003_granules import L2TLSTE
|
15
|
+
|
16
|
+
from .constants import *
|
17
|
+
|
18
|
+
logger = logging.getLogger(__name__)
|
19
|
+
|
20
|
+
def generate_L2T_STARS_runconfig(
|
21
|
+
L2T_LSTE_filename: str,
|
22
|
+
prior_L2T_STARS_filename: str = "",
|
23
|
+
orbit: int = None,
|
24
|
+
scene: int = None,
|
25
|
+
tile: str = None,
|
26
|
+
time_UTC: Union[datetime, str] = None,
|
27
|
+
working_directory: str = None,
|
28
|
+
sources_directory: str = None,
|
29
|
+
indices_directory: str = None,
|
30
|
+
model_directory: str = None,
|
31
|
+
executable_filename: str = None,
|
32
|
+
output_directory: str = None,
|
33
|
+
runconfig_filename: str = None,
|
34
|
+
log_filename: str = None,
|
35
|
+
build: str = None,
|
36
|
+
processing_node: str = None,
|
37
|
+
production_datetime: datetime = None,
|
38
|
+
job_ID: str = None,
|
39
|
+
instance_ID: str = None,
|
40
|
+
product_counter: int = None,
|
41
|
+
template_filename: str = None,
|
42
|
+
) -> str:
|
43
|
+
"""
|
44
|
+
Generates an XML run-configuration file for the L2T_STARS processing.
|
45
|
+
|
46
|
+
This function dynamically creates an XML run-config file by populating a template
|
47
|
+
with provided or default parameters. It also checks for and returns existing
|
48
|
+
run-configs to prevent redundant generation.
|
49
|
+
|
50
|
+
Args:
|
51
|
+
L2T_LSTE_filename (str): Path to the input ECOSTRESS L2T LSTE granule file.
|
52
|
+
prior_L2T_STARS_filename (str, optional): Path to a prior L2T_STARS product file.
|
53
|
+
Defaults to "".
|
54
|
+
orbit (int, optional): Orbit number. If None, derived from L2T_LSTE_filename.
|
55
|
+
scene (int, optional): Scene ID. If None, derived from L2T_LSTE_filename.
|
56
|
+
tile (str, optional): HLS tile ID (e.g., '11SPS'). If None, derived from L2T_LSTE_filename.
|
57
|
+
time_UTC (Union[datetime, str], optional): UTC time of the L2T_LSTE granule. If None,
|
58
|
+
derived from L2T_LSTE_filename.
|
59
|
+
working_directory (str, optional): Root directory for all processing outputs.
|
60
|
+
Defaults to ".".
|
61
|
+
sources_directory (str, optional): Directory for downloaded source data (HLS, VIIRS).
|
62
|
+
indices_directory (str, optional): Directory for intermediate index products.
|
63
|
+
model_directory (str, optional): Directory for model state files (priors, posteriors).
|
64
|
+
executable_filename (str, optional): Path to the L2T_STARS executable. If None,
|
65
|
+
'L2T_STARS' is assumed to be in the system's PATH.
|
66
|
+
output_directory (str, optional): Directory for final L2T_STARS products.
|
67
|
+
runconfig_filename (str, optional): Specific filename for the generated run-config.
|
68
|
+
If None, a default name based on granule ID is used.
|
69
|
+
log_filename (str, optional): Specific filename for the log file. If None,
|
70
|
+
a default name based on granule ID is used.
|
71
|
+
build (str, optional): Build ID of the PGE. Defaults to DEFAULT_BUILD.
|
72
|
+
processing_node (str, optional): Name of the processing node. Defaults to system hostname.
|
73
|
+
production_datetime (datetime, optional): Production date and time. Defaults to now (UTC).
|
74
|
+
job_ID (str, optional): Job identifier. Defaults to a timestamp.
|
75
|
+
instance_ID (str, optional): Unique instance identifier. Defaults to a UUID.
|
76
|
+
product_counter (int, optional): Counter for product generation. Defaults to 1.
|
77
|
+
template_filename (str, optional): Path to the XML run-config template file.
|
78
|
+
Defaults to L2T_STARS_TEMPLATE.
|
79
|
+
|
80
|
+
Returns:
|
81
|
+
str: The absolute path to the generated or existing run-configuration file.
|
82
|
+
"""
|
83
|
+
timer = TicToc() # Initialize pytictoc timer
|
84
|
+
timer.tic() # Start the timer
|
85
|
+
|
86
|
+
# Load the L2T_LSTE granule to extract necessary metadata if not provided
|
87
|
+
l2t_lste_granule = L2TLSTE(L2T_LSTE_filename)
|
88
|
+
|
89
|
+
# Use values from L2T_LSTE granule if not explicitly provided
|
90
|
+
if orbit is None:
|
91
|
+
orbit = l2t_lste_granule.orbit
|
92
|
+
if scene is None:
|
93
|
+
scene = l2t_lste_granule.scene
|
94
|
+
if tile is None:
|
95
|
+
tile = l2t_lste_granule.tile
|
96
|
+
if time_UTC is None:
|
97
|
+
time_UTC = l2t_lste_granule.time_UTC
|
98
|
+
|
99
|
+
# Set default values for other parameters if not provided
|
100
|
+
if build is None:
|
101
|
+
build = DEFAULT_BUILD
|
102
|
+
if working_directory is None:
|
103
|
+
working_directory = "."
|
104
|
+
|
105
|
+
date_UTC = time_UTC.date()
|
106
|
+
|
107
|
+
logger.info(
|
108
|
+
f"Started generating L2T_STARS run-config for tile {cl.val(tile)} on date {cl.time(date_UTC)}"
|
109
|
+
)
|
110
|
+
|
111
|
+
# Check for previous run-configs to avoid re-generating
|
112
|
+
pattern = join(
|
113
|
+
working_directory, "runconfig", f"ECOv003_L2T_STARS_{tile}_*_{build}_*.xml"
|
114
|
+
)
|
115
|
+
logger.info(f"Scanning for previous run-configs: {cl.val(pattern)}")
|
116
|
+
previous_runconfigs = glob(pattern)
|
117
|
+
previous_runconfig_count = len(previous_runconfigs)
|
118
|
+
|
119
|
+
if previous_runconfig_count > 0:
|
120
|
+
logger.info(f"Found {cl.val(previous_runconfig_count)} previous run-configs")
|
121
|
+
# Return the most recent run-config if found
|
122
|
+
previous_runconfig = sorted(previous_runconfigs)[-1]
|
123
|
+
logger.info(f"Previous run-config: {cl.file(previous_runconfig)}")
|
124
|
+
return previous_runconfig
|
125
|
+
|
126
|
+
# Resolve the path to the run-config template
|
127
|
+
if template_filename is None:
|
128
|
+
template_filename = L2T_STARS_TEMPLATE
|
129
|
+
template_filename = abspath(expanduser(template_filename))
|
130
|
+
|
131
|
+
# Set production datetime if not provided
|
132
|
+
if production_datetime is None:
|
133
|
+
production_datetime = datetime.now(timezone.utc)
|
134
|
+
|
135
|
+
# Set product counter if not provided
|
136
|
+
if product_counter is None:
|
137
|
+
product_counter = 1
|
138
|
+
|
139
|
+
# Format timestamp and generate granule ID
|
140
|
+
timestamp = f"{time_UTC:%Y%m%d}"
|
141
|
+
granule_ID = (
|
142
|
+
f"ECOv003_L2T_STARS_{tile}_{timestamp}_{build}_{product_counter:02d}"
|
143
|
+
)
|
144
|
+
|
145
|
+
# Define run-config filename and resolve absolute path
|
146
|
+
if runconfig_filename is None:
|
147
|
+
runconfig_filename = join(working_directory, "runconfig", f"{granule_ID}.xml")
|
148
|
+
runconfig_filename = abspath(expanduser(runconfig_filename))
|
149
|
+
|
150
|
+
# If the run-config file already exists, log and return its path
|
151
|
+
if exists(runconfig_filename):
|
152
|
+
logger.info(f"Run-config already exists {cl.file(runconfig_filename)}")
|
153
|
+
return runconfig_filename
|
154
|
+
|
155
|
+
# Resolve absolute paths for various directories if not already defined
|
156
|
+
working_directory = abspath(expanduser(working_directory))
|
157
|
+
if sources_directory is None:
|
158
|
+
sources_directory = join(working_directory, DEFAULT_STARS_SOURCES_DIRECTORY)
|
159
|
+
if indices_directory is None:
|
160
|
+
indices_directory = join(working_directory, DEFAULT_STARS_INDICES_DIRECTORY)
|
161
|
+
if model_directory is None:
|
162
|
+
model_directory = join(working_directory, DEFAULT_STARS_MODEL_DIRECTORY)
|
163
|
+
|
164
|
+
# Determine executable path; fall back to just the name if not found in PATH
|
165
|
+
if executable_filename is None:
|
166
|
+
executable_filename = which("L2T_STARS")
|
167
|
+
if executable_filename is None:
|
168
|
+
executable_filename = "L2T_STARS"
|
169
|
+
|
170
|
+
# Define output and log file paths
|
171
|
+
if output_directory is None:
|
172
|
+
output_directory = join(working_directory, DEFAULT_OUTPUT_DIRECTORY)
|
173
|
+
output_directory = abspath(expanduser(output_directory))
|
174
|
+
if log_filename is None:
|
175
|
+
log_filename = join(working_directory, "log", f"{granule_ID}.log")
|
176
|
+
log_filename = abspath(expanduser(log_filename))
|
177
|
+
|
178
|
+
# Get processing node hostname
|
179
|
+
if processing_node is None:
|
180
|
+
processing_node = socket.gethostname()
|
181
|
+
|
182
|
+
# Set Job ID and Instance ID
|
183
|
+
if job_ID is None:
|
184
|
+
job_ID = timestamp
|
185
|
+
if instance_ID is None:
|
186
|
+
instance_ID = str(uuid4()) # Generate a unique UUID for the instance
|
187
|
+
|
188
|
+
# Resolve absolute path for the input L2T_LSTE file
|
189
|
+
L2T_LSTE_filename = abspath(expanduser(L2T_LSTE_filename))
|
190
|
+
|
191
|
+
logger.info(f"Loading L2T_STARS template: {cl.file(template_filename)}")
|
192
|
+
|
193
|
+
# Read the XML template file content
|
194
|
+
with open(template_filename, "r") as file:
|
195
|
+
template = file.read()
|
196
|
+
|
197
|
+
# Replace placeholders in the template with actual values
|
198
|
+
logger.info(f"Orbit: {cl.val(orbit)}")
|
199
|
+
template = template.replace("orbit_number", f"{orbit:05d}")
|
200
|
+
logger.info(f"Scene: {cl.val(scene)}")
|
201
|
+
template = template.replace("scene_ID", f"{scene:03d}")
|
202
|
+
logger.info(f"Tile: {cl.val(tile)}")
|
203
|
+
template = template.replace("tile_ID", f"{tile}")
|
204
|
+
logger.info(f"L2T_LSTE file: {cl.file(L2T_LSTE_filename)}")
|
205
|
+
template = template.replace("L2T_LSTE_filename", L2T_LSTE_filename)
|
206
|
+
logger.info(f"Prior L2T_STARS file: {cl.file(prior_L2T_STARS_filename)}")
|
207
|
+
template = template.replace("prior_L2T_STARS_filename", prior_L2T_STARS_filename)
|
208
|
+
logger.info(f"Working directory: {cl.dir(working_directory)}")
|
209
|
+
template = template.replace("working_directory", working_directory)
|
210
|
+
logger.info(f"Sources directory: {cl.dir(sources_directory)}")
|
211
|
+
template = template.replace("sources_directory", sources_directory)
|
212
|
+
logger.info(f"Indices directory: {cl.dir(indices_directory)}")
|
213
|
+
template = template.replace("indices_directory", indices_directory)
|
214
|
+
logger.info(f"Model directory: {cl.dir(model_directory)}")
|
215
|
+
template = template.replace("model_directory", model_directory)
|
216
|
+
logger.info(f"Executable: {cl.file(executable_filename)}")
|
217
|
+
template = template.replace("executable_filename", executable_filename)
|
218
|
+
logger.info(f"Output directory: {cl.dir(output_directory)}")
|
219
|
+
template = template.replace("output_directory", output_directory)
|
220
|
+
logger.info(f"Run-config: {cl.file(runconfig_filename)}")
|
221
|
+
template = template.replace("runconfig_filename", runconfig_filename)
|
222
|
+
logger.info(f"Log: {cl.file(log_filename)}")
|
223
|
+
template = template.replace("log_filename", log_filename)
|
224
|
+
logger.info(f"Build: {cl.val(build)}")
|
225
|
+
template = template.replace("build_ID", build)
|
226
|
+
logger.info(f"Processing node: {cl.val(processing_node)}")
|
227
|
+
template = template.replace("processing_node", processing_node)
|
228
|
+
logger.info(f"Production date/time: {cl.time(production_datetime)}")
|
229
|
+
template = template.replace("production_datetime", f"{production_datetime:%Y-%m-%dT%H:%M:%SZ}")
|
230
|
+
logger.info(f"Job ID: {cl.val(job_ID)}")
|
231
|
+
template = template.replace("job_ID", job_ID)
|
232
|
+
logger.info(f"Instance ID: {cl.val(instance_ID)}")
|
233
|
+
template = template.replace("instance_ID", instance_ID)
|
234
|
+
logger.info(f"Product counter: {cl.val(product_counter)}")
|
235
|
+
template = template.replace("product_counter", f"{product_counter:02d}")
|
236
|
+
|
237
|
+
# Create the directory for the run-config file if it doesn't exist
|
238
|
+
makedirs(dirname(abspath(runconfig_filename)), exist_ok=True)
|
239
|
+
logger.info(f"Writing run-config file: {cl.file(runconfig_filename)}")
|
240
|
+
|
241
|
+
# Write the populated template to the run-config file
|
242
|
+
with open(runconfig_filename, "w") as file:
|
243
|
+
file.write(template)
|
244
|
+
|
245
|
+
logger.info(
|
246
|
+
f"Finished generating L2T_STARS run-config for orbit {cl.val(orbit)} scene {cl.val(scene)} ({timer.tocvalue():.2f} seconds)"
|
247
|
+
)
|
248
|
+
|
249
|
+
return
|
@@ -0,0 +1,21 @@
|
|
1
|
+
from .generate_input_staging_directory import generate_input_staging_directory
|
2
|
+
|
3
|
+
def generate_NDVI_coarse_directory(
|
4
|
+
input_staging_directory: str,
|
5
|
+
tile: str) -> str:
|
6
|
+
"""
|
7
|
+
Generates the specific staging directory for coarse NDVI images.
|
8
|
+
|
9
|
+
Args:
|
10
|
+
input_staging_directory (str): The base input staging directory.
|
11
|
+
tile (str): The HLS tile ID.
|
12
|
+
|
13
|
+
Returns:
|
14
|
+
str: The full path to the coarse NDVI staging directory.
|
15
|
+
"""
|
16
|
+
return generate_input_staging_directory(
|
17
|
+
input_staging_directory,
|
18
|
+
tile,
|
19
|
+
"NDVI_coarse"
|
20
|
+
)
|
21
|
+
|
@@ -0,0 +1,30 @@
|
|
1
|
+
from typing import Union
|
2
|
+
from datetime import date
|
3
|
+
import numpy as np
|
4
|
+
|
5
|
+
import rasters as rt
|
6
|
+
from rasters import Raster, RasterGeometry
|
7
|
+
|
8
|
+
from .VIIRS import VIIRSDownloaderAlbedo, VIIRSDownloaderNDVI
|
9
|
+
|
10
|
+
def generate_NDVI_coarse_image(
|
11
|
+
date_UTC: Union[date, str],
|
12
|
+
VIIRS_connection: VIIRSDownloaderNDVI,
|
13
|
+
geometry: RasterGeometry = None) -> Raster:
|
14
|
+
"""
|
15
|
+
Generates a coarse-resolution NDVI image from VIIRS data.
|
16
|
+
|
17
|
+
Args:
|
18
|
+
date_UTC (Union[date, str]): The UTC date for which to retrieve NDVI data.
|
19
|
+
VIIRS_connection (VIIRSDownloaderNDVI): An initialized VIIRS NDVI downloader object.
|
20
|
+
geometry (RasterGeometry, optional): The target geometry for the VIIRS image.
|
21
|
+
If None, the native VIIRS geometry is used.
|
22
|
+
|
23
|
+
Returns:
|
24
|
+
Raster: A Raster object representing the coarse-resolution NDVI image.
|
25
|
+
Zero values are converted to NaN.
|
26
|
+
"""
|
27
|
+
coarse_image = VIIRS_connection.NDVI(date_UTC=date_UTC, geometry=geometry)
|
28
|
+
# Convert zero values (often used as NoData in some datasets) to NaN for proper handling
|
29
|
+
coarse_image = rt.where(coarse_image == 0, np.nan, coarse_image)
|
30
|
+
return coarse_image
|
@@ -0,0 +1,14 @@
|
|
1
|
+
from .generate_input_staging_directory import generate_input_staging_directory
|
2
|
+
|
3
|
+
def generate_NDVI_fine_directory(input_staging_directory: str, tile: str) -> str:
|
4
|
+
"""
|
5
|
+
Generates the specific staging directory for fine NDVI images.
|
6
|
+
|
7
|
+
Args:
|
8
|
+
input_staging_directory (str): The base input staging directory.
|
9
|
+
tile (str): The HLS tile ID.
|
10
|
+
|
11
|
+
Returns:
|
12
|
+
str: The full path to the fine NDVI staging directory.
|
13
|
+
"""
|
14
|
+
return generate_input_staging_directory(input_staging_directory, tile, "NDVI_fine")
|
@@ -0,0 +1,28 @@
|
|
1
|
+
from typing import Union
|
2
|
+
from datetime import date
|
3
|
+
import numpy as np
|
4
|
+
import rasters as rt
|
5
|
+
from rasters import Raster
|
6
|
+
|
7
|
+
from harmonized_landsat_sentinel import HLS
|
8
|
+
|
9
|
+
def generate_NDVI_fine_image(
|
10
|
+
date_UTC: Union[date, str],
|
11
|
+
tile: str,
|
12
|
+
HLS_connection: HLS) -> Raster:
|
13
|
+
"""
|
14
|
+
Generates a fine-resolution NDVI image from HLS data.
|
15
|
+
|
16
|
+
Args:
|
17
|
+
date_UTC (Union[date, str]): The UTC date for which to retrieve NDVI data.
|
18
|
+
tile (str): The HLS tile ID.
|
19
|
+
HLS_connection (HLS): An initialized HLS data connection object.
|
20
|
+
|
21
|
+
Returns:
|
22
|
+
Raster: A Raster object representing the fine-resolution NDVI image.
|
23
|
+
Zero values are converted to NaN.
|
24
|
+
"""
|
25
|
+
fine_image = HLS_connection.NDVI(tile=tile, date_UTC=date_UTC)
|
26
|
+
# Convert zero values to NaN for consistency
|
27
|
+
fine_image = rt.where(fine_image == 0, np.nan, fine_image)
|
28
|
+
return fine_image
|
@@ -0,0 +1,231 @@
|
|
1
|
+
from typing import Union
|
2
|
+
from datetime import date, datetime
|
3
|
+
from dateutil.rrule import rrule, DAILY
|
4
|
+
import logging
|
5
|
+
|
6
|
+
import colored_logging as cl
|
7
|
+
from rasters import Raster, RasterGeometry
|
8
|
+
from harmonized_landsat_sentinel import HLS2CMR
|
9
|
+
|
10
|
+
from ECOv003_exit_codes import AuxiliaryLatency
|
11
|
+
|
12
|
+
from .constants import VIIRS_GIVEUP_DAYS
|
13
|
+
from .generate_filename import generate_filename
|
14
|
+
from .daterange import get_date
|
15
|
+
from .generate_NDVI_coarse_image import generate_NDVI_coarse_image
|
16
|
+
from .generate_NDVI_fine_image import generate_NDVI_fine_image
|
17
|
+
from .generate_albedo_coarse_image import generate_albedo_coarse_image
|
18
|
+
from .generate_albedo_fine_image import generate_albedo_fine_image
|
19
|
+
from .calibrate_fine_to_coarse import calibrate_fine_to_coarse
|
20
|
+
from .VIIRS.VIIRSDownloader import VIIRSDownloaderAlbedo, VIIRSDownloaderNDVI
|
21
|
+
|
22
|
+
logger = logging.getLogger(__name__)
|
23
|
+
|
24
|
+
def generate_STARS_inputs(
|
25
|
+
tile: str,
|
26
|
+
date_UTC: date,
|
27
|
+
HLS_start_date: date,
|
28
|
+
HLS_end_date: date,
|
29
|
+
VIIRS_start_date: date,
|
30
|
+
VIIRS_end_date: date,
|
31
|
+
NDVI_resolution: int,
|
32
|
+
albedo_resolution: int,
|
33
|
+
target_resolution: int,
|
34
|
+
NDVI_coarse_geometry: RasterGeometry,
|
35
|
+
albedo_coarse_geometry: RasterGeometry,
|
36
|
+
working_directory: str,
|
37
|
+
NDVI_coarse_directory: str,
|
38
|
+
NDVI_fine_directory: str,
|
39
|
+
albedo_coarse_directory: str,
|
40
|
+
albedo_fine_directory: str,
|
41
|
+
HLS_connection: HLS2CMR,
|
42
|
+
NDVI_VIIRS_connection: VIIRSDownloaderNDVI,
|
43
|
+
albedo_VIIRS_connection: VIIRSDownloaderAlbedo,
|
44
|
+
calibrate_fine: bool = False,
|
45
|
+
):
|
46
|
+
"""
|
47
|
+
Generates and stages the necessary coarse and fine resolution input images
|
48
|
+
for the STARS data fusion process.
|
49
|
+
|
50
|
+
This function iterates through the VIIRS date range, retrieving and saving
|
51
|
+
coarse NDVI and albedo images. For dates within the HLS range, it also
|
52
|
+
retrieves and saves fine NDVI and albedo images. It can optionally
|
53
|
+
calibrate the fine images to the coarse images.
|
54
|
+
|
55
|
+
Args:
|
56
|
+
tile (str): The HLS tile ID.
|
57
|
+
date_UTC (date): The target UTC date for the L2T_STARS product.
|
58
|
+
HLS_start_date (date): The start date for HLS data retrieval for the fusion period.
|
59
|
+
HLS_end_date (date): The end date for HLS data retrieval for the fusion period.
|
60
|
+
VIIRS_start_date (date): The start date for VIIRS data retrieval for the fusion period.
|
61
|
+
VIIRS_end_date (date): The end date for VIIRS data retrieval for the fusion period.
|
62
|
+
NDVI_resolution (int): The resolution of the coarse NDVI data.
|
63
|
+
albedo_resolution (int): The resolution of the coarse albedo data.
|
64
|
+
target_resolution (int): The desired output resolution of the fused product.
|
65
|
+
NDVI_coarse_geometry (RasterGeometry): The target geometry for coarse NDVI images.
|
66
|
+
albedo_coarse_geometry (RasterGeometry): The target geometry for coarse albedo images.
|
67
|
+
working_directory (str): The main working directory.
|
68
|
+
NDVI_coarse_directory (str): Directory for staging coarse NDVI images.
|
69
|
+
NDVI_fine_directory (str): Directory for staging fine NDVI images.
|
70
|
+
albedo_coarse_directory (str): Directory for staging coarse albedo images.
|
71
|
+
albedo_fine_directory (str): Directory for staging fine albedo images.
|
72
|
+
HLS_connection (HLS2CMR): An initialized HLS data connection object.
|
73
|
+
NDVI_VIIRS_connection (VIIRSDownloaderNDVI): An initialized VIIRS NDVI downloader.
|
74
|
+
albedo_VIIRS_connection (VIIRSDownloaderAlbedo): An initialized VIIRS albedo downloader.
|
75
|
+
calibrate_fine (bool, optional): If True, calibrate fine images to coarse images.
|
76
|
+
Defaults to False.
|
77
|
+
|
78
|
+
Raises:
|
79
|
+
AuxiliaryLatency: If coarse VIIRS data is missing within the VIIRS_GIVEUP_DAYS window.
|
80
|
+
"""
|
81
|
+
missing_coarse_dates = set() # Track dates where coarse data could not be generated
|
82
|
+
|
83
|
+
# Process each day within the VIIRS data fusion window
|
84
|
+
for processing_date in [
|
85
|
+
get_date(dt) for dt in rrule(DAILY, dtstart=VIIRS_start_date, until=VIIRS_end_date)
|
86
|
+
]:
|
87
|
+
logger.info(
|
88
|
+
f"Preparing coarse image for STARS NDVI at {cl.place(tile)} on {cl.time(processing_date)}"
|
89
|
+
)
|
90
|
+
|
91
|
+
try:
|
92
|
+
# Generate coarse NDVI image
|
93
|
+
NDVI_coarse_image = generate_NDVI_coarse_image(
|
94
|
+
date_UTC=processing_date,
|
95
|
+
VIIRS_connection=NDVI_VIIRS_connection,
|
96
|
+
geometry=NDVI_coarse_geometry,
|
97
|
+
)
|
98
|
+
|
99
|
+
# Define filename for coarse NDVI and save
|
100
|
+
NDVI_coarse_filename = generate_filename(
|
101
|
+
directory=NDVI_coarse_directory,
|
102
|
+
variable="NDVI",
|
103
|
+
date_UTC=processing_date,
|
104
|
+
tile=tile,
|
105
|
+
cell_size=NDVI_resolution,
|
106
|
+
)
|
107
|
+
logger.info(
|
108
|
+
f"Saving coarse image for STARS NDVI at {cl.place(tile)} on {cl.time(processing_date)}: {NDVI_coarse_filename}"
|
109
|
+
)
|
110
|
+
NDVI_coarse_image.to_geotiff(NDVI_coarse_filename)
|
111
|
+
|
112
|
+
# If the processing date is within the HLS range, generate fine NDVI
|
113
|
+
if processing_date >= HLS_start_date:
|
114
|
+
logger.info(
|
115
|
+
f"Preparing fine image for STARS NDVI at {cl.place(tile)} on {cl.time(processing_date)}"
|
116
|
+
)
|
117
|
+
try:
|
118
|
+
NDVI_fine_image = generate_NDVI_fine_image(
|
119
|
+
date_UTC=processing_date,
|
120
|
+
tile=tile,
|
121
|
+
HLS_connection=HLS_connection,
|
122
|
+
)
|
123
|
+
|
124
|
+
# Optionally calibrate the fine NDVI image to the coarse NDVI image
|
125
|
+
if calibrate_fine:
|
126
|
+
logger.info(
|
127
|
+
f"Calibrating fine image for STARS NDVI at {cl.place(tile)} on {cl.time(processing_date)}"
|
128
|
+
)
|
129
|
+
NDVI_fine_image = calibrate_fine_to_coarse(
|
130
|
+
NDVI_fine_image, NDVI_coarse_image
|
131
|
+
)
|
132
|
+
|
133
|
+
# Define filename for fine NDVI and save
|
134
|
+
NDVI_fine_filename = generate_filename(
|
135
|
+
directory=NDVI_fine_directory,
|
136
|
+
variable="NDVI",
|
137
|
+
date_UTC=processing_date,
|
138
|
+
tile=tile,
|
139
|
+
cell_size=target_resolution,
|
140
|
+
)
|
141
|
+
logger.info(
|
142
|
+
f"Saving fine image for STARS NDVI at {cl.place(tile)} on {cl.time(processing_date)}: {NDVI_fine_filename}"
|
143
|
+
)
|
144
|
+
NDVI_fine_image.to_geotiff(NDVI_fine_filename)
|
145
|
+
except Exception: # Catch any exception during HLS fine image generation
|
146
|
+
logger.info(f"HLS NDVI is not available on {processing_date}")
|
147
|
+
except Exception as e:
|
148
|
+
logger.exception(e)
|
149
|
+
logger.warning(
|
150
|
+
f"Unable to produce coarse NDVI for date {processing_date}"
|
151
|
+
)
|
152
|
+
missing_coarse_dates.add(processing_date) # Add date to missing set
|
153
|
+
|
154
|
+
logger.info(
|
155
|
+
f"Preparing coarse image for STARS albedo at {cl.place(tile)} on {cl.time(processing_date)}"
|
156
|
+
)
|
157
|
+
try:
|
158
|
+
# Generate coarse albedo image
|
159
|
+
albedo_coarse_image = generate_albedo_coarse_image(
|
160
|
+
date_UTC=processing_date,
|
161
|
+
VIIRS_connection=albedo_VIIRS_connection,
|
162
|
+
geometry=albedo_coarse_geometry,
|
163
|
+
)
|
164
|
+
|
165
|
+
# Define filename for coarse albedo and save
|
166
|
+
albedo_coarse_filename = generate_filename(
|
167
|
+
directory=albedo_coarse_directory,
|
168
|
+
variable="albedo",
|
169
|
+
date_UTC=processing_date,
|
170
|
+
tile=tile,
|
171
|
+
cell_size=albedo_resolution,
|
172
|
+
)
|
173
|
+
logger.info(
|
174
|
+
f"Saving coarse image for STARS albedo at {cl.place(tile)} on {cl.time(processing_date)}: {albedo_coarse_filename}"
|
175
|
+
)
|
176
|
+
albedo_coarse_image.to_geotiff(albedo_coarse_filename)
|
177
|
+
|
178
|
+
# If the processing date is within the HLS range, generate fine albedo
|
179
|
+
if processing_date >= HLS_start_date:
|
180
|
+
logger.info(
|
181
|
+
f"Preparing fine image for STARS albedo at {cl.place(tile)} on {cl.time(processing_date)}"
|
182
|
+
)
|
183
|
+
try:
|
184
|
+
albedo_fine_image = generate_albedo_fine_image(
|
185
|
+
date_UTC=processing_date,
|
186
|
+
tile=tile,
|
187
|
+
HLS_connection=HLS_connection,
|
188
|
+
)
|
189
|
+
|
190
|
+
# Optionally calibrate the fine albedo image to the coarse albedo image
|
191
|
+
if calibrate_fine:
|
192
|
+
logger.info(
|
193
|
+
f"Calibrating fine image for STARS albedo at {cl.place(tile)} on {cl.time(processing_date)}"
|
194
|
+
)
|
195
|
+
albedo_fine_image = calibrate_fine_to_coarse(
|
196
|
+
albedo_fine_image, albedo_coarse_image
|
197
|
+
)
|
198
|
+
|
199
|
+
# Define filename for fine albedo and save
|
200
|
+
albedo_fine_filename = generate_filename(
|
201
|
+
directory=albedo_fine_directory,
|
202
|
+
variable="albedo",
|
203
|
+
date_UTC=processing_date,
|
204
|
+
tile=tile,
|
205
|
+
cell_size=target_resolution,
|
206
|
+
)
|
207
|
+
logger.info(
|
208
|
+
f"Saving fine image for STARS albedo at {cl.place(tile)} on {cl.time(processing_date)}: {albedo_fine_filename}"
|
209
|
+
)
|
210
|
+
albedo_fine_image.to_geotiff(albedo_fine_filename)
|
211
|
+
except Exception: # Catch any exception during HLS fine image generation
|
212
|
+
logger.info(f"HLS albedo is not available on {processing_date}")
|
213
|
+
except Exception as e:
|
214
|
+
logger.exception(e)
|
215
|
+
logger.warning(
|
216
|
+
f"Unable to produce coarse albedo for date {processing_date}"
|
217
|
+
)
|
218
|
+
missing_coarse_dates.add(processing_date) # Add date to missing set
|
219
|
+
|
220
|
+
# Check for missing coarse dates within the give-up window
|
221
|
+
coarse_latency_dates = [
|
222
|
+
d
|
223
|
+
for d in missing_coarse_dates
|
224
|
+
if (datetime.utcnow().date() - d).days <= VIIRS_GIVEUP_DAYS
|
225
|
+
]
|
226
|
+
|
227
|
+
if len(coarse_latency_dates) > 0:
|
228
|
+
raise AuxiliaryLatency(
|
229
|
+
f"Missing coarse dates within {VIIRS_GIVEUP_DAYS}-day window: "
|
230
|
+
f"{', '.join([str(d) for d in sorted(list(coarse_latency_dates))])}"
|
231
|
+
)
|
@@ -0,0 +1,18 @@
|
|
1
|
+
|
2
|
+
from .generate_input_staging_directory import generate_input_staging_directory
|
3
|
+
|
4
|
+
|
5
|
+
def generate_albedo_coarse_directory(input_staging_directory: str, tile: str) -> str:
|
6
|
+
"""
|
7
|
+
Generates the specific staging directory for coarse albedo images.
|
8
|
+
|
9
|
+
Args:
|
10
|
+
input_staging_directory (str): The base input staging directory.
|
11
|
+
tile (str): The HLS tile ID.
|
12
|
+
|
13
|
+
Returns:
|
14
|
+
str: The full path to the coarse albedo staging directory.
|
15
|
+
"""
|
16
|
+
return generate_input_staging_directory(
|
17
|
+
input_staging_directory, tile, "albedo_coarse"
|
18
|
+
)
|
@@ -0,0 +1,30 @@
|
|
1
|
+
from typing import Union
|
2
|
+
from datetime import date
|
3
|
+
|
4
|
+
import numpy as np
|
5
|
+
|
6
|
+
import rasters as rt
|
7
|
+
from rasters import Raster, RasterGeometry
|
8
|
+
|
9
|
+
from .VIIRS.VIIRSDownloader import VIIRSDownloaderAlbedo
|
10
|
+
|
11
|
+
def generate_albedo_coarse_image(
|
12
|
+
date_UTC: Union[date, str], VIIRS_connection: VIIRSDownloaderAlbedo, geometry: RasterGeometry = None
|
13
|
+
) -> Raster:
|
14
|
+
"""
|
15
|
+
Generates a coarse-resolution albedo image from VIIRS data.
|
16
|
+
|
17
|
+
Args:
|
18
|
+
date_UTC (Union[date, str]): The UTC date for which to retrieve albedo data.
|
19
|
+
VIIRS_connection (VIIRSDownloaderAlbedo): An initialized VIIRS albedo downloader object.
|
20
|
+
geometry (RasterGeometry, optional): The target geometry for the VIIRS image.
|
21
|
+
If None, the native VIIRS geometry is used.
|
22
|
+
|
23
|
+
Returns:
|
24
|
+
Raster: A Raster object representing the coarse-resolution albedo image.
|
25
|
+
Zero values are converted to NaN.
|
26
|
+
"""
|
27
|
+
coarse_image = VIIRS_connection.albedo(date_UTC=date_UTC, geometry=geometry)
|
28
|
+
# Convert zero values to NaN for consistency
|
29
|
+
coarse_image = rt.where(coarse_image == 0, np.nan, coarse_image)
|
30
|
+
return coarse_image
|
@@ -0,0 +1,17 @@
|
|
1
|
+
|
2
|
+
from .generate_input_staging_directory import generate_input_staging_directory
|
3
|
+
|
4
|
+
def generate_albedo_fine_directory(input_staging_directory: str, tile: str) -> str:
|
5
|
+
"""
|
6
|
+
Generates the specific staging directory for fine albedo images.
|
7
|
+
|
8
|
+
Args:
|
9
|
+
input_staging_directory (str): The base input staging directory.
|
10
|
+
tile (str): The HLS tile ID.
|
11
|
+
|
12
|
+
Returns:
|
13
|
+
str: The full path to the fine albedo staging directory.
|
14
|
+
"""
|
15
|
+
return generate_input_staging_directory(
|
16
|
+
input_staging_directory, tile, "albedo_fine"
|
17
|
+
)
|
@@ -0,0 +1,30 @@
|
|
1
|
+
from typing import Union
|
2
|
+
from datetime import date
|
3
|
+
|
4
|
+
import numpy as np
|
5
|
+
|
6
|
+
import rasters as rt
|
7
|
+
from rasters import Raster
|
8
|
+
|
9
|
+
from harmonized_landsat_sentinel import HLS
|
10
|
+
|
11
|
+
def generate_albedo_fine_image(
|
12
|
+
date_UTC: Union[date, str],
|
13
|
+
tile: str,
|
14
|
+
HLS_connection: HLS) -> Raster:
|
15
|
+
"""
|
16
|
+
Generates a fine-resolution albedo image from HLS data.
|
17
|
+
|
18
|
+
Args:
|
19
|
+
date_UTC (Union[date, str]): The UTC date for which to retrieve albedo data.
|
20
|
+
tile (str): The HLS tile ID.
|
21
|
+
HLS_connection (HLS): An initialized HLS data connection object.
|
22
|
+
|
23
|
+
Returns:
|
24
|
+
Raster: A Raster object representing the fine-resolution albedo image.
|
25
|
+
Zero values are converted to NaN.
|
26
|
+
"""
|
27
|
+
fine_image = HLS_connection.albedo(tile=tile, date_UTC=date_UTC)
|
28
|
+
# Convert zero values to NaN for consistency
|
29
|
+
fine_image = rt.where(fine_image == 0, np.nan, fine_image)
|
30
|
+
return fine_image
|