ECOv003-L2T-STARS 1.0.1__py3-none-any.whl → 1.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ECOv003_L2T_STARS/BRDF/BRDF.py +57 -0
- ECOv003_L2T_STARS/BRDF/SZA.py +65 -0
- ECOv003_L2T_STARS/BRDF/__init__.py +1 -0
- ECOv003_L2T_STARS/BRDF/statistical_radiative_transport.txt +90 -0
- ECOv003_L2T_STARS/BRDF/version.txt +1 -0
- ECOv003_L2T_STARS/ECOv003_DL.py +527 -0
- ECOv003_L2T_STARS/ECOv003_DL.xml +47 -0
- ECOv003_L2T_STARS/ECOv003_L2T_STARS.py +169 -0
- ECOv003_L2T_STARS/ECOv003_L2T_STARS.xml +47 -0
- ECOv003_L2T_STARS/L2TSTARSConfig.py +190 -0
- ECOv003_L2T_STARS/L2T_STARS.py +503 -0
- ECOv003_L2T_STARS/LPDAAC/LPDAACDataPool.py +444 -0
- ECOv003_L2T_STARS/LPDAAC/__init__.py +9 -0
- ECOv003_L2T_STARS/LPDAAC/version.txt +1 -0
- ECOv003_L2T_STARS/Manifest.toml +2332 -0
- ECOv003_L2T_STARS/Project.toml +14 -0
- ECOv003_L2T_STARS/VIIRS/VIIRSDataPool.py +294 -0
- ECOv003_L2T_STARS/VIIRS/VIIRSDownloader.py +26 -0
- ECOv003_L2T_STARS/VIIRS/VIIRS_CMR_LOGIN.py +36 -0
- ECOv003_L2T_STARS/VIIRS/VNP09GA.py +1278 -0
- ECOv003_L2T_STARS/VIIRS/VNP43IA4.py +288 -0
- ECOv003_L2T_STARS/VIIRS/VNP43MA3.py +323 -0
- ECOv003_L2T_STARS/VIIRS/__init__.py +9 -0
- ECOv003_L2T_STARS/VIIRS/version.txt +1 -0
- ECOv003_L2T_STARS/VNP43NRT/VNP43NRT.py +863 -0
- ECOv003_L2T_STARS/VNP43NRT/__init__.py +1 -0
- ECOv003_L2T_STARS/VNP43NRT/process_VNP43NRT.jl +169 -0
- ECOv003_L2T_STARS/VNP43NRT/version.txt +1 -0
- ECOv003_L2T_STARS/VNP43NRT_jl/Manifest.toml +995 -0
- ECOv003_L2T_STARS/VNP43NRT_jl/Project.toml +15 -0
- ECOv003_L2T_STARS/VNP43NRT_jl/__init__.py +0 -0
- ECOv003_L2T_STARS/VNP43NRT_jl/instantiate.jl +25 -0
- ECOv003_L2T_STARS/VNP43NRT_jl/instantiate.py +13 -0
- ECOv003_L2T_STARS/VNP43NRT_jl/src/VNP43NRT.jl +411 -0
- ECOv003_L2T_STARS/VNP43NRT_jl/src/__init__.py +0 -0
- ECOv003_L2T_STARS/__init__.py +3 -0
- ECOv003_L2T_STARS/calibrate_fine_to_coarse.py +60 -0
- ECOv003_L2T_STARS/constants.py +38 -0
- ECOv003_L2T_STARS/daterange/__init__.py +1 -0
- ECOv003_L2T_STARS/daterange/daterange.py +35 -0
- ECOv003_L2T_STARS/generate_L2T_STARS_runconfig.py +249 -0
- ECOv003_L2T_STARS/generate_NDVI_coarse_directory.py +21 -0
- ECOv003_L2T_STARS/generate_NDVI_coarse_image.py +30 -0
- ECOv003_L2T_STARS/generate_NDVI_fine_directory.py +14 -0
- ECOv003_L2T_STARS/generate_NDVI_fine_image.py +28 -0
- ECOv003_L2T_STARS/generate_STARS_inputs.py +231 -0
- ECOv003_L2T_STARS/generate_albedo_coarse_directory.py +18 -0
- ECOv003_L2T_STARS/generate_albedo_coarse_image.py +30 -0
- ECOv003_L2T_STARS/generate_albedo_fine_directory.py +17 -0
- ECOv003_L2T_STARS/generate_albedo_fine_image.py +30 -0
- ECOv003_L2T_STARS/generate_filename.py +37 -0
- ECOv003_L2T_STARS/generate_input_staging_directory.py +23 -0
- ECOv003_L2T_STARS/generate_model_state_tile_date_directory.py +28 -0
- ECOv003_L2T_STARS/generate_output_directory.py +28 -0
- ECOv003_L2T_STARS/install_STARS_jl.py +43 -0
- ECOv003_L2T_STARS/instantiate_STARS_jl.py +38 -0
- ECOv003_L2T_STARS/load_prior.py +250 -0
- ECOv003_L2T_STARS/prior.py +56 -0
- ECOv003_L2T_STARS/process_ECOSTRESS_data_fusion_distributed_bias.jl +420 -0
- ECOv003_L2T_STARS/process_STARS_product.py +507 -0
- ECOv003_L2T_STARS/process_julia_data_fusion.py +110 -0
- ECOv003_L2T_STARS/retrieve_STARS_sources.py +101 -0
- ECOv003_L2T_STARS/runconfig.py +70 -0
- ECOv003_L2T_STARS/timer/__init__.py +1 -0
- ECOv003_L2T_STARS/timer/timer.py +77 -0
- ECOv003_L2T_STARS/version.py +8 -0
- ECOv003_L2T_STARS/version.txt +1 -0
- {ECOv003_L2T_STARS-1.0.1.dist-info → ecov003_l2t_stars-1.2.0.dist-info}/METADATA +31 -24
- ecov003_l2t_stars-1.2.0.dist-info/RECORD +73 -0
- {ECOv003_L2T_STARS-1.0.1.dist-info → ecov003_l2t_stars-1.2.0.dist-info}/WHEEL +1 -1
- ecov003_l2t_stars-1.2.0.dist-info/entry_points.txt +3 -0
- ecov003_l2t_stars-1.2.0.dist-info/top_level.txt +1 -0
- ECOv003_L2T_STARS-1.0.1.dist-info/RECORD +0 -5
- ECOv003_L2T_STARS-1.0.1.dist-info/top_level.txt +0 -1
- {ECOv003_L2T_STARS-1.0.1.dist-info → ecov003_l2t_stars-1.2.0.dist-info/licenses}/LICENSE +0 -0
@@ -0,0 +1,527 @@
|
|
1
|
+
import logging
|
2
|
+
import socket
|
3
|
+
import sys
|
4
|
+
from datetime import datetime
|
5
|
+
from os import makedirs
|
6
|
+
from os.path import join, abspath, dirname, expanduser, exists, splitext, basename
|
7
|
+
from shutil import which
|
8
|
+
from typing import List
|
9
|
+
from uuid import uuid4
|
10
|
+
from dateutil import parser
|
11
|
+
import colored_logging as cl
|
12
|
+
import argparse # Import argparse for command-line argument parsing
|
13
|
+
|
14
|
+
from sentinel_tiles import SentinelTileGrid
|
15
|
+
|
16
|
+
from ECOv003_granules import L2TLSTE
|
17
|
+
from ECOv003_exit_codes import *
|
18
|
+
|
19
|
+
from .constants import *
|
20
|
+
from .generate_L2T_STARS_runconfig import generate_L2T_STARS_runconfig
|
21
|
+
from .runconfig import ECOSTRESSRunConfig, read_runconfig
|
22
|
+
from .L2T_STARS import L2T_STARS
|
23
|
+
|
24
|
+
# Read the version from the version.txt file
|
25
|
+
with open(join(abspath(dirname(__file__)), "version.txt")) as f:
|
26
|
+
version = f.read()
|
27
|
+
|
28
|
+
__version__ = version
|
29
|
+
|
30
|
+
# Configure the logger for the module
|
31
|
+
logger = logging.getLogger(__name__)
|
32
|
+
|
33
|
+
# Define the template for the ECOv003_DL run-config XML
|
34
|
+
ECOv003_DL_TEMPLATE = join(abspath(dirname(__file__)), "ECOv003_DL.xml")
|
35
|
+
# Default build ID
|
36
|
+
DEFAULT_BUILD = "0700"
|
37
|
+
|
38
|
+
def generate_downloader_runconfig(
|
39
|
+
L2G_LSTE_filename: str,
|
40
|
+
L2T_LSTE_filenames: List[str],
|
41
|
+
orbit: int = None,
|
42
|
+
scene: int = None,
|
43
|
+
working_directory: str = None,
|
44
|
+
L2T_STARS_sources_directory: str = None,
|
45
|
+
L2T_STARS_indices_directory: str = None,
|
46
|
+
L2T_STARS_model_directory: str = None,
|
47
|
+
executable_filename: str = None,
|
48
|
+
runconfig_filename: str = None,
|
49
|
+
log_filename: str = None,
|
50
|
+
build: str = None,
|
51
|
+
processing_node: str = None,
|
52
|
+
production_datetime: datetime = None,
|
53
|
+
job_ID: str = None,
|
54
|
+
instance_ID: str = None,
|
55
|
+
product_counter: int = None,
|
56
|
+
template_filename: str = None) -> str:
|
57
|
+
"""
|
58
|
+
Generates an ECOv003 Downloader run-config XML file.
|
59
|
+
|
60
|
+
Args:
|
61
|
+
L2G_LSTE_filename (str): Path to the L2G LSTE input file.
|
62
|
+
L2T_LSTE_filenames (List[str]): List of paths to L2T LSTE input files.
|
63
|
+
orbit (int, optional): Orbit number. Defaults to None, extracted from L2G filename.
|
64
|
+
scene (int, optional): Scene ID. Defaults to None, extracted from L2G filename.
|
65
|
+
working_directory (str, optional): Working directory for the PGE. Defaults to None,
|
66
|
+
derived from run ID.
|
67
|
+
L2T_STARS_sources_directory (str, optional): Directory for L2T STARS sources. Defaults to None,
|
68
|
+
derived from working directory.
|
69
|
+
L2T_STARS_indices_directory (str, optional): Directory for L2T STARS indices. Defaults to None,
|
70
|
+
derived from working directory.
|
71
|
+
L2T_STARS_model_directory (str, optional): Directory for L2T STARS model files. Defaults to None,
|
72
|
+
derived from working directory.
|
73
|
+
executable_filename (str, optional): Path to the ECOv003_DL executable. Defaults to None,
|
74
|
+
searches PATH or uses "ECOv003_DL".
|
75
|
+
runconfig_filename (str, optional): Output run-config filename. Defaults to None,
|
76
|
+
derived from working directory and run ID.
|
77
|
+
log_filename (str, optional): Output log filename. Defaults to None,
|
78
|
+
derived from working directory and run ID.
|
79
|
+
build (str, optional): Build ID. Defaults to DEFAULT_BUILD.
|
80
|
+
processing_node (str, optional): Name of the processing node. Defaults to current hostname.
|
81
|
+
production_datetime (datetime, optional): Production date and time. Defaults to UTC now.
|
82
|
+
job_ID (str, optional): Job ID. Defaults to production_datetime.
|
83
|
+
instance_ID (str, optional): Instance ID. Defaults to a new UUID.
|
84
|
+
product_counter (int, optional): Product counter. Defaults to 1.
|
85
|
+
template_filename (str, optional): Path to the run-config XML template. Defaults to ECOv003_DL_TEMPLATE.
|
86
|
+
|
87
|
+
Returns:
|
88
|
+
str: The absolute path to the generated run-config XML file.
|
89
|
+
|
90
|
+
Raises:
|
91
|
+
IOError: If the L2G LSTE file is not found.
|
92
|
+
ValueError: If no L2T LSTE filenames are provided.
|
93
|
+
"""
|
94
|
+
# Resolve absolute path for L2G LSTE filename
|
95
|
+
L2G_LSTE_filename = abspath(expanduser(L2G_LSTE_filename))
|
96
|
+
|
97
|
+
# Check if L2G LSTE file exists
|
98
|
+
if not exists(L2G_LSTE_filename):
|
99
|
+
raise IOError(f"L2G LSTE file not found: {L2G_LSTE_filename}")
|
100
|
+
|
101
|
+
logger.info(f"L2G LSTE file: {cl.file(L2G_LSTE_filename)}")
|
102
|
+
# Extract source granule ID from L2G LSTE filename
|
103
|
+
source_granule_ID = splitext(basename(L2G_LSTE_filename))[0]
|
104
|
+
logger.info(f"source granule ID: {cl.name(source_granule_ID)}")
|
105
|
+
|
106
|
+
# Determine orbit number
|
107
|
+
if orbit is None:
|
108
|
+
orbit = int(source_granule_ID.split("_")[-5])
|
109
|
+
logger.info(f"orbit: {cl.val(orbit)}")
|
110
|
+
|
111
|
+
# Determine scene ID
|
112
|
+
if scene is None:
|
113
|
+
scene = int(source_granule_ID.split("_")[-4])
|
114
|
+
logger.info(f"scene: {cl.val(scene)}")
|
115
|
+
|
116
|
+
# Set template filename
|
117
|
+
if template_filename is None:
|
118
|
+
template_filename = ECOv003_DL_TEMPLATE
|
119
|
+
template_filename = abspath(expanduser(template_filename))
|
120
|
+
|
121
|
+
# Generate run ID
|
122
|
+
run_ID = f"ECOv003_DL_{orbit:05d}_{scene:05d}"
|
123
|
+
|
124
|
+
# Determine working directory
|
125
|
+
if working_directory is None:
|
126
|
+
working_directory = run_ID
|
127
|
+
working_directory = abspath(expanduser(working_directory))
|
128
|
+
|
129
|
+
# Determine run-config filename
|
130
|
+
if runconfig_filename is None:
|
131
|
+
runconfig_filename = join(working_directory, "runconfig", f"{run_ID}.xml")
|
132
|
+
runconfig_filename = abspath(expanduser(runconfig_filename))
|
133
|
+
|
134
|
+
# Determine L2T STARS sources directory
|
135
|
+
if L2T_STARS_sources_directory is None:
|
136
|
+
L2T_STARS_sources_directory = join(working_directory, DEFAULT_STARS_SOURCES_DIRECTORY)
|
137
|
+
L2T_STARS_sources_directory = abspath(expanduser(L2T_STARS_sources_directory))
|
138
|
+
|
139
|
+
# Determine L2T STARS indices directory
|
140
|
+
if L2T_STARS_indices_directory is None:
|
141
|
+
L2T_STARS_indices_directory = join(working_directory, DEFAULT_STARS_INDICES_DIRECTORY)
|
142
|
+
L2T_STARS_indices_directory = abspath(expanduser(L2T_STARS_indices_directory))
|
143
|
+
|
144
|
+
# Determine L2T STARS model directory
|
145
|
+
if L2T_STARS_model_directory is None:
|
146
|
+
L2T_STARS_model_directory = join(working_directory, DEFAULT_STARS_MODEL_DIRECTORY)
|
147
|
+
L2T_STARS_model_directory = abspath(expanduser(L2T_STARS_model_directory))
|
148
|
+
|
149
|
+
# Determine executable filename
|
150
|
+
if executable_filename is None:
|
151
|
+
executable_filename = which("ECOv003_DL")
|
152
|
+
if executable_filename is None:
|
153
|
+
executable_filename = "ECOv003_DL" # Fallback if not found in PATH
|
154
|
+
|
155
|
+
# Determine log filename
|
156
|
+
if log_filename is None:
|
157
|
+
log_filename = join(working_directory, f"{run_ID}.log")
|
158
|
+
log_filename = abspath(expanduser(log_filename))
|
159
|
+
|
160
|
+
# Set build ID
|
161
|
+
if build is None:
|
162
|
+
build = DEFAULT_BUILD
|
163
|
+
|
164
|
+
# Set processing node
|
165
|
+
if processing_node is None:
|
166
|
+
processing_node = socket.gethostname()
|
167
|
+
|
168
|
+
# Set production datetime
|
169
|
+
if production_datetime is None:
|
170
|
+
production_datetime = datetime.utcnow()
|
171
|
+
# Convert datetime object to string if it's not already
|
172
|
+
if isinstance(production_datetime, datetime):
|
173
|
+
production_datetime = str(production_datetime)
|
174
|
+
|
175
|
+
# Set job ID
|
176
|
+
if job_ID is None:
|
177
|
+
job_ID = production_datetime
|
178
|
+
|
179
|
+
# Set instance ID
|
180
|
+
if instance_ID is None:
|
181
|
+
instance_ID = str(uuid4())
|
182
|
+
|
183
|
+
# Set product counter
|
184
|
+
if product_counter is None:
|
185
|
+
product_counter = 1
|
186
|
+
|
187
|
+
logger.info(f"generating run-config for orbit {cl.val(orbit)} scene {cl.val(scene)}")
|
188
|
+
logger.info(f"loading ECOv003_DL template: {cl.file(template_filename)}")
|
189
|
+
|
190
|
+
# Read the template file content
|
191
|
+
with open(template_filename, "r") as file:
|
192
|
+
template = file.read()
|
193
|
+
|
194
|
+
# Replace placeholders in the template with actual values
|
195
|
+
logger.info(f"orbit: {cl.val(orbit)}")
|
196
|
+
template = template.replace("orbit_number", f"{orbit:05d}")
|
197
|
+
logger.info(f"scene: {cl.val(scene)}")
|
198
|
+
template = template.replace("scene_ID", f"{scene:03d}")
|
199
|
+
|
200
|
+
# Ensure L2G_LSTE_filename is absolute
|
201
|
+
L2G_LSTE_filename = abspath(expanduser(L2G_LSTE_filename))
|
202
|
+
logger.info(f"L2G_LSTE file: {cl.file(L2G_LSTE_filename)}")
|
203
|
+
template = template.replace("L2G_LSTE_filename", L2G_LSTE_filename)
|
204
|
+
|
205
|
+
# Check if L2T LSTE filenames are provided
|
206
|
+
if len(L2T_LSTE_filenames) == 0:
|
207
|
+
raise ValueError(f"no L2T LSTE filenames given")
|
208
|
+
|
209
|
+
logger.info(f"listing {len(L2T_LSTE_filenames)} L2T_LSTE files: ")
|
210
|
+
|
211
|
+
# Format L2T LSTE filenames into XML elements
|
212
|
+
L2T_LSTE_filenames_XML = "\n ".join([
|
213
|
+
f"<element>{abspath(expanduser(filename))}</element>"
|
214
|
+
for filename
|
215
|
+
in L2T_LSTE_filenames
|
216
|
+
])
|
217
|
+
template = template.replace("<element>L2T_LSTE_filename1</element>", L2T_LSTE_filenames_XML)
|
218
|
+
|
219
|
+
logger.info(f"working directory: {cl.dir(working_directory)}")
|
220
|
+
template = template.replace("working_directory", working_directory)
|
221
|
+
logger.info(f"L2T STARS sources directory: {cl.dir(L2T_STARS_sources_directory)}")
|
222
|
+
template = template.replace("L2T_STARS_sources_directory", L2T_STARS_sources_directory)
|
223
|
+
logger.info(f"L2T STARS indices directory: {cl.dir(L2T_STARS_indices_directory)}")
|
224
|
+
template = template.replace("L2T_STARS_indices_directory", L2T_STARS_indices_directory)
|
225
|
+
logger.info(f"L2T STARS model directory: {cl.dir(L2T_STARS_model_directory)}")
|
226
|
+
template = template.replace("L2T_STARS_model_directory", L2T_STARS_model_directory)
|
227
|
+
logger.info(f"executable: {cl.file(executable_filename)}")
|
228
|
+
template = template.replace("executable_filename", executable_filename)
|
229
|
+
logger.info(f"run-config: {cl.file(runconfig_filename)}")
|
230
|
+
template = template.replace("runconfig_filename", runconfig_filename)
|
231
|
+
logger.info(f"log: {cl.file(log_filename)}")
|
232
|
+
template = template.replace("log_filename", log_filename)
|
233
|
+
logger.info(f"build: {cl.val(build)}")
|
234
|
+
template = template.replace("build_ID", build)
|
235
|
+
logger.info(f"processing node: {cl.val(processing_node)}")
|
236
|
+
template = template.replace("processing_node", processing_node)
|
237
|
+
logger.info(f"production date/time: {cl.time(production_datetime)}")
|
238
|
+
template = template.replace("production_datetime", production_datetime)
|
239
|
+
logger.info(f"job ID: {cl.val(job_ID)}")
|
240
|
+
template = template.replace("job_ID", job_ID)
|
241
|
+
logger.info(f"instance ID: {cl.val(instance_ID)}")
|
242
|
+
template = template.replace("instance_ID", instance_ID)
|
243
|
+
logger.info(f"product counter: {cl.val(product_counter)}")
|
244
|
+
template = template.replace("product_counter", f"{product_counter:02d}")
|
245
|
+
|
246
|
+
# Create directory for the run-config file if it doesn't exist
|
247
|
+
makedirs(dirname(abspath(runconfig_filename)), exist_ok=True)
|
248
|
+
logger.info(f"writing run-config file: {cl.file(runconfig_filename)}")
|
249
|
+
|
250
|
+
# Write the modified template to the run-config file
|
251
|
+
with open(runconfig_filename, "w") as file:
|
252
|
+
file.write(template)
|
253
|
+
|
254
|
+
return runconfig_filename
|
255
|
+
|
256
|
+
|
257
|
+
class ECOv003DLConfig(ECOSTRESSRunConfig):
|
258
|
+
"""
|
259
|
+
Parses and holds the configuration for the ECOv003 Downloader PGE from a run-config XML file.
|
260
|
+
Inherits from ECOSTRESSRunConfig for common run-config parsing functionalities.
|
261
|
+
"""
|
262
|
+
def __init__(self, filename: str):
|
263
|
+
try:
|
264
|
+
logger.info(f"loading ECOv003_DL run-config: {cl.file(filename)}")
|
265
|
+
runconfig = read_runconfig(filename)
|
266
|
+
|
267
|
+
# Validate and extract working directory
|
268
|
+
if "StaticAncillaryFileGroup" not in runconfig:
|
269
|
+
raise MissingRunConfigValue(f"missing StaticAncillaryFileGroup in ECOv003_DL run-config: {filename}")
|
270
|
+
if "ECOv003_DL_WORKING" not in runconfig["StaticAncillaryFileGroup"]:
|
271
|
+
raise MissingRunConfigValue(
|
272
|
+
f"missing StaticAncillaryFileGroup/ECOv003_DL_WORKING in ECOv003_DL run-config: {filename}")
|
273
|
+
working_directory = abspath(runconfig["StaticAncillaryFileGroup"]["ECOv003_DL_WORKING"])
|
274
|
+
logger.info(f"working directory: {cl.dir(working_directory)}")
|
275
|
+
|
276
|
+
# Validate and extract L2T STARS sources directory
|
277
|
+
if "L2T_STARS_SOURCES" not in runconfig["StaticAncillaryFileGroup"]:
|
278
|
+
raise MissingRunConfigValue(
|
279
|
+
f"missing StaticAncillaryFileGroup/L2T_STARS_SOURCES in ECOv003_DL run-config: {filename}")
|
280
|
+
L2T_STARS_sources_directory = abspath(runconfig["StaticAncillaryFileGroup"]["L2T_STARS_SOURCES"])
|
281
|
+
logger.info(f"L2T STARS sources directory: {cl.dir(L2T_STARS_sources_directory)}")
|
282
|
+
|
283
|
+
# Validate and extract L2T STARS indices directory
|
284
|
+
if "L2T_STARS_INDICES" not in runconfig["StaticAncillaryFileGroup"]:
|
285
|
+
raise MissingRunConfigValue(
|
286
|
+
f"missing StaticAncillaryFileGroup/L2T_STARS_INDICES in ECOv003_DL run-config: {filename}")
|
287
|
+
L2T_STARS_indices_directory = abspath(runconfig["StaticAncillaryFileGroup"]["L2T_STARS_INDICES"])
|
288
|
+
logger.info(f"L2T STARS indices directory: {cl.dir(L2T_STARS_indices_directory)}")
|
289
|
+
|
290
|
+
# Validate and extract L2T STARS model directory
|
291
|
+
if "L2T_STARS_MODEL" not in runconfig["StaticAncillaryFileGroup"]:
|
292
|
+
raise MissingRunConfigValue(
|
293
|
+
f"missing StaticAncillaryFileGroup/L2T_STARS_MODEL in ECOv003_DL run-config: {filename}")
|
294
|
+
L2T_STARS_model_directory = abspath(runconfig["StaticAncillaryFileGroup"]["L2T_STARS_MODEL"])
|
295
|
+
logger.info(f"L2T STARS model directory: {cl.dir(L2T_STARS_model_directory)}")
|
296
|
+
|
297
|
+
# Validate ProductPathGroup
|
298
|
+
if "ProductPathGroup" not in runconfig:
|
299
|
+
raise MissingRunConfigValue(f"missing ProductPathGroup in ECOv003_DL run-config: {filename}")
|
300
|
+
|
301
|
+
# Validate and extract InputFileGroup and L2G_LSTE
|
302
|
+
if "InputFileGroup" not in runconfig:
|
303
|
+
raise MissingRunConfigValue(f"missing InputFileGroup in ECOv003_DL run-config: {filename}")
|
304
|
+
if "L2G_LSTE" not in runconfig["InputFileGroup"]:
|
305
|
+
raise MissingRunConfigValue(f"missing InputFileGroup/L2G_LSTE in ECOv003_DL run-config: {filename}")
|
306
|
+
L2G_LSTE_filename = abspath(runconfig["InputFileGroup"]["L2G_LSTE"])
|
307
|
+
logger.info(f"L2G_LSTE file: {cl.file(L2G_LSTE_filename)}")
|
308
|
+
|
309
|
+
# Validate and extract L2T_LSTE filenames
|
310
|
+
if "L2T_LSTE" not in runconfig["InputFileGroup"]:
|
311
|
+
raise MissingRunConfigValue(
|
312
|
+
f"missing InputFileGroup/L2T_LSTE in ECOv003_DL run-config: {filename}")
|
313
|
+
L2T_LSTE_filenames = runconfig["InputFileGroup"]["L2T_LSTE"]
|
314
|
+
logger.info(f"reading {len(L2T_LSTE_filenames)} L2T_LSTE files")
|
315
|
+
|
316
|
+
# Extract orbit and scene IDs
|
317
|
+
orbit = int(runconfig["Geometry"]["OrbitNumber"])
|
318
|
+
logger.info(f"orbit: {cl.val(orbit)}")
|
319
|
+
if "SceneId" not in runconfig["Geometry"]:
|
320
|
+
raise MissingRunConfigValue(f"missing Geometry/SceneId in L2T_STARS run-config: {filename}")
|
321
|
+
scene = int(runconfig["Geometry"]["SceneId"])
|
322
|
+
logger.info(f"scene: {cl.val(scene)}")
|
323
|
+
|
324
|
+
# Extract build ID
|
325
|
+
if "BuildID" not in runconfig["PrimaryExecutable"]:
|
326
|
+
raise MissingRunConfigValue(
|
327
|
+
f"missing PrimaryExecutable/BuildID in L1_L2_RAD_LSTE run-config {filename}")
|
328
|
+
build = str(runconfig["PrimaryExecutable"]["BuildID"])
|
329
|
+
|
330
|
+
# Extract product counter
|
331
|
+
if "ProductCounter" not in runconfig["ProductPathGroup"]:
|
332
|
+
raise MissingRunConfigValue(
|
333
|
+
f"missing ProductPathGroup/ProductCounter in L1_L2_RAD_LSTE run-config {filename}")
|
334
|
+
product_counter = int(runconfig["ProductPathGroup"]["ProductCounter"])
|
335
|
+
|
336
|
+
# Determine UTC time from L2G LSTE filename (assuming a specific naming convention)
|
337
|
+
time_UTC = parser.parse(basename(L2G_LSTE_filename).split("_")[-3])
|
338
|
+
|
339
|
+
# Define PGE name and version
|
340
|
+
PGE_name = "DOWNLOADER"
|
341
|
+
PGE_version = __version__
|
342
|
+
|
343
|
+
# Store extracted configuration values as attributes
|
344
|
+
self.working_directory = working_directory
|
345
|
+
self.L2G_LSTE_filename = L2G_LSTE_filename
|
346
|
+
self.L2T_LSTE_filenames = L2T_LSTE_filenames
|
347
|
+
self.L2T_STARS_sources_directory = L2T_STARS_sources_directory
|
348
|
+
self.L2T_STARS_indices_directory = L2T_STARS_indices_directory
|
349
|
+
self.L2T_STARS_model_directory = L2T_STARS_model_directory
|
350
|
+
self.orbit = orbit
|
351
|
+
self.scene = scene
|
352
|
+
self.product_counter = product_counter
|
353
|
+
self.time_UTC = time_UTC
|
354
|
+
self.PGE_name = PGE_name
|
355
|
+
self.PGE_version = PGE_version
|
356
|
+
except MissingRunConfigValue as e:
|
357
|
+
# Re-raise specific run-config value errors
|
358
|
+
raise e
|
359
|
+
except ECOSTRESSExitCodeException as e:
|
360
|
+
# Re-raise specific ECOSTRESS exit code exceptions
|
361
|
+
raise e
|
362
|
+
except Exception as e:
|
363
|
+
# Catch all other exceptions and wrap them in UnableToParseRunConfig
|
364
|
+
logger.exception(e)
|
365
|
+
raise UnableToParseRunConfig(f"unable to parse run-config file: {filename}")
|
366
|
+
|
367
|
+
|
368
|
+
def ECOv003_DL(runconfig_filename: str, tiles: List[str] = None) -> int:
|
369
|
+
"""
|
370
|
+
ECOSTRESS Collection 3 Downloader PGE.
|
371
|
+
This function orchestrates the download process for data required for L2T LSTE product generation.
|
372
|
+
|
373
|
+
Args:
|
374
|
+
runconfig_filename (str): Filename for the XML run-config.
|
375
|
+
tiles (List[str], optional): A list of specific Sentinel tile IDs to process. If None, all tiles
|
376
|
+
listed in the run-config will be considered. Defaults to None.
|
377
|
+
|
378
|
+
Returns:
|
379
|
+
int: An exit code indicating the success or failure of the operation.
|
380
|
+
"""
|
381
|
+
exit_code = SUCCESS_EXIT_CODE
|
382
|
+
|
383
|
+
# Configure colored logging for better readability in the console
|
384
|
+
cl.configure()
|
385
|
+
# Get logger for this function (redundant if already configured globally, but good practice)
|
386
|
+
logger = logging.getLogger(__name__)
|
387
|
+
|
388
|
+
try:
|
389
|
+
logger.info(f"ECOSTRESS Collection 2 Downloader PGE ({cl.val(__version__)})")
|
390
|
+
logger.info(f"run-config: {cl.file(runconfig_filename)}")
|
391
|
+
# Load and parse the run-config file
|
392
|
+
runconfig = ECOv003DLConfig(runconfig_filename)
|
393
|
+
|
394
|
+
# Extract parameters from the loaded run-config
|
395
|
+
working_directory = runconfig.working_directory
|
396
|
+
logger.info(f"working directory: {cl.dir(working_directory)}")
|
397
|
+
L2T_STARS_sources_directory = runconfig.L2T_STARS_sources_directory
|
398
|
+
logger.info(f"L2T STARS sources directory: {cl.dir(L2T_STARS_sources_directory)}")
|
399
|
+
L2T_STARS_indices_directory = runconfig.L2T_STARS_indices_directory
|
400
|
+
logger.info(f"L2T STARS indices directory: {cl.dir(L2T_STARS_indices_directory)}")
|
401
|
+
L2T_STARS_model_directory = runconfig.L2T_STARS_model_directory
|
402
|
+
logger.info(f"L2T STARS model directory: {cl.dir(L2T_STARS_model_directory)}")
|
403
|
+
L2G_LSTE_filename = runconfig.L2G_LSTE_filename
|
404
|
+
logger.info(f"L2G LSTE file: {cl.file(L2G_LSTE_filename)}")
|
405
|
+
|
406
|
+
orbit = runconfig.orbit
|
407
|
+
logger.info(f"orbit: {cl.val(orbit)}")
|
408
|
+
scene = runconfig.scene
|
409
|
+
logger.info(f"scene: {cl.val(scene)}")
|
410
|
+
|
411
|
+
time_UTC = runconfig.time_UTC # Already parsed in ECOv003DLConfig
|
412
|
+
|
413
|
+
L2T_LSTE_filenames = runconfig.L2T_LSTE_filenames
|
414
|
+
logger.info(
|
415
|
+
f"processing {cl.val(len(L2T_LSTE_filenames))} tiles for orbit {cl.val(orbit)} scene {cl.val(scene)}")
|
416
|
+
|
417
|
+
# Iterate through each L2T LSTE filename (representing a tile)
|
418
|
+
for L2T_LSTE_filename in L2T_LSTE_filenames:
|
419
|
+
# Create an L2TLSTE granule object to extract tile information
|
420
|
+
L2T_LSTE_granule = L2TLSTE(L2T_LSTE_filename)
|
421
|
+
tile = L2T_LSTE_granule.tile
|
422
|
+
# Initialize SentinelTileGrid to check if the tile is on land
|
423
|
+
sentinel_tiles = SentinelTileGrid(target_resolution=70)
|
424
|
+
|
425
|
+
# Check if the Sentinel tile is on land
|
426
|
+
if not sentinel_tiles.land(tile):
|
427
|
+
logger.warning(f"Sentinel tile {tile} is not on land. Skipping processing for this tile.")
|
428
|
+
continue # Skip to the next tile
|
429
|
+
|
430
|
+
# If specific tiles are provided via arguments, skip if current tile is not in the list
|
431
|
+
if tiles is not None and tile not in tiles:
|
432
|
+
logger.info(f"Skipping tile {tile} as it's not in the specified --tiles list.")
|
433
|
+
continue
|
434
|
+
|
435
|
+
logger.info(f"L2T LSTE filename: {cl.file(L2T_LSTE_filename)}")
|
436
|
+
logger.info(f"orbit: {cl.val(orbit)} scene: {cl.val(scene)} tile: {cl.val(tile)}")
|
437
|
+
|
438
|
+
# Generate the run-config for the L2T_STARS process for the current tile
|
439
|
+
L2T_STARS_runconfig_filename = generate_L2T_STARS_runconfig(
|
440
|
+
L2T_LSTE_filename=L2T_LSTE_filename,
|
441
|
+
orbit=orbit,
|
442
|
+
scene=scene,
|
443
|
+
tile=tile,
|
444
|
+
time_UTC=time_UTC,
|
445
|
+
working_directory=working_directory,
|
446
|
+
sources_directory=L2T_STARS_sources_directory,
|
447
|
+
indices_directory=L2T_STARS_indices_directory,
|
448
|
+
model_directory=L2T_STARS_model_directory
|
449
|
+
)
|
450
|
+
|
451
|
+
# Execute the L2T_STARS process (downloader mode)
|
452
|
+
# The 'sources_only=True' argument indicates that this call is primarily for downloading
|
453
|
+
# or preparing necessary source data for STARS, not full processing.
|
454
|
+
exit_code = L2T_STARS(
|
455
|
+
runconfig_filename=L2T_STARS_runconfig_filename,
|
456
|
+
sources_only=True
|
457
|
+
)
|
458
|
+
|
459
|
+
# Handle specific exit codes from L2T_STARS
|
460
|
+
if exit_code == LAND_FILTER:
|
461
|
+
logger.warning(f"L2T_STARS reported that Sentinel tile {tile} is not on land. Skipping.")
|
462
|
+
continue # Continue to the next tile if it's a land filter issue
|
463
|
+
|
464
|
+
# If L2T_STARS failed for any other reason, return the error code immediately
|
465
|
+
if exit_code != 0:
|
466
|
+
logger.error(f"L2T_STARS failed for tile {tile} with exit code {exit_code}. Aborting.")
|
467
|
+
return exit_code
|
468
|
+
|
469
|
+
except ECOSTRESSExitCodeException as exception:
|
470
|
+
# Catch and log specific ECOSTRESS exit code exceptions
|
471
|
+
logger.exception(f"An ECOSTRESS specific error occurred: {exception}")
|
472
|
+
exit_code = exception.exit_code
|
473
|
+
|
474
|
+
return exit_code
|
475
|
+
|
476
|
+
|
477
|
+
def main(argv=sys.argv):
|
478
|
+
"""
|
479
|
+
Main entry point for the ECOv003_DL PGE.
|
480
|
+
Parses command-line arguments and initiates the downloader process.
|
481
|
+
"""
|
482
|
+
# Configure colored logging for the main function
|
483
|
+
cl.configure()
|
484
|
+
# Get logger for the main function
|
485
|
+
logger = logging.getLogger(__name__)
|
486
|
+
|
487
|
+
parser = argparse.ArgumentParser(
|
488
|
+
description=f"ECOSTRESS Collection 2 Downloader PGE ({__version__}).",
|
489
|
+
formatter_class=argparse.RawTextHelpFormatter
|
490
|
+
)
|
491
|
+
parser.add_argument(
|
492
|
+
"runconfig_filename",
|
493
|
+
type=str,
|
494
|
+
help="Path to the XML run-config file for ECOv003_DL."
|
495
|
+
)
|
496
|
+
parser.add_argument(
|
497
|
+
"--tiles",
|
498
|
+
nargs='*', # 0 or more arguments
|
499
|
+
default=None,
|
500
|
+
help="Optional: Space-separated list of specific Sentinel tile IDs (e.g., '30SWJ 30SXJ') "
|
501
|
+
"to process. If not provided, all tiles in the run-config will be processed."
|
502
|
+
)
|
503
|
+
parser.add_argument(
|
504
|
+
"--version",
|
505
|
+
action="version",
|
506
|
+
version=f"%(prog)s {__version__}",
|
507
|
+
help="Show program's version number and exit."
|
508
|
+
)
|
509
|
+
|
510
|
+
args = parser.parse_args(argv[1:]) # Parse arguments starting from the second element (skip script name)
|
511
|
+
|
512
|
+
# Validate that the runconfig file exists
|
513
|
+
if not exists(args.runconfig_filename):
|
514
|
+
logger.error(f"Run-config file not found: {cl.file(args.runconfig_filename)}")
|
515
|
+
return RUNCONFIG_FILENAME_NOT_SUPPLIED
|
516
|
+
|
517
|
+
logger.info(f"Starting ECOSTRESS Collection 2 Downloader PGE ({cl.val(__version__)})")
|
518
|
+
# Call the core ECOv003_DL function with parsed arguments
|
519
|
+
exit_code = ECOv003_DL(runconfig_filename=args.runconfig_filename, tiles=args.tiles)
|
520
|
+
logger.info(f"ECOSTRESS Collection 2 Downloader PGE finished with exit code: {cl.val(exit_code)}")
|
521
|
+
|
522
|
+
return exit_code
|
523
|
+
|
524
|
+
|
525
|
+
if __name__ == "__main__":
|
526
|
+
# Entry point when the script is executed directly
|
527
|
+
sys.exit(main(argv=sys.argv))
|
@@ -0,0 +1,47 @@
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
2
|
+
<input xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
3
|
+
xmlns:xalan="http://xml.apache.org/xslt"
|
4
|
+
xmlns:cas="http://oodt.jpl.nasa.gov/1.0/cas"
|
5
|
+
xsi:noNamespaceSchemaLocation="">
|
6
|
+
<group name="LogMetadata">
|
7
|
+
<vector name="CommandLineParameters">
|
8
|
+
<element>executable_filename</element>
|
9
|
+
<element>runconfig_filename</element>
|
10
|
+
<element>log_filename</element>
|
11
|
+
</vector>
|
12
|
+
</group>
|
13
|
+
<group name="JobIdentification">
|
14
|
+
<scalar name="ProductionLocation">ECOSTRESS Science Computing Facility</scalar>
|
15
|
+
<scalar name="TaskId">urn:ecostress:L2GL2TLSTETask</scalar>
|
16
|
+
<scalar name="ProcessingNode">processing_node</scalar>
|
17
|
+
<scalar name="ProductionDateTime">production_datetime</scalar>
|
18
|
+
<scalar name="JobId">job_ID</scalar>
|
19
|
+
<scalar name="WorkflowInstanceId">instance_ID</scalar>
|
20
|
+
</group>
|
21
|
+
<group name="Geometry">
|
22
|
+
<scalar name="SceneId">scene_ID</scalar>
|
23
|
+
<scalar name="OrbitNumber">orbit_number</scalar>
|
24
|
+
</group>
|
25
|
+
<group name="InputFileGroup">
|
26
|
+
<scalar name="L2G_LSTE">L2G_LSTE_filename</scalar>
|
27
|
+
<vector name="L2T_LSTE">
|
28
|
+
<element>L2T_LSTE_filename1</element>
|
29
|
+
</vector>
|
30
|
+
</group>
|
31
|
+
<group name="StaticAncillaryFileGroup">
|
32
|
+
<scalar name="ECOv003_DL_WORKING">working_directory</scalar>
|
33
|
+
<scalar name="L2T_STARS_SOURCES">L2T_STARS_sources_directory</scalar>
|
34
|
+
<scalar name="L2T_STARS_INDICES">L2T_STARS_indices_directory</scalar>
|
35
|
+
<scalar name="L2T_STARS_MODEL">L2T_STARS_model_directory</scalar>
|
36
|
+
</group>
|
37
|
+
<group name="PrimaryExecutable">
|
38
|
+
<scalar name="BuildID">build_ID</scalar>
|
39
|
+
<scalar name="FullPathname">executable_filename</scalar>
|
40
|
+
</group>
|
41
|
+
<group name="ProductPathGroup">
|
42
|
+
<scalar name="ProductCounter">product_counter</scalar>
|
43
|
+
</group>
|
44
|
+
<group name="PGENameGroup">
|
45
|
+
<scalar name="PGEName">ECOv003_DL</scalar>
|
46
|
+
</group>
|
47
|
+
</input>
|