rs-client-libraries 0.2a3.post31.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rs_client/__init__.py +18 -0
- rs_client/auxip_client.py +61 -0
- rs_client/cadip_client.py +142 -0
- rs_client/rs_client.py +433 -0
- rs_client/stac_client.py +390 -0
- rs_client_libraries-0.2a3.post31.dev0.dist-info/LICENSE +201 -0
- rs_client_libraries-0.2a3.post31.dev0.dist-info/METADATA +41 -0
- rs_client_libraries-0.2a3.post31.dev0.dist-info/RECORD +20 -0
- rs_client_libraries-0.2a3.post31.dev0.dist-info/WHEEL +4 -0
- rs_common/__init__.py +20 -0
- rs_common/config.py +62 -0
- rs_common/logging.py +108 -0
- rs_common/opentelemetry.py +114 -0
- rs_common/serialization.py +86 -0
- rs_common/utils.py +46 -0
- rs_workflows/__init__.py +18 -0
- rs_workflows/config/dpr_config_template.yaml +43 -0
- rs_workflows/example.py +43 -0
- rs_workflows/s1_l0.py +497 -0
- rs_workflows/staging.py +512 -0
rs_workflows/s1_l0.py
ADDED
|
@@ -0,0 +1,497 @@
|
|
|
1
|
+
# Copyright 2024 CS Group
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""Prefect flow for processing a S1 L0 product"""
|
|
16
|
+
import json
|
|
17
|
+
import os.path as osp
|
|
18
|
+
from datetime import datetime
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
from typing import List
|
|
21
|
+
|
|
22
|
+
import requests
|
|
23
|
+
import yaml
|
|
24
|
+
from prefect import flow, task
|
|
25
|
+
from prefect_dask.task_runners import DaskTaskRunner
|
|
26
|
+
from pystac import Collection, Extent, SpatialExtent, TemporalExtent
|
|
27
|
+
|
|
28
|
+
from rs_client.stac_client import StacClient
|
|
29
|
+
from rs_common.config import AUXIP_STATION, ECadipStation
|
|
30
|
+
from rs_common.logging import Logging
|
|
31
|
+
from rs_common.serialization import RsClientSerialization
|
|
32
|
+
from rs_workflows.staging import (
|
|
33
|
+
CATALOG_REQUEST_TIMEOUT,
|
|
34
|
+
create_collection_name,
|
|
35
|
+
get_prefect_logger,
|
|
36
|
+
update_stac_catalog,
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
CONFIG_DIR = Path(osp.realpath(osp.dirname(__file__))) / "config"
|
|
40
|
+
YAML_TEMPLATE_FILE = "dpr_config_template.yaml"
|
|
41
|
+
LOGGER_NAME = "s1_l0_wf"
|
|
42
|
+
DPR_PROCESSING_TIMEOUT = 14400 # 4 hours
|
|
43
|
+
asset_types = {"zarr": "zarr", "cog": "cog", ".nc": "netcdf"}
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@task
|
|
47
|
+
def start_dpr(dpr_endpoint: str, yaml_dpr_input: dict):
|
|
48
|
+
"""Starts the DPR processing with the given YAML input.
|
|
49
|
+
|
|
50
|
+
Args:
|
|
51
|
+
dpr_endpoint (str): The endpoint of the DPR processor
|
|
52
|
+
yaml_dpr_input (dict): The YAML input for DPR processing.
|
|
53
|
+
|
|
54
|
+
Returns:
|
|
55
|
+
response (dict): The response JSON from the DPR simulator if successful, else None.
|
|
56
|
+
"""
|
|
57
|
+
logger = Logging.default(LOGGER_NAME)
|
|
58
|
+
logger.debug("Task start_dpr STARTED")
|
|
59
|
+
logger.info("Faking dpr processing with the following input file:")
|
|
60
|
+
logger.debug(yaml.dump(yaml_dpr_input))
|
|
61
|
+
try:
|
|
62
|
+
response = requests.post(
|
|
63
|
+
dpr_endpoint.rstrip("/") + "/run",
|
|
64
|
+
json=yaml.safe_load(yaml.dump(yaml_dpr_input)),
|
|
65
|
+
timeout=DPR_PROCESSING_TIMEOUT,
|
|
66
|
+
)
|
|
67
|
+
except (requests.exceptions.RequestException, requests.exceptions.Timeout) as e:
|
|
68
|
+
logger.exception("Calling the dpr simulator resulted in exception: %s", e)
|
|
69
|
+
return None
|
|
70
|
+
|
|
71
|
+
if int(response.status_code) not in [201, 200]:
|
|
72
|
+
try:
|
|
73
|
+
body = json.dumps(response.json(), indent=2)
|
|
74
|
+
except requests.JSONDecodeError:
|
|
75
|
+
body = ""
|
|
76
|
+
logger.error(f"The dpr simulator endpoint failed with status code: {response.status_code}\n{body}")
|
|
77
|
+
return None
|
|
78
|
+
|
|
79
|
+
logger.debug("DPR processor results: \n\n")
|
|
80
|
+
for attr in response.json():
|
|
81
|
+
logger.debug(json.dumps(attr, indent=2))
|
|
82
|
+
logger.debug("Task start_dpr FINISHED")
|
|
83
|
+
return response.json()
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
@task
|
|
87
|
+
def build_eopf_triggering_yaml(cadip_files: dict, adgs_files: dict, product_types: list, temp_s3_path: str):
|
|
88
|
+
"""Builds the EOPF triggering YAML file using CADIP and ADGS file paths.
|
|
89
|
+
|
|
90
|
+
Args:
|
|
91
|
+
cadip_files (dict): CADIP files metadata.
|
|
92
|
+
adgs_files (dict): ADGS files metadata.
|
|
93
|
+
product_types (list): The types for the output products
|
|
94
|
+
temp_s3_path (str): Temporary S3 path.
|
|
95
|
+
|
|
96
|
+
Returns:
|
|
97
|
+
response (dict): The generated YAML template with updated inputs and I/O products, or None if unsuccessful.
|
|
98
|
+
Raises:
|
|
99
|
+
FileNotFoundError: If the YAML template file is not found.
|
|
100
|
+
yaml.YAMLError: If there is an error loading the YAML template file.
|
|
101
|
+
IOError: If there is an input/output error while accessing the YAML template file.
|
|
102
|
+
"""
|
|
103
|
+
logger = Logging.default(LOGGER_NAME)
|
|
104
|
+
logger.debug("Task build_eopf_triggering_yaml STARTED")
|
|
105
|
+
# Load YAML template
|
|
106
|
+
try:
|
|
107
|
+
with open(CONFIG_DIR / YAML_TEMPLATE_FILE, encoding="utf-8") as yaml_file:
|
|
108
|
+
yaml_template = yaml.safe_load(yaml_file)
|
|
109
|
+
except FileNotFoundError as e:
|
|
110
|
+
logger.exception(f"Could not find the YAML template file: {e}")
|
|
111
|
+
return None
|
|
112
|
+
except yaml.YAMLError as e:
|
|
113
|
+
logger.exception(f"Could not load the YAML template file: {e}")
|
|
114
|
+
return None
|
|
115
|
+
except IOError as e:
|
|
116
|
+
logger.exception(f"Could not find the YAML template file: {e}")
|
|
117
|
+
return None
|
|
118
|
+
|
|
119
|
+
# Extract paths for CADIP and ADGS files
|
|
120
|
+
cadip_paths = [
|
|
121
|
+
file_prop["assets"][asset]["alternate"]["s3"]["href"]
|
|
122
|
+
for file_prop in cadip_files
|
|
123
|
+
for asset in file_prop["assets"]
|
|
124
|
+
]
|
|
125
|
+
|
|
126
|
+
adgs_paths = [
|
|
127
|
+
file_prop["assets"][asset]["alternate"]["s3"]["href"]
|
|
128
|
+
for file_prop in adgs_files
|
|
129
|
+
for asset in file_prop["assets"]
|
|
130
|
+
]
|
|
131
|
+
|
|
132
|
+
# cadip_paths = [file_prop["assets"]["file"]["alternate"]["s3"]["href"] for file_prop in cadip_files]
|
|
133
|
+
# adgs_paths = [file_prop["assets"]["file"]["alternate"]["s3"]["href"] for file_prop in adgs_files]
|
|
134
|
+
# create the dictionaries to insert within the yaml template
|
|
135
|
+
|
|
136
|
+
# Update the YAML template with inputs and I/O products
|
|
137
|
+
yaml_template["workflow"][0]["inputs"], yaml_template["I/O"]["inputs_products"] = gen_payload_inputs(
|
|
138
|
+
cadip_paths,
|
|
139
|
+
adgs_paths,
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
# Update the YAML with the outpurs
|
|
143
|
+
yaml_template["workflow"][0]["parameters"]["product_types"] = product_types
|
|
144
|
+
yaml_template["workflow"][0]["outputs"], yaml_template["I/O"]["output_products"] = gen_payload_outputs(
|
|
145
|
+
product_types,
|
|
146
|
+
temp_s3_path,
|
|
147
|
+
)
|
|
148
|
+
|
|
149
|
+
logger.debug("Task build_eopf_triggering_yaml FINISHED")
|
|
150
|
+
return yaml_template
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def gen_payload_inputs(cadu_list: list, adgs_list: list):
|
|
154
|
+
"""Generate payload inputs for the EOPF triggering YAML file.
|
|
155
|
+
|
|
156
|
+
Args:
|
|
157
|
+
cadu_list (list): List of CADU file paths.
|
|
158
|
+
adgs_list (list): List of ADGS file paths.
|
|
159
|
+
|
|
160
|
+
Returns:
|
|
161
|
+
tuple ([list, list]): A tuple containing the composer dictionary and the YAML content list.
|
|
162
|
+
"""
|
|
163
|
+
input_body = []
|
|
164
|
+
composer = []
|
|
165
|
+
|
|
166
|
+
def add_input(file_list, prefix, start_cnt):
|
|
167
|
+
nonlocal composer
|
|
168
|
+
nonlocal input_body
|
|
169
|
+
|
|
170
|
+
for cnt, file in enumerate(file_list, start=start_cnt):
|
|
171
|
+
file_id = f"in{cnt}"
|
|
172
|
+
input_id = f"{prefix}{cnt}"
|
|
173
|
+
composer.append({file_id: input_id})
|
|
174
|
+
yaml_template = {"id": input_id, "path": file, "store_type": file.split(".")[-1], "store_params": {}}
|
|
175
|
+
input_body.append(yaml_template)
|
|
176
|
+
|
|
177
|
+
add_input(cadu_list, "CADU", 1)
|
|
178
|
+
add_input(adgs_list, "ADGS", len(cadu_list) + 1)
|
|
179
|
+
|
|
180
|
+
return composer, input_body
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def gen_payload_outputs(product_types, temp_s3_path: str):
|
|
184
|
+
"""Generate payload outputs for product types.
|
|
185
|
+
|
|
186
|
+
This function generates composer and output_body payloads for a list of product types,
|
|
187
|
+
each associated with a temporary S3 path.
|
|
188
|
+
|
|
189
|
+
Args:
|
|
190
|
+
product_types (list): A list of product types.
|
|
191
|
+
temp_s3_path (str): The temporary S3 path where the products will be stored.
|
|
192
|
+
|
|
193
|
+
Returns:
|
|
194
|
+
Tuple ([list, list]): A tuple containing composer and output_body payloads.
|
|
195
|
+
|
|
196
|
+
"""
|
|
197
|
+
composer = []
|
|
198
|
+
output_body = []
|
|
199
|
+
|
|
200
|
+
temp_s3_path = temp_s3_path.rstrip("/")
|
|
201
|
+
for typecnt, ptype in enumerate(product_types):
|
|
202
|
+
composer.append({f"out{typecnt}": ptype})
|
|
203
|
+
output_body.append(
|
|
204
|
+
{
|
|
205
|
+
"id": ptype,
|
|
206
|
+
"path": f"{temp_s3_path}/{ptype}/",
|
|
207
|
+
"type": "folder|zip",
|
|
208
|
+
"store_type": "zarr",
|
|
209
|
+
"store_params": {},
|
|
210
|
+
},
|
|
211
|
+
)
|
|
212
|
+
|
|
213
|
+
return composer, output_body
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def get_yaml_outputs(template: dict):
|
|
217
|
+
"""Extract the paths of YAML outputs from the template.
|
|
218
|
+
|
|
219
|
+
Args:
|
|
220
|
+
template (dict): The YAML template.
|
|
221
|
+
|
|
222
|
+
Returns:
|
|
223
|
+
list (list): A list of paths for YAML outputs.
|
|
224
|
+
"""
|
|
225
|
+
return [out["path"] for out in template["I/O"]["output_products"]]
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def create_cql2_filter(properties: dict, op: str = "and"):
|
|
229
|
+
"""
|
|
230
|
+
Create a CQL2 filter based on provided properties,
|
|
231
|
+
see: https://pystac-client.readthedocs.io/en/stable/tutorials/cql2-filter.html
|
|
232
|
+
|
|
233
|
+
Args:
|
|
234
|
+
properties (dict): Dictionary containing field-value pairs for filtering.
|
|
235
|
+
op (str, optional): Logical operator to combine filter conditions. Defaults to "and".
|
|
236
|
+
|
|
237
|
+
Returns:
|
|
238
|
+
ret (dict): CQL2 filter.
|
|
239
|
+
"""
|
|
240
|
+
args = [{"op": "=", "args": [{"property": field}, value]} for field, value in properties.items()]
|
|
241
|
+
# args.append("collecttion=test_user_s1_chunk")
|
|
242
|
+
# this filter is used for getting the files for one CADIP seesion id
|
|
243
|
+
# There can't be more than 60 files for a cadip session, so a hardcoded limit of 1000 seems
|
|
244
|
+
# to be ok for the time being
|
|
245
|
+
return {"filter-lang": "cql2-json", "limit": "1000", "filter": {"op": op, "args": args}}
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
@task
|
|
249
|
+
def get_cadip_catalog_data(
|
|
250
|
+
stac_client: StacClient, # NOTE: maybe use RsClientSerialization instead
|
|
251
|
+
collection: str,
|
|
252
|
+
session_id: str,
|
|
253
|
+
):
|
|
254
|
+
"""Prefect task to retrieve CADIP catalog data for a specific collection and session ID.
|
|
255
|
+
|
|
256
|
+
This task retrieves CADIP catalog data by sending a request to the CADIP
|
|
257
|
+
catalog search endpoint with a filter based on the collection and session ID.
|
|
258
|
+
|
|
259
|
+
Args:
|
|
260
|
+
stac_client (StacClient): The StacClient instance for accessing the CADIP catalog.
|
|
261
|
+
collection (str): The name of the collection.
|
|
262
|
+
session_id (str): The session ID associated with the collection.
|
|
263
|
+
|
|
264
|
+
Returns:
|
|
265
|
+
response (dict): The CADIP catalog data if retrieval is successful, otherwise None.
|
|
266
|
+
|
|
267
|
+
"""
|
|
268
|
+
|
|
269
|
+
logger = stac_client.logger
|
|
270
|
+
logger.debug("Task get_cadip_catalog_data STARTED")
|
|
271
|
+
|
|
272
|
+
_filter = create_cql2_filter({"collection": f"{stac_client.owner_id}_{collection}", "cadip:session_id": session_id})
|
|
273
|
+
|
|
274
|
+
search = stac_client.search(filter=_filter)
|
|
275
|
+
data = list(search.items_as_dicts())
|
|
276
|
+
logger.debug("Task get_cadip_catalog_data FINISHED")
|
|
277
|
+
return data
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
def merge_assets_to_one_feature(assets_dict: List[dict]) -> dict:
|
|
281
|
+
"""Combine output of DPR into one multi-asset STAC Feature."""
|
|
282
|
+
matching_stac = assets_dict[0]
|
|
283
|
+
|
|
284
|
+
# Create a dictionary of assets using comprehension
|
|
285
|
+
matching_stac["stac_discovery"]["assets"] = {
|
|
286
|
+
value: {"href": asset["stac_discovery"]["id"]}
|
|
287
|
+
for asset in assets_dict
|
|
288
|
+
for key, value in asset_types.items()
|
|
289
|
+
if key in asset["stac_discovery"]["id"]
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
return matching_stac
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
@task
|
|
296
|
+
def get_adgs_catalog_data(
|
|
297
|
+
stac_client: StacClient, # NOTE: maybe use RsClientSerialization instead
|
|
298
|
+
collection: str,
|
|
299
|
+
files: list,
|
|
300
|
+
):
|
|
301
|
+
"""Prefect task to retrieve ADGS catalog data for specific files in a collection.
|
|
302
|
+
|
|
303
|
+
This task retrieves ADGS catalog data by sending a request to the ADGS
|
|
304
|
+
catalog search endpoint with filters based on the collection and file IDs.
|
|
305
|
+
|
|
306
|
+
Args:
|
|
307
|
+
stac_client (StacClient): The StacClient instance for accessing the ADGS catalog.
|
|
308
|
+
collection (str): The name of the collection.
|
|
309
|
+
files (list): A list of file IDs to retrieve from the catalog.
|
|
310
|
+
|
|
311
|
+
Returns:
|
|
312
|
+
response (dict): The ADGS catalog data if retrieval is successful, otherwise None.
|
|
313
|
+
|
|
314
|
+
"""
|
|
315
|
+
|
|
316
|
+
logger = stac_client.logger
|
|
317
|
+
logger.debug("Task get_adgs_catalog_data STARTED")
|
|
318
|
+
|
|
319
|
+
_filter = {
|
|
320
|
+
"op": "and",
|
|
321
|
+
"args": [
|
|
322
|
+
{"op": "=", "args": [{"property": "collection"}, f"{stac_client.owner_id}_{collection}"]},
|
|
323
|
+
{"op": "=", "args": [{"property": "owner"}, stac_client.owner_id]},
|
|
324
|
+
{"op": "in", "args": [{"property": "id"}, files]},
|
|
325
|
+
],
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
search = stac_client.search(filter=_filter)
|
|
329
|
+
data = list(search.items_as_dicts())
|
|
330
|
+
logger.debug("Task get_adgs_catalog_data FINISHED")
|
|
331
|
+
return data
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
class PrefectS1L0FlowConfig: # pylint: disable=too-few-public-methods, too-many-instance-attributes
|
|
335
|
+
"""Configuration for Prefect flow related to S1 Level 0 data."""
|
|
336
|
+
|
|
337
|
+
def __init__( # pylint: disable=too-many-arguments
|
|
338
|
+
self,
|
|
339
|
+
stac_client: StacClient,
|
|
340
|
+
url_dpr: str,
|
|
341
|
+
mission: str,
|
|
342
|
+
cadip_session_id: str,
|
|
343
|
+
product_types: list,
|
|
344
|
+
adgs_files: list,
|
|
345
|
+
s3_path: str,
|
|
346
|
+
temp_s3_path: str,
|
|
347
|
+
):
|
|
348
|
+
"""
|
|
349
|
+
Initialize the PrefectS1L0FlowConfig object with provided parameters.
|
|
350
|
+
|
|
351
|
+
Args:
|
|
352
|
+
stac_client (StacClient): StacClient instance
|
|
353
|
+
url_dpr (str): The URL of the dpr endpoint
|
|
354
|
+
mission (str): The mission name.
|
|
355
|
+
cadip_session_id (str): The CADIP session ID.
|
|
356
|
+
product_types (list): The list with the products types to be processed by the DPR
|
|
357
|
+
adgs_files (list): ADGS files in the catalog.
|
|
358
|
+
s3_path (str): The S3 path.
|
|
359
|
+
temp_s3_path (str): The temporary S3 path.
|
|
360
|
+
"""
|
|
361
|
+
self.stac_client: StacClient | None = None # don't save this instance
|
|
362
|
+
self.rs_client_serialization = RsClientSerialization(stac_client) # save the serialization parameters instead
|
|
363
|
+
self.url_dpr = url_dpr
|
|
364
|
+
self.mission = mission
|
|
365
|
+
self.cadip_session_id = cadip_session_id
|
|
366
|
+
self.product_types = product_types
|
|
367
|
+
self.adgs_files = adgs_files
|
|
368
|
+
self.s3_path = s3_path
|
|
369
|
+
self.temp_s3_path = temp_s3_path
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
# At the time being, no more than 2 workers are required, because this flow runs at most 2 tasks in in parallel
|
|
373
|
+
# @flow # TO DEBUG THE CODE, JUST USE @flow
|
|
374
|
+
@flow(
|
|
375
|
+
flow_run_name="{config.rs_client_serialization.get_flow_name}",
|
|
376
|
+
task_runner=DaskTaskRunner(cluster_kwargs={"n_workers": 2, "threads_per_worker": 1}),
|
|
377
|
+
)
|
|
378
|
+
def s1_l0_flow(config: PrefectS1L0FlowConfig): # pylint: disable=too-many-locals
|
|
379
|
+
"""Constructs a Prefect Flow for Sentinel-1 Level 0 processing.
|
|
380
|
+
|
|
381
|
+
This flow oversees the execution of tasks involved in processing Sentinel-1 Level 0 data. It coordinates sequential
|
|
382
|
+
and parallel tasks, including retrieving catalog data concurrently from CADIP and ADGS stations, generating YAML
|
|
383
|
+
configuration based on the retrieved data, initiating the processing based on the YAML configuration, updating the
|
|
384
|
+
STAC catalog according to processing results, and publishing the processed files contingent upon catalog updates.
|
|
385
|
+
|
|
386
|
+
Args:
|
|
387
|
+
config (PrefectS1L0FlowConfig): Configuration for the flow.
|
|
388
|
+
|
|
389
|
+
"""
|
|
390
|
+
logger = get_prefect_logger(LOGGER_NAME)
|
|
391
|
+
|
|
392
|
+
# Deserialize the RsClient instance
|
|
393
|
+
config.stac_client = config.rs_client_serialization.deserialize(logger) # type: ignore
|
|
394
|
+
|
|
395
|
+
# TODO: the station for CADIP should come as an input
|
|
396
|
+
cadip_collection = create_collection_name(config.mission, ECadipStation["CADIP"])
|
|
397
|
+
adgs_collection = create_collection_name(config.mission, AUXIP_STATION)
|
|
398
|
+
logger.debug(f"Collections: {cadip_collection} | {adgs_collection}")
|
|
399
|
+
|
|
400
|
+
# gather the data for cadip session id
|
|
401
|
+
logger.debug("Starting task get_cadip_catalog_data")
|
|
402
|
+
cadip_catalog_data = get_cadip_catalog_data.submit(
|
|
403
|
+
config.stac_client, # type: ignore # NOTE: maybe use RsClientSerialization instead
|
|
404
|
+
cadip_collection,
|
|
405
|
+
config.cadip_session_id,
|
|
406
|
+
)
|
|
407
|
+
# logger.debug(f"cadip_catalog_data = {cadip_catalog_data} | {cadip_catalog_data.result()}")
|
|
408
|
+
logger.debug("Starting task get_adgs_catalog_data")
|
|
409
|
+
adgs_catalog_data = get_adgs_catalog_data.submit(
|
|
410
|
+
config.stac_client, # type: ignore # NOTE: maybe use RsClientSerialization instead
|
|
411
|
+
adgs_collection,
|
|
412
|
+
config.adgs_files,
|
|
413
|
+
)
|
|
414
|
+
|
|
415
|
+
# the previous tasks may be launched in parallel. The next task depends on the results from these previous tasks
|
|
416
|
+
if not cadip_catalog_data.result():
|
|
417
|
+
logger.error(f"No Cadip files were found in the catalog for Cadip session ID: {config.cadip_session_id!r}")
|
|
418
|
+
return
|
|
419
|
+
|
|
420
|
+
if not adgs_catalog_data.result():
|
|
421
|
+
logger.error( # pylint: disable=logging-not-lazy
|
|
422
|
+
"None of these Auxip files were found in the catalog:\n" + "\n".join(config.adgs_files),
|
|
423
|
+
)
|
|
424
|
+
return
|
|
425
|
+
|
|
426
|
+
logger.debug("Starting task build_eopf_triggering_yaml ")
|
|
427
|
+
yaml_dpr_input = build_eopf_triggering_yaml.submit(
|
|
428
|
+
cadip_catalog_data.result(),
|
|
429
|
+
adgs_catalog_data.result(),
|
|
430
|
+
config.product_types,
|
|
431
|
+
config.temp_s3_path,
|
|
432
|
+
wait_for=[cadip_catalog_data, adgs_catalog_data],
|
|
433
|
+
)
|
|
434
|
+
# the output product list should be :S1SEWRAW S1SIWRAW S1SSMRAW S1SWVRAW
|
|
435
|
+
# according to the jira story
|
|
436
|
+
|
|
437
|
+
# this task depends on the result from the previous task
|
|
438
|
+
logger.debug("Starting task start_dpr")
|
|
439
|
+
files_stac = start_dpr.submit(config.url_dpr, yaml_dpr_input.result(), wait_for=[yaml_dpr_input])
|
|
440
|
+
if not files_stac.result():
|
|
441
|
+
logger.error("DPR did not processed anything")
|
|
442
|
+
return
|
|
443
|
+
|
|
444
|
+
# The final collection where the processed products will be stored is assumed to be named "mission_dpr".
|
|
445
|
+
# However, it might be necessary to allow the users to input a specific collection name
|
|
446
|
+
# if they wish to do so. There is no established guide in the US for this matter.
|
|
447
|
+
collection_name = f"{config.mission}_dpr"
|
|
448
|
+
now = datetime.now()
|
|
449
|
+
config.stac_client.add_collection(
|
|
450
|
+
Collection(
|
|
451
|
+
id=collection_name,
|
|
452
|
+
description=None, # rs-client will provide a default description for us
|
|
453
|
+
extent=Extent(
|
|
454
|
+
spatial=SpatialExtent(bboxes=[-180.0, -90.0, 180.0, 90.0]),
|
|
455
|
+
temporal=TemporalExtent([now, now]),
|
|
456
|
+
),
|
|
457
|
+
license="esa-domain",
|
|
458
|
+
),
|
|
459
|
+
timeout=CATALOG_REQUEST_TIMEOUT,
|
|
460
|
+
)
|
|
461
|
+
|
|
462
|
+
fin_res = []
|
|
463
|
+
for output_product in get_yaml_outputs(yaml_dpr_input.result()):
|
|
464
|
+
assets = [d for d in files_stac.result() if d["stac_discovery"]["properties"]["eopf:type"] in output_product]
|
|
465
|
+
if len(assets) > 1:
|
|
466
|
+
matching_stac = merge_assets_to_one_feature(assets)
|
|
467
|
+
else:
|
|
468
|
+
matching_stac = assets[0]
|
|
469
|
+
assets_dict = {
|
|
470
|
+
asset_pt: {"href": ""} for asset_pt in asset_types if asset_pt in matching_stac["stac_discovery"]["id"]
|
|
471
|
+
}
|
|
472
|
+
# Check if assets_dict is empty, set default if so
|
|
473
|
+
matching_stac["stac_discovery"]["assets"] = assets_dict
|
|
474
|
+
|
|
475
|
+
# Update catalog (it moves the products from temporary bucket to the final one)
|
|
476
|
+
logger.info("Starting task update_stac_catalog")
|
|
477
|
+
fin_res.append(
|
|
478
|
+
(
|
|
479
|
+
update_stac_catalog.submit(
|
|
480
|
+
config.stac_client, # NOTE: maybe use RsClientSerialization instead
|
|
481
|
+
collection_name,
|
|
482
|
+
matching_stac["stac_discovery"],
|
|
483
|
+
output_product,
|
|
484
|
+
update_assets=len(assets) > 1,
|
|
485
|
+
wait_for=[files_stac],
|
|
486
|
+
),
|
|
487
|
+
output_product,
|
|
488
|
+
),
|
|
489
|
+
)
|
|
490
|
+
logger.debug("All tasks submitted")
|
|
491
|
+
for tsk in fin_res:
|
|
492
|
+
if tsk[0].result():
|
|
493
|
+
logger.info(f"File well published: {tsk[1]}")
|
|
494
|
+
else:
|
|
495
|
+
logger.error(f"File could not be published: {tsk[1]}")
|
|
496
|
+
|
|
497
|
+
logger.info("S1 L0 prefect flow finished")
|