rs-client-libraries 0.2a9__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
rs_workflows/s1_l0.py ADDED
@@ -0,0 +1,497 @@
1
+ # Copyright 2024 CS Group
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """Prefect flow for processing a S1 L0 product"""
16
+ import json
17
+ import os.path as osp
18
+ from datetime import datetime
19
+ from pathlib import Path
20
+ from typing import List
21
+
22
+ import requests
23
+ import yaml
24
+ from prefect import flow, task
25
+ from prefect_dask.task_runners import DaskTaskRunner
26
+ from pystac import Collection, Extent, SpatialExtent, TemporalExtent
27
+
28
+ from rs_client.stac_client import StacClient
29
+ from rs_common.config import AUXIP_STATION, ECadipStation
30
+ from rs_common.logging import Logging
31
+ from rs_common.serialization import RsClientSerialization
32
+ from rs_workflows.staging import (
33
+ CATALOG_REQUEST_TIMEOUT,
34
+ create_collection_name,
35
+ get_prefect_logger,
36
+ update_stac_catalog,
37
+ )
38
+
39
+ CONFIG_DIR = Path(osp.realpath(osp.dirname(__file__))) / "config"
40
+ YAML_TEMPLATE_FILE = "dpr_config_template.yaml"
41
+ LOGGER_NAME = "s1_l0_wf"
42
+ DPR_PROCESSING_TIMEOUT = 14400 # 4 hours
43
+ asset_types = {"zarr": "zarr", "cog": "cog", ".nc": "netcdf"}
44
+
45
+
46
+ @task
47
+ def start_dpr(dpr_endpoint: str, yaml_dpr_input: dict):
48
+ """Starts the DPR processing with the given YAML input.
49
+
50
+ Args:
51
+ dpr_endpoint (str): The endpoint of the DPR processor
52
+ yaml_dpr_input (dict): The YAML input for DPR processing.
53
+
54
+ Returns:
55
+ response (dict): The response JSON from the DPR simulator if successful, else None.
56
+ """
57
+ logger = Logging.default(LOGGER_NAME)
58
+ logger.debug("Task start_dpr STARTED")
59
+ logger.info("Faking dpr processing with the following input file:")
60
+ logger.debug(yaml.dump(yaml_dpr_input))
61
+ try:
62
+ response = requests.post(
63
+ dpr_endpoint.rstrip("/") + "/run",
64
+ json=yaml.safe_load(yaml.dump(yaml_dpr_input)),
65
+ timeout=DPR_PROCESSING_TIMEOUT,
66
+ )
67
+ except (requests.exceptions.RequestException, requests.exceptions.Timeout) as e:
68
+ logger.exception("Calling the dpr simulator resulted in exception: %s", e)
69
+ return None
70
+
71
+ if int(response.status_code) not in [201, 200]:
72
+ try:
73
+ body = json.dumps(response.json(), indent=2)
74
+ except requests.JSONDecodeError:
75
+ body = ""
76
+ logger.error(f"The dpr simulator endpoint failed with status code: {response.status_code}\n{body}")
77
+ return None
78
+
79
+ logger.debug("DPR processor results: \n\n")
80
+ for attr in response.json():
81
+ logger.debug(json.dumps(attr, indent=2))
82
+ logger.debug("Task start_dpr FINISHED")
83
+ return response.json()
84
+
85
+
86
+ @task
87
+ def build_eopf_triggering_yaml(cadip_files: dict, adgs_files: dict, product_types: list, temp_s3_path: str):
88
+ """Builds the EOPF triggering YAML file using CADIP and ADGS file paths.
89
+
90
+ Args:
91
+ cadip_files (dict): CADIP files metadata.
92
+ adgs_files (dict): ADGS files metadata.
93
+ product_types (list): The types for the output products
94
+ temp_s3_path (str): Temporary S3 path.
95
+
96
+ Returns:
97
+ response (dict): The generated YAML template with updated inputs and I/O products, or None if unsuccessful.
98
+ Raises:
99
+ FileNotFoundError: If the YAML template file is not found.
100
+ yaml.YAMLError: If there is an error loading the YAML template file.
101
+ IOError: If there is an input/output error while accessing the YAML template file.
102
+ """
103
+ logger = Logging.default(LOGGER_NAME)
104
+ logger.debug("Task build_eopf_triggering_yaml STARTED")
105
+ # Load YAML template
106
+ try:
107
+ with open(CONFIG_DIR / YAML_TEMPLATE_FILE, encoding="utf-8") as yaml_file:
108
+ yaml_template = yaml.safe_load(yaml_file)
109
+ except FileNotFoundError as e:
110
+ logger.exception(f"Could not find the YAML template file: {e}")
111
+ return None
112
+ except yaml.YAMLError as e:
113
+ logger.exception(f"Could not load the YAML template file: {e}")
114
+ return None
115
+ except IOError as e:
116
+ logger.exception(f"Could not find the YAML template file: {e}")
117
+ return None
118
+
119
+ # Extract paths for CADIP and ADGS files
120
+ cadip_paths = [
121
+ file_prop["assets"][asset]["alternate"]["s3"]["href"]
122
+ for file_prop in cadip_files
123
+ for asset in file_prop["assets"]
124
+ ]
125
+
126
+ adgs_paths = [
127
+ file_prop["assets"][asset]["alternate"]["s3"]["href"]
128
+ for file_prop in adgs_files
129
+ for asset in file_prop["assets"]
130
+ ]
131
+
132
+ # cadip_paths = [file_prop["assets"]["file"]["alternate"]["s3"]["href"] for file_prop in cadip_files]
133
+ # adgs_paths = [file_prop["assets"]["file"]["alternate"]["s3"]["href"] for file_prop in adgs_files]
134
+ # create the dictionaries to insert within the yaml template
135
+
136
+ # Update the YAML template with inputs and I/O products
137
+ yaml_template["workflow"][0]["inputs"], yaml_template["I/O"]["inputs_products"] = gen_payload_inputs(
138
+ cadip_paths,
139
+ adgs_paths,
140
+ )
141
+
142
+ # Update the YAML with the outpurs
143
+ yaml_template["workflow"][0]["parameters"]["product_types"] = product_types
144
+ yaml_template["workflow"][0]["outputs"], yaml_template["I/O"]["output_products"] = gen_payload_outputs(
145
+ product_types,
146
+ temp_s3_path,
147
+ )
148
+
149
+ logger.debug("Task build_eopf_triggering_yaml FINISHED")
150
+ return yaml_template
151
+
152
+
153
+ def gen_payload_inputs(cadu_list: list, adgs_list: list):
154
+ """Generate payload inputs for the EOPF triggering YAML file.
155
+
156
+ Args:
157
+ cadu_list (list): List of CADU file paths.
158
+ adgs_list (list): List of ADGS file paths.
159
+
160
+ Returns:
161
+ tuple ([list, list]): A tuple containing the composer dictionary and the YAML content list.
162
+ """
163
+ input_body = []
164
+ composer = []
165
+
166
+ def add_input(file_list, prefix, start_cnt):
167
+ nonlocal composer
168
+ nonlocal input_body
169
+
170
+ for cnt, file in enumerate(file_list, start=start_cnt):
171
+ file_id = f"in{cnt}"
172
+ input_id = f"{prefix}{cnt}"
173
+ composer.append({file_id: input_id})
174
+ yaml_template = {"id": input_id, "path": file, "store_type": file.split(".")[-1], "store_params": {}}
175
+ input_body.append(yaml_template)
176
+
177
+ add_input(cadu_list, "CADU", 1)
178
+ add_input(adgs_list, "ADGS", len(cadu_list) + 1)
179
+
180
+ return composer, input_body
181
+
182
+
183
+ def gen_payload_outputs(product_types, temp_s3_path: str):
184
+ """Generate payload outputs for product types.
185
+
186
+ This function generates composer and output_body payloads for a list of product types,
187
+ each associated with a temporary S3 path.
188
+
189
+ Args:
190
+ product_types (list): A list of product types.
191
+ temp_s3_path (str): The temporary S3 path where the products will be stored.
192
+
193
+ Returns:
194
+ Tuple ([list, list]): A tuple containing composer and output_body payloads.
195
+
196
+ """
197
+ composer = []
198
+ output_body = []
199
+
200
+ temp_s3_path = temp_s3_path.rstrip("/")
201
+ for typecnt, ptype in enumerate(product_types):
202
+ composer.append({f"out{typecnt}": ptype})
203
+ output_body.append(
204
+ {
205
+ "id": ptype,
206
+ "path": f"{temp_s3_path}/{ptype}/",
207
+ "type": "folder|zip",
208
+ "store_type": "zarr",
209
+ "store_params": {},
210
+ },
211
+ )
212
+
213
+ return composer, output_body
214
+
215
+
216
+ def get_yaml_outputs(template: dict):
217
+ """Extract the paths of YAML outputs from the template.
218
+
219
+ Args:
220
+ template (dict): The YAML template.
221
+
222
+ Returns:
223
+ list (list): A list of paths for YAML outputs.
224
+ """
225
+ return [out["path"] for out in template["I/O"]["output_products"]]
226
+
227
+
228
+ def create_cql2_filter(properties: dict, op: str = "and"):
229
+ """
230
+ Create a CQL2 filter based on provided properties,
231
+ see: https://pystac-client.readthedocs.io/en/stable/tutorials/cql2-filter.html
232
+
233
+ Args:
234
+ properties (dict): Dictionary containing field-value pairs for filtering.
235
+ op (str, optional): Logical operator to combine filter conditions. Defaults to "and".
236
+
237
+ Returns:
238
+ ret (dict): CQL2 filter.
239
+ """
240
+ args = [{"op": "=", "args": [{"property": field}, value]} for field, value in properties.items()]
241
+ # args.append("collecttion=test_user_s1_chunk")
242
+ # this filter is used for getting the files for one CADIP seesion id
243
+ # There can't be more than 60 files for a cadip session, so a hardcoded limit of 1000 seems
244
+ # to be ok for the time being
245
+ return {"filter-lang": "cql2-json", "limit": "1000", "filter": {"op": op, "args": args}}
246
+
247
+
248
+ @task
249
+ def get_cadip_catalog_data(
250
+ stac_client: StacClient, # NOTE: maybe use RsClientSerialization instead
251
+ collection: str,
252
+ session_id: str,
253
+ ):
254
+ """Prefect task to retrieve CADIP catalog data for a specific collection and session ID.
255
+
256
+ This task retrieves CADIP catalog data by sending a request to the CADIP
257
+ catalog search endpoint with a filter based on the collection and session ID.
258
+
259
+ Args:
260
+ stac_client (StacClient): The StacClient instance for accessing the CADIP catalog.
261
+ collection (str): The name of the collection.
262
+ session_id (str): The session ID associated with the collection.
263
+
264
+ Returns:
265
+ response (dict): The CADIP catalog data if retrieval is successful, otherwise None.
266
+
267
+ """
268
+
269
+ logger = stac_client.logger
270
+ logger.debug("Task get_cadip_catalog_data STARTED")
271
+
272
+ _filter = create_cql2_filter({"collection": f"{stac_client.owner_id}_{collection}", "cadip:session_id": session_id})
273
+
274
+ search = stac_client.search(filter=_filter)
275
+ data = list(search.items_as_dicts())
276
+ logger.debug("Task get_cadip_catalog_data FINISHED")
277
+ return data
278
+
279
+
280
+ def merge_assets_to_one_feature(assets_dict: List[dict]) -> dict:
281
+ """Combine output of DPR into one multi-asset STAC Feature."""
282
+ matching_stac = assets_dict[0]
283
+
284
+ # Create a dictionary of assets using comprehension
285
+ matching_stac["stac_discovery"]["assets"] = {
286
+ value: {"href": asset["stac_discovery"]["id"]}
287
+ for asset in assets_dict
288
+ for key, value in asset_types.items()
289
+ if key in asset["stac_discovery"]["id"]
290
+ }
291
+
292
+ return matching_stac
293
+
294
+
295
+ @task
296
+ def get_adgs_catalog_data(
297
+ stac_client: StacClient, # NOTE: maybe use RsClientSerialization instead
298
+ collection: str,
299
+ files: list,
300
+ ):
301
+ """Prefect task to retrieve ADGS catalog data for specific files in a collection.
302
+
303
+ This task retrieves ADGS catalog data by sending a request to the ADGS
304
+ catalog search endpoint with filters based on the collection and file IDs.
305
+
306
+ Args:
307
+ stac_client (StacClient): The StacClient instance for accessing the ADGS catalog.
308
+ collection (str): The name of the collection.
309
+ files (list): A list of file IDs to retrieve from the catalog.
310
+
311
+ Returns:
312
+ response (dict): The ADGS catalog data if retrieval is successful, otherwise None.
313
+
314
+ """
315
+
316
+ logger = stac_client.logger
317
+ logger.debug("Task get_adgs_catalog_data STARTED")
318
+
319
+ _filter = {
320
+ "op": "and",
321
+ "args": [
322
+ {"op": "=", "args": [{"property": "collection"}, f"{stac_client.owner_id}_{collection}"]},
323
+ {"op": "=", "args": [{"property": "owner"}, stac_client.owner_id]},
324
+ {"op": "in", "args": [{"property": "id"}, files]},
325
+ ],
326
+ }
327
+
328
+ search = stac_client.search(filter=_filter)
329
+ data = list(search.items_as_dicts())
330
+ logger.debug("Task get_adgs_catalog_data FINISHED")
331
+ return data
332
+
333
+
334
+ class PrefectS1L0FlowConfig: # pylint: disable=too-few-public-methods, too-many-instance-attributes
335
+ """Configuration for Prefect flow related to S1 Level 0 data."""
336
+
337
+ def __init__( # pylint: disable=too-many-arguments, too-many-positional-arguments
338
+ self,
339
+ stac_client: StacClient,
340
+ url_dpr: str,
341
+ mission: str,
342
+ cadip_session_id: str,
343
+ product_types: list,
344
+ adgs_files: list,
345
+ s3_path: str,
346
+ temp_s3_path: str,
347
+ ):
348
+ """
349
+ Initialize the PrefectS1L0FlowConfig object with provided parameters.
350
+
351
+ Args:
352
+ stac_client (StacClient): StacClient instance
353
+ url_dpr (str): The URL of the dpr endpoint
354
+ mission (str): The mission name.
355
+ cadip_session_id (str): The CADIP session ID.
356
+ product_types (list): The list with the products types to be processed by the DPR
357
+ adgs_files (list): ADGS files in the catalog.
358
+ s3_path (str): The S3 path.
359
+ temp_s3_path (str): The temporary S3 path.
360
+ """
361
+ self.stac_client: StacClient | None = None # don't save this instance
362
+ self.rs_client_serialization = RsClientSerialization(stac_client) # save the serialization parameters instead
363
+ self.url_dpr = url_dpr
364
+ self.mission = mission
365
+ self.cadip_session_id = cadip_session_id
366
+ self.product_types = product_types
367
+ self.adgs_files = adgs_files
368
+ self.s3_path = s3_path
369
+ self.temp_s3_path = temp_s3_path
370
+
371
+
372
+ # At the time being, no more than 2 workers are required, because this flow runs at most 2 tasks in in parallel
373
+ # @flow # TO DEBUG THE CODE, JUST USE @flow
374
+ @flow(
375
+ flow_run_name="{config.rs_client_serialization.get_flow_name}",
376
+ task_runner=DaskTaskRunner(cluster_kwargs={"n_workers": 2, "threads_per_worker": 1}),
377
+ )
378
+ def s1_l0_flow(config: PrefectS1L0FlowConfig): # pylint: disable=too-many-locals
379
+ """Constructs a Prefect Flow for Sentinel-1 Level 0 processing.
380
+
381
+ This flow oversees the execution of tasks involved in processing Sentinel-1 Level 0 data. It coordinates sequential
382
+ and parallel tasks, including retrieving catalog data concurrently from CADIP and ADGS stations, generating YAML
383
+ configuration based on the retrieved data, initiating the processing based on the YAML configuration, updating the
384
+ STAC catalog according to processing results, and publishing the processed files contingent upon catalog updates.
385
+
386
+ Args:
387
+ config (PrefectS1L0FlowConfig): Configuration for the flow.
388
+
389
+ """
390
+ logger = get_prefect_logger(LOGGER_NAME)
391
+
392
+ # Deserialize the RsClient instance
393
+ config.stac_client = config.rs_client_serialization.deserialize(logger) # type: ignore
394
+
395
+ # TODO: the station for CADIP should come as an input
396
+ cadip_collection = create_collection_name(config.mission, ECadipStation["CADIP"])
397
+ adgs_collection = create_collection_name(config.mission, AUXIP_STATION)
398
+ logger.debug(f"Collections: {cadip_collection} | {adgs_collection}")
399
+
400
+ # gather the data for cadip session id
401
+ logger.debug("Starting task get_cadip_catalog_data")
402
+ cadip_catalog_data = get_cadip_catalog_data.submit(
403
+ config.stac_client, # type: ignore # NOTE: maybe use RsClientSerialization instead
404
+ cadip_collection,
405
+ config.cadip_session_id,
406
+ )
407
+ # logger.debug(f"cadip_catalog_data = {cadip_catalog_data} | {cadip_catalog_data.result()}")
408
+ logger.debug("Starting task get_adgs_catalog_data")
409
+ adgs_catalog_data = get_adgs_catalog_data.submit(
410
+ config.stac_client, # type: ignore # NOTE: maybe use RsClientSerialization instead
411
+ adgs_collection,
412
+ config.adgs_files,
413
+ )
414
+
415
+ # the previous tasks may be launched in parallel. The next task depends on the results from these previous tasks
416
+ if not cadip_catalog_data.result():
417
+ logger.error(f"No Cadip files were found in the catalog for Cadip session ID: {config.cadip_session_id!r}")
418
+ return
419
+
420
+ if not adgs_catalog_data.result():
421
+ logger.error( # pylint: disable=logging-not-lazy
422
+ "None of these Auxip files were found in the catalog:\n" + "\n".join(config.adgs_files),
423
+ )
424
+ return
425
+
426
+ logger.debug("Starting task build_eopf_triggering_yaml ")
427
+ yaml_dpr_input = build_eopf_triggering_yaml.submit(
428
+ cadip_catalog_data.result(),
429
+ adgs_catalog_data.result(),
430
+ config.product_types,
431
+ config.temp_s3_path,
432
+ wait_for=[cadip_catalog_data, adgs_catalog_data],
433
+ )
434
+ # the output product list should be :S1SEWRAW S1SIWRAW S1SSMRAW S1SWVRAW
435
+ # according to the jira story
436
+
437
+ # this task depends on the result from the previous task
438
+ logger.debug("Starting task start_dpr")
439
+ files_stac = start_dpr.submit(config.url_dpr, yaml_dpr_input.result(), wait_for=[yaml_dpr_input])
440
+ if not files_stac.result():
441
+ logger.error("DPR did not processed anything")
442
+ return
443
+
444
+ # The final collection where the processed products will be stored is assumed to be named "mission_dpr".
445
+ # However, it might be necessary to allow the users to input a specific collection name
446
+ # if they wish to do so. There is no established guide in the US for this matter.
447
+ collection_name = f"{config.mission}_dpr"
448
+ now = datetime.now()
449
+ config.stac_client.add_collection(
450
+ Collection(
451
+ id=collection_name,
452
+ description=None, # rs-client will provide a default description for us
453
+ extent=Extent(
454
+ spatial=SpatialExtent(bboxes=[-180.0, -90.0, 180.0, 90.0]),
455
+ temporal=TemporalExtent([now, now]),
456
+ ),
457
+ license="esa-domain",
458
+ ),
459
+ timeout=CATALOG_REQUEST_TIMEOUT,
460
+ )
461
+
462
+ fin_res = []
463
+ for output_product in get_yaml_outputs(yaml_dpr_input.result()):
464
+ assets = [d for d in files_stac.result() if d["stac_discovery"]["properties"]["eopf:type"] in output_product]
465
+ if len(assets) > 1:
466
+ matching_stac = merge_assets_to_one_feature(assets)
467
+ else:
468
+ matching_stac = assets[0]
469
+ assets_dict = {
470
+ asset_pt: {"href": ""} for asset_pt in asset_types if asset_pt in matching_stac["stac_discovery"]["id"]
471
+ }
472
+ # Check if assets_dict is empty, set default if so
473
+ matching_stac["stac_discovery"]["assets"] = assets_dict
474
+
475
+ # Update catalog (it moves the products from temporary bucket to the final one)
476
+ logger.info("Starting task update_stac_catalog")
477
+ fin_res.append(
478
+ (
479
+ update_stac_catalog.submit(
480
+ config.stac_client, # NOTE: maybe use RsClientSerialization instead
481
+ collection_name,
482
+ matching_stac["stac_discovery"],
483
+ output_product,
484
+ update_assets=len(assets) > 1,
485
+ wait_for=[files_stac],
486
+ ),
487
+ output_product,
488
+ ),
489
+ )
490
+ logger.debug("All tasks submitted")
491
+ for tsk in fin_res:
492
+ if tsk[0].result():
493
+ logger.info(f"File well published: {tsk[1]}")
494
+ else:
495
+ logger.error(f"File could not be published: {tsk[1]}")
496
+
497
+ logger.info("S1 L0 prefect flow finished")