rs-client-libraries 0.2a9__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,511 @@
1
+ # Copyright 2024 CS Group
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """Common workflows for general usage"""
16
+
17
+ import time
18
+ from datetime import datetime
19
+
20
+ import dateutil.parser
21
+ import numpy as np
22
+ import requests
23
+ from prefect import exceptions, flow, get_run_logger, task
24
+ from prefect_dask.task_runners import DaskTaskRunner
25
+ from pystac.asset import Asset
26
+ from pystac.item import Item
27
+ from shapely import geometry as geom
28
+
29
+ from rs_client.auxip_client import AuxipClient
30
+ from rs_client.cadip_client import CadipClient
31
+ from rs_client.stac_client import StacClient
32
+ from rs_common.config import (
33
+ AUXIP_STATION,
34
+ DATETIME_FORMAT_MS,
35
+ ECadipStation,
36
+ EDownloadStatus,
37
+ )
38
+ from rs_common.logging import Logging
39
+ from rs_common.serialization import RsClientSerialization
40
+
41
+ DOWNLOAD_FILE_TIMEOUT = 360 # in seconds
42
+ SET_PREFECT_LOGGING_LEVEL = "DEBUG"
43
+ ENDPOINT_TIMEOUT = 10 # in seconds
44
+ SEARCH_ENDPOINT_TIMEOUT = 60 # in seconds
45
+ CATALOG_REQUEST_TIMEOUT = 20 # in seconds
46
+
47
+
48
+ def get_prefect_logger(general_logger_name):
49
+ """
50
+ It returns the Prefect logger. If this can't be taken due to the missing
51
+ Prefect context (i.e. the flow/task is executed as a single function, from Pytest for example),
52
+ the general logger is returned
53
+
54
+ Args:
55
+ general_logger_name (str): The name of the general logger if Prefect logger can't be returned.
56
+
57
+ Returns:
58
+ logger (logger): A Prefect logger instance or a general logger instance.
59
+ """
60
+ try:
61
+ logger = get_run_logger()
62
+ logger.setLevel(SET_PREFECT_LOGGING_LEVEL)
63
+ except exceptions.MissingContextError:
64
+ logger = Logging.default(general_logger_name)
65
+ logger.warning("Could not get the prefect logger due to missing context. Using the general one")
66
+ return logger
67
+
68
+
69
+ @task
70
+ def update_stac_catalog( # pylint: disable=too-many-locals
71
+ stac_client: StacClient, # NOTE: maybe use RsClientSerialization instead
72
+ collection_name: str,
73
+ stac_file_info: dict,
74
+ obs: str,
75
+ **kwargs,
76
+ ):
77
+ """Update the STAC catalog with file information.
78
+
79
+ This task updates the STAC catalog with the information of a file that has been processed
80
+ and saved to a specific location. It adds the mandatory fields such as geometry, collection name and object storage
81
+ href. It sends a POST request to add the newly composed item into the STAC catalog.
82
+
83
+ Args:
84
+ stac_client (StacClient): The client for accessing the STAC catalog.
85
+ collection_name (str): The name of the collection in the STAC catalog.
86
+ stac_file_info (dict): Information about the STAC file to be updated.
87
+ obs (str): The bucket location where the file has been saved.
88
+
89
+ Returns:
90
+ bool (bool): True if the STAC catalog is successfully updated, False otherwise.
91
+
92
+ """
93
+ item_id = stac_file_info["id"]
94
+ now = datetime.now()
95
+
96
+ # The file path from the temp s3 bucket is given in the assets
97
+ assets = {}
98
+ update_assets = kwargs.get("update_assets", None)
99
+
100
+ if update_assets:
101
+ assets = {
102
+ stac_file_info["assets"][asset]["href"].split("/")[-1]: Asset(
103
+ href=f"{obs.rstrip('/')}/{stac_file_info['assets'][asset]['href']}",
104
+ )
105
+ for asset in stac_file_info["assets"]
106
+ }
107
+ else:
108
+ new_href_value = Asset(href=f"{obs.rstrip('/')}/{stac_file_info['id']}")
109
+ assets = (
110
+ {stac_file_info["id"]: new_href_value}
111
+ if "file" in stac_file_info["assets"]
112
+ else {
113
+ stac_file_info["id"].split("/")[-1]: new_href_value
114
+ for _, asset_value in stac_file_info["assets"].items()
115
+ if "href" in asset_value
116
+ }
117
+ )
118
+
119
+ # Copy properties from the input stac file, or use default values
120
+ properties = stac_file_info.get("properties") or {}
121
+ geometry = stac_file_info.get("geometry") or { # NOTE: override the geometry if it is set to None
122
+ "type": "Polygon",
123
+ "coordinates": [[[-180, -90], [180, -90], [180, 90], [-180, 90], [-180, -90]]],
124
+ }
125
+ bbox = stac_file_info.get("bbox") or [-180.0, -90.0, 180.0, 90.0]
126
+ datetime_value = now
127
+
128
+ # Check if the geometry is a valid polygon. If not we correct the geometry (Temporary solution)
129
+ if geometry["type"] == "Polygon":
130
+ poly = geom.Polygon(geometry["coordinates"][0])
131
+ if not (poly.is_valid and poly.is_closed):
132
+ coords = poly.exterior.coords.xy
133
+ geometry["coordinates"] = [[list(coord) for coord in zip(coords[0], coords[1])]]
134
+
135
+ # Try to format each property date or datetime
136
+ for key, value in properties.items():
137
+ try:
138
+ properties[key] = dateutil.parser.parse(value).strftime(DATETIME_FORMAT_MS)
139
+ # If this is not a datetime, do nothing
140
+ except (dateutil.parser.ParserError, TypeError):
141
+ pass
142
+
143
+ # Add item to the STAC catalog collection, check status is OK
144
+ item = Item(id=item_id, geometry=geometry, bbox=bbox, datetime=datetime_value, properties=properties, assets=assets)
145
+ try:
146
+ response = stac_client.add_item(collection_name, item, timeout=CATALOG_REQUEST_TIMEOUT)
147
+ except (requests.exceptions.RequestException, requests.exceptions.Timeout) as e:
148
+ stac_client.logger.exception("Request exception caught: %s", e)
149
+ return False
150
+ try:
151
+ if not response.ok:
152
+ stac_client.logger.error(f"The catalog update endpoint: {response.json()}")
153
+ except requests.exceptions.JSONDecodeError:
154
+ stac_client.logger.exception(f"Could not get the json body from response: {response}")
155
+ return response.status_code == 200
156
+
157
+
158
+ class PrefectCommonConfig: # pylint: disable=too-few-public-methods, too-many-instance-attributes,
159
+ """Common configuration for Prefect tasks and flows.
160
+ Base class for configuration used in Prefect task and flow
161
+ used in staging the files from different stations (cadip, adgs...)
162
+
163
+ Attributes:
164
+ rs_client (AuxipClient | CadipClient): The client for accessing the Auxip or Cadip service.
165
+ mission (str): The mission identifier.
166
+ tmp_download_path (str): The local path where temporary files will be downloaded.
167
+ s3_path (str): The S3 bucket path where the files will be stored.
168
+ """
169
+
170
+ def __init__(
171
+ self,
172
+ rs_client: AuxipClient | CadipClient,
173
+ mission,
174
+ tmp_download_path,
175
+ s3_path,
176
+ ):
177
+ self.rs_client: AuxipClient | CadipClient | None = None # don't save this instance
178
+ self.rs_client_serialization = RsClientSerialization(rs_client) # save the serialization parameters instead
179
+ self.mission: str = mission
180
+ self.tmp_download_path: str = tmp_download_path
181
+ self.s3_path: str = s3_path
182
+
183
+
184
+ class PrefectTaskConfig(PrefectCommonConfig): # pylint: disable=too-few-public-methods
185
+ """Configuration class for Prefect tasks.
186
+
187
+ This class extends the `PrefectCommonConfig` class with additional attributes
188
+ specific to Prefect tasks. It includes the configuration for the task, such as the
189
+ files to be processed by the task and the maximum number of retries allowed.
190
+
191
+ Attributes: see :py:class:`PrefectCommonConfig`
192
+ task_files_stac (List[Dict]): A list of dictionaries containing information about the files to be processed
193
+ by the task. This info is in STAC format
194
+ max_retries (int): The maximum number of retries allowed for the task.
195
+
196
+ """
197
+
198
+ def __init__( # pylint: disable=too-many-arguments, too-many-positional-arguments
199
+ self,
200
+ rs_client: AuxipClient | CadipClient,
201
+ mission,
202
+ tmp_download_path,
203
+ s3_path,
204
+ task_files_stac,
205
+ max_retries: int = 3,
206
+ ):
207
+ """
208
+ Initialize the PrefectTaskConfig object with provided parameters.
209
+ """
210
+
211
+ super().__init__(rs_client, mission, tmp_download_path, s3_path)
212
+ self.task_files_stac: list[dict] = task_files_stac
213
+ self.max_retries: int = max_retries
214
+
215
+
216
+ @task
217
+ def staging(config: PrefectTaskConfig):
218
+ """Prefect task function to stage (=download/ingest) files from a specified station.
219
+
220
+ This task function stages files for processing by calling the appropriate
221
+ rs-server endpoint for each requested file. It monitors the status of the file until it is
222
+ completely staged before initiating the next stage request.
223
+
224
+ Args:
225
+ config (PrefectTaskConfig): The configuration object containing the necessary parameters
226
+ for staging the files.
227
+
228
+ Returns:
229
+ List ([dict]): A list containing information about files that FAILED to be staged. The files within this
230
+ list don't appear in the catalog as published
231
+
232
+ """
233
+
234
+ logger = get_prefect_logger("task_dwn")
235
+
236
+ # Deserialize the RsClient instance
237
+ rs_client: AuxipClient | CadipClient = config.rs_client_serialization.deserialize(logger) # type: ignore
238
+
239
+ # list with failed files
240
+ failed_files = config.task_files_stac.copy()
241
+
242
+ downloaded_files_indices = []
243
+
244
+ # Call the download endpoint for each requested file
245
+ for i, file_stac in enumerate(config.task_files_stac):
246
+ # update the filename to be ingested
247
+ try:
248
+ rs_client.staging(file_stac["id"], config.s3_path, config.tmp_download_path, timeout=ENDPOINT_TIMEOUT)
249
+ except RuntimeError as e:
250
+ # TODO: Continue? Stop ?
251
+ logger.exception(f"Could not stage file %s. Exception: {e}")
252
+ continue
253
+
254
+ # monitor the status of the file until it is completely downloaded before initiating the next download request
255
+ status = rs_client.staging_status(file_stac["id"], ENDPOINT_TIMEOUT)
256
+ # just for the demo the timeout is hardcoded, it should be otherwise defined somewhere in the configuration
257
+ timeout = DOWNLOAD_FILE_TIMEOUT # 6 minutes
258
+ while status in [EDownloadStatus.NOT_STARTED, EDownloadStatus.IN_PROGRESS] and timeout > 0:
259
+ logger.info(
260
+ "The download progress for file %s is %s",
261
+ file_stac["id"],
262
+ status.name,
263
+ )
264
+ time.sleep(2)
265
+ timeout -= 2
266
+ status = rs_client.staging_status(file_stac["id"], timeout=ENDPOINT_TIMEOUT)
267
+ if status == EDownloadStatus.DONE:
268
+ logger.info("File %s has been properly downloaded...", file_stac["id"])
269
+ # TODO: either move the code from filter_unpublished_files to RsClient
270
+ # or use the new PgstacClient ?
271
+ if update_stac_catalog.fn(
272
+ rs_client.get_stac_client(), # NOTE: maybe use RsClientSerialization instead
273
+ create_collection_name(config.mission, rs_client.station_name),
274
+ file_stac,
275
+ config.s3_path,
276
+ ):
277
+ logger.info(f"File well published: {file_stac['id']}\n")
278
+ # save the index of the well ingested file
279
+ downloaded_files_indices.append(i)
280
+ else:
281
+ logger.error(f"Could not publish file: {file_stac['id']}")
282
+ else:
283
+ logger.error(
284
+ "Error in downloading the file %s (status %s). Timeout was %s from %s\n",
285
+ file_stac["id"],
286
+ status.name,
287
+ timeout,
288
+ DOWNLOAD_FILE_TIMEOUT,
289
+ )
290
+ # remove all the well ingested files
291
+ # return only those that failed
292
+ for idx in sorted(downloaded_files_indices, reverse=True):
293
+ del failed_files[idx]
294
+
295
+ return failed_files
296
+
297
+
298
+ @task
299
+ def filter_unpublished_files(
300
+ stac_client: StacClient, # NOTE: maybe use RsClientSerialization instead
301
+ collection_name: str,
302
+ files_stac: list,
303
+ ) -> list:
304
+ """Checks for unpublished files in the STAC catalog.
305
+
306
+ This function takes a list of files and checks if they are already published in a specified
307
+ STAC (SpatioTemporal Asset Catalog) collection. It returns a list of files that are not yet published.
308
+
309
+ Parameters:
310
+ stac_client (StacClient): An instance of `StacClient` to interact with the STAC catalog.
311
+ collection_name (str): The name of the collection in which the search is performed.
312
+ files_stac (list of dict): A list of files to be checked for publication. Each file is represented as a
313
+ dictionary with at least an "id" key.
314
+
315
+ Returns:
316
+ list of dict: A list of files that are not yet published in the catalog. Each file is represented as a
317
+ dictionary.
318
+
319
+ Examples:
320
+ >>> rs_client = StacClient(...)
321
+ >>> collection_name = "example_collection"
322
+ >>> files_stac = [{"id": "file1.raw"}, {"id": "file2.raw"}]
323
+ >>> unpublished_files = filter_unpublished_files(rs_client, collection_name, files_stac)
324
+ >>> print(unpublished_files)
325
+ [{"id": "file1.raw"}]
326
+ """
327
+
328
+ # Get files IDs (no duplicate IDs)
329
+ ids = set()
330
+ for fs in files_stac:
331
+ ids.add(str(fs["id"]))
332
+ ids = list(ids) # type: ignore # set to list conversion
333
+
334
+ # For searching, we need to prefix our collection name by <owner_id>_
335
+ owner_collection = f"{stac_client.owner_id}_{collection_name}"
336
+
337
+ # Search using a CQL2 filter, see: https://pystac-client.readthedocs.io/en/stable/tutorials/cql2-filter.html
338
+ filter_ = {
339
+ "op": "and",
340
+ "args": [
341
+ {"op": "=", "args": [{"property": "collection"}, owner_collection]},
342
+ {"op": "=", "args": [{"property": "owner"}, stac_client.owner_id]},
343
+ {"op": "in", "args": [{"property": "id"}, ids]},
344
+ ],
345
+ }
346
+ try:
347
+ search = stac_client.search(filter=filter_)
348
+ existing = list(search.items_as_dicts())
349
+
350
+ # In case of any error, try to ingest everything anyway
351
+ except NotImplementedError as e:
352
+ stac_client.logger.exception("Search exception caught: %s", e)
353
+ return files_stac
354
+
355
+ # Only keep the files that do not alreay exist in the catalog
356
+ existing_ids = [item["id"] for item in existing]
357
+ return [file_stac for file_stac in files_stac if file_stac["id"] not in existing_ids]
358
+
359
+
360
+ def create_collection_name(mission: str, station: str) -> str:
361
+ """Create the name of a catalog collection
362
+
363
+ This function constructs and returns a specific name for the catalog collection.
364
+ For ADGS station type should be "mission_name"_aux
365
+ For CADIP stations types should be "mission_name"_chunk
366
+
367
+ For other values, a RuntimeError is raised.
368
+
369
+ Args:
370
+ mission (str): The name of the mission.
371
+ station (str): The type of station . Supported
372
+ values are "AUX" and "CADIP", "INS", "MPS", "MTI", "NSG", "SGS".
373
+
374
+ Returns:
375
+ str (str): The name of the collection
376
+
377
+ Raises:
378
+ RuntimeError: If the provided station type is not supported.
379
+
380
+ """
381
+ if station == AUXIP_STATION:
382
+ return f"{mission}_aux"
383
+ # check CADIP
384
+ try:
385
+ ECadipStation(station)
386
+ return f"{mission}_chunk"
387
+ except ValueError as e:
388
+ raise RuntimeError(f"Unknown station: {station}") from e
389
+
390
+
391
+ class PrefectFlowConfig(PrefectCommonConfig): # pylint: disable=too-few-public-methods
392
+ """Configuration class for Prefect flow.
393
+
394
+ This class inherits the PrefectCommonConfig and represents the configuration for a
395
+ Prefect flow
396
+
397
+ Attributes: see :py:class:`PrefectCommonConfig`
398
+ max_workers (int): The maximum number of workers for the Prefect flow.
399
+ start_datetime (datetime): The start datetime of the files that the station should return
400
+ stop_datetime (datetime): The stop datetime of the files that the station should return
401
+ limit: The limit for the number of the files in the list retrieved from the ADGS/CADIP station (optional).
402
+ """
403
+
404
+ def __init__( # pylint: disable=too-many-arguments, too-many-positional-arguments
405
+ self,
406
+ rs_client: AuxipClient | CadipClient,
407
+ mission,
408
+ tmp_download_path,
409
+ s3_path,
410
+ max_workers,
411
+ start_datetime,
412
+ stop_datetime,
413
+ limit=None,
414
+ ):
415
+ """
416
+ Initialize the PrefectFlowConfig object with provided parameters.
417
+ """
418
+ super().__init__(rs_client, mission, tmp_download_path, s3_path)
419
+ self.max_workers: int = max_workers
420
+ self.start_datetime: datetime = start_datetime
421
+ self.stop_datetime: datetime = stop_datetime
422
+ self.limit = limit
423
+
424
+
425
+ # @flow # TO DEBUG THE CODE, JUST USE @flow
426
+ @flow(
427
+ flow_run_name="{config.rs_client_serialization.get_flow_name}",
428
+ task_runner=DaskTaskRunner(cluster_kwargs={"n_workers": 5, "threads_per_worker": 1}),
429
+ )
430
+ def staging_flow(config: PrefectFlowConfig):
431
+ """
432
+ Prefect flow for staging (downloading/ingesting) files from a station.
433
+
434
+ This flow orchestrates the download process by obtaining the list of files from the search endpoint (provided
435
+ station), splitting the list into tasks based on the number of workers, and submitting tasks for ingestion.
436
+
437
+ Args:
438
+ config (PrefectFlowConfig): Configuration object containing details about the download flow, such as
439
+ start and stop datetime, limit, rs_client, mission, temporary download path, S3 path, and max_workers.
440
+
441
+ Returns:
442
+ bool (bool): True if the flow execution is successful, False otherwise.
443
+
444
+ Raises:
445
+ None: This function does not raise any exceptions.
446
+ """
447
+ # get the Prefect logger
448
+ logger = get_prefect_logger("flow_dwn")
449
+ logger.info(f"The staging flow is starting. Received workers:{config.max_workers}")
450
+ try:
451
+ # Deserialize the RsClient instance
452
+ rs_client: AuxipClient | CadipClient = config.rs_client_serialization.deserialize(logger) # type: ignore
453
+
454
+ # get the list with files from the search endpoint
455
+ try:
456
+ files_stac = rs_client.search_stations(
457
+ config.start_datetime,
458
+ config.stop_datetime,
459
+ config.limit,
460
+ timeout=SEARCH_ENDPOINT_TIMEOUT,
461
+ )
462
+ except RuntimeError as e:
463
+ logger.exception(f"Unable to get the list with files for staging: {e}")
464
+ return False
465
+
466
+ # check if the list with files returned from the station is not empty
467
+ if len(files_stac) == 0:
468
+ logger.warning(
469
+ f"The station {rs_client.station_name} did not return any \
470
+ element for time interval {config.start_datetime} - {config.stop_datetime}",
471
+ )
472
+ return True
473
+ # create the collection name
474
+
475
+ # filter those that are already existing
476
+
477
+ files_stac = filter_unpublished_files( # type: ignore
478
+ rs_client.get_stac_client(), # NOTE: maybe use RsClientSerialization instead
479
+ create_collection_name(config.mission, rs_client.station_name),
480
+ files_stac,
481
+ wait_for=[files_stac],
482
+ )
483
+ # distribute the filenames evenly in a number of lists equal with
484
+ # the minimum between number of runners and files to be downloaded
485
+ try:
486
+ tasks_files_stac = [
487
+ x.tolist() for x in [*np.array_split(files_stac, min(config.max_workers, len(files_stac)))]
488
+ ]
489
+ except ValueError:
490
+ logger.warning("No task will be started, the requested number of tasks is 0 !")
491
+ tasks_files_stac = []
492
+ logger.info("List with files to be downloaded (after filtering against the catalog)")
493
+ for f in files_stac:
494
+ logger.info(" %s", f["id"])
495
+
496
+ for files_stac in tasks_files_stac:
497
+ staging.submit(
498
+ PrefectTaskConfig(
499
+ rs_client,
500
+ config.mission,
501
+ config.tmp_download_path,
502
+ config.s3_path,
503
+ files_stac,
504
+ ),
505
+ )
506
+
507
+ except (RuntimeError, TypeError) as e:
508
+ logger.error("Exception caught: %s", e)
509
+ return False
510
+
511
+ return True