rs-client-libraries 0.2a3.post31.dev0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,512 @@
1
+ # Copyright 2024 CS Group
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """Common workflows for general usage"""
16
+
17
+ import time
18
+ from datetime import datetime
19
+ from shapely import geometry as geom
20
+
21
+ import dateutil.parser
22
+ import numpy as np
23
+ import requests
24
+ from prefect import exceptions, flow, get_run_logger, task
25
+ from prefect_dask.task_runners import DaskTaskRunner
26
+ from pystac.asset import Asset
27
+ from pystac.item import Item
28
+
29
+ from rs_client.auxip_client import AuxipClient
30
+ from rs_client.cadip_client import CadipClient
31
+ from rs_client.stac_client import StacClient
32
+ from rs_common.config import (
33
+ AUXIP_STATION,
34
+ DATETIME_FORMAT_MS,
35
+ ECadipStation,
36
+ EDownloadStatus,
37
+ )
38
+ from rs_common.logging import Logging
39
+ from rs_common.serialization import RsClientSerialization
40
+
41
+ DOWNLOAD_FILE_TIMEOUT = 360 # in seconds
42
+ SET_PREFECT_LOGGING_LEVEL = "DEBUG"
43
+ ENDPOINT_TIMEOUT = 10 # in seconds
44
+ SEARCH_ENDPOINT_TIMEOUT = 60 # in seconds
45
+ CATALOG_REQUEST_TIMEOUT = 20 # in seconds
46
+
47
+
48
+ def get_prefect_logger(general_logger_name):
49
+ """
50
+ It returns the Prefect logger. If this can't be taken due to the missing
51
+ Prefect context (i.e. the flow/task is executed as a single function, from Pytest for example),
52
+ the general logger is returned
53
+
54
+ Args:
55
+ general_logger_name (str): The name of the general logger if Prefect logger can't be returned.
56
+
57
+ Returns:
58
+ logger (logger): A Prefect logger instance or a general logger instance.
59
+ """
60
+ try:
61
+ logger = get_run_logger()
62
+ logger.setLevel(SET_PREFECT_LOGGING_LEVEL)
63
+ except exceptions.MissingContextError:
64
+ logger = Logging.default(general_logger_name)
65
+ logger.warning("Could not get the prefect logger due to missing context. Using the general one")
66
+ return logger
67
+
68
+
69
+ @task
70
+ def update_stac_catalog( # pylint: disable=too-many-locals
71
+ stac_client: StacClient, # NOTE: maybe use RsClientSerialization instead
72
+ collection_name: str,
73
+ stac_file_info: dict,
74
+ obs: str,
75
+ **kwargs,
76
+ ):
77
+ """Update the STAC catalog with file information.
78
+
79
+ This task updates the STAC catalog with the information of a file that has been processed
80
+ and saved to a specific location. It adds the mandatory fields such as geometry, collection name and object storage
81
+ href. It sends a POST request to add the newly composed item into the STAC catalog.
82
+
83
+ Args:
84
+ stac_client (StacClient): The client for accessing the STAC catalog.
85
+ collection_name (str): The name of the collection in the STAC catalog.
86
+ stac_file_info (dict): Information about the STAC file to be updated.
87
+ obs (str): The bucket location where the file has been saved.
88
+
89
+ Returns:
90
+ bool (bool): True if the STAC catalog is successfully updated, False otherwise.
91
+
92
+ """
93
+ item_id = stac_file_info["id"]
94
+ now = datetime.now()
95
+
96
+ # The file path from the temp s3 bucket is given in the assets
97
+ assets = {}
98
+ update_assets = kwargs.get("update_assets", None)
99
+
100
+ if update_assets:
101
+ assets = {
102
+ stac_file_info["assets"][asset]["href"].split("/")[-1]: Asset(
103
+ href=f"{obs.rstrip('/')}/{stac_file_info['assets'][asset]['href']}",
104
+ )
105
+ for asset in stac_file_info["assets"]
106
+ }
107
+ else:
108
+ new_href_value = Asset(href=f"{obs.rstrip('/')}/{stac_file_info['id']}")
109
+ assets = (
110
+ {stac_file_info["id"]: new_href_value}
111
+ if "file" in stac_file_info["assets"]
112
+ else {
113
+ stac_file_info["id"].split("/")[-1]: new_href_value
114
+ for _, asset_value in stac_file_info["assets"].items()
115
+ if "href" in asset_value
116
+ }
117
+ )
118
+
119
+ # Copy properties from the input stac file, or use default values
120
+ properties = stac_file_info.get("properties") or {}
121
+ geometry = stac_file_info.get("geometry") or { # NOTE: override the geometry if it is set to None
122
+ "type": "Polygon",
123
+ "coordinates": [[[-180, -90], [180, -90], [180, 90], [-180, 90], [-180, -90]]],
124
+ }
125
+ bbox = stac_file_info.get("bbox") or [-180.0, -90.0, 180.0, 90.0]
126
+ datetime_value = now
127
+
128
+ # Check if the geometry is a valid polygon. If not we correct the geometry (Temporary solution)
129
+ if geometry["type"] == "Polygon":
130
+ poly = geom.Polygon(geometry["coordinates"][0])
131
+ if not (poly.is_valid and poly.is_closed):
132
+
133
+ coords = poly.exterior.coords.xy
134
+ geometry["coordinates"] = [[list(coord) for coord in zip(coords[0], coords[1])]]
135
+
136
+ # Try to format each property date or datetime
137
+ for key, value in properties.items():
138
+ try:
139
+ properties[key] = dateutil.parser.parse(value).strftime(DATETIME_FORMAT_MS)
140
+ # If this is not a datetime, do nothing
141
+ except (dateutil.parser.ParserError, TypeError):
142
+ pass
143
+
144
+ # Add item to the STAC catalog collection, check status is OK
145
+ item = Item(id=item_id, geometry=geometry, bbox=bbox, datetime=datetime_value, properties=properties, assets=assets)
146
+ try:
147
+ response = stac_client.add_item(collection_name, item, timeout=CATALOG_REQUEST_TIMEOUT)
148
+ except (requests.exceptions.RequestException, requests.exceptions.Timeout) as e:
149
+ stac_client.logger.exception("Request exception caught: %s", e)
150
+ return False
151
+ try:
152
+ if not response.ok:
153
+ stac_client.logger.error(f"The catalog update endpoint: {response.json()}")
154
+ except requests.exceptions.JSONDecodeError:
155
+ stac_client.logger.exception(f"Could not get the json body from response: {response}")
156
+ return response.status_code == 200
157
+
158
+
159
+ class PrefectCommonConfig: # pylint: disable=too-few-public-methods, too-many-instance-attributes,
160
+ """Common configuration for Prefect tasks and flows.
161
+ Base class for configuration used in Prefect task and flow
162
+ used in staging the files from different stations (cadip, adgs...)
163
+
164
+ Attributes:
165
+ rs_client (AuxipClient | CadipClient): The client for accessing the Auxip or Cadip service.
166
+ mission (str): The mission identifier.
167
+ tmp_download_path (str): The local path where temporary files will be downloaded.
168
+ s3_path (str): The S3 bucket path where the files will be stored.
169
+ """
170
+
171
+ def __init__(
172
+ self,
173
+ rs_client: AuxipClient | CadipClient,
174
+ mission,
175
+ tmp_download_path,
176
+ s3_path,
177
+ ):
178
+ self.rs_client: AuxipClient | CadipClient | None = None # don't save this instance
179
+ self.rs_client_serialization = RsClientSerialization(rs_client) # save the serialization parameters instead
180
+ self.mission: str = mission
181
+ self.tmp_download_path: str = tmp_download_path
182
+ self.s3_path: str = s3_path
183
+
184
+
185
+ class PrefectTaskConfig(PrefectCommonConfig): # pylint: disable=too-few-public-methods
186
+ """Configuration class for Prefect tasks.
187
+
188
+ This class extends the `PrefectCommonConfig` class with additional attributes
189
+ specific to Prefect tasks. It includes the configuration for the task, such as the
190
+ files to be processed by the task and the maximum number of retries allowed.
191
+
192
+ Attributes: see :py:class:`PrefectCommonConfig`
193
+ task_files_stac (List[Dict]): A list of dictionaries containing information about the files to be processed
194
+ by the task. This info is in STAC format
195
+ max_retries (int): The maximum number of retries allowed for the task.
196
+
197
+ """
198
+
199
+ def __init__( # pylint: disable=too-many-arguments
200
+ self,
201
+ rs_client: AuxipClient | CadipClient,
202
+ mission,
203
+ tmp_download_path,
204
+ s3_path,
205
+ task_files_stac,
206
+ max_retries: int = 3,
207
+ ):
208
+ """
209
+ Initialize the PrefectTaskConfig object with provided parameters.
210
+ """
211
+
212
+ super().__init__(rs_client, mission, tmp_download_path, s3_path)
213
+ self.task_files_stac: list[dict] = task_files_stac
214
+ self.max_retries: int = max_retries
215
+
216
+
217
+ @task
218
+ def staging(config: PrefectTaskConfig):
219
+ """Prefect task function to stage (=download/ingest) files from a specified station.
220
+
221
+ This task function stages files for processing by calling the appropriate
222
+ rs-server endpoint for each requested file. It monitors the status of the file until it is
223
+ completely staged before initiating the next stage request.
224
+
225
+ Args:
226
+ config (PrefectTaskConfig): The configuration object containing the necessary parameters
227
+ for staging the files.
228
+
229
+ Returns:
230
+ List ([dict]): A list containing information about files that FAILED to be staged. The files within this
231
+ list don't appear in the catalog as published
232
+
233
+ """
234
+
235
+ logger = get_prefect_logger("task_dwn")
236
+
237
+ # Deserialize the RsClient instance
238
+ rs_client: AuxipClient | CadipClient = config.rs_client_serialization.deserialize(logger) # type: ignore
239
+
240
+ # list with failed files
241
+ failed_files = config.task_files_stac.copy()
242
+
243
+ downloaded_files_indices = []
244
+
245
+ # Call the download endpoint for each requested file
246
+ for i, file_stac in enumerate(config.task_files_stac):
247
+ # update the filename to be ingested
248
+ try:
249
+ rs_client.staging(file_stac["id"], config.s3_path, config.tmp_download_path, timeout=ENDPOINT_TIMEOUT)
250
+ except RuntimeError as e:
251
+ # TODO: Continue? Stop ?
252
+ logger.exception(f"Could not stage file %s. Exception: {e}")
253
+ continue
254
+
255
+ # monitor the status of the file until it is completely downloaded before initiating the next download request
256
+ status = rs_client.staging_status(file_stac["id"], ENDPOINT_TIMEOUT)
257
+ # just for the demo the timeout is hardcoded, it should be otherwise defined somewhere in the configuration
258
+ timeout = DOWNLOAD_FILE_TIMEOUT # 6 minutes
259
+ while status in [EDownloadStatus.NOT_STARTED, EDownloadStatus.IN_PROGRESS] and timeout > 0:
260
+ logger.info(
261
+ "The download progress for file %s is %s",
262
+ file_stac["id"],
263
+ status.name,
264
+ )
265
+ time.sleep(2)
266
+ timeout -= 2
267
+ status = rs_client.staging_status(file_stac["id"], timeout=ENDPOINT_TIMEOUT)
268
+ if status == EDownloadStatus.DONE:
269
+ logger.info("File %s has been properly downloaded...", file_stac["id"])
270
+ # TODO: either move the code from filter_unpublished_files to RsClient
271
+ # or use the new PgstacClient ?
272
+ if update_stac_catalog.fn(
273
+ rs_client.get_stac_client(), # NOTE: maybe use RsClientSerialization instead
274
+ create_collection_name(config.mission, rs_client.station_name),
275
+ file_stac,
276
+ config.s3_path,
277
+ ):
278
+ logger.info(f"File well published: {file_stac['id']}\n")
279
+ # save the index of the well ingested file
280
+ downloaded_files_indices.append(i)
281
+ else:
282
+ logger.error(f"Could not publish file: {file_stac['id']}")
283
+ else:
284
+ logger.error(
285
+ "Error in downloading the file %s (status %s). Timeout was %s from %s\n",
286
+ file_stac["id"],
287
+ status.name,
288
+ timeout,
289
+ DOWNLOAD_FILE_TIMEOUT,
290
+ )
291
+ # remove all the well ingested files
292
+ # return only those that failed
293
+ for idx in sorted(downloaded_files_indices, reverse=True):
294
+ del failed_files[idx]
295
+
296
+ return failed_files
297
+
298
+
299
+ @task
300
+ def filter_unpublished_files(
301
+ stac_client: StacClient, # NOTE: maybe use RsClientSerialization instead
302
+ collection_name: str,
303
+ files_stac: list,
304
+ ) -> list:
305
+ """Checks for unpublished files in the STAC catalog.
306
+
307
+ This function takes a list of files and checks if they are already published in a specified
308
+ STAC (SpatioTemporal Asset Catalog) collection. It returns a list of files that are not yet published.
309
+
310
+ Parameters:
311
+ stac_client (StacClient): An instance of `StacClient` to interact with the STAC catalog.
312
+ collection_name (str): The name of the collection in which the search is performed.
313
+ files_stac (list of dict): A list of files to be checked for publication. Each file is represented as a
314
+ dictionary with at least an "id" key.
315
+
316
+ Returns:
317
+ list of dict: A list of files that are not yet published in the catalog. Each file is represented as a
318
+ dictionary.
319
+
320
+ Examples:
321
+ >>> rs_client = StacClient(...)
322
+ >>> collection_name = "example_collection"
323
+ >>> files_stac = [{"id": "file1.raw"}, {"id": "file2.raw"}]
324
+ >>> unpublished_files = filter_unpublished_files(rs_client, collection_name, files_stac)
325
+ >>> print(unpublished_files)
326
+ [{"id": "file1.raw"}]
327
+ """
328
+
329
+ # Get files IDs (no duplicate IDs)
330
+ ids = set()
331
+ for fs in files_stac:
332
+ ids.add(str(fs["id"]))
333
+ ids = list(ids) # type: ignore # set to list conversion
334
+
335
+ # For searching, we need to prefix our collection name by <owner_id>_
336
+ owner_collection = f"{stac_client.owner_id}_{collection_name}"
337
+
338
+ # Search using a CQL2 filter, see: https://pystac-client.readthedocs.io/en/stable/tutorials/cql2-filter.html
339
+ filter_ = {
340
+ "op": "and",
341
+ "args": [
342
+ {"op": "=", "args": [{"property": "collection"}, owner_collection]},
343
+ {"op": "=", "args": [{"property": "owner"}, stac_client.owner_id]},
344
+ {"op": "in", "args": [{"property": "id"}, ids]},
345
+ ],
346
+ }
347
+ try:
348
+ search = stac_client.search(filter=filter_)
349
+ existing = list(search.items_as_dicts())
350
+
351
+ # In case of any error, try to ingest everything anyway
352
+ except NotImplementedError as e:
353
+ stac_client.logger.exception("Search exception caught: %s", e)
354
+ return files_stac
355
+
356
+ # Only keep the files that do not alreay exist in the catalog
357
+ existing_ids = [item["id"] for item in existing]
358
+ return [file_stac for file_stac in files_stac if file_stac["id"] not in existing_ids]
359
+
360
+
361
+ def create_collection_name(mission: str, station: str) -> str:
362
+ """Create the name of a catalog collection
363
+
364
+ This function constructs and returns a specific name for the catalog collection.
365
+ For ADGS station type should be "mission_name"_aux
366
+ For CADIP stations types should be "mission_name"_chunk
367
+
368
+ For other values, a RuntimeError is raised.
369
+
370
+ Args:
371
+ mission (str): The name of the mission.
372
+ station (str): The type of station . Supported
373
+ values are "AUX" and "CADIP", "INS", "MPS", "MTI", "NSG", "SGS".
374
+
375
+ Returns:
376
+ str (str): The name of the collection
377
+
378
+ Raises:
379
+ RuntimeError: If the provided station type is not supported.
380
+
381
+ """
382
+ if station == AUXIP_STATION:
383
+ return f"{mission}_aux"
384
+ # check CADIP
385
+ try:
386
+ ECadipStation(station)
387
+ return f"{mission}_chunk"
388
+ except ValueError as e:
389
+ raise RuntimeError(f"Unknown station: {station}") from e
390
+
391
+
392
+ class PrefectFlowConfig(PrefectCommonConfig): # pylint: disable=too-few-public-methods
393
+ """Configuration class for Prefect flow.
394
+
395
+ This class inherits the PrefectCommonConfig and represents the configuration for a
396
+ Prefect flow
397
+
398
+ Attributes: see :py:class:`PrefectCommonConfig`
399
+ max_workers (int): The maximum number of workers for the Prefect flow.
400
+ start_datetime (datetime): The start datetime of the files that the station should return
401
+ stop_datetime (datetime): The stop datetime of the files that the station should return
402
+ limit: The limit for the number of the files in the list retrieved from the ADGS/CADIP station (optional).
403
+ """
404
+
405
+ def __init__( # pylint: disable=too-many-arguments
406
+ self,
407
+ rs_client: AuxipClient | CadipClient,
408
+ mission,
409
+ tmp_download_path,
410
+ s3_path,
411
+ max_workers,
412
+ start_datetime,
413
+ stop_datetime,
414
+ limit=None,
415
+ ):
416
+ """
417
+ Initialize the PrefectFlowConfig object with provided parameters.
418
+ """
419
+ super().__init__(rs_client, mission, tmp_download_path, s3_path)
420
+ self.max_workers: int = max_workers
421
+ self.start_datetime: datetime = start_datetime
422
+ self.stop_datetime: datetime = stop_datetime
423
+ self.limit = limit
424
+
425
+
426
+ # @flow # TO DEBUG THE CODE, JUST USE @flow
427
+ @flow(
428
+ flow_run_name="{config.rs_client_serialization.get_flow_name}",
429
+ task_runner=DaskTaskRunner(cluster_kwargs={"n_workers": 5, "threads_per_worker": 1}),
430
+ )
431
+ def staging_flow(config: PrefectFlowConfig):
432
+ """
433
+ Prefect flow for staging (downloading/ingesting) files from a station.
434
+
435
+ This flow orchestrates the download process by obtaining the list of files from the search endpoint (provided
436
+ station), splitting the list into tasks based on the number of workers, and submitting tasks for ingestion.
437
+
438
+ Args:
439
+ config (PrefectFlowConfig): Configuration object containing details about the download flow, such as
440
+ start and stop datetime, limit, rs_client, mission, temporary download path, S3 path, and max_workers.
441
+
442
+ Returns:
443
+ bool (bool): True if the flow execution is successful, False otherwise.
444
+
445
+ Raises:
446
+ None: This function does not raise any exceptions.
447
+ """
448
+ # get the Prefect logger
449
+ logger = get_prefect_logger("flow_dwn")
450
+ logger.info(f"The staging flow is starting. Received workers:{config.max_workers}")
451
+ try:
452
+ # Deserialize the RsClient instance
453
+ rs_client: AuxipClient | CadipClient = config.rs_client_serialization.deserialize(logger) # type: ignore
454
+
455
+ # get the list with files from the search endpoint
456
+ try:
457
+ files_stac = rs_client.search_stations(
458
+ config.start_datetime,
459
+ config.stop_datetime,
460
+ config.limit,
461
+ timeout=SEARCH_ENDPOINT_TIMEOUT,
462
+ )
463
+ except RuntimeError as e:
464
+ logger.exception(f"Unable to get the list with files for staging: {e}")
465
+ return False
466
+
467
+ # check if the list with files returned from the station is not empty
468
+ if len(files_stac) == 0:
469
+ logger.warning(
470
+ f"The station {rs_client.station_name} did not return any \
471
+ element for time interval {config.start_datetime} - {config.stop_datetime}",
472
+ )
473
+ return True
474
+ # create the collection name
475
+
476
+ # filter those that are already existing
477
+
478
+ files_stac = filter_unpublished_files( # type: ignore
479
+ rs_client.get_stac_client(), # NOTE: maybe use RsClientSerialization instead
480
+ create_collection_name(config.mission, rs_client.station_name),
481
+ files_stac,
482
+ wait_for=[files_stac],
483
+ )
484
+ # distribute the filenames evenly in a number of lists equal with
485
+ # the minimum between number of runners and files to be downloaded
486
+ try:
487
+ tasks_files_stac = [
488
+ x.tolist() for x in [*np.array_split(files_stac, min(config.max_workers, len(files_stac)))]
489
+ ]
490
+ except ValueError:
491
+ logger.warning("No task will be started, the requested number of tasks is 0 !")
492
+ tasks_files_stac = []
493
+ logger.info("List with files to be downloaded (after filtering against the catalog)")
494
+ for f in files_stac:
495
+ logger.info(" %s", f["id"])
496
+
497
+ for files_stac in tasks_files_stac:
498
+ staging.submit(
499
+ PrefectTaskConfig(
500
+ rs_client,
501
+ config.mission,
502
+ config.tmp_download_path,
503
+ config.s3_path,
504
+ files_stac,
505
+ ),
506
+ )
507
+
508
+ except (RuntimeError, TypeError) as e:
509
+ logger.error("Exception caught: %s", e)
510
+ return False
511
+
512
+ return True