rs-client-libraries 0.2a3.post31.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rs_client/__init__.py +18 -0
- rs_client/auxip_client.py +61 -0
- rs_client/cadip_client.py +142 -0
- rs_client/rs_client.py +433 -0
- rs_client/stac_client.py +390 -0
- rs_client_libraries-0.2a3.post31.dev0.dist-info/LICENSE +201 -0
- rs_client_libraries-0.2a3.post31.dev0.dist-info/METADATA +41 -0
- rs_client_libraries-0.2a3.post31.dev0.dist-info/RECORD +20 -0
- rs_client_libraries-0.2a3.post31.dev0.dist-info/WHEEL +4 -0
- rs_common/__init__.py +20 -0
- rs_common/config.py +62 -0
- rs_common/logging.py +108 -0
- rs_common/opentelemetry.py +114 -0
- rs_common/serialization.py +86 -0
- rs_common/utils.py +46 -0
- rs_workflows/__init__.py +18 -0
- rs_workflows/config/dpr_config_template.yaml +43 -0
- rs_workflows/example.py +43 -0
- rs_workflows/s1_l0.py +497 -0
- rs_workflows/staging.py +512 -0
rs_workflows/staging.py
ADDED
|
@@ -0,0 +1,512 @@
|
|
|
1
|
+
# Copyright 2024 CS Group
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""Common workflows for general usage"""
|
|
16
|
+
|
|
17
|
+
import time
|
|
18
|
+
from datetime import datetime
|
|
19
|
+
from shapely import geometry as geom
|
|
20
|
+
|
|
21
|
+
import dateutil.parser
|
|
22
|
+
import numpy as np
|
|
23
|
+
import requests
|
|
24
|
+
from prefect import exceptions, flow, get_run_logger, task
|
|
25
|
+
from prefect_dask.task_runners import DaskTaskRunner
|
|
26
|
+
from pystac.asset import Asset
|
|
27
|
+
from pystac.item import Item
|
|
28
|
+
|
|
29
|
+
from rs_client.auxip_client import AuxipClient
|
|
30
|
+
from rs_client.cadip_client import CadipClient
|
|
31
|
+
from rs_client.stac_client import StacClient
|
|
32
|
+
from rs_common.config import (
|
|
33
|
+
AUXIP_STATION,
|
|
34
|
+
DATETIME_FORMAT_MS,
|
|
35
|
+
ECadipStation,
|
|
36
|
+
EDownloadStatus,
|
|
37
|
+
)
|
|
38
|
+
from rs_common.logging import Logging
|
|
39
|
+
from rs_common.serialization import RsClientSerialization
|
|
40
|
+
|
|
41
|
+
DOWNLOAD_FILE_TIMEOUT = 360 # in seconds
|
|
42
|
+
SET_PREFECT_LOGGING_LEVEL = "DEBUG"
|
|
43
|
+
ENDPOINT_TIMEOUT = 10 # in seconds
|
|
44
|
+
SEARCH_ENDPOINT_TIMEOUT = 60 # in seconds
|
|
45
|
+
CATALOG_REQUEST_TIMEOUT = 20 # in seconds
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def get_prefect_logger(general_logger_name):
|
|
49
|
+
"""
|
|
50
|
+
It returns the Prefect logger. If this can't be taken due to the missing
|
|
51
|
+
Prefect context (i.e. the flow/task is executed as a single function, from Pytest for example),
|
|
52
|
+
the general logger is returned
|
|
53
|
+
|
|
54
|
+
Args:
|
|
55
|
+
general_logger_name (str): The name of the general logger if Prefect logger can't be returned.
|
|
56
|
+
|
|
57
|
+
Returns:
|
|
58
|
+
logger (logger): A Prefect logger instance or a general logger instance.
|
|
59
|
+
"""
|
|
60
|
+
try:
|
|
61
|
+
logger = get_run_logger()
|
|
62
|
+
logger.setLevel(SET_PREFECT_LOGGING_LEVEL)
|
|
63
|
+
except exceptions.MissingContextError:
|
|
64
|
+
logger = Logging.default(general_logger_name)
|
|
65
|
+
logger.warning("Could not get the prefect logger due to missing context. Using the general one")
|
|
66
|
+
return logger
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
@task
|
|
70
|
+
def update_stac_catalog( # pylint: disable=too-many-locals
|
|
71
|
+
stac_client: StacClient, # NOTE: maybe use RsClientSerialization instead
|
|
72
|
+
collection_name: str,
|
|
73
|
+
stac_file_info: dict,
|
|
74
|
+
obs: str,
|
|
75
|
+
**kwargs,
|
|
76
|
+
):
|
|
77
|
+
"""Update the STAC catalog with file information.
|
|
78
|
+
|
|
79
|
+
This task updates the STAC catalog with the information of a file that has been processed
|
|
80
|
+
and saved to a specific location. It adds the mandatory fields such as geometry, collection name and object storage
|
|
81
|
+
href. It sends a POST request to add the newly composed item into the STAC catalog.
|
|
82
|
+
|
|
83
|
+
Args:
|
|
84
|
+
stac_client (StacClient): The client for accessing the STAC catalog.
|
|
85
|
+
collection_name (str): The name of the collection in the STAC catalog.
|
|
86
|
+
stac_file_info (dict): Information about the STAC file to be updated.
|
|
87
|
+
obs (str): The bucket location where the file has been saved.
|
|
88
|
+
|
|
89
|
+
Returns:
|
|
90
|
+
bool (bool): True if the STAC catalog is successfully updated, False otherwise.
|
|
91
|
+
|
|
92
|
+
"""
|
|
93
|
+
item_id = stac_file_info["id"]
|
|
94
|
+
now = datetime.now()
|
|
95
|
+
|
|
96
|
+
# The file path from the temp s3 bucket is given in the assets
|
|
97
|
+
assets = {}
|
|
98
|
+
update_assets = kwargs.get("update_assets", None)
|
|
99
|
+
|
|
100
|
+
if update_assets:
|
|
101
|
+
assets = {
|
|
102
|
+
stac_file_info["assets"][asset]["href"].split("/")[-1]: Asset(
|
|
103
|
+
href=f"{obs.rstrip('/')}/{stac_file_info['assets'][asset]['href']}",
|
|
104
|
+
)
|
|
105
|
+
for asset in stac_file_info["assets"]
|
|
106
|
+
}
|
|
107
|
+
else:
|
|
108
|
+
new_href_value = Asset(href=f"{obs.rstrip('/')}/{stac_file_info['id']}")
|
|
109
|
+
assets = (
|
|
110
|
+
{stac_file_info["id"]: new_href_value}
|
|
111
|
+
if "file" in stac_file_info["assets"]
|
|
112
|
+
else {
|
|
113
|
+
stac_file_info["id"].split("/")[-1]: new_href_value
|
|
114
|
+
for _, asset_value in stac_file_info["assets"].items()
|
|
115
|
+
if "href" in asset_value
|
|
116
|
+
}
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
# Copy properties from the input stac file, or use default values
|
|
120
|
+
properties = stac_file_info.get("properties") or {}
|
|
121
|
+
geometry = stac_file_info.get("geometry") or { # NOTE: override the geometry if it is set to None
|
|
122
|
+
"type": "Polygon",
|
|
123
|
+
"coordinates": [[[-180, -90], [180, -90], [180, 90], [-180, 90], [-180, -90]]],
|
|
124
|
+
}
|
|
125
|
+
bbox = stac_file_info.get("bbox") or [-180.0, -90.0, 180.0, 90.0]
|
|
126
|
+
datetime_value = now
|
|
127
|
+
|
|
128
|
+
# Check if the geometry is a valid polygon. If not we correct the geometry (Temporary solution)
|
|
129
|
+
if geometry["type"] == "Polygon":
|
|
130
|
+
poly = geom.Polygon(geometry["coordinates"][0])
|
|
131
|
+
if not (poly.is_valid and poly.is_closed):
|
|
132
|
+
|
|
133
|
+
coords = poly.exterior.coords.xy
|
|
134
|
+
geometry["coordinates"] = [[list(coord) for coord in zip(coords[0], coords[1])]]
|
|
135
|
+
|
|
136
|
+
# Try to format each property date or datetime
|
|
137
|
+
for key, value in properties.items():
|
|
138
|
+
try:
|
|
139
|
+
properties[key] = dateutil.parser.parse(value).strftime(DATETIME_FORMAT_MS)
|
|
140
|
+
# If this is not a datetime, do nothing
|
|
141
|
+
except (dateutil.parser.ParserError, TypeError):
|
|
142
|
+
pass
|
|
143
|
+
|
|
144
|
+
# Add item to the STAC catalog collection, check status is OK
|
|
145
|
+
item = Item(id=item_id, geometry=geometry, bbox=bbox, datetime=datetime_value, properties=properties, assets=assets)
|
|
146
|
+
try:
|
|
147
|
+
response = stac_client.add_item(collection_name, item, timeout=CATALOG_REQUEST_TIMEOUT)
|
|
148
|
+
except (requests.exceptions.RequestException, requests.exceptions.Timeout) as e:
|
|
149
|
+
stac_client.logger.exception("Request exception caught: %s", e)
|
|
150
|
+
return False
|
|
151
|
+
try:
|
|
152
|
+
if not response.ok:
|
|
153
|
+
stac_client.logger.error(f"The catalog update endpoint: {response.json()}")
|
|
154
|
+
except requests.exceptions.JSONDecodeError:
|
|
155
|
+
stac_client.logger.exception(f"Could not get the json body from response: {response}")
|
|
156
|
+
return response.status_code == 200
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
class PrefectCommonConfig: # pylint: disable=too-few-public-methods, too-many-instance-attributes,
|
|
160
|
+
"""Common configuration for Prefect tasks and flows.
|
|
161
|
+
Base class for configuration used in Prefect task and flow
|
|
162
|
+
used in staging the files from different stations (cadip, adgs...)
|
|
163
|
+
|
|
164
|
+
Attributes:
|
|
165
|
+
rs_client (AuxipClient | CadipClient): The client for accessing the Auxip or Cadip service.
|
|
166
|
+
mission (str): The mission identifier.
|
|
167
|
+
tmp_download_path (str): The local path where temporary files will be downloaded.
|
|
168
|
+
s3_path (str): The S3 bucket path where the files will be stored.
|
|
169
|
+
"""
|
|
170
|
+
|
|
171
|
+
def __init__(
|
|
172
|
+
self,
|
|
173
|
+
rs_client: AuxipClient | CadipClient,
|
|
174
|
+
mission,
|
|
175
|
+
tmp_download_path,
|
|
176
|
+
s3_path,
|
|
177
|
+
):
|
|
178
|
+
self.rs_client: AuxipClient | CadipClient | None = None # don't save this instance
|
|
179
|
+
self.rs_client_serialization = RsClientSerialization(rs_client) # save the serialization parameters instead
|
|
180
|
+
self.mission: str = mission
|
|
181
|
+
self.tmp_download_path: str = tmp_download_path
|
|
182
|
+
self.s3_path: str = s3_path
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
class PrefectTaskConfig(PrefectCommonConfig): # pylint: disable=too-few-public-methods
|
|
186
|
+
"""Configuration class for Prefect tasks.
|
|
187
|
+
|
|
188
|
+
This class extends the `PrefectCommonConfig` class with additional attributes
|
|
189
|
+
specific to Prefect tasks. It includes the configuration for the task, such as the
|
|
190
|
+
files to be processed by the task and the maximum number of retries allowed.
|
|
191
|
+
|
|
192
|
+
Attributes: see :py:class:`PrefectCommonConfig`
|
|
193
|
+
task_files_stac (List[Dict]): A list of dictionaries containing information about the files to be processed
|
|
194
|
+
by the task. This info is in STAC format
|
|
195
|
+
max_retries (int): The maximum number of retries allowed for the task.
|
|
196
|
+
|
|
197
|
+
"""
|
|
198
|
+
|
|
199
|
+
def __init__( # pylint: disable=too-many-arguments
|
|
200
|
+
self,
|
|
201
|
+
rs_client: AuxipClient | CadipClient,
|
|
202
|
+
mission,
|
|
203
|
+
tmp_download_path,
|
|
204
|
+
s3_path,
|
|
205
|
+
task_files_stac,
|
|
206
|
+
max_retries: int = 3,
|
|
207
|
+
):
|
|
208
|
+
"""
|
|
209
|
+
Initialize the PrefectTaskConfig object with provided parameters.
|
|
210
|
+
"""
|
|
211
|
+
|
|
212
|
+
super().__init__(rs_client, mission, tmp_download_path, s3_path)
|
|
213
|
+
self.task_files_stac: list[dict] = task_files_stac
|
|
214
|
+
self.max_retries: int = max_retries
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
@task
|
|
218
|
+
def staging(config: PrefectTaskConfig):
|
|
219
|
+
"""Prefect task function to stage (=download/ingest) files from a specified station.
|
|
220
|
+
|
|
221
|
+
This task function stages files for processing by calling the appropriate
|
|
222
|
+
rs-server endpoint for each requested file. It monitors the status of the file until it is
|
|
223
|
+
completely staged before initiating the next stage request.
|
|
224
|
+
|
|
225
|
+
Args:
|
|
226
|
+
config (PrefectTaskConfig): The configuration object containing the necessary parameters
|
|
227
|
+
for staging the files.
|
|
228
|
+
|
|
229
|
+
Returns:
|
|
230
|
+
List ([dict]): A list containing information about files that FAILED to be staged. The files within this
|
|
231
|
+
list don't appear in the catalog as published
|
|
232
|
+
|
|
233
|
+
"""
|
|
234
|
+
|
|
235
|
+
logger = get_prefect_logger("task_dwn")
|
|
236
|
+
|
|
237
|
+
# Deserialize the RsClient instance
|
|
238
|
+
rs_client: AuxipClient | CadipClient = config.rs_client_serialization.deserialize(logger) # type: ignore
|
|
239
|
+
|
|
240
|
+
# list with failed files
|
|
241
|
+
failed_files = config.task_files_stac.copy()
|
|
242
|
+
|
|
243
|
+
downloaded_files_indices = []
|
|
244
|
+
|
|
245
|
+
# Call the download endpoint for each requested file
|
|
246
|
+
for i, file_stac in enumerate(config.task_files_stac):
|
|
247
|
+
# update the filename to be ingested
|
|
248
|
+
try:
|
|
249
|
+
rs_client.staging(file_stac["id"], config.s3_path, config.tmp_download_path, timeout=ENDPOINT_TIMEOUT)
|
|
250
|
+
except RuntimeError as e:
|
|
251
|
+
# TODO: Continue? Stop ?
|
|
252
|
+
logger.exception(f"Could not stage file %s. Exception: {e}")
|
|
253
|
+
continue
|
|
254
|
+
|
|
255
|
+
# monitor the status of the file until it is completely downloaded before initiating the next download request
|
|
256
|
+
status = rs_client.staging_status(file_stac["id"], ENDPOINT_TIMEOUT)
|
|
257
|
+
# just for the demo the timeout is hardcoded, it should be otherwise defined somewhere in the configuration
|
|
258
|
+
timeout = DOWNLOAD_FILE_TIMEOUT # 6 minutes
|
|
259
|
+
while status in [EDownloadStatus.NOT_STARTED, EDownloadStatus.IN_PROGRESS] and timeout > 0:
|
|
260
|
+
logger.info(
|
|
261
|
+
"The download progress for file %s is %s",
|
|
262
|
+
file_stac["id"],
|
|
263
|
+
status.name,
|
|
264
|
+
)
|
|
265
|
+
time.sleep(2)
|
|
266
|
+
timeout -= 2
|
|
267
|
+
status = rs_client.staging_status(file_stac["id"], timeout=ENDPOINT_TIMEOUT)
|
|
268
|
+
if status == EDownloadStatus.DONE:
|
|
269
|
+
logger.info("File %s has been properly downloaded...", file_stac["id"])
|
|
270
|
+
# TODO: either move the code from filter_unpublished_files to RsClient
|
|
271
|
+
# or use the new PgstacClient ?
|
|
272
|
+
if update_stac_catalog.fn(
|
|
273
|
+
rs_client.get_stac_client(), # NOTE: maybe use RsClientSerialization instead
|
|
274
|
+
create_collection_name(config.mission, rs_client.station_name),
|
|
275
|
+
file_stac,
|
|
276
|
+
config.s3_path,
|
|
277
|
+
):
|
|
278
|
+
logger.info(f"File well published: {file_stac['id']}\n")
|
|
279
|
+
# save the index of the well ingested file
|
|
280
|
+
downloaded_files_indices.append(i)
|
|
281
|
+
else:
|
|
282
|
+
logger.error(f"Could not publish file: {file_stac['id']}")
|
|
283
|
+
else:
|
|
284
|
+
logger.error(
|
|
285
|
+
"Error in downloading the file %s (status %s). Timeout was %s from %s\n",
|
|
286
|
+
file_stac["id"],
|
|
287
|
+
status.name,
|
|
288
|
+
timeout,
|
|
289
|
+
DOWNLOAD_FILE_TIMEOUT,
|
|
290
|
+
)
|
|
291
|
+
# remove all the well ingested files
|
|
292
|
+
# return only those that failed
|
|
293
|
+
for idx in sorted(downloaded_files_indices, reverse=True):
|
|
294
|
+
del failed_files[idx]
|
|
295
|
+
|
|
296
|
+
return failed_files
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
@task
|
|
300
|
+
def filter_unpublished_files(
|
|
301
|
+
stac_client: StacClient, # NOTE: maybe use RsClientSerialization instead
|
|
302
|
+
collection_name: str,
|
|
303
|
+
files_stac: list,
|
|
304
|
+
) -> list:
|
|
305
|
+
"""Checks for unpublished files in the STAC catalog.
|
|
306
|
+
|
|
307
|
+
This function takes a list of files and checks if they are already published in a specified
|
|
308
|
+
STAC (SpatioTemporal Asset Catalog) collection. It returns a list of files that are not yet published.
|
|
309
|
+
|
|
310
|
+
Parameters:
|
|
311
|
+
stac_client (StacClient): An instance of `StacClient` to interact with the STAC catalog.
|
|
312
|
+
collection_name (str): The name of the collection in which the search is performed.
|
|
313
|
+
files_stac (list of dict): A list of files to be checked for publication. Each file is represented as a
|
|
314
|
+
dictionary with at least an "id" key.
|
|
315
|
+
|
|
316
|
+
Returns:
|
|
317
|
+
list of dict: A list of files that are not yet published in the catalog. Each file is represented as a
|
|
318
|
+
dictionary.
|
|
319
|
+
|
|
320
|
+
Examples:
|
|
321
|
+
>>> rs_client = StacClient(...)
|
|
322
|
+
>>> collection_name = "example_collection"
|
|
323
|
+
>>> files_stac = [{"id": "file1.raw"}, {"id": "file2.raw"}]
|
|
324
|
+
>>> unpublished_files = filter_unpublished_files(rs_client, collection_name, files_stac)
|
|
325
|
+
>>> print(unpublished_files)
|
|
326
|
+
[{"id": "file1.raw"}]
|
|
327
|
+
"""
|
|
328
|
+
|
|
329
|
+
# Get files IDs (no duplicate IDs)
|
|
330
|
+
ids = set()
|
|
331
|
+
for fs in files_stac:
|
|
332
|
+
ids.add(str(fs["id"]))
|
|
333
|
+
ids = list(ids) # type: ignore # set to list conversion
|
|
334
|
+
|
|
335
|
+
# For searching, we need to prefix our collection name by <owner_id>_
|
|
336
|
+
owner_collection = f"{stac_client.owner_id}_{collection_name}"
|
|
337
|
+
|
|
338
|
+
# Search using a CQL2 filter, see: https://pystac-client.readthedocs.io/en/stable/tutorials/cql2-filter.html
|
|
339
|
+
filter_ = {
|
|
340
|
+
"op": "and",
|
|
341
|
+
"args": [
|
|
342
|
+
{"op": "=", "args": [{"property": "collection"}, owner_collection]},
|
|
343
|
+
{"op": "=", "args": [{"property": "owner"}, stac_client.owner_id]},
|
|
344
|
+
{"op": "in", "args": [{"property": "id"}, ids]},
|
|
345
|
+
],
|
|
346
|
+
}
|
|
347
|
+
try:
|
|
348
|
+
search = stac_client.search(filter=filter_)
|
|
349
|
+
existing = list(search.items_as_dicts())
|
|
350
|
+
|
|
351
|
+
# In case of any error, try to ingest everything anyway
|
|
352
|
+
except NotImplementedError as e:
|
|
353
|
+
stac_client.logger.exception("Search exception caught: %s", e)
|
|
354
|
+
return files_stac
|
|
355
|
+
|
|
356
|
+
# Only keep the files that do not alreay exist in the catalog
|
|
357
|
+
existing_ids = [item["id"] for item in existing]
|
|
358
|
+
return [file_stac for file_stac in files_stac if file_stac["id"] not in existing_ids]
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
def create_collection_name(mission: str, station: str) -> str:
|
|
362
|
+
"""Create the name of a catalog collection
|
|
363
|
+
|
|
364
|
+
This function constructs and returns a specific name for the catalog collection.
|
|
365
|
+
For ADGS station type should be "mission_name"_aux
|
|
366
|
+
For CADIP stations types should be "mission_name"_chunk
|
|
367
|
+
|
|
368
|
+
For other values, a RuntimeError is raised.
|
|
369
|
+
|
|
370
|
+
Args:
|
|
371
|
+
mission (str): The name of the mission.
|
|
372
|
+
station (str): The type of station . Supported
|
|
373
|
+
values are "AUX" and "CADIP", "INS", "MPS", "MTI", "NSG", "SGS".
|
|
374
|
+
|
|
375
|
+
Returns:
|
|
376
|
+
str (str): The name of the collection
|
|
377
|
+
|
|
378
|
+
Raises:
|
|
379
|
+
RuntimeError: If the provided station type is not supported.
|
|
380
|
+
|
|
381
|
+
"""
|
|
382
|
+
if station == AUXIP_STATION:
|
|
383
|
+
return f"{mission}_aux"
|
|
384
|
+
# check CADIP
|
|
385
|
+
try:
|
|
386
|
+
ECadipStation(station)
|
|
387
|
+
return f"{mission}_chunk"
|
|
388
|
+
except ValueError as e:
|
|
389
|
+
raise RuntimeError(f"Unknown station: {station}") from e
|
|
390
|
+
|
|
391
|
+
|
|
392
|
+
class PrefectFlowConfig(PrefectCommonConfig): # pylint: disable=too-few-public-methods
|
|
393
|
+
"""Configuration class for Prefect flow.
|
|
394
|
+
|
|
395
|
+
This class inherits the PrefectCommonConfig and represents the configuration for a
|
|
396
|
+
Prefect flow
|
|
397
|
+
|
|
398
|
+
Attributes: see :py:class:`PrefectCommonConfig`
|
|
399
|
+
max_workers (int): The maximum number of workers for the Prefect flow.
|
|
400
|
+
start_datetime (datetime): The start datetime of the files that the station should return
|
|
401
|
+
stop_datetime (datetime): The stop datetime of the files that the station should return
|
|
402
|
+
limit: The limit for the number of the files in the list retrieved from the ADGS/CADIP station (optional).
|
|
403
|
+
"""
|
|
404
|
+
|
|
405
|
+
def __init__( # pylint: disable=too-many-arguments
|
|
406
|
+
self,
|
|
407
|
+
rs_client: AuxipClient | CadipClient,
|
|
408
|
+
mission,
|
|
409
|
+
tmp_download_path,
|
|
410
|
+
s3_path,
|
|
411
|
+
max_workers,
|
|
412
|
+
start_datetime,
|
|
413
|
+
stop_datetime,
|
|
414
|
+
limit=None,
|
|
415
|
+
):
|
|
416
|
+
"""
|
|
417
|
+
Initialize the PrefectFlowConfig object with provided parameters.
|
|
418
|
+
"""
|
|
419
|
+
super().__init__(rs_client, mission, tmp_download_path, s3_path)
|
|
420
|
+
self.max_workers: int = max_workers
|
|
421
|
+
self.start_datetime: datetime = start_datetime
|
|
422
|
+
self.stop_datetime: datetime = stop_datetime
|
|
423
|
+
self.limit = limit
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
# @flow # TO DEBUG THE CODE, JUST USE @flow
|
|
427
|
+
@flow(
|
|
428
|
+
flow_run_name="{config.rs_client_serialization.get_flow_name}",
|
|
429
|
+
task_runner=DaskTaskRunner(cluster_kwargs={"n_workers": 5, "threads_per_worker": 1}),
|
|
430
|
+
)
|
|
431
|
+
def staging_flow(config: PrefectFlowConfig):
|
|
432
|
+
"""
|
|
433
|
+
Prefect flow for staging (downloading/ingesting) files from a station.
|
|
434
|
+
|
|
435
|
+
This flow orchestrates the download process by obtaining the list of files from the search endpoint (provided
|
|
436
|
+
station), splitting the list into tasks based on the number of workers, and submitting tasks for ingestion.
|
|
437
|
+
|
|
438
|
+
Args:
|
|
439
|
+
config (PrefectFlowConfig): Configuration object containing details about the download flow, such as
|
|
440
|
+
start and stop datetime, limit, rs_client, mission, temporary download path, S3 path, and max_workers.
|
|
441
|
+
|
|
442
|
+
Returns:
|
|
443
|
+
bool (bool): True if the flow execution is successful, False otherwise.
|
|
444
|
+
|
|
445
|
+
Raises:
|
|
446
|
+
None: This function does not raise any exceptions.
|
|
447
|
+
"""
|
|
448
|
+
# get the Prefect logger
|
|
449
|
+
logger = get_prefect_logger("flow_dwn")
|
|
450
|
+
logger.info(f"The staging flow is starting. Received workers:{config.max_workers}")
|
|
451
|
+
try:
|
|
452
|
+
# Deserialize the RsClient instance
|
|
453
|
+
rs_client: AuxipClient | CadipClient = config.rs_client_serialization.deserialize(logger) # type: ignore
|
|
454
|
+
|
|
455
|
+
# get the list with files from the search endpoint
|
|
456
|
+
try:
|
|
457
|
+
files_stac = rs_client.search_stations(
|
|
458
|
+
config.start_datetime,
|
|
459
|
+
config.stop_datetime,
|
|
460
|
+
config.limit,
|
|
461
|
+
timeout=SEARCH_ENDPOINT_TIMEOUT,
|
|
462
|
+
)
|
|
463
|
+
except RuntimeError as e:
|
|
464
|
+
logger.exception(f"Unable to get the list with files for staging: {e}")
|
|
465
|
+
return False
|
|
466
|
+
|
|
467
|
+
# check if the list with files returned from the station is not empty
|
|
468
|
+
if len(files_stac) == 0:
|
|
469
|
+
logger.warning(
|
|
470
|
+
f"The station {rs_client.station_name} did not return any \
|
|
471
|
+
element for time interval {config.start_datetime} - {config.stop_datetime}",
|
|
472
|
+
)
|
|
473
|
+
return True
|
|
474
|
+
# create the collection name
|
|
475
|
+
|
|
476
|
+
# filter those that are already existing
|
|
477
|
+
|
|
478
|
+
files_stac = filter_unpublished_files( # type: ignore
|
|
479
|
+
rs_client.get_stac_client(), # NOTE: maybe use RsClientSerialization instead
|
|
480
|
+
create_collection_name(config.mission, rs_client.station_name),
|
|
481
|
+
files_stac,
|
|
482
|
+
wait_for=[files_stac],
|
|
483
|
+
)
|
|
484
|
+
# distribute the filenames evenly in a number of lists equal with
|
|
485
|
+
# the minimum between number of runners and files to be downloaded
|
|
486
|
+
try:
|
|
487
|
+
tasks_files_stac = [
|
|
488
|
+
x.tolist() for x in [*np.array_split(files_stac, min(config.max_workers, len(files_stac)))]
|
|
489
|
+
]
|
|
490
|
+
except ValueError:
|
|
491
|
+
logger.warning("No task will be started, the requested number of tasks is 0 !")
|
|
492
|
+
tasks_files_stac = []
|
|
493
|
+
logger.info("List with files to be downloaded (after filtering against the catalog)")
|
|
494
|
+
for f in files_stac:
|
|
495
|
+
logger.info(" %s", f["id"])
|
|
496
|
+
|
|
497
|
+
for files_stac in tasks_files_stac:
|
|
498
|
+
staging.submit(
|
|
499
|
+
PrefectTaskConfig(
|
|
500
|
+
rs_client,
|
|
501
|
+
config.mission,
|
|
502
|
+
config.tmp_download_path,
|
|
503
|
+
config.s3_path,
|
|
504
|
+
files_stac,
|
|
505
|
+
),
|
|
506
|
+
)
|
|
507
|
+
|
|
508
|
+
except (RuntimeError, TypeError) as e:
|
|
509
|
+
logger.error("Exception caught: %s", e)
|
|
510
|
+
return False
|
|
511
|
+
|
|
512
|
+
return True
|