rs-client-libraries 0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- config/staging_templates/README.md +17 -0
- config/staging_templates/yaml/additionalParameter.yaml +30 -0
- config/staging_templates/yaml/bbox.yaml +34 -0
- config/staging_templates/yaml/binaryInputValue.yaml +15 -0
- config/staging_templates/yaml/confClasses.yaml +22 -0
- config/staging_templates/yaml/descriptionType.yaml +36 -0
- config/staging_templates/yaml/dpr_openapi_schema.yaml +223 -0
- config/staging_templates/yaml/exception.yaml +31 -0
- config/staging_templates/yaml/execute.yaml +34 -0
- config/staging_templates/yaml/format.yaml +25 -0
- config/staging_templates/yaml/inlineOrRefData.yaml +18 -0
- config/staging_templates/yaml/inputDescription.yaml +32 -0
- config/staging_templates/yaml/inputValue.yaml +17 -0
- config/staging_templates/yaml/inputValueNoObject.yaml +22 -0
- config/staging_templates/yaml/jobControlOptions.yaml +19 -0
- config/staging_templates/yaml/jobList.yaml +27 -0
- config/staging_templates/yaml/landingPage.yaml +28 -0
- config/staging_templates/yaml/link.yaml +31 -0
- config/staging_templates/yaml/metadata.yaml +22 -0
- config/staging_templates/yaml/output.yaml +20 -0
- config/staging_templates/yaml/outputDescription.yaml +22 -0
- config/staging_templates/yaml/process.yaml +23 -0
- config/staging_templates/yaml/processList.yaml +27 -0
- config/staging_templates/yaml/processSummary.yaml +37 -0
- config/staging_templates/yaml/qualifiedInputValue.yaml +22 -0
- config/staging_templates/yaml/reference.yaml +21 -0
- config/staging_templates/yaml/results.yaml +16 -0
- config/staging_templates/yaml/schema.yaml +144 -0
- config/staging_templates/yaml/staging_openapi_schema.yaml +223 -0
- config/staging_templates/yaml/statusCode.yaml +22 -0
- config/staging_templates/yaml/statusInfo.yaml +52 -0
- config/staging_templates/yaml/subscriber.yaml +33 -0
- config/staging_templates/yaml/transmissionMode.yaml +19 -0
- rs_client/__init__.py +18 -0
- rs_client/ogcapi/__init__.py +15 -0
- rs_client/ogcapi/dpr_client.py +231 -0
- rs_client/ogcapi/ogcapi_client.py +282 -0
- rs_client/ogcapi/staging_client.py +178 -0
- rs_client/rs_client.py +277 -0
- rs_client/stac/__init__.py +15 -0
- rs_client/stac/auxip_client.py +70 -0
- rs_client/stac/cadip_client.py +70 -0
- rs_client/stac/catalog_client.py +479 -0
- rs_client/stac/stac_base.py +313 -0
- rs_client_libraries-0.2.dist-info/LICENSE +201 -0
- rs_client_libraries-0.2.dist-info/METADATA +44 -0
- rs_client_libraries-0.2.dist-info/RECORD +62 -0
- rs_client_libraries-0.2.dist-info/WHEEL +4 -0
- rs_common/__init__.py +20 -0
- rs_common/config.py +69 -0
- rs_common/init_opentelemetry.py +250 -0
- rs_common/logging.py +107 -0
- rs_common/prefect_utils.py +407 -0
- rs_common/utils.py +70 -0
- rs_workflows/__init__.py +15 -0
- rs_workflows/auxip_flow.py +70 -0
- rs_workflows/cadip_flow.py +69 -0
- rs_workflows/catalog_flow.py +71 -0
- rs_workflows/dpr_flow.py +272 -0
- rs_workflows/flow_utils.py +127 -0
- rs_workflows/on_demand_processing.py +243 -0
- rs_workflows/staging_flow.py +78 -0
|
@@ -0,0 +1,407 @@
|
|
|
1
|
+
# Copyright 2024 CS Group
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""Utility Python module for the tutorials, to be shared with the prefect or dask workers.
|
|
16
|
+
|
|
17
|
+
WARNING: AFTER EACH MODIFICATION, RESTART THE JUPYTER NOTEBOOK KERNEL !
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
import asyncio
|
|
21
|
+
import getpass
|
|
22
|
+
import json
|
|
23
|
+
import os
|
|
24
|
+
import re
|
|
25
|
+
import socket
|
|
26
|
+
import tempfile
|
|
27
|
+
from collections.abc import Callable
|
|
28
|
+
from pathlib import Path
|
|
29
|
+
from typing import Any
|
|
30
|
+
|
|
31
|
+
from fastapi.concurrency import run_in_threadpool
|
|
32
|
+
from prefect.blocks.system import Secret
|
|
33
|
+
from prefect.client.orchestration import get_client
|
|
34
|
+
from prefect.exceptions import ObjectNotFound
|
|
35
|
+
from prefect.utilities.asyncutils import sync_compatible
|
|
36
|
+
from prefect_aws import AwsCredentials, S3Bucket
|
|
37
|
+
|
|
38
|
+
from rs_common.utils import env_bool
|
|
39
|
+
|
|
40
|
+
# In local mode, all your services are running locally.
|
|
41
|
+
# In cluster mode, we use the services deployed on the RS-Server website.
|
|
42
|
+
# This configuration is set in an environment variable.
|
|
43
|
+
local_mode: bool = False
|
|
44
|
+
cluster_mode: bool = not local_mode
|
|
45
|
+
|
|
46
|
+
# Current user
|
|
47
|
+
owner_id: str | None = ""
|
|
48
|
+
|
|
49
|
+
# Prefect block names
|
|
50
|
+
BLOCK_NAME_ENV_GLOBAL: str = "env-vars"
|
|
51
|
+
BLOCK_NAME_ENV_USER: str = "env-vars-{0}" # env variables for the user/owner_id
|
|
52
|
+
BLOCK_NAME_SHARE_BUCKET: str = "share-bucket"
|
|
53
|
+
|
|
54
|
+
# S3 bucket object for each bucket name.
|
|
55
|
+
S3_BUCKETS: dict[str, S3Bucket] = {}
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def init_global_env(new_owner_id: str | None = None):
|
|
59
|
+
"""
|
|
60
|
+
Init the global variables above from the env vars.
|
|
61
|
+
Needs to be called from the client, then called again from the prefect workers.
|
|
62
|
+
|
|
63
|
+
Args:
|
|
64
|
+
new_owner_id: new ower/user id, if known.
|
|
65
|
+
"""
|
|
66
|
+
global local_mode, cluster_mode, owner_id
|
|
67
|
+
|
|
68
|
+
local_mode = env_bool("RSPY_LOCAL_MODE", default=False)
|
|
69
|
+
cluster_mode = not local_mode
|
|
70
|
+
|
|
71
|
+
# Override the owner id or read it from the env vars
|
|
72
|
+
if new_owner_id:
|
|
73
|
+
owner_id = new_owner_id
|
|
74
|
+
else:
|
|
75
|
+
owner_id = os.getenv("JUPYTERHUB_USER") if cluster_mode else os.getenv("RSPY_HOST_USER")
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
# Init the global variables
|
|
79
|
+
init_global_env()
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def get_ip_address() -> str:
|
|
83
|
+
"""Return IP address, see: https://stackoverflow.com/a/166520"""
|
|
84
|
+
return socket.gethostbyname(socket.gethostname())
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def format_env_user(any_owner_id: str):
|
|
88
|
+
"""Format the Prefect secret block for the current user"""
|
|
89
|
+
name = BLOCK_NAME_ENV_USER.format(any_owner_id).lower()
|
|
90
|
+
return re.sub("[^a-zA-Z0-9]", "-", name) # replace special characters by dash
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
@sync_compatible
|
|
94
|
+
async def read_apikey(optional: bool = True, save_to_env: bool = True) -> None:
|
|
95
|
+
"""
|
|
96
|
+
Read the API key, either from the environment variable or from an interactive input form.
|
|
97
|
+
|
|
98
|
+
Args:
|
|
99
|
+
optional (bool): If False and if the env var is missing, ask it from an interactive input form.
|
|
100
|
+
save_to_env (bool): If True, saves the API key to the ~/.env file.
|
|
101
|
+
"""
|
|
102
|
+
# No API key in local mode
|
|
103
|
+
if local_mode:
|
|
104
|
+
return
|
|
105
|
+
|
|
106
|
+
# If the API is saved as an env var in the ~/.env file, then it has already
|
|
107
|
+
# been read automatically by rs-infra-core/.github/jupyter/resources/00-read-env.py
|
|
108
|
+
apikey = os.getenv("RSPY_APIKEY")
|
|
109
|
+
if (not apikey) and (not optional):
|
|
110
|
+
|
|
111
|
+
# Else read it from user input
|
|
112
|
+
apikey = getpass.getpass("Enter your API key:")
|
|
113
|
+
|
|
114
|
+
# Save the env var
|
|
115
|
+
os.environ["RSPY_APIKEY"] = apikey
|
|
116
|
+
|
|
117
|
+
# Append it to the ~/.env file, if requested.
|
|
118
|
+
# Don't overwrite the full ~/.env file because it can contain other user info.
|
|
119
|
+
if save_to_env:
|
|
120
|
+
with open(os.path.expanduser("~/.env"), "a", encoding="utf-8") as env_file:
|
|
121
|
+
env_file.write(f"\nRSPY_APIKEY={apikey}\n")
|
|
122
|
+
print("API key saved to ~/.env.")
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
@sync_compatible
|
|
126
|
+
async def init_prefect_blocks():
|
|
127
|
+
"""Init prefect blocks from the client environment (= from jupyter)"""
|
|
128
|
+
|
|
129
|
+
#
|
|
130
|
+
# Env vars for all users
|
|
131
|
+
|
|
132
|
+
# In local mode, create the block that contains the environment variables for all users.
|
|
133
|
+
# In cluster mode, this block has already been created by an admin.
|
|
134
|
+
# All fields are mandatory.
|
|
135
|
+
if local_mode:
|
|
136
|
+
await Secret(
|
|
137
|
+
value={ # type: ignore[arg-type]
|
|
138
|
+
"RSPY_LOCAL_MODE": "1",
|
|
139
|
+
"PREFECT_BUCKET_NAME": os.getenv("PREFECT_BUCKET_NAME") or os.environ["RSPY_TEMP_BUCKET"],
|
|
140
|
+
"PREFECT_BUCKET_FOLDER": os.getenv("PREFECT_BUCKET_FOLDER", "prefect-share"),
|
|
141
|
+
"S3_ACCESSKEY": os.environ["S3_ACCESSKEY"],
|
|
142
|
+
"S3_SECRETKEY": os.environ["S3_SECRETKEY"],
|
|
143
|
+
"S3_REGION": os.environ["S3_REGION"],
|
|
144
|
+
"S3_ENDPOINT": os.environ["S3_ENDPOINT"],
|
|
145
|
+
"LOCAL_DASK_USERNAME": os.environ["LOCAL_DASK_USERNAME"],
|
|
146
|
+
"LOCAL_DASK_PASSWORD": os.environ["LOCAL_DASK_PASSWORD"],
|
|
147
|
+
},
|
|
148
|
+
).save(BLOCK_NAME_ENV_GLOBAL, overwrite=True)
|
|
149
|
+
|
|
150
|
+
#
|
|
151
|
+
# Env vars for current user/owner_id
|
|
152
|
+
|
|
153
|
+
# In cluster mode, read the API key
|
|
154
|
+
if cluster_mode:
|
|
155
|
+
await read_apikey()
|
|
156
|
+
|
|
157
|
+
# Read environment variables that are available from the client env in both local and cluster mode.
|
|
158
|
+
# They are optional.
|
|
159
|
+
env_vars = {}
|
|
160
|
+
for key in (
|
|
161
|
+
"AWS_REQUEST_CHECKSUM_CALCULATION",
|
|
162
|
+
"AWS_RESPONSE_CHECKSUM_VALIDATION",
|
|
163
|
+
"OTEL_PYTHON_REQUESTS_TRACE_HEADERS",
|
|
164
|
+
"OTEL_PYTHON_REQUESTS_TRACE_BODY",
|
|
165
|
+
"RSPY_APIKEY",
|
|
166
|
+
"RSPY_OAUTH2_COOKIE",
|
|
167
|
+
"RSPY_UAC_CHECK_URL",
|
|
168
|
+
"RSPY_WEBSITE",
|
|
169
|
+
"TEMPO_ENDPOINT",
|
|
170
|
+
# dask in cluster mode
|
|
171
|
+
"DASK_GATEWAY_ADDRESS",
|
|
172
|
+
# dask in local mode
|
|
173
|
+
"DASK_GATEWAY_STAGING_ADDRESS",
|
|
174
|
+
"DASK_GATEWAY_EOPF_ADDRESS",
|
|
175
|
+
"DASK_GATEWAY_EOPF_MOCKUP_ADDRESS",
|
|
176
|
+
"DASK_GATEWAY_STAGING_PUBLIC",
|
|
177
|
+
"DASK_GATEWAY_EOPF_PUBLIC",
|
|
178
|
+
"DASK_GATEWAY_EOPF_MOCKUP_PUBLIC",
|
|
179
|
+
):
|
|
180
|
+
if value := os.getenv(key):
|
|
181
|
+
env_vars[key] = value
|
|
182
|
+
|
|
183
|
+
# Save env vars in a secret block for the current user
|
|
184
|
+
await Secret(value=env_vars).save(format_env_user(owner_id), overwrite=True) # type: ignore
|
|
185
|
+
|
|
186
|
+
# Now read back the blocks so we are sure our env vars are up-to-date
|
|
187
|
+
await read_prefect_blocks(owner_id)
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
@sync_compatible
|
|
191
|
+
async def read_prefect_blocks(any_owner_id: str | None = None):
|
|
192
|
+
"""
|
|
193
|
+
Read prefect blocks from the prefect flow and tasks into env vars and global vars.
|
|
194
|
+
|
|
195
|
+
Args:
|
|
196
|
+
owner_id: Read prefect blocks for a specific user.
|
|
197
|
+
"""
|
|
198
|
+
|
|
199
|
+
# Read the env vars for all users
|
|
200
|
+
os.environ.update((await Secret.load(BLOCK_NAME_ENV_GLOBAL)).get())
|
|
201
|
+
|
|
202
|
+
# Read the env vars for the given user
|
|
203
|
+
if any_owner_id:
|
|
204
|
+
os.environ.update((await Secret.load(format_env_user(any_owner_id))).get())
|
|
205
|
+
|
|
206
|
+
# Init the env of the current module from the env vars we have just read
|
|
207
|
+
init_global_env(any_owner_id)
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def hack_for_jupyter(func: Callable, *args, **kwargs) -> asyncio.Task:
|
|
211
|
+
"""From Jupyter we need this hack to deploy prefect flows"""
|
|
212
|
+
coroutine = run_in_threadpool(func, *args, **kwargs)
|
|
213
|
+
return asyncio.create_task(coroutine)
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
async def wait_for_deployment(name: str, wait: int | float = 1, max_retry: int = 30):
|
|
217
|
+
"""Wait for prefect deployment to be finished."""
|
|
218
|
+
# Taken from prefect/cli/deployment.py::inspect
|
|
219
|
+
retry = 0
|
|
220
|
+
async with get_client() as client:
|
|
221
|
+
while True:
|
|
222
|
+
try:
|
|
223
|
+
await client.read_deployment_by_name(name)
|
|
224
|
+
print(f"Finished deploying prefect flow: {name!r}")
|
|
225
|
+
return
|
|
226
|
+
except ObjectNotFound:
|
|
227
|
+
retry += 1
|
|
228
|
+
if retry >= max_retry:
|
|
229
|
+
raise
|
|
230
|
+
print(f"Wait for deployment of prefect flow: {name!r} ...")
|
|
231
|
+
await asyncio.sleep(wait)
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
#
|
|
235
|
+
# Utility functions for s3 bucket operations.
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def get_s3_bucket(s3_path: str) -> tuple[S3Bucket, str]:
|
|
239
|
+
"""
|
|
240
|
+
Return a prefect S3 bucket object and S3 "object name" (= S3 path without s3://bucket-name)
|
|
241
|
+
from the given S3 path.
|
|
242
|
+
We use the prefect higher-level functions instead of those from boto3.
|
|
243
|
+
"""
|
|
244
|
+
|
|
245
|
+
# Remove the s3:// prefix and split by /
|
|
246
|
+
split = s3_path.removeprefix("s3").removeprefix("S3").strip(":/").split("/")
|
|
247
|
+
|
|
248
|
+
# Filter empty elements (if we had double //)
|
|
249
|
+
split = list(filter(None, split))
|
|
250
|
+
|
|
251
|
+
if not split:
|
|
252
|
+
raise ValueError(f"Invalid S3 path: {s3_path!r}")
|
|
253
|
+
|
|
254
|
+
bucket_name = split[0]
|
|
255
|
+
object_name = "/".join(split[1:])
|
|
256
|
+
|
|
257
|
+
# Try to return an existing bucket object
|
|
258
|
+
try:
|
|
259
|
+
return S3_BUCKETS[bucket_name], object_name
|
|
260
|
+
|
|
261
|
+
# Else create a new one
|
|
262
|
+
except KeyError:
|
|
263
|
+
aws_credentials = AwsCredentials(
|
|
264
|
+
aws_access_key_id=os.environ["S3_ACCESSKEY"],
|
|
265
|
+
aws_secret_access_key=os.environ["S3_SECRETKEY"],
|
|
266
|
+
region_name=os.environ["S3_REGION"],
|
|
267
|
+
aws_client_parameters={"endpoint_url": os.environ["S3_ENDPOINT"]},
|
|
268
|
+
)
|
|
269
|
+
s3_bucket = S3Bucket(
|
|
270
|
+
bucket_name=bucket_name,
|
|
271
|
+
credentials=aws_credentials,
|
|
272
|
+
bucket_folder="", # no prefixed folder
|
|
273
|
+
)
|
|
274
|
+
S3_BUCKETS[bucket_name] = s3_bucket
|
|
275
|
+
return s3_bucket, object_name
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
@sync_compatible
|
|
279
|
+
async def get_share_bucket() -> S3Bucket:
|
|
280
|
+
"""Get the prefect share bucket folder"""
|
|
281
|
+
|
|
282
|
+
# Try to read it from the prefect block
|
|
283
|
+
try:
|
|
284
|
+
return await S3Bucket.load(BLOCK_NAME_SHARE_BUCKET)
|
|
285
|
+
|
|
286
|
+
# If it doesn't exist yet, create and return it
|
|
287
|
+
except ValueError:
|
|
288
|
+
pass
|
|
289
|
+
|
|
290
|
+
# Read the prefect blocks that contain the S3 authentication
|
|
291
|
+
bucket_name = os.environ["PREFECT_BUCKET_NAME"]
|
|
292
|
+
bucket_folder = os.environ["PREFECT_BUCKET_FOLDER"]
|
|
293
|
+
|
|
294
|
+
# Get a s3 bucket object from its name
|
|
295
|
+
generic_bucket, _ = get_s3_bucket(bucket_name)
|
|
296
|
+
|
|
297
|
+
# Create a new object with the same credentials and a prefixed folder
|
|
298
|
+
share_bucket = S3Bucket(
|
|
299
|
+
bucket_name=bucket_name,
|
|
300
|
+
bucket_folder=bucket_folder,
|
|
301
|
+
credentials=generic_bucket.credentials,
|
|
302
|
+
)
|
|
303
|
+
|
|
304
|
+
# Save it as a prefect block and return it
|
|
305
|
+
await share_bucket.save(BLOCK_NAME_SHARE_BUCKET, overwrite=True)
|
|
306
|
+
return share_bucket
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
@sync_compatible
|
|
310
|
+
async def s3_upload_file(
|
|
311
|
+
from_path: str | Path,
|
|
312
|
+
s3_path: str,
|
|
313
|
+
**upload_kwargs: dict[str, Any],
|
|
314
|
+
) -> str:
|
|
315
|
+
"""See: S3Bucket.upload_from_path"""
|
|
316
|
+
s3_bucket, to_path = get_s3_bucket(s3_path)
|
|
317
|
+
return await s3_bucket.upload_from_path(from_path, to_path, **upload_kwargs)
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
@sync_compatible
|
|
321
|
+
async def s3_upload_empty_file(
|
|
322
|
+
s3_path: str,
|
|
323
|
+
**upload_kwargs: dict[str, Any],
|
|
324
|
+
) -> str:
|
|
325
|
+
"""Upload an empty temp file to the S3 bucket."""
|
|
326
|
+
|
|
327
|
+
# Create a tmp file
|
|
328
|
+
with tempfile.NamedTemporaryFile() as tmp:
|
|
329
|
+
|
|
330
|
+
# Add contents to the file or boto3 has a strange behavior after uploading an empty file
|
|
331
|
+
tmp.write(b"empty")
|
|
332
|
+
tmp.flush()
|
|
333
|
+
|
|
334
|
+
# Upload the file
|
|
335
|
+
return await s3_upload_file(tmp.name, s3_path, **upload_kwargs)
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
@sync_compatible
|
|
339
|
+
async def s3_upload_dir(
|
|
340
|
+
from_folder: str | Path,
|
|
341
|
+
s3_path: str,
|
|
342
|
+
**upload_kwargs: dict[str, Any],
|
|
343
|
+
) -> str | None:
|
|
344
|
+
"""
|
|
345
|
+
See: S3Bucket.upload_from_folder
|
|
346
|
+
|
|
347
|
+
Uploads files *within* a folder (excluding the folder itself) to the object storage service folder.
|
|
348
|
+
"""
|
|
349
|
+
s3_bucket, to_path = get_s3_bucket(s3_path)
|
|
350
|
+
return await s3_bucket.upload_from_folder(from_folder, to_path, **upload_kwargs)
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
@sync_compatible
|
|
354
|
+
async def s3_download_file(
|
|
355
|
+
s3_path: str,
|
|
356
|
+
to_path: str | Path | None,
|
|
357
|
+
**download_kwargs: dict[str, Any],
|
|
358
|
+
) -> Path:
|
|
359
|
+
"""See: S3Bucket.download_object_to_path"""
|
|
360
|
+
s3_bucket, from_path = get_s3_bucket(s3_path)
|
|
361
|
+
return await s3_bucket.download_object_to_path(from_path, to_path, **download_kwargs)
|
|
362
|
+
|
|
363
|
+
|
|
364
|
+
@sync_compatible
|
|
365
|
+
async def s3_download_dir(
|
|
366
|
+
s3_path: str,
|
|
367
|
+
local_path: str | None = None,
|
|
368
|
+
) -> None:
|
|
369
|
+
"""See: S3Bucket.get_directory"""
|
|
370
|
+
s3_bucket, from_path = get_s3_bucket(s3_path)
|
|
371
|
+
await s3_bucket.get_directory(from_path, local_path)
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
def s3_delete(s3_prefix: str, log: bool = False):
|
|
375
|
+
"""Remove all files from S3 bucket with the given prefix, using low-level client."""
|
|
376
|
+
s3_bucket, prefix = get_s3_bucket(s3_prefix)
|
|
377
|
+
objects = s3_bucket._get_bucket_resource().objects # pylint: disable=protected-access
|
|
378
|
+
|
|
379
|
+
# objects.filter(Prefix=prefix) makes a recursive search and returns only files, not folders.
|
|
380
|
+
# If we want to delete a file, call the filter and only keep the exact matches.
|
|
381
|
+
try_file = [obj.key for obj in objects.filter(Prefix=prefix)]
|
|
382
|
+
if prefix in try_file:
|
|
383
|
+
objects_to_delete = [{"Key": prefix}]
|
|
384
|
+
|
|
385
|
+
# If we want to delete a folder, make sure the prefix ends by /, call the filter and keep everything
|
|
386
|
+
else:
|
|
387
|
+
objects_to_delete = [{"Key": obj.key} for obj in objects.filter(Prefix=prefix.rstrip("/") + "/")]
|
|
388
|
+
|
|
389
|
+
if not objects_to_delete:
|
|
390
|
+
return
|
|
391
|
+
|
|
392
|
+
s3_client = s3_bucket._get_s3_client() # pylint: disable=protected-access
|
|
393
|
+
endpoint_url = s3_bucket.credentials.aws_client_parameters.endpoint_url
|
|
394
|
+
|
|
395
|
+
if log:
|
|
396
|
+
keys = [f"s3://{s3_bucket.bucket_name}/{o.get('Key')}" for o in objects_to_delete]
|
|
397
|
+
s3_bucket.logger.info(
|
|
398
|
+
f"Delete from {endpoint_url!r}: {json.dumps(keys, indent=2)}", # nosec hardcoded_sql_expressions
|
|
399
|
+
)
|
|
400
|
+
|
|
401
|
+
# Split the list of objects to delete
|
|
402
|
+
chunk_size = 1000
|
|
403
|
+
for i in range(0, len(objects_to_delete), chunk_size):
|
|
404
|
+
s3_client.delete_objects(
|
|
405
|
+
Bucket=s3_bucket.bucket_name,
|
|
406
|
+
Delete={"Objects": objects_to_delete[i : i + chunk_size], "Quiet": True}, # noqa: E203
|
|
407
|
+
)
|
rs_common/utils.py
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
# Copyright 2024 CS Group
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""This module is used to share common functions between apis"""
|
|
16
|
+
|
|
17
|
+
import os
|
|
18
|
+
from dataclasses import dataclass
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass
|
|
22
|
+
class AuthInfo:
|
|
23
|
+
"""User authentication information in Keycloak."""
|
|
24
|
+
|
|
25
|
+
# User login (preferred username)
|
|
26
|
+
user_login: str
|
|
27
|
+
|
|
28
|
+
# IAM roles
|
|
29
|
+
iam_roles: list[str]
|
|
30
|
+
|
|
31
|
+
# Configuration associated to the API key (not implemented for now)
|
|
32
|
+
apikey_config: dict
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def read_response_error(response) -> str:
|
|
36
|
+
"""Read and return an HTTP response error detail."""
|
|
37
|
+
|
|
38
|
+
# Try to read the response detail or error
|
|
39
|
+
try:
|
|
40
|
+
json = response.json()
|
|
41
|
+
if isinstance(json, str):
|
|
42
|
+
return json
|
|
43
|
+
return json.get("detail") or json.get("description") or json["error"]
|
|
44
|
+
|
|
45
|
+
# If this fail, get the full response content
|
|
46
|
+
except Exception: # pylint: disable=broad-exception-caught
|
|
47
|
+
return response.content.decode("utf-8", errors="ignore")
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def get_href_service(rs_server_href, env_var) -> str:
|
|
51
|
+
"""Get specific href link."""
|
|
52
|
+
if from_env := os.getenv(env_var, None):
|
|
53
|
+
return from_env.rstrip("/")
|
|
54
|
+
if not rs_server_href:
|
|
55
|
+
raise RuntimeError("RS-Server URL is undefined")
|
|
56
|
+
return rs_server_href.rstrip("/")
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def env_bool(var: str, default: bool) -> bool:
|
|
60
|
+
"""
|
|
61
|
+
Return True if an environemnt variable is set to 1, true or yes (case insensitive).
|
|
62
|
+
Return False if set to 0, false or no (case insensitive).
|
|
63
|
+
Return the default value if not set or set to a different value.
|
|
64
|
+
"""
|
|
65
|
+
val = os.getenv(var, str(default)).lower()
|
|
66
|
+
if val in ("y", "yes", "t", "true", "on", "1"):
|
|
67
|
+
return True
|
|
68
|
+
if val in ("n", "no", "f", "false", "off", "0"):
|
|
69
|
+
return False
|
|
70
|
+
return default
|
rs_workflows/__init__.py
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# Copyright 2025 CS Group
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""Prefect tasks and workflows"""
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
# Copyright 2025 CS Group
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""Auxip flow implementation"""
|
|
16
|
+
|
|
17
|
+
import json
|
|
18
|
+
|
|
19
|
+
from prefect import flow, get_run_logger, task
|
|
20
|
+
from pystac import ItemCollection
|
|
21
|
+
|
|
22
|
+
from rs_client.stac.auxip_client import AuxipClient
|
|
23
|
+
from rs_workflows.flow_utils import FlowEnv, FlowEnvArgs
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@flow(name="Auxip search")
|
|
27
|
+
async def search(
|
|
28
|
+
env: FlowEnvArgs,
|
|
29
|
+
auxip_cql2: dict,
|
|
30
|
+
error_if_empty: bool = False,
|
|
31
|
+
) -> ItemCollection | None:
|
|
32
|
+
"""
|
|
33
|
+
Search Auxip products.
|
|
34
|
+
|
|
35
|
+
Args:
|
|
36
|
+
env: Prefect flow environment (at least the owner_id is required)
|
|
37
|
+
auxip_cql2: Auxip CQL2 filter read from the processor tasktable.
|
|
38
|
+
error_if_empty: Raise a ValueError if the results are empty.
|
|
39
|
+
"""
|
|
40
|
+
logger = get_run_logger()
|
|
41
|
+
|
|
42
|
+
# Init flow environment and opentelemetry span
|
|
43
|
+
flow_env = FlowEnv(env)
|
|
44
|
+
with flow_env.start_span(__name__, "auxip-search"):
|
|
45
|
+
|
|
46
|
+
logger.info("Start Auxip search")
|
|
47
|
+
auxip_client: AuxipClient = flow_env.rs_client.get_auxip_client()
|
|
48
|
+
found = auxip_client.search(
|
|
49
|
+
method="POST",
|
|
50
|
+
stac_filter=auxip_cql2.get("filter"),
|
|
51
|
+
max_items=auxip_cql2.get("limit"),
|
|
52
|
+
sortby=auxip_cql2.get("sortby"),
|
|
53
|
+
)
|
|
54
|
+
if (not found) and error_if_empty:
|
|
55
|
+
raise ValueError(
|
|
56
|
+
f"No Auxip product found for CQL2 filter: {json.dumps(auxip_cql2, indent=2)}",
|
|
57
|
+
)
|
|
58
|
+
logger.info(f"Auxip search found {len(found)} results: {found}")
|
|
59
|
+
return found
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
###########################
|
|
63
|
+
# Call the flows as tasks #
|
|
64
|
+
###########################
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@task(name="Auxip search")
|
|
68
|
+
async def search_task(*args, **kwargs) -> ItemCollection | None:
|
|
69
|
+
"""See: search"""
|
|
70
|
+
return await search.fn(*args, **kwargs)
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
# Copyright 2025 CS Group
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""Cadip flow implementation"""
|
|
16
|
+
|
|
17
|
+
from prefect import flow, get_run_logger, task
|
|
18
|
+
from pystac import ItemCollection
|
|
19
|
+
|
|
20
|
+
from rs_client.stac.cadip_client import CadipClient
|
|
21
|
+
from rs_workflows.flow_utils import FlowEnv, FlowEnvArgs
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@flow(name="Cadip search")
|
|
25
|
+
async def search(
|
|
26
|
+
env: FlowEnvArgs,
|
|
27
|
+
cadip_collection_identifier: str,
|
|
28
|
+
session_identifier: str,
|
|
29
|
+
error_if_empty: bool = False,
|
|
30
|
+
) -> ItemCollection:
|
|
31
|
+
"""
|
|
32
|
+
Search Cadip sessions.
|
|
33
|
+
|
|
34
|
+
Args:
|
|
35
|
+
env: Prefect flow environment (at least the owner_id is required)
|
|
36
|
+
cadip_collection_identifier: CADIP collection identifier (to know the station)
|
|
37
|
+
session_identifier: Session identifier
|
|
38
|
+
error_if_empty: Raise a ValueError if the results are empty.
|
|
39
|
+
"""
|
|
40
|
+
logger = get_run_logger()
|
|
41
|
+
|
|
42
|
+
# Init flow environment and opentelemetry span
|
|
43
|
+
flow_env = FlowEnv(env)
|
|
44
|
+
with flow_env.start_span(__name__, "cadip-search"):
|
|
45
|
+
|
|
46
|
+
logger.info("Start Cadip search")
|
|
47
|
+
cadip_client: CadipClient = flow_env.rs_client.get_cadip_client()
|
|
48
|
+
found = cadip_client.search(
|
|
49
|
+
method="GET",
|
|
50
|
+
ids=[session_identifier],
|
|
51
|
+
collections=[cadip_collection_identifier],
|
|
52
|
+
)
|
|
53
|
+
if (not found) and error_if_empty:
|
|
54
|
+
raise ValueError(
|
|
55
|
+
f"No Cadip session found for id={session_identifier!r} collection={cadip_collection_identifier!r}",
|
|
56
|
+
)
|
|
57
|
+
logger.info(f"Cadip search found {len(found)} results: {found}")
|
|
58
|
+
return found
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
###########################
|
|
62
|
+
# Call the flows as tasks #
|
|
63
|
+
###########################
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
@task(name="Cadip search")
|
|
67
|
+
async def search_task(*args, **kwargs) -> ItemCollection | None:
|
|
68
|
+
"""See: search"""
|
|
69
|
+
return await search.fn(*args, **kwargs)
|