rs-client-libraries 0.2a9__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rs_client/__init__.py +18 -0
- rs_client/auxip_client.py +61 -0
- rs_client/cadip_client.py +142 -0
- rs_client/rs_client.py +433 -0
- rs_client/stac_client.py +391 -0
- rs_client_libraries-0.2a9.dist-info/LICENSE +201 -0
- rs_client_libraries-0.2a9.dist-info/METADATA +42 -0
- rs_client_libraries-0.2a9.dist-info/RECORD +20 -0
- rs_client_libraries-0.2a9.dist-info/WHEEL +4 -0
- rs_common/__init__.py +20 -0
- rs_common/config.py +62 -0
- rs_common/logging.py +108 -0
- rs_common/opentelemetry.py +114 -0
- rs_common/serialization.py +86 -0
- rs_common/utils.py +46 -0
- rs_workflows/__init__.py +18 -0
- rs_workflows/config/dpr_config_template.yaml +43 -0
- rs_workflows/example.py +43 -0
- rs_workflows/s1_l0.py +497 -0
- rs_workflows/staging.py +511 -0
rs_client/__init__.py
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# Copyright 2024 CS Group
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""rsclient main module."""
|
|
16
|
+
|
|
17
|
+
# Set automatically by running `poetry dynamic-versioning`
|
|
18
|
+
__version__ = "0.2a9"
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
# Copyright 2024 CS Group
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""AuxipClient class implementation."""
|
|
16
|
+
|
|
17
|
+
import os
|
|
18
|
+
|
|
19
|
+
from rs_client.rs_client import RsClient
|
|
20
|
+
from rs_common.config import AUXIP_STATION
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class AuxipClient(RsClient):
|
|
24
|
+
"""
|
|
25
|
+
AuxipClient class implementation.
|
|
26
|
+
|
|
27
|
+
Attributes: see :py:class:`RsClient`
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
@property
|
|
31
|
+
def href_adgs(self) -> str:
|
|
32
|
+
"""
|
|
33
|
+
Return the RS-Server ADGS URL hostname.
|
|
34
|
+
This URL can be overwritten using the RSPY_HOST_ADGS env variable (used e.g. for local mode).
|
|
35
|
+
Otherwise it should just be the RS-Server URL.
|
|
36
|
+
"""
|
|
37
|
+
if from_env := os.getenv("RSPY_HOST_ADGS", None):
|
|
38
|
+
return from_env.rstrip("/")
|
|
39
|
+
if not self.rs_server_href:
|
|
40
|
+
raise RuntimeError("RS-Server URL is undefined")
|
|
41
|
+
return self.rs_server_href.rstrip("/")
|
|
42
|
+
|
|
43
|
+
@property
|
|
44
|
+
def href_search(self) -> str:
|
|
45
|
+
"""Return the RS-Server hostname and path where the ADGS search endpoint is deployed."""
|
|
46
|
+
return f"{self.href_adgs}/adgs/aux/search"
|
|
47
|
+
|
|
48
|
+
@property
|
|
49
|
+
def href_staging(self) -> str:
|
|
50
|
+
"""Return the RS-Server hostname and path where the ADGS staging endpoint is deployed."""
|
|
51
|
+
return f"{self.href_adgs}/adgs/aux"
|
|
52
|
+
|
|
53
|
+
@property
|
|
54
|
+
def href_status(self) -> str:
|
|
55
|
+
"""Return the RS-Server hostname and path where the ADGS status endpoint is deployed."""
|
|
56
|
+
return f"{self.href_adgs}/adgs/aux/status"
|
|
57
|
+
|
|
58
|
+
@property
|
|
59
|
+
def station_name(self) -> str:
|
|
60
|
+
"""Return "AUXIP"."""
|
|
61
|
+
return AUXIP_STATION
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
# Copyright 2024 CS Group
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""CadipClient class implementation."""
|
|
16
|
+
|
|
17
|
+
import logging
|
|
18
|
+
import os
|
|
19
|
+
from datetime import datetime
|
|
20
|
+
|
|
21
|
+
import requests
|
|
22
|
+
|
|
23
|
+
from rs_client.rs_client import TIMEOUT, RsClient
|
|
24
|
+
from rs_common.config import DATETIME_FORMAT, ECadipStation, EPlatform
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class CadipClient(RsClient):
|
|
28
|
+
"""
|
|
29
|
+
CadipClient class implementation.
|
|
30
|
+
|
|
31
|
+
Attributes: see :py:class:`RsClient`
|
|
32
|
+
station (ECadipStation): Cadip station
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
def __init__( # pylint: disable=too-many-arguments, too-many-positional-arguments
|
|
36
|
+
self,
|
|
37
|
+
rs_server_href: str | None,
|
|
38
|
+
rs_server_api_key: str | None,
|
|
39
|
+
owner_id: str | None,
|
|
40
|
+
station: ECadipStation,
|
|
41
|
+
logger: logging.Logger | None = None,
|
|
42
|
+
):
|
|
43
|
+
"""CadipClient class constructor."""
|
|
44
|
+
super().__init__(rs_server_href, rs_server_api_key, owner_id, logger)
|
|
45
|
+
try:
|
|
46
|
+
self.station: ECadipStation = ECadipStation[station]
|
|
47
|
+
except KeyError as e:
|
|
48
|
+
self.logger.exception(f"There is no such CADIP station: {station}")
|
|
49
|
+
raise RuntimeError(f"There is no such CADIP station: {station}") from e
|
|
50
|
+
|
|
51
|
+
@property
|
|
52
|
+
def href_cadip(self) -> str:
|
|
53
|
+
"""
|
|
54
|
+
Return the RS-Server CADIP URL hostname.
|
|
55
|
+
This URL can be overwritten using the RSPY_HOST_CADIP env variable (used e.g. for local mode).
|
|
56
|
+
Otherwise it should just be the RS-Server URL.
|
|
57
|
+
"""
|
|
58
|
+
if from_env := os.getenv("RSPY_HOST_CADIP", None):
|
|
59
|
+
return from_env.rstrip("/")
|
|
60
|
+
if not self.rs_server_href:
|
|
61
|
+
raise RuntimeError("RS-Server URL is undefined")
|
|
62
|
+
return self.rs_server_href.rstrip("/")
|
|
63
|
+
|
|
64
|
+
@property
|
|
65
|
+
def href_search(self) -> str:
|
|
66
|
+
"""Return the RS-Server hostname and path where the CADIP search endpoint is deployed."""
|
|
67
|
+
return f"{self.href_cadip}/cadip/{self.station.value}/cadu/search"
|
|
68
|
+
|
|
69
|
+
@property
|
|
70
|
+
def href_session(self) -> str:
|
|
71
|
+
"""Return the RS-Server hostname and path where the CADIP search session endpoint is deployed."""
|
|
72
|
+
return f"{self.href_cadip}/cadip/{self.station.value}/session"
|
|
73
|
+
|
|
74
|
+
@property
|
|
75
|
+
def href_staging(self) -> str:
|
|
76
|
+
"""Return the RS-Server hostname and path where the CADIP staging endpoint is deployed."""
|
|
77
|
+
return f"{self.href_cadip}/cadip/{self.station.value}/cadu"
|
|
78
|
+
|
|
79
|
+
@property
|
|
80
|
+
def href_status(self) -> str:
|
|
81
|
+
"""Return the RS-Server hostname and path where the CADIP status endpoint is deployed."""
|
|
82
|
+
return f"{self.href_cadip}/cadip/{self.station.value}/cadu/status"
|
|
83
|
+
|
|
84
|
+
@property
|
|
85
|
+
def station_name(self) -> str:
|
|
86
|
+
"""Return the station name."""
|
|
87
|
+
return self.station.value # TO BE DISCUSSED: maybe just return "CADIP"
|
|
88
|
+
|
|
89
|
+
############################
|
|
90
|
+
# Call RS-Server endpoints #
|
|
91
|
+
############################
|
|
92
|
+
|
|
93
|
+
def search_sessions( # pylint: disable=too-many-arguments, too-many-positional-arguments
|
|
94
|
+
self,
|
|
95
|
+
session_ids: list[str] | None = None,
|
|
96
|
+
start_date: datetime | None = None,
|
|
97
|
+
stop_date: datetime | None = None,
|
|
98
|
+
platforms: list[EPlatform] | None = None,
|
|
99
|
+
timeout: int = TIMEOUT,
|
|
100
|
+
) -> list[dict]: # TODO return pystac.ItemCollection instead
|
|
101
|
+
"""Endpoint to retrieve list of sessions from any CADIP station.
|
|
102
|
+
|
|
103
|
+
Args:
|
|
104
|
+
timeout (int): The timeout duration for the HTTP request.
|
|
105
|
+
session_ids (list[str]): Session identifiers
|
|
106
|
+
(eg: ["S1A_20170501121534062343"] or ["S1A_20170501121534062343, S1A_20240328185208053186"])
|
|
107
|
+
start_date (datetime): Start date of the time interval
|
|
108
|
+
stop_date (datetime): Stop date of the time interval
|
|
109
|
+
platforms (list[PlatformEnum]): platform list
|
|
110
|
+
"""
|
|
111
|
+
|
|
112
|
+
payload = {}
|
|
113
|
+
if session_ids:
|
|
114
|
+
payload["id"] = ",".join(session_ids)
|
|
115
|
+
if platforms:
|
|
116
|
+
payload["platform"] = ",".join([platform.value for platform in platforms])
|
|
117
|
+
if start_date:
|
|
118
|
+
payload["start_date"] = start_date.strftime(DATETIME_FORMAT)
|
|
119
|
+
if stop_date:
|
|
120
|
+
payload["stop_date"] = stop_date.strftime(DATETIME_FORMAT)
|
|
121
|
+
try:
|
|
122
|
+
response = self.http_session.get(
|
|
123
|
+
self.href_session,
|
|
124
|
+
params=payload,
|
|
125
|
+
timeout=timeout,
|
|
126
|
+
**self.apikey_headers,
|
|
127
|
+
)
|
|
128
|
+
except (requests.exceptions.RequestException, requests.exceptions.Timeout) as e:
|
|
129
|
+
self.logger.exception(f"Could not get the response from the session search endpoint: {e}")
|
|
130
|
+
raise RuntimeError("Could not get the response from the session search endpoint") from e
|
|
131
|
+
|
|
132
|
+
sessions = []
|
|
133
|
+
try:
|
|
134
|
+
if response.ok:
|
|
135
|
+
for session_info in response.json()["features"]:
|
|
136
|
+
sessions.append(session_info)
|
|
137
|
+
else:
|
|
138
|
+
self.logger.error(f"Error: {response.status_code} : {response.json()}")
|
|
139
|
+
except KeyError as e:
|
|
140
|
+
raise RuntimeError("Wrong format of session search endpoint answer") from e
|
|
141
|
+
|
|
142
|
+
return sessions
|
rs_client/rs_client.py
ADDED
|
@@ -0,0 +1,433 @@
|
|
|
1
|
+
# Copyright 2024 CS Group
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""RsClient class implementation."""
|
|
16
|
+
|
|
17
|
+
import getpass
|
|
18
|
+
import logging
|
|
19
|
+
import os
|
|
20
|
+
import re
|
|
21
|
+
import sys
|
|
22
|
+
from datetime import datetime
|
|
23
|
+
from typing import Union
|
|
24
|
+
|
|
25
|
+
import requests
|
|
26
|
+
from cachetools import TTLCache, cached
|
|
27
|
+
|
|
28
|
+
from rs_common import utils
|
|
29
|
+
from rs_common.config import DATETIME_FORMAT, ECadipStation, EDownloadStatus
|
|
30
|
+
from rs_common.logging import Logging
|
|
31
|
+
from rs_common.utils import AuthInfo
|
|
32
|
+
|
|
33
|
+
APIKEY_HEADER = "x-api-key"
|
|
34
|
+
|
|
35
|
+
# Timeout in seconds
|
|
36
|
+
TIMEOUT = 30
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class RsClient: # pylint: disable=too-many-instance-attributes
|
|
40
|
+
"""
|
|
41
|
+
RsClient class implementation.
|
|
42
|
+
|
|
43
|
+
Attributes:
|
|
44
|
+
rs_server_href (str): RS-Server URL. In local mode, pass None.
|
|
45
|
+
rs_server_api_key (str): API key for RS-Server authentication.
|
|
46
|
+
rs_server_oauth2_cookie (str): session cookie that contains the OAuth2 authentication read from the
|
|
47
|
+
RSPY_OAUTH2_COOKIE environment variable.
|
|
48
|
+
owner_id (str): ID of the owner of the STAC catalog collections (no special characters allowoed).
|
|
49
|
+
By default, this is the user login from the keycloak account, associated to the API key.
|
|
50
|
+
Or, in local mode, this is the local system username.
|
|
51
|
+
Else, your API Key must give you the rights to read/write on this catalog owner.
|
|
52
|
+
This owner ID is also used in the RS-Client logging.
|
|
53
|
+
logger (logging.Logger): Logging instance.
|
|
54
|
+
local_mode (bool): Local mode or hybrid/cluster mode.
|
|
55
|
+
apikey_headers (dict): API key in a dict, ready-to-use in HTTP request headers.
|
|
56
|
+
http_session (Session): HTTP requests session with cookies.
|
|
57
|
+
"""
|
|
58
|
+
|
|
59
|
+
def __init__(
|
|
60
|
+
self,
|
|
61
|
+
rs_server_href: str | None,
|
|
62
|
+
rs_server_api_key: str | None = None,
|
|
63
|
+
owner_id: str | None = None,
|
|
64
|
+
logger: logging.Logger | None = None,
|
|
65
|
+
):
|
|
66
|
+
"""RsClient class constructor."""
|
|
67
|
+
self.rs_server_href: str | None = rs_server_href
|
|
68
|
+
self.rs_server_api_key: str | None = rs_server_api_key
|
|
69
|
+
self.rs_server_oauth2_cookie: str | None = os.getenv("RSPY_OAUTH2_COOKIE")
|
|
70
|
+
self.owner_id: str | None = owner_id or ""
|
|
71
|
+
self.logger: logging.Logger = logger or Logging.default(__name__)
|
|
72
|
+
|
|
73
|
+
# Remove trailing / character(s) from the URL
|
|
74
|
+
if self.rs_server_href:
|
|
75
|
+
self.rs_server_href = self.rs_server_href.strip().rstrip("/").strip()
|
|
76
|
+
|
|
77
|
+
# We are in local mode if the URL is undefined.
|
|
78
|
+
# Env vars are used instead to determine the different services URL.
|
|
79
|
+
self.local_mode = not bool(self.rs_server_href)
|
|
80
|
+
|
|
81
|
+
if (not self.local_mode) and (not self.rs_server_api_key) and (not self.rs_server_oauth2_cookie):
|
|
82
|
+
raise RuntimeError("API key or OAuth2 cookie is mandatory for RS-Server authentication")
|
|
83
|
+
|
|
84
|
+
# For HTTP request headers
|
|
85
|
+
self.apikey_headers: dict = (
|
|
86
|
+
{"headers": {APIKEY_HEADER: self.rs_server_api_key}} if self.rs_server_api_key else {}
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
# HTTP requests session with cookies
|
|
90
|
+
self.http_session = requests.Session()
|
|
91
|
+
if self.rs_server_oauth2_cookie:
|
|
92
|
+
self.http_session.cookies.set("session", self.rs_server_oauth2_cookie)
|
|
93
|
+
|
|
94
|
+
# Determine automatically the owner id
|
|
95
|
+
if not self.owner_id:
|
|
96
|
+
# In local mode, we use the local system username
|
|
97
|
+
if self.local_mode:
|
|
98
|
+
self.owner_id = getpass.getuser()
|
|
99
|
+
|
|
100
|
+
# In hybrid/cluster mode, we retrieve the OAuth2 or API key login
|
|
101
|
+
else:
|
|
102
|
+
self.owner_id = self.apikey_user_login if self.rs_server_api_key else self.oauth2_user_login
|
|
103
|
+
|
|
104
|
+
# Remove special characters
|
|
105
|
+
self.owner_id = re.sub(r"[^a-zA-Z0-9]+", "", self.owner_id)
|
|
106
|
+
|
|
107
|
+
if not self.owner_id:
|
|
108
|
+
raise RuntimeError("The owner ID is empty or only contains special characters")
|
|
109
|
+
|
|
110
|
+
self.logger.debug(f"Owner ID: {self.owner_id!r}")
|
|
111
|
+
|
|
112
|
+
def oauth2_security(self) -> AuthInfo:
|
|
113
|
+
"""
|
|
114
|
+
Returns:
|
|
115
|
+
Authentication information from the user keycloak account, associated to the authentication cookie.
|
|
116
|
+
"""
|
|
117
|
+
|
|
118
|
+
# In local mode, we have no authentication, so return empty results
|
|
119
|
+
if self.local_mode:
|
|
120
|
+
return AuthInfo(user_login="", iam_roles=[], apikey_config={})
|
|
121
|
+
|
|
122
|
+
# Call the endpoint to retrieve the user information
|
|
123
|
+
response = self.http_session.get(f"{self.rs_server_href}/auth/me")
|
|
124
|
+
if not response.ok:
|
|
125
|
+
raise RuntimeError(f"OAuth2 status code {response.status_code}: {utils.read_response_error(response)}")
|
|
126
|
+
|
|
127
|
+
# Decode the JSON response
|
|
128
|
+
contents = response.json()
|
|
129
|
+
return AuthInfo(
|
|
130
|
+
user_login=contents["user_login"],
|
|
131
|
+
iam_roles=contents["iam_roles"],
|
|
132
|
+
apikey_config={}, # no API key config here
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
# The following variable is needed for the tests to pass
|
|
136
|
+
apikey_security_cache: TTLCache = TTLCache(maxsize=sys.maxsize, ttl=120)
|
|
137
|
+
|
|
138
|
+
@cached(cache=apikey_security_cache)
|
|
139
|
+
def apikey_security(self) -> AuthInfo:
|
|
140
|
+
"""
|
|
141
|
+
Check the api key validity. Cache an infinite (sys.maxsize) number of results for 120 seconds.
|
|
142
|
+
|
|
143
|
+
Returns:
|
|
144
|
+
Authentication information from the keycloak account, associated to the api key.
|
|
145
|
+
"""
|
|
146
|
+
|
|
147
|
+
# In local mode, we have no API key, so return empty results
|
|
148
|
+
if self.local_mode:
|
|
149
|
+
return AuthInfo(user_login="", iam_roles=[], apikey_config={})
|
|
150
|
+
|
|
151
|
+
# self.logger.warning(
|
|
152
|
+
# f"TODO: use {self.rs_server_href}/apikeymanager/auth/check_key instead, see: "
|
|
153
|
+
# "https://pforge-exchange2.astrium.eads.net/jira/browse/RSPY-257",
|
|
154
|
+
# )
|
|
155
|
+
# Does not work in hybrid mode for now because this URL is not exposed.
|
|
156
|
+
check_url = os.environ["RSPY_UAC_CHECK_URL"]
|
|
157
|
+
|
|
158
|
+
# Request the API key manager, pass user-defined api key in http header
|
|
159
|
+
# check_url = f"{self.rs_server_href}/apikeymanager/auth/check_key"
|
|
160
|
+
self.logger.debug("Call the API key manager")
|
|
161
|
+
response = self.http_session.get(check_url, **self.apikey_headers, timeout=TIMEOUT)
|
|
162
|
+
if not response.ok:
|
|
163
|
+
raise RuntimeError(
|
|
164
|
+
f"API key manager status code {response.status_code}: {utils.read_response_error(response)}",
|
|
165
|
+
)
|
|
166
|
+
|
|
167
|
+
# Read the api key info.
|
|
168
|
+
# Note: for now, config is an empty dict.
|
|
169
|
+
contents = response.json()
|
|
170
|
+
return AuthInfo(
|
|
171
|
+
user_login=contents["user_login"],
|
|
172
|
+
iam_roles=contents["iam_roles"],
|
|
173
|
+
apikey_config=contents["config"],
|
|
174
|
+
)
|
|
175
|
+
|
|
176
|
+
@property
|
|
177
|
+
def oauth2_user_login(self) -> str:
|
|
178
|
+
"""Return the user login from the keycloak account, associated to the authentication cookie."""
|
|
179
|
+
return self.oauth2_security().user_login
|
|
180
|
+
|
|
181
|
+
@property
|
|
182
|
+
def apikey_user_login(self) -> str:
|
|
183
|
+
"""Return the user login from the keycloak account, associated to the api key."""
|
|
184
|
+
return self.apikey_security().user_login
|
|
185
|
+
|
|
186
|
+
@property
|
|
187
|
+
def oauth2_iam_roles(self) -> list[str]:
|
|
188
|
+
"""
|
|
189
|
+
Return the IAM (Identity and Access Management) roles from the keycloak account,
|
|
190
|
+
associated to the authentication cookie
|
|
191
|
+
"""
|
|
192
|
+
return self.oauth2_security().iam_roles
|
|
193
|
+
|
|
194
|
+
@property
|
|
195
|
+
def apikey_iam_roles(self) -> list[str]:
|
|
196
|
+
"""
|
|
197
|
+
Return the IAM (Identity and Access Management) roles from the keycloak account,
|
|
198
|
+
associated to the api key.
|
|
199
|
+
"""
|
|
200
|
+
return self.apikey_security().iam_roles
|
|
201
|
+
|
|
202
|
+
@property
|
|
203
|
+
def apikey_config(self) -> dict:
|
|
204
|
+
"""Return the config from the keycloak account, associated to the api key."""
|
|
205
|
+
return self.apikey_security().apikey_config
|
|
206
|
+
|
|
207
|
+
#############################
|
|
208
|
+
# Get child class instances #
|
|
209
|
+
#############################
|
|
210
|
+
|
|
211
|
+
def get_auxip_client(self) -> "AuxipClient": # type: ignore # noqa: F821
|
|
212
|
+
"""
|
|
213
|
+
Return an instance of the child class AuxipClient, with the same attributes as this "self" instance.
|
|
214
|
+
"""
|
|
215
|
+
from rs_client.auxip_client import ( # pylint: disable=import-outside-toplevel,cyclic-import
|
|
216
|
+
AuxipClient,
|
|
217
|
+
)
|
|
218
|
+
|
|
219
|
+
return AuxipClient(self.rs_server_href, self.rs_server_api_key, self.owner_id, self.logger)
|
|
220
|
+
|
|
221
|
+
def get_cadip_client(
|
|
222
|
+
self,
|
|
223
|
+
station: ECadipStation,
|
|
224
|
+
) -> "CadipClient": # type: ignore # noqa: F821
|
|
225
|
+
"""
|
|
226
|
+
Return an instance of the child class CadipClient, with the same attributes as this "self" instance.
|
|
227
|
+
|
|
228
|
+
Args:
|
|
229
|
+
station (ECadipStation): Cadip station
|
|
230
|
+
"""
|
|
231
|
+
from rs_client.cadip_client import ( # pylint: disable=import-outside-toplevel,cyclic-import
|
|
232
|
+
CadipClient,
|
|
233
|
+
)
|
|
234
|
+
|
|
235
|
+
return CadipClient(self.rs_server_href, self.rs_server_api_key, self.owner_id, station, self.logger)
|
|
236
|
+
|
|
237
|
+
def get_stac_client(self, *args, **kwargs) -> "StacClient": # type: ignore # noqa: F821
|
|
238
|
+
"""
|
|
239
|
+
Return an instance of the child class StacClient, with the same attributes as this "self" instance.
|
|
240
|
+
"""
|
|
241
|
+
from rs_client.stac_client import ( # pylint: disable=import-outside-toplevel,cyclic-import
|
|
242
|
+
StacClient,
|
|
243
|
+
)
|
|
244
|
+
|
|
245
|
+
return StacClient.open(
|
|
246
|
+
self.rs_server_href,
|
|
247
|
+
self.rs_server_api_key,
|
|
248
|
+
self.rs_server_oauth2_cookie,
|
|
249
|
+
self.owner_id,
|
|
250
|
+
self.logger,
|
|
251
|
+
*args,
|
|
252
|
+
**kwargs,
|
|
253
|
+
)
|
|
254
|
+
|
|
255
|
+
############################
|
|
256
|
+
# Call RS-Server endpoints #
|
|
257
|
+
############################
|
|
258
|
+
|
|
259
|
+
def staging_status(self, filename, timeout: int = TIMEOUT) -> EDownloadStatus:
|
|
260
|
+
"""Check the status of a file download from the specified rs-server endpoint.
|
|
261
|
+
|
|
262
|
+
This function sends a GET request to the rs-server endpoint with the filename as a query parameter
|
|
263
|
+
to retrieve the status of the file download. If the response is successful and contains a 'status'
|
|
264
|
+
key in the JSON response, the function returns the corresponding download status enum value. If the
|
|
265
|
+
response is not successful or does not contain the 'status' key, the function returns a FAILED status.
|
|
266
|
+
|
|
267
|
+
Args:
|
|
268
|
+
filename (str): The name of the file for which to check the status.
|
|
269
|
+
timeout (int): The timeout duration for the HTTP request.
|
|
270
|
+
|
|
271
|
+
Returns:
|
|
272
|
+
EDownloadStatus: The download status enum value based on the response from the endpoint.
|
|
273
|
+
"""
|
|
274
|
+
|
|
275
|
+
# TODO: check the status for a certain timeout if http returns NOK ?
|
|
276
|
+
try:
|
|
277
|
+
response = self.http_session.get(
|
|
278
|
+
self.href_status, # pylint: disable=no-member # ("self" is AuxipClient or CadipClient)
|
|
279
|
+
params={"name": filename},
|
|
280
|
+
timeout=timeout,
|
|
281
|
+
**self.apikey_headers,
|
|
282
|
+
)
|
|
283
|
+
|
|
284
|
+
eval_response = response.json()
|
|
285
|
+
if (
|
|
286
|
+
response.ok
|
|
287
|
+
and "name" in eval_response.keys()
|
|
288
|
+
and filename == eval_response["name"]
|
|
289
|
+
and "status" in eval_response.keys()
|
|
290
|
+
):
|
|
291
|
+
return EDownloadStatus(eval_response["status"])
|
|
292
|
+
|
|
293
|
+
except (requests.exceptions.RequestException, requests.exceptions.Timeout) as e:
|
|
294
|
+
self.logger.exception(f"Status endpoint exception: {e}")
|
|
295
|
+
|
|
296
|
+
return EDownloadStatus.FAILED
|
|
297
|
+
|
|
298
|
+
def staging(self, filename: str, s3_path: str = "", tmp_download_path: str = "", timeout: int = TIMEOUT):
|
|
299
|
+
"""Stage a file for download.
|
|
300
|
+
|
|
301
|
+
This method stages a file for download by sending a request to the staging endpoint
|
|
302
|
+
with optional parameters for S3 path and temporary download path.
|
|
303
|
+
|
|
304
|
+
Args:
|
|
305
|
+
filename (str): The name of the file to be staged for download.
|
|
306
|
+
timeout (int): The timeout duration for the HTTP request.
|
|
307
|
+
s3_path (str, optional): The S3 path where the file will be stored after download.
|
|
308
|
+
Defaults to an empty string.
|
|
309
|
+
tmp_download_path (str, optional): The temporary download path for the file.
|
|
310
|
+
Defaults to an empty string.
|
|
311
|
+
|
|
312
|
+
Raises:
|
|
313
|
+
RuntimeError: If an error occurs while staging the file.
|
|
314
|
+
|
|
315
|
+
"""
|
|
316
|
+
|
|
317
|
+
# dictionary to be used for payload request
|
|
318
|
+
payload = {}
|
|
319
|
+
# some protections for the optional args
|
|
320
|
+
if s3_path:
|
|
321
|
+
payload["obs"] = s3_path
|
|
322
|
+
if tmp_download_path:
|
|
323
|
+
payload["local"] = tmp_download_path
|
|
324
|
+
|
|
325
|
+
# update the filename to be ingested
|
|
326
|
+
payload["name"] = filename
|
|
327
|
+
try:
|
|
328
|
+
# logger.debug(f"Calling {endpoint} with payload {payload}")
|
|
329
|
+
response = self.http_session.get(
|
|
330
|
+
self.href_staging, # pylint: disable=no-member # ("self" is AuxipClient or CadipClient)
|
|
331
|
+
params=payload,
|
|
332
|
+
timeout=timeout,
|
|
333
|
+
**self.apikey_headers,
|
|
334
|
+
)
|
|
335
|
+
self.logger.debug(f"Download start endpoint returned in {response.elapsed.total_seconds()}")
|
|
336
|
+
if not response.ok:
|
|
337
|
+
self.logger.error(f"The download endpoint returned error for file {filename}\n")
|
|
338
|
+
raise RuntimeError(f"The download endpoint returned error for file {filename}")
|
|
339
|
+
except (
|
|
340
|
+
requests.exceptions.RequestException,
|
|
341
|
+
requests.exceptions.Timeout,
|
|
342
|
+
requests.exceptions.ReadTimeout,
|
|
343
|
+
) as e:
|
|
344
|
+
self.logger.exception(f"Staging file exception for {filename}:", e)
|
|
345
|
+
raise RuntimeError(f"Staging file exception for {filename}") from e
|
|
346
|
+
|
|
347
|
+
def search_stations( # pylint: disable=too-many-arguments, too-many-positional-arguments
|
|
348
|
+
self,
|
|
349
|
+
start_date: datetime,
|
|
350
|
+
stop_date: datetime,
|
|
351
|
+
limit: Union[int, None] = None,
|
|
352
|
+
sortby: Union[str, None] = None,
|
|
353
|
+
timeout: int = TIMEOUT,
|
|
354
|
+
) -> list:
|
|
355
|
+
"""Retrieve a list of files from the specified endpoint.
|
|
356
|
+
|
|
357
|
+
This function queries the specified endpoint to retrieve a list of files available in the
|
|
358
|
+
station (CADIP, ADGS, LTA ...) that were collected by the satellite within the provided time range,
|
|
359
|
+
starting from 'start_date' up to 'stop_date' (inclusive).
|
|
360
|
+
|
|
361
|
+
Args:
|
|
362
|
+
start_date (datetime): The start date of the time range.
|
|
363
|
+
stop_date (datetime): The stop date of the time range.
|
|
364
|
+
timeout (int): The timeout duration for the HTTP request.
|
|
365
|
+
limit (int, optional): The maximum number of results to return. Defaults to None.
|
|
366
|
+
sortby (str, optional): The attribute to sort the results by. Defaults to None.
|
|
367
|
+
|
|
368
|
+
Returns:
|
|
369
|
+
files (list): The list of files available at the station within the specified time range.
|
|
370
|
+
|
|
371
|
+
Raises:
|
|
372
|
+
RuntimeError: if the endpoint can't be reached
|
|
373
|
+
|
|
374
|
+
Notes:
|
|
375
|
+
- This function queries the specified endpoint with a time range to retrieve information about
|
|
376
|
+
available files.
|
|
377
|
+
- It constructs a payload with the start and stop dates in ISO 8601 format and sends a GET
|
|
378
|
+
request to the endpoint.
|
|
379
|
+
- The response is expected to be a STAC Compatible formatted JSONResponse, containing information about
|
|
380
|
+
available files.
|
|
381
|
+
- The function converts a STAC FeatureCollection to a Python list.
|
|
382
|
+
"""
|
|
383
|
+
|
|
384
|
+
payload = {
|
|
385
|
+
"datetime": start_date.strftime(DATETIME_FORMAT)
|
|
386
|
+
+ "/" # 2014-01-01T12:00:00Z/2023-12-30T12:00:00Z",
|
|
387
|
+
+ stop_date.strftime(DATETIME_FORMAT),
|
|
388
|
+
}
|
|
389
|
+
if limit:
|
|
390
|
+
payload["limit"] = str(limit)
|
|
391
|
+
if sortby:
|
|
392
|
+
payload["sortby"] = str(sortby)
|
|
393
|
+
try:
|
|
394
|
+
response = self.http_session.get(
|
|
395
|
+
self.href_search, # pylint: disable=no-member # ("self" is AuxipClient or CadipClient)
|
|
396
|
+
params=payload,
|
|
397
|
+
timeout=timeout,
|
|
398
|
+
**self.apikey_headers,
|
|
399
|
+
)
|
|
400
|
+
except (requests.exceptions.RequestException, requests.exceptions.Timeout) as e:
|
|
401
|
+
self.logger.exception(f"Could not get the response from the station search endpoint: {e}")
|
|
402
|
+
raise RuntimeError("Could not get the response from the station search endpoint") from e
|
|
403
|
+
|
|
404
|
+
files = []
|
|
405
|
+
try:
|
|
406
|
+
if response.ok:
|
|
407
|
+
for file_info in response.json()["features"]:
|
|
408
|
+
files.append(file_info)
|
|
409
|
+
else:
|
|
410
|
+
self.logger.error(f"Error: {response.status_code} : {response.json()}")
|
|
411
|
+
except KeyError as e:
|
|
412
|
+
raise RuntimeError("Wrong format of search endpoint answer") from e
|
|
413
|
+
|
|
414
|
+
return files
|
|
415
|
+
|
|
416
|
+
##############################################
|
|
417
|
+
# Methods to be implemented by child classes #
|
|
418
|
+
##############################################
|
|
419
|
+
|
|
420
|
+
@property
|
|
421
|
+
def href_search(self) -> str:
|
|
422
|
+
"""Implemented by AuxipClient and CadipClient."""
|
|
423
|
+
raise NotImplementedError
|
|
424
|
+
|
|
425
|
+
@property
|
|
426
|
+
def href_staging(self) -> str:
|
|
427
|
+
"""Implemented by AuxipClient and CadipClient."""
|
|
428
|
+
raise NotImplementedError
|
|
429
|
+
|
|
430
|
+
@property
|
|
431
|
+
def href_status(self) -> str:
|
|
432
|
+
"""Implemented by AuxipClient and CadipClient."""
|
|
433
|
+
raise NotImplementedError
|