rs-client-libraries 0.2a11__tar.gz → 0.2a12__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {rs_client_libraries-0.2a11 → rs_client_libraries-0.2a12}/PKG-INFO +10 -10
- {rs_client_libraries-0.2a11 → rs_client_libraries-0.2a12}/pyproject.toml +13 -13
- {rs_client_libraries-0.2a11 → rs_client_libraries-0.2a12}/rs_client/__init__.py +1 -1
- {rs_client_libraries-0.2a11 → rs_client_libraries-0.2a12}/rs_client/auxip_client.py +10 -2
- {rs_client_libraries-0.2a11 → rs_client_libraries-0.2a12}/rs_client/cadip_client.py +23 -2
- {rs_client_libraries-0.2a11 → rs_client_libraries-0.2a12}/rs_client/catalog_client.py +8 -8
- {rs_client_libraries-0.2a11 → rs_client_libraries-0.2a12}/rs_client/rs_client.py +27 -19
- {rs_client_libraries-0.2a11 → rs_client_libraries-0.2a12}/rs_client/stac_base.py +14 -13
- {rs_client_libraries-0.2a11 → rs_client_libraries-0.2a12}/rs_client/staging_client.py +87 -68
- {rs_client_libraries-0.2a11 → rs_client_libraries-0.2a12}/rs_common/opentelemetry.py +0 -4
- {rs_client_libraries-0.2a11 → rs_client_libraries-0.2a12}/LICENSE +0 -0
- {rs_client_libraries-0.2a11 → rs_client_libraries-0.2a12}/README.adoc +0 -0
- {rs_client_libraries-0.2a11 → rs_client_libraries-0.2a12}/rs_common/__init__.py +0 -0
- {rs_client_libraries-0.2a11 → rs_client_libraries-0.2a12}/rs_common/config.py +0 -0
- {rs_client_libraries-0.2a11 → rs_client_libraries-0.2a12}/rs_common/logging.py +0 -0
- {rs_client_libraries-0.2a11 → rs_client_libraries-0.2a12}/rs_common/utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: rs-client-libraries
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2a12
|
|
4
4
|
Summary: Copernicus Sentinel rs-client library and processing workflows.
|
|
5
5
|
Home-page: https://home.rs-python.eu/
|
|
6
6
|
License: Apache-2.0
|
|
@@ -19,17 +19,17 @@ Classifier: Programming Language :: Python :: 3.12
|
|
|
19
19
|
Classifier: Programming Language :: Python :: 3.13
|
|
20
20
|
Classifier: Topic :: Scientific/Engineering :: Image Processing
|
|
21
21
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
22
|
-
Requires-Dist: cachetools (>=5.5.
|
|
23
|
-
Requires-Dist: openapi-core (>=0.19.
|
|
24
|
-
Requires-Dist: opentelemetry-distro (>=0.
|
|
25
|
-
Requires-Dist: opentelemetry-exporter-otlp (>=1.
|
|
26
|
-
Requires-Dist: opentelemetry-instrumentation-aws-lambda (>=0.
|
|
27
|
-
Requires-Dist: opentelemetry-test-utils (>=0.
|
|
28
|
-
Requires-Dist: pystac (>=1.12.
|
|
29
|
-
Requires-Dist: pystac-client (>=0.8.
|
|
22
|
+
Requires-Dist: cachetools (>=5.5.2,<6.0.0)
|
|
23
|
+
Requires-Dist: openapi-core (>=0.19.5,<0.20.0)
|
|
24
|
+
Requires-Dist: opentelemetry-distro (>=0.53b0,<0.54)
|
|
25
|
+
Requires-Dist: opentelemetry-exporter-otlp (>=1.32.0,<2.0.0)
|
|
26
|
+
Requires-Dist: opentelemetry-instrumentation-aws-lambda (>=0.53b0,<0.54)
|
|
27
|
+
Requires-Dist: opentelemetry-test-utils (>=0.53b0,<0.54)
|
|
28
|
+
Requires-Dist: pystac (>=1.12.2,<2.0.0)
|
|
29
|
+
Requires-Dist: pystac-client (>=0.8.6,<0.9.0)
|
|
30
30
|
Requires-Dist: python-dateutil (>=2.9.0.post0,<3.0.0)
|
|
31
31
|
Requires-Dist: python-logging-loki (>=0.3.1,<0.4.0)
|
|
32
|
-
Requires-Dist: stac-pydantic (>=3.1.
|
|
32
|
+
Requires-Dist: stac-pydantic (>=3.1.5,<4.0.0)
|
|
33
33
|
Project-URL: Documentation, https://home.rs-python.eu/rs-documentation/rs-client-libraries/docs/doc/
|
|
34
34
|
Project-URL: Repository, https://github.com/RS-PYTHON/rs-client-libraries
|
|
35
35
|
Description-Content-Type: text/plain
|
|
@@ -4,7 +4,7 @@ version = "0.0.0" # see: https://github.com/mtkennerly/poetry-dynamic-versioning
|
|
|
4
4
|
|
|
5
5
|
[tool.poetry]
|
|
6
6
|
name = "rs-client-libraries"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.2a12"
|
|
8
8
|
description = "Copernicus Sentinel rs-client library and processing workflows."
|
|
9
9
|
license = "Apache-2.0"
|
|
10
10
|
authors = ["ESA under contract with Airbus and CS <ops_coprs@airbus.com>"]
|
|
@@ -36,23 +36,23 @@ include = [
|
|
|
36
36
|
|
|
37
37
|
[tool.poetry.dependencies]
|
|
38
38
|
python = "^3.11"
|
|
39
|
-
opentelemetry-distro = "^0.
|
|
40
|
-
opentelemetry-instrumentation-aws-lambda = "^0.
|
|
41
|
-
opentelemetry-test-utils = "^0.
|
|
42
|
-
opentelemetry-exporter-otlp = "^1.
|
|
39
|
+
opentelemetry-distro = "^0.53b0" # then run 'poetry run opentelemetry-bootstrap -a install'
|
|
40
|
+
opentelemetry-instrumentation-aws-lambda = "^0.53b0"
|
|
41
|
+
opentelemetry-test-utils = "^0.53b0"
|
|
42
|
+
opentelemetry-exporter-otlp = "^1.32.0"
|
|
43
43
|
python-logging-loki = "^0.3.1" # to avoid otl error: No module named 'logging_loki'
|
|
44
|
-
cachetools = "^5.5.
|
|
45
|
-
pystac = "^1.12.
|
|
46
|
-
pystac-client = "^0.8.
|
|
44
|
+
cachetools = "^5.5.2"
|
|
45
|
+
pystac = "^1.12.2"
|
|
46
|
+
pystac-client = "^0.8.6"
|
|
47
47
|
python-dateutil = "^2.9.0.post0"
|
|
48
|
-
stac-pydantic = "^3.1.
|
|
49
|
-
openapi-core = "^0.19.
|
|
48
|
+
stac-pydantic = "^3.1.5"
|
|
49
|
+
openapi-core = "^0.19.5"
|
|
50
50
|
|
|
51
51
|
[tool.poetry.group.dev]
|
|
52
52
|
optional = true
|
|
53
53
|
|
|
54
54
|
[tool.poetry.group.dev.dependencies]
|
|
55
|
-
pytest = "^8.3.
|
|
55
|
+
pytest = "^8.3.5"
|
|
56
56
|
pytest-cov = "^6.0.0"
|
|
57
57
|
pytest-mock = "^3.14.0"
|
|
58
58
|
pytest-error-for-skips = "^2.0.2"
|
|
@@ -67,12 +67,12 @@ sphinx = "^8.1.3"
|
|
|
67
67
|
sphinx-rtd-theme = "^3.0.2"
|
|
68
68
|
flake8 = "^7.1.1"
|
|
69
69
|
flake8-pyproject = "^1.2.3"
|
|
70
|
-
responses = "^0.25.
|
|
70
|
+
responses = "^0.25.7"
|
|
71
71
|
starlette = "^0.45.3"
|
|
72
72
|
|
|
73
73
|
[tool.poetry.group.demo.dependencies] # for hybrid mode
|
|
74
74
|
jupyterlab = "^4.2.1"
|
|
75
|
-
boto3 = "^1.
|
|
75
|
+
boto3 = "^1.37.11"
|
|
76
76
|
papermill = "^2.6.0"
|
|
77
77
|
|
|
78
78
|
[tool.poetry-dynamic-versioning] # needs: poetry self add "poetry-dynamic-versioning[plugin]"
|
|
@@ -15,6 +15,7 @@
|
|
|
15
15
|
"""AuxipClient class implementation."""
|
|
16
16
|
|
|
17
17
|
import logging
|
|
18
|
+
from typing import Any
|
|
18
19
|
|
|
19
20
|
from rs_client.stac_base import StacBase
|
|
20
21
|
from rs_common.config import EAuxipStation
|
|
@@ -35,7 +36,7 @@ class AuxipClient(StacBase):
|
|
|
35
36
|
owner_id: str | None,
|
|
36
37
|
station: EAuxipStation | str,
|
|
37
38
|
logger: logging.Logger | None = None,
|
|
38
|
-
**kwargs,
|
|
39
|
+
**kwargs: dict[str, Any],
|
|
39
40
|
):
|
|
40
41
|
"""
|
|
41
42
|
Initializes an AuxipClient instance.
|
|
@@ -46,7 +47,14 @@ class AuxipClient(StacBase):
|
|
|
46
47
|
owner_id (str | None): ID of the catalog owner.
|
|
47
48
|
station (EAuxipStation | str): The AUXIP station identifier.
|
|
48
49
|
logger (logging.Logger | None, optional): Logger instance (default: None).
|
|
49
|
-
**kwargs:
|
|
50
|
+
**kwargs: Arbitrary keyword arguments that may include:
|
|
51
|
+
- `headers` (Optional[Dict[str, str]])
|
|
52
|
+
- `parameters` (Optional[Dict[str, Any]])
|
|
53
|
+
- `ignore_conformance` (Optional[bool])
|
|
54
|
+
- `modifier` (Callable[[Collection | Item | ItemCollection | dict[Any, Any]], None] | None)
|
|
55
|
+
- `request_modifier` (Optional[Callable[[Request], Union[Request, None]]])
|
|
56
|
+
- `stac_io` (Optional[StacApiIO])
|
|
57
|
+
- `timeout` (Optional[Timeout])
|
|
50
58
|
|
|
51
59
|
Raises:
|
|
52
60
|
RuntimeError: If the provided station is not a valid AUXIP station.
|
|
@@ -15,6 +15,7 @@
|
|
|
15
15
|
"""CadipClient class implementation."""
|
|
16
16
|
|
|
17
17
|
import logging
|
|
18
|
+
from typing import Any
|
|
18
19
|
|
|
19
20
|
from rs_client.stac_base import StacBase
|
|
20
21
|
from rs_common.config import ECadipStation
|
|
@@ -36,9 +37,29 @@ class CadipClient(StacBase):
|
|
|
36
37
|
owner_id: str | None,
|
|
37
38
|
station: str,
|
|
38
39
|
logger: logging.Logger | None = None,
|
|
39
|
-
**kwargs,
|
|
40
|
+
**kwargs: dict[str, Any],
|
|
40
41
|
):
|
|
41
|
-
"""
|
|
42
|
+
"""
|
|
43
|
+
Initializes a CadipClient instance.
|
|
44
|
+
|
|
45
|
+
Args:
|
|
46
|
+
rs_server_href (str | None): The URL of the RS-Server. Pass None for local mode.
|
|
47
|
+
rs_server_api_key (str | None): API key for authentication.
|
|
48
|
+
owner_id (str | None): ID of the catalog owner.
|
|
49
|
+
station (EAuxipStation | str): The CADIP station identifier.
|
|
50
|
+
logger (logging.Logger | None, optional): Logger instance (default: None).
|
|
51
|
+
**kwargs: Arbitrary keyword arguments that may include:
|
|
52
|
+
- `headers` (Optional[Dict[str, str]])
|
|
53
|
+
- `parameters` (Optional[Dict[str, Any]])
|
|
54
|
+
- `ignore_conformance` (Optional[bool])
|
|
55
|
+
- `modifier` (Callable[[Collection | Item | ItemCollection | dict[Any, Any]], None] | None)
|
|
56
|
+
- `request_modifier` (Optional[Callable[[Request], Union[Request, None]]])
|
|
57
|
+
- `stac_io` (Optional[StacApiIO])
|
|
58
|
+
- `timeout` (Optional[Timeout])
|
|
59
|
+
|
|
60
|
+
Raises:
|
|
61
|
+
RuntimeError: If the provided station is not a valid CADIP station.
|
|
62
|
+
"""
|
|
42
63
|
super().__init__(
|
|
43
64
|
rs_server_href,
|
|
44
65
|
rs_server_api_key,
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
from __future__ import annotations
|
|
18
18
|
|
|
19
19
|
import logging
|
|
20
|
-
from
|
|
20
|
+
from collections.abc import Iterator
|
|
21
21
|
|
|
22
22
|
import pystac
|
|
23
23
|
from pystac import Collection, Item, Link, RelType
|
|
@@ -70,7 +70,7 @@ class CatalogClient(StacBase): # type: ignore # pylint: disable=too-many-ancest
|
|
|
70
70
|
|
|
71
71
|
return get_href_service(self.rs_server_href, "RSPY_HOST_CATALOG")
|
|
72
72
|
|
|
73
|
-
def full_collection_id(self, owner_id: str | None, collection_id: str, concat_char: str | None = None):
|
|
73
|
+
def full_collection_id(self, owner_id: str | None, collection_id: str, concat_char: str | None = None) -> str:
|
|
74
74
|
"""
|
|
75
75
|
Generates a full collection identifier by concatenating the owner ID and collection ID.
|
|
76
76
|
|
|
@@ -90,11 +90,11 @@ class CatalogClient(StacBase): # type: ignore # pylint: disable=too-many-ancest
|
|
|
90
90
|
|
|
91
91
|
Raises:
|
|
92
92
|
- **AttributeError**: If `self.owner_id` is not set and `owner_id` is `None`,
|
|
93
|
-
|
|
93
|
+
causing an attempt to concatenate a `NoneType` with a string.
|
|
94
94
|
|
|
95
95
|
Notes:
|
|
96
96
|
- This function is useful in scenarios where collections are stored with unique
|
|
97
|
-
|
|
97
|
+
identifiers that require owner prefixes for proper scoping.
|
|
98
98
|
"""
|
|
99
99
|
|
|
100
100
|
if not concat_char:
|
|
@@ -111,16 +111,16 @@ class CatalogClient(StacBase): # type: ignore # pylint: disable=too-many-ancest
|
|
|
111
111
|
self,
|
|
112
112
|
collection_id: str,
|
|
113
113
|
owner_id: str | None = None,
|
|
114
|
-
) ->
|
|
114
|
+
) -> Collection | CollectionClient:
|
|
115
115
|
"""Get the requested collection"""
|
|
116
116
|
return super().get_collection(self.full_collection_id(owner_id, collection_id, ":"))
|
|
117
117
|
|
|
118
118
|
def get_items(
|
|
119
119
|
self,
|
|
120
120
|
collection_id: str,
|
|
121
|
-
items_ids:
|
|
121
|
+
items_ids: str | None = None,
|
|
122
122
|
owner_id: str | None = None,
|
|
123
|
-
) -> Iterator[
|
|
123
|
+
) -> Iterator[Item]:
|
|
124
124
|
"""Get all items from a specific collection."""
|
|
125
125
|
return super().get_items(self.full_collection_id(owner_id, collection_id, ":"), items_ids)
|
|
126
126
|
|
|
@@ -131,7 +131,7 @@ class CatalogClient(StacBase): # type: ignore # pylint: disable=too-many-ancest
|
|
|
131
131
|
def search( # type: ignore # pylint: disable=too-many-arguments, arguments-differ
|
|
132
132
|
self,
|
|
133
133
|
**kwargs,
|
|
134
|
-
) ->
|
|
134
|
+
) -> ItemSearch | None:
|
|
135
135
|
"""Search items inside a specific collection."""
|
|
136
136
|
|
|
137
137
|
kwargs["collections"] = [
|
|
@@ -33,6 +33,10 @@ APIKEY_HEADER = "x-api-key"
|
|
|
33
33
|
# Timeout in seconds
|
|
34
34
|
TIMEOUT = 30
|
|
35
35
|
|
|
36
|
+
# API Key Manager URL used to get an API Key information.
|
|
37
|
+
# Works only in cluster mode. This endpoint is not exposed outside the cluster.
|
|
38
|
+
RSPY_UAC_CHECK_URL = os.getenv("RSPY_UAC_CHECK_URL", "")
|
|
39
|
+
|
|
36
40
|
|
|
37
41
|
class RsClient: # pylint: disable=too-many-instance-attributes
|
|
38
42
|
"""
|
|
@@ -49,14 +53,15 @@ class RsClient: # pylint: disable=too-many-instance-attributes
|
|
|
49
53
|
rs_server_href (str | None): RS-Server URL. Pass None for local mode.
|
|
50
54
|
rs_server_api_key (str | None): API key for RS-Server authentication.
|
|
51
55
|
rs_server_oauth2_cookie (str | None): OAuth2 session cookie read from
|
|
52
|
-
|
|
53
|
-
owner_id (str): The owner of the STAC catalog collections (no special characters allowed)
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
56
|
+
the `RSPY_OAUTH2_COOKIE` environment variable.
|
|
57
|
+
owner_id (str): The owner of the STAC catalog collections (no special characters allowed).
|
|
58
|
+
If not set, we try to read it from the RSPY_HOST_USER environment variable. If still not set:
|
|
59
|
+
- In local mode, it takes the system username.
|
|
60
|
+
- In cluster mode, it is deduced from the API key or OAuth2 login = your keycloak username.
|
|
61
|
+
- In hybrid mode, we raise an Exception.
|
|
62
|
+
If owner_id is different than your keycloak username, then make sure that your keycloak account has
|
|
63
|
+
the rights to read/write on this catalog owner.
|
|
64
|
+
owner_id is also used in the RS-Client logging.
|
|
60
65
|
logger (logging.Logger): Logger instance for logging messages.
|
|
61
66
|
local_mode (bool): Indicates whether the client is running in local mode.
|
|
62
67
|
apikey_headers (dict): API key headers for HTTP requests.
|
|
@@ -86,7 +91,7 @@ class RsClient: # pylint: disable=too-many-instance-attributes
|
|
|
86
91
|
self.rs_server_href: str | None = rs_server_href
|
|
87
92
|
self.rs_server_api_key: str | None = rs_server_api_key
|
|
88
93
|
self.rs_server_oauth2_cookie: str | None = os.getenv("RSPY_OAUTH2_COOKIE")
|
|
89
|
-
self.owner_id: str | None = owner_id or ""
|
|
94
|
+
self.owner_id: str | None = owner_id or os.getenv("RSPY_HOST_USER")
|
|
90
95
|
self.logger: logging.Logger = logger or Logging.default(__name__)
|
|
91
96
|
|
|
92
97
|
# Remove trailing / character(s) from the URL
|
|
@@ -97,6 +102,10 @@ class RsClient: # pylint: disable=too-many-instance-attributes
|
|
|
97
102
|
# Env vars are used instead to determine the different services URL.
|
|
98
103
|
self.local_mode = not bool(self.rs_server_href)
|
|
99
104
|
|
|
105
|
+
# We are in hybrid mode if not on local mode and the API Key Manager check URL is undefined.
|
|
106
|
+
# NOTE: maybe later we could define this hybrid mode in a different way.
|
|
107
|
+
self.hybrid_mode = (not self.local_mode) and (not RSPY_UAC_CHECK_URL)
|
|
108
|
+
|
|
100
109
|
if (not self.local_mode) and (not self.rs_server_api_key) and (not self.rs_server_oauth2_cookie):
|
|
101
110
|
raise RuntimeError("API key or OAuth2 cookie is mandatory for RS-Server authentication")
|
|
102
111
|
|
|
@@ -116,7 +125,14 @@ class RsClient: # pylint: disable=too-many-instance-attributes
|
|
|
116
125
|
if self.local_mode:
|
|
117
126
|
self.owner_id = getpass.getuser()
|
|
118
127
|
|
|
119
|
-
# In hybrid
|
|
128
|
+
# In hybrid mode, the API Key Manager check URL is not accessible and there is no OAuth2
|
|
129
|
+
# so the owner id must be set explicitly by the user.
|
|
130
|
+
elif self.hybrid_mode:
|
|
131
|
+
raise RuntimeError(
|
|
132
|
+
"In hybrid mode, the owner_id must be set explicitly by parameter or environment variable",
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
# In cluster mode, we retrieve the OAuth2 or API key login
|
|
120
136
|
else:
|
|
121
137
|
self.owner_id = self.apikey_user_login if self.rs_server_api_key else self.oauth2_user_login
|
|
122
138
|
|
|
@@ -184,17 +200,9 @@ class RsClient: # pylint: disable=too-many-instance-attributes
|
|
|
184
200
|
if self.local_mode:
|
|
185
201
|
return AuthInfo(user_login="", iam_roles=[], apikey_config={})
|
|
186
202
|
|
|
187
|
-
# self.logger.warning(
|
|
188
|
-
# f"TODO: use {self.rs_server_href}/apikeymanager/auth/check_key instead, see: "
|
|
189
|
-
# "https://pforge-exchange2.astrium.eads.net/jira/browse/RSPY-257",
|
|
190
|
-
# )
|
|
191
|
-
# Does not work in hybrid mode for now because this URL is not exposed.
|
|
192
|
-
check_url = os.environ["RSPY_UAC_CHECK_URL"]
|
|
193
|
-
|
|
194
203
|
# Request the API key manager, pass user-defined api key in http header
|
|
195
|
-
# check_url = f"{self.rs_server_href}/apikeymanager/auth/check_key"
|
|
196
204
|
self.logger.debug("Call the API key manager")
|
|
197
|
-
response = self.http_session.get(
|
|
205
|
+
response = self.http_session.get(RSPY_UAC_CHECK_URL, **self.apikey_headers, timeout=TIMEOUT)
|
|
198
206
|
if not response.ok:
|
|
199
207
|
raise RuntimeError(
|
|
200
208
|
f"API key manager status code {response.status_code}: {utils.read_response_error(response)}",
|
|
@@ -15,8 +15,9 @@
|
|
|
15
15
|
"""StacBase class implementation."""
|
|
16
16
|
|
|
17
17
|
import logging
|
|
18
|
+
from collections.abc import Callable, Iterator
|
|
18
19
|
from functools import lru_cache, wraps
|
|
19
|
-
from typing import Any,
|
|
20
|
+
from typing import Any, cast
|
|
20
21
|
|
|
21
22
|
import requests
|
|
22
23
|
from pystac import Collection, Item, ItemCollection
|
|
@@ -76,13 +77,13 @@ class StacBase(RsClient):
|
|
|
76
77
|
owner_id: str | None = None,
|
|
77
78
|
logger: logging.Logger | None = None,
|
|
78
79
|
stac_href: str | None = None, # Flag to enable pystac_client for specific subclasses
|
|
79
|
-
headers:
|
|
80
|
-
parameters:
|
|
81
|
-
ignore_conformance:
|
|
80
|
+
headers: dict[str, str] | None = None,
|
|
81
|
+
parameters: dict[str, Any] | None = None,
|
|
82
|
+
ignore_conformance: bool | None = None,
|
|
82
83
|
modifier: Callable[[Collection | Item | ItemCollection | dict[Any, Any]], None] | None = None,
|
|
83
|
-
request_modifier:
|
|
84
|
-
stac_io:
|
|
85
|
-
timeout:
|
|
84
|
+
request_modifier: Callable[[Request], Request | None] | None = None,
|
|
85
|
+
stac_io: StacApiIO | None = None,
|
|
86
|
+
timeout: Timeout | None = TIMEOUT,
|
|
86
87
|
):
|
|
87
88
|
"""
|
|
88
89
|
Initialize the StacBase instance.
|
|
@@ -167,9 +168,9 @@ class StacBase(RsClient):
|
|
|
167
168
|
# Get all the available collections
|
|
168
169
|
return self.ps_client.get_collections()
|
|
169
170
|
|
|
170
|
-
@lru_cache
|
|
171
|
+
@lru_cache
|
|
171
172
|
@handle_api_error
|
|
172
|
-
def get_collection(self, collection_id: str) ->
|
|
173
|
+
def get_collection(self, collection_id: str) -> Collection | CollectionClient:
|
|
173
174
|
"""
|
|
174
175
|
Retrieve a specific STAC collection by ID.
|
|
175
176
|
|
|
@@ -185,7 +186,7 @@ class StacBase(RsClient):
|
|
|
185
186
|
return self.ps_client.get_collection(collection_id)
|
|
186
187
|
|
|
187
188
|
@handle_api_error
|
|
188
|
-
def get_items(self, collection_id: str, items_ids:
|
|
189
|
+
def get_items(self, collection_id: str, items_ids: str | None = None) -> Iterator["Item"]:
|
|
189
190
|
"""
|
|
190
191
|
Retrieve all items or specific items from a collection.
|
|
191
192
|
|
|
@@ -233,7 +234,7 @@ class StacBase(RsClient):
|
|
|
233
234
|
return item
|
|
234
235
|
|
|
235
236
|
@handle_api_error
|
|
236
|
-
def get_collection_queryables(self, collection_id) ->
|
|
237
|
+
def get_collection_queryables(self, collection_id) -> dict[str, Any]:
|
|
237
238
|
"""
|
|
238
239
|
Retrieve queryable fields for a specific collection.
|
|
239
240
|
|
|
@@ -249,7 +250,7 @@ class StacBase(RsClient):
|
|
|
249
250
|
|
|
250
251
|
return self.ps_client.get_merged_queryables([collection_id])
|
|
251
252
|
|
|
252
|
-
def get_queryables(self) ->
|
|
253
|
+
def get_queryables(self) -> dict[str, Any]:
|
|
253
254
|
"""
|
|
254
255
|
Retrieve queryable fields for all collections in the STAC API. These are the available terms for
|
|
255
256
|
usage when writing filter expressions in /search endpoint for all the collections
|
|
@@ -278,7 +279,7 @@ class StacBase(RsClient):
|
|
|
278
279
|
raise RuntimeError(f"Could not get queryables from {href_queryables}")
|
|
279
280
|
try:
|
|
280
281
|
json_data = response.json()
|
|
281
|
-
return cast(
|
|
282
|
+
return cast(dict[str, Any], json_data) # Explicitly cast to Dict[str, Any]
|
|
282
283
|
except ValueError as e:
|
|
283
284
|
raise RuntimeError(f"Invalid JSON response from {href_queryables}") from e
|
|
284
285
|
|
|
@@ -17,7 +17,8 @@
|
|
|
17
17
|
import json
|
|
18
18
|
import os
|
|
19
19
|
import os.path as osp
|
|
20
|
-
from typing import Any
|
|
20
|
+
from typing import Any
|
|
21
|
+
from urllib.parse import urlparse
|
|
21
22
|
|
|
22
23
|
# openapi_core libraries used for endpoints validation
|
|
23
24
|
import requests
|
|
@@ -37,7 +38,6 @@ from rs_common.utils import get_href_service
|
|
|
37
38
|
# with a valid format according to ogc standard. In the meantime, we don't perform validation to make all staging
|
|
38
39
|
# notebooks pass. If this env variable if not specified (for example that is the case when we launch the pytest),
|
|
39
40
|
# we perform this validation by default
|
|
40
|
-
DO_VALIDATE: bool = os.getenv("RSPY_APPLY_STAGING_ENDPOINTS_VALIDATION", "1") == "1"
|
|
41
41
|
|
|
42
42
|
PATH_TO_YAML_OPENAPI = osp.join(
|
|
43
43
|
osp.realpath(osp.dirname(__file__)),
|
|
@@ -46,15 +46,17 @@ PATH_TO_YAML_OPENAPI = osp.join(
|
|
|
46
46
|
"yaml",
|
|
47
47
|
"staging_openapi_schema.yaml",
|
|
48
48
|
)
|
|
49
|
-
PATH_TO_STAGING_BODY = osp.join(
|
|
50
|
-
osp.realpath(osp.dirname(__file__)),
|
|
51
|
-
"../config",
|
|
52
|
-
"staging_templates",
|
|
53
|
-
"staging_body.json",
|
|
54
|
-
)
|
|
55
49
|
RESOURCE = "staging"
|
|
56
50
|
|
|
57
51
|
|
|
52
|
+
def is_url(path):
|
|
53
|
+
"""
|
|
54
|
+
Function to check if a string is an url
|
|
55
|
+
"""
|
|
56
|
+
parsed = urlparse(path)
|
|
57
|
+
return bool(parsed.scheme and parsed.netloc)
|
|
58
|
+
|
|
59
|
+
|
|
58
60
|
class StagingValidationException(Exception):
|
|
59
61
|
"""
|
|
60
62
|
Exception raised when an error occurs during the OGC validation
|
|
@@ -93,8 +95,6 @@ class StagingClient(RsClient):
|
|
|
93
95
|
ResponseUnmarshalResult.data: data validated by the openapi_core
|
|
94
96
|
unmarshal_response method
|
|
95
97
|
"""
|
|
96
|
-
if not DO_VALIDATE:
|
|
97
|
-
return request.body
|
|
98
98
|
|
|
99
99
|
if not os.path.isfile(PATH_TO_YAML_OPENAPI):
|
|
100
100
|
raise FileNotFoundError(f"The following file path was not found: {PATH_TO_YAML_OPENAPI}")
|
|
@@ -129,11 +129,6 @@ class StagingClient(RsClient):
|
|
|
129
129
|
ResponseUnmarshalResult.data: data validated by the openapi_core
|
|
130
130
|
unmarshal_response method
|
|
131
131
|
"""
|
|
132
|
-
if not DO_VALIDATE:
|
|
133
|
-
if not response.content:
|
|
134
|
-
raise StagingValidationException("Response content is empty !")
|
|
135
|
-
return json.loads(response.content)
|
|
136
|
-
|
|
137
132
|
if not os.path.isfile(PATH_TO_YAML_OPENAPI):
|
|
138
133
|
raise FileNotFoundError(f"The following file path was not found: {PATH_TO_YAML_OPENAPI}")
|
|
139
134
|
|
|
@@ -146,8 +141,9 @@ class StagingClient(RsClient):
|
|
|
146
141
|
result = openapi.unmarshal_response(openapi_request, openapi_response) # type: ignore
|
|
147
142
|
if result.errors:
|
|
148
143
|
raise StagingValidationException( # type: ignore
|
|
149
|
-
f"Error validating the response of the enpoint "
|
|
150
|
-
f"
|
|
144
|
+
f"Error validating the response of the enpoint {openapi_request.path} - "
|
|
145
|
+
f"Server response content: {response.json()} - "
|
|
146
|
+
f"Validation error of the server response: {str(result.errors[0])}", # type: ignore
|
|
151
147
|
)
|
|
152
148
|
if not result.data:
|
|
153
149
|
raise StagingValidationException(
|
|
@@ -155,13 +151,13 @@ class StagingClient(RsClient):
|
|
|
155
151
|
f"{openapi_request.path}: 'data' field of ResponseUnmarshalResult"
|
|
156
152
|
f"object is empty",
|
|
157
153
|
)
|
|
158
|
-
return
|
|
154
|
+
return json.loads(response.content)
|
|
159
155
|
|
|
160
156
|
############################
|
|
161
157
|
# Call RS-Server endpoints #
|
|
162
158
|
############################
|
|
163
159
|
|
|
164
|
-
def get_processes(self) ->
|
|
160
|
+
def get_processes(self) -> dict:
|
|
165
161
|
"""_summary_
|
|
166
162
|
|
|
167
163
|
Returns:
|
|
@@ -174,7 +170,7 @@ class StagingClient(RsClient):
|
|
|
174
170
|
)
|
|
175
171
|
return self.validate_and_unmarshal_response(response)
|
|
176
172
|
|
|
177
|
-
def get_process(self, process_id: str) ->
|
|
173
|
+
def get_process(self, process_id: str) -> dict:
|
|
178
174
|
"""
|
|
179
175
|
Wrapper to get a specific process
|
|
180
176
|
Args:
|
|
@@ -191,7 +187,7 @@ class StagingClient(RsClient):
|
|
|
191
187
|
self,
|
|
192
188
|
stac_input: dict[Any, Any] | str,
|
|
193
189
|
out_coll_name: str,
|
|
194
|
-
) ->
|
|
190
|
+
) -> dict:
|
|
195
191
|
"""Method to start the staging process from rs-client - Call the endpoint /processes/staging/execution
|
|
196
192
|
|
|
197
193
|
Args:
|
|
@@ -200,55 +196,77 @@ class StagingClient(RsClient):
|
|
|
200
196
|
the output of a search for Cadip or Auxip sessions)
|
|
201
197
|
- A json string corresponding to a Feature or a FeatureCollection
|
|
202
198
|
- A string corresponding to a path to a json file containing a Feature or a FeatureCollection
|
|
203
|
-
|
|
204
|
-
|
|
199
|
+
- A single link that returns a STAC itemCollection: this link should be an url to search a
|
|
200
|
+
itemCollection, for example:
|
|
201
|
+
http://localhost:8002/cadip/search?ids=S1A_20231120061537234567&collections=cadip_sentinel1
|
|
202
|
+
out_coll_name (str): name of the output collection
|
|
205
203
|
Return:
|
|
206
204
|
job_id (int, str): Returns the status code of the staging request + the identifier
|
|
207
205
|
(or None if staging endpoint fails) of the running job
|
|
208
206
|
"""
|
|
209
207
|
stac_input_dict = {}
|
|
210
|
-
|
|
208
|
+
is_an_url = False
|
|
209
|
+
|
|
210
|
+
# ----- Case 1: we only load a link (that refers to a STAC itemCollection) in the staging request body
|
|
211
211
|
if isinstance(stac_input, str):
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
stac_input_dict = json.loads(stac_input)
|
|
221
|
-
else:
|
|
222
|
-
stac_input_dict = stac_input
|
|
223
|
-
|
|
224
|
-
if "type" not in stac_input_dict:
|
|
225
|
-
raise KeyError("Key 'type' is missing from the staging input data")
|
|
226
|
-
|
|
227
|
-
# Validate input data using Pydantic
|
|
228
|
-
if stac_input_dict["type"] == "Feature":
|
|
229
|
-
stac_item = Item(**stac_input_dict)
|
|
230
|
-
stac_item_collection = ItemCollection(
|
|
231
|
-
**{
|
|
232
|
-
"type": "FeatureCollection",
|
|
233
|
-
"context": {"limit": 1000, "returned": 2},
|
|
234
|
-
"features": [stac_item],
|
|
235
|
-
}, # type: ignore
|
|
236
|
-
)
|
|
212
|
+
try:
|
|
213
|
+
is_an_url = is_url(stac_input)
|
|
214
|
+
except ValueError:
|
|
215
|
+
is_an_url = False
|
|
216
|
+
if is_an_url:
|
|
217
|
+
staging_body = {"inputs": {"collection": out_coll_name, "items": {"href": stac_input}}}
|
|
218
|
+
|
|
219
|
+
# ----- Case 2: we directly load a STAC ItemCollection in the staging request body
|
|
237
220
|
else:
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
221
|
+
# If stac_input is a file, load this file to a dictionary
|
|
222
|
+
if isinstance(stac_input, str):
|
|
223
|
+
# If the input is a valid path to a json_file, load this file
|
|
224
|
+
if os.path.exists(os.path.dirname(stac_input)) and stac_input.endswith(".json"):
|
|
225
|
+
# Read the yaml or json file
|
|
226
|
+
with open(stac_input, encoding="utf-8") as opened:
|
|
227
|
+
stac_file_to_dict = json.loads(opened.read())
|
|
228
|
+
stac_input_dict = stac_file_to_dict
|
|
229
|
+
# If the input string is not a path, try to convert the content of the string to a json dictionary
|
|
230
|
+
else:
|
|
231
|
+
try:
|
|
232
|
+
stac_input_dict = json.loads(stac_input)
|
|
233
|
+
except json.JSONDecodeError as e:
|
|
234
|
+
raise StagingValidationException(
|
|
235
|
+
f"""Invalid input format: {stac_input} - Input data must be either:
|
|
236
|
+
- A Python dictionary corresponding to a Feature or a FeatureCollection
|
|
237
|
+
(that can be for example the output of a search for Cadip or Auxip sessions)
|
|
238
|
+
- A json string corresponding to a Feature or a FeatureCollection
|
|
239
|
+
- A string corresponding to a path to a json file containing a Feature or a
|
|
240
|
+
FeatureCollection
|
|
241
|
+
- A single link that returns a STAC ItemCollection: this link should be an
|
|
242
|
+
url to search an ItemCollection
|
|
243
|
+
""".strip(),
|
|
244
|
+
) from e
|
|
245
|
+
else:
|
|
246
|
+
stac_input_dict = stac_input
|
|
247
|
+
|
|
248
|
+
if "type" not in stac_input_dict:
|
|
249
|
+
raise KeyError("Key 'type' is missing from the staging input data")
|
|
250
|
+
|
|
251
|
+
# Validate input data using Pydantic
|
|
252
|
+
if stac_input_dict["type"] == "Feature":
|
|
253
|
+
stac_item = Item(**stac_input_dict)
|
|
254
|
+
stac_item_collection = ItemCollection(
|
|
255
|
+
**{
|
|
256
|
+
"type": "FeatureCollection",
|
|
257
|
+
"context": {"limit": 1000, "returned": 2},
|
|
258
|
+
"features": [stac_item],
|
|
259
|
+
}, # type: ignore
|
|
260
|
+
)
|
|
261
|
+
else:
|
|
262
|
+
stac_item_collection = ItemCollection(**stac_input_dict)
|
|
243
263
|
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
# TODO: replace the staging_body "outputs" field with the following when rs-server-staging is updated
|
|
251
|
-
# TODO: "outputs": {"featureCollectionOutput": {"transmissionMode": "value"}},
|
|
264
|
+
staging_body = {
|
|
265
|
+
"inputs": {
|
|
266
|
+
"collection": out_coll_name,
|
|
267
|
+
"items": {"value": stac_item_collection.model_dump(mode="json")},
|
|
268
|
+
},
|
|
269
|
+
}
|
|
252
270
|
|
|
253
271
|
# Check that the request containing the staging body is valid
|
|
254
272
|
request = requests.Request( # pylint: disable=W0612 # noqa: F841
|
|
@@ -256,8 +274,9 @@ class StagingClient(RsClient):
|
|
|
256
274
|
url=f"{self.href_service}/processes/{RESOURCE}/execution", # L'URL de l'endpoint
|
|
257
275
|
json=staging_body, # Corps de la requête en JSON
|
|
258
276
|
).prepare()
|
|
259
|
-
|
|
260
|
-
#
|
|
277
|
+
|
|
278
|
+
# Validate the body of the request that will be sent to the staging
|
|
279
|
+
self.validate_and_unmarshal_request(request)
|
|
261
280
|
|
|
262
281
|
response = self.http_session.post(
|
|
263
282
|
url=f"{self.href_service}/processes/staging/execution",
|
|
@@ -267,7 +286,7 @@ class StagingClient(RsClient):
|
|
|
267
286
|
)
|
|
268
287
|
return self.validate_and_unmarshal_response(response)
|
|
269
288
|
|
|
270
|
-
def get_jobs(self) ->
|
|
289
|
+
def get_jobs(self) -> dict:
|
|
271
290
|
"""Method to get running jobs"""
|
|
272
291
|
response = self.http_session.get(
|
|
273
292
|
url=f"{self.href_service}/jobs",
|
|
@@ -276,7 +295,7 @@ class StagingClient(RsClient):
|
|
|
276
295
|
)
|
|
277
296
|
return self.validate_and_unmarshal_response(response)
|
|
278
297
|
|
|
279
|
-
def get_job_info(self, job_id: str) ->
|
|
298
|
+
def get_job_info(self, job_id: str) -> dict: # pylint: disable=too-many-locals
|
|
280
299
|
"""Method to get a specific job response"""
|
|
281
300
|
response = self.http_session.get(
|
|
282
301
|
url=f"{self.href_service}/jobs/{job_id}",
|
|
@@ -285,7 +304,7 @@ class StagingClient(RsClient):
|
|
|
285
304
|
)
|
|
286
305
|
return self.validate_and_unmarshal_response(response)
|
|
287
306
|
|
|
288
|
-
def delete_job(self, job_id: str) ->
|
|
307
|
+
def delete_job(self, job_id: str) -> dict: # pylint: disable=too-many-locals
|
|
289
308
|
"""Method to get a specific job response"""
|
|
290
309
|
response = self.http_session.delete(
|
|
291
310
|
url=f"{self.href_service}/jobs/{job_id}",
|
|
@@ -294,7 +313,7 @@ class StagingClient(RsClient):
|
|
|
294
313
|
)
|
|
295
314
|
return self.validate_and_unmarshal_response(response)
|
|
296
315
|
|
|
297
|
-
def get_job_results(self, job_id: str) ->
|
|
316
|
+
def get_job_results(self, job_id: str) -> dict:
|
|
298
317
|
"""Wrapper to get the result of a specfific job
|
|
299
318
|
|
|
300
319
|
Args:
|
|
@@ -29,10 +29,6 @@ from opentelemetry.sdk.resources import Resource
|
|
|
29
29
|
from opentelemetry.sdk.trace import TracerProvider
|
|
30
30
|
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
|
31
31
|
|
|
32
|
-
from rs_common.logging import Logging
|
|
33
|
-
|
|
34
|
-
logger = Logging.default(__name__)
|
|
35
|
-
|
|
36
32
|
FROM_PYTEST = False
|
|
37
33
|
|
|
38
34
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|