rs-client-libraries 0.2a11__tar.gz → 0.2a13__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: rs-client-libraries
3
- Version: 0.2a11
3
+ Version: 0.2a13
4
4
  Summary: Copernicus Sentinel rs-client library and processing workflows.
5
5
  Home-page: https://home.rs-python.eu/
6
6
  License: Apache-2.0
@@ -19,17 +19,17 @@ Classifier: Programming Language :: Python :: 3.12
19
19
  Classifier: Programming Language :: Python :: 3.13
20
20
  Classifier: Topic :: Scientific/Engineering :: Image Processing
21
21
  Classifier: Topic :: Software Development :: Libraries :: Python Modules
22
- Requires-Dist: cachetools (>=5.5.1,<6.0.0)
23
- Requires-Dist: openapi-core (>=0.19.4,<0.20.0)
24
- Requires-Dist: opentelemetry-distro (>=0.51b0,<0.52)
25
- Requires-Dist: opentelemetry-exporter-otlp (>=1.30.0,<2.0.0)
26
- Requires-Dist: opentelemetry-instrumentation-aws-lambda (>=0.51b0,<0.52)
27
- Requires-Dist: opentelemetry-test-utils (>=0.51b0,<0.52)
28
- Requires-Dist: pystac (>=1.12.1,<2.0.0)
29
- Requires-Dist: pystac-client (>=0.8.5,<0.9.0)
22
+ Requires-Dist: cachetools (>=5.5.2,<6.0.0)
23
+ Requires-Dist: openapi-core (>=0.19.5,<0.20.0)
24
+ Requires-Dist: opentelemetry-distro (>=0.53b1,<0.54)
25
+ Requires-Dist: opentelemetry-exporter-otlp (>=1.32.1,<2.0.0)
26
+ Requires-Dist: opentelemetry-instrumentation-aws-lambda (>=0.53b1,<0.54)
27
+ Requires-Dist: opentelemetry-test-utils (>=0.53b1,<0.54)
28
+ Requires-Dist: pystac (>=1.13.0,<2.0.0)
29
+ Requires-Dist: pystac-client (>=0.8.6,<0.9.0)
30
30
  Requires-Dist: python-dateutil (>=2.9.0.post0,<3.0.0)
31
31
  Requires-Dist: python-logging-loki (>=0.3.1,<0.4.0)
32
- Requires-Dist: stac-pydantic (>=3.1.4,<4.0.0)
32
+ Requires-Dist: stac-pydantic (>=3.1.5,<4.0.0)
33
33
  Project-URL: Documentation, https://home.rs-python.eu/rs-documentation/rs-client-libraries/docs/doc/
34
34
  Project-URL: Repository, https://github.com/RS-PYTHON/rs-client-libraries
35
35
  Description-Content-Type: text/plain
@@ -4,7 +4,7 @@ version = "0.0.0" # see: https://github.com/mtkennerly/poetry-dynamic-versioning
4
4
 
5
5
  [tool.poetry]
6
6
  name = "rs-client-libraries"
7
- version = "0.2a11"
7
+ version = "0.2a13"
8
8
  description = "Copernicus Sentinel rs-client library and processing workflows."
9
9
  license = "Apache-2.0"
10
10
  authors = ["ESA under contract with Airbus and CS <ops_coprs@airbus.com>"]
@@ -36,23 +36,23 @@ include = [
36
36
 
37
37
  [tool.poetry.dependencies]
38
38
  python = "^3.11"
39
- opentelemetry-distro = "^0.51b0" # then run 'poetry run opentelemetry-bootstrap -a install'
40
- opentelemetry-instrumentation-aws-lambda = "^0.51b0"
41
- opentelemetry-test-utils = "^0.51b0"
42
- opentelemetry-exporter-otlp = "^1.30.0"
39
+ opentelemetry-distro = "^0.53b1" # then run 'poetry run opentelemetry-bootstrap -a install'
40
+ opentelemetry-instrumentation-aws-lambda = "^0.53b1"
41
+ opentelemetry-test-utils = "^0.53b1"
42
+ opentelemetry-exporter-otlp = "^1.32.1"
43
43
  python-logging-loki = "^0.3.1" # to avoid otl error: No module named 'logging_loki'
44
- cachetools = "^5.5.1"
45
- pystac = "^1.12.1"
46
- pystac-client = "^0.8.5"
44
+ cachetools = "^5.5.2"
45
+ pystac = "^1.13.0"
46
+ pystac-client = "^0.8.6"
47
47
  python-dateutil = "^2.9.0.post0"
48
- stac-pydantic = "^3.1.4"
49
- openapi-core = "^0.19.4"
48
+ stac-pydantic = "^3.1.5"
49
+ openapi-core = "^0.19.5"
50
50
 
51
51
  [tool.poetry.group.dev]
52
52
  optional = true
53
53
 
54
54
  [tool.poetry.group.dev.dependencies]
55
- pytest = "^8.3.4"
55
+ pytest = "^8.3.5"
56
56
  pytest-cov = "^6.0.0"
57
57
  pytest-mock = "^3.14.0"
58
58
  pytest-error-for-skips = "^2.0.2"
@@ -67,12 +67,12 @@ sphinx = "^8.1.3"
67
67
  sphinx-rtd-theme = "^3.0.2"
68
68
  flake8 = "^7.1.1"
69
69
  flake8-pyproject = "^1.2.3"
70
- responses = "^0.25.6"
70
+ responses = "^0.25.7"
71
71
  starlette = "^0.45.3"
72
72
 
73
73
  [tool.poetry.group.demo.dependencies] # for hybrid mode
74
74
  jupyterlab = "^4.2.1"
75
- boto3 = "^1.36.13"
75
+ boto3 = "^1.37.11"
76
76
  papermill = "^2.6.0"
77
77
 
78
78
  [tool.poetry-dynamic-versioning] # needs: poetry self add "poetry-dynamic-versioning[plugin]"
@@ -15,4 +15,4 @@
15
15
  """rsclient main module."""
16
16
 
17
17
  # Set automatically by running `poetry dynamic-versioning`
18
- __version__ = "0.2a11"
18
+ __version__ = "0.2a13"
@@ -15,9 +15,9 @@
15
15
  """AuxipClient class implementation."""
16
16
 
17
17
  import logging
18
+ from typing import Any
18
19
 
19
20
  from rs_client.stac_base import StacBase
20
- from rs_common.config import EAuxipStation
21
21
  from rs_common.utils import get_href_service
22
22
 
23
23
 
@@ -33,9 +33,8 @@ class AuxipClient(StacBase):
33
33
  rs_server_href: str | None,
34
34
  rs_server_api_key: str | None,
35
35
  owner_id: str | None,
36
- station: EAuxipStation | str,
37
36
  logger: logging.Logger | None = None,
38
- **kwargs,
37
+ **kwargs: dict[str, Any],
39
38
  ):
40
39
  """
41
40
  Initializes an AuxipClient instance.
@@ -44,12 +43,15 @@ class AuxipClient(StacBase):
44
43
  rs_server_href (str | None): The URL of the RS-Server. Pass None for local mode.
45
44
  rs_server_api_key (str | None): API key for authentication.
46
45
  owner_id (str | None): ID of the catalog owner.
47
- station (EAuxipStation | str): The AUXIP station identifier.
48
46
  logger (logging.Logger | None, optional): Logger instance (default: None).
49
- **kwargs: Additional keyword arguments.
50
-
51
- Raises:
52
- RuntimeError: If the provided station is not a valid AUXIP station.
47
+ **kwargs: Arbitrary keyword arguments that may include:
48
+ - `headers` (Optional[Dict[str, str]])
49
+ - `parameters` (Optional[Dict[str, Any]])
50
+ - `ignore_conformance` (Optional[bool])
51
+ - `modifier` (Callable[[Collection | Item | ItemCollection | dict[Any, Any]], None] | None)
52
+ - `request_modifier` (Optional[Callable[[Request], Union[Request, None]]])
53
+ - `stac_io` (Optional[StacApiIO])
54
+ - `timeout` (Optional[Timeout])
53
55
  """
54
56
  super().__init__(
55
57
  rs_server_href,
@@ -59,10 +61,6 @@ class AuxipClient(StacBase):
59
61
  get_href_service(rs_server_href, "RSPY_HOST_ADGS") + "/auxip/",
60
62
  **kwargs,
61
63
  )
62
- try:
63
- self.station: EAuxipStation = EAuxipStation[station] if isinstance(station, str) else station
64
- except KeyError as e:
65
- self.log_and_raise(f"There is no such AUXIP station: {station}", e)
66
64
 
67
65
  @property
68
66
  def href_service(self) -> str:
@@ -72,8 +70,3 @@ class AuxipClient(StacBase):
72
70
  Otherwise it should just be the RS-Server URL.
73
71
  """
74
72
  return get_href_service(self.rs_server_href, "RSPY_HOST_ADGS") + "/auxip"
75
-
76
- @property
77
- def station_name(self) -> str:
78
- """Return the station name."""
79
- return self.station.value
@@ -15,9 +15,9 @@
15
15
  """CadipClient class implementation."""
16
16
 
17
17
  import logging
18
+ from typing import Any
18
19
 
19
20
  from rs_client.stac_base import StacBase
20
- from rs_common.config import ECadipStation
21
21
  from rs_common.utils import get_href_service
22
22
 
23
23
 
@@ -26,7 +26,6 @@ class CadipClient(StacBase):
26
26
  CadipClient class implementation.
27
27
 
28
28
  Attributes: see :py:class:`RsClient`
29
- station (ECadipStation): Cadip station
30
29
  """
31
30
 
32
31
  def __init__( # pylint: disable=too-many-arguments, too-many-positional-arguments
@@ -34,11 +33,26 @@ class CadipClient(StacBase):
34
33
  rs_server_href: str | None,
35
34
  rs_server_api_key: str | None,
36
35
  owner_id: str | None,
37
- station: str,
38
36
  logger: logging.Logger | None = None,
39
- **kwargs,
37
+ **kwargs: dict[str, Any],
40
38
  ):
41
- """CadipClient class constructor."""
39
+ """
40
+ Initializes a CadipClient instance.
41
+
42
+ Args:
43
+ rs_server_href (str | None): The URL of the RS-Server. Pass None for local mode.
44
+ rs_server_api_key (str | None): API key for authentication.
45
+ owner_id (str | None): ID of the catalog owner.
46
+ logger (logging.Logger | None, optional): Logger instance (default: None).
47
+ **kwargs: Arbitrary keyword arguments that may include:
48
+ - `headers` (Optional[Dict[str, str]])
49
+ - `parameters` (Optional[Dict[str, Any]])
50
+ - `ignore_conformance` (Optional[bool])
51
+ - `modifier` (Callable[[Collection | Item | ItemCollection | dict[Any, Any]], None] | None)
52
+ - `request_modifier` (Optional[Callable[[Request], Union[Request, None]]])
53
+ - `stac_io` (Optional[StacApiIO])
54
+ - `timeout` (Optional[Timeout])
55
+ """
42
56
  super().__init__(
43
57
  rs_server_href,
44
58
  rs_server_api_key,
@@ -47,10 +61,6 @@ class CadipClient(StacBase):
47
61
  get_href_service(rs_server_href, "RSPY_HOST_CADIP") + "/cadip/",
48
62
  **kwargs,
49
63
  )
50
- try:
51
- self.station: ECadipStation = ECadipStation[station] if isinstance(station, str) else station
52
- except KeyError as e:
53
- self.log_and_raise(f"There is no such CADIP station: {station}", e)
54
64
 
55
65
  @property
56
66
  def href_service(self) -> str:
@@ -60,8 +70,3 @@ class CadipClient(StacBase):
60
70
  Otherwise it should just be the RS-Server URL.
61
71
  """
62
72
  return get_href_service(self.rs_server_href, "RSPY_HOST_CADIP") + "/cadip"
63
-
64
- @property
65
- def station_name(self) -> str:
66
- """Return the station name."""
67
- return self.station.value
@@ -17,7 +17,7 @@
17
17
  from __future__ import annotations
18
18
 
19
19
  import logging
20
- from typing import Iterator, Union
20
+ from collections.abc import Iterator
21
21
 
22
22
  import pystac
23
23
  from pystac import Collection, Item, Link, RelType
@@ -70,7 +70,7 @@ class CatalogClient(StacBase): # type: ignore # pylint: disable=too-many-ancest
70
70
 
71
71
  return get_href_service(self.rs_server_href, "RSPY_HOST_CATALOG")
72
72
 
73
- def full_collection_id(self, owner_id: str | None, collection_id: str, concat_char: str | None = None):
73
+ def full_collection_id(self, owner_id: str | None, collection_id: str, concat_char: str | None = None) -> str:
74
74
  """
75
75
  Generates a full collection identifier by concatenating the owner ID and collection ID.
76
76
 
@@ -90,11 +90,11 @@ class CatalogClient(StacBase): # type: ignore # pylint: disable=too-many-ancest
90
90
 
91
91
  Raises:
92
92
  - **AttributeError**: If `self.owner_id` is not set and `owner_id` is `None`,
93
- causing an attempt to concatenate a `NoneType` with a string.
93
+ causing an attempt to concatenate a `NoneType` with a string.
94
94
 
95
95
  Notes:
96
96
  - This function is useful in scenarios where collections are stored with unique
97
- identifiers that require owner prefixes for proper scoping.
97
+ identifiers that require owner prefixes for proper scoping.
98
98
  """
99
99
 
100
100
  if not concat_char:
@@ -111,16 +111,16 @@ class CatalogClient(StacBase): # type: ignore # pylint: disable=too-many-ancest
111
111
  self,
112
112
  collection_id: str,
113
113
  owner_id: str | None = None,
114
- ) -> Union[Collection, CollectionClient]:
114
+ ) -> Collection | CollectionClient:
115
115
  """Get the requested collection"""
116
116
  return super().get_collection(self.full_collection_id(owner_id, collection_id, ":"))
117
117
 
118
118
  def get_items(
119
119
  self,
120
120
  collection_id: str,
121
- items_ids: Union[str, None] = None,
121
+ items_ids: str | None = None,
122
122
  owner_id: str | None = None,
123
- ) -> Iterator["Item"]:
123
+ ) -> Iterator[Item]:
124
124
  """Get all items from a specific collection."""
125
125
  return super().get_items(self.full_collection_id(owner_id, collection_id, ":"), items_ids)
126
126
 
@@ -131,7 +131,7 @@ class CatalogClient(StacBase): # type: ignore # pylint: disable=too-many-ancest
131
131
  def search( # type: ignore # pylint: disable=too-many-arguments, arguments-differ
132
132
  self,
133
133
  **kwargs,
134
- ) -> Union[ItemSearch, None]:
134
+ ) -> ItemSearch | None:
135
135
  """Search items inside a specific collection."""
136
136
 
137
137
  kwargs["collections"] = [
@@ -24,7 +24,6 @@ import requests
24
24
  from cachetools import TTLCache, cached
25
25
 
26
26
  from rs_common import utils
27
- from rs_common.config import EAuxipStation, ECadipStation
28
27
  from rs_common.logging import Logging
29
28
  from rs_common.utils import AuthInfo
30
29
 
@@ -33,6 +32,10 @@ APIKEY_HEADER = "x-api-key"
33
32
  # Timeout in seconds
34
33
  TIMEOUT = 30
35
34
 
35
+ # API Key Manager URL used to get an API Key information.
36
+ # Works only in cluster mode. This endpoint is not exposed outside the cluster.
37
+ RSPY_UAC_CHECK_URL = os.getenv("RSPY_UAC_CHECK_URL", "")
38
+
36
39
 
37
40
  class RsClient: # pylint: disable=too-many-instance-attributes
38
41
  """
@@ -49,14 +52,15 @@ class RsClient: # pylint: disable=too-many-instance-attributes
49
52
  rs_server_href (str | None): RS-Server URL. Pass None for local mode.
50
53
  rs_server_api_key (str | None): API key for RS-Server authentication.
51
54
  rs_server_oauth2_cookie (str | None): OAuth2 session cookie read from
52
- the `RSPY_OAUTH2_COOKIE` environment variable.
53
- owner_id (str): The owner of the STAC catalog collections (no special characters allowed):
54
- - In local mode, this is the system username.
55
- - In remote mode, this is derived from the API key or OAuth2 login. .
56
- By default, this is the user login from the keycloak account, associated to the API key.
57
- Or, in local mode, this is the local system username.
58
- Else, your API Key must give you the rights to read/write on this catalog owner.
59
- This owner ID is also used in the RS-Client logging.
55
+ the `RSPY_OAUTH2_COOKIE` environment variable.
56
+ owner_id (str): The owner of the STAC catalog collections (no special characters allowed).
57
+ If not set, we try to read it from the RSPY_HOST_USER environment variable. If still not set:
58
+ - In local mode, it takes the system username.
59
+ - In cluster mode, it is deduced from the API key or OAuth2 login = your keycloak username.
60
+ - In hybrid mode, we raise an Exception.
61
+ If owner_id is different than your keycloak username, then make sure that your keycloak account has
62
+ the rights to read/write on this catalog owner.
63
+ owner_id is also used in the RS-Client logging.
60
64
  logger (logging.Logger): Logger instance for logging messages.
61
65
  local_mode (bool): Indicates whether the client is running in local mode.
62
66
  apikey_headers (dict): API key headers for HTTP requests.
@@ -86,7 +90,7 @@ class RsClient: # pylint: disable=too-many-instance-attributes
86
90
  self.rs_server_href: str | None = rs_server_href
87
91
  self.rs_server_api_key: str | None = rs_server_api_key
88
92
  self.rs_server_oauth2_cookie: str | None = os.getenv("RSPY_OAUTH2_COOKIE")
89
- self.owner_id: str | None = owner_id or ""
93
+ self.owner_id: str | None = owner_id or os.getenv("RSPY_HOST_USER")
90
94
  self.logger: logging.Logger = logger or Logging.default(__name__)
91
95
 
92
96
  # Remove trailing / character(s) from the URL
@@ -97,6 +101,10 @@ class RsClient: # pylint: disable=too-many-instance-attributes
97
101
  # Env vars are used instead to determine the different services URL.
98
102
  self.local_mode = not bool(self.rs_server_href)
99
103
 
104
+ # We are in hybrid mode if not on local mode and the API Key Manager check URL is undefined.
105
+ # NOTE: maybe later we could define this hybrid mode in a different way.
106
+ self.hybrid_mode = (not self.local_mode) and (not RSPY_UAC_CHECK_URL)
107
+
100
108
  if (not self.local_mode) and (not self.rs_server_api_key) and (not self.rs_server_oauth2_cookie):
101
109
  raise RuntimeError("API key or OAuth2 cookie is mandatory for RS-Server authentication")
102
110
 
@@ -116,7 +124,14 @@ class RsClient: # pylint: disable=too-many-instance-attributes
116
124
  if self.local_mode:
117
125
  self.owner_id = getpass.getuser()
118
126
 
119
- # In hybrid/cluster mode, we retrieve the OAuth2 or API key login
127
+ # In hybrid mode, the API Key Manager check URL is not accessible and there is no OAuth2
128
+ # so the owner id must be set explicitly by the user.
129
+ elif self.hybrid_mode:
130
+ raise RuntimeError(
131
+ "In hybrid mode, the owner_id must be set explicitly by parameter or environment variable",
132
+ )
133
+
134
+ # In cluster mode, we retrieve the OAuth2 or API key login
120
135
  else:
121
136
  self.owner_id = self.apikey_user_login if self.rs_server_api_key else self.oauth2_user_login
122
137
 
@@ -184,17 +199,9 @@ class RsClient: # pylint: disable=too-many-instance-attributes
184
199
  if self.local_mode:
185
200
  return AuthInfo(user_login="", iam_roles=[], apikey_config={})
186
201
 
187
- # self.logger.warning(
188
- # f"TODO: use {self.rs_server_href}/apikeymanager/auth/check_key instead, see: "
189
- # "https://pforge-exchange2.astrium.eads.net/jira/browse/RSPY-257",
190
- # )
191
- # Does not work in hybrid mode for now because this URL is not exposed.
192
- check_url = os.environ["RSPY_UAC_CHECK_URL"]
193
-
194
202
  # Request the API key manager, pass user-defined api key in http header
195
- # check_url = f"{self.rs_server_href}/apikeymanager/auth/check_key"
196
203
  self.logger.debug("Call the API key manager")
197
- response = self.http_session.get(check_url, **self.apikey_headers, timeout=TIMEOUT)
204
+ response = self.http_session.get(RSPY_UAC_CHECK_URL, **self.apikey_headers, timeout=TIMEOUT)
198
205
  if not response.ok:
199
206
  raise RuntimeError(
200
207
  f"API key manager status code {response.status_code}: {utils.read_response_error(response)}",
@@ -248,30 +255,25 @@ class RsClient: # pylint: disable=too-many-instance-attributes
248
255
  # Get child class instances #
249
256
  #############################
250
257
 
251
- def get_auxip_client(self, station: EAuxipStation, **kwargs) -> "AuxipClient": # type: ignore # noqa: F821
258
+ def get_auxip_client(self, **kwargs) -> "AuxipClient": # type: ignore # noqa: F821
252
259
  """
253
260
  Return an instance of the child class AuxipClient, with the same attributes as this "self" instance.
254
- Args:
255
- station (EAuxipStation): Auxip station
256
261
  """
257
262
  from rs_client.auxip_client import ( # pylint: disable=import-outside-toplevel,cyclic-import
258
263
  AuxipClient,
259
264
  )
260
265
 
261
- return AuxipClient(self.rs_server_href, self.rs_server_api_key, self.owner_id, station, self.logger, **kwargs)
266
+ return AuxipClient(self.rs_server_href, self.rs_server_api_key, self.owner_id, self.logger, **kwargs)
262
267
 
263
- def get_cadip_client(self, station: ECadipStation, **kwargs) -> "CadipClient": # type: ignore # noqa: F821
268
+ def get_cadip_client(self, **kwargs) -> "CadipClient": # type: ignore # noqa: F821
264
269
  """
265
270
  Return an instance of the child class CadipClient, with the same attributes as this "self" instance.
266
-
267
- Args:
268
- station (ECadipStation): Cadip station
269
271
  """
270
272
  from rs_client.cadip_client import ( # pylint: disable=import-outside-toplevel,cyclic-import
271
273
  CadipClient,
272
274
  )
273
275
 
274
- return CadipClient(self.rs_server_href, self.rs_server_api_key, self.owner_id, station, self.logger, **kwargs)
276
+ return CadipClient(self.rs_server_href, self.rs_server_api_key, self.owner_id, self.logger, **kwargs)
275
277
 
276
278
  def get_catalog_client(self, **kwargs) -> "CatalogClient": # type: ignore # noqa: F821
277
279
  """
@@ -15,8 +15,9 @@
15
15
  """StacBase class implementation."""
16
16
 
17
17
  import logging
18
+ from collections.abc import Callable, Iterator
18
19
  from functools import lru_cache, wraps
19
- from typing import Any, Callable, Dict, Iterator, Optional, Union, cast
20
+ from typing import Any, cast
20
21
 
21
22
  import requests
22
23
  from pystac import Collection, Item, ItemCollection
@@ -76,13 +77,13 @@ class StacBase(RsClient):
76
77
  owner_id: str | None = None,
77
78
  logger: logging.Logger | None = None,
78
79
  stac_href: str | None = None, # Flag to enable pystac_client for specific subclasses
79
- headers: Optional[Dict[str, str]] = None,
80
- parameters: Optional[Dict[str, Any]] = None,
81
- ignore_conformance: Optional[bool] = None,
80
+ headers: dict[str, str] | None = None,
81
+ parameters: dict[str, Any] | None = None,
82
+ ignore_conformance: bool | None = None,
82
83
  modifier: Callable[[Collection | Item | ItemCollection | dict[Any, Any]], None] | None = None,
83
- request_modifier: Optional[Callable[[Request], Union[Request, None]]] = None,
84
- stac_io: Optional[StacApiIO] = None,
85
- timeout: Optional[Timeout] = TIMEOUT,
84
+ request_modifier: Callable[[Request], Request | None] | None = None,
85
+ stac_io: StacApiIO | None = None,
86
+ timeout: Timeout | None = TIMEOUT,
86
87
  ):
87
88
  """
88
89
  Initialize the StacBase instance.
@@ -167,9 +168,9 @@ class StacBase(RsClient):
167
168
  # Get all the available collections
168
169
  return self.ps_client.get_collections()
169
170
 
170
- @lru_cache()
171
+ @lru_cache
171
172
  @handle_api_error
172
- def get_collection(self, collection_id: str) -> Union[Collection, CollectionClient]:
173
+ def get_collection(self, collection_id: str) -> Collection | CollectionClient:
173
174
  """
174
175
  Retrieve a specific STAC collection by ID.
175
176
 
@@ -185,7 +186,7 @@ class StacBase(RsClient):
185
186
  return self.ps_client.get_collection(collection_id)
186
187
 
187
188
  @handle_api_error
188
- def get_items(self, collection_id: str, items_ids: Union[str, None] = None) -> Iterator["Item"]:
189
+ def get_items(self, collection_id: str, items_ids: str | None = None) -> Iterator["Item"]:
189
190
  """
190
191
  Retrieve all items or specific items from a collection.
191
192
 
@@ -233,7 +234,7 @@ class StacBase(RsClient):
233
234
  return item
234
235
 
235
236
  @handle_api_error
236
- def get_collection_queryables(self, collection_id) -> Dict[str, Any]:
237
+ def get_collection_queryables(self, collection_id) -> dict[str, Any]:
237
238
  """
238
239
  Retrieve queryable fields for a specific collection.
239
240
 
@@ -249,7 +250,7 @@ class StacBase(RsClient):
249
250
 
250
251
  return self.ps_client.get_merged_queryables([collection_id])
251
252
 
252
- def get_queryables(self) -> Dict[str, Any]:
253
+ def get_queryables(self) -> dict[str, Any]:
253
254
  """
254
255
  Retrieve queryable fields for all collections in the STAC API. These are the available terms for
255
256
  usage when writing filter expressions in /search endpoint for all the collections
@@ -278,7 +279,7 @@ class StacBase(RsClient):
278
279
  raise RuntimeError(f"Could not get queryables from {href_queryables}")
279
280
  try:
280
281
  json_data = response.json()
281
- return cast(Dict[str, Any], json_data) # Explicitly cast to Dict[str, Any]
282
+ return cast(dict[str, Any], json_data) # Explicitly cast to Dict[str, Any]
282
283
  except ValueError as e:
283
284
  raise RuntimeError(f"Invalid JSON response from {href_queryables}") from e
284
285
 
@@ -17,7 +17,8 @@
17
17
  import json
18
18
  import os
19
19
  import os.path as osp
20
- from typing import Any, Dict
20
+ from typing import Any
21
+ from urllib.parse import urlparse
21
22
 
22
23
  # openapi_core libraries used for endpoints validation
23
24
  import requests
@@ -37,7 +38,6 @@ from rs_common.utils import get_href_service
37
38
  # with a valid format according to ogc standard. In the meantime, we don't perform validation to make all staging
38
39
  # notebooks pass. If this env variable if not specified (for example that is the case when we launch the pytest),
39
40
  # we perform this validation by default
40
- DO_VALIDATE: bool = os.getenv("RSPY_APPLY_STAGING_ENDPOINTS_VALIDATION", "1") == "1"
41
41
 
42
42
  PATH_TO_YAML_OPENAPI = osp.join(
43
43
  osp.realpath(osp.dirname(__file__)),
@@ -46,15 +46,17 @@ PATH_TO_YAML_OPENAPI = osp.join(
46
46
  "yaml",
47
47
  "staging_openapi_schema.yaml",
48
48
  )
49
- PATH_TO_STAGING_BODY = osp.join(
50
- osp.realpath(osp.dirname(__file__)),
51
- "../config",
52
- "staging_templates",
53
- "staging_body.json",
54
- )
55
49
  RESOURCE = "staging"
56
50
 
57
51
 
52
+ def is_url(path):
53
+ """
54
+ Function to check if a string is an url
55
+ """
56
+ parsed = urlparse(path)
57
+ return bool(parsed.scheme and parsed.netloc)
58
+
59
+
58
60
  class StagingValidationException(Exception):
59
61
  """
60
62
  Exception raised when an error occurs during the OGC validation
@@ -93,8 +95,6 @@ class StagingClient(RsClient):
93
95
  ResponseUnmarshalResult.data: data validated by the openapi_core
94
96
  unmarshal_response method
95
97
  """
96
- if not DO_VALIDATE:
97
- return request.body
98
98
 
99
99
  if not os.path.isfile(PATH_TO_YAML_OPENAPI):
100
100
  raise FileNotFoundError(f"The following file path was not found: {PATH_TO_YAML_OPENAPI}")
@@ -129,11 +129,6 @@ class StagingClient(RsClient):
129
129
  ResponseUnmarshalResult.data: data validated by the openapi_core
130
130
  unmarshal_response method
131
131
  """
132
- if not DO_VALIDATE:
133
- if not response.content:
134
- raise StagingValidationException("Response content is empty !")
135
- return json.loads(response.content)
136
-
137
132
  if not os.path.isfile(PATH_TO_YAML_OPENAPI):
138
133
  raise FileNotFoundError(f"The following file path was not found: {PATH_TO_YAML_OPENAPI}")
139
134
 
@@ -146,8 +141,9 @@ class StagingClient(RsClient):
146
141
  result = openapi.unmarshal_response(openapi_request, openapi_response) # type: ignore
147
142
  if result.errors:
148
143
  raise StagingValidationException( # type: ignore
149
- f"Error validating the response of the enpoint "
150
- f"{openapi_request.path}: {str(result.errors[0])}", # type: ignore
144
+ f"Error validating the response of the enpoint {openapi_request.path} - "
145
+ f"Server response content: {response.json()} - "
146
+ f"Validation error of the server response: {str(result.errors[0])}", # type: ignore
151
147
  )
152
148
  if not result.data:
153
149
  raise StagingValidationException(
@@ -155,13 +151,13 @@ class StagingClient(RsClient):
155
151
  f"{openapi_request.path}: 'data' field of ResponseUnmarshalResult"
156
152
  f"object is empty",
157
153
  )
158
- return result.data
154
+ return json.loads(response.content)
159
155
 
160
156
  ############################
161
157
  # Call RS-Server endpoints #
162
158
  ############################
163
159
 
164
- def get_processes(self) -> Dict:
160
+ def get_processes(self) -> dict:
165
161
  """_summary_
166
162
 
167
163
  Returns:
@@ -174,7 +170,7 @@ class StagingClient(RsClient):
174
170
  )
175
171
  return self.validate_and_unmarshal_response(response)
176
172
 
177
- def get_process(self, process_id: str) -> Dict:
173
+ def get_process(self, process_id: str) -> dict:
178
174
  """
179
175
  Wrapper to get a specific process
180
176
  Args:
@@ -191,7 +187,7 @@ class StagingClient(RsClient):
191
187
  self,
192
188
  stac_input: dict[Any, Any] | str,
193
189
  out_coll_name: str,
194
- ) -> Dict:
190
+ ) -> dict:
195
191
  """Method to start the staging process from rs-client - Call the endpoint /processes/staging/execution
196
192
 
197
193
  Args:
@@ -200,55 +196,77 @@ class StagingClient(RsClient):
200
196
  the output of a search for Cadip or Auxip sessions)
201
197
  - A json string corresponding to a Feature or a FeatureCollection
202
198
  - A string corresponding to a path to a json file containing a Feature or a FeatureCollection
203
- out_coll_name (): _description_
204
-
199
+ - A single link that returns a STAC itemCollection: this link should be an url to search a
200
+ itemCollection, for example:
201
+ http://localhost:8002/cadip/search?ids=S1A_20231120061537234567&collections=cadip_sentinel1
202
+ out_coll_name (str): name of the output collection
205
203
  Return:
206
204
  job_id (int, str): Returns the status code of the staging request + the identifier
207
205
  (or None if staging endpoint fails) of the running job
208
206
  """
209
207
  stac_input_dict = {}
210
- # If stac_input is a file, load this file to a dictionary
208
+ is_an_url = False
209
+
210
+ # ----- Case 1: we only load a link (that refers to a STAC itemCollection) in the staging request body
211
211
  if isinstance(stac_input, str):
212
- # If the input is a valid path to a json_file, load this file
213
- if os.path.exists(os.path.dirname(stac_input)) and stac_input.endswith(".json"):
214
- # Read the yaml or json file
215
- with open(stac_input, encoding="utf-8") as opened:
216
- stac_file_to_dict = json.loads(opened.read())
217
- stac_input_dict = stac_file_to_dict
218
- # If the input string is not a path, try to convert the content of the string to a json dictionary
219
- else:
220
- stac_input_dict = json.loads(stac_input)
221
- else:
222
- stac_input_dict = stac_input
223
-
224
- if "type" not in stac_input_dict:
225
- raise KeyError("Key 'type' is missing from the staging input data")
226
-
227
- # Validate input data using Pydantic
228
- if stac_input_dict["type"] == "Feature":
229
- stac_item = Item(**stac_input_dict)
230
- stac_item_collection = ItemCollection(
231
- **{
232
- "type": "FeatureCollection",
233
- "context": {"limit": 1000, "returned": 2},
234
- "features": [stac_item],
235
- }, # type: ignore
236
- )
212
+ try:
213
+ is_an_url = is_url(stac_input)
214
+ except ValueError:
215
+ is_an_url = False
216
+ if is_an_url:
217
+ staging_body = {"inputs": {"collection": out_coll_name, "items": {"href": stac_input}}}
218
+
219
+ # ----- Case 2: we directly load a STAC ItemCollection in the staging request body
237
220
  else:
238
- stac_item_collection = ItemCollection(**stac_input_dict)
239
-
240
- # Load staging body base structure and fill it with the staging content
241
- if not os.path.isfile(PATH_TO_STAGING_BODY):
242
- raise FileNotFoundError(f"The following file path was not found: {PATH_TO_STAGING_BODY}")
221
+ # If stac_input is a file, load this file to a dictionary
222
+ if isinstance(stac_input, str):
223
+ # If the input is a valid path to a json_file, load this file
224
+ if os.path.exists(os.path.dirname(stac_input)) and stac_input.endswith(".json"):
225
+ # Read the yaml or json file
226
+ with open(stac_input, encoding="utf-8") as opened:
227
+ stac_file_to_dict = json.loads(opened.read())
228
+ stac_input_dict = stac_file_to_dict
229
+ # If the input string is not a path, try to convert the content of the string to a json dictionary
230
+ else:
231
+ try:
232
+ stac_input_dict = json.loads(stac_input)
233
+ except json.JSONDecodeError as e:
234
+ raise StagingValidationException(
235
+ f"""Invalid input format: {stac_input} - Input data must be either:
236
+ - A Python dictionary corresponding to a Feature or a FeatureCollection
237
+ (that can be for example the output of a search for Cadip or Auxip sessions)
238
+ - A json string corresponding to a Feature or a FeatureCollection
239
+ - A string corresponding to a path to a json file containing a Feature or a
240
+ FeatureCollection
241
+ - A single link that returns a STAC ItemCollection: this link should be an
242
+ url to search an ItemCollection
243
+ """.strip(),
244
+ ) from e
245
+ else:
246
+ stac_input_dict = stac_input
247
+
248
+ if "type" not in stac_input_dict:
249
+ raise KeyError("Key 'type' is missing from the staging input data")
250
+
251
+ # Validate input data using Pydantic
252
+ if stac_input_dict["type"] == "Feature":
253
+ stac_item = Item(**stac_input_dict)
254
+ stac_item_collection = ItemCollection(
255
+ **{
256
+ "type": "FeatureCollection",
257
+ "context": {"limit": 1000, "returned": 2},
258
+ "features": [stac_item],
259
+ }, # type: ignore
260
+ )
261
+ else:
262
+ stac_item_collection = ItemCollection(**stac_input_dict)
243
263
 
244
- # TODO: this staging_body content will be validated with the self.validate_and_unmarshal_request(request)
245
- # once rs-server-staging will be updated
246
- with open(PATH_TO_STAGING_BODY, encoding="utf-8") as f:
247
- staging_body = json.load(f)
248
- staging_body["inputs"]["collection"]["id"] = out_coll_name
249
- staging_body["inputs"]["items"].update(stac_item_collection.model_dump(mode="json"))
250
- # TODO: replace the staging_body "outputs" field with the following when rs-server-staging is updated
251
- # TODO: "outputs": {"featureCollectionOutput": {"transmissionMode": "value"}},
264
+ staging_body = {
265
+ "inputs": {
266
+ "collection": out_coll_name,
267
+ "items": {"value": stac_item_collection.model_dump(mode="json")},
268
+ },
269
+ }
252
270
 
253
271
  # Check that the request containing the staging body is valid
254
272
  request = requests.Request( # pylint: disable=W0612 # noqa: F841
@@ -256,8 +274,9 @@ class StagingClient(RsClient):
256
274
  url=f"{self.href_service}/processes/{RESOURCE}/execution", # L'URL de l'endpoint
257
275
  json=staging_body, # Corps de la requête en JSON
258
276
  ).prepare()
259
- # TODO: uncomment when rs-server-staging is updated
260
- # TODO: self.validate_and_unmarshal_request(request)
277
+
278
+ # Validate the body of the request that will be sent to the staging
279
+ self.validate_and_unmarshal_request(request)
261
280
 
262
281
  response = self.http_session.post(
263
282
  url=f"{self.href_service}/processes/staging/execution",
@@ -267,7 +286,7 @@ class StagingClient(RsClient):
267
286
  )
268
287
  return self.validate_and_unmarshal_response(response)
269
288
 
270
- def get_jobs(self) -> Dict:
289
+ def get_jobs(self) -> dict:
271
290
  """Method to get running jobs"""
272
291
  response = self.http_session.get(
273
292
  url=f"{self.href_service}/jobs",
@@ -276,7 +295,7 @@ class StagingClient(RsClient):
276
295
  )
277
296
  return self.validate_and_unmarshal_response(response)
278
297
 
279
- def get_job_info(self, job_id: str) -> Dict: # pylint: disable=too-many-locals
298
+ def get_job_info(self, job_id: str) -> dict: # pylint: disable=too-many-locals
280
299
  """Method to get a specific job response"""
281
300
  response = self.http_session.get(
282
301
  url=f"{self.href_service}/jobs/{job_id}",
@@ -285,7 +304,7 @@ class StagingClient(RsClient):
285
304
  )
286
305
  return self.validate_and_unmarshal_response(response)
287
306
 
288
- def delete_job(self, job_id: str) -> Dict: # pylint: disable=too-many-locals
307
+ def delete_job(self, job_id: str) -> dict: # pylint: disable=too-many-locals
289
308
  """Method to get a specific job response"""
290
309
  response = self.http_session.delete(
291
310
  url=f"{self.href_service}/jobs/{job_id}",
@@ -294,7 +313,7 @@ class StagingClient(RsClient):
294
313
  )
295
314
  return self.validate_and_unmarshal_response(response)
296
315
 
297
- def get_job_results(self, job_id: str) -> Dict:
316
+ def get_job_results(self, job_id: str) -> dict:
298
317
  """Wrapper to get the result of a specfific job
299
318
 
300
319
  Args:
@@ -15,6 +15,6 @@
15
15
  """Common modules."""
16
16
 
17
17
  # Configure OpenTelemetry
18
- from rs_common import opentelemetry
18
+ from rs_common import init_opentelemetry
19
19
 
20
- opentelemetry.init_traces("rs.client")
20
+ init_opentelemetry.init_traces("rs.client")
@@ -15,6 +15,7 @@
15
15
  """OpenTelemetry utility"""
16
16
 
17
17
  import inspect
18
+ import json
18
19
  import os
19
20
  import pkgutil
20
21
  import sys
@@ -25,32 +26,113 @@ from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExport
25
26
  from opentelemetry.instrumentation.asyncio import AsyncioInstrumentor
26
27
  from opentelemetry.instrumentation.aws_lambda import AwsLambdaInstrumentor
27
28
  from opentelemetry.instrumentation.instrumentor import BaseInstrumentor # type: ignore
29
+ from opentelemetry.instrumentation.requests import RequestsInstrumentor
28
30
  from opentelemetry.sdk.resources import Resource
29
31
  from opentelemetry.sdk.trace import TracerProvider
30
32
  from opentelemetry.sdk.trace.export import BatchSpanProcessor
33
+ from opentelemetry.trace.span import NonRecordingSpan, SpanContext, TraceFlags
34
+ from opentelemetry.util._decorator import _agnosticcontextmanager
31
35
 
32
36
  from rs_common.logging import Logging
37
+ from rs_common.utils import env_bool
33
38
 
34
- logger = Logging.default(__name__)
39
+ default_logger = Logging.default(__name__)
35
40
 
36
41
  FROM_PYTEST = False
37
42
 
38
43
 
39
- def init_traces(service_name: str):
44
+ # Show details of http headers and body/content in tempo/grafana ?
45
+ # Don't store results in global variables because the env var values can change
46
+ # after this module was loaded.
47
+ def trace_headers():
48
+ """Trace request headers ?"""
49
+ return env_bool("OTEL_PYTHON_REQUESTS_TRACE_HEADERS", default=False)
50
+
51
+
52
+ def trace_body():
53
+ """Trace request bodies and response contents ?"""
54
+ return env_bool("OTEL_PYTHON_REQUESTS_TRACE_BODY", default=False)
55
+
56
+
57
+ def parse_data(data) -> str:
58
+ """Convert data to a string representation"""
59
+
60
+ if not data:
61
+ return ""
62
+
63
+ # Try to decode bytes
64
+ if isinstance(data, bytes):
65
+ data = data.decode("utf-8")
66
+
67
+ # Try to convert to a dict
68
+ try:
69
+ data = dict(data)
70
+ except Exception: # pylint: disable=broad-exception-caught # nosec
71
+ pass
72
+
73
+ # Or to parse to a dict
74
+ try:
75
+ data = json.loads(data)
76
+ except Exception: # pylint: disable=broad-exception-caught # nosec
77
+ pass
78
+
79
+ # If we have a dict, try to format it as json
80
+ if isinstance(data, dict):
81
+ data = json.dumps(data, indent=2)
82
+
83
+ return data or ""
84
+
85
+
86
+ def request_hook(span, request):
87
+ """
88
+ HTTP requests intrumentation
89
+ """
90
+ if not span:
91
+ return
92
+
93
+ # Copy the http.url attribute into _url so it appears at the
94
+ # top in the grafana UI, it's more readable
95
+ span.set_attribute("_url", span.attributes.get("http.url"))
96
+
97
+ if trace_headers():
98
+ span.set_attribute("http.request.headers", parse_data(request.headers))
99
+
100
+ if trace_body():
101
+ span.set_attribute("http.request.body", parse_data(request.body))
102
+
103
+
104
+ def response_hook(span, request, response): # pylint: disable=W0613
105
+ """
106
+ HTTP responses intrumentation
107
+ """
108
+ if not span:
109
+ return
110
+
111
+ if trace_headers():
112
+ span.set_attribute("http.response.headers", parse_data(response.headers))
113
+
114
+ if trace_body():
115
+ span.set_attribute("http.response.content", parse_data(response.content))
116
+
117
+
118
+ def init_traces(service_name: str, logger=None):
40
119
  """
41
120
  Init instrumentation of OpenTelemetry traces.
42
121
 
43
122
  Args:
44
123
  service_name (str): service name
124
+ logger: non-default logger to user
45
125
  """
46
-
47
126
  # See: https://github.com/softwarebloat/python-tracing-demo/tree/main
48
127
 
128
+ logger = logger or default_logger
129
+
49
130
  # Don't call this line from pytest because it causes errors:
50
131
  # Transient error StatusCode.UNAVAILABLE encountered while exporting metrics to localhost:4317, retrying in ..s.
51
132
  if not FROM_PYTEST:
52
133
  tempo_endpoint = os.getenv("TEMPO_ENDPOINT")
53
134
  if not tempo_endpoint:
135
+ logger.warning("'TEMPO_ENDPOINT' variable is missing, cannot initialize OpenTelemetry")
54
136
  return
55
137
 
56
138
  # TODO: to avoid errors in local mode:
@@ -83,6 +165,7 @@ def init_traces(service_name: str):
83
165
 
84
166
  # Recursively find all package modules
85
167
  for _, module_str, _ in pkgutil.walk_packages(path=package.__path__, prefix=prefix, onerror=None):
168
+
86
169
  # Don't instrument these modules, they have errors, maybe we should see why
87
170
  if module_str in [
88
171
  "opentelemetry.instrumentation.tortoiseorm",
@@ -106,8 +189,53 @@ def init_traces(service_name: str):
106
189
  # If the "instrument" method exists, call it
107
190
  _instrument = getattr(_class, "instrument", None)
108
191
  if callable(_instrument):
192
+
109
193
  _class_instance = _class()
110
- if not _class_instance.is_instrumented_by_opentelemetry:
194
+ if _class == RequestsInstrumentor and (trace_headers() or trace_body()):
195
+ _class_instance.instrument(
196
+ tracer_provider=otel_tracer,
197
+ request_hook=request_hook,
198
+ response_hook=response_hook,
199
+ )
200
+ elif not _class_instance.is_instrumented_by_opentelemetry:
111
201
  _class_instance.instrument(tracer_provider=otel_tracer)
112
202
  # name = f"{module_str}.{_class.__name__}".removeprefix(prefix)
113
203
  # logger.debug(f"OpenTelemetry instrumentation of {name!r}")
204
+
205
+
206
+ @_agnosticcontextmanager
207
+ def start_span(
208
+ instrumenting_module_name: str,
209
+ name: str,
210
+ span_context: SpanContext | None = None,
211
+ ):
212
+ """
213
+ Context manager for creating a new main or child OpenTelemetry span and set it
214
+ as the current span in this tracer's context.
215
+
216
+ Args:
217
+ instrumenting_module_name: Caller module name, just pass __name__
218
+ name: The name of the span to be created (use a custom name)
219
+ span_context: Parent span context. Only to create a child span.
220
+ """
221
+ tracer = trace.get_tracer(instrumenting_module_name)
222
+
223
+ # Create a main span
224
+ if not span_context:
225
+ with tracer.start_as_current_span(name) as span:
226
+ yield span
227
+
228
+ # Create a child span
229
+ else:
230
+ main_span_context = SpanContext(
231
+ trace_id=span_context.trace_id,
232
+ span_id=span_context.span_id,
233
+ is_remote=True,
234
+ trace_flags=TraceFlags(TraceFlags.SAMPLED),
235
+ )
236
+ main_span = NonRecordingSpan(main_span_context)
237
+ with trace.use_span(main_span): # pylint: disable=not-context-manager
238
+ # Optionnaly, we could use the main span instead of creating
239
+ # a new one, to be discussed.
240
+ with tracer.start_as_current_span(name) as span:
241
+ yield span
@@ -15,13 +15,10 @@
15
15
  """Logging utility."""
16
16
 
17
17
  import logging
18
- import logging.handlers
19
18
  import os
20
19
  from multiprocessing import Queue
21
20
  from threading import Lock
22
21
 
23
- import logging_loki
24
-
25
22
 
26
23
  class Logging: # pylint: disable=too-few-public-methods
27
24
  """
@@ -64,6 +61,8 @@ class Logging: # pylint: disable=too-few-public-methods
64
61
  # Export logs to Loki, see: https://pypi.org/project/python-logging-loki/
65
62
  loki_endpoint = os.getenv("LOKI_ENDPOINT")
66
63
  if loki_endpoint:
64
+ import logging_loki # pylint: disable=import-outside-toplevel
65
+
67
66
  handler = logging_loki.LokiQueueHandler(
68
67
  Queue(-1),
69
68
  url=loki_endpoint,
@@ -54,3 +54,17 @@ def get_href_service(rs_server_href, env_var) -> str:
54
54
  if not rs_server_href:
55
55
  raise RuntimeError("RS-Server URL is undefined")
56
56
  return rs_server_href.rstrip("/")
57
+
58
+
59
+ def env_bool(var: str, default: bool) -> bool:
60
+ """
61
+ Return True if an environemnt variable is set to 1, true or yes (case insensitive).
62
+ Return False if set to 0, false or no (case insensitive).
63
+ Return the default value if not set or set to a different value.
64
+ """
65
+ val = os.getenv(var, str(default)).lower()
66
+ if val in ("y", "yes", "t", "true", "on", "1"):
67
+ return True
68
+ if val in ("n", "no", "f", "false", "off", "0"):
69
+ return False
70
+ return default