rs-client-libraries 0.2a11__tar.gz → 0.2a13__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {rs_client_libraries-0.2a11 → rs_client_libraries-0.2a13}/PKG-INFO +10 -10
- {rs_client_libraries-0.2a11 → rs_client_libraries-0.2a13}/pyproject.toml +13 -13
- {rs_client_libraries-0.2a11 → rs_client_libraries-0.2a13}/rs_client/__init__.py +1 -1
- {rs_client_libraries-0.2a11 → rs_client_libraries-0.2a13}/rs_client/auxip_client.py +10 -17
- {rs_client_libraries-0.2a11 → rs_client_libraries-0.2a13}/rs_client/cadip_client.py +19 -14
- {rs_client_libraries-0.2a11 → rs_client_libraries-0.2a13}/rs_client/catalog_client.py +8 -8
- {rs_client_libraries-0.2a11 → rs_client_libraries-0.2a13}/rs_client/rs_client.py +31 -29
- {rs_client_libraries-0.2a11 → rs_client_libraries-0.2a13}/rs_client/stac_base.py +14 -13
- {rs_client_libraries-0.2a11 → rs_client_libraries-0.2a13}/rs_client/staging_client.py +87 -68
- {rs_client_libraries-0.2a11 → rs_client_libraries-0.2a13}/rs_common/__init__.py +2 -2
- rs_client_libraries-0.2a11/rs_common/opentelemetry.py → rs_client_libraries-0.2a13/rs_common/init_opentelemetry.py +132 -4
- {rs_client_libraries-0.2a11 → rs_client_libraries-0.2a13}/rs_common/logging.py +2 -3
- {rs_client_libraries-0.2a11 → rs_client_libraries-0.2a13}/rs_common/utils.py +14 -0
- {rs_client_libraries-0.2a11 → rs_client_libraries-0.2a13}/LICENSE +0 -0
- {rs_client_libraries-0.2a11 → rs_client_libraries-0.2a13}/README.adoc +0 -0
- {rs_client_libraries-0.2a11 → rs_client_libraries-0.2a13}/rs_common/config.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: rs-client-libraries
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2a13
|
|
4
4
|
Summary: Copernicus Sentinel rs-client library and processing workflows.
|
|
5
5
|
Home-page: https://home.rs-python.eu/
|
|
6
6
|
License: Apache-2.0
|
|
@@ -19,17 +19,17 @@ Classifier: Programming Language :: Python :: 3.12
|
|
|
19
19
|
Classifier: Programming Language :: Python :: 3.13
|
|
20
20
|
Classifier: Topic :: Scientific/Engineering :: Image Processing
|
|
21
21
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
22
|
-
Requires-Dist: cachetools (>=5.5.
|
|
23
|
-
Requires-Dist: openapi-core (>=0.19.
|
|
24
|
-
Requires-Dist: opentelemetry-distro (>=0.
|
|
25
|
-
Requires-Dist: opentelemetry-exporter-otlp (>=1.
|
|
26
|
-
Requires-Dist: opentelemetry-instrumentation-aws-lambda (>=0.
|
|
27
|
-
Requires-Dist: opentelemetry-test-utils (>=0.
|
|
28
|
-
Requires-Dist: pystac (>=1.
|
|
29
|
-
Requires-Dist: pystac-client (>=0.8.
|
|
22
|
+
Requires-Dist: cachetools (>=5.5.2,<6.0.0)
|
|
23
|
+
Requires-Dist: openapi-core (>=0.19.5,<0.20.0)
|
|
24
|
+
Requires-Dist: opentelemetry-distro (>=0.53b1,<0.54)
|
|
25
|
+
Requires-Dist: opentelemetry-exporter-otlp (>=1.32.1,<2.0.0)
|
|
26
|
+
Requires-Dist: opentelemetry-instrumentation-aws-lambda (>=0.53b1,<0.54)
|
|
27
|
+
Requires-Dist: opentelemetry-test-utils (>=0.53b1,<0.54)
|
|
28
|
+
Requires-Dist: pystac (>=1.13.0,<2.0.0)
|
|
29
|
+
Requires-Dist: pystac-client (>=0.8.6,<0.9.0)
|
|
30
30
|
Requires-Dist: python-dateutil (>=2.9.0.post0,<3.0.0)
|
|
31
31
|
Requires-Dist: python-logging-loki (>=0.3.1,<0.4.0)
|
|
32
|
-
Requires-Dist: stac-pydantic (>=3.1.
|
|
32
|
+
Requires-Dist: stac-pydantic (>=3.1.5,<4.0.0)
|
|
33
33
|
Project-URL: Documentation, https://home.rs-python.eu/rs-documentation/rs-client-libraries/docs/doc/
|
|
34
34
|
Project-URL: Repository, https://github.com/RS-PYTHON/rs-client-libraries
|
|
35
35
|
Description-Content-Type: text/plain
|
|
@@ -4,7 +4,7 @@ version = "0.0.0" # see: https://github.com/mtkennerly/poetry-dynamic-versioning
|
|
|
4
4
|
|
|
5
5
|
[tool.poetry]
|
|
6
6
|
name = "rs-client-libraries"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.2a13"
|
|
8
8
|
description = "Copernicus Sentinel rs-client library and processing workflows."
|
|
9
9
|
license = "Apache-2.0"
|
|
10
10
|
authors = ["ESA under contract with Airbus and CS <ops_coprs@airbus.com>"]
|
|
@@ -36,23 +36,23 @@ include = [
|
|
|
36
36
|
|
|
37
37
|
[tool.poetry.dependencies]
|
|
38
38
|
python = "^3.11"
|
|
39
|
-
opentelemetry-distro = "^0.
|
|
40
|
-
opentelemetry-instrumentation-aws-lambda = "^0.
|
|
41
|
-
opentelemetry-test-utils = "^0.
|
|
42
|
-
opentelemetry-exporter-otlp = "^1.
|
|
39
|
+
opentelemetry-distro = "^0.53b1" # then run 'poetry run opentelemetry-bootstrap -a install'
|
|
40
|
+
opentelemetry-instrumentation-aws-lambda = "^0.53b1"
|
|
41
|
+
opentelemetry-test-utils = "^0.53b1"
|
|
42
|
+
opentelemetry-exporter-otlp = "^1.32.1"
|
|
43
43
|
python-logging-loki = "^0.3.1" # to avoid otl error: No module named 'logging_loki'
|
|
44
|
-
cachetools = "^5.5.
|
|
45
|
-
pystac = "^1.
|
|
46
|
-
pystac-client = "^0.8.
|
|
44
|
+
cachetools = "^5.5.2"
|
|
45
|
+
pystac = "^1.13.0"
|
|
46
|
+
pystac-client = "^0.8.6"
|
|
47
47
|
python-dateutil = "^2.9.0.post0"
|
|
48
|
-
stac-pydantic = "^3.1.
|
|
49
|
-
openapi-core = "^0.19.
|
|
48
|
+
stac-pydantic = "^3.1.5"
|
|
49
|
+
openapi-core = "^0.19.5"
|
|
50
50
|
|
|
51
51
|
[tool.poetry.group.dev]
|
|
52
52
|
optional = true
|
|
53
53
|
|
|
54
54
|
[tool.poetry.group.dev.dependencies]
|
|
55
|
-
pytest = "^8.3.
|
|
55
|
+
pytest = "^8.3.5"
|
|
56
56
|
pytest-cov = "^6.0.0"
|
|
57
57
|
pytest-mock = "^3.14.0"
|
|
58
58
|
pytest-error-for-skips = "^2.0.2"
|
|
@@ -67,12 +67,12 @@ sphinx = "^8.1.3"
|
|
|
67
67
|
sphinx-rtd-theme = "^3.0.2"
|
|
68
68
|
flake8 = "^7.1.1"
|
|
69
69
|
flake8-pyproject = "^1.2.3"
|
|
70
|
-
responses = "^0.25.
|
|
70
|
+
responses = "^0.25.7"
|
|
71
71
|
starlette = "^0.45.3"
|
|
72
72
|
|
|
73
73
|
[tool.poetry.group.demo.dependencies] # for hybrid mode
|
|
74
74
|
jupyterlab = "^4.2.1"
|
|
75
|
-
boto3 = "^1.
|
|
75
|
+
boto3 = "^1.37.11"
|
|
76
76
|
papermill = "^2.6.0"
|
|
77
77
|
|
|
78
78
|
[tool.poetry-dynamic-versioning] # needs: poetry self add "poetry-dynamic-versioning[plugin]"
|
|
@@ -15,9 +15,9 @@
|
|
|
15
15
|
"""AuxipClient class implementation."""
|
|
16
16
|
|
|
17
17
|
import logging
|
|
18
|
+
from typing import Any
|
|
18
19
|
|
|
19
20
|
from rs_client.stac_base import StacBase
|
|
20
|
-
from rs_common.config import EAuxipStation
|
|
21
21
|
from rs_common.utils import get_href_service
|
|
22
22
|
|
|
23
23
|
|
|
@@ -33,9 +33,8 @@ class AuxipClient(StacBase):
|
|
|
33
33
|
rs_server_href: str | None,
|
|
34
34
|
rs_server_api_key: str | None,
|
|
35
35
|
owner_id: str | None,
|
|
36
|
-
station: EAuxipStation | str,
|
|
37
36
|
logger: logging.Logger | None = None,
|
|
38
|
-
**kwargs,
|
|
37
|
+
**kwargs: dict[str, Any],
|
|
39
38
|
):
|
|
40
39
|
"""
|
|
41
40
|
Initializes an AuxipClient instance.
|
|
@@ -44,12 +43,15 @@ class AuxipClient(StacBase):
|
|
|
44
43
|
rs_server_href (str | None): The URL of the RS-Server. Pass None for local mode.
|
|
45
44
|
rs_server_api_key (str | None): API key for authentication.
|
|
46
45
|
owner_id (str | None): ID of the catalog owner.
|
|
47
|
-
station (EAuxipStation | str): The AUXIP station identifier.
|
|
48
46
|
logger (logging.Logger | None, optional): Logger instance (default: None).
|
|
49
|
-
**kwargs:
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
47
|
+
**kwargs: Arbitrary keyword arguments that may include:
|
|
48
|
+
- `headers` (Optional[Dict[str, str]])
|
|
49
|
+
- `parameters` (Optional[Dict[str, Any]])
|
|
50
|
+
- `ignore_conformance` (Optional[bool])
|
|
51
|
+
- `modifier` (Callable[[Collection | Item | ItemCollection | dict[Any, Any]], None] | None)
|
|
52
|
+
- `request_modifier` (Optional[Callable[[Request], Union[Request, None]]])
|
|
53
|
+
- `stac_io` (Optional[StacApiIO])
|
|
54
|
+
- `timeout` (Optional[Timeout])
|
|
53
55
|
"""
|
|
54
56
|
super().__init__(
|
|
55
57
|
rs_server_href,
|
|
@@ -59,10 +61,6 @@ class AuxipClient(StacBase):
|
|
|
59
61
|
get_href_service(rs_server_href, "RSPY_HOST_ADGS") + "/auxip/",
|
|
60
62
|
**kwargs,
|
|
61
63
|
)
|
|
62
|
-
try:
|
|
63
|
-
self.station: EAuxipStation = EAuxipStation[station] if isinstance(station, str) else station
|
|
64
|
-
except KeyError as e:
|
|
65
|
-
self.log_and_raise(f"There is no such AUXIP station: {station}", e)
|
|
66
64
|
|
|
67
65
|
@property
|
|
68
66
|
def href_service(self) -> str:
|
|
@@ -72,8 +70,3 @@ class AuxipClient(StacBase):
|
|
|
72
70
|
Otherwise it should just be the RS-Server URL.
|
|
73
71
|
"""
|
|
74
72
|
return get_href_service(self.rs_server_href, "RSPY_HOST_ADGS") + "/auxip"
|
|
75
|
-
|
|
76
|
-
@property
|
|
77
|
-
def station_name(self) -> str:
|
|
78
|
-
"""Return the station name."""
|
|
79
|
-
return self.station.value
|
|
@@ -15,9 +15,9 @@
|
|
|
15
15
|
"""CadipClient class implementation."""
|
|
16
16
|
|
|
17
17
|
import logging
|
|
18
|
+
from typing import Any
|
|
18
19
|
|
|
19
20
|
from rs_client.stac_base import StacBase
|
|
20
|
-
from rs_common.config import ECadipStation
|
|
21
21
|
from rs_common.utils import get_href_service
|
|
22
22
|
|
|
23
23
|
|
|
@@ -26,7 +26,6 @@ class CadipClient(StacBase):
|
|
|
26
26
|
CadipClient class implementation.
|
|
27
27
|
|
|
28
28
|
Attributes: see :py:class:`RsClient`
|
|
29
|
-
station (ECadipStation): Cadip station
|
|
30
29
|
"""
|
|
31
30
|
|
|
32
31
|
def __init__( # pylint: disable=too-many-arguments, too-many-positional-arguments
|
|
@@ -34,11 +33,26 @@ class CadipClient(StacBase):
|
|
|
34
33
|
rs_server_href: str | None,
|
|
35
34
|
rs_server_api_key: str | None,
|
|
36
35
|
owner_id: str | None,
|
|
37
|
-
station: str,
|
|
38
36
|
logger: logging.Logger | None = None,
|
|
39
|
-
**kwargs,
|
|
37
|
+
**kwargs: dict[str, Any],
|
|
40
38
|
):
|
|
41
|
-
"""
|
|
39
|
+
"""
|
|
40
|
+
Initializes a CadipClient instance.
|
|
41
|
+
|
|
42
|
+
Args:
|
|
43
|
+
rs_server_href (str | None): The URL of the RS-Server. Pass None for local mode.
|
|
44
|
+
rs_server_api_key (str | None): API key for authentication.
|
|
45
|
+
owner_id (str | None): ID of the catalog owner.
|
|
46
|
+
logger (logging.Logger | None, optional): Logger instance (default: None).
|
|
47
|
+
**kwargs: Arbitrary keyword arguments that may include:
|
|
48
|
+
- `headers` (Optional[Dict[str, str]])
|
|
49
|
+
- `parameters` (Optional[Dict[str, Any]])
|
|
50
|
+
- `ignore_conformance` (Optional[bool])
|
|
51
|
+
- `modifier` (Callable[[Collection | Item | ItemCollection | dict[Any, Any]], None] | None)
|
|
52
|
+
- `request_modifier` (Optional[Callable[[Request], Union[Request, None]]])
|
|
53
|
+
- `stac_io` (Optional[StacApiIO])
|
|
54
|
+
- `timeout` (Optional[Timeout])
|
|
55
|
+
"""
|
|
42
56
|
super().__init__(
|
|
43
57
|
rs_server_href,
|
|
44
58
|
rs_server_api_key,
|
|
@@ -47,10 +61,6 @@ class CadipClient(StacBase):
|
|
|
47
61
|
get_href_service(rs_server_href, "RSPY_HOST_CADIP") + "/cadip/",
|
|
48
62
|
**kwargs,
|
|
49
63
|
)
|
|
50
|
-
try:
|
|
51
|
-
self.station: ECadipStation = ECadipStation[station] if isinstance(station, str) else station
|
|
52
|
-
except KeyError as e:
|
|
53
|
-
self.log_and_raise(f"There is no such CADIP station: {station}", e)
|
|
54
64
|
|
|
55
65
|
@property
|
|
56
66
|
def href_service(self) -> str:
|
|
@@ -60,8 +70,3 @@ class CadipClient(StacBase):
|
|
|
60
70
|
Otherwise it should just be the RS-Server URL.
|
|
61
71
|
"""
|
|
62
72
|
return get_href_service(self.rs_server_href, "RSPY_HOST_CADIP") + "/cadip"
|
|
63
|
-
|
|
64
|
-
@property
|
|
65
|
-
def station_name(self) -> str:
|
|
66
|
-
"""Return the station name."""
|
|
67
|
-
return self.station.value
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
from __future__ import annotations
|
|
18
18
|
|
|
19
19
|
import logging
|
|
20
|
-
from
|
|
20
|
+
from collections.abc import Iterator
|
|
21
21
|
|
|
22
22
|
import pystac
|
|
23
23
|
from pystac import Collection, Item, Link, RelType
|
|
@@ -70,7 +70,7 @@ class CatalogClient(StacBase): # type: ignore # pylint: disable=too-many-ancest
|
|
|
70
70
|
|
|
71
71
|
return get_href_service(self.rs_server_href, "RSPY_HOST_CATALOG")
|
|
72
72
|
|
|
73
|
-
def full_collection_id(self, owner_id: str | None, collection_id: str, concat_char: str | None = None):
|
|
73
|
+
def full_collection_id(self, owner_id: str | None, collection_id: str, concat_char: str | None = None) -> str:
|
|
74
74
|
"""
|
|
75
75
|
Generates a full collection identifier by concatenating the owner ID and collection ID.
|
|
76
76
|
|
|
@@ -90,11 +90,11 @@ class CatalogClient(StacBase): # type: ignore # pylint: disable=too-many-ancest
|
|
|
90
90
|
|
|
91
91
|
Raises:
|
|
92
92
|
- **AttributeError**: If `self.owner_id` is not set and `owner_id` is `None`,
|
|
93
|
-
|
|
93
|
+
causing an attempt to concatenate a `NoneType` with a string.
|
|
94
94
|
|
|
95
95
|
Notes:
|
|
96
96
|
- This function is useful in scenarios where collections are stored with unique
|
|
97
|
-
|
|
97
|
+
identifiers that require owner prefixes for proper scoping.
|
|
98
98
|
"""
|
|
99
99
|
|
|
100
100
|
if not concat_char:
|
|
@@ -111,16 +111,16 @@ class CatalogClient(StacBase): # type: ignore # pylint: disable=too-many-ancest
|
|
|
111
111
|
self,
|
|
112
112
|
collection_id: str,
|
|
113
113
|
owner_id: str | None = None,
|
|
114
|
-
) ->
|
|
114
|
+
) -> Collection | CollectionClient:
|
|
115
115
|
"""Get the requested collection"""
|
|
116
116
|
return super().get_collection(self.full_collection_id(owner_id, collection_id, ":"))
|
|
117
117
|
|
|
118
118
|
def get_items(
|
|
119
119
|
self,
|
|
120
120
|
collection_id: str,
|
|
121
|
-
items_ids:
|
|
121
|
+
items_ids: str | None = None,
|
|
122
122
|
owner_id: str | None = None,
|
|
123
|
-
) -> Iterator[
|
|
123
|
+
) -> Iterator[Item]:
|
|
124
124
|
"""Get all items from a specific collection."""
|
|
125
125
|
return super().get_items(self.full_collection_id(owner_id, collection_id, ":"), items_ids)
|
|
126
126
|
|
|
@@ -131,7 +131,7 @@ class CatalogClient(StacBase): # type: ignore # pylint: disable=too-many-ancest
|
|
|
131
131
|
def search( # type: ignore # pylint: disable=too-many-arguments, arguments-differ
|
|
132
132
|
self,
|
|
133
133
|
**kwargs,
|
|
134
|
-
) ->
|
|
134
|
+
) -> ItemSearch | None:
|
|
135
135
|
"""Search items inside a specific collection."""
|
|
136
136
|
|
|
137
137
|
kwargs["collections"] = [
|
|
@@ -24,7 +24,6 @@ import requests
|
|
|
24
24
|
from cachetools import TTLCache, cached
|
|
25
25
|
|
|
26
26
|
from rs_common import utils
|
|
27
|
-
from rs_common.config import EAuxipStation, ECadipStation
|
|
28
27
|
from rs_common.logging import Logging
|
|
29
28
|
from rs_common.utils import AuthInfo
|
|
30
29
|
|
|
@@ -33,6 +32,10 @@ APIKEY_HEADER = "x-api-key"
|
|
|
33
32
|
# Timeout in seconds
|
|
34
33
|
TIMEOUT = 30
|
|
35
34
|
|
|
35
|
+
# API Key Manager URL used to get an API Key information.
|
|
36
|
+
# Works only in cluster mode. This endpoint is not exposed outside the cluster.
|
|
37
|
+
RSPY_UAC_CHECK_URL = os.getenv("RSPY_UAC_CHECK_URL", "")
|
|
38
|
+
|
|
36
39
|
|
|
37
40
|
class RsClient: # pylint: disable=too-many-instance-attributes
|
|
38
41
|
"""
|
|
@@ -49,14 +52,15 @@ class RsClient: # pylint: disable=too-many-instance-attributes
|
|
|
49
52
|
rs_server_href (str | None): RS-Server URL. Pass None for local mode.
|
|
50
53
|
rs_server_api_key (str | None): API key for RS-Server authentication.
|
|
51
54
|
rs_server_oauth2_cookie (str | None): OAuth2 session cookie read from
|
|
52
|
-
|
|
53
|
-
owner_id (str): The owner of the STAC catalog collections (no special characters allowed)
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
55
|
+
the `RSPY_OAUTH2_COOKIE` environment variable.
|
|
56
|
+
owner_id (str): The owner of the STAC catalog collections (no special characters allowed).
|
|
57
|
+
If not set, we try to read it from the RSPY_HOST_USER environment variable. If still not set:
|
|
58
|
+
- In local mode, it takes the system username.
|
|
59
|
+
- In cluster mode, it is deduced from the API key or OAuth2 login = your keycloak username.
|
|
60
|
+
- In hybrid mode, we raise an Exception.
|
|
61
|
+
If owner_id is different than your keycloak username, then make sure that your keycloak account has
|
|
62
|
+
the rights to read/write on this catalog owner.
|
|
63
|
+
owner_id is also used in the RS-Client logging.
|
|
60
64
|
logger (logging.Logger): Logger instance for logging messages.
|
|
61
65
|
local_mode (bool): Indicates whether the client is running in local mode.
|
|
62
66
|
apikey_headers (dict): API key headers for HTTP requests.
|
|
@@ -86,7 +90,7 @@ class RsClient: # pylint: disable=too-many-instance-attributes
|
|
|
86
90
|
self.rs_server_href: str | None = rs_server_href
|
|
87
91
|
self.rs_server_api_key: str | None = rs_server_api_key
|
|
88
92
|
self.rs_server_oauth2_cookie: str | None = os.getenv("RSPY_OAUTH2_COOKIE")
|
|
89
|
-
self.owner_id: str | None = owner_id or ""
|
|
93
|
+
self.owner_id: str | None = owner_id or os.getenv("RSPY_HOST_USER")
|
|
90
94
|
self.logger: logging.Logger = logger or Logging.default(__name__)
|
|
91
95
|
|
|
92
96
|
# Remove trailing / character(s) from the URL
|
|
@@ -97,6 +101,10 @@ class RsClient: # pylint: disable=too-many-instance-attributes
|
|
|
97
101
|
# Env vars are used instead to determine the different services URL.
|
|
98
102
|
self.local_mode = not bool(self.rs_server_href)
|
|
99
103
|
|
|
104
|
+
# We are in hybrid mode if not on local mode and the API Key Manager check URL is undefined.
|
|
105
|
+
# NOTE: maybe later we could define this hybrid mode in a different way.
|
|
106
|
+
self.hybrid_mode = (not self.local_mode) and (not RSPY_UAC_CHECK_URL)
|
|
107
|
+
|
|
100
108
|
if (not self.local_mode) and (not self.rs_server_api_key) and (not self.rs_server_oauth2_cookie):
|
|
101
109
|
raise RuntimeError("API key or OAuth2 cookie is mandatory for RS-Server authentication")
|
|
102
110
|
|
|
@@ -116,7 +124,14 @@ class RsClient: # pylint: disable=too-many-instance-attributes
|
|
|
116
124
|
if self.local_mode:
|
|
117
125
|
self.owner_id = getpass.getuser()
|
|
118
126
|
|
|
119
|
-
# In hybrid
|
|
127
|
+
# In hybrid mode, the API Key Manager check URL is not accessible and there is no OAuth2
|
|
128
|
+
# so the owner id must be set explicitly by the user.
|
|
129
|
+
elif self.hybrid_mode:
|
|
130
|
+
raise RuntimeError(
|
|
131
|
+
"In hybrid mode, the owner_id must be set explicitly by parameter or environment variable",
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
# In cluster mode, we retrieve the OAuth2 or API key login
|
|
120
135
|
else:
|
|
121
136
|
self.owner_id = self.apikey_user_login if self.rs_server_api_key else self.oauth2_user_login
|
|
122
137
|
|
|
@@ -184,17 +199,9 @@ class RsClient: # pylint: disable=too-many-instance-attributes
|
|
|
184
199
|
if self.local_mode:
|
|
185
200
|
return AuthInfo(user_login="", iam_roles=[], apikey_config={})
|
|
186
201
|
|
|
187
|
-
# self.logger.warning(
|
|
188
|
-
# f"TODO: use {self.rs_server_href}/apikeymanager/auth/check_key instead, see: "
|
|
189
|
-
# "https://pforge-exchange2.astrium.eads.net/jira/browse/RSPY-257",
|
|
190
|
-
# )
|
|
191
|
-
# Does not work in hybrid mode for now because this URL is not exposed.
|
|
192
|
-
check_url = os.environ["RSPY_UAC_CHECK_URL"]
|
|
193
|
-
|
|
194
202
|
# Request the API key manager, pass user-defined api key in http header
|
|
195
|
-
# check_url = f"{self.rs_server_href}/apikeymanager/auth/check_key"
|
|
196
203
|
self.logger.debug("Call the API key manager")
|
|
197
|
-
response = self.http_session.get(
|
|
204
|
+
response = self.http_session.get(RSPY_UAC_CHECK_URL, **self.apikey_headers, timeout=TIMEOUT)
|
|
198
205
|
if not response.ok:
|
|
199
206
|
raise RuntimeError(
|
|
200
207
|
f"API key manager status code {response.status_code}: {utils.read_response_error(response)}",
|
|
@@ -248,30 +255,25 @@ class RsClient: # pylint: disable=too-many-instance-attributes
|
|
|
248
255
|
# Get child class instances #
|
|
249
256
|
#############################
|
|
250
257
|
|
|
251
|
-
def get_auxip_client(self,
|
|
258
|
+
def get_auxip_client(self, **kwargs) -> "AuxipClient": # type: ignore # noqa: F821
|
|
252
259
|
"""
|
|
253
260
|
Return an instance of the child class AuxipClient, with the same attributes as this "self" instance.
|
|
254
|
-
Args:
|
|
255
|
-
station (EAuxipStation): Auxip station
|
|
256
261
|
"""
|
|
257
262
|
from rs_client.auxip_client import ( # pylint: disable=import-outside-toplevel,cyclic-import
|
|
258
263
|
AuxipClient,
|
|
259
264
|
)
|
|
260
265
|
|
|
261
|
-
return AuxipClient(self.rs_server_href, self.rs_server_api_key, self.owner_id,
|
|
266
|
+
return AuxipClient(self.rs_server_href, self.rs_server_api_key, self.owner_id, self.logger, **kwargs)
|
|
262
267
|
|
|
263
|
-
def get_cadip_client(self,
|
|
268
|
+
def get_cadip_client(self, **kwargs) -> "CadipClient": # type: ignore # noqa: F821
|
|
264
269
|
"""
|
|
265
270
|
Return an instance of the child class CadipClient, with the same attributes as this "self" instance.
|
|
266
|
-
|
|
267
|
-
Args:
|
|
268
|
-
station (ECadipStation): Cadip station
|
|
269
271
|
"""
|
|
270
272
|
from rs_client.cadip_client import ( # pylint: disable=import-outside-toplevel,cyclic-import
|
|
271
273
|
CadipClient,
|
|
272
274
|
)
|
|
273
275
|
|
|
274
|
-
return CadipClient(self.rs_server_href, self.rs_server_api_key, self.owner_id,
|
|
276
|
+
return CadipClient(self.rs_server_href, self.rs_server_api_key, self.owner_id, self.logger, **kwargs)
|
|
275
277
|
|
|
276
278
|
def get_catalog_client(self, **kwargs) -> "CatalogClient": # type: ignore # noqa: F821
|
|
277
279
|
"""
|
|
@@ -15,8 +15,9 @@
|
|
|
15
15
|
"""StacBase class implementation."""
|
|
16
16
|
|
|
17
17
|
import logging
|
|
18
|
+
from collections.abc import Callable, Iterator
|
|
18
19
|
from functools import lru_cache, wraps
|
|
19
|
-
from typing import Any,
|
|
20
|
+
from typing import Any, cast
|
|
20
21
|
|
|
21
22
|
import requests
|
|
22
23
|
from pystac import Collection, Item, ItemCollection
|
|
@@ -76,13 +77,13 @@ class StacBase(RsClient):
|
|
|
76
77
|
owner_id: str | None = None,
|
|
77
78
|
logger: logging.Logger | None = None,
|
|
78
79
|
stac_href: str | None = None, # Flag to enable pystac_client for specific subclasses
|
|
79
|
-
headers:
|
|
80
|
-
parameters:
|
|
81
|
-
ignore_conformance:
|
|
80
|
+
headers: dict[str, str] | None = None,
|
|
81
|
+
parameters: dict[str, Any] | None = None,
|
|
82
|
+
ignore_conformance: bool | None = None,
|
|
82
83
|
modifier: Callable[[Collection | Item | ItemCollection | dict[Any, Any]], None] | None = None,
|
|
83
|
-
request_modifier:
|
|
84
|
-
stac_io:
|
|
85
|
-
timeout:
|
|
84
|
+
request_modifier: Callable[[Request], Request | None] | None = None,
|
|
85
|
+
stac_io: StacApiIO | None = None,
|
|
86
|
+
timeout: Timeout | None = TIMEOUT,
|
|
86
87
|
):
|
|
87
88
|
"""
|
|
88
89
|
Initialize the StacBase instance.
|
|
@@ -167,9 +168,9 @@ class StacBase(RsClient):
|
|
|
167
168
|
# Get all the available collections
|
|
168
169
|
return self.ps_client.get_collections()
|
|
169
170
|
|
|
170
|
-
@lru_cache
|
|
171
|
+
@lru_cache
|
|
171
172
|
@handle_api_error
|
|
172
|
-
def get_collection(self, collection_id: str) ->
|
|
173
|
+
def get_collection(self, collection_id: str) -> Collection | CollectionClient:
|
|
173
174
|
"""
|
|
174
175
|
Retrieve a specific STAC collection by ID.
|
|
175
176
|
|
|
@@ -185,7 +186,7 @@ class StacBase(RsClient):
|
|
|
185
186
|
return self.ps_client.get_collection(collection_id)
|
|
186
187
|
|
|
187
188
|
@handle_api_error
|
|
188
|
-
def get_items(self, collection_id: str, items_ids:
|
|
189
|
+
def get_items(self, collection_id: str, items_ids: str | None = None) -> Iterator["Item"]:
|
|
189
190
|
"""
|
|
190
191
|
Retrieve all items or specific items from a collection.
|
|
191
192
|
|
|
@@ -233,7 +234,7 @@ class StacBase(RsClient):
|
|
|
233
234
|
return item
|
|
234
235
|
|
|
235
236
|
@handle_api_error
|
|
236
|
-
def get_collection_queryables(self, collection_id) ->
|
|
237
|
+
def get_collection_queryables(self, collection_id) -> dict[str, Any]:
|
|
237
238
|
"""
|
|
238
239
|
Retrieve queryable fields for a specific collection.
|
|
239
240
|
|
|
@@ -249,7 +250,7 @@ class StacBase(RsClient):
|
|
|
249
250
|
|
|
250
251
|
return self.ps_client.get_merged_queryables([collection_id])
|
|
251
252
|
|
|
252
|
-
def get_queryables(self) ->
|
|
253
|
+
def get_queryables(self) -> dict[str, Any]:
|
|
253
254
|
"""
|
|
254
255
|
Retrieve queryable fields for all collections in the STAC API. These are the available terms for
|
|
255
256
|
usage when writing filter expressions in /search endpoint for all the collections
|
|
@@ -278,7 +279,7 @@ class StacBase(RsClient):
|
|
|
278
279
|
raise RuntimeError(f"Could not get queryables from {href_queryables}")
|
|
279
280
|
try:
|
|
280
281
|
json_data = response.json()
|
|
281
|
-
return cast(
|
|
282
|
+
return cast(dict[str, Any], json_data) # Explicitly cast to Dict[str, Any]
|
|
282
283
|
except ValueError as e:
|
|
283
284
|
raise RuntimeError(f"Invalid JSON response from {href_queryables}") from e
|
|
284
285
|
|
|
@@ -17,7 +17,8 @@
|
|
|
17
17
|
import json
|
|
18
18
|
import os
|
|
19
19
|
import os.path as osp
|
|
20
|
-
from typing import Any
|
|
20
|
+
from typing import Any
|
|
21
|
+
from urllib.parse import urlparse
|
|
21
22
|
|
|
22
23
|
# openapi_core libraries used for endpoints validation
|
|
23
24
|
import requests
|
|
@@ -37,7 +38,6 @@ from rs_common.utils import get_href_service
|
|
|
37
38
|
# with a valid format according to ogc standard. In the meantime, we don't perform validation to make all staging
|
|
38
39
|
# notebooks pass. If this env variable if not specified (for example that is the case when we launch the pytest),
|
|
39
40
|
# we perform this validation by default
|
|
40
|
-
DO_VALIDATE: bool = os.getenv("RSPY_APPLY_STAGING_ENDPOINTS_VALIDATION", "1") == "1"
|
|
41
41
|
|
|
42
42
|
PATH_TO_YAML_OPENAPI = osp.join(
|
|
43
43
|
osp.realpath(osp.dirname(__file__)),
|
|
@@ -46,15 +46,17 @@ PATH_TO_YAML_OPENAPI = osp.join(
|
|
|
46
46
|
"yaml",
|
|
47
47
|
"staging_openapi_schema.yaml",
|
|
48
48
|
)
|
|
49
|
-
PATH_TO_STAGING_BODY = osp.join(
|
|
50
|
-
osp.realpath(osp.dirname(__file__)),
|
|
51
|
-
"../config",
|
|
52
|
-
"staging_templates",
|
|
53
|
-
"staging_body.json",
|
|
54
|
-
)
|
|
55
49
|
RESOURCE = "staging"
|
|
56
50
|
|
|
57
51
|
|
|
52
|
+
def is_url(path):
|
|
53
|
+
"""
|
|
54
|
+
Function to check if a string is an url
|
|
55
|
+
"""
|
|
56
|
+
parsed = urlparse(path)
|
|
57
|
+
return bool(parsed.scheme and parsed.netloc)
|
|
58
|
+
|
|
59
|
+
|
|
58
60
|
class StagingValidationException(Exception):
|
|
59
61
|
"""
|
|
60
62
|
Exception raised when an error occurs during the OGC validation
|
|
@@ -93,8 +95,6 @@ class StagingClient(RsClient):
|
|
|
93
95
|
ResponseUnmarshalResult.data: data validated by the openapi_core
|
|
94
96
|
unmarshal_response method
|
|
95
97
|
"""
|
|
96
|
-
if not DO_VALIDATE:
|
|
97
|
-
return request.body
|
|
98
98
|
|
|
99
99
|
if not os.path.isfile(PATH_TO_YAML_OPENAPI):
|
|
100
100
|
raise FileNotFoundError(f"The following file path was not found: {PATH_TO_YAML_OPENAPI}")
|
|
@@ -129,11 +129,6 @@ class StagingClient(RsClient):
|
|
|
129
129
|
ResponseUnmarshalResult.data: data validated by the openapi_core
|
|
130
130
|
unmarshal_response method
|
|
131
131
|
"""
|
|
132
|
-
if not DO_VALIDATE:
|
|
133
|
-
if not response.content:
|
|
134
|
-
raise StagingValidationException("Response content is empty !")
|
|
135
|
-
return json.loads(response.content)
|
|
136
|
-
|
|
137
132
|
if not os.path.isfile(PATH_TO_YAML_OPENAPI):
|
|
138
133
|
raise FileNotFoundError(f"The following file path was not found: {PATH_TO_YAML_OPENAPI}")
|
|
139
134
|
|
|
@@ -146,8 +141,9 @@ class StagingClient(RsClient):
|
|
|
146
141
|
result = openapi.unmarshal_response(openapi_request, openapi_response) # type: ignore
|
|
147
142
|
if result.errors:
|
|
148
143
|
raise StagingValidationException( # type: ignore
|
|
149
|
-
f"Error validating the response of the enpoint "
|
|
150
|
-
f"
|
|
144
|
+
f"Error validating the response of the enpoint {openapi_request.path} - "
|
|
145
|
+
f"Server response content: {response.json()} - "
|
|
146
|
+
f"Validation error of the server response: {str(result.errors[0])}", # type: ignore
|
|
151
147
|
)
|
|
152
148
|
if not result.data:
|
|
153
149
|
raise StagingValidationException(
|
|
@@ -155,13 +151,13 @@ class StagingClient(RsClient):
|
|
|
155
151
|
f"{openapi_request.path}: 'data' field of ResponseUnmarshalResult"
|
|
156
152
|
f"object is empty",
|
|
157
153
|
)
|
|
158
|
-
return
|
|
154
|
+
return json.loads(response.content)
|
|
159
155
|
|
|
160
156
|
############################
|
|
161
157
|
# Call RS-Server endpoints #
|
|
162
158
|
############################
|
|
163
159
|
|
|
164
|
-
def get_processes(self) ->
|
|
160
|
+
def get_processes(self) -> dict:
|
|
165
161
|
"""_summary_
|
|
166
162
|
|
|
167
163
|
Returns:
|
|
@@ -174,7 +170,7 @@ class StagingClient(RsClient):
|
|
|
174
170
|
)
|
|
175
171
|
return self.validate_and_unmarshal_response(response)
|
|
176
172
|
|
|
177
|
-
def get_process(self, process_id: str) ->
|
|
173
|
+
def get_process(self, process_id: str) -> dict:
|
|
178
174
|
"""
|
|
179
175
|
Wrapper to get a specific process
|
|
180
176
|
Args:
|
|
@@ -191,7 +187,7 @@ class StagingClient(RsClient):
|
|
|
191
187
|
self,
|
|
192
188
|
stac_input: dict[Any, Any] | str,
|
|
193
189
|
out_coll_name: str,
|
|
194
|
-
) ->
|
|
190
|
+
) -> dict:
|
|
195
191
|
"""Method to start the staging process from rs-client - Call the endpoint /processes/staging/execution
|
|
196
192
|
|
|
197
193
|
Args:
|
|
@@ -200,55 +196,77 @@ class StagingClient(RsClient):
|
|
|
200
196
|
the output of a search for Cadip or Auxip sessions)
|
|
201
197
|
- A json string corresponding to a Feature or a FeatureCollection
|
|
202
198
|
- A string corresponding to a path to a json file containing a Feature or a FeatureCollection
|
|
203
|
-
|
|
204
|
-
|
|
199
|
+
- A single link that returns a STAC itemCollection: this link should be an url to search a
|
|
200
|
+
itemCollection, for example:
|
|
201
|
+
http://localhost:8002/cadip/search?ids=S1A_20231120061537234567&collections=cadip_sentinel1
|
|
202
|
+
out_coll_name (str): name of the output collection
|
|
205
203
|
Return:
|
|
206
204
|
job_id (int, str): Returns the status code of the staging request + the identifier
|
|
207
205
|
(or None if staging endpoint fails) of the running job
|
|
208
206
|
"""
|
|
209
207
|
stac_input_dict = {}
|
|
210
|
-
|
|
208
|
+
is_an_url = False
|
|
209
|
+
|
|
210
|
+
# ----- Case 1: we only load a link (that refers to a STAC itemCollection) in the staging request body
|
|
211
211
|
if isinstance(stac_input, str):
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
stac_input_dict = json.loads(stac_input)
|
|
221
|
-
else:
|
|
222
|
-
stac_input_dict = stac_input
|
|
223
|
-
|
|
224
|
-
if "type" not in stac_input_dict:
|
|
225
|
-
raise KeyError("Key 'type' is missing from the staging input data")
|
|
226
|
-
|
|
227
|
-
# Validate input data using Pydantic
|
|
228
|
-
if stac_input_dict["type"] == "Feature":
|
|
229
|
-
stac_item = Item(**stac_input_dict)
|
|
230
|
-
stac_item_collection = ItemCollection(
|
|
231
|
-
**{
|
|
232
|
-
"type": "FeatureCollection",
|
|
233
|
-
"context": {"limit": 1000, "returned": 2},
|
|
234
|
-
"features": [stac_item],
|
|
235
|
-
}, # type: ignore
|
|
236
|
-
)
|
|
212
|
+
try:
|
|
213
|
+
is_an_url = is_url(stac_input)
|
|
214
|
+
except ValueError:
|
|
215
|
+
is_an_url = False
|
|
216
|
+
if is_an_url:
|
|
217
|
+
staging_body = {"inputs": {"collection": out_coll_name, "items": {"href": stac_input}}}
|
|
218
|
+
|
|
219
|
+
# ----- Case 2: we directly load a STAC ItemCollection in the staging request body
|
|
237
220
|
else:
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
221
|
+
# If stac_input is a file, load this file to a dictionary
|
|
222
|
+
if isinstance(stac_input, str):
|
|
223
|
+
# If the input is a valid path to a json_file, load this file
|
|
224
|
+
if os.path.exists(os.path.dirname(stac_input)) and stac_input.endswith(".json"):
|
|
225
|
+
# Read the yaml or json file
|
|
226
|
+
with open(stac_input, encoding="utf-8") as opened:
|
|
227
|
+
stac_file_to_dict = json.loads(opened.read())
|
|
228
|
+
stac_input_dict = stac_file_to_dict
|
|
229
|
+
# If the input string is not a path, try to convert the content of the string to a json dictionary
|
|
230
|
+
else:
|
|
231
|
+
try:
|
|
232
|
+
stac_input_dict = json.loads(stac_input)
|
|
233
|
+
except json.JSONDecodeError as e:
|
|
234
|
+
raise StagingValidationException(
|
|
235
|
+
f"""Invalid input format: {stac_input} - Input data must be either:
|
|
236
|
+
- A Python dictionary corresponding to a Feature or a FeatureCollection
|
|
237
|
+
(that can be for example the output of a search for Cadip or Auxip sessions)
|
|
238
|
+
- A json string corresponding to a Feature or a FeatureCollection
|
|
239
|
+
- A string corresponding to a path to a json file containing a Feature or a
|
|
240
|
+
FeatureCollection
|
|
241
|
+
- A single link that returns a STAC ItemCollection: this link should be an
|
|
242
|
+
url to search an ItemCollection
|
|
243
|
+
""".strip(),
|
|
244
|
+
) from e
|
|
245
|
+
else:
|
|
246
|
+
stac_input_dict = stac_input
|
|
247
|
+
|
|
248
|
+
if "type" not in stac_input_dict:
|
|
249
|
+
raise KeyError("Key 'type' is missing from the staging input data")
|
|
250
|
+
|
|
251
|
+
# Validate input data using Pydantic
|
|
252
|
+
if stac_input_dict["type"] == "Feature":
|
|
253
|
+
stac_item = Item(**stac_input_dict)
|
|
254
|
+
stac_item_collection = ItemCollection(
|
|
255
|
+
**{
|
|
256
|
+
"type": "FeatureCollection",
|
|
257
|
+
"context": {"limit": 1000, "returned": 2},
|
|
258
|
+
"features": [stac_item],
|
|
259
|
+
}, # type: ignore
|
|
260
|
+
)
|
|
261
|
+
else:
|
|
262
|
+
stac_item_collection = ItemCollection(**stac_input_dict)
|
|
243
263
|
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
# TODO: replace the staging_body "outputs" field with the following when rs-server-staging is updated
|
|
251
|
-
# TODO: "outputs": {"featureCollectionOutput": {"transmissionMode": "value"}},
|
|
264
|
+
staging_body = {
|
|
265
|
+
"inputs": {
|
|
266
|
+
"collection": out_coll_name,
|
|
267
|
+
"items": {"value": stac_item_collection.model_dump(mode="json")},
|
|
268
|
+
},
|
|
269
|
+
}
|
|
252
270
|
|
|
253
271
|
# Check that the request containing the staging body is valid
|
|
254
272
|
request = requests.Request( # pylint: disable=W0612 # noqa: F841
|
|
@@ -256,8 +274,9 @@ class StagingClient(RsClient):
|
|
|
256
274
|
url=f"{self.href_service}/processes/{RESOURCE}/execution", # L'URL de l'endpoint
|
|
257
275
|
json=staging_body, # Corps de la requête en JSON
|
|
258
276
|
).prepare()
|
|
259
|
-
|
|
260
|
-
#
|
|
277
|
+
|
|
278
|
+
# Validate the body of the request that will be sent to the staging
|
|
279
|
+
self.validate_and_unmarshal_request(request)
|
|
261
280
|
|
|
262
281
|
response = self.http_session.post(
|
|
263
282
|
url=f"{self.href_service}/processes/staging/execution",
|
|
@@ -267,7 +286,7 @@ class StagingClient(RsClient):
|
|
|
267
286
|
)
|
|
268
287
|
return self.validate_and_unmarshal_response(response)
|
|
269
288
|
|
|
270
|
-
def get_jobs(self) ->
|
|
289
|
+
def get_jobs(self) -> dict:
|
|
271
290
|
"""Method to get running jobs"""
|
|
272
291
|
response = self.http_session.get(
|
|
273
292
|
url=f"{self.href_service}/jobs",
|
|
@@ -276,7 +295,7 @@ class StagingClient(RsClient):
|
|
|
276
295
|
)
|
|
277
296
|
return self.validate_and_unmarshal_response(response)
|
|
278
297
|
|
|
279
|
-
def get_job_info(self, job_id: str) ->
|
|
298
|
+
def get_job_info(self, job_id: str) -> dict: # pylint: disable=too-many-locals
|
|
280
299
|
"""Method to get a specific job response"""
|
|
281
300
|
response = self.http_session.get(
|
|
282
301
|
url=f"{self.href_service}/jobs/{job_id}",
|
|
@@ -285,7 +304,7 @@ class StagingClient(RsClient):
|
|
|
285
304
|
)
|
|
286
305
|
return self.validate_and_unmarshal_response(response)
|
|
287
306
|
|
|
288
|
-
def delete_job(self, job_id: str) ->
|
|
307
|
+
def delete_job(self, job_id: str) -> dict: # pylint: disable=too-many-locals
|
|
289
308
|
"""Method to get a specific job response"""
|
|
290
309
|
response = self.http_session.delete(
|
|
291
310
|
url=f"{self.href_service}/jobs/{job_id}",
|
|
@@ -294,7 +313,7 @@ class StagingClient(RsClient):
|
|
|
294
313
|
)
|
|
295
314
|
return self.validate_and_unmarshal_response(response)
|
|
296
315
|
|
|
297
|
-
def get_job_results(self, job_id: str) ->
|
|
316
|
+
def get_job_results(self, job_id: str) -> dict:
|
|
298
317
|
"""Wrapper to get the result of a specfific job
|
|
299
318
|
|
|
300
319
|
Args:
|
|
@@ -15,6 +15,7 @@
|
|
|
15
15
|
"""OpenTelemetry utility"""
|
|
16
16
|
|
|
17
17
|
import inspect
|
|
18
|
+
import json
|
|
18
19
|
import os
|
|
19
20
|
import pkgutil
|
|
20
21
|
import sys
|
|
@@ -25,32 +26,113 @@ from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExport
|
|
|
25
26
|
from opentelemetry.instrumentation.asyncio import AsyncioInstrumentor
|
|
26
27
|
from opentelemetry.instrumentation.aws_lambda import AwsLambdaInstrumentor
|
|
27
28
|
from opentelemetry.instrumentation.instrumentor import BaseInstrumentor # type: ignore
|
|
29
|
+
from opentelemetry.instrumentation.requests import RequestsInstrumentor
|
|
28
30
|
from opentelemetry.sdk.resources import Resource
|
|
29
31
|
from opentelemetry.sdk.trace import TracerProvider
|
|
30
32
|
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
|
33
|
+
from opentelemetry.trace.span import NonRecordingSpan, SpanContext, TraceFlags
|
|
34
|
+
from opentelemetry.util._decorator import _agnosticcontextmanager
|
|
31
35
|
|
|
32
36
|
from rs_common.logging import Logging
|
|
37
|
+
from rs_common.utils import env_bool
|
|
33
38
|
|
|
34
|
-
|
|
39
|
+
default_logger = Logging.default(__name__)
|
|
35
40
|
|
|
36
41
|
FROM_PYTEST = False
|
|
37
42
|
|
|
38
43
|
|
|
39
|
-
|
|
44
|
+
# Show details of http headers and body/content in tempo/grafana ?
|
|
45
|
+
# Don't store results in global variables because the env var values can change
|
|
46
|
+
# after this module was loaded.
|
|
47
|
+
def trace_headers():
|
|
48
|
+
"""Trace request headers ?"""
|
|
49
|
+
return env_bool("OTEL_PYTHON_REQUESTS_TRACE_HEADERS", default=False)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def trace_body():
|
|
53
|
+
"""Trace request bodies and response contents ?"""
|
|
54
|
+
return env_bool("OTEL_PYTHON_REQUESTS_TRACE_BODY", default=False)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def parse_data(data) -> str:
|
|
58
|
+
"""Convert data to a string representation"""
|
|
59
|
+
|
|
60
|
+
if not data:
|
|
61
|
+
return ""
|
|
62
|
+
|
|
63
|
+
# Try to decode bytes
|
|
64
|
+
if isinstance(data, bytes):
|
|
65
|
+
data = data.decode("utf-8")
|
|
66
|
+
|
|
67
|
+
# Try to convert to a dict
|
|
68
|
+
try:
|
|
69
|
+
data = dict(data)
|
|
70
|
+
except Exception: # pylint: disable=broad-exception-caught # nosec
|
|
71
|
+
pass
|
|
72
|
+
|
|
73
|
+
# Or to parse to a dict
|
|
74
|
+
try:
|
|
75
|
+
data = json.loads(data)
|
|
76
|
+
except Exception: # pylint: disable=broad-exception-caught # nosec
|
|
77
|
+
pass
|
|
78
|
+
|
|
79
|
+
# If we have a dict, try to format it as json
|
|
80
|
+
if isinstance(data, dict):
|
|
81
|
+
data = json.dumps(data, indent=2)
|
|
82
|
+
|
|
83
|
+
return data or ""
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def request_hook(span, request):
|
|
87
|
+
"""
|
|
88
|
+
HTTP requests intrumentation
|
|
89
|
+
"""
|
|
90
|
+
if not span:
|
|
91
|
+
return
|
|
92
|
+
|
|
93
|
+
# Copy the http.url attribute into _url so it appears at the
|
|
94
|
+
# top in the grafana UI, it's more readable
|
|
95
|
+
span.set_attribute("_url", span.attributes.get("http.url"))
|
|
96
|
+
|
|
97
|
+
if trace_headers():
|
|
98
|
+
span.set_attribute("http.request.headers", parse_data(request.headers))
|
|
99
|
+
|
|
100
|
+
if trace_body():
|
|
101
|
+
span.set_attribute("http.request.body", parse_data(request.body))
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def response_hook(span, request, response): # pylint: disable=W0613
|
|
105
|
+
"""
|
|
106
|
+
HTTP responses intrumentation
|
|
107
|
+
"""
|
|
108
|
+
if not span:
|
|
109
|
+
return
|
|
110
|
+
|
|
111
|
+
if trace_headers():
|
|
112
|
+
span.set_attribute("http.response.headers", parse_data(response.headers))
|
|
113
|
+
|
|
114
|
+
if trace_body():
|
|
115
|
+
span.set_attribute("http.response.content", parse_data(response.content))
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def init_traces(service_name: str, logger=None):
|
|
40
119
|
"""
|
|
41
120
|
Init instrumentation of OpenTelemetry traces.
|
|
42
121
|
|
|
43
122
|
Args:
|
|
44
123
|
service_name (str): service name
|
|
124
|
+
logger: non-default logger to user
|
|
45
125
|
"""
|
|
46
|
-
|
|
47
126
|
# See: https://github.com/softwarebloat/python-tracing-demo/tree/main
|
|
48
127
|
|
|
128
|
+
logger = logger or default_logger
|
|
129
|
+
|
|
49
130
|
# Don't call this line from pytest because it causes errors:
|
|
50
131
|
# Transient error StatusCode.UNAVAILABLE encountered while exporting metrics to localhost:4317, retrying in ..s.
|
|
51
132
|
if not FROM_PYTEST:
|
|
52
133
|
tempo_endpoint = os.getenv("TEMPO_ENDPOINT")
|
|
53
134
|
if not tempo_endpoint:
|
|
135
|
+
logger.warning("'TEMPO_ENDPOINT' variable is missing, cannot initialize OpenTelemetry")
|
|
54
136
|
return
|
|
55
137
|
|
|
56
138
|
# TODO: to avoid errors in local mode:
|
|
@@ -83,6 +165,7 @@ def init_traces(service_name: str):
|
|
|
83
165
|
|
|
84
166
|
# Recursively find all package modules
|
|
85
167
|
for _, module_str, _ in pkgutil.walk_packages(path=package.__path__, prefix=prefix, onerror=None):
|
|
168
|
+
|
|
86
169
|
# Don't instrument these modules, they have errors, maybe we should see why
|
|
87
170
|
if module_str in [
|
|
88
171
|
"opentelemetry.instrumentation.tortoiseorm",
|
|
@@ -106,8 +189,53 @@ def init_traces(service_name: str):
|
|
|
106
189
|
# If the "instrument" method exists, call it
|
|
107
190
|
_instrument = getattr(_class, "instrument", None)
|
|
108
191
|
if callable(_instrument):
|
|
192
|
+
|
|
109
193
|
_class_instance = _class()
|
|
110
|
-
if
|
|
194
|
+
if _class == RequestsInstrumentor and (trace_headers() or trace_body()):
|
|
195
|
+
_class_instance.instrument(
|
|
196
|
+
tracer_provider=otel_tracer,
|
|
197
|
+
request_hook=request_hook,
|
|
198
|
+
response_hook=response_hook,
|
|
199
|
+
)
|
|
200
|
+
elif not _class_instance.is_instrumented_by_opentelemetry:
|
|
111
201
|
_class_instance.instrument(tracer_provider=otel_tracer)
|
|
112
202
|
# name = f"{module_str}.{_class.__name__}".removeprefix(prefix)
|
|
113
203
|
# logger.debug(f"OpenTelemetry instrumentation of {name!r}")
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
@_agnosticcontextmanager
|
|
207
|
+
def start_span(
|
|
208
|
+
instrumenting_module_name: str,
|
|
209
|
+
name: str,
|
|
210
|
+
span_context: SpanContext | None = None,
|
|
211
|
+
):
|
|
212
|
+
"""
|
|
213
|
+
Context manager for creating a new main or child OpenTelemetry span and set it
|
|
214
|
+
as the current span in this tracer's context.
|
|
215
|
+
|
|
216
|
+
Args:
|
|
217
|
+
instrumenting_module_name: Caller module name, just pass __name__
|
|
218
|
+
name: The name of the span to be created (use a custom name)
|
|
219
|
+
span_context: Parent span context. Only to create a child span.
|
|
220
|
+
"""
|
|
221
|
+
tracer = trace.get_tracer(instrumenting_module_name)
|
|
222
|
+
|
|
223
|
+
# Create a main span
|
|
224
|
+
if not span_context:
|
|
225
|
+
with tracer.start_as_current_span(name) as span:
|
|
226
|
+
yield span
|
|
227
|
+
|
|
228
|
+
# Create a child span
|
|
229
|
+
else:
|
|
230
|
+
main_span_context = SpanContext(
|
|
231
|
+
trace_id=span_context.trace_id,
|
|
232
|
+
span_id=span_context.span_id,
|
|
233
|
+
is_remote=True,
|
|
234
|
+
trace_flags=TraceFlags(TraceFlags.SAMPLED),
|
|
235
|
+
)
|
|
236
|
+
main_span = NonRecordingSpan(main_span_context)
|
|
237
|
+
with trace.use_span(main_span): # pylint: disable=not-context-manager
|
|
238
|
+
# Optionnaly, we could use the main span instead of creating
|
|
239
|
+
# a new one, to be discussed.
|
|
240
|
+
with tracer.start_as_current_span(name) as span:
|
|
241
|
+
yield span
|
|
@@ -15,13 +15,10 @@
|
|
|
15
15
|
"""Logging utility."""
|
|
16
16
|
|
|
17
17
|
import logging
|
|
18
|
-
import logging.handlers
|
|
19
18
|
import os
|
|
20
19
|
from multiprocessing import Queue
|
|
21
20
|
from threading import Lock
|
|
22
21
|
|
|
23
|
-
import logging_loki
|
|
24
|
-
|
|
25
22
|
|
|
26
23
|
class Logging: # pylint: disable=too-few-public-methods
|
|
27
24
|
"""
|
|
@@ -64,6 +61,8 @@ class Logging: # pylint: disable=too-few-public-methods
|
|
|
64
61
|
# Export logs to Loki, see: https://pypi.org/project/python-logging-loki/
|
|
65
62
|
loki_endpoint = os.getenv("LOKI_ENDPOINT")
|
|
66
63
|
if loki_endpoint:
|
|
64
|
+
import logging_loki # pylint: disable=import-outside-toplevel
|
|
65
|
+
|
|
67
66
|
handler = logging_loki.LokiQueueHandler(
|
|
68
67
|
Queue(-1),
|
|
69
68
|
url=loki_endpoint,
|
|
@@ -54,3 +54,17 @@ def get_href_service(rs_server_href, env_var) -> str:
|
|
|
54
54
|
if not rs_server_href:
|
|
55
55
|
raise RuntimeError("RS-Server URL is undefined")
|
|
56
56
|
return rs_server_href.rstrip("/")
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def env_bool(var: str, default: bool) -> bool:
|
|
60
|
+
"""
|
|
61
|
+
Return True if an environemnt variable is set to 1, true or yes (case insensitive).
|
|
62
|
+
Return False if set to 0, false or no (case insensitive).
|
|
63
|
+
Return the default value if not set or set to a different value.
|
|
64
|
+
"""
|
|
65
|
+
val = os.getenv(var, str(default)).lower()
|
|
66
|
+
if val in ("y", "yes", "t", "true", "on", "1"):
|
|
67
|
+
return True
|
|
68
|
+
if val in ("n", "no", "f", "false", "off", "0"):
|
|
69
|
+
return False
|
|
70
|
+
return default
|
|
File without changes
|
|
File without changes
|
|
File without changes
|