rs-client-libraries 0.2a12__tar.gz → 0.2a13__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: rs-client-libraries
3
- Version: 0.2a12
3
+ Version: 0.2a13
4
4
  Summary: Copernicus Sentinel rs-client library and processing workflows.
5
5
  Home-page: https://home.rs-python.eu/
6
6
  License: Apache-2.0
@@ -21,11 +21,11 @@ Classifier: Topic :: Scientific/Engineering :: Image Processing
21
21
  Classifier: Topic :: Software Development :: Libraries :: Python Modules
22
22
  Requires-Dist: cachetools (>=5.5.2,<6.0.0)
23
23
  Requires-Dist: openapi-core (>=0.19.5,<0.20.0)
24
- Requires-Dist: opentelemetry-distro (>=0.53b0,<0.54)
25
- Requires-Dist: opentelemetry-exporter-otlp (>=1.32.0,<2.0.0)
26
- Requires-Dist: opentelemetry-instrumentation-aws-lambda (>=0.53b0,<0.54)
27
- Requires-Dist: opentelemetry-test-utils (>=0.53b0,<0.54)
28
- Requires-Dist: pystac (>=1.12.2,<2.0.0)
24
+ Requires-Dist: opentelemetry-distro (>=0.53b1,<0.54)
25
+ Requires-Dist: opentelemetry-exporter-otlp (>=1.32.1,<2.0.0)
26
+ Requires-Dist: opentelemetry-instrumentation-aws-lambda (>=0.53b1,<0.54)
27
+ Requires-Dist: opentelemetry-test-utils (>=0.53b1,<0.54)
28
+ Requires-Dist: pystac (>=1.13.0,<2.0.0)
29
29
  Requires-Dist: pystac-client (>=0.8.6,<0.9.0)
30
30
  Requires-Dist: python-dateutil (>=2.9.0.post0,<3.0.0)
31
31
  Requires-Dist: python-logging-loki (>=0.3.1,<0.4.0)
@@ -4,7 +4,7 @@ version = "0.0.0" # see: https://github.com/mtkennerly/poetry-dynamic-versioning
4
4
 
5
5
  [tool.poetry]
6
6
  name = "rs-client-libraries"
7
- version = "0.2a12"
7
+ version = "0.2a13"
8
8
  description = "Copernicus Sentinel rs-client library and processing workflows."
9
9
  license = "Apache-2.0"
10
10
  authors = ["ESA under contract with Airbus and CS <ops_coprs@airbus.com>"]
@@ -36,13 +36,13 @@ include = [
36
36
 
37
37
  [tool.poetry.dependencies]
38
38
  python = "^3.11"
39
- opentelemetry-distro = "^0.53b0" # then run 'poetry run opentelemetry-bootstrap -a install'
40
- opentelemetry-instrumentation-aws-lambda = "^0.53b0"
41
- opentelemetry-test-utils = "^0.53b0"
42
- opentelemetry-exporter-otlp = "^1.32.0"
39
+ opentelemetry-distro = "^0.53b1" # then run 'poetry run opentelemetry-bootstrap -a install'
40
+ opentelemetry-instrumentation-aws-lambda = "^0.53b1"
41
+ opentelemetry-test-utils = "^0.53b1"
42
+ opentelemetry-exporter-otlp = "^1.32.1"
43
43
  python-logging-loki = "^0.3.1" # to avoid otl error: No module named 'logging_loki'
44
44
  cachetools = "^5.5.2"
45
- pystac = "^1.12.2"
45
+ pystac = "^1.13.0"
46
46
  pystac-client = "^0.8.6"
47
47
  python-dateutil = "^2.9.0.post0"
48
48
  stac-pydantic = "^3.1.5"
@@ -15,4 +15,4 @@
15
15
  """rsclient main module."""
16
16
 
17
17
  # Set automatically by running `poetry dynamic-versioning`
18
- __version__ = "0.2a12"
18
+ __version__ = "0.2a13"
@@ -18,7 +18,6 @@ import logging
18
18
  from typing import Any
19
19
 
20
20
  from rs_client.stac_base import StacBase
21
- from rs_common.config import EAuxipStation
22
21
  from rs_common.utils import get_href_service
23
22
 
24
23
 
@@ -34,7 +33,6 @@ class AuxipClient(StacBase):
34
33
  rs_server_href: str | None,
35
34
  rs_server_api_key: str | None,
36
35
  owner_id: str | None,
37
- station: EAuxipStation | str,
38
36
  logger: logging.Logger | None = None,
39
37
  **kwargs: dict[str, Any],
40
38
  ):
@@ -45,7 +43,6 @@ class AuxipClient(StacBase):
45
43
  rs_server_href (str | None): The URL of the RS-Server. Pass None for local mode.
46
44
  rs_server_api_key (str | None): API key for authentication.
47
45
  owner_id (str | None): ID of the catalog owner.
48
- station (EAuxipStation | str): The AUXIP station identifier.
49
46
  logger (logging.Logger | None, optional): Logger instance (default: None).
50
47
  **kwargs: Arbitrary keyword arguments that may include:
51
48
  - `headers` (Optional[Dict[str, str]])
@@ -55,9 +52,6 @@ class AuxipClient(StacBase):
55
52
  - `request_modifier` (Optional[Callable[[Request], Union[Request, None]]])
56
53
  - `stac_io` (Optional[StacApiIO])
57
54
  - `timeout` (Optional[Timeout])
58
-
59
- Raises:
60
- RuntimeError: If the provided station is not a valid AUXIP station.
61
55
  """
62
56
  super().__init__(
63
57
  rs_server_href,
@@ -67,10 +61,6 @@ class AuxipClient(StacBase):
67
61
  get_href_service(rs_server_href, "RSPY_HOST_ADGS") + "/auxip/",
68
62
  **kwargs,
69
63
  )
70
- try:
71
- self.station: EAuxipStation = EAuxipStation[station] if isinstance(station, str) else station
72
- except KeyError as e:
73
- self.log_and_raise(f"There is no such AUXIP station: {station}", e)
74
64
 
75
65
  @property
76
66
  def href_service(self) -> str:
@@ -80,8 +70,3 @@ class AuxipClient(StacBase):
80
70
  Otherwise it should just be the RS-Server URL.
81
71
  """
82
72
  return get_href_service(self.rs_server_href, "RSPY_HOST_ADGS") + "/auxip"
83
-
84
- @property
85
- def station_name(self) -> str:
86
- """Return the station name."""
87
- return self.station.value
@@ -18,7 +18,6 @@ import logging
18
18
  from typing import Any
19
19
 
20
20
  from rs_client.stac_base import StacBase
21
- from rs_common.config import ECadipStation
22
21
  from rs_common.utils import get_href_service
23
22
 
24
23
 
@@ -27,7 +26,6 @@ class CadipClient(StacBase):
27
26
  CadipClient class implementation.
28
27
 
29
28
  Attributes: see :py:class:`RsClient`
30
- station (ECadipStation): Cadip station
31
29
  """
32
30
 
33
31
  def __init__( # pylint: disable=too-many-arguments, too-many-positional-arguments
@@ -35,7 +33,6 @@ class CadipClient(StacBase):
35
33
  rs_server_href: str | None,
36
34
  rs_server_api_key: str | None,
37
35
  owner_id: str | None,
38
- station: str,
39
36
  logger: logging.Logger | None = None,
40
37
  **kwargs: dict[str, Any],
41
38
  ):
@@ -46,7 +43,6 @@ class CadipClient(StacBase):
46
43
  rs_server_href (str | None): The URL of the RS-Server. Pass None for local mode.
47
44
  rs_server_api_key (str | None): API key for authentication.
48
45
  owner_id (str | None): ID of the catalog owner.
49
- station (EAuxipStation | str): The CADIP station identifier.
50
46
  logger (logging.Logger | None, optional): Logger instance (default: None).
51
47
  **kwargs: Arbitrary keyword arguments that may include:
52
48
  - `headers` (Optional[Dict[str, str]])
@@ -56,9 +52,6 @@ class CadipClient(StacBase):
56
52
  - `request_modifier` (Optional[Callable[[Request], Union[Request, None]]])
57
53
  - `stac_io` (Optional[StacApiIO])
58
54
  - `timeout` (Optional[Timeout])
59
-
60
- Raises:
61
- RuntimeError: If the provided station is not a valid CADIP station.
62
55
  """
63
56
  super().__init__(
64
57
  rs_server_href,
@@ -68,10 +61,6 @@ class CadipClient(StacBase):
68
61
  get_href_service(rs_server_href, "RSPY_HOST_CADIP") + "/cadip/",
69
62
  **kwargs,
70
63
  )
71
- try:
72
- self.station: ECadipStation = ECadipStation[station] if isinstance(station, str) else station
73
- except KeyError as e:
74
- self.log_and_raise(f"There is no such CADIP station: {station}", e)
75
64
 
76
65
  @property
77
66
  def href_service(self) -> str:
@@ -81,8 +70,3 @@ class CadipClient(StacBase):
81
70
  Otherwise it should just be the RS-Server URL.
82
71
  """
83
72
  return get_href_service(self.rs_server_href, "RSPY_HOST_CADIP") + "/cadip"
84
-
85
- @property
86
- def station_name(self) -> str:
87
- """Return the station name."""
88
- return self.station.value
@@ -24,7 +24,6 @@ import requests
24
24
  from cachetools import TTLCache, cached
25
25
 
26
26
  from rs_common import utils
27
- from rs_common.config import EAuxipStation, ECadipStation
28
27
  from rs_common.logging import Logging
29
28
  from rs_common.utils import AuthInfo
30
29
 
@@ -256,30 +255,25 @@ class RsClient: # pylint: disable=too-many-instance-attributes
256
255
  # Get child class instances #
257
256
  #############################
258
257
 
259
- def get_auxip_client(self, station: EAuxipStation, **kwargs) -> "AuxipClient": # type: ignore # noqa: F821
258
+ def get_auxip_client(self, **kwargs) -> "AuxipClient": # type: ignore # noqa: F821
260
259
  """
261
260
  Return an instance of the child class AuxipClient, with the same attributes as this "self" instance.
262
- Args:
263
- station (EAuxipStation): Auxip station
264
261
  """
265
262
  from rs_client.auxip_client import ( # pylint: disable=import-outside-toplevel,cyclic-import
266
263
  AuxipClient,
267
264
  )
268
265
 
269
- return AuxipClient(self.rs_server_href, self.rs_server_api_key, self.owner_id, station, self.logger, **kwargs)
266
+ return AuxipClient(self.rs_server_href, self.rs_server_api_key, self.owner_id, self.logger, **kwargs)
270
267
 
271
- def get_cadip_client(self, station: ECadipStation, **kwargs) -> "CadipClient": # type: ignore # noqa: F821
268
+ def get_cadip_client(self, **kwargs) -> "CadipClient": # type: ignore # noqa: F821
272
269
  """
273
270
  Return an instance of the child class CadipClient, with the same attributes as this "self" instance.
274
-
275
- Args:
276
- station (ECadipStation): Cadip station
277
271
  """
278
272
  from rs_client.cadip_client import ( # pylint: disable=import-outside-toplevel,cyclic-import
279
273
  CadipClient,
280
274
  )
281
275
 
282
- return CadipClient(self.rs_server_href, self.rs_server_api_key, self.owner_id, station, self.logger, **kwargs)
276
+ return CadipClient(self.rs_server_href, self.rs_server_api_key, self.owner_id, self.logger, **kwargs)
283
277
 
284
278
  def get_catalog_client(self, **kwargs) -> "CatalogClient": # type: ignore # noqa: F821
285
279
  """
@@ -15,6 +15,6 @@
15
15
  """Common modules."""
16
16
 
17
17
  # Configure OpenTelemetry
18
- from rs_common import opentelemetry
18
+ from rs_common import init_opentelemetry
19
19
 
20
- opentelemetry.init_traces("rs.client")
20
+ init_opentelemetry.init_traces("rs.client")
@@ -15,6 +15,7 @@
15
15
  """OpenTelemetry utility"""
16
16
 
17
17
  import inspect
18
+ import json
18
19
  import os
19
20
  import pkgutil
20
21
  import sys
@@ -25,28 +26,113 @@ from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExport
25
26
  from opentelemetry.instrumentation.asyncio import AsyncioInstrumentor
26
27
  from opentelemetry.instrumentation.aws_lambda import AwsLambdaInstrumentor
27
28
  from opentelemetry.instrumentation.instrumentor import BaseInstrumentor # type: ignore
29
+ from opentelemetry.instrumentation.requests import RequestsInstrumentor
28
30
  from opentelemetry.sdk.resources import Resource
29
31
  from opentelemetry.sdk.trace import TracerProvider
30
32
  from opentelemetry.sdk.trace.export import BatchSpanProcessor
33
+ from opentelemetry.trace.span import NonRecordingSpan, SpanContext, TraceFlags
34
+ from opentelemetry.util._decorator import _agnosticcontextmanager
35
+
36
+ from rs_common.logging import Logging
37
+ from rs_common.utils import env_bool
38
+
39
+ default_logger = Logging.default(__name__)
31
40
 
32
41
  FROM_PYTEST = False
33
42
 
34
43
 
35
- def init_traces(service_name: str):
44
+ # Show details of http headers and body/content in tempo/grafana ?
45
+ # Don't store results in global variables because the env var values can change
46
+ # after this module was loaded.
47
+ def trace_headers():
48
+ """Trace request headers ?"""
49
+ return env_bool("OTEL_PYTHON_REQUESTS_TRACE_HEADERS", default=False)
50
+
51
+
52
+ def trace_body():
53
+ """Trace request bodies and response contents ?"""
54
+ return env_bool("OTEL_PYTHON_REQUESTS_TRACE_BODY", default=False)
55
+
56
+
57
+ def parse_data(data) -> str:
58
+ """Convert data to a string representation"""
59
+
60
+ if not data:
61
+ return ""
62
+
63
+ # Try to decode bytes
64
+ if isinstance(data, bytes):
65
+ data = data.decode("utf-8")
66
+
67
+ # Try to convert to a dict
68
+ try:
69
+ data = dict(data)
70
+ except Exception: # pylint: disable=broad-exception-caught # nosec
71
+ pass
72
+
73
+ # Or to parse to a dict
74
+ try:
75
+ data = json.loads(data)
76
+ except Exception: # pylint: disable=broad-exception-caught # nosec
77
+ pass
78
+
79
+ # If we have a dict, try to format it as json
80
+ if isinstance(data, dict):
81
+ data = json.dumps(data, indent=2)
82
+
83
+ return data or ""
84
+
85
+
86
+ def request_hook(span, request):
87
+ """
88
+ HTTP requests intrumentation
89
+ """
90
+ if not span:
91
+ return
92
+
93
+ # Copy the http.url attribute into _url so it appears at the
94
+ # top in the grafana UI, it's more readable
95
+ span.set_attribute("_url", span.attributes.get("http.url"))
96
+
97
+ if trace_headers():
98
+ span.set_attribute("http.request.headers", parse_data(request.headers))
99
+
100
+ if trace_body():
101
+ span.set_attribute("http.request.body", parse_data(request.body))
102
+
103
+
104
+ def response_hook(span, request, response): # pylint: disable=W0613
105
+ """
106
+ HTTP responses intrumentation
107
+ """
108
+ if not span:
109
+ return
110
+
111
+ if trace_headers():
112
+ span.set_attribute("http.response.headers", parse_data(response.headers))
113
+
114
+ if trace_body():
115
+ span.set_attribute("http.response.content", parse_data(response.content))
116
+
117
+
118
+ def init_traces(service_name: str, logger=None):
36
119
  """
37
120
  Init instrumentation of OpenTelemetry traces.
38
121
 
39
122
  Args:
40
123
  service_name (str): service name
124
+ logger: non-default logger to user
41
125
  """
42
-
43
126
  # See: https://github.com/softwarebloat/python-tracing-demo/tree/main
44
127
 
128
+ logger = logger or default_logger
129
+
45
130
  # Don't call this line from pytest because it causes errors:
46
131
  # Transient error StatusCode.UNAVAILABLE encountered while exporting metrics to localhost:4317, retrying in ..s.
47
132
  if not FROM_PYTEST:
48
133
  tempo_endpoint = os.getenv("TEMPO_ENDPOINT")
49
134
  if not tempo_endpoint:
135
+ logger.warning("'TEMPO_ENDPOINT' variable is missing, cannot initialize OpenTelemetry")
50
136
  return
51
137
 
52
138
  # TODO: to avoid errors in local mode:
@@ -79,6 +165,7 @@ def init_traces(service_name: str):
79
165
 
80
166
  # Recursively find all package modules
81
167
  for _, module_str, _ in pkgutil.walk_packages(path=package.__path__, prefix=prefix, onerror=None):
168
+
82
169
  # Don't instrument these modules, they have errors, maybe we should see why
83
170
  if module_str in [
84
171
  "opentelemetry.instrumentation.tortoiseorm",
@@ -102,8 +189,53 @@ def init_traces(service_name: str):
102
189
  # If the "instrument" method exists, call it
103
190
  _instrument = getattr(_class, "instrument", None)
104
191
  if callable(_instrument):
192
+
105
193
  _class_instance = _class()
106
- if not _class_instance.is_instrumented_by_opentelemetry:
194
+ if _class == RequestsInstrumentor and (trace_headers() or trace_body()):
195
+ _class_instance.instrument(
196
+ tracer_provider=otel_tracer,
197
+ request_hook=request_hook,
198
+ response_hook=response_hook,
199
+ )
200
+ elif not _class_instance.is_instrumented_by_opentelemetry:
107
201
  _class_instance.instrument(tracer_provider=otel_tracer)
108
202
  # name = f"{module_str}.{_class.__name__}".removeprefix(prefix)
109
203
  # logger.debug(f"OpenTelemetry instrumentation of {name!r}")
204
+
205
+
206
+ @_agnosticcontextmanager
207
+ def start_span(
208
+ instrumenting_module_name: str,
209
+ name: str,
210
+ span_context: SpanContext | None = None,
211
+ ):
212
+ """
213
+ Context manager for creating a new main or child OpenTelemetry span and set it
214
+ as the current span in this tracer's context.
215
+
216
+ Args:
217
+ instrumenting_module_name: Caller module name, just pass __name__
218
+ name: The name of the span to be created (use a custom name)
219
+ span_context: Parent span context. Only to create a child span.
220
+ """
221
+ tracer = trace.get_tracer(instrumenting_module_name)
222
+
223
+ # Create a main span
224
+ if not span_context:
225
+ with tracer.start_as_current_span(name) as span:
226
+ yield span
227
+
228
+ # Create a child span
229
+ else:
230
+ main_span_context = SpanContext(
231
+ trace_id=span_context.trace_id,
232
+ span_id=span_context.span_id,
233
+ is_remote=True,
234
+ trace_flags=TraceFlags(TraceFlags.SAMPLED),
235
+ )
236
+ main_span = NonRecordingSpan(main_span_context)
237
+ with trace.use_span(main_span): # pylint: disable=not-context-manager
238
+ # Optionnaly, we could use the main span instead of creating
239
+ # a new one, to be discussed.
240
+ with tracer.start_as_current_span(name) as span:
241
+ yield span
@@ -15,13 +15,10 @@
15
15
  """Logging utility."""
16
16
 
17
17
  import logging
18
- import logging.handlers
19
18
  import os
20
19
  from multiprocessing import Queue
21
20
  from threading import Lock
22
21
 
23
- import logging_loki
24
-
25
22
 
26
23
  class Logging: # pylint: disable=too-few-public-methods
27
24
  """
@@ -64,6 +61,8 @@ class Logging: # pylint: disable=too-few-public-methods
64
61
  # Export logs to Loki, see: https://pypi.org/project/python-logging-loki/
65
62
  loki_endpoint = os.getenv("LOKI_ENDPOINT")
66
63
  if loki_endpoint:
64
+ import logging_loki # pylint: disable=import-outside-toplevel
65
+
67
66
  handler = logging_loki.LokiQueueHandler(
68
67
  Queue(-1),
69
68
  url=loki_endpoint,
@@ -54,3 +54,17 @@ def get_href_service(rs_server_href, env_var) -> str:
54
54
  if not rs_server_href:
55
55
  raise RuntimeError("RS-Server URL is undefined")
56
56
  return rs_server_href.rstrip("/")
57
+
58
+
59
+ def env_bool(var: str, default: bool) -> bool:
60
+ """
61
+ Return True if an environemnt variable is set to 1, true or yes (case insensitive).
62
+ Return False if set to 0, false or no (case insensitive).
63
+ Return the default value if not set or set to a different value.
64
+ """
65
+ val = os.getenv(var, str(default)).lower()
66
+ if val in ("y", "yes", "t", "true", "on", "1"):
67
+ return True
68
+ if val in ("n", "no", "f", "false", "off", "0"):
69
+ return False
70
+ return default