rs-client-libraries 0.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. config/staging_templates/README.md +17 -0
  2. config/staging_templates/yaml/additionalParameter.yaml +30 -0
  3. config/staging_templates/yaml/bbox.yaml +34 -0
  4. config/staging_templates/yaml/binaryInputValue.yaml +15 -0
  5. config/staging_templates/yaml/confClasses.yaml +22 -0
  6. config/staging_templates/yaml/descriptionType.yaml +36 -0
  7. config/staging_templates/yaml/dpr_openapi_schema.yaml +223 -0
  8. config/staging_templates/yaml/exception.yaml +31 -0
  9. config/staging_templates/yaml/execute.yaml +34 -0
  10. config/staging_templates/yaml/format.yaml +25 -0
  11. config/staging_templates/yaml/inlineOrRefData.yaml +18 -0
  12. config/staging_templates/yaml/inputDescription.yaml +32 -0
  13. config/staging_templates/yaml/inputValue.yaml +17 -0
  14. config/staging_templates/yaml/inputValueNoObject.yaml +22 -0
  15. config/staging_templates/yaml/jobControlOptions.yaml +19 -0
  16. config/staging_templates/yaml/jobList.yaml +27 -0
  17. config/staging_templates/yaml/landingPage.yaml +28 -0
  18. config/staging_templates/yaml/link.yaml +31 -0
  19. config/staging_templates/yaml/metadata.yaml +22 -0
  20. config/staging_templates/yaml/output.yaml +20 -0
  21. config/staging_templates/yaml/outputDescription.yaml +22 -0
  22. config/staging_templates/yaml/process.yaml +23 -0
  23. config/staging_templates/yaml/processList.yaml +27 -0
  24. config/staging_templates/yaml/processSummary.yaml +37 -0
  25. config/staging_templates/yaml/qualifiedInputValue.yaml +22 -0
  26. config/staging_templates/yaml/reference.yaml +21 -0
  27. config/staging_templates/yaml/results.yaml +16 -0
  28. config/staging_templates/yaml/schema.yaml +144 -0
  29. config/staging_templates/yaml/staging_openapi_schema.yaml +223 -0
  30. config/staging_templates/yaml/statusCode.yaml +22 -0
  31. config/staging_templates/yaml/statusInfo.yaml +52 -0
  32. config/staging_templates/yaml/subscriber.yaml +33 -0
  33. config/staging_templates/yaml/transmissionMode.yaml +19 -0
  34. rs_client/__init__.py +18 -0
  35. rs_client/ogcapi/__init__.py +15 -0
  36. rs_client/ogcapi/dpr_client.py +231 -0
  37. rs_client/ogcapi/ogcapi_client.py +282 -0
  38. rs_client/ogcapi/staging_client.py +178 -0
  39. rs_client/rs_client.py +277 -0
  40. rs_client/stac/__init__.py +15 -0
  41. rs_client/stac/auxip_client.py +70 -0
  42. rs_client/stac/cadip_client.py +70 -0
  43. rs_client/stac/catalog_client.py +479 -0
  44. rs_client/stac/stac_base.py +313 -0
  45. rs_client_libraries-0.2.dist-info/LICENSE +201 -0
  46. rs_client_libraries-0.2.dist-info/METADATA +44 -0
  47. rs_client_libraries-0.2.dist-info/RECORD +62 -0
  48. rs_client_libraries-0.2.dist-info/WHEEL +4 -0
  49. rs_common/__init__.py +20 -0
  50. rs_common/config.py +69 -0
  51. rs_common/init_opentelemetry.py +250 -0
  52. rs_common/logging.py +107 -0
  53. rs_common/prefect_utils.py +407 -0
  54. rs_common/utils.py +70 -0
  55. rs_workflows/__init__.py +15 -0
  56. rs_workflows/auxip_flow.py +70 -0
  57. rs_workflows/cadip_flow.py +69 -0
  58. rs_workflows/catalog_flow.py +71 -0
  59. rs_workflows/dpr_flow.py +272 -0
  60. rs_workflows/flow_utils.py +127 -0
  61. rs_workflows/on_demand_processing.py +243 -0
  62. rs_workflows/staging_flow.py +78 -0
rs_common/config.py ADDED
@@ -0,0 +1,69 @@
1
+ # Copyright 2024 CS Group
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """Configuration objects and functions."""
16
+
17
+ from enum import Enum
18
+
19
+ #############
20
+ # Constants #
21
+ #############
22
+
23
+ # datetime format used for HTTP requests
24
+ DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ"
25
+ DATETIME_FORMAT_MS = "%Y-%m-%dT%H:%M:%S.%fZ"
26
+
27
+ # name of the Auxip station, used in the same logic as the ECadipStation enum (see below)
28
+ AUXIP_STATION = "AUXIP"
29
+
30
+ #########
31
+ # Enums #
32
+ #########
33
+
34
+
35
+ class EPlatform(str, Enum):
36
+ """Platform enumeration."""
37
+
38
+ S1A = "S1A"
39
+ S2A = "S2A"
40
+ S2B = "S2B"
41
+ S3A = "S3A"
42
+ S3B = "S3B"
43
+
44
+
45
+ class ECadipStation(str, Enum):
46
+ """Cadip stations enumeration."""
47
+
48
+ CADIP = "CADIP"
49
+ INS = "INS"
50
+ MPS = "MPS"
51
+ MTI = "MTI"
52
+ NSG = "NSG"
53
+ SGS = "SGS"
54
+
55
+
56
+ class EAuxipStation(str, Enum):
57
+ """Auxip stations enumeration."""
58
+
59
+ ADGS = "ADGS"
60
+ ADGS2 = "ADGS2"
61
+
62
+
63
+ class EDownloadStatus(str, Enum):
64
+ """Download status enumeration."""
65
+
66
+ NOT_STARTED = "NOT_STARTED"
67
+ IN_PROGRESS = "IN_PROGRESS"
68
+ FAILED = "FAILED"
69
+ DONE = "DONE"
@@ -0,0 +1,250 @@
1
+ # Copyright 2024 CS Group
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """OpenTelemetry utility"""
16
+
17
+ import inspect
18
+ import json
19
+ import os
20
+ import pkgutil
21
+ import sys
22
+ from collections.abc import Iterator
23
+ from threading import Lock
24
+
25
+ import opentelemetry.instrumentation
26
+ from opentelemetry import trace
27
+ from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
28
+ from opentelemetry.instrumentation.asyncio import AsyncioInstrumentor
29
+ from opentelemetry.instrumentation.aws_lambda import AwsLambdaInstrumentor
30
+ from opentelemetry.instrumentation.instrumentor import BaseInstrumentor # type: ignore
31
+ from opentelemetry.instrumentation.requests import RequestsInstrumentor
32
+ from opentelemetry.sdk.resources import Resource
33
+ from opentelemetry.sdk.trace import TracerProvider
34
+ from opentelemetry.sdk.trace.export import BatchSpanProcessor
35
+ from opentelemetry.trace.span import NonRecordingSpan, Span, SpanContext, TraceFlags
36
+ from opentelemetry.util._decorator import _agnosticcontextmanager
37
+
38
+ from rs_common.logging import Logging
39
+ from rs_common.utils import env_bool
40
+
41
+ lock = Lock()
42
+
43
+ default_logger = Logging.default(__name__)
44
+
45
+ FROM_PYTEST = False
46
+
47
+
48
+ # Show details of http headers and body/content in tempo/grafana ?
49
+ # Don't store results in global variables because the env var values can change
50
+ # after this module was loaded.
51
+ def trace_headers():
52
+ """Trace request headers ?"""
53
+ return env_bool("OTEL_PYTHON_REQUESTS_TRACE_HEADERS", default=False)
54
+
55
+
56
+ def trace_body():
57
+ """Trace request bodies and response contents ?"""
58
+ return env_bool("OTEL_PYTHON_REQUESTS_TRACE_BODY", default=False)
59
+
60
+
61
+ def parse_data(data) -> str:
62
+ """Convert data to a string representation"""
63
+
64
+ if not data:
65
+ return ""
66
+
67
+ # Try to decode bytes
68
+ if isinstance(data, bytes):
69
+ data = data.decode("utf-8")
70
+
71
+ # Try to convert to a dict
72
+ try:
73
+ data = dict(data)
74
+ except Exception: # pylint: disable=broad-exception-caught # nosec
75
+ pass
76
+
77
+ # Or to parse to a dict
78
+ try:
79
+ data = json.loads(data)
80
+ except Exception: # pylint: disable=broad-exception-caught # nosec
81
+ pass
82
+
83
+ # If we have a dict, try to format it as json
84
+ if isinstance(data, dict):
85
+ data = json.dumps(data, indent=2)
86
+
87
+ return data or ""
88
+
89
+
90
+ def request_hook(span, request):
91
+ """
92
+ HTTP requests intrumentation
93
+ """
94
+ if not span:
95
+ return
96
+
97
+ # Copy the http.url attribute into _url so it appears at the
98
+ # top in the grafana UI, it's more readable
99
+ span.set_attribute("_url", span.attributes.get("http.url"))
100
+
101
+ if trace_headers():
102
+ span.set_attribute("http.request.headers", parse_data(request.headers))
103
+
104
+ if trace_body():
105
+ span.set_attribute("http.request.body", parse_data(request.body))
106
+
107
+
108
+ def response_hook(span, request, response): # pylint: disable=W0613
109
+ """
110
+ HTTP responses intrumentation
111
+ """
112
+ if not span:
113
+ return
114
+
115
+ if trace_headers():
116
+ span.set_attribute("http.response.headers", parse_data(response.headers))
117
+
118
+ if trace_body():
119
+ span.set_attribute("http.response.content", parse_data(response.content))
120
+
121
+
122
+ def init_traces(service_name: str, logger=None):
123
+ """
124
+ Init instrumentation of OpenTelemetry traces.
125
+
126
+ Args:
127
+ service_name (str): service name
128
+ logger: non-default logger to user
129
+ """
130
+ # See: https://github.com/softwarebloat/python-tracing-demo/tree/main
131
+
132
+ # No concurrent threads
133
+ with lock:
134
+ logger = logger or default_logger
135
+
136
+ # Don't call this line from pytest because it causes errors:
137
+ # Transient error StatusCode.UNAVAILABLE encountered while exporting metrics to ...
138
+ if not FROM_PYTEST:
139
+ tempo_endpoint = os.getenv("TEMPO_ENDPOINT")
140
+ if not tempo_endpoint:
141
+ logger.warning("'TEMPO_ENDPOINT' variable is missing, cannot initialize OpenTelemetry")
142
+ return
143
+
144
+ # TODO: to avoid errors in local mode:
145
+ # Transient error StatusCode.UNAVAILABLE encountered while exporting metrics to ...
146
+ #
147
+ # The below line does not work either but at least we have less error messages.
148
+ # See: https://pforge-exchange2.astrium.eads.net/jira/browse/RSPY-221?focusedId=162092&
149
+ # page=com.atlassian.jira.plugin.system.issuetabpanels%3Acomment-tabpanel#comment-162092
150
+ #
151
+ # Now we have a single line error, which is less worst:
152
+ # Failed to export metrics to tempo:4317, error code: StatusCode.UNIMPLEMENTED
153
+ os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] = tempo_endpoint
154
+
155
+ otel_resource = Resource(attributes={"service.name": service_name})
156
+ otel_tracer = TracerProvider(resource=otel_resource)
157
+ trace.set_tracer_provider(otel_tracer)
158
+
159
+ if not FROM_PYTEST:
160
+ otel_tracer.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(endpoint=tempo_endpoint)))
161
+
162
+ # Instrument all the dependencies under opentelemetry.instrumentation.*
163
+ # NOTE: we need 'poetry run opentelemetry-bootstrap -a install' to install these.
164
+
165
+ package = opentelemetry.instrumentation
166
+ prefix = package.__name__ + "."
167
+ classes = set()
168
+
169
+ # We need an empty PYTHONPATH if the env var is missing
170
+ os.environ["PYTHONPATH"] = os.getenv("PYTHONPATH", "")
171
+
172
+ # Recursively find all package modules
173
+ for _, module_str, _ in pkgutil.walk_packages(path=package.__path__, prefix=prefix, onerror=None):
174
+
175
+ # Don't instrument these modules, they have errors, maybe we should see why
176
+ if module_str in [
177
+ "opentelemetry.instrumentation.tortoiseorm",
178
+ "opentelemetry.instrumentation.auto_instrumentation.sitecustomize",
179
+ ]:
180
+ continue
181
+
182
+ # Import and find all module classes
183
+ __import__(module_str)
184
+ for _, _class in inspect.getmembers(sys.modules[module_str]):
185
+ if (not inspect.isclass(_class)) or (_class in classes):
186
+ continue
187
+
188
+ # Save the class (classes are found several times when imported by other modules)
189
+ classes.add(_class)
190
+
191
+ # Don't instrument these classes, they have errors, maybe we should see why
192
+ if _class in [AsyncioInstrumentor, AwsLambdaInstrumentor, BaseInstrumentor]:
193
+ continue
194
+
195
+ # If the "instrument" method exists, call it
196
+ _instrument = getattr(_class, "instrument", None)
197
+ if callable(_instrument):
198
+
199
+ _class_instance = _class()
200
+ if _class == RequestsInstrumentor and (trace_headers() or trace_body()):
201
+ _class_instance.instrument(
202
+ tracer_provider=otel_tracer,
203
+ request_hook=request_hook,
204
+ response_hook=response_hook,
205
+ )
206
+ elif not _class_instance.is_instrumented_by_opentelemetry:
207
+ _class_instance.instrument(tracer_provider=otel_tracer)
208
+ # name = f"{module_str}.{_class.__name__}".removeprefix(prefix)
209
+ # logger.debug(f"OpenTelemetry instrumentation of {name!r}")
210
+
211
+
212
+ @_agnosticcontextmanager
213
+ def start_span(
214
+ instrumenting_module_name: str,
215
+ name: str,
216
+ span_context: SpanContext | None = None,
217
+ ) -> Iterator[Span]:
218
+ """
219
+ Context manager for creating a new main or child OpenTelemetry span and set it
220
+ as the current span in this tracer's context.
221
+
222
+ Args:
223
+ instrumenting_module_name: Caller module name, just pass __name__
224
+ name: The name of the span to be created (use a custom name)
225
+ span_context: Parent span context. Only to create a child span.
226
+
227
+ Yields:
228
+ The newly-created span.
229
+ """
230
+ tracer = trace.get_tracer(instrumenting_module_name)
231
+
232
+ # Create a main span
233
+ if not span_context:
234
+ with tracer.start_as_current_span(name) as span:
235
+ yield span
236
+
237
+ # Create a child span
238
+ else:
239
+ main_span_context = SpanContext(
240
+ trace_id=span_context.trace_id,
241
+ span_id=span_context.span_id,
242
+ is_remote=True,
243
+ trace_flags=TraceFlags(TraceFlags.SAMPLED),
244
+ )
245
+ main_span = NonRecordingSpan(main_span_context)
246
+ with trace.use_span(main_span): # pylint: disable=not-context-manager
247
+ # Optionnaly, we could use the main span instead of creating
248
+ # a new one, to be discussed.
249
+ with tracer.start_as_current_span(name) as span:
250
+ yield span
rs_common/logging.py ADDED
@@ -0,0 +1,107 @@
1
+ # Copyright 2024 CS Group
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """Logging utility."""
16
+
17
+ import logging
18
+ import os
19
+ from multiprocessing import Queue
20
+ from threading import Lock
21
+
22
+
23
+ class Logging: # pylint: disable=too-few-public-methods
24
+ """
25
+ Logging utility.
26
+
27
+ Attributes:
28
+ lock: For code synchronization
29
+ level: Minimal log level to use for all new logging instances.
30
+ """
31
+
32
+ lock = Lock()
33
+ level = logging.DEBUG
34
+
35
+ @classmethod
36
+ def default(cls, name="rspy"):
37
+ """
38
+ Return a default Logger class instance.
39
+
40
+ Args:
41
+ name: Logger name. You can pass __name__ to use your current module name.
42
+ """
43
+ logger = logging.getLogger(name=name)
44
+
45
+ with cls.lock:
46
+ # Don't propagate to root logger
47
+ logger.propagate = False
48
+
49
+ # If we have already set the handlers for the logger with this name, do nothing more
50
+ if logger.hasHandlers():
51
+ return logger
52
+
53
+ # Set the minimal log level to use for all new logging instances.
54
+ logger.setLevel(cls.level)
55
+
56
+ # Create console handler
57
+ handler = logging.StreamHandler()
58
+ handler.setFormatter(CustomFormatter())
59
+ logger.addHandler(handler)
60
+
61
+ # Export logs to Loki, see: https://pypi.org/project/python-logging-loki/
62
+ loki_endpoint = os.getenv("LOKI_ENDPOINT")
63
+ if loki_endpoint:
64
+ import logging_loki # pylint: disable=import-outside-toplevel
65
+
66
+ handler = logging_loki.LokiQueueHandler(
67
+ Queue(-1),
68
+ url=loki_endpoint,
69
+ tags={"service": "rs.client"},
70
+ # auth=("username", "password"),
71
+ version="1",
72
+ )
73
+ handler.setFormatter(CustomFormatter())
74
+ logger.addHandler(handler)
75
+
76
+ return logger
77
+
78
+
79
+ class CustomFormatter(logging.Formatter):
80
+ """
81
+ Custom logging formatter with colored text.
82
+ See: https://stackoverflow.com/a/56944256
83
+ """
84
+
85
+ _RED = "\x1b[31m"
86
+ _BOLD_RED = "\x1b[31;1m"
87
+ _GREEN = "\x1b[32m"
88
+ _YELLOW = "\x1b[33m"
89
+ _PURPLE = "\x1b[35m"
90
+ _RESET = "\x1b[0m"
91
+
92
+ _FORMAT = f"%(asctime)s.%(msecs)03d [{{color}}%(levelname)s{_RESET}] (%(name)s) %(message)s"
93
+ _DATETIME = "%H:%M:%S"
94
+
95
+ _FORMATS = {
96
+ logging.NOTSET: _FORMAT.format(color=""),
97
+ logging.DEBUG: _FORMAT.format(color=_PURPLE),
98
+ logging.INFO: _FORMAT.format(color=_GREEN),
99
+ logging.WARNING: _FORMAT.format(color=_YELLOW),
100
+ logging.ERROR: _FORMAT.format(color=_BOLD_RED),
101
+ logging.CRITICAL: _FORMAT.format(color=_RED),
102
+ }
103
+
104
+ def format(self, record):
105
+ level_format = self._FORMATS.get(record.levelno)
106
+ formatter = logging.Formatter(level_format, self._DATETIME)
107
+ return formatter.format(record)