rs-client-libraries 0.2a9__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
rs_common/config.py ADDED
@@ -0,0 +1,62 @@
1
+ # Copyright 2024 CS Group
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """Configuration objects and functions."""
16
+
17
+ from enum import Enum
18
+
19
+ #############
20
+ # Constants #
21
+ #############
22
+
23
+ # datetime format used for HTTP requests
24
+ DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ"
25
+ DATETIME_FORMAT_MS = "%Y-%m-%dT%H:%M:%S.%fZ"
26
+
27
+ # name of the Auxip station, used in the same logic as the ECadipStation enum (see below)
28
+ AUXIP_STATION = "AUXIP"
29
+
30
+ #########
31
+ # Enums #
32
+ #########
33
+
34
+
35
+ class EPlatform(str, Enum):
36
+ """Platform enumeration."""
37
+
38
+ S1A = "S1A"
39
+ S2A = "S2A"
40
+ S2B = "S2B"
41
+ S3A = "S3A"
42
+ S3B = "S3B"
43
+
44
+
45
+ class ECadipStation(str, Enum):
46
+ """Cadip stations enumeration."""
47
+
48
+ CADIP = "CADIP"
49
+ INS = "INS"
50
+ MPS = "MPS"
51
+ MTI = "MTI"
52
+ NSG = "NSG"
53
+ SGS = "SGS"
54
+
55
+
56
+ class EDownloadStatus(str, Enum):
57
+ """Download status enumeration."""
58
+
59
+ NOT_STARTED = "NOT_STARTED"
60
+ IN_PROGRESS = "IN_PROGRESS"
61
+ FAILED = "FAILED"
62
+ DONE = "DONE"
rs_common/logging.py ADDED
@@ -0,0 +1,108 @@
1
+ # Copyright 2024 CS Group
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """Logging utility."""
16
+
17
+ import logging
18
+ import logging.handlers
19
+ import os
20
+ from multiprocessing import Queue
21
+ from threading import Lock
22
+
23
+ import logging_loki
24
+
25
+
26
+ class Logging: # pylint: disable=too-few-public-methods
27
+ """
28
+ Logging utility.
29
+
30
+ Attributes:
31
+ lock: For code synchronization
32
+ level: Minimal log level to use for all new logging instances.
33
+ """
34
+
35
+ lock = Lock()
36
+ level = logging.DEBUG
37
+
38
+ @classmethod
39
+ def default(cls, name="rspy"):
40
+ """
41
+ Return a default Logger class instance.
42
+
43
+ Args:
44
+ name: Logger name. You can pass __name__ to use your current module name.
45
+ """
46
+ logger = logging.getLogger(name=name)
47
+
48
+ with cls.lock:
49
+ # Don't propagate to root logger
50
+ logger.propagate = False
51
+
52
+ # If we have already set the handlers for the logger with this name, do nothing more
53
+ if logger.hasHandlers():
54
+ return logger
55
+
56
+ # Set the minimal log level to use for all new logging instances.
57
+ logger.setLevel(cls.level)
58
+
59
+ # Create console handler
60
+ handler = logging.StreamHandler()
61
+ handler.setFormatter(CustomFormatter())
62
+ logger.addHandler(handler)
63
+
64
+ # Export logs to Loki, see: https://pypi.org/project/python-logging-loki/
65
+ loki_endpoint = os.getenv("LOKI_ENDPOINT")
66
+ if loki_endpoint:
67
+ handler = logging_loki.LokiQueueHandler(
68
+ Queue(-1),
69
+ url=loki_endpoint,
70
+ tags={"service": "rs.client"},
71
+ # auth=("username", "password"),
72
+ version="1",
73
+ )
74
+ handler.setFormatter(CustomFormatter())
75
+ logger.addHandler(handler)
76
+
77
+ return logger
78
+
79
+
80
+ class CustomFormatter(logging.Formatter):
81
+ """
82
+ Custom logging formatter with colored text.
83
+ See: https://stackoverflow.com/a/56944256
84
+ """
85
+
86
+ _RED = "\x1b[31m"
87
+ _BOLD_RED = "\x1b[31;1m"
88
+ _GREEN = "\x1b[32m"
89
+ _YELLOW = "\x1b[33m"
90
+ _PURPLE = "\x1b[35m"
91
+ _RESET = "\x1b[0m"
92
+
93
+ _FORMAT = f"%(asctime)s.%(msecs)03d [{{color}}%(levelname)s{_RESET}] (%(name)s) %(message)s"
94
+ _DATETIME = "%H:%M:%S"
95
+
96
+ _FORMATS = {
97
+ logging.NOTSET: _FORMAT.format(color=""),
98
+ logging.DEBUG: _FORMAT.format(color=_PURPLE),
99
+ logging.INFO: _FORMAT.format(color=_GREEN),
100
+ logging.WARNING: _FORMAT.format(color=_YELLOW),
101
+ logging.ERROR: _FORMAT.format(color=_BOLD_RED),
102
+ logging.CRITICAL: _FORMAT.format(color=_RED),
103
+ }
104
+
105
+ def format(self, record):
106
+ level_format = self._FORMATS.get(record.levelno)
107
+ formatter = logging.Formatter(level_format, self._DATETIME)
108
+ return formatter.format(record)
@@ -0,0 +1,114 @@
1
+ # Copyright 2024 CS Group
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """OpenTelemetry utility"""
16
+
17
+ import inspect
18
+ import os
19
+ import pkgutil
20
+ import sys
21
+
22
+ import opentelemetry.instrumentation
23
+ from opentelemetry import trace
24
+ from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
25
+ from opentelemetry.instrumentation.asyncio import AsyncioInstrumentor
26
+ from opentelemetry.instrumentation.aws_lambda import AwsLambdaInstrumentor
27
+ from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor
28
+ from opentelemetry.instrumentation.instrumentor import BaseInstrumentor # type: ignore
29
+ from opentelemetry.sdk.resources import Resource
30
+ from opentelemetry.sdk.trace import TracerProvider
31
+ from opentelemetry.sdk.trace.export import BatchSpanProcessor
32
+
33
+ from rs_common.logging import Logging
34
+
35
+ logger = Logging.default(__name__)
36
+
37
+ FROM_PYTEST = False
38
+
39
+
40
+ def init_traces(service_name: str):
41
+ """
42
+ Init instrumentation of OpenTelemetry traces.
43
+
44
+ Args:
45
+ service_name (str): service name
46
+ """
47
+
48
+ # See: https://github.com/softwarebloat/python-tracing-demo/tree/main
49
+
50
+ # Don't call this line from pytest because it causes errors:
51
+ # Transient error StatusCode.UNAVAILABLE encountered while exporting metrics to localhost:4317, retrying in ..s.
52
+ if not FROM_PYTEST:
53
+ tempo_endpoint = os.getenv("TEMPO_ENDPOINT")
54
+ if not tempo_endpoint:
55
+ return
56
+
57
+ # TODO: to avoid errors in local mode:
58
+ # Transient error StatusCode.UNAVAILABLE encountered while exporting metrics to localhost:4317, retrying in ..s.
59
+ #
60
+ # The below line does not work either but at least we have less error messages.
61
+ # See: https://pforge-exchange2.astrium.eads.net/jira/browse/RSPY-221?focusedId=162092&
62
+ # page=com.atlassian.jira.plugin.system.issuetabpanels%3Acomment-tabpanel#comment-162092
63
+ #
64
+ # Now we have a single line error, which is less worst:
65
+ # Failed to export metrics to tempo:4317, error code: StatusCode.UNIMPLEMENTED
66
+ os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] = tempo_endpoint
67
+
68
+ otel_resource = Resource(attributes={"service.name": service_name})
69
+ otel_tracer = TracerProvider(resource=otel_resource)
70
+ trace.set_tracer_provider(otel_tracer)
71
+
72
+ if not FROM_PYTEST:
73
+ otel_tracer.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(endpoint=tempo_endpoint)))
74
+
75
+ # Instrument all the dependencies under opentelemetry.instrumentation.*
76
+ # NOTE: we need 'poetry run opentelemetry-bootstrap -a install' to install these.
77
+
78
+ package = opentelemetry.instrumentation
79
+ prefix = package.__name__ + "."
80
+ classes = set()
81
+
82
+ # We need an empty PYTHONPATH if the env var is missing
83
+ os.environ["PYTHONPATH"] = os.getenv("PYTHONPATH", "")
84
+
85
+ # Recursively find all package modules
86
+ for _, module_str, _ in pkgutil.walk_packages(path=package.__path__, prefix=prefix, onerror=None):
87
+ # Don't instrument these modules, they have errors, maybe we should see why
88
+ if module_str in [
89
+ "opentelemetry.instrumentation.tortoiseorm",
90
+ "opentelemetry.instrumentation.auto_instrumentation.sitecustomize",
91
+ ]:
92
+ continue
93
+
94
+ # Import and find all module classes
95
+ __import__(module_str)
96
+ for _, _class in inspect.getmembers(sys.modules[module_str]):
97
+ if (not inspect.isclass(_class)) or (_class in classes):
98
+ continue
99
+
100
+ # Save the class (classes are found several times when imported by other modules)
101
+ classes.add(_class)
102
+
103
+ # Don't instrument these classes, they have errors, maybe we should see why
104
+ if _class in [AsyncioInstrumentor, AwsLambdaInstrumentor, BaseInstrumentor, HTTPXClientInstrumentor]:
105
+ continue
106
+
107
+ # If the "instrument" method exists, call it
108
+ _instrument = getattr(_class, "instrument", None)
109
+ if callable(_instrument):
110
+ _class_instance = _class()
111
+ if not _class_instance.is_instrumented_by_opentelemetry:
112
+ _class_instance.instrument(tracer_provider=otel_tracer)
113
+ # name = f"{module_str}.{_class.__name__}".removeprefix(prefix)
114
+ # logger.debug(f"OpenTelemetry instrumentation of {name!r}")
@@ -0,0 +1,86 @@
1
+ # Copyright 2024 CS Group
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """(De-)Serialization utility functions for the workflows."""
16
+
17
+ from datetime import datetime, timezone
18
+ from typing import Type, cast
19
+
20
+ from rs_client.auxip_client import AuxipClient
21
+ from rs_client.cadip_client import CadipClient
22
+ from rs_client.rs_client import RsClient
23
+ from rs_client.stac_client import StacClient
24
+ from rs_common.config import DATETIME_FORMAT, ECadipStation
25
+
26
+
27
+ class RsClientSerialization: # pylint: disable=too-few-public-methods
28
+ """
29
+ We can't pass a RsClient instance to the workflow because it causes (de-)serialization issues.
30
+ So instead we pass the instance parameters, that will be used to recreate a new instance
31
+ from the workflow.
32
+
33
+ TODO: also (de-)serialize the pystac_client.Client.open(...) parameters ?
34
+
35
+ Attributes:
36
+ cls (Class): RsClient child class type
37
+ rs_server_href (str): RS-Server URL. In local mode, pass None.
38
+ rs_server_api_key (str): API key for RS-Server authentication.
39
+ owner_id (str): ID of the owner of the STAC catalog collections (no special characters allowoed).
40
+ station (ECadipStation): Cadip station (if applicable)
41
+
42
+ """
43
+
44
+ def __init__(self, client: RsClient):
45
+ """
46
+ Serialize a RsClient instance.
47
+
48
+ Args:
49
+ client (RsClient): RsClient instance
50
+ """
51
+
52
+ # Save the parameters, except the logging (which should not be (de-)serialized neither.
53
+ self.cls: Type[RsClient] = type(client)
54
+ self.rs_server_href: str | None = client.rs_server_href
55
+ self.rs_server_api_key: str | None = client.rs_server_api_key
56
+ self.owner_id: str | None = client.owner_id
57
+
58
+ self.station: ECadipStation | None = None
59
+ if self.cls == CadipClient:
60
+ self.station = cast(CadipClient, client).station
61
+
62
+ def deserialize(self, logger=None) -> RsClient:
63
+ """
64
+ Recreate a new RsClient instance from the serialized parameters.
65
+
66
+ Return:
67
+ client (RsClient): RsClient instance
68
+ logger (logging.Logger): Logging instance.
69
+ """
70
+
71
+ # Init parent class
72
+ client = RsClient(self.rs_server_href, self.rs_server_api_key, self.owner_id, logger)
73
+
74
+ # Return child instance
75
+ if self.cls == AuxipClient:
76
+ return client.get_auxip_client()
77
+ if self.cls == CadipClient:
78
+ return client.get_cadip_client(self.station) # type: ignore
79
+ if self.cls == StacClient:
80
+ return client.get_stac_client()
81
+ raise ValueError(f"Unknown RsClient type: {self.cls}")
82
+
83
+ @property
84
+ def get_flow_name(self):
85
+ """Return a unique flow name for Prefect."""
86
+ return f"{self.owner_id}_{datetime.now(timezone.utc).strftime(DATETIME_FORMAT)}"
rs_common/utils.py ADDED
@@ -0,0 +1,46 @@
1
+ # Copyright 2024 CS Group
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """This module is used to share common functions between apis"""
16
+
17
+ from dataclasses import dataclass
18
+
19
+
20
+ @dataclass
21
+ class AuthInfo:
22
+ """User authentication information in KeyCloak."""
23
+
24
+ # User login (preferred username)
25
+ user_login: str
26
+
27
+ # IAM roles
28
+ iam_roles: list[str]
29
+
30
+ # Configuration associated to the API key (not implemented for now)
31
+ apikey_config: dict
32
+
33
+
34
+ def read_response_error(response):
35
+ """Read and return an HTTP response error detail."""
36
+
37
+ # Try to read the response detail or error
38
+ try:
39
+ json = response.json()
40
+ detail = json.get("detail") or json["error"]
41
+
42
+ # If this fail, get the full response content
43
+ except Exception: # pylint: disable=broad-exception-caught
44
+ detail = response.content.decode("utf-8", errors="ignore")
45
+
46
+ return detail
@@ -0,0 +1,18 @@
1
+ # Copyright 2024 CS Group
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """rs-workflows main module."""
16
+
17
+ # Set automatically by running `poetry dynamic-versioning`
18
+ __version__ = "0.2a9"
@@ -0,0 +1,43 @@
1
+ # Copyright 2024 CS Group
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ general_configuration:
16
+ logging:
17
+ level: DEBUG
18
+ triggering__validate_run: true
19
+ triggering__use_default_filename: true
20
+ triggering__use_basic_logging: true
21
+ triggering__load_default_logging: false
22
+ breakpoints:
23
+ workflow:
24
+ - step: 1
25
+ active: true
26
+ module: rs.dpr.mockup # string corresponding to the python path of the module
27
+ processing_unit: DprMockupProcessor # EOProcessingUnit class name
28
+ name: DprMockupProcessor # identifier for the processing unit
29
+ inputs:
30
+ -
31
+ outputs:
32
+ -
33
+ parameters:
34
+ product_types: # List of EOPF product types we want to generate. In this example we simulate S1L0 processor that generates 4 products
35
+ -
36
+ I/O:
37
+ inputs_products:
38
+ -
39
+ output_products:
40
+ -
41
+ dask_context: {}
42
+ logging: {}
43
+ config: {}
@@ -0,0 +1,43 @@
1
+ # Copyright 2024 CS Group
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """Prefect flow example for rs-client-libraries"""
16
+ from prefect import flow, task
17
+
18
+
19
+ @task
20
+ def print_hello(name: str):
21
+ """Example task to be used in a prefect flow
22
+
23
+ This prefect task may be used as start point in creating your own prefect tasks
24
+
25
+ Args:
26
+ name (str): Username to be printed
27
+ """
28
+ print(f"Hello {name}!")
29
+
30
+
31
+ @flow(name="Hello RS-Server flow ")
32
+ def hello_world(name="COPERNICUS", tasks_number=2):
33
+ """Example flow that can be use in a COPERNICUS chain
34
+
35
+ This prefect flow may be used as start point in creating your own prefect flows. It runs in parallel
36
+ tasks_number of print_hello
37
+
38
+ Args:
39
+ name (str): Username to be printed. Default COPERNICUS
40
+ tasks_number (int): Number of tasks to be run. Default 2
41
+ """
42
+ for idx in range(0, tasks_number):
43
+ print_hello(name + "_" + str(idx))