rs-client-libraries 0.2a9__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rs_client/__init__.py +18 -0
- rs_client/auxip_client.py +61 -0
- rs_client/cadip_client.py +142 -0
- rs_client/rs_client.py +433 -0
- rs_client/stac_client.py +391 -0
- rs_client_libraries-0.2a9.dist-info/LICENSE +201 -0
- rs_client_libraries-0.2a9.dist-info/METADATA +42 -0
- rs_client_libraries-0.2a9.dist-info/RECORD +20 -0
- rs_client_libraries-0.2a9.dist-info/WHEEL +4 -0
- rs_common/__init__.py +20 -0
- rs_common/config.py +62 -0
- rs_common/logging.py +108 -0
- rs_common/opentelemetry.py +114 -0
- rs_common/serialization.py +86 -0
- rs_common/utils.py +46 -0
- rs_workflows/__init__.py +18 -0
- rs_workflows/config/dpr_config_template.yaml +43 -0
- rs_workflows/example.py +43 -0
- rs_workflows/s1_l0.py +497 -0
- rs_workflows/staging.py +511 -0
rs_common/config.py
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
# Copyright 2024 CS Group
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""Configuration objects and functions."""
|
|
16
|
+
|
|
17
|
+
from enum import Enum
|
|
18
|
+
|
|
19
|
+
#############
|
|
20
|
+
# Constants #
|
|
21
|
+
#############
|
|
22
|
+
|
|
23
|
+
# datetime format used for HTTP requests
|
|
24
|
+
DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ"
|
|
25
|
+
DATETIME_FORMAT_MS = "%Y-%m-%dT%H:%M:%S.%fZ"
|
|
26
|
+
|
|
27
|
+
# name of the Auxip station, used in the same logic as the ECadipStation enum (see below)
|
|
28
|
+
AUXIP_STATION = "AUXIP"
|
|
29
|
+
|
|
30
|
+
#########
|
|
31
|
+
# Enums #
|
|
32
|
+
#########
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class EPlatform(str, Enum):
|
|
36
|
+
"""Platform enumeration."""
|
|
37
|
+
|
|
38
|
+
S1A = "S1A"
|
|
39
|
+
S2A = "S2A"
|
|
40
|
+
S2B = "S2B"
|
|
41
|
+
S3A = "S3A"
|
|
42
|
+
S3B = "S3B"
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class ECadipStation(str, Enum):
|
|
46
|
+
"""Cadip stations enumeration."""
|
|
47
|
+
|
|
48
|
+
CADIP = "CADIP"
|
|
49
|
+
INS = "INS"
|
|
50
|
+
MPS = "MPS"
|
|
51
|
+
MTI = "MTI"
|
|
52
|
+
NSG = "NSG"
|
|
53
|
+
SGS = "SGS"
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class EDownloadStatus(str, Enum):
|
|
57
|
+
"""Download status enumeration."""
|
|
58
|
+
|
|
59
|
+
NOT_STARTED = "NOT_STARTED"
|
|
60
|
+
IN_PROGRESS = "IN_PROGRESS"
|
|
61
|
+
FAILED = "FAILED"
|
|
62
|
+
DONE = "DONE"
|
rs_common/logging.py
ADDED
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
# Copyright 2024 CS Group
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""Logging utility."""
|
|
16
|
+
|
|
17
|
+
import logging
|
|
18
|
+
import logging.handlers
|
|
19
|
+
import os
|
|
20
|
+
from multiprocessing import Queue
|
|
21
|
+
from threading import Lock
|
|
22
|
+
|
|
23
|
+
import logging_loki
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class Logging: # pylint: disable=too-few-public-methods
|
|
27
|
+
"""
|
|
28
|
+
Logging utility.
|
|
29
|
+
|
|
30
|
+
Attributes:
|
|
31
|
+
lock: For code synchronization
|
|
32
|
+
level: Minimal log level to use for all new logging instances.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
lock = Lock()
|
|
36
|
+
level = logging.DEBUG
|
|
37
|
+
|
|
38
|
+
@classmethod
|
|
39
|
+
def default(cls, name="rspy"):
|
|
40
|
+
"""
|
|
41
|
+
Return a default Logger class instance.
|
|
42
|
+
|
|
43
|
+
Args:
|
|
44
|
+
name: Logger name. You can pass __name__ to use your current module name.
|
|
45
|
+
"""
|
|
46
|
+
logger = logging.getLogger(name=name)
|
|
47
|
+
|
|
48
|
+
with cls.lock:
|
|
49
|
+
# Don't propagate to root logger
|
|
50
|
+
logger.propagate = False
|
|
51
|
+
|
|
52
|
+
# If we have already set the handlers for the logger with this name, do nothing more
|
|
53
|
+
if logger.hasHandlers():
|
|
54
|
+
return logger
|
|
55
|
+
|
|
56
|
+
# Set the minimal log level to use for all new logging instances.
|
|
57
|
+
logger.setLevel(cls.level)
|
|
58
|
+
|
|
59
|
+
# Create console handler
|
|
60
|
+
handler = logging.StreamHandler()
|
|
61
|
+
handler.setFormatter(CustomFormatter())
|
|
62
|
+
logger.addHandler(handler)
|
|
63
|
+
|
|
64
|
+
# Export logs to Loki, see: https://pypi.org/project/python-logging-loki/
|
|
65
|
+
loki_endpoint = os.getenv("LOKI_ENDPOINT")
|
|
66
|
+
if loki_endpoint:
|
|
67
|
+
handler = logging_loki.LokiQueueHandler(
|
|
68
|
+
Queue(-1),
|
|
69
|
+
url=loki_endpoint,
|
|
70
|
+
tags={"service": "rs.client"},
|
|
71
|
+
# auth=("username", "password"),
|
|
72
|
+
version="1",
|
|
73
|
+
)
|
|
74
|
+
handler.setFormatter(CustomFormatter())
|
|
75
|
+
logger.addHandler(handler)
|
|
76
|
+
|
|
77
|
+
return logger
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class CustomFormatter(logging.Formatter):
|
|
81
|
+
"""
|
|
82
|
+
Custom logging formatter with colored text.
|
|
83
|
+
See: https://stackoverflow.com/a/56944256
|
|
84
|
+
"""
|
|
85
|
+
|
|
86
|
+
_RED = "\x1b[31m"
|
|
87
|
+
_BOLD_RED = "\x1b[31;1m"
|
|
88
|
+
_GREEN = "\x1b[32m"
|
|
89
|
+
_YELLOW = "\x1b[33m"
|
|
90
|
+
_PURPLE = "\x1b[35m"
|
|
91
|
+
_RESET = "\x1b[0m"
|
|
92
|
+
|
|
93
|
+
_FORMAT = f"%(asctime)s.%(msecs)03d [{{color}}%(levelname)s{_RESET}] (%(name)s) %(message)s"
|
|
94
|
+
_DATETIME = "%H:%M:%S"
|
|
95
|
+
|
|
96
|
+
_FORMATS = {
|
|
97
|
+
logging.NOTSET: _FORMAT.format(color=""),
|
|
98
|
+
logging.DEBUG: _FORMAT.format(color=_PURPLE),
|
|
99
|
+
logging.INFO: _FORMAT.format(color=_GREEN),
|
|
100
|
+
logging.WARNING: _FORMAT.format(color=_YELLOW),
|
|
101
|
+
logging.ERROR: _FORMAT.format(color=_BOLD_RED),
|
|
102
|
+
logging.CRITICAL: _FORMAT.format(color=_RED),
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
def format(self, record):
|
|
106
|
+
level_format = self._FORMATS.get(record.levelno)
|
|
107
|
+
formatter = logging.Formatter(level_format, self._DATETIME)
|
|
108
|
+
return formatter.format(record)
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
# Copyright 2024 CS Group
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""OpenTelemetry utility"""
|
|
16
|
+
|
|
17
|
+
import inspect
|
|
18
|
+
import os
|
|
19
|
+
import pkgutil
|
|
20
|
+
import sys
|
|
21
|
+
|
|
22
|
+
import opentelemetry.instrumentation
|
|
23
|
+
from opentelemetry import trace
|
|
24
|
+
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
|
|
25
|
+
from opentelemetry.instrumentation.asyncio import AsyncioInstrumentor
|
|
26
|
+
from opentelemetry.instrumentation.aws_lambda import AwsLambdaInstrumentor
|
|
27
|
+
from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor
|
|
28
|
+
from opentelemetry.instrumentation.instrumentor import BaseInstrumentor # type: ignore
|
|
29
|
+
from opentelemetry.sdk.resources import Resource
|
|
30
|
+
from opentelemetry.sdk.trace import TracerProvider
|
|
31
|
+
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
|
32
|
+
|
|
33
|
+
from rs_common.logging import Logging
|
|
34
|
+
|
|
35
|
+
logger = Logging.default(__name__)
|
|
36
|
+
|
|
37
|
+
FROM_PYTEST = False
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def init_traces(service_name: str):
|
|
41
|
+
"""
|
|
42
|
+
Init instrumentation of OpenTelemetry traces.
|
|
43
|
+
|
|
44
|
+
Args:
|
|
45
|
+
service_name (str): service name
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
# See: https://github.com/softwarebloat/python-tracing-demo/tree/main
|
|
49
|
+
|
|
50
|
+
# Don't call this line from pytest because it causes errors:
|
|
51
|
+
# Transient error StatusCode.UNAVAILABLE encountered while exporting metrics to localhost:4317, retrying in ..s.
|
|
52
|
+
if not FROM_PYTEST:
|
|
53
|
+
tempo_endpoint = os.getenv("TEMPO_ENDPOINT")
|
|
54
|
+
if not tempo_endpoint:
|
|
55
|
+
return
|
|
56
|
+
|
|
57
|
+
# TODO: to avoid errors in local mode:
|
|
58
|
+
# Transient error StatusCode.UNAVAILABLE encountered while exporting metrics to localhost:4317, retrying in ..s.
|
|
59
|
+
#
|
|
60
|
+
# The below line does not work either but at least we have less error messages.
|
|
61
|
+
# See: https://pforge-exchange2.astrium.eads.net/jira/browse/RSPY-221?focusedId=162092&
|
|
62
|
+
# page=com.atlassian.jira.plugin.system.issuetabpanels%3Acomment-tabpanel#comment-162092
|
|
63
|
+
#
|
|
64
|
+
# Now we have a single line error, which is less worst:
|
|
65
|
+
# Failed to export metrics to tempo:4317, error code: StatusCode.UNIMPLEMENTED
|
|
66
|
+
os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] = tempo_endpoint
|
|
67
|
+
|
|
68
|
+
otel_resource = Resource(attributes={"service.name": service_name})
|
|
69
|
+
otel_tracer = TracerProvider(resource=otel_resource)
|
|
70
|
+
trace.set_tracer_provider(otel_tracer)
|
|
71
|
+
|
|
72
|
+
if not FROM_PYTEST:
|
|
73
|
+
otel_tracer.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(endpoint=tempo_endpoint)))
|
|
74
|
+
|
|
75
|
+
# Instrument all the dependencies under opentelemetry.instrumentation.*
|
|
76
|
+
# NOTE: we need 'poetry run opentelemetry-bootstrap -a install' to install these.
|
|
77
|
+
|
|
78
|
+
package = opentelemetry.instrumentation
|
|
79
|
+
prefix = package.__name__ + "."
|
|
80
|
+
classes = set()
|
|
81
|
+
|
|
82
|
+
# We need an empty PYTHONPATH if the env var is missing
|
|
83
|
+
os.environ["PYTHONPATH"] = os.getenv("PYTHONPATH", "")
|
|
84
|
+
|
|
85
|
+
# Recursively find all package modules
|
|
86
|
+
for _, module_str, _ in pkgutil.walk_packages(path=package.__path__, prefix=prefix, onerror=None):
|
|
87
|
+
# Don't instrument these modules, they have errors, maybe we should see why
|
|
88
|
+
if module_str in [
|
|
89
|
+
"opentelemetry.instrumentation.tortoiseorm",
|
|
90
|
+
"opentelemetry.instrumentation.auto_instrumentation.sitecustomize",
|
|
91
|
+
]:
|
|
92
|
+
continue
|
|
93
|
+
|
|
94
|
+
# Import and find all module classes
|
|
95
|
+
__import__(module_str)
|
|
96
|
+
for _, _class in inspect.getmembers(sys.modules[module_str]):
|
|
97
|
+
if (not inspect.isclass(_class)) or (_class in classes):
|
|
98
|
+
continue
|
|
99
|
+
|
|
100
|
+
# Save the class (classes are found several times when imported by other modules)
|
|
101
|
+
classes.add(_class)
|
|
102
|
+
|
|
103
|
+
# Don't instrument these classes, they have errors, maybe we should see why
|
|
104
|
+
if _class in [AsyncioInstrumentor, AwsLambdaInstrumentor, BaseInstrumentor, HTTPXClientInstrumentor]:
|
|
105
|
+
continue
|
|
106
|
+
|
|
107
|
+
# If the "instrument" method exists, call it
|
|
108
|
+
_instrument = getattr(_class, "instrument", None)
|
|
109
|
+
if callable(_instrument):
|
|
110
|
+
_class_instance = _class()
|
|
111
|
+
if not _class_instance.is_instrumented_by_opentelemetry:
|
|
112
|
+
_class_instance.instrument(tracer_provider=otel_tracer)
|
|
113
|
+
# name = f"{module_str}.{_class.__name__}".removeprefix(prefix)
|
|
114
|
+
# logger.debug(f"OpenTelemetry instrumentation of {name!r}")
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
# Copyright 2024 CS Group
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""(De-)Serialization utility functions for the workflows."""
|
|
16
|
+
|
|
17
|
+
from datetime import datetime, timezone
|
|
18
|
+
from typing import Type, cast
|
|
19
|
+
|
|
20
|
+
from rs_client.auxip_client import AuxipClient
|
|
21
|
+
from rs_client.cadip_client import CadipClient
|
|
22
|
+
from rs_client.rs_client import RsClient
|
|
23
|
+
from rs_client.stac_client import StacClient
|
|
24
|
+
from rs_common.config import DATETIME_FORMAT, ECadipStation
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class RsClientSerialization: # pylint: disable=too-few-public-methods
|
|
28
|
+
"""
|
|
29
|
+
We can't pass a RsClient instance to the workflow because it causes (de-)serialization issues.
|
|
30
|
+
So instead we pass the instance parameters, that will be used to recreate a new instance
|
|
31
|
+
from the workflow.
|
|
32
|
+
|
|
33
|
+
TODO: also (de-)serialize the pystac_client.Client.open(...) parameters ?
|
|
34
|
+
|
|
35
|
+
Attributes:
|
|
36
|
+
cls (Class): RsClient child class type
|
|
37
|
+
rs_server_href (str): RS-Server URL. In local mode, pass None.
|
|
38
|
+
rs_server_api_key (str): API key for RS-Server authentication.
|
|
39
|
+
owner_id (str): ID of the owner of the STAC catalog collections (no special characters allowoed).
|
|
40
|
+
station (ECadipStation): Cadip station (if applicable)
|
|
41
|
+
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
def __init__(self, client: RsClient):
|
|
45
|
+
"""
|
|
46
|
+
Serialize a RsClient instance.
|
|
47
|
+
|
|
48
|
+
Args:
|
|
49
|
+
client (RsClient): RsClient instance
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
# Save the parameters, except the logging (which should not be (de-)serialized neither.
|
|
53
|
+
self.cls: Type[RsClient] = type(client)
|
|
54
|
+
self.rs_server_href: str | None = client.rs_server_href
|
|
55
|
+
self.rs_server_api_key: str | None = client.rs_server_api_key
|
|
56
|
+
self.owner_id: str | None = client.owner_id
|
|
57
|
+
|
|
58
|
+
self.station: ECadipStation | None = None
|
|
59
|
+
if self.cls == CadipClient:
|
|
60
|
+
self.station = cast(CadipClient, client).station
|
|
61
|
+
|
|
62
|
+
def deserialize(self, logger=None) -> RsClient:
|
|
63
|
+
"""
|
|
64
|
+
Recreate a new RsClient instance from the serialized parameters.
|
|
65
|
+
|
|
66
|
+
Return:
|
|
67
|
+
client (RsClient): RsClient instance
|
|
68
|
+
logger (logging.Logger): Logging instance.
|
|
69
|
+
"""
|
|
70
|
+
|
|
71
|
+
# Init parent class
|
|
72
|
+
client = RsClient(self.rs_server_href, self.rs_server_api_key, self.owner_id, logger)
|
|
73
|
+
|
|
74
|
+
# Return child instance
|
|
75
|
+
if self.cls == AuxipClient:
|
|
76
|
+
return client.get_auxip_client()
|
|
77
|
+
if self.cls == CadipClient:
|
|
78
|
+
return client.get_cadip_client(self.station) # type: ignore
|
|
79
|
+
if self.cls == StacClient:
|
|
80
|
+
return client.get_stac_client()
|
|
81
|
+
raise ValueError(f"Unknown RsClient type: {self.cls}")
|
|
82
|
+
|
|
83
|
+
@property
|
|
84
|
+
def get_flow_name(self):
|
|
85
|
+
"""Return a unique flow name for Prefect."""
|
|
86
|
+
return f"{self.owner_id}_{datetime.now(timezone.utc).strftime(DATETIME_FORMAT)}"
|
rs_common/utils.py
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# Copyright 2024 CS Group
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""This module is used to share common functions between apis"""
|
|
16
|
+
|
|
17
|
+
from dataclasses import dataclass
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass
|
|
21
|
+
class AuthInfo:
|
|
22
|
+
"""User authentication information in KeyCloak."""
|
|
23
|
+
|
|
24
|
+
# User login (preferred username)
|
|
25
|
+
user_login: str
|
|
26
|
+
|
|
27
|
+
# IAM roles
|
|
28
|
+
iam_roles: list[str]
|
|
29
|
+
|
|
30
|
+
# Configuration associated to the API key (not implemented for now)
|
|
31
|
+
apikey_config: dict
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def read_response_error(response):
|
|
35
|
+
"""Read and return an HTTP response error detail."""
|
|
36
|
+
|
|
37
|
+
# Try to read the response detail or error
|
|
38
|
+
try:
|
|
39
|
+
json = response.json()
|
|
40
|
+
detail = json.get("detail") or json["error"]
|
|
41
|
+
|
|
42
|
+
# If this fail, get the full response content
|
|
43
|
+
except Exception: # pylint: disable=broad-exception-caught
|
|
44
|
+
detail = response.content.decode("utf-8", errors="ignore")
|
|
45
|
+
|
|
46
|
+
return detail
|
rs_workflows/__init__.py
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# Copyright 2024 CS Group
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""rs-workflows main module."""
|
|
16
|
+
|
|
17
|
+
# Set automatically by running `poetry dynamic-versioning`
|
|
18
|
+
__version__ = "0.2a9"
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# Copyright 2024 CS Group
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
general_configuration:
|
|
16
|
+
logging:
|
|
17
|
+
level: DEBUG
|
|
18
|
+
triggering__validate_run: true
|
|
19
|
+
triggering__use_default_filename: true
|
|
20
|
+
triggering__use_basic_logging: true
|
|
21
|
+
triggering__load_default_logging: false
|
|
22
|
+
breakpoints:
|
|
23
|
+
workflow:
|
|
24
|
+
- step: 1
|
|
25
|
+
active: true
|
|
26
|
+
module: rs.dpr.mockup # string corresponding to the python path of the module
|
|
27
|
+
processing_unit: DprMockupProcessor # EOProcessingUnit class name
|
|
28
|
+
name: DprMockupProcessor # identifier for the processing unit
|
|
29
|
+
inputs:
|
|
30
|
+
-
|
|
31
|
+
outputs:
|
|
32
|
+
-
|
|
33
|
+
parameters:
|
|
34
|
+
product_types: # List of EOPF product types we want to generate. In this example we simulate S1L0 processor that generates 4 products
|
|
35
|
+
-
|
|
36
|
+
I/O:
|
|
37
|
+
inputs_products:
|
|
38
|
+
-
|
|
39
|
+
output_products:
|
|
40
|
+
-
|
|
41
|
+
dask_context: {}
|
|
42
|
+
logging: {}
|
|
43
|
+
config: {}
|
rs_workflows/example.py
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# Copyright 2024 CS Group
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""Prefect flow example for rs-client-libraries"""
|
|
16
|
+
from prefect import flow, task
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@task
|
|
20
|
+
def print_hello(name: str):
|
|
21
|
+
"""Example task to be used in a prefect flow
|
|
22
|
+
|
|
23
|
+
This prefect task may be used as start point in creating your own prefect tasks
|
|
24
|
+
|
|
25
|
+
Args:
|
|
26
|
+
name (str): Username to be printed
|
|
27
|
+
"""
|
|
28
|
+
print(f"Hello {name}!")
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@flow(name="Hello RS-Server flow ")
|
|
32
|
+
def hello_world(name="COPERNICUS", tasks_number=2):
|
|
33
|
+
"""Example flow that can be use in a COPERNICUS chain
|
|
34
|
+
|
|
35
|
+
This prefect flow may be used as start point in creating your own prefect flows. It runs in parallel
|
|
36
|
+
tasks_number of print_hello
|
|
37
|
+
|
|
38
|
+
Args:
|
|
39
|
+
name (str): Username to be printed. Default COPERNICUS
|
|
40
|
+
tasks_number (int): Number of tasks to be run. Default 2
|
|
41
|
+
"""
|
|
42
|
+
for idx in range(0, tasks_number):
|
|
43
|
+
print_hello(name + "_" + str(idx))
|