rs-client-libraries 0.2a9__tar.gz → 0.2a10__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {rs_client_libraries-0.2a9 → rs_client_libraries-0.2a10}/PKG-INFO +10 -13
- {rs_client_libraries-0.2a9 → rs_client_libraries-0.2a10}/pyproject.toml +31 -34
- {rs_client_libraries-0.2a9 → rs_client_libraries-0.2a10}/rs_client/__init__.py +1 -1
- {rs_client_libraries-0.2a9 → rs_client_libraries-0.2a10}/rs_client/rs_client.py +13 -7
- rs_client_libraries-0.2a10/rs_client/staging_client.py +311 -0
- {rs_client_libraries-0.2a9 → rs_client_libraries-0.2a10}/rs_common/opentelemetry.py +1 -2
- rs_client_libraries-0.2a9/rs_workflows/__init__.py +0 -18
- rs_client_libraries-0.2a9/rs_workflows/config/dpr_config_template.yaml +0 -43
- rs_client_libraries-0.2a9/rs_workflows/example.py +0 -43
- rs_client_libraries-0.2a9/rs_workflows/s1_l0.py +0 -497
- rs_client_libraries-0.2a9/rs_workflows/staging.py +0 -511
- {rs_client_libraries-0.2a9 → rs_client_libraries-0.2a10}/LICENSE +0 -0
- {rs_client_libraries-0.2a9 → rs_client_libraries-0.2a10}/README.adoc +0 -0
- {rs_client_libraries-0.2a9 → rs_client_libraries-0.2a10}/rs_client/auxip_client.py +0 -0
- {rs_client_libraries-0.2a9 → rs_client_libraries-0.2a10}/rs_client/cadip_client.py +0 -0
- {rs_client_libraries-0.2a9 → rs_client_libraries-0.2a10}/rs_client/stac_client.py +0 -0
- {rs_client_libraries-0.2a9 → rs_client_libraries-0.2a10}/rs_common/__init__.py +0 -0
- {rs_client_libraries-0.2a9 → rs_client_libraries-0.2a10}/rs_common/config.py +0 -0
- {rs_client_libraries-0.2a9 → rs_client_libraries-0.2a10}/rs_common/logging.py +0 -0
- {rs_client_libraries-0.2a9 → rs_client_libraries-0.2a10}/rs_common/serialization.py +0 -0
- {rs_client_libraries-0.2a9 → rs_client_libraries-0.2a10}/rs_common/utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: rs-client-libraries
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2a10
|
|
4
4
|
Summary: Copernicus Sentinel rs-client library and processing workflows.
|
|
5
5
|
Home-page: https://home.rs-python.eu/
|
|
6
6
|
License: Apache-2.0
|
|
@@ -19,20 +19,17 @@ Classifier: Programming Language :: Python :: 3.12
|
|
|
19
19
|
Classifier: Programming Language :: Python :: 3.13
|
|
20
20
|
Classifier: Topic :: Scientific/Engineering :: Image Processing
|
|
21
21
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
22
|
-
Requires-Dist: cachetools (>=5.
|
|
23
|
-
Requires-Dist:
|
|
24
|
-
Requires-Dist: opentelemetry-distro (>=0.
|
|
25
|
-
Requires-Dist: opentelemetry-exporter-otlp (>=1.
|
|
26
|
-
Requires-Dist: opentelemetry-instrumentation-aws-lambda (>=0.
|
|
27
|
-
Requires-Dist: opentelemetry-test-utils (>=0.
|
|
28
|
-
Requires-Dist:
|
|
29
|
-
Requires-Dist:
|
|
30
|
-
Requires-Dist: pystac (>=1.10.1,<2.0.0)
|
|
31
|
-
Requires-Dist: pystac-client (>=0.8.1,<0.9.0)
|
|
22
|
+
Requires-Dist: cachetools (>=5.5.1,<6.0.0)
|
|
23
|
+
Requires-Dist: openapi-core (>=0.19.4,<0.20.0)
|
|
24
|
+
Requires-Dist: opentelemetry-distro (>=0.51b0,<0.52)
|
|
25
|
+
Requires-Dist: opentelemetry-exporter-otlp (>=1.30.0,<2.0.0)
|
|
26
|
+
Requires-Dist: opentelemetry-instrumentation-aws-lambda (>=0.51b0,<0.52)
|
|
27
|
+
Requires-Dist: opentelemetry-test-utils (>=0.51b0,<0.52)
|
|
28
|
+
Requires-Dist: pystac (>=1.12.1,<2.0.0)
|
|
29
|
+
Requires-Dist: pystac-client (>=0.8.5,<0.9.0)
|
|
32
30
|
Requires-Dist: python-dateutil (>=2.9.0.post0,<3.0.0)
|
|
33
31
|
Requires-Dist: python-logging-loki (>=0.3.1,<0.4.0)
|
|
34
|
-
Requires-Dist:
|
|
35
|
-
Requires-Dist: starlette (>=0.38.4,<0.39.0)
|
|
32
|
+
Requires-Dist: stac-pydantic (>=3.1.4,<4.0.0)
|
|
36
33
|
Project-URL: Documentation, https://home.rs-python.eu/rs-documentation/rs-client-libraries/docs/doc/
|
|
37
34
|
Project-URL: Repository, https://github.com/RS-PYTHON/rs-client-libraries
|
|
38
35
|
Description-Content-Type: text/plain
|
|
@@ -4,7 +4,7 @@ version = "0.0.0" # see: https://github.com/mtkennerly/poetry-dynamic-versioning
|
|
|
4
4
|
|
|
5
5
|
[tool.poetry]
|
|
6
6
|
name = "rs-client-libraries"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.2a10"
|
|
8
8
|
description = "Copernicus Sentinel rs-client library and processing workflows."
|
|
9
9
|
license = "Apache-2.0"
|
|
10
10
|
authors = ["ESA under contract with Airbus and CS <ops_coprs@airbus.com>"]
|
|
@@ -27,55 +27,52 @@ classifiers = [
|
|
|
27
27
|
packages = [
|
|
28
28
|
{ include = "rs_client" },
|
|
29
29
|
{ include = "rs_common" },
|
|
30
|
-
{ include = "rs_workflows" },
|
|
31
30
|
]
|
|
32
31
|
|
|
33
|
-
#
|
|
34
|
-
|
|
32
|
+
# Add configuration files to the .whl package
|
|
33
|
+
include = [
|
|
34
|
+
{ path = "config", format = ["wheel"] }
|
|
35
|
+
]
|
|
35
36
|
|
|
36
|
-
# All the dependencies for release are set in the sub-projects
|
|
37
37
|
[tool.poetry.dependencies]
|
|
38
38
|
python = "^3.11"
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
opentelemetry-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
cachetools = "^5.3.3"
|
|
48
|
-
pystac = "^1.10.1"
|
|
49
|
-
pystac-client = "^0.8.1"
|
|
50
|
-
starlette = "^0.38.4"
|
|
39
|
+
opentelemetry-distro = "^0.51b0" # then run 'poetry run opentelemetry-bootstrap -a install'
|
|
40
|
+
opentelemetry-instrumentation-aws-lambda = "^0.51b0"
|
|
41
|
+
opentelemetry-test-utils = "^0.51b0"
|
|
42
|
+
opentelemetry-exporter-otlp = "^1.30.0"
|
|
43
|
+
python-logging-loki = "^0.3.1" # to avoid otl error: No module named 'logging_loki'
|
|
44
|
+
cachetools = "^5.5.1"
|
|
45
|
+
pystac = "^1.12.1"
|
|
46
|
+
pystac-client = "^0.8.5"
|
|
51
47
|
python-dateutil = "^2.9.0.post0"
|
|
52
|
-
|
|
48
|
+
stac-pydantic = "^3.1.4"
|
|
49
|
+
openapi-core = "^0.19.4"
|
|
53
50
|
|
|
54
51
|
[tool.poetry.group.dev]
|
|
55
52
|
optional = true
|
|
56
53
|
|
|
57
|
-
# All the dev dependencies are set here so they can be used by the CI.
|
|
58
54
|
[tool.poetry.group.dev.dependencies]
|
|
59
|
-
pytest = "^
|
|
60
|
-
pytest-cov = "^
|
|
61
|
-
pytest-mock = "^3.
|
|
55
|
+
pytest = "^8.3.4"
|
|
56
|
+
pytest-cov = "^6.0.0"
|
|
57
|
+
pytest-mock = "^3.14.0"
|
|
62
58
|
pytest-error-for-skips = "^2.0.2"
|
|
63
|
-
black = "==
|
|
64
|
-
mypy = "^1.
|
|
65
|
-
pre-commit = "^
|
|
66
|
-
bandit = "^1.
|
|
67
|
-
safety = "^3.2.
|
|
68
|
-
pylint = "^3.
|
|
69
|
-
pytest-docker = "^
|
|
70
|
-
sphinx = "^
|
|
71
|
-
sphinx-rtd-theme = "^
|
|
72
|
-
flake8 = "^
|
|
59
|
+
black = "==25.1.0"
|
|
60
|
+
mypy = "^1.15.0"
|
|
61
|
+
pre-commit = "^4.1.0"
|
|
62
|
+
bandit = "^1.8.2"
|
|
63
|
+
safety = "^3.2.14"
|
|
64
|
+
pylint = "^3.3.4"
|
|
65
|
+
pytest-docker = "^3.1.2"
|
|
66
|
+
sphinx = "^8.1.3"
|
|
67
|
+
sphinx-rtd-theme = "^3.0.2"
|
|
68
|
+
flake8 = "^7.1.1"
|
|
73
69
|
flake8-pyproject = "^1.2.3"
|
|
74
|
-
responses = "^0.25.
|
|
70
|
+
responses = "^0.25.6"
|
|
71
|
+
starlette = "^0.45.3"
|
|
75
72
|
|
|
76
73
|
[tool.poetry.group.demo.dependencies] # for hybrid mode
|
|
77
74
|
jupyterlab = "^4.2.1"
|
|
78
|
-
boto3 = "^1.
|
|
75
|
+
boto3 = "^1.36.13"
|
|
79
76
|
papermill = "^2.6.0"
|
|
80
77
|
|
|
81
78
|
[tool.poetry-dynamic-versioning] # needs: poetry self add "poetry-dynamic-versioning[plugin]"
|
|
@@ -252,6 +252,16 @@ class RsClient: # pylint: disable=too-many-instance-attributes
|
|
|
252
252
|
**kwargs,
|
|
253
253
|
)
|
|
254
254
|
|
|
255
|
+
def get_staging_client(self) -> "StagingClient": # type: ignore # noqa: F821
|
|
256
|
+
"""
|
|
257
|
+
Return an instance of the child class AuxipClient, with the same attributes as this "self" instance.
|
|
258
|
+
"""
|
|
259
|
+
from rs_client.staging_client import ( # pylint: disable=import-outside-toplevel,cyclic-import
|
|
260
|
+
StagingClient,
|
|
261
|
+
)
|
|
262
|
+
|
|
263
|
+
return StagingClient(self.rs_server_href, self.rs_server_api_key, self.owner_id, self.logger)
|
|
264
|
+
|
|
255
265
|
############################
|
|
256
266
|
# Call RS-Server endpoints #
|
|
257
267
|
############################
|
|
@@ -311,7 +321,6 @@ class RsClient: # pylint: disable=too-many-instance-attributes
|
|
|
311
321
|
|
|
312
322
|
Raises:
|
|
313
323
|
RuntimeError: If an error occurs while staging the file.
|
|
314
|
-
|
|
315
324
|
"""
|
|
316
325
|
|
|
317
326
|
# dictionary to be used for payload request
|
|
@@ -418,16 +427,13 @@ class RsClient: # pylint: disable=too-many-instance-attributes
|
|
|
418
427
|
##############################################
|
|
419
428
|
|
|
420
429
|
@property
|
|
421
|
-
def href_search(self)
|
|
430
|
+
def href_search(self):
|
|
422
431
|
"""Implemented by AuxipClient and CadipClient."""
|
|
423
|
-
raise NotImplementedError
|
|
424
432
|
|
|
425
433
|
@property
|
|
426
|
-
def href_staging(self)
|
|
434
|
+
def href_staging(self):
|
|
427
435
|
"""Implemented by AuxipClient and CadipClient."""
|
|
428
|
-
raise NotImplementedError
|
|
429
436
|
|
|
430
437
|
@property
|
|
431
|
-
def href_status(self)
|
|
438
|
+
def href_status(self):
|
|
432
439
|
"""Implemented by AuxipClient and CadipClient."""
|
|
433
|
-
raise NotImplementedError
|
|
@@ -0,0 +1,311 @@
|
|
|
1
|
+
# Copyright 2024 CS Group
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""Lauch staging with rs-client-libraries"""
|
|
16
|
+
|
|
17
|
+
import json
|
|
18
|
+
import os
|
|
19
|
+
import os.path as osp
|
|
20
|
+
from typing import Any, Dict
|
|
21
|
+
|
|
22
|
+
# openapi_core libraries used for endpoints validation
|
|
23
|
+
import requests
|
|
24
|
+
from openapi_core import OpenAPI # Spec, validate_request, validate_response
|
|
25
|
+
from openapi_core.contrib.requests import (
|
|
26
|
+
RequestsOpenAPIRequest,
|
|
27
|
+
RequestsOpenAPIResponse,
|
|
28
|
+
)
|
|
29
|
+
from requests import Response
|
|
30
|
+
from requests.models import PreparedRequest
|
|
31
|
+
from stac_pydantic.api import Item, ItemCollection
|
|
32
|
+
|
|
33
|
+
from rs_client.rs_client import TIMEOUT, RsClient
|
|
34
|
+
|
|
35
|
+
# WARNING: this env variable is temporarily added until the response of rssrver-staging endpoints are corrected
|
|
36
|
+
# with a valid format according to ogc standard. In the meantime, we don't perform validation to make all staging
|
|
37
|
+
# notebooks pass. If this env variable if not specified (for example that is the case when we launch the pytest),
|
|
38
|
+
# we perform this validation by default
|
|
39
|
+
DO_VALIDATE: bool = os.getenv("RSPY_APPLY_STAGING_ENDPOINTS_VALIDATION", "1") == "1"
|
|
40
|
+
|
|
41
|
+
PATH_TO_YAML_OPENAPI = osp.join(
|
|
42
|
+
osp.realpath(osp.dirname(__file__)),
|
|
43
|
+
"../config",
|
|
44
|
+
"staging_templates",
|
|
45
|
+
"yaml",
|
|
46
|
+
"staging_openapi_schema.yaml",
|
|
47
|
+
)
|
|
48
|
+
PATH_TO_STAGING_BODY = osp.join(
|
|
49
|
+
osp.realpath(osp.dirname(__file__)),
|
|
50
|
+
"../config",
|
|
51
|
+
"staging_templates",
|
|
52
|
+
"staging_body.json",
|
|
53
|
+
)
|
|
54
|
+
RESOURCE = "staging"
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class StagingValidationException(Exception):
|
|
58
|
+
"""
|
|
59
|
+
Exception raised when an error occurs during the OGC validation
|
|
60
|
+
of the staging endpoints
|
|
61
|
+
"""
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class StagingClient(RsClient):
|
|
65
|
+
"""
|
|
66
|
+
Class to handle the staging process in rs-client-libraries
|
|
67
|
+
|
|
68
|
+
This class provides python methods to call the different endpoints of the rs-server-staging method.
|
|
69
|
+
|
|
70
|
+
Remark: this class don't inherits from the owslib.ogcapi.processes.Processes class because the latter
|
|
71
|
+
doesn't provide wrapping for all endpoints defined in rs-server-staging (it only provides the /processes
|
|
72
|
+
and /processes/{processId}/execution endpoints + it doesn't allow to manage apikey_header parameter which
|
|
73
|
+
is passed as an extra argument).
|
|
74
|
+
"""
|
|
75
|
+
|
|
76
|
+
@property
|
|
77
|
+
def href_staging(self) -> str:
|
|
78
|
+
"""
|
|
79
|
+
Return the RS-Server staging URL hostname.
|
|
80
|
+
This URL can be overwritten using the RSPY_HOST_STAGING env variable (used e.g. for local mode).
|
|
81
|
+
Otherwise it should just be the RS-Server URL.
|
|
82
|
+
"""
|
|
83
|
+
if from_env := os.getenv("RSPY_HOST_STAGING", None):
|
|
84
|
+
return from_env.rstrip("/")
|
|
85
|
+
if not self.rs_server_href:
|
|
86
|
+
raise RuntimeError("RS-Server URL is undefined")
|
|
87
|
+
return self.rs_server_href.rstrip("/")
|
|
88
|
+
|
|
89
|
+
def validate_and_unmarshal_request(self, request: PreparedRequest) -> Any:
|
|
90
|
+
"""Validate an endpoint request according to the ogc specifications
|
|
91
|
+
|
|
92
|
+
Args:
|
|
93
|
+
request (Request): endpoint request
|
|
94
|
+
|
|
95
|
+
Returns:
|
|
96
|
+
ResponseUnmarshalResult.data: data validated by the openapi_core
|
|
97
|
+
unmarshal_response method
|
|
98
|
+
"""
|
|
99
|
+
if not DO_VALIDATE:
|
|
100
|
+
return request.body
|
|
101
|
+
|
|
102
|
+
if not os.path.isfile(PATH_TO_YAML_OPENAPI):
|
|
103
|
+
raise FileNotFoundError(f"The following file path was not found: {PATH_TO_YAML_OPENAPI}")
|
|
104
|
+
|
|
105
|
+
openapi = OpenAPI.from_file_path(PATH_TO_YAML_OPENAPI)
|
|
106
|
+
openapi_request = RequestsOpenAPIRequest(request)
|
|
107
|
+
|
|
108
|
+
# validate_request(request, spec=Spec.from_file_path(PATH_TO_YAML_OPENAPI))
|
|
109
|
+
result = openapi.unmarshal_request(openapi_request)
|
|
110
|
+
|
|
111
|
+
if result.errors:
|
|
112
|
+
raise StagingValidationException(
|
|
113
|
+
f"Error validating the request of the enpoint "
|
|
114
|
+
f"{openapi_request.path}: {str(result.errors[0])}", # type: ignore
|
|
115
|
+
)
|
|
116
|
+
if not result.body:
|
|
117
|
+
raise StagingValidationException(
|
|
118
|
+
f"Error validating the request of the enpoint "
|
|
119
|
+
f"{openapi_request.path}: 'data' field of ResponseUnmarshalResult"
|
|
120
|
+
f"object is empty",
|
|
121
|
+
)
|
|
122
|
+
return result.body
|
|
123
|
+
|
|
124
|
+
def validate_and_unmarshal_response(self, response: Response) -> Any:
|
|
125
|
+
"""
|
|
126
|
+
Validate an endpoint response according to the ogc specifications
|
|
127
|
+
(described as yaml schemas)
|
|
128
|
+
|
|
129
|
+
Args:
|
|
130
|
+
response (Response): endpoint response
|
|
131
|
+
Returns:
|
|
132
|
+
ResponseUnmarshalResult.data: data validated by the openapi_core
|
|
133
|
+
unmarshal_response method
|
|
134
|
+
"""
|
|
135
|
+
if not DO_VALIDATE:
|
|
136
|
+
if not response.content:
|
|
137
|
+
raise StagingValidationException("Response content is empty !")
|
|
138
|
+
return json.loads(response.content)
|
|
139
|
+
|
|
140
|
+
if not os.path.isfile(PATH_TO_YAML_OPENAPI):
|
|
141
|
+
raise FileNotFoundError(f"The following file path was not found: {PATH_TO_YAML_OPENAPI}")
|
|
142
|
+
|
|
143
|
+
openapi = OpenAPI.from_file_path(PATH_TO_YAML_OPENAPI)
|
|
144
|
+
openapi_request = RequestsOpenAPIRequest(response.request)
|
|
145
|
+
openapi_response = RequestsOpenAPIResponse(response)
|
|
146
|
+
|
|
147
|
+
# Alternative method to validate the response
|
|
148
|
+
# validate_response(response=response, spec= Spec.from_file_path(PATH_TO_YAML_OPENAPI), request=request)
|
|
149
|
+
result = openapi.unmarshal_response(openapi_request, openapi_response) # type: ignore
|
|
150
|
+
if result.errors:
|
|
151
|
+
raise StagingValidationException( # type: ignore
|
|
152
|
+
f"Error validating the response of the enpoint "
|
|
153
|
+
f"{openapi_request.path}: {str(result.errors[0])}", # type: ignore
|
|
154
|
+
)
|
|
155
|
+
if not result.data:
|
|
156
|
+
raise StagingValidationException(
|
|
157
|
+
f"Error validating the response of the enpoint "
|
|
158
|
+
f"{openapi_request.path}: 'data' field of ResponseUnmarshalResult"
|
|
159
|
+
f"object is empty",
|
|
160
|
+
)
|
|
161
|
+
return result.data
|
|
162
|
+
|
|
163
|
+
############################
|
|
164
|
+
# Call RS-Server endpoints #
|
|
165
|
+
############################
|
|
166
|
+
|
|
167
|
+
def get_processes(self) -> Dict:
|
|
168
|
+
"""_summary_
|
|
169
|
+
|
|
170
|
+
Returns:
|
|
171
|
+
dict: dictionary containing the content of the response
|
|
172
|
+
"""
|
|
173
|
+
response = self.http_session.get(
|
|
174
|
+
url=f"{self.href_staging}/processes",
|
|
175
|
+
timeout=TIMEOUT,
|
|
176
|
+
**self.apikey_headers,
|
|
177
|
+
)
|
|
178
|
+
return self.validate_and_unmarshal_response(response)
|
|
179
|
+
|
|
180
|
+
def get_process(self, process_id: str) -> Dict:
|
|
181
|
+
"""
|
|
182
|
+
Wrapper to get a specific process
|
|
183
|
+
Args:
|
|
184
|
+
process_id (str): name of the resource
|
|
185
|
+
"""
|
|
186
|
+
response = self.http_session.get(
|
|
187
|
+
url=f"{self.href_staging}/processes/{process_id}",
|
|
188
|
+
timeout=TIMEOUT,
|
|
189
|
+
**self.apikey_headers,
|
|
190
|
+
)
|
|
191
|
+
return self.validate_and_unmarshal_response(response)
|
|
192
|
+
|
|
193
|
+
def run_staging( # pylint: disable=too-many-locals
|
|
194
|
+
self,
|
|
195
|
+
stac_input: dict[Any, Any] | str,
|
|
196
|
+
out_coll_name: str,
|
|
197
|
+
) -> Dict:
|
|
198
|
+
"""Method to start the staging process from rs-client - Call the endpoint /processes/staging/execution
|
|
199
|
+
|
|
200
|
+
Args:
|
|
201
|
+
stac_input (dict | str): input dictionary: the stac_input can have different format. It can be:
|
|
202
|
+
- A Python dictionary corresponding to a Feature or a FeatureCollection (that can be for example
|
|
203
|
+
the output of a search for Cadip or Auxip sessions)
|
|
204
|
+
- A json string corresponding to a Feature or a FeatureCollection
|
|
205
|
+
- A string corresponding to a path to a json file containing a Feature or a FeatureCollection
|
|
206
|
+
out_coll_name (): _description_
|
|
207
|
+
|
|
208
|
+
Return:
|
|
209
|
+
job_id (int, str): Returns the status code of the staging request + the identifier
|
|
210
|
+
(or None if staging endpoint fails) of the running job
|
|
211
|
+
"""
|
|
212
|
+
stac_input_dict = {}
|
|
213
|
+
# If stac_input is a file, load this file to a dictionary
|
|
214
|
+
if isinstance(stac_input, str):
|
|
215
|
+
# If the input is a valid path to a json_file, load this file
|
|
216
|
+
if os.path.exists(os.path.dirname(stac_input)) and stac_input.endswith(".json"):
|
|
217
|
+
# Read the yaml or json file
|
|
218
|
+
with open(stac_input, encoding="utf-8") as opened:
|
|
219
|
+
stac_file_to_dict = json.loads(opened.read())
|
|
220
|
+
stac_input_dict = stac_file_to_dict
|
|
221
|
+
# If the input string is not a path, try to convert the content of the string to a json dictionary
|
|
222
|
+
else:
|
|
223
|
+
stac_input_dict = json.loads(stac_input)
|
|
224
|
+
else:
|
|
225
|
+
stac_input_dict = stac_input
|
|
226
|
+
|
|
227
|
+
if "type" not in stac_input_dict:
|
|
228
|
+
raise KeyError("Key 'type' is missing from the staging input data")
|
|
229
|
+
|
|
230
|
+
# Validate input data using Pydantic
|
|
231
|
+
if stac_input_dict["type"] == "Feature":
|
|
232
|
+
stac_item = Item(**stac_input_dict)
|
|
233
|
+
stac_item_collection = ItemCollection(
|
|
234
|
+
**{
|
|
235
|
+
"type": "FeatureCollection",
|
|
236
|
+
"context": {"limit": 1000, "returned": 2},
|
|
237
|
+
"features": [stac_item],
|
|
238
|
+
}, # type: ignore
|
|
239
|
+
)
|
|
240
|
+
else:
|
|
241
|
+
stac_item_collection = ItemCollection(**stac_input_dict)
|
|
242
|
+
|
|
243
|
+
# Load staging body base structure and fill it with the staging content
|
|
244
|
+
if not os.path.isfile(PATH_TO_STAGING_BODY):
|
|
245
|
+
raise FileNotFoundError(f"The following file path was not found: {PATH_TO_STAGING_BODY}")
|
|
246
|
+
|
|
247
|
+
# TODO: this staging_body content will be validated with the self.validate_and_unmarshal_request(request)
|
|
248
|
+
# once rs-server-staging will be updated
|
|
249
|
+
with open(PATH_TO_STAGING_BODY, encoding="utf-8") as f:
|
|
250
|
+
staging_body = json.load(f)
|
|
251
|
+
staging_body["inputs"]["collection"]["id"] = out_coll_name
|
|
252
|
+
staging_body["inputs"]["items"].update(stac_item_collection.model_dump(mode="json"))
|
|
253
|
+
# TODO: replace the staging_body "outputs" field with the following when rs-server-staging is updated
|
|
254
|
+
# TODO: "outputs": {"featureCollectionOutput": {"transmissionMode": "value"}},
|
|
255
|
+
|
|
256
|
+
# Check that the request containing the staging body is valid
|
|
257
|
+
request = requests.Request( # pylint: disable=W0612 # noqa: F841
|
|
258
|
+
method="POST", # Méthode HTTP, peut être 'POST', 'GET', etc.
|
|
259
|
+
url=f"{self.href_staging}/processes/{RESOURCE}/execution", # L'URL de l'endpoint
|
|
260
|
+
json=staging_body, # Corps de la requête en JSON
|
|
261
|
+
).prepare()
|
|
262
|
+
# TODO: uncomment when rs-server-staging is updated
|
|
263
|
+
# TODO: self.validate_and_unmarshal_request(request)
|
|
264
|
+
|
|
265
|
+
response = self.http_session.post(
|
|
266
|
+
url=f"{self.href_staging}/processes/staging/execution",
|
|
267
|
+
json=staging_body,
|
|
268
|
+
**self.apikey_headers,
|
|
269
|
+
timeout=TIMEOUT,
|
|
270
|
+
)
|
|
271
|
+
return self.validate_and_unmarshal_response(response)
|
|
272
|
+
|
|
273
|
+
def get_jobs(self) -> Dict:
|
|
274
|
+
"""Method to get running jobs"""
|
|
275
|
+
response = self.http_session.get(
|
|
276
|
+
url=f"{self.href_staging}/jobs",
|
|
277
|
+
**self.apikey_headers,
|
|
278
|
+
timeout=TIMEOUT,
|
|
279
|
+
)
|
|
280
|
+
return self.validate_and_unmarshal_response(response)
|
|
281
|
+
|
|
282
|
+
def get_job_info(self, job_id: str) -> Dict: # pylint: disable=too-many-locals
|
|
283
|
+
"""Method to get a specific job response"""
|
|
284
|
+
response = self.http_session.get(
|
|
285
|
+
url=f"{self.href_staging}/jobs/{job_id}",
|
|
286
|
+
**self.apikey_headers,
|
|
287
|
+
timeout=TIMEOUT,
|
|
288
|
+
)
|
|
289
|
+
return self.validate_and_unmarshal_response(response)
|
|
290
|
+
|
|
291
|
+
def delete_job(self, job_id: str) -> Dict: # pylint: disable=too-many-locals
|
|
292
|
+
"""Method to get a specific job response"""
|
|
293
|
+
response = self.http_session.delete(
|
|
294
|
+
url=f"{self.href_staging}/jobs/{job_id}",
|
|
295
|
+
**self.apikey_headers,
|
|
296
|
+
timeout=TIMEOUT,
|
|
297
|
+
)
|
|
298
|
+
return self.validate_and_unmarshal_response(response)
|
|
299
|
+
|
|
300
|
+
def get_job_results(self, job_id: str) -> Dict:
|
|
301
|
+
"""Wrapper to get the result of a specfific job
|
|
302
|
+
|
|
303
|
+
Args:
|
|
304
|
+
job_id (str): _description_
|
|
305
|
+
"""
|
|
306
|
+
response = self.http_session.get(
|
|
307
|
+
url=f"{self.href_staging}/jobs/{job_id}/results",
|
|
308
|
+
timeout=TIMEOUT,
|
|
309
|
+
**self.apikey_headers,
|
|
310
|
+
)
|
|
311
|
+
return self.validate_and_unmarshal_response(response)
|
|
@@ -24,7 +24,6 @@ from opentelemetry import trace
|
|
|
24
24
|
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
|
|
25
25
|
from opentelemetry.instrumentation.asyncio import AsyncioInstrumentor
|
|
26
26
|
from opentelemetry.instrumentation.aws_lambda import AwsLambdaInstrumentor
|
|
27
|
-
from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor
|
|
28
27
|
from opentelemetry.instrumentation.instrumentor import BaseInstrumentor # type: ignore
|
|
29
28
|
from opentelemetry.sdk.resources import Resource
|
|
30
29
|
from opentelemetry.sdk.trace import TracerProvider
|
|
@@ -101,7 +100,7 @@ def init_traces(service_name: str):
|
|
|
101
100
|
classes.add(_class)
|
|
102
101
|
|
|
103
102
|
# Don't instrument these classes, they have errors, maybe we should see why
|
|
104
|
-
if _class in [AsyncioInstrumentor, AwsLambdaInstrumentor, BaseInstrumentor
|
|
103
|
+
if _class in [AsyncioInstrumentor, AwsLambdaInstrumentor, BaseInstrumentor]:
|
|
105
104
|
continue
|
|
106
105
|
|
|
107
106
|
# If the "instrument" method exists, call it
|
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
# Copyright 2024 CS Group
|
|
2
|
-
#
|
|
3
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
-
# you may not use this file except in compliance with the License.
|
|
5
|
-
# You may obtain a copy of the License at
|
|
6
|
-
#
|
|
7
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
-
#
|
|
9
|
-
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
-
# See the License for the specific language governing permissions and
|
|
13
|
-
# limitations under the License.
|
|
14
|
-
|
|
15
|
-
"""rs-workflows main module."""
|
|
16
|
-
|
|
17
|
-
# Set automatically by running `poetry dynamic-versioning`
|
|
18
|
-
__version__ = "0.2a9"
|
|
@@ -1,43 +0,0 @@
|
|
|
1
|
-
# Copyright 2024 CS Group
|
|
2
|
-
#
|
|
3
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
-
# you may not use this file except in compliance with the License.
|
|
5
|
-
# You may obtain a copy of the License at
|
|
6
|
-
#
|
|
7
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
-
#
|
|
9
|
-
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
-
# See the License for the specific language governing permissions and
|
|
13
|
-
# limitations under the License.
|
|
14
|
-
|
|
15
|
-
general_configuration:
|
|
16
|
-
logging:
|
|
17
|
-
level: DEBUG
|
|
18
|
-
triggering__validate_run: true
|
|
19
|
-
triggering__use_default_filename: true
|
|
20
|
-
triggering__use_basic_logging: true
|
|
21
|
-
triggering__load_default_logging: false
|
|
22
|
-
breakpoints:
|
|
23
|
-
workflow:
|
|
24
|
-
- step: 1
|
|
25
|
-
active: true
|
|
26
|
-
module: rs.dpr.mockup # string corresponding to the python path of the module
|
|
27
|
-
processing_unit: DprMockupProcessor # EOProcessingUnit class name
|
|
28
|
-
name: DprMockupProcessor # identifier for the processing unit
|
|
29
|
-
inputs:
|
|
30
|
-
-
|
|
31
|
-
outputs:
|
|
32
|
-
-
|
|
33
|
-
parameters:
|
|
34
|
-
product_types: # List of EOPF product types we want to generate. In this example we simulate S1L0 processor that generates 4 products
|
|
35
|
-
-
|
|
36
|
-
I/O:
|
|
37
|
-
inputs_products:
|
|
38
|
-
-
|
|
39
|
-
output_products:
|
|
40
|
-
-
|
|
41
|
-
dask_context: {}
|
|
42
|
-
logging: {}
|
|
43
|
-
config: {}
|
|
@@ -1,43 +0,0 @@
|
|
|
1
|
-
# Copyright 2024 CS Group
|
|
2
|
-
#
|
|
3
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
-
# you may not use this file except in compliance with the License.
|
|
5
|
-
# You may obtain a copy of the License at
|
|
6
|
-
#
|
|
7
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
-
#
|
|
9
|
-
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
-
# See the License for the specific language governing permissions and
|
|
13
|
-
# limitations under the License.
|
|
14
|
-
|
|
15
|
-
"""Prefect flow example for rs-client-libraries"""
|
|
16
|
-
from prefect import flow, task
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
@task
|
|
20
|
-
def print_hello(name: str):
|
|
21
|
-
"""Example task to be used in a prefect flow
|
|
22
|
-
|
|
23
|
-
This prefect task may be used as start point in creating your own prefect tasks
|
|
24
|
-
|
|
25
|
-
Args:
|
|
26
|
-
name (str): Username to be printed
|
|
27
|
-
"""
|
|
28
|
-
print(f"Hello {name}!")
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
@flow(name="Hello RS-Server flow ")
|
|
32
|
-
def hello_world(name="COPERNICUS", tasks_number=2):
|
|
33
|
-
"""Example flow that can be use in a COPERNICUS chain
|
|
34
|
-
|
|
35
|
-
This prefect flow may be used as start point in creating your own prefect flows. It runs in parallel
|
|
36
|
-
tasks_number of print_hello
|
|
37
|
-
|
|
38
|
-
Args:
|
|
39
|
-
name (str): Username to be printed. Default COPERNICUS
|
|
40
|
-
tasks_number (int): Number of tasks to be run. Default 2
|
|
41
|
-
"""
|
|
42
|
-
for idx in range(0, tasks_number):
|
|
43
|
-
print_hello(name + "_" + str(idx))
|