rs-client-libraries 0.2a13__tar.gz → 0.2a14__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {rs_client_libraries-0.2a13 → rs_client_libraries-0.2a14}/PKG-INFO +9 -7
- {rs_client_libraries-0.2a13 → rs_client_libraries-0.2a14}/pyproject.toml +25 -8
- {rs_client_libraries-0.2a13 → rs_client_libraries-0.2a14}/rs_client/__init__.py +1 -1
- rs_client_libraries-0.2a14/rs_client/ogcapi/__init__.py +15 -0
- rs_client_libraries-0.2a14/rs_client/ogcapi/dpr_client.py +80 -0
- rs_client_libraries-0.2a14/rs_client/ogcapi/ogcapi_client.py +291 -0
- rs_client_libraries-0.2a14/rs_client/ogcapi/staging_client.py +142 -0
- {rs_client_libraries-0.2a13 → rs_client_libraries-0.2a14}/rs_client/rs_client.py +19 -44
- rs_client_libraries-0.2a14/rs_client/stac/__init__.py +15 -0
- {rs_client_libraries-0.2a13/rs_client → rs_client_libraries-0.2a14/rs_client/stac}/auxip_client.py +2 -4
- {rs_client_libraries-0.2a13/rs_client → rs_client_libraries-0.2a14/rs_client/stac}/cadip_client.py +2 -4
- {rs_client_libraries-0.2a13/rs_client → rs_client_libraries-0.2a14/rs_client/stac}/catalog_client.py +50 -2
- {rs_client_libraries-0.2a13/rs_client → rs_client_libraries-0.2a14/rs_client/stac}/stac_base.py +1 -1
- {rs_client_libraries-0.2a13 → rs_client_libraries-0.2a14}/rs_common/init_opentelemetry.py +82 -73
- rs_client_libraries-0.2a14/rs_common/prefect_utils.py +393 -0
- rs_client_libraries-0.2a14/rs_workflows/__init__.py +15 -0
- rs_client_libraries-0.2a14/rs_workflows/auxip_flow.py +68 -0
- rs_client_libraries-0.2a14/rs_workflows/cadip_flow.py +67 -0
- rs_client_libraries-0.2a14/rs_workflows/catalog_flow.py +70 -0
- rs_client_libraries-0.2a14/rs_workflows/dpr_flow.py +270 -0
- rs_client_libraries-0.2a14/rs_workflows/flow_utils.py +127 -0
- rs_client_libraries-0.2a14/rs_workflows/on_demand_processing.py +130 -0
- rs_client_libraries-0.2a14/rs_workflows/staging_flow.py +92 -0
- rs_client_libraries-0.2a13/rs_client/staging_client.py +0 -327
- {rs_client_libraries-0.2a13 → rs_client_libraries-0.2a14}/LICENSE +0 -0
- {rs_client_libraries-0.2a13 → rs_client_libraries-0.2a14}/README.adoc +0 -0
- {rs_client_libraries-0.2a13 → rs_client_libraries-0.2a14}/rs_common/__init__.py +0 -0
- {rs_client_libraries-0.2a13 → rs_client_libraries-0.2a14}/rs_common/config.py +0 -0
- {rs_client_libraries-0.2a13 → rs_client_libraries-0.2a14}/rs_common/logging.py +0 -0
- {rs_client_libraries-0.2a13 → rs_client_libraries-0.2a14}/rs_common/utils.py +0 -0
|
@@ -1,8 +1,7 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
2
|
Name: rs-client-libraries
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2a14
|
|
4
4
|
Summary: Copernicus Sentinel rs-client library and processing workflows.
|
|
5
|
-
Home-page: https://home.rs-python.eu/
|
|
6
5
|
License: Apache-2.0
|
|
7
6
|
Keywords: stac,copernicus,sentinel,processing
|
|
8
7
|
Author: ESA under contract with Airbus and CS
|
|
@@ -20,17 +19,20 @@ Classifier: Programming Language :: Python :: 3.13
|
|
|
20
19
|
Classifier: Topic :: Scientific/Engineering :: Image Processing
|
|
21
20
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
22
21
|
Requires-Dist: cachetools (>=5.5.2,<6.0.0)
|
|
22
|
+
Requires-Dist: click (==8.1.8)
|
|
23
23
|
Requires-Dist: openapi-core (>=0.19.5,<0.20.0)
|
|
24
|
-
Requires-Dist: opentelemetry-distro (>=0.
|
|
25
|
-
Requires-Dist: opentelemetry-exporter-otlp (>=1.
|
|
26
|
-
Requires-Dist: opentelemetry-instrumentation-aws-lambda (>=0.
|
|
27
|
-
Requires-Dist: opentelemetry-test-utils (>=0.
|
|
24
|
+
Requires-Dist: opentelemetry-distro (>=0.55b0,<0.56)
|
|
25
|
+
Requires-Dist: opentelemetry-exporter-otlp (>=1.34.0,<2.0.0)
|
|
26
|
+
Requires-Dist: opentelemetry-instrumentation-aws-lambda (>=0.55b0,<0.56)
|
|
27
|
+
Requires-Dist: opentelemetry-test-utils (>=0.55b0,<0.56)
|
|
28
|
+
Requires-Dist: prefect[aws] (==3.2.13)
|
|
28
29
|
Requires-Dist: pystac (>=1.13.0,<2.0.0)
|
|
29
30
|
Requires-Dist: pystac-client (>=0.8.6,<0.9.0)
|
|
30
31
|
Requires-Dist: python-dateutil (>=2.9.0.post0,<3.0.0)
|
|
31
32
|
Requires-Dist: python-logging-loki (>=0.3.1,<0.4.0)
|
|
32
33
|
Requires-Dist: stac-pydantic (>=3.1.5,<4.0.0)
|
|
33
34
|
Project-URL: Documentation, https://home.rs-python.eu/rs-documentation/rs-client-libraries/docs/doc/
|
|
35
|
+
Project-URL: Homepage, https://home.rs-python.eu/
|
|
34
36
|
Project-URL: Repository, https://github.com/RS-PYTHON/rs-client-libraries
|
|
35
37
|
Description-Content-Type: text/plain
|
|
36
38
|
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "rs-client-libraries"
|
|
3
|
-
|
|
3
|
+
dynamic = ["version"] # see: https://github.com/mtkennerly/poetry-dynamic-versioning
|
|
4
4
|
|
|
5
5
|
[tool.poetry]
|
|
6
6
|
name = "rs-client-libraries"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.2a14"
|
|
8
8
|
description = "Copernicus Sentinel rs-client library and processing workflows."
|
|
9
9
|
license = "Apache-2.0"
|
|
10
10
|
authors = ["ESA under contract with Airbus and CS <ops_coprs@airbus.com>"]
|
|
@@ -27,6 +27,7 @@ classifiers = [
|
|
|
27
27
|
packages = [
|
|
28
28
|
{ include = "rs_client" },
|
|
29
29
|
{ include = "rs_common" },
|
|
30
|
+
{ include = "rs_workflows" },
|
|
30
31
|
]
|
|
31
32
|
|
|
32
33
|
# Add configuration files to the .whl package
|
|
@@ -36,10 +37,10 @@ include = [
|
|
|
36
37
|
|
|
37
38
|
[tool.poetry.dependencies]
|
|
38
39
|
python = "^3.11"
|
|
39
|
-
opentelemetry-distro = "^0.
|
|
40
|
-
opentelemetry-instrumentation-aws-lambda = "^0.
|
|
41
|
-
opentelemetry-test-utils = "^0.
|
|
42
|
-
opentelemetry-exporter-otlp = "^1.
|
|
40
|
+
opentelemetry-distro = "^0.55b0" # then run 'poetry run opentelemetry-bootstrap -a install'
|
|
41
|
+
opentelemetry-instrumentation-aws-lambda = "^0.55b0"
|
|
42
|
+
opentelemetry-test-utils = "^0.55b0"
|
|
43
|
+
opentelemetry-exporter-otlp = "^1.34.0"
|
|
43
44
|
python-logging-loki = "^0.3.1" # to avoid otl error: No module named 'logging_loki'
|
|
44
45
|
cachetools = "^5.5.2"
|
|
45
46
|
pystac = "^1.13.0"
|
|
@@ -47,6 +48,8 @@ pystac-client = "^0.8.6"
|
|
|
47
48
|
python-dateutil = "^2.9.0.post0"
|
|
48
49
|
stac-pydantic = "^3.1.5"
|
|
49
50
|
openapi-core = "^0.19.5"
|
|
51
|
+
click = "8.1.8"
|
|
52
|
+
prefect = { version = "3.2.13", extras = ["aws"] }
|
|
50
53
|
|
|
51
54
|
[tool.poetry.group.dev]
|
|
52
55
|
optional = true
|
|
@@ -74,8 +77,12 @@ starlette = "^0.45.3"
|
|
|
74
77
|
jupyterlab = "^4.2.1"
|
|
75
78
|
boto3 = "^1.37.11"
|
|
76
79
|
papermill = "^2.6.0"
|
|
80
|
+
tornado = "^6.5" # fix trivy vulnerability
|
|
77
81
|
|
|
78
|
-
[tool.poetry-
|
|
82
|
+
[tool.poetry.requires-plugins]
|
|
83
|
+
poetry-dynamic-versioning = { version = ">=1.0.0,<2.0.0", extras = ["plugin"] }
|
|
84
|
+
|
|
85
|
+
[tool.poetry-dynamic-versioning]
|
|
79
86
|
enable = false
|
|
80
87
|
metadata = false
|
|
81
88
|
|
|
@@ -98,6 +105,7 @@ warn_redundant_casts = true
|
|
|
98
105
|
# Force analyze of partially annotate function signatures
|
|
99
106
|
check_untyped_defs = true
|
|
100
107
|
ignore_missing_imports = true
|
|
108
|
+
disable_error_code = ["call-overload"]
|
|
101
109
|
|
|
102
110
|
[tool.flake8]
|
|
103
111
|
max-line-length = 120
|
|
@@ -107,7 +115,16 @@ max-line-length = 120
|
|
|
107
115
|
|
|
108
116
|
[tool.pylint.messages_control]
|
|
109
117
|
max-line-length=120
|
|
110
|
-
disable =
|
|
118
|
+
disable = [
|
|
119
|
+
"logging-fstring-interpolation",
|
|
120
|
+
"global-statement",
|
|
121
|
+
"too-many-arguments",
|
|
122
|
+
"too-many-positional-arguments",
|
|
123
|
+
"not-context-manager",
|
|
124
|
+
"too-many-locals",
|
|
125
|
+
"too-many-branches",
|
|
126
|
+
"too-many-statements",
|
|
127
|
+
]
|
|
111
128
|
|
|
112
129
|
[tool.pylint.MASTER]
|
|
113
130
|
ignore-paths = '^docs/.*$'
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# Copyright 2024 CS Group
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""ogc api modules."""
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
# Copyright 2024 CS Group
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""Use "DPR as a service" implemented by rs-dpr-service"""
|
|
16
|
+
|
|
17
|
+
import ast
|
|
18
|
+
import os.path as osp
|
|
19
|
+
|
|
20
|
+
from openapi_core import OpenAPI # Spec, validate_request, validate_response
|
|
21
|
+
|
|
22
|
+
from rs_client.ogcapi.ogcapi_client import OgcApiClient
|
|
23
|
+
from rs_common.utils import get_href_service
|
|
24
|
+
|
|
25
|
+
PATH_TO_YAML_OPENAPI = osp.realpath(
|
|
26
|
+
osp.join(
|
|
27
|
+
osp.dirname(__file__),
|
|
28
|
+
"../../config",
|
|
29
|
+
"staging_templates",
|
|
30
|
+
"yaml",
|
|
31
|
+
"dpr_openapi_schema.yaml",
|
|
32
|
+
),
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class DprClient(OgcApiClient):
|
|
37
|
+
"""Implement the OGC API client for 'DPR as a service'."""
|
|
38
|
+
|
|
39
|
+
# Init the OpenAPI instance from config file
|
|
40
|
+
openapi = OpenAPI.from_file_path(PATH_TO_YAML_OPENAPI)
|
|
41
|
+
|
|
42
|
+
@property
|
|
43
|
+
def endpoint_prefix(self) -> str:
|
|
44
|
+
"""Return the endpoints prefix, if any."""
|
|
45
|
+
return "dpr/"
|
|
46
|
+
|
|
47
|
+
@property
|
|
48
|
+
def href_service(self) -> str:
|
|
49
|
+
"""
|
|
50
|
+
Return the rs-dpr-service URL hostname.
|
|
51
|
+
This URL can be overwritten using the RSPY_HOST_DPR_SERVICE env variable (used e.g. for local mode).
|
|
52
|
+
Otherwise it should just be the RS-Server URL.
|
|
53
|
+
"""
|
|
54
|
+
return get_href_service(self.rs_server_href, "RSPY_HOST_DPR_SERVICE")
|
|
55
|
+
|
|
56
|
+
def wait_for_job(self, *args, **kwargs) -> list[dict]: # type: ignore
|
|
57
|
+
"""
|
|
58
|
+
Wait for job to finish.
|
|
59
|
+
|
|
60
|
+
Returns:
|
|
61
|
+
EOPF results
|
|
62
|
+
"""
|
|
63
|
+
# Call parent method and parse results
|
|
64
|
+
job_status = super().wait_for_job(*args, **kwargs)
|
|
65
|
+
return ast.literal_eval(job_status["message"])
|
|
66
|
+
|
|
67
|
+
#
|
|
68
|
+
# These endpoints are not implemented by the service
|
|
69
|
+
|
|
70
|
+
def get_processes(self) -> dict:
|
|
71
|
+
raise NotImplementedError
|
|
72
|
+
|
|
73
|
+
def get_jobs(self) -> dict:
|
|
74
|
+
raise NotImplementedError
|
|
75
|
+
|
|
76
|
+
def delete_job(self, _: str) -> dict:
|
|
77
|
+
raise NotImplementedError
|
|
78
|
+
|
|
79
|
+
def get_job_results(self, _: str) -> dict:
|
|
80
|
+
raise NotImplementedError
|
|
@@ -0,0 +1,291 @@
|
|
|
1
|
+
# Copyright 2024 CS Group
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""Use OGC API services with rs-client-libraries"""
|
|
16
|
+
|
|
17
|
+
import json
|
|
18
|
+
import math
|
|
19
|
+
import time
|
|
20
|
+
from typing import Any
|
|
21
|
+
from urllib.parse import urlparse
|
|
22
|
+
|
|
23
|
+
# openapi_core libraries used for endpoints validation
|
|
24
|
+
import requests
|
|
25
|
+
from openapi_core import OpenAPI # Spec, validate_request, validate_response
|
|
26
|
+
from openapi_core.contrib.requests import (
|
|
27
|
+
RequestsOpenAPIRequest,
|
|
28
|
+
RequestsOpenAPIResponse,
|
|
29
|
+
)
|
|
30
|
+
from requests import Response
|
|
31
|
+
from requests.models import PreparedRequest
|
|
32
|
+
|
|
33
|
+
from rs_client.rs_client import TIMEOUT, RsClient
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class OgcValidationException(Exception):
|
|
37
|
+
"""
|
|
38
|
+
Exception raised when an error occurs during the OGC validation of the endpoints
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class OgcApiClient(RsClient):
|
|
43
|
+
"""
|
|
44
|
+
Class to handle OGC API processes in rs-client-libraries
|
|
45
|
+
|
|
46
|
+
This class provides python methods to call the different endpoints of the OGC API service.
|
|
47
|
+
|
|
48
|
+
Remark: this class don't inherits from the owslib.ogcapi.processes.Processes class because the latter
|
|
49
|
+
doesn't provide wrapping for all endpoints defined in our services (it only provides the /processes
|
|
50
|
+
and /processes/{processId}/execution endpoints + it doesn't allow to manage apikey_header parameter which
|
|
51
|
+
is passed as an extra argument).
|
|
52
|
+
"""
|
|
53
|
+
|
|
54
|
+
@staticmethod
|
|
55
|
+
def is_url(path):
|
|
56
|
+
"""
|
|
57
|
+
Function to check if a string is an url
|
|
58
|
+
"""
|
|
59
|
+
parsed = urlparse(path)
|
|
60
|
+
return bool(parsed.scheme and parsed.netloc)
|
|
61
|
+
|
|
62
|
+
@property
|
|
63
|
+
def endpoint_prefix(self) -> str:
|
|
64
|
+
"""Return the endpoints prefix, if any."""
|
|
65
|
+
return "" # no prefix by default
|
|
66
|
+
|
|
67
|
+
@classmethod
|
|
68
|
+
def get_openapi(cls) -> OpenAPI:
|
|
69
|
+
"""Return the OpenAPI instance of the subclass"""
|
|
70
|
+
return cls.openapi # type: ignore # pylint: disable=no-member
|
|
71
|
+
|
|
72
|
+
def validate_and_unmarshal_request(self, request: PreparedRequest) -> Any:
|
|
73
|
+
"""Validate an endpoint request according to the ogc specifications
|
|
74
|
+
|
|
75
|
+
Args:
|
|
76
|
+
request (Request): endpoint request
|
|
77
|
+
|
|
78
|
+
Returns:
|
|
79
|
+
ResponseUnmarshalResult.data: data validated by the openapi_core
|
|
80
|
+
unmarshal_response method
|
|
81
|
+
"""
|
|
82
|
+
openapi_request = RequestsOpenAPIRequest(request)
|
|
83
|
+
|
|
84
|
+
# validate_request(request, spec=Spec.from_file_path(PATH_TO_YAML_OPENAPI))
|
|
85
|
+
result = self.get_openapi().unmarshal_request(openapi_request)
|
|
86
|
+
|
|
87
|
+
if result.errors:
|
|
88
|
+
raise OgcValidationException(
|
|
89
|
+
f"Error validating the request of the enpoint "
|
|
90
|
+
f"{openapi_request.path}: {str(result.errors[0])}", # type: ignore
|
|
91
|
+
)
|
|
92
|
+
if not result.body:
|
|
93
|
+
raise OgcValidationException(
|
|
94
|
+
f"Error validating the request of the enpoint "
|
|
95
|
+
f"{openapi_request.path}: 'data' field of ResponseUnmarshalResult"
|
|
96
|
+
f"object is empty",
|
|
97
|
+
)
|
|
98
|
+
return result.body
|
|
99
|
+
|
|
100
|
+
def validate_and_unmarshal_response(self, response: Response) -> Any:
|
|
101
|
+
"""
|
|
102
|
+
Validate an endpoint response according to the ogc specifications
|
|
103
|
+
(described as yaml schemas)
|
|
104
|
+
|
|
105
|
+
Args:
|
|
106
|
+
response (Response): endpoint response
|
|
107
|
+
Returns:
|
|
108
|
+
ResponseUnmarshalResult.data: data validated by the openapi_core
|
|
109
|
+
unmarshal_response method
|
|
110
|
+
"""
|
|
111
|
+
openapi_request = RequestsOpenAPIRequest(response.request)
|
|
112
|
+
openapi_response = RequestsOpenAPIResponse(response)
|
|
113
|
+
|
|
114
|
+
# Alternative method to validate the response
|
|
115
|
+
# validate_response(response=response, spec= Spec.from_file_path(PATH_TO_YAML_OPENAPI), request=request)
|
|
116
|
+
result = self.get_openapi().unmarshal_response(openapi_request, openapi_response) # type: ignore
|
|
117
|
+
if result.errors:
|
|
118
|
+
raise OgcValidationException( # type: ignore
|
|
119
|
+
f"Error validating the response of the enpoint {openapi_request.path} - "
|
|
120
|
+
f"Server response content: {response.json()} - "
|
|
121
|
+
f"Validation error of the server response: {str(result.errors[0])}", # type: ignore
|
|
122
|
+
)
|
|
123
|
+
if not result.data:
|
|
124
|
+
raise OgcValidationException(
|
|
125
|
+
f"Error validating the response of the enpoint "
|
|
126
|
+
f"{openapi_request.path}: 'data' field of ResponseUnmarshalResult"
|
|
127
|
+
f"object is empty",
|
|
128
|
+
)
|
|
129
|
+
return json.loads(response.content)
|
|
130
|
+
|
|
131
|
+
##################################
|
|
132
|
+
# Call OGC API service endpoints #
|
|
133
|
+
##################################
|
|
134
|
+
|
|
135
|
+
def get_processes(self) -> dict:
|
|
136
|
+
"""_summary_
|
|
137
|
+
|
|
138
|
+
Returns:
|
|
139
|
+
dict: dictionary containing the content of the response
|
|
140
|
+
"""
|
|
141
|
+
response = self.http_session.get(
|
|
142
|
+
url=f"{self.href_service}/{self.endpoint_prefix}processes",
|
|
143
|
+
timeout=TIMEOUT,
|
|
144
|
+
**self.apikey_headers,
|
|
145
|
+
)
|
|
146
|
+
return self.validate_and_unmarshal_response(response)
|
|
147
|
+
|
|
148
|
+
def get_process(self, process_id: str) -> dict:
|
|
149
|
+
"""
|
|
150
|
+
Wrapper to get a specific process
|
|
151
|
+
Args:
|
|
152
|
+
process_id (str): name of the resource
|
|
153
|
+
"""
|
|
154
|
+
response = self.http_session.get(
|
|
155
|
+
url=f"{self.href_service}/{self.endpoint_prefix}processes/{process_id}",
|
|
156
|
+
timeout=TIMEOUT,
|
|
157
|
+
**self.apikey_headers,
|
|
158
|
+
)
|
|
159
|
+
return self.validate_and_unmarshal_response(response)
|
|
160
|
+
|
|
161
|
+
def run_process(self, process: str, body: dict) -> dict:
|
|
162
|
+
"""Method to start the process from rs-client - Call the endpoint /processes/{process}/execution
|
|
163
|
+
|
|
164
|
+
Args:
|
|
165
|
+
process: Process name
|
|
166
|
+
body: POST HTTP request body as a JSON dict
|
|
167
|
+
|
|
168
|
+
Return:
|
|
169
|
+
job_id (int, str): Returns the status code of the request + the identifier
|
|
170
|
+
(or None if endpoint fails) of the running job
|
|
171
|
+
"""
|
|
172
|
+
|
|
173
|
+
# Check that the request containing the body is valid
|
|
174
|
+
request = requests.Request(
|
|
175
|
+
method="POST",
|
|
176
|
+
url=f"{self.href_service}/{self.endpoint_prefix}processes/{process}/execution",
|
|
177
|
+
json=body,
|
|
178
|
+
).prepare()
|
|
179
|
+
|
|
180
|
+
# Validate the body of the request that will be sent to the service
|
|
181
|
+
self.validate_and_unmarshal_request(request)
|
|
182
|
+
|
|
183
|
+
response = self.http_session.post(
|
|
184
|
+
url=f"{self.href_service}/{self.endpoint_prefix}processes/{process}/execution",
|
|
185
|
+
json=body,
|
|
186
|
+
**self.apikey_headers,
|
|
187
|
+
timeout=TIMEOUT,
|
|
188
|
+
)
|
|
189
|
+
return self.validate_and_unmarshal_response(response)
|
|
190
|
+
|
|
191
|
+
def get_jobs(self) -> dict:
|
|
192
|
+
"""Method to get running jobs"""
|
|
193
|
+
response = self.http_session.get(
|
|
194
|
+
url=f"{self.href_service}/{self.endpoint_prefix}jobs",
|
|
195
|
+
**self.apikey_headers,
|
|
196
|
+
timeout=TIMEOUT,
|
|
197
|
+
)
|
|
198
|
+
return self.validate_and_unmarshal_response(response)
|
|
199
|
+
|
|
200
|
+
def get_job_info(self, job_id: str) -> dict: # pylint: disable=too-many-locals
|
|
201
|
+
"""Method to get a specific job response"""
|
|
202
|
+
response = self.http_session.get(
|
|
203
|
+
url=f"{self.href_service}/{self.endpoint_prefix}jobs/{job_id}",
|
|
204
|
+
**self.apikey_headers,
|
|
205
|
+
timeout=TIMEOUT,
|
|
206
|
+
)
|
|
207
|
+
return self.validate_and_unmarshal_response(response)
|
|
208
|
+
|
|
209
|
+
def delete_job(self, job_id: str) -> dict: # pylint: disable=too-many-locals
|
|
210
|
+
"""Method to get a specific job response"""
|
|
211
|
+
response = self.http_session.delete(
|
|
212
|
+
url=f"{self.href_service}/{self.endpoint_prefix}jobs/{job_id}",
|
|
213
|
+
**self.apikey_headers,
|
|
214
|
+
timeout=TIMEOUT,
|
|
215
|
+
)
|
|
216
|
+
return self.validate_and_unmarshal_response(response)
|
|
217
|
+
|
|
218
|
+
def get_job_results(self, job_id: str) -> dict:
|
|
219
|
+
"""Wrapper to get the result of a specfific job
|
|
220
|
+
|
|
221
|
+
Args:
|
|
222
|
+
job_id (str): _description_
|
|
223
|
+
"""
|
|
224
|
+
response = self.http_session.get(
|
|
225
|
+
url=f"{self.href_service}/{self.endpoint_prefix}jobs/{job_id}/results",
|
|
226
|
+
timeout=TIMEOUT,
|
|
227
|
+
**self.apikey_headers,
|
|
228
|
+
)
|
|
229
|
+
return self.validate_and_unmarshal_response(response)
|
|
230
|
+
|
|
231
|
+
def wait_for_job(
|
|
232
|
+
self,
|
|
233
|
+
job_status: dict,
|
|
234
|
+
logger=None,
|
|
235
|
+
job_name: str = "",
|
|
236
|
+
timeout: int | float = math.inf,
|
|
237
|
+
poll_interval: int = 2,
|
|
238
|
+
) -> dict:
|
|
239
|
+
"""
|
|
240
|
+
Wait for job to finish.
|
|
241
|
+
|
|
242
|
+
Args:
|
|
243
|
+
job_status: Returned by `run_process`
|
|
244
|
+
logger: To show advancement in logger
|
|
245
|
+
job_name: Job name to show in the logger
|
|
246
|
+
timeout: Job completion timeout in seconds
|
|
247
|
+
poll_interval: When to check again for job completion in seconds
|
|
248
|
+
|
|
249
|
+
Returns:
|
|
250
|
+
Job status
|
|
251
|
+
|
|
252
|
+
Raises:
|
|
253
|
+
RuntimeError in case of error
|
|
254
|
+
"""
|
|
255
|
+
try:
|
|
256
|
+
status_type, job_identifier = job_status["status"], job_status["jobID"]
|
|
257
|
+
if not job_identifier:
|
|
258
|
+
raise RuntimeError("Job identifier is missing.")
|
|
259
|
+
|
|
260
|
+
while True:
|
|
261
|
+
job_status = self.get_job_info(job_identifier)
|
|
262
|
+
if logger:
|
|
263
|
+
logger.info(f"job_status: {job_status}")
|
|
264
|
+
status_type = job_status.get("status")
|
|
265
|
+
if logger:
|
|
266
|
+
logger.info(
|
|
267
|
+
f"----- {job_name} job {job_identifier!r}: {status_type.upper()} \n",
|
|
268
|
+
)
|
|
269
|
+
|
|
270
|
+
# Exit the loop
|
|
271
|
+
if status_type in {"successful", "failed", "dismissed"}:
|
|
272
|
+
break
|
|
273
|
+
|
|
274
|
+
# Or sleep n seconds and try again
|
|
275
|
+
timeout -= poll_interval
|
|
276
|
+
if timeout <= 0:
|
|
277
|
+
raise TimeoutError(f"Timed out while waiting for {job_name} job {job_identifier!r}")
|
|
278
|
+
time.sleep(poll_interval)
|
|
279
|
+
|
|
280
|
+
# Log all exceptions except the timeout
|
|
281
|
+
except TimeoutError:
|
|
282
|
+
raise
|
|
283
|
+
except Exception as e:
|
|
284
|
+
raise RuntimeError(f"Exception while monitoring job {job_name}: {job_status}") from e
|
|
285
|
+
|
|
286
|
+
if status_type == "successful":
|
|
287
|
+
if logger:
|
|
288
|
+
logger.info(f"----- {job_name} job {job_identifier!r}: COMPLETED \n")
|
|
289
|
+
return job_status
|
|
290
|
+
|
|
291
|
+
raise RuntimeError(f"{job_name} job {job_identifier!r}: FAILED")
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
# Copyright 2024 CS Group
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""Launch staging with rs-client-libraries"""
|
|
16
|
+
|
|
17
|
+
import json
|
|
18
|
+
import os
|
|
19
|
+
import os.path as osp
|
|
20
|
+
from typing import Any
|
|
21
|
+
|
|
22
|
+
# openapi_core libraries used for endpoints validation
|
|
23
|
+
from openapi_core import OpenAPI # Spec, validate_request, validate_response
|
|
24
|
+
from stac_pydantic.api import Item, ItemCollection
|
|
25
|
+
|
|
26
|
+
from rs_client.ogcapi.ogcapi_client import OgcApiClient, OgcValidationException
|
|
27
|
+
from rs_common.utils import get_href_service
|
|
28
|
+
|
|
29
|
+
PATH_TO_YAML_OPENAPI = osp.realpath(
|
|
30
|
+
osp.join(
|
|
31
|
+
osp.dirname(__file__),
|
|
32
|
+
"../../config",
|
|
33
|
+
"staging_templates",
|
|
34
|
+
"yaml",
|
|
35
|
+
"staging_openapi_schema.yaml",
|
|
36
|
+
),
|
|
37
|
+
)
|
|
38
|
+
RESOURCE = "staging"
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class StagingClient(OgcApiClient):
|
|
42
|
+
"""Implement the OGC API client for the staging."""
|
|
43
|
+
|
|
44
|
+
# Init the OpenAPI instance from config file
|
|
45
|
+
openapi = OpenAPI.from_file_path(PATH_TO_YAML_OPENAPI)
|
|
46
|
+
|
|
47
|
+
@property
|
|
48
|
+
def href_service(self) -> str:
|
|
49
|
+
"""
|
|
50
|
+
Return the RS-Server staging URL hostname.
|
|
51
|
+
This URL can be overwritten using the RSPY_HOST_STAGING env variable (used e.g. for local mode).
|
|
52
|
+
Otherwise it should just be the RS-Server URL.
|
|
53
|
+
"""
|
|
54
|
+
return get_href_service(self.rs_server_href, "RSPY_HOST_STAGING")
|
|
55
|
+
|
|
56
|
+
def run_staging( # pylint: disable=too-many-locals
|
|
57
|
+
self,
|
|
58
|
+
stac_input: dict[Any, Any] | str,
|
|
59
|
+
out_coll_name: str,
|
|
60
|
+
) -> dict:
|
|
61
|
+
"""Method to start the staging process from rs-client - Call the endpoint /processes/staging/execution
|
|
62
|
+
|
|
63
|
+
Args:
|
|
64
|
+
stac_input (dict | str): input dictionary: the stac_input can have different format. It can be:
|
|
65
|
+
- A Python dictionary corresponding to a Feature or a FeatureCollection (that can be for example
|
|
66
|
+
the output of a search for Cadip or Auxip sessions)
|
|
67
|
+
- A json string corresponding to a Feature or a FeatureCollection
|
|
68
|
+
- A string corresponding to a path to a json file containing a Feature or a FeatureCollection
|
|
69
|
+
- A single link that returns a STAC itemCollection: this link should be an url to search a
|
|
70
|
+
itemCollection, for example:
|
|
71
|
+
http://localhost:8002/cadip/search?ids=S1A_20231120061537234567&collections=cadip_sentinel1
|
|
72
|
+
out_coll_name (str): name of the output collection
|
|
73
|
+
Return:
|
|
74
|
+
job_id (int, str): Returns the status code of the staging request + the identifier
|
|
75
|
+
(or None if staging endpoint fails) of the running job
|
|
76
|
+
"""
|
|
77
|
+
stac_input_dict = {}
|
|
78
|
+
is_an_url = False
|
|
79
|
+
|
|
80
|
+
# ----- Case 1: we only load a link (that refers to a STAC itemCollection) in the staging request body
|
|
81
|
+
if isinstance(stac_input, str):
|
|
82
|
+
try:
|
|
83
|
+
is_an_url = self.is_url(stac_input)
|
|
84
|
+
except ValueError:
|
|
85
|
+
is_an_url = False
|
|
86
|
+
if is_an_url:
|
|
87
|
+
staging_body = {"inputs": {"collection": out_coll_name, "items": {"href": stac_input}}}
|
|
88
|
+
|
|
89
|
+
# ----- Case 2: we directly load a STAC ItemCollection in the staging request body
|
|
90
|
+
else:
|
|
91
|
+
# If stac_input is a file, load this file to a dictionary
|
|
92
|
+
if isinstance(stac_input, str):
|
|
93
|
+
# If the input is a valid path to a json_file, load this file
|
|
94
|
+
if os.path.exists(os.path.dirname(stac_input)) and stac_input.endswith(".json"):
|
|
95
|
+
# Read the yaml or json file
|
|
96
|
+
with open(stac_input, encoding="utf-8") as opened:
|
|
97
|
+
stac_file_to_dict = json.loads(opened.read())
|
|
98
|
+
stac_input_dict = stac_file_to_dict
|
|
99
|
+
# If the input string is not a path, try to convert the content of the string to a json dictionary
|
|
100
|
+
else:
|
|
101
|
+
try:
|
|
102
|
+
stac_input_dict = json.loads(stac_input)
|
|
103
|
+
except json.JSONDecodeError as e:
|
|
104
|
+
raise OgcValidationException(
|
|
105
|
+
f"""Invalid input format: {stac_input} - Input data must be either:
|
|
106
|
+
- A Python dictionary corresponding to a Feature or a FeatureCollection
|
|
107
|
+
(that can be for example the output of a search for Cadip or Auxip sessions)
|
|
108
|
+
- A json string corresponding to a Feature or a FeatureCollection
|
|
109
|
+
- A string corresponding to a path to a json file containing a Feature or a
|
|
110
|
+
FeatureCollection
|
|
111
|
+
- A single link that returns a STAC ItemCollection: this link should be an
|
|
112
|
+
url to search an ItemCollection
|
|
113
|
+
""".strip(),
|
|
114
|
+
) from e
|
|
115
|
+
else:
|
|
116
|
+
stac_input_dict = stac_input
|
|
117
|
+
|
|
118
|
+
if "type" not in stac_input_dict:
|
|
119
|
+
raise KeyError("Key 'type' is missing from the staging input data")
|
|
120
|
+
|
|
121
|
+
# Validate input data using Pydantic
|
|
122
|
+
if stac_input_dict["type"] == "Feature":
|
|
123
|
+
stac_item = Item(**stac_input_dict)
|
|
124
|
+
stac_item_collection = ItemCollection(
|
|
125
|
+
**{
|
|
126
|
+
"type": "FeatureCollection",
|
|
127
|
+
"context": {"limit": 1000, "returned": 2},
|
|
128
|
+
"features": [stac_item],
|
|
129
|
+
}, # type: ignore
|
|
130
|
+
)
|
|
131
|
+
else:
|
|
132
|
+
stac_item_collection = ItemCollection(**stac_input_dict)
|
|
133
|
+
|
|
134
|
+
staging_body = {
|
|
135
|
+
"inputs": {
|
|
136
|
+
"collection": out_coll_name,
|
|
137
|
+
"items": {"value": stac_item_collection.model_dump(mode="json")},
|
|
138
|
+
},
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
# Run the process
|
|
142
|
+
return self.run_process(RESOURCE, staging_body)
|