rs-client-libraries 0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- config/staging_templates/README.md +17 -0
- config/staging_templates/yaml/additionalParameter.yaml +30 -0
- config/staging_templates/yaml/bbox.yaml +34 -0
- config/staging_templates/yaml/binaryInputValue.yaml +15 -0
- config/staging_templates/yaml/confClasses.yaml +22 -0
- config/staging_templates/yaml/descriptionType.yaml +36 -0
- config/staging_templates/yaml/dpr_openapi_schema.yaml +223 -0
- config/staging_templates/yaml/exception.yaml +31 -0
- config/staging_templates/yaml/execute.yaml +34 -0
- config/staging_templates/yaml/format.yaml +25 -0
- config/staging_templates/yaml/inlineOrRefData.yaml +18 -0
- config/staging_templates/yaml/inputDescription.yaml +32 -0
- config/staging_templates/yaml/inputValue.yaml +17 -0
- config/staging_templates/yaml/inputValueNoObject.yaml +22 -0
- config/staging_templates/yaml/jobControlOptions.yaml +19 -0
- config/staging_templates/yaml/jobList.yaml +27 -0
- config/staging_templates/yaml/landingPage.yaml +28 -0
- config/staging_templates/yaml/link.yaml +31 -0
- config/staging_templates/yaml/metadata.yaml +22 -0
- config/staging_templates/yaml/output.yaml +20 -0
- config/staging_templates/yaml/outputDescription.yaml +22 -0
- config/staging_templates/yaml/process.yaml +23 -0
- config/staging_templates/yaml/processList.yaml +27 -0
- config/staging_templates/yaml/processSummary.yaml +37 -0
- config/staging_templates/yaml/qualifiedInputValue.yaml +22 -0
- config/staging_templates/yaml/reference.yaml +21 -0
- config/staging_templates/yaml/results.yaml +16 -0
- config/staging_templates/yaml/schema.yaml +144 -0
- config/staging_templates/yaml/staging_openapi_schema.yaml +223 -0
- config/staging_templates/yaml/statusCode.yaml +22 -0
- config/staging_templates/yaml/statusInfo.yaml +52 -0
- config/staging_templates/yaml/subscriber.yaml +33 -0
- config/staging_templates/yaml/transmissionMode.yaml +19 -0
- rs_client/__init__.py +18 -0
- rs_client/ogcapi/__init__.py +15 -0
- rs_client/ogcapi/dpr_client.py +231 -0
- rs_client/ogcapi/ogcapi_client.py +282 -0
- rs_client/ogcapi/staging_client.py +178 -0
- rs_client/rs_client.py +277 -0
- rs_client/stac/__init__.py +15 -0
- rs_client/stac/auxip_client.py +70 -0
- rs_client/stac/cadip_client.py +70 -0
- rs_client/stac/catalog_client.py +479 -0
- rs_client/stac/stac_base.py +313 -0
- rs_client_libraries-0.2.dist-info/LICENSE +201 -0
- rs_client_libraries-0.2.dist-info/METADATA +44 -0
- rs_client_libraries-0.2.dist-info/RECORD +62 -0
- rs_client_libraries-0.2.dist-info/WHEEL +4 -0
- rs_common/__init__.py +20 -0
- rs_common/config.py +69 -0
- rs_common/init_opentelemetry.py +250 -0
- rs_common/logging.py +107 -0
- rs_common/prefect_utils.py +407 -0
- rs_common/utils.py +70 -0
- rs_workflows/__init__.py +15 -0
- rs_workflows/auxip_flow.py +70 -0
- rs_workflows/cadip_flow.py +69 -0
- rs_workflows/catalog_flow.py +71 -0
- rs_workflows/dpr_flow.py +272 -0
- rs_workflows/flow_utils.py +127 -0
- rs_workflows/on_demand_processing.py +243 -0
- rs_workflows/staging_flow.py +78 -0
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# Copyright 2024 CS Group
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
type: string
|
|
16
|
+
enum:
|
|
17
|
+
- value
|
|
18
|
+
- reference
|
|
19
|
+
default: value
|
rs_client/__init__.py
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# Copyright 2024 CS Group
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""rsclient main module."""
|
|
16
|
+
|
|
17
|
+
# Set automatically by running `poetry dynamic-versioning`
|
|
18
|
+
__version__ = "0.2"
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# Copyright 2024 CS Group
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""ogc api modules."""
|
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
# Copyright 2024 CS Group
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""Use "DPR as a service" implemented by rs-dpr-service"""
|
|
16
|
+
|
|
17
|
+
import ast
|
|
18
|
+
import os.path as osp
|
|
19
|
+
import tempfile
|
|
20
|
+
|
|
21
|
+
import yaml
|
|
22
|
+
from openapi_core import OpenAPI # Spec, validate_request, validate_response
|
|
23
|
+
|
|
24
|
+
from rs_client.ogcapi.ogcapi_client import OgcApiClient
|
|
25
|
+
from rs_common import prefect_utils
|
|
26
|
+
from rs_common.utils import get_href_service
|
|
27
|
+
|
|
28
|
+
PATH_TO_YAML_OPENAPI = osp.realpath(
|
|
29
|
+
osp.join(
|
|
30
|
+
osp.dirname(__file__),
|
|
31
|
+
"../../config",
|
|
32
|
+
"staging_templates",
|
|
33
|
+
"yaml",
|
|
34
|
+
"dpr_openapi_schema.yaml",
|
|
35
|
+
),
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class DprClient(OgcApiClient):
|
|
40
|
+
"""Implement the OGC API client for 'DPR as a service'."""
|
|
41
|
+
|
|
42
|
+
##########################################
|
|
43
|
+
# Override parent methods and attributes #
|
|
44
|
+
##########################################
|
|
45
|
+
|
|
46
|
+
# Init the OpenAPI instance from config file
|
|
47
|
+
openapi = OpenAPI.from_file_path(PATH_TO_YAML_OPENAPI)
|
|
48
|
+
|
|
49
|
+
@property
|
|
50
|
+
def endpoint_prefix(self) -> str:
|
|
51
|
+
"""Return the endpoints prefix, if any."""
|
|
52
|
+
return "dpr/"
|
|
53
|
+
|
|
54
|
+
@property
|
|
55
|
+
def href_service(self) -> str:
|
|
56
|
+
"""
|
|
57
|
+
Return the rs-dpr-service URL hostname.
|
|
58
|
+
This URL can be overwritten using the RSPY_HOST_DPR_SERVICE env variable (used e.g. for local mode).
|
|
59
|
+
Otherwise it should just be the RS-Server URL.
|
|
60
|
+
"""
|
|
61
|
+
return get_href_service(self.rs_server_href, "RSPY_HOST_DPR_SERVICE")
|
|
62
|
+
|
|
63
|
+
def run_process(
|
|
64
|
+
self,
|
|
65
|
+
process: str,
|
|
66
|
+
s3_config_dir: str,
|
|
67
|
+
payload_subpath: str,
|
|
68
|
+
s3_report_dir: str | None,
|
|
69
|
+
use_dpr_mockup: bool = False,
|
|
70
|
+
) -> dict:
|
|
71
|
+
"""Method to start the process from rs-client - Call the endpoint /processes/{process}/execution
|
|
72
|
+
|
|
73
|
+
Args:
|
|
74
|
+
process: Process name
|
|
75
|
+
s3_config_dir: S3 bucket folder that contains the payload and configuration files to pass to the processor
|
|
76
|
+
payload_subpath: Payload file path, relative to the config folder
|
|
77
|
+
s3_report_dir: S3 bucket folder were the processor report files will be written (optional). All the eopf
|
|
78
|
+
local files written in the local "./reports" directory will be pushed to this S3 bucket folder.
|
|
79
|
+
use_dpr_mockup: Use the real or the mockup DPR processor ?
|
|
80
|
+
|
|
81
|
+
Return:
|
|
82
|
+
job_id (int, str): Returns the status code of the request + the identifier
|
|
83
|
+
(or None if endpoint fails) of the running job
|
|
84
|
+
"""
|
|
85
|
+
|
|
86
|
+
# Data to pass to the real processor
|
|
87
|
+
data = {}
|
|
88
|
+
if not use_dpr_mockup:
|
|
89
|
+
data = {
|
|
90
|
+
"s3_config_dir": s3_config_dir,
|
|
91
|
+
"payload_subpath": payload_subpath,
|
|
92
|
+
"s3_report_dir": s3_report_dir,
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
# For the mockup processor, pass the payload contents.
|
|
96
|
+
# Download the payload file into a temp file.
|
|
97
|
+
else:
|
|
98
|
+
with tempfile.NamedTemporaryFile() as temp:
|
|
99
|
+
prefect_utils.s3_download_file( # type: ignore
|
|
100
|
+
osp.join(s3_config_dir, payload_subpath),
|
|
101
|
+
temp.name,
|
|
102
|
+
_sync=True, # type: ignore
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
# Read it as a yaml file
|
|
106
|
+
with open(temp.name, encoding="utf-8") as opened:
|
|
107
|
+
data = yaml.safe_load(opened)
|
|
108
|
+
|
|
109
|
+
# Add extra info
|
|
110
|
+
data.update({"use_mockup": use_dpr_mockup})
|
|
111
|
+
|
|
112
|
+
# Call the parent method
|
|
113
|
+
return super()._run_process(process, data)
|
|
114
|
+
|
|
115
|
+
def run_conv_safe_zarr(self, payload: dict):
|
|
116
|
+
"""Method to start the safe to zarr conversion process from rs-client -
|
|
117
|
+
Call the endpoint /processes/conv_safe_zarr/execution
|
|
118
|
+
|
|
119
|
+
Args:
|
|
120
|
+
payload: Dictionary to pass to the processor,
|
|
121
|
+
containing input_safe_path - the s3 path of legacy product and
|
|
122
|
+
output_zarr_dir_path - the s3 path for the new zarr
|
|
123
|
+
Return:
|
|
124
|
+
job_id (int, str): Returns the status code of the request + the identifier
|
|
125
|
+
(or None if endpoint fails) of the running job
|
|
126
|
+
"""
|
|
127
|
+
return super()._run_process("conv_safe_zarr", payload)
|
|
128
|
+
|
|
129
|
+
def wait_for_job(self, *args, **kwargs) -> list[dict]: # type: ignore
|
|
130
|
+
"""
|
|
131
|
+
Wait for job to finish.
|
|
132
|
+
|
|
133
|
+
Returns:
|
|
134
|
+
EOPF results
|
|
135
|
+
"""
|
|
136
|
+
# Call parent method and parse results
|
|
137
|
+
job_status = super().wait_for_job(*args, **kwargs)
|
|
138
|
+
return ast.literal_eval(job_status["message"])
|
|
139
|
+
|
|
140
|
+
######################################################
|
|
141
|
+
# These endpoints are not implemented by the service #
|
|
142
|
+
######################################################
|
|
143
|
+
|
|
144
|
+
def get_processes(self) -> dict:
|
|
145
|
+
raise NotImplementedError
|
|
146
|
+
|
|
147
|
+
def get_jobs(self) -> dict:
|
|
148
|
+
raise NotImplementedError
|
|
149
|
+
|
|
150
|
+
def delete_job(self, _: str) -> dict:
|
|
151
|
+
raise NotImplementedError
|
|
152
|
+
|
|
153
|
+
def get_job_results(self, _: str) -> dict:
|
|
154
|
+
raise NotImplementedError
|
|
155
|
+
|
|
156
|
+
####################
|
|
157
|
+
# Specific methods #
|
|
158
|
+
####################
|
|
159
|
+
|
|
160
|
+
async def update_configuration(self, local_path: str, s3_path: str, is_payload: bool = False, **kwargs):
|
|
161
|
+
"""
|
|
162
|
+
Update local configuration file depending on the environment, upload it to the s3 bucket,
|
|
163
|
+
and initialize output bucket folders.
|
|
164
|
+
|
|
165
|
+
Args:
|
|
166
|
+
local_path: Local configuration file path
|
|
167
|
+
s3_path: S3 bucket path where to upload to modified updated configuration file
|
|
168
|
+
is_payload: Specific behavior for the processor payload files
|
|
169
|
+
kwargs: Specific environment variables to expand in the configuration file
|
|
170
|
+
"""
|
|
171
|
+
|
|
172
|
+
to_expand = {}
|
|
173
|
+
|
|
174
|
+
if self.local_mode:
|
|
175
|
+
# In local mode, replace the S3_xxx_CLUSTER env vars, by the env vars from the ~/.s3cfg
|
|
176
|
+
# config file, that contains access to the "real" s3 bucket
|
|
177
|
+
to_expand = {
|
|
178
|
+
"S3_ACCESSKEY_CLUSTER": "${access_key}",
|
|
179
|
+
"S3_SECRETKEY_CLUSTER": "${secret_key}",
|
|
180
|
+
"S3_ENDPOINT_CLUSTER": "${host_bucket}",
|
|
181
|
+
"S3_REGION_CLUSTER": "${bucket_location}",
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
else:
|
|
185
|
+
# In cluster mode, just use the "real" s3 bucket
|
|
186
|
+
to_expand = {
|
|
187
|
+
"S3_ACCESSKEY_CLUSTER": "${S3_ACCESSKEY}",
|
|
188
|
+
"S3_SECRETKEY_CLUSTER": "${S3_SECRETKEY}",
|
|
189
|
+
"S3_ENDPOINT_CLUSTER": "${S3_ENDPOINT}",
|
|
190
|
+
"S3_REGION_CLUSTER": "${S3_REGION}",
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
# Also expand the user-given parameters
|
|
194
|
+
to_expand.update(kwargs)
|
|
195
|
+
|
|
196
|
+
# Open the input local file
|
|
197
|
+
with open(local_path, encoding="utf-8") as opened:
|
|
198
|
+
contents = opened.read()
|
|
199
|
+
|
|
200
|
+
# Expand the env vars as $key, ${key} or %key%
|
|
201
|
+
for key, value in to_expand.items():
|
|
202
|
+
for key2 in f"${key}", f"${{{key}}}", f"%{key}%":
|
|
203
|
+
contents = contents.replace(key2, str(value))
|
|
204
|
+
|
|
205
|
+
# Read the payload contents
|
|
206
|
+
if is_payload:
|
|
207
|
+
payload = yaml.safe_load(contents)
|
|
208
|
+
|
|
209
|
+
# We need to create the output S3 folder with a dummy file before running DPR
|
|
210
|
+
for output_product in payload["I/O"]["output_products"]:
|
|
211
|
+
s3_output_dir = output_product["path"]
|
|
212
|
+
s3_empty_file = f"{s3_output_dir}/.empty"
|
|
213
|
+
self.logger.info(f"Write empty file: {self.logger.level} {s3_empty_file!r}")
|
|
214
|
+
await prefect_utils.s3_upload_empty_file(s3_empty_file)
|
|
215
|
+
|
|
216
|
+
# Change the dask authentication for local mode
|
|
217
|
+
cluster_config = payload["dask_context"]["cluster_config"]
|
|
218
|
+
if self.local_mode:
|
|
219
|
+
cluster_config["auth"] = cluster_config["auth_local_mode"]
|
|
220
|
+
del cluster_config["auth_local_mode"]
|
|
221
|
+
|
|
222
|
+
# yaml to str conversion
|
|
223
|
+
contents = yaml.dump(payload, default_flow_style=False, sort_keys=False)
|
|
224
|
+
|
|
225
|
+
# Write the modified contents to a temp file
|
|
226
|
+
with tempfile.NamedTemporaryFile() as tmp:
|
|
227
|
+
tmp.write(contents.encode("utf-8"))
|
|
228
|
+
tmp.flush()
|
|
229
|
+
|
|
230
|
+
# Upload the temp file to the s3 bucket
|
|
231
|
+
return await prefect_utils.s3_upload_file(tmp.name, s3_path)
|
|
@@ -0,0 +1,282 @@
|
|
|
1
|
+
# Copyright 2024 CS Group
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""Use OGC API services with rs-client-libraries"""
|
|
16
|
+
|
|
17
|
+
import json
|
|
18
|
+
import math
|
|
19
|
+
import time
|
|
20
|
+
from typing import Any
|
|
21
|
+
|
|
22
|
+
# openapi_core libraries used for endpoints validation
|
|
23
|
+
import requests
|
|
24
|
+
from openapi_core import OpenAPI # Spec, validate_request, validate_response
|
|
25
|
+
from openapi_core.contrib.requests import (
|
|
26
|
+
RequestsOpenAPIRequest,
|
|
27
|
+
RequestsOpenAPIResponse,
|
|
28
|
+
)
|
|
29
|
+
from requests import Response
|
|
30
|
+
from requests.models import PreparedRequest
|
|
31
|
+
|
|
32
|
+
from rs_client.rs_client import TIMEOUT, RsClient
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class OgcValidationException(Exception):
|
|
36
|
+
"""
|
|
37
|
+
Exception raised when an error occurs during the OGC validation of the endpoints
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class OgcApiClient(RsClient):
|
|
42
|
+
"""
|
|
43
|
+
Class to handle OGC API processes in rs-client-libraries
|
|
44
|
+
|
|
45
|
+
This class provides python methods to call the different endpoints of the OGC API service.
|
|
46
|
+
|
|
47
|
+
Remark: this class don't inherits from the owslib.ogcapi.processes.Processes class because the latter
|
|
48
|
+
doesn't provide wrapping for all endpoints defined in our services (it only provides the /processes
|
|
49
|
+
and /processes/{processId}/execution endpoints + it doesn't allow to manage apikey_header parameter which
|
|
50
|
+
is passed as an extra argument).
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
@property
|
|
54
|
+
def endpoint_prefix(self) -> str:
|
|
55
|
+
"""Return the endpoints prefix, if any."""
|
|
56
|
+
return "" # no prefix by default
|
|
57
|
+
|
|
58
|
+
@classmethod
|
|
59
|
+
def get_openapi(cls) -> OpenAPI:
|
|
60
|
+
"""Return the OpenAPI instance of the subclass"""
|
|
61
|
+
return cls.openapi # type: ignore # pylint: disable=no-member
|
|
62
|
+
|
|
63
|
+
def validate_and_unmarshal_request(self, request: PreparedRequest) -> Any:
|
|
64
|
+
"""Validate an endpoint request according to the ogc specifications
|
|
65
|
+
|
|
66
|
+
Args:
|
|
67
|
+
request (Request): endpoint request
|
|
68
|
+
|
|
69
|
+
Returns:
|
|
70
|
+
RequestUnmarshalResult.data: data validated by the openapi_core
|
|
71
|
+
unmarshal_response method
|
|
72
|
+
"""
|
|
73
|
+
openapi_request = RequestsOpenAPIRequest(request)
|
|
74
|
+
|
|
75
|
+
# validate request
|
|
76
|
+
result = self.get_openapi().unmarshal_request(openapi_request)
|
|
77
|
+
|
|
78
|
+
if result.errors:
|
|
79
|
+
raise OgcValidationException(
|
|
80
|
+
f"Error validating the request of the endpoint "
|
|
81
|
+
f"{openapi_request.path}: {', '.join(str(x) for x in result.errors)}",
|
|
82
|
+
)
|
|
83
|
+
if not result.body:
|
|
84
|
+
raise OgcValidationException(
|
|
85
|
+
f"Error validating the request of the endpoint "
|
|
86
|
+
f"{openapi_request.path}: 'data' field of RequestUnmarshalResult"
|
|
87
|
+
f"object is empty",
|
|
88
|
+
)
|
|
89
|
+
return result.body
|
|
90
|
+
|
|
91
|
+
def validate_and_unmarshal_response(self, response: Response) -> Any:
|
|
92
|
+
"""
|
|
93
|
+
Validate an endpoint response according to the ogc specifications
|
|
94
|
+
(described as yaml schemas)
|
|
95
|
+
|
|
96
|
+
Args:
|
|
97
|
+
response (Response): endpoint response
|
|
98
|
+
Returns:
|
|
99
|
+
ResponseUnmarshalResult.data: data validated by the openapi_core
|
|
100
|
+
unmarshal_response method
|
|
101
|
+
"""
|
|
102
|
+
openapi_request = RequestsOpenAPIRequest(response.request)
|
|
103
|
+
openapi_response = RequestsOpenAPIResponse(response)
|
|
104
|
+
|
|
105
|
+
# validate response
|
|
106
|
+
result = self.get_openapi().unmarshal_response(openapi_request, openapi_response) # type: ignore
|
|
107
|
+
if result.errors:
|
|
108
|
+
raise OgcValidationException( # type: ignore
|
|
109
|
+
f"Error validating the response of the endpoint {openapi_request.path} - "
|
|
110
|
+
f"Server response content: {response.json()} - "
|
|
111
|
+
f"Validation error of the server response: {', '.join(str(x) for x in result.errors)}",
|
|
112
|
+
)
|
|
113
|
+
if not result.data:
|
|
114
|
+
raise OgcValidationException(
|
|
115
|
+
f"Error validating the response of the endpoint "
|
|
116
|
+
f"{openapi_request.path}: 'data' field of ResponseUnmarshalResult"
|
|
117
|
+
f"object is empty",
|
|
118
|
+
)
|
|
119
|
+
return json.loads(response.content)
|
|
120
|
+
|
|
121
|
+
##################################
|
|
122
|
+
# Call OGC API service endpoints #
|
|
123
|
+
##################################
|
|
124
|
+
|
|
125
|
+
def get_processes(self) -> dict:
|
|
126
|
+
"""_summary_
|
|
127
|
+
|
|
128
|
+
Returns:
|
|
129
|
+
dict: dictionary containing the content of the response
|
|
130
|
+
"""
|
|
131
|
+
response = self.http_session.get(
|
|
132
|
+
url=f"{self.href_service}/{self.endpoint_prefix}processes",
|
|
133
|
+
timeout=TIMEOUT,
|
|
134
|
+
**self.apikey_headers,
|
|
135
|
+
)
|
|
136
|
+
return self.validate_and_unmarshal_response(response)
|
|
137
|
+
|
|
138
|
+
def get_process(self, process_id: str) -> dict:
|
|
139
|
+
"""
|
|
140
|
+
Wrapper to get a specific process
|
|
141
|
+
Args:
|
|
142
|
+
process_id (str): name of the resource
|
|
143
|
+
"""
|
|
144
|
+
response = self.http_session.get(
|
|
145
|
+
url=f"{self.href_service}/{self.endpoint_prefix}processes/{process_id}",
|
|
146
|
+
timeout=TIMEOUT,
|
|
147
|
+
**self.apikey_headers,
|
|
148
|
+
)
|
|
149
|
+
return self.validate_and_unmarshal_response(response)
|
|
150
|
+
|
|
151
|
+
def _run_process(self, process: str, body: dict) -> dict:
|
|
152
|
+
"""Method to start the process from rs-client - Call the endpoint /processes/{process}/execution
|
|
153
|
+
|
|
154
|
+
Args:
|
|
155
|
+
process: Process name
|
|
156
|
+
body: POST HTTP request body as a JSON dict
|
|
157
|
+
|
|
158
|
+
Return:
|
|
159
|
+
job_id (int, str): Returns the status code of the request + the identifier
|
|
160
|
+
(or None if endpoint fails) of the running job
|
|
161
|
+
"""
|
|
162
|
+
|
|
163
|
+
# Check that the request containing the body is valid
|
|
164
|
+
request = requests.Request(
|
|
165
|
+
method="POST",
|
|
166
|
+
url=f"{self.href_service}/{self.endpoint_prefix}processes/{process}/execution",
|
|
167
|
+
json=body,
|
|
168
|
+
).prepare()
|
|
169
|
+
|
|
170
|
+
# Validate the body of the request that will be sent to the service
|
|
171
|
+
self.validate_and_unmarshal_request(request)
|
|
172
|
+
|
|
173
|
+
response = self.http_session.post(
|
|
174
|
+
url=f"{self.href_service}/{self.endpoint_prefix}processes/{process}/execution",
|
|
175
|
+
json=body,
|
|
176
|
+
**self.apikey_headers,
|
|
177
|
+
timeout=TIMEOUT,
|
|
178
|
+
)
|
|
179
|
+
return self.validate_and_unmarshal_response(response)
|
|
180
|
+
|
|
181
|
+
def get_jobs(self) -> dict:
|
|
182
|
+
"""Method to get running jobs"""
|
|
183
|
+
response = self.http_session.get(
|
|
184
|
+
url=f"{self.href_service}/{self.endpoint_prefix}jobs",
|
|
185
|
+
**self.apikey_headers,
|
|
186
|
+
timeout=TIMEOUT,
|
|
187
|
+
)
|
|
188
|
+
return self.validate_and_unmarshal_response(response)
|
|
189
|
+
|
|
190
|
+
def get_job_info(self, job_id: str) -> dict: # pylint: disable=too-many-locals
|
|
191
|
+
"""Method to get a specific job response"""
|
|
192
|
+
response = self.http_session.get(
|
|
193
|
+
url=f"{self.href_service}/{self.endpoint_prefix}jobs/{job_id}",
|
|
194
|
+
**self.apikey_headers,
|
|
195
|
+
timeout=TIMEOUT,
|
|
196
|
+
)
|
|
197
|
+
return self.validate_and_unmarshal_response(response)
|
|
198
|
+
|
|
199
|
+
def delete_job(self, job_id: str) -> dict: # pylint: disable=too-many-locals
|
|
200
|
+
"""Method to get a specific job response"""
|
|
201
|
+
response = self.http_session.delete(
|
|
202
|
+
url=f"{self.href_service}/{self.endpoint_prefix}jobs/{job_id}",
|
|
203
|
+
**self.apikey_headers,
|
|
204
|
+
timeout=TIMEOUT,
|
|
205
|
+
)
|
|
206
|
+
return self.validate_and_unmarshal_response(response)
|
|
207
|
+
|
|
208
|
+
def get_job_results(self, job_id: str) -> dict:
|
|
209
|
+
"""Wrapper to get the result of a specfific job
|
|
210
|
+
|
|
211
|
+
Args:
|
|
212
|
+
job_id (str): _description_
|
|
213
|
+
"""
|
|
214
|
+
response = self.http_session.get(
|
|
215
|
+
url=f"{self.href_service}/{self.endpoint_prefix}jobs/{job_id}/results",
|
|
216
|
+
timeout=TIMEOUT,
|
|
217
|
+
**self.apikey_headers,
|
|
218
|
+
)
|
|
219
|
+
return self.validate_and_unmarshal_response(response)
|
|
220
|
+
|
|
221
|
+
def wait_for_job(
|
|
222
|
+
self,
|
|
223
|
+
job_status: dict,
|
|
224
|
+
logger=None,
|
|
225
|
+
job_name: str = "",
|
|
226
|
+
timeout: int | float = math.inf,
|
|
227
|
+
poll_interval: int = 2,
|
|
228
|
+
) -> dict:
|
|
229
|
+
"""
|
|
230
|
+
Wait for job to finish.
|
|
231
|
+
|
|
232
|
+
Args:
|
|
233
|
+
job_status: Returned by `_run_process`
|
|
234
|
+
logger: To show advancement in logger
|
|
235
|
+
job_name: Job name to show in the logger
|
|
236
|
+
timeout: Job completion timeout in seconds
|
|
237
|
+
poll_interval: When to check again for job completion in seconds
|
|
238
|
+
|
|
239
|
+
Returns:
|
|
240
|
+
Job status
|
|
241
|
+
|
|
242
|
+
Raises:
|
|
243
|
+
RuntimeError in case of error
|
|
244
|
+
"""
|
|
245
|
+
try:
|
|
246
|
+
status_type = ""
|
|
247
|
+
job_identifier = job_status.get("jobID")
|
|
248
|
+
if not job_identifier:
|
|
249
|
+
raise RuntimeError("Job identifier is missing.")
|
|
250
|
+
|
|
251
|
+
while True:
|
|
252
|
+
job_status = self.get_job_info(job_identifier)
|
|
253
|
+
if logger:
|
|
254
|
+
logger.info(f"job_status: {job_status}")
|
|
255
|
+
status_type = job_status.get("status", "")
|
|
256
|
+
if logger:
|
|
257
|
+
logger.info(
|
|
258
|
+
f"----- {job_name} job {job_identifier!r}: {status_type.upper()} \n",
|
|
259
|
+
)
|
|
260
|
+
|
|
261
|
+
# Exit the loop
|
|
262
|
+
if status_type in {"successful", "failed", "dismissed"}:
|
|
263
|
+
break
|
|
264
|
+
|
|
265
|
+
# Or sleep n seconds and try again
|
|
266
|
+
timeout -= poll_interval
|
|
267
|
+
if timeout <= 0:
|
|
268
|
+
raise TimeoutError(f"Timed out while waiting for {job_name} job {job_identifier!r}")
|
|
269
|
+
time.sleep(poll_interval)
|
|
270
|
+
|
|
271
|
+
# Log all exceptions except the timeout
|
|
272
|
+
except TimeoutError:
|
|
273
|
+
raise
|
|
274
|
+
except Exception as e:
|
|
275
|
+
raise RuntimeError(f"Exception while monitoring job {job_name}: {job_status}") from e
|
|
276
|
+
|
|
277
|
+
if status_type == "successful":
|
|
278
|
+
if logger:
|
|
279
|
+
logger.info(f"----- {job_name} job {job_identifier!r}: COMPLETED \n")
|
|
280
|
+
return job_status
|
|
281
|
+
|
|
282
|
+
raise RuntimeError(f"{job_name} job {job_identifier!r}: FAILED")
|