rs-client-libraries 0.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. config/staging_templates/README.md +17 -0
  2. config/staging_templates/yaml/additionalParameter.yaml +30 -0
  3. config/staging_templates/yaml/bbox.yaml +34 -0
  4. config/staging_templates/yaml/binaryInputValue.yaml +15 -0
  5. config/staging_templates/yaml/confClasses.yaml +22 -0
  6. config/staging_templates/yaml/descriptionType.yaml +36 -0
  7. config/staging_templates/yaml/dpr_openapi_schema.yaml +223 -0
  8. config/staging_templates/yaml/exception.yaml +31 -0
  9. config/staging_templates/yaml/execute.yaml +34 -0
  10. config/staging_templates/yaml/format.yaml +25 -0
  11. config/staging_templates/yaml/inlineOrRefData.yaml +18 -0
  12. config/staging_templates/yaml/inputDescription.yaml +32 -0
  13. config/staging_templates/yaml/inputValue.yaml +17 -0
  14. config/staging_templates/yaml/inputValueNoObject.yaml +22 -0
  15. config/staging_templates/yaml/jobControlOptions.yaml +19 -0
  16. config/staging_templates/yaml/jobList.yaml +27 -0
  17. config/staging_templates/yaml/landingPage.yaml +28 -0
  18. config/staging_templates/yaml/link.yaml +31 -0
  19. config/staging_templates/yaml/metadata.yaml +22 -0
  20. config/staging_templates/yaml/output.yaml +20 -0
  21. config/staging_templates/yaml/outputDescription.yaml +22 -0
  22. config/staging_templates/yaml/process.yaml +23 -0
  23. config/staging_templates/yaml/processList.yaml +27 -0
  24. config/staging_templates/yaml/processSummary.yaml +37 -0
  25. config/staging_templates/yaml/qualifiedInputValue.yaml +22 -0
  26. config/staging_templates/yaml/reference.yaml +21 -0
  27. config/staging_templates/yaml/results.yaml +16 -0
  28. config/staging_templates/yaml/schema.yaml +144 -0
  29. config/staging_templates/yaml/staging_openapi_schema.yaml +223 -0
  30. config/staging_templates/yaml/statusCode.yaml +22 -0
  31. config/staging_templates/yaml/statusInfo.yaml +52 -0
  32. config/staging_templates/yaml/subscriber.yaml +33 -0
  33. config/staging_templates/yaml/transmissionMode.yaml +19 -0
  34. rs_client/__init__.py +18 -0
  35. rs_client/ogcapi/__init__.py +15 -0
  36. rs_client/ogcapi/dpr_client.py +231 -0
  37. rs_client/ogcapi/ogcapi_client.py +282 -0
  38. rs_client/ogcapi/staging_client.py +178 -0
  39. rs_client/rs_client.py +277 -0
  40. rs_client/stac/__init__.py +15 -0
  41. rs_client/stac/auxip_client.py +70 -0
  42. rs_client/stac/cadip_client.py +70 -0
  43. rs_client/stac/catalog_client.py +479 -0
  44. rs_client/stac/stac_base.py +313 -0
  45. rs_client_libraries-0.2.dist-info/LICENSE +201 -0
  46. rs_client_libraries-0.2.dist-info/METADATA +44 -0
  47. rs_client_libraries-0.2.dist-info/RECORD +62 -0
  48. rs_client_libraries-0.2.dist-info/WHEEL +4 -0
  49. rs_common/__init__.py +20 -0
  50. rs_common/config.py +69 -0
  51. rs_common/init_opentelemetry.py +250 -0
  52. rs_common/logging.py +107 -0
  53. rs_common/prefect_utils.py +407 -0
  54. rs_common/utils.py +70 -0
  55. rs_workflows/__init__.py +15 -0
  56. rs_workflows/auxip_flow.py +70 -0
  57. rs_workflows/cadip_flow.py +69 -0
  58. rs_workflows/catalog_flow.py +71 -0
  59. rs_workflows/dpr_flow.py +272 -0
  60. rs_workflows/flow_utils.py +127 -0
  61. rs_workflows/on_demand_processing.py +243 -0
  62. rs_workflows/staging_flow.py +78 -0
@@ -0,0 +1,19 @@
1
+ # Copyright 2024 CS Group
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ type: string
16
+ enum:
17
+ - value
18
+ - reference
19
+ default: value
rs_client/__init__.py ADDED
@@ -0,0 +1,18 @@
1
+ # Copyright 2024 CS Group
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """rsclient main module."""
16
+
17
+ # Set automatically by running `poetry dynamic-versioning`
18
+ __version__ = "0.2"
@@ -0,0 +1,15 @@
1
+ # Copyright 2024 CS Group
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """ogc api modules."""
@@ -0,0 +1,231 @@
1
+ # Copyright 2024 CS Group
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """Use "DPR as a service" implemented by rs-dpr-service"""
16
+
17
+ import ast
18
+ import os.path as osp
19
+ import tempfile
20
+
21
+ import yaml
22
+ from openapi_core import OpenAPI # Spec, validate_request, validate_response
23
+
24
+ from rs_client.ogcapi.ogcapi_client import OgcApiClient
25
+ from rs_common import prefect_utils
26
+ from rs_common.utils import get_href_service
27
+
28
+ PATH_TO_YAML_OPENAPI = osp.realpath(
29
+ osp.join(
30
+ osp.dirname(__file__),
31
+ "../../config",
32
+ "staging_templates",
33
+ "yaml",
34
+ "dpr_openapi_schema.yaml",
35
+ ),
36
+ )
37
+
38
+
39
+ class DprClient(OgcApiClient):
40
+ """Implement the OGC API client for 'DPR as a service'."""
41
+
42
+ ##########################################
43
+ # Override parent methods and attributes #
44
+ ##########################################
45
+
46
+ # Init the OpenAPI instance from config file
47
+ openapi = OpenAPI.from_file_path(PATH_TO_YAML_OPENAPI)
48
+
49
+ @property
50
+ def endpoint_prefix(self) -> str:
51
+ """Return the endpoints prefix, if any."""
52
+ return "dpr/"
53
+
54
+ @property
55
+ def href_service(self) -> str:
56
+ """
57
+ Return the rs-dpr-service URL hostname.
58
+ This URL can be overwritten using the RSPY_HOST_DPR_SERVICE env variable (used e.g. for local mode).
59
+ Otherwise it should just be the RS-Server URL.
60
+ """
61
+ return get_href_service(self.rs_server_href, "RSPY_HOST_DPR_SERVICE")
62
+
63
+ def run_process(
64
+ self,
65
+ process: str,
66
+ s3_config_dir: str,
67
+ payload_subpath: str,
68
+ s3_report_dir: str | None,
69
+ use_dpr_mockup: bool = False,
70
+ ) -> dict:
71
+ """Method to start the process from rs-client - Call the endpoint /processes/{process}/execution
72
+
73
+ Args:
74
+ process: Process name
75
+ s3_config_dir: S3 bucket folder that contains the payload and configuration files to pass to the processor
76
+ payload_subpath: Payload file path, relative to the config folder
77
+ s3_report_dir: S3 bucket folder were the processor report files will be written (optional). All the eopf
78
+ local files written in the local "./reports" directory will be pushed to this S3 bucket folder.
79
+ use_dpr_mockup: Use the real or the mockup DPR processor ?
80
+
81
+ Return:
82
+ job_id (int, str): Returns the status code of the request + the identifier
83
+ (or None if endpoint fails) of the running job
84
+ """
85
+
86
+ # Data to pass to the real processor
87
+ data = {}
88
+ if not use_dpr_mockup:
89
+ data = {
90
+ "s3_config_dir": s3_config_dir,
91
+ "payload_subpath": payload_subpath,
92
+ "s3_report_dir": s3_report_dir,
93
+ }
94
+
95
+ # For the mockup processor, pass the payload contents.
96
+ # Download the payload file into a temp file.
97
+ else:
98
+ with tempfile.NamedTemporaryFile() as temp:
99
+ prefect_utils.s3_download_file( # type: ignore
100
+ osp.join(s3_config_dir, payload_subpath),
101
+ temp.name,
102
+ _sync=True, # type: ignore
103
+ )
104
+
105
+ # Read it as a yaml file
106
+ with open(temp.name, encoding="utf-8") as opened:
107
+ data = yaml.safe_load(opened)
108
+
109
+ # Add extra info
110
+ data.update({"use_mockup": use_dpr_mockup})
111
+
112
+ # Call the parent method
113
+ return super()._run_process(process, data)
114
+
115
+ def run_conv_safe_zarr(self, payload: dict):
116
+ """Method to start the safe to zarr conversion process from rs-client -
117
+ Call the endpoint /processes/conv_safe_zarr/execution
118
+
119
+ Args:
120
+ payload: Dictionary to pass to the processor,
121
+ containing input_safe_path - the s3 path of legacy product and
122
+ output_zarr_dir_path - the s3 path for the new zarr
123
+ Return:
124
+ job_id (int, str): Returns the status code of the request + the identifier
125
+ (or None if endpoint fails) of the running job
126
+ """
127
+ return super()._run_process("conv_safe_zarr", payload)
128
+
129
+ def wait_for_job(self, *args, **kwargs) -> list[dict]: # type: ignore
130
+ """
131
+ Wait for job to finish.
132
+
133
+ Returns:
134
+ EOPF results
135
+ """
136
+ # Call parent method and parse results
137
+ job_status = super().wait_for_job(*args, **kwargs)
138
+ return ast.literal_eval(job_status["message"])
139
+
140
+ ######################################################
141
+ # These endpoints are not implemented by the service #
142
+ ######################################################
143
+
144
+ def get_processes(self) -> dict:
145
+ raise NotImplementedError
146
+
147
+ def get_jobs(self) -> dict:
148
+ raise NotImplementedError
149
+
150
+ def delete_job(self, _: str) -> dict:
151
+ raise NotImplementedError
152
+
153
+ def get_job_results(self, _: str) -> dict:
154
+ raise NotImplementedError
155
+
156
+ ####################
157
+ # Specific methods #
158
+ ####################
159
+
160
+ async def update_configuration(self, local_path: str, s3_path: str, is_payload: bool = False, **kwargs):
161
+ """
162
+ Update local configuration file depending on the environment, upload it to the s3 bucket,
163
+ and initialize output bucket folders.
164
+
165
+ Args:
166
+ local_path: Local configuration file path
167
+ s3_path: S3 bucket path where to upload to modified updated configuration file
168
+ is_payload: Specific behavior for the processor payload files
169
+ kwargs: Specific environment variables to expand in the configuration file
170
+ """
171
+
172
+ to_expand = {}
173
+
174
+ if self.local_mode:
175
+ # In local mode, replace the S3_xxx_CLUSTER env vars, by the env vars from the ~/.s3cfg
176
+ # config file, that contains access to the "real" s3 bucket
177
+ to_expand = {
178
+ "S3_ACCESSKEY_CLUSTER": "${access_key}",
179
+ "S3_SECRETKEY_CLUSTER": "${secret_key}",
180
+ "S3_ENDPOINT_CLUSTER": "${host_bucket}",
181
+ "S3_REGION_CLUSTER": "${bucket_location}",
182
+ }
183
+
184
+ else:
185
+ # In cluster mode, just use the "real" s3 bucket
186
+ to_expand = {
187
+ "S3_ACCESSKEY_CLUSTER": "${S3_ACCESSKEY}",
188
+ "S3_SECRETKEY_CLUSTER": "${S3_SECRETKEY}",
189
+ "S3_ENDPOINT_CLUSTER": "${S3_ENDPOINT}",
190
+ "S3_REGION_CLUSTER": "${S3_REGION}",
191
+ }
192
+
193
+ # Also expand the user-given parameters
194
+ to_expand.update(kwargs)
195
+
196
+ # Open the input local file
197
+ with open(local_path, encoding="utf-8") as opened:
198
+ contents = opened.read()
199
+
200
+ # Expand the env vars as $key, ${key} or %key%
201
+ for key, value in to_expand.items():
202
+ for key2 in f"${key}", f"${{{key}}}", f"%{key}%":
203
+ contents = contents.replace(key2, str(value))
204
+
205
+ # Read the payload contents
206
+ if is_payload:
207
+ payload = yaml.safe_load(contents)
208
+
209
+ # We need to create the output S3 folder with a dummy file before running DPR
210
+ for output_product in payload["I/O"]["output_products"]:
211
+ s3_output_dir = output_product["path"]
212
+ s3_empty_file = f"{s3_output_dir}/.empty"
213
+ self.logger.info(f"Write empty file: {self.logger.level} {s3_empty_file!r}")
214
+ await prefect_utils.s3_upload_empty_file(s3_empty_file)
215
+
216
+ # Change the dask authentication for local mode
217
+ cluster_config = payload["dask_context"]["cluster_config"]
218
+ if self.local_mode:
219
+ cluster_config["auth"] = cluster_config["auth_local_mode"]
220
+ del cluster_config["auth_local_mode"]
221
+
222
+ # yaml to str conversion
223
+ contents = yaml.dump(payload, default_flow_style=False, sort_keys=False)
224
+
225
+ # Write the modified contents to a temp file
226
+ with tempfile.NamedTemporaryFile() as tmp:
227
+ tmp.write(contents.encode("utf-8"))
228
+ tmp.flush()
229
+
230
+ # Upload the temp file to the s3 bucket
231
+ return await prefect_utils.s3_upload_file(tmp.name, s3_path)
@@ -0,0 +1,282 @@
1
+ # Copyright 2024 CS Group
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """Use OGC API services with rs-client-libraries"""
16
+
17
+ import json
18
+ import math
19
+ import time
20
+ from typing import Any
21
+
22
+ # openapi_core libraries used for endpoints validation
23
+ import requests
24
+ from openapi_core import OpenAPI # Spec, validate_request, validate_response
25
+ from openapi_core.contrib.requests import (
26
+ RequestsOpenAPIRequest,
27
+ RequestsOpenAPIResponse,
28
+ )
29
+ from requests import Response
30
+ from requests.models import PreparedRequest
31
+
32
+ from rs_client.rs_client import TIMEOUT, RsClient
33
+
34
+
35
+ class OgcValidationException(Exception):
36
+ """
37
+ Exception raised when an error occurs during the OGC validation of the endpoints
38
+ """
39
+
40
+
41
+ class OgcApiClient(RsClient):
42
+ """
43
+ Class to handle OGC API processes in rs-client-libraries
44
+
45
+ This class provides python methods to call the different endpoints of the OGC API service.
46
+
47
+ Remark: this class don't inherits from the owslib.ogcapi.processes.Processes class because the latter
48
+ doesn't provide wrapping for all endpoints defined in our services (it only provides the /processes
49
+ and /processes/{processId}/execution endpoints + it doesn't allow to manage apikey_header parameter which
50
+ is passed as an extra argument).
51
+ """
52
+
53
+ @property
54
+ def endpoint_prefix(self) -> str:
55
+ """Return the endpoints prefix, if any."""
56
+ return "" # no prefix by default
57
+
58
+ @classmethod
59
+ def get_openapi(cls) -> OpenAPI:
60
+ """Return the OpenAPI instance of the subclass"""
61
+ return cls.openapi # type: ignore # pylint: disable=no-member
62
+
63
+ def validate_and_unmarshal_request(self, request: PreparedRequest) -> Any:
64
+ """Validate an endpoint request according to the ogc specifications
65
+
66
+ Args:
67
+ request (Request): endpoint request
68
+
69
+ Returns:
70
+ RequestUnmarshalResult.data: data validated by the openapi_core
71
+ unmarshal_response method
72
+ """
73
+ openapi_request = RequestsOpenAPIRequest(request)
74
+
75
+ # validate request
76
+ result = self.get_openapi().unmarshal_request(openapi_request)
77
+
78
+ if result.errors:
79
+ raise OgcValidationException(
80
+ f"Error validating the request of the endpoint "
81
+ f"{openapi_request.path}: {', '.join(str(x) for x in result.errors)}",
82
+ )
83
+ if not result.body:
84
+ raise OgcValidationException(
85
+ f"Error validating the request of the endpoint "
86
+ f"{openapi_request.path}: 'data' field of RequestUnmarshalResult"
87
+ f"object is empty",
88
+ )
89
+ return result.body
90
+
91
+ def validate_and_unmarshal_response(self, response: Response) -> Any:
92
+ """
93
+ Validate an endpoint response according to the ogc specifications
94
+ (described as yaml schemas)
95
+
96
+ Args:
97
+ response (Response): endpoint response
98
+ Returns:
99
+ ResponseUnmarshalResult.data: data validated by the openapi_core
100
+ unmarshal_response method
101
+ """
102
+ openapi_request = RequestsOpenAPIRequest(response.request)
103
+ openapi_response = RequestsOpenAPIResponse(response)
104
+
105
+ # validate response
106
+ result = self.get_openapi().unmarshal_response(openapi_request, openapi_response) # type: ignore
107
+ if result.errors:
108
+ raise OgcValidationException( # type: ignore
109
+ f"Error validating the response of the endpoint {openapi_request.path} - "
110
+ f"Server response content: {response.json()} - "
111
+ f"Validation error of the server response: {', '.join(str(x) for x in result.errors)}",
112
+ )
113
+ if not result.data:
114
+ raise OgcValidationException(
115
+ f"Error validating the response of the endpoint "
116
+ f"{openapi_request.path}: 'data' field of ResponseUnmarshalResult"
117
+ f"object is empty",
118
+ )
119
+ return json.loads(response.content)
120
+
121
+ ##################################
122
+ # Call OGC API service endpoints #
123
+ ##################################
124
+
125
+ def get_processes(self) -> dict:
126
+ """_summary_
127
+
128
+ Returns:
129
+ dict: dictionary containing the content of the response
130
+ """
131
+ response = self.http_session.get(
132
+ url=f"{self.href_service}/{self.endpoint_prefix}processes",
133
+ timeout=TIMEOUT,
134
+ **self.apikey_headers,
135
+ )
136
+ return self.validate_and_unmarshal_response(response)
137
+
138
+ def get_process(self, process_id: str) -> dict:
139
+ """
140
+ Wrapper to get a specific process
141
+ Args:
142
+ process_id (str): name of the resource
143
+ """
144
+ response = self.http_session.get(
145
+ url=f"{self.href_service}/{self.endpoint_prefix}processes/{process_id}",
146
+ timeout=TIMEOUT,
147
+ **self.apikey_headers,
148
+ )
149
+ return self.validate_and_unmarshal_response(response)
150
+
151
+ def _run_process(self, process: str, body: dict) -> dict:
152
+ """Method to start the process from rs-client - Call the endpoint /processes/{process}/execution
153
+
154
+ Args:
155
+ process: Process name
156
+ body: POST HTTP request body as a JSON dict
157
+
158
+ Return:
159
+ job_id (int, str): Returns the status code of the request + the identifier
160
+ (or None if endpoint fails) of the running job
161
+ """
162
+
163
+ # Check that the request containing the body is valid
164
+ request = requests.Request(
165
+ method="POST",
166
+ url=f"{self.href_service}/{self.endpoint_prefix}processes/{process}/execution",
167
+ json=body,
168
+ ).prepare()
169
+
170
+ # Validate the body of the request that will be sent to the service
171
+ self.validate_and_unmarshal_request(request)
172
+
173
+ response = self.http_session.post(
174
+ url=f"{self.href_service}/{self.endpoint_prefix}processes/{process}/execution",
175
+ json=body,
176
+ **self.apikey_headers,
177
+ timeout=TIMEOUT,
178
+ )
179
+ return self.validate_and_unmarshal_response(response)
180
+
181
+ def get_jobs(self) -> dict:
182
+ """Method to get running jobs"""
183
+ response = self.http_session.get(
184
+ url=f"{self.href_service}/{self.endpoint_prefix}jobs",
185
+ **self.apikey_headers,
186
+ timeout=TIMEOUT,
187
+ )
188
+ return self.validate_and_unmarshal_response(response)
189
+
190
+ def get_job_info(self, job_id: str) -> dict: # pylint: disable=too-many-locals
191
+ """Method to get a specific job response"""
192
+ response = self.http_session.get(
193
+ url=f"{self.href_service}/{self.endpoint_prefix}jobs/{job_id}",
194
+ **self.apikey_headers,
195
+ timeout=TIMEOUT,
196
+ )
197
+ return self.validate_and_unmarshal_response(response)
198
+
199
+ def delete_job(self, job_id: str) -> dict: # pylint: disable=too-many-locals
200
+ """Method to get a specific job response"""
201
+ response = self.http_session.delete(
202
+ url=f"{self.href_service}/{self.endpoint_prefix}jobs/{job_id}",
203
+ **self.apikey_headers,
204
+ timeout=TIMEOUT,
205
+ )
206
+ return self.validate_and_unmarshal_response(response)
207
+
208
+ def get_job_results(self, job_id: str) -> dict:
209
+ """Wrapper to get the result of a specfific job
210
+
211
+ Args:
212
+ job_id (str): _description_
213
+ """
214
+ response = self.http_session.get(
215
+ url=f"{self.href_service}/{self.endpoint_prefix}jobs/{job_id}/results",
216
+ timeout=TIMEOUT,
217
+ **self.apikey_headers,
218
+ )
219
+ return self.validate_and_unmarshal_response(response)
220
+
221
+ def wait_for_job(
222
+ self,
223
+ job_status: dict,
224
+ logger=None,
225
+ job_name: str = "",
226
+ timeout: int | float = math.inf,
227
+ poll_interval: int = 2,
228
+ ) -> dict:
229
+ """
230
+ Wait for job to finish.
231
+
232
+ Args:
233
+ job_status: Returned by `_run_process`
234
+ logger: To show advancement in logger
235
+ job_name: Job name to show in the logger
236
+ timeout: Job completion timeout in seconds
237
+ poll_interval: When to check again for job completion in seconds
238
+
239
+ Returns:
240
+ Job status
241
+
242
+ Raises:
243
+ RuntimeError in case of error
244
+ """
245
+ try:
246
+ status_type = ""
247
+ job_identifier = job_status.get("jobID")
248
+ if not job_identifier:
249
+ raise RuntimeError("Job identifier is missing.")
250
+
251
+ while True:
252
+ job_status = self.get_job_info(job_identifier)
253
+ if logger:
254
+ logger.info(f"job_status: {job_status}")
255
+ status_type = job_status.get("status", "")
256
+ if logger:
257
+ logger.info(
258
+ f"----- {job_name} job {job_identifier!r}: {status_type.upper()} \n",
259
+ )
260
+
261
+ # Exit the loop
262
+ if status_type in {"successful", "failed", "dismissed"}:
263
+ break
264
+
265
+ # Or sleep n seconds and try again
266
+ timeout -= poll_interval
267
+ if timeout <= 0:
268
+ raise TimeoutError(f"Timed out while waiting for {job_name} job {job_identifier!r}")
269
+ time.sleep(poll_interval)
270
+
271
+ # Log all exceptions except the timeout
272
+ except TimeoutError:
273
+ raise
274
+ except Exception as e:
275
+ raise RuntimeError(f"Exception while monitoring job {job_name}: {job_status}") from e
276
+
277
+ if status_type == "successful":
278
+ if logger:
279
+ logger.info(f"----- {job_name} job {job_identifier!r}: COMPLETED \n")
280
+ return job_status
281
+
282
+ raise RuntimeError(f"{job_name} job {job_identifier!r}: FAILED")