rs-client-libraries 0.2a15.post17.dev0__tar.gz → 1.0a2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a2}/PKG-INFO +11 -8
- {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a2}/pyproject.toml +28 -13
- {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a2}/rs_client/__init__.py +1 -1
- {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a2}/rs_client/ogcapi/dpr_client.py +56 -14
- {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a2}/rs_client/ogcapi/ogcapi_client.py +33 -14
- {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a2}/rs_client/ogcapi/staging_client.py +5 -4
- {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a2}/rs_client/rs_client.py +11 -0
- {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a2}/rs_client/stac/catalog_client.py +164 -32
- rs_client_libraries-1.0a2/rs_client/stac/prip_client.py +70 -0
- {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a2}/rs_common/prefect_utils.py +14 -26
- {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a2}/rs_common/utils.py +5 -5
- {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a2}/rs_workflows/dpr_flow.py +16 -8
- {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a2}/rs_workflows/flow_utils.py +1 -0
- rs_client_libraries-1.0a2/rs_workflows/init_pi_db_flow.py +140 -0
- {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a2}/rs_workflows/on_demand_processing.py +87 -28
- rs_client_libraries-1.0a2/rs_workflows/pi_db_models.py +156 -0
- rs_client_libraries-1.0a2/rs_workflows/prip_flow.py +68 -0
- rs_client_libraries-1.0a2/rs_workflows/record_performance.py +327 -0
- {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a2}/rs_workflows/staging_flow.py +14 -2
- {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a2}/LICENSE +0 -0
- {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a2}/README.adoc +0 -0
- {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a2}/rs_client/ogcapi/__init__.py +0 -0
- {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a2}/rs_client/stac/__init__.py +0 -0
- {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a2}/rs_client/stac/auxip_client.py +0 -0
- {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a2}/rs_client/stac/cadip_client.py +0 -0
- {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a2}/rs_client/stac/stac_base.py +0 -0
- {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a2}/rs_common/__init__.py +0 -0
- {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a2}/rs_common/config.py +0 -0
- {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a2}/rs_common/init_opentelemetry.py +0 -0
- {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a2}/rs_common/logging.py +0 -0
- {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a2}/rs_workflows/__init__.py +0 -0
- {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a2}/rs_workflows/auxip_flow.py +0 -0
- {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a2}/rs_workflows/cadip_flow.py +0 -0
- {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a2}/rs_workflows/catalog_flow.py +0 -0
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: rs-client-libraries
|
|
3
|
-
Version:
|
|
3
|
+
Version: 1.0a2
|
|
4
4
|
Summary: Copernicus Sentinel rs-client library and processing workflows.
|
|
5
5
|
License: Apache-2.0
|
|
6
|
+
License-File: LICENSE
|
|
6
7
|
Keywords: stac,copernicus,sentinel,processing
|
|
7
8
|
Author: ESA under contract with Airbus and CS
|
|
8
9
|
Author-email: ops_coprs@airbus.com
|
|
@@ -16,23 +17,25 @@ Classifier: Programming Language :: Python :: 3
|
|
|
16
17
|
Classifier: Programming Language :: Python :: 3.11
|
|
17
18
|
Classifier: Programming Language :: Python :: 3.12
|
|
18
19
|
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
19
21
|
Classifier: Topic :: Scientific/Engineering :: Image Processing
|
|
20
22
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
23
|
+
Requires-Dist: SQLAlchemy (>=2.0.37,<3.0.0)
|
|
21
24
|
Requires-Dist: cachetools (>=5.5.2,<6.0.0)
|
|
22
25
|
Requires-Dist: click (==8.1.8)
|
|
23
26
|
Requires-Dist: openapi-core (>=0.19.5,<0.20.0)
|
|
24
|
-
Requires-Dist: opentelemetry-distro (>=0.
|
|
25
|
-
Requires-Dist: opentelemetry-exporter-otlp (>=1.
|
|
26
|
-
Requires-Dist: opentelemetry-instrumentation-aws-lambda (>=0.
|
|
27
|
-
Requires-Dist: opentelemetry-test-utils (>=0.
|
|
27
|
+
Requires-Dist: opentelemetry-distro (>=0.58b0,<0.59)
|
|
28
|
+
Requires-Dist: opentelemetry-exporter-otlp (>=1.37.0,<2.0.0)
|
|
29
|
+
Requires-Dist: opentelemetry-instrumentation-aws-lambda (>=0.58b0,<0.59)
|
|
30
|
+
Requires-Dist: opentelemetry-test-utils (>=0.58b0,<0.59)
|
|
28
31
|
Requires-Dist: prefect[aws] (==3.2.13)
|
|
29
32
|
Requires-Dist: protobuf (>=5.29.5)
|
|
30
33
|
Requires-Dist: pystac (>=1.13.0,<2.0.0)
|
|
31
|
-
Requires-Dist: pystac-client (>=0.
|
|
34
|
+
Requires-Dist: pystac-client (>=0.9.0,<0.10.0)
|
|
32
35
|
Requires-Dist: python-dateutil (>=2.9.0.post0,<3.0.0)
|
|
33
36
|
Requires-Dist: python-logging-loki (>=0.3.1,<0.4.0)
|
|
34
37
|
Requires-Dist: requests (>=2.32.4,<3.0.0)
|
|
35
|
-
Requires-Dist: stac-pydantic (>=3.
|
|
38
|
+
Requires-Dist: stac-pydantic (>=3.4.0,<4.0.0)
|
|
36
39
|
Requires-Dist: urllib3 (>=2.5.0)
|
|
37
40
|
Project-URL: Documentation, https://home.rs-python.eu/rs-documentation/rs-client-libraries/docs/doc/
|
|
38
41
|
Project-URL: Homepage, https://home.rs-python.eu/
|
|
@@ -1,10 +1,24 @@
|
|
|
1
|
+
# Copyright 2025 CS Group
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
1
15
|
[project]
|
|
2
16
|
name = "rs-client-libraries"
|
|
3
17
|
dynamic = ["version"] # see: https://github.com/mtkennerly/poetry-dynamic-versioning
|
|
4
18
|
|
|
5
19
|
[tool.poetry]
|
|
6
20
|
name = "rs-client-libraries"
|
|
7
|
-
version = "
|
|
21
|
+
version = "1.0a2"
|
|
8
22
|
description = "Copernicus Sentinel rs-client library and processing workflows."
|
|
9
23
|
license = "Apache-2.0"
|
|
10
24
|
authors = ["ESA under contract with Airbus and CS <ops_coprs@airbus.com>"]
|
|
@@ -37,45 +51,46 @@ include = [
|
|
|
37
51
|
|
|
38
52
|
[tool.poetry.dependencies]
|
|
39
53
|
python = "^3.11"
|
|
40
|
-
opentelemetry-distro = "^0.
|
|
41
|
-
opentelemetry-instrumentation-aws-lambda = "^0.
|
|
42
|
-
opentelemetry-test-utils = "^0.
|
|
43
|
-
opentelemetry-exporter-otlp = "^1.
|
|
54
|
+
opentelemetry-distro = "^0.58b0" # then run 'poetry run opentelemetry-bootstrap -a install'
|
|
55
|
+
opentelemetry-instrumentation-aws-lambda = "^0.58b0"
|
|
56
|
+
opentelemetry-test-utils = "^0.58b0"
|
|
57
|
+
opentelemetry-exporter-otlp = "^1.37.0"
|
|
44
58
|
protobuf = ">=5.29.5"
|
|
45
59
|
python-logging-loki = "^0.3.1" # to avoid otl error: No module named 'logging_loki'
|
|
46
60
|
cachetools = "^5.5.2"
|
|
47
61
|
pystac = "^1.13.0"
|
|
48
|
-
pystac-client = "^0.
|
|
62
|
+
pystac-client = "^0.9.0"
|
|
49
63
|
python-dateutil = "^2.9.0.post0"
|
|
50
|
-
stac-pydantic = "^3.
|
|
64
|
+
stac-pydantic = "^3.4.0"
|
|
51
65
|
openapi-core = "^0.19.5"
|
|
52
66
|
click = "8.1.8"
|
|
53
67
|
prefect = { version = "3.2.13", extras = ["aws"] }
|
|
54
68
|
requests = "^2.32.4"
|
|
55
69
|
urllib3 = ">=2.5.0"
|
|
70
|
+
SQLAlchemy = "^2.0.37"
|
|
56
71
|
|
|
57
72
|
[tool.poetry.group.dev]
|
|
58
73
|
optional = true
|
|
59
74
|
|
|
60
75
|
[tool.poetry.group.dev.dependencies]
|
|
61
|
-
pytest = "^8.
|
|
62
|
-
pytest-
|
|
63
|
-
pytest-
|
|
76
|
+
pytest = "^8.4.2"
|
|
77
|
+
pytest-asyncio = "^1.2.0"
|
|
78
|
+
pytest-cov = "^7.0.0"
|
|
79
|
+
pytest-docker = "^3.2.3"
|
|
64
80
|
pytest-error-for-skips = "^2.0.2"
|
|
81
|
+
pytest-mock = "^3.15.1"
|
|
65
82
|
black = "==25.1.0"
|
|
66
83
|
mypy = "^1.16.0"
|
|
67
84
|
pre-commit = "^4.1.0"
|
|
68
85
|
bandit = "^1.8.3"
|
|
69
86
|
safety = "^3.5.2"
|
|
70
87
|
pylint = "^3.3.4"
|
|
71
|
-
pytest-docker = "^3.1.2"
|
|
72
88
|
sphinx = "^8.1.3"
|
|
73
89
|
sphinx-rtd-theme = "^3.0.2"
|
|
74
90
|
flake8 = "^7.1.1"
|
|
75
91
|
flake8-pyproject = "^1.2.3"
|
|
76
92
|
responses = "^0.25.7"
|
|
77
|
-
starlette = "^0.
|
|
78
|
-
pytest-asyncio = "^1.0.0"
|
|
93
|
+
starlette = "^0.47.2"
|
|
79
94
|
|
|
80
95
|
[tool.poetry.group.demo.dependencies] # for hybrid mode
|
|
81
96
|
jupyterlab = "^4.2.1"
|
{rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a2}/rs_client/ogcapi/dpr_client.py
RENAMED
|
@@ -17,7 +17,10 @@
|
|
|
17
17
|
import ast
|
|
18
18
|
import os.path as osp
|
|
19
19
|
import tempfile
|
|
20
|
+
from enum import Enum
|
|
21
|
+
from pathlib import Path
|
|
20
22
|
|
|
23
|
+
import anyio
|
|
21
24
|
import yaml
|
|
22
25
|
from openapi_core import OpenAPI # Spec, validate_request, validate_response
|
|
23
26
|
|
|
@@ -36,6 +39,17 @@ PATH_TO_YAML_OPENAPI = osp.realpath(
|
|
|
36
39
|
)
|
|
37
40
|
|
|
38
41
|
|
|
42
|
+
class DprProcess(str, Enum):
|
|
43
|
+
"""DPR processor name"""
|
|
44
|
+
|
|
45
|
+
# String value = resource name in the rs-dpr-service
|
|
46
|
+
CONV_SAFE_ZARR = "conv_safe_zarr"
|
|
47
|
+
MOCKUP = "mockup"
|
|
48
|
+
S1L0 = "s1_l0"
|
|
49
|
+
S3L0 = "s3_l0"
|
|
50
|
+
S1ARD = "s1_ard"
|
|
51
|
+
|
|
52
|
+
|
|
39
53
|
class DprClient(OgcApiClient):
|
|
40
54
|
"""Implement the OGC API client for 'DPR as a service'."""
|
|
41
55
|
|
|
@@ -62,35 +76,37 @@ class DprClient(OgcApiClient):
|
|
|
62
76
|
|
|
63
77
|
def run_process(
|
|
64
78
|
self,
|
|
65
|
-
process:
|
|
79
|
+
process: DprProcess,
|
|
66
80
|
s3_config_dir: str,
|
|
67
81
|
payload_subpath: str,
|
|
68
82
|
s3_report_dir: str | None,
|
|
69
|
-
|
|
83
|
+
extra_data: dict | None = None,
|
|
70
84
|
) -> dict:
|
|
71
85
|
"""Method to start the process from rs-client - Call the endpoint /processes/{process}/execution
|
|
72
86
|
|
|
73
87
|
Args:
|
|
74
|
-
process:
|
|
88
|
+
process: DPR process
|
|
75
89
|
s3_config_dir: S3 bucket folder that contains the payload and configuration files to pass to the processor
|
|
76
90
|
payload_subpath: Payload file path, relative to the config folder
|
|
77
91
|
s3_report_dir: S3 bucket folder were the processor report files will be written (optional). All the eopf
|
|
78
92
|
local files written in the local "./reports" directory will be pushed to this S3 bucket folder.
|
|
79
|
-
|
|
93
|
+
extra_data: Extra data to pass to the processor.
|
|
80
94
|
|
|
81
95
|
Return:
|
|
82
96
|
job_id (int, str): Returns the status code of the request + the identifier
|
|
83
97
|
(or None if endpoint fails) of the running job
|
|
84
98
|
"""
|
|
85
99
|
|
|
100
|
+
use_mockup = process == DprProcess.MOCKUP
|
|
101
|
+
|
|
86
102
|
# Data to pass to the real processor
|
|
87
103
|
data = {}
|
|
88
|
-
if not
|
|
104
|
+
if not use_mockup:
|
|
89
105
|
data = {
|
|
90
106
|
"s3_config_dir": s3_config_dir,
|
|
91
107
|
"payload_subpath": payload_subpath,
|
|
92
108
|
"s3_report_dir": s3_report_dir,
|
|
93
|
-
}
|
|
109
|
+
} | (extra_data or {})
|
|
94
110
|
|
|
95
111
|
# For the mockup processor, pass the payload contents.
|
|
96
112
|
# Download the payload file into a temp file.
|
|
@@ -107,10 +123,24 @@ class DprClient(OgcApiClient):
|
|
|
107
123
|
data = yaml.safe_load(opened)
|
|
108
124
|
|
|
109
125
|
# Add extra info
|
|
110
|
-
data.update({"use_mockup":
|
|
126
|
+
data.update({"use_mockup": use_mockup})
|
|
111
127
|
|
|
112
128
|
# Call the parent method
|
|
113
|
-
return super()._run_process(process, data)
|
|
129
|
+
return super()._run_process(process.value, data)
|
|
130
|
+
|
|
131
|
+
def run_conv_safe_zarr(self, payload: dict):
|
|
132
|
+
"""Method to start the safe to zarr conversion process from rs-client -
|
|
133
|
+
Call the endpoint /processes/conv_safe_zarr/execution
|
|
134
|
+
|
|
135
|
+
Args:
|
|
136
|
+
payload: Dictionary to pass to the processor,
|
|
137
|
+
containing input_safe_path - the s3 path of legacy product and
|
|
138
|
+
output_zarr_dir_path - the s3 path for the new zarr
|
|
139
|
+
Return:
|
|
140
|
+
job_id (int, str): Returns the status code of the request + the identifier
|
|
141
|
+
(or None if endpoint fails) of the running job
|
|
142
|
+
"""
|
|
143
|
+
return super()._run_process("conv_safe_zarr", payload)
|
|
114
144
|
|
|
115
145
|
def wait_for_job(self, *args, **kwargs) -> list[dict]: # type: ignore
|
|
116
146
|
"""
|
|
@@ -128,10 +158,16 @@ class DprClient(OgcApiClient):
|
|
|
128
158
|
######################################################
|
|
129
159
|
|
|
130
160
|
def get_processes(self) -> dict:
|
|
131
|
-
|
|
161
|
+
"""Get all defined processes with logging."""
|
|
162
|
+
processes = super().get_processes()
|
|
163
|
+
self.logger.debug("Fetched %d processes", len(processes))
|
|
164
|
+
return processes
|
|
132
165
|
|
|
133
166
|
def get_jobs(self) -> dict:
|
|
134
|
-
|
|
167
|
+
"""Get all registered jobs with logging."""
|
|
168
|
+
jobs = super().get_jobs()
|
|
169
|
+
self.logger.debug("Fetched %d jobs", len(jobs))
|
|
170
|
+
return jobs
|
|
135
171
|
|
|
136
172
|
def delete_job(self, _: str) -> dict:
|
|
137
173
|
raise NotImplementedError
|
|
@@ -143,7 +179,13 @@ class DprClient(OgcApiClient):
|
|
|
143
179
|
# Specific methods #
|
|
144
180
|
####################
|
|
145
181
|
|
|
146
|
-
async def update_configuration(
|
|
182
|
+
async def update_configuration(
|
|
183
|
+
self,
|
|
184
|
+
local_path: str | Path,
|
|
185
|
+
s3_path: str | Path,
|
|
186
|
+
is_payload: bool = False,
|
|
187
|
+
**kwargs,
|
|
188
|
+
):
|
|
147
189
|
"""
|
|
148
190
|
Update local configuration file depending on the environment, upload it to the s3 bucket,
|
|
149
191
|
and initialize output bucket folders.
|
|
@@ -180,8 +222,8 @@ class DprClient(OgcApiClient):
|
|
|
180
222
|
to_expand.update(kwargs)
|
|
181
223
|
|
|
182
224
|
# Open the input local file
|
|
183
|
-
with
|
|
184
|
-
contents = opened.read()
|
|
225
|
+
async with await anyio.open_file(str(local_path), encoding="utf-8") as opened:
|
|
226
|
+
contents = await opened.read()
|
|
185
227
|
|
|
186
228
|
# Expand the env vars as $key, ${key} or %key%
|
|
187
229
|
for key, value in to_expand.items():
|
|
@@ -214,4 +256,4 @@ class DprClient(OgcApiClient):
|
|
|
214
256
|
tmp.flush()
|
|
215
257
|
|
|
216
258
|
# Upload the temp file to the s3 bucket
|
|
217
|
-
return await prefect_utils.s3_upload_file(tmp.name, s3_path)
|
|
259
|
+
return await prefect_utils.s3_upload_file(tmp.name, str(s3_path))
|
|
@@ -67,23 +67,23 @@ class OgcApiClient(RsClient):
|
|
|
67
67
|
request (Request): endpoint request
|
|
68
68
|
|
|
69
69
|
Returns:
|
|
70
|
-
|
|
70
|
+
RequestUnmarshalResult.data: data validated by the openapi_core
|
|
71
71
|
unmarshal_response method
|
|
72
72
|
"""
|
|
73
73
|
openapi_request = RequestsOpenAPIRequest(request)
|
|
74
74
|
|
|
75
|
-
#
|
|
75
|
+
# validate request
|
|
76
76
|
result = self.get_openapi().unmarshal_request(openapi_request)
|
|
77
77
|
|
|
78
78
|
if result.errors:
|
|
79
79
|
raise OgcValidationException(
|
|
80
|
-
f"Error validating the request of the
|
|
81
|
-
f"{openapi_request.path}: {str(result.errors
|
|
80
|
+
f"Error validating the request of the endpoint "
|
|
81
|
+
f"{openapi_request.path}: {', '.join(str(x) for x in result.errors)}",
|
|
82
82
|
)
|
|
83
83
|
if not result.body:
|
|
84
84
|
raise OgcValidationException(
|
|
85
|
-
f"Error validating the request of the
|
|
86
|
-
f"{openapi_request.path}: 'data' field of
|
|
85
|
+
f"Error validating the request of the endpoint "
|
|
86
|
+
f"{openapi_request.path}: 'data' field of RequestUnmarshalResult"
|
|
87
87
|
f"object is empty",
|
|
88
88
|
)
|
|
89
89
|
return result.body
|
|
@@ -102,18 +102,17 @@ class OgcApiClient(RsClient):
|
|
|
102
102
|
openapi_request = RequestsOpenAPIRequest(response.request)
|
|
103
103
|
openapi_response = RequestsOpenAPIResponse(response)
|
|
104
104
|
|
|
105
|
-
#
|
|
106
|
-
# validate_response(response=response, spec= Spec.from_file_path(PATH_TO_YAML_OPENAPI), request=request)
|
|
105
|
+
# validate response
|
|
107
106
|
result = self.get_openapi().unmarshal_response(openapi_request, openapi_response) # type: ignore
|
|
108
107
|
if result.errors:
|
|
109
108
|
raise OgcValidationException( # type: ignore
|
|
110
|
-
f"Error validating the response of the
|
|
109
|
+
f"Error validating the response of the endpoint {openapi_request.path} - "
|
|
111
110
|
f"Server response content: {response.json()} - "
|
|
112
|
-
f"Validation error of the server response: {str(result.errors
|
|
111
|
+
f"Validation error of the server response: {', '.join(str(x) for x in result.errors)}",
|
|
113
112
|
)
|
|
114
113
|
if not result.data:
|
|
115
114
|
raise OgcValidationException(
|
|
116
|
-
f"Error validating the response of the
|
|
115
|
+
f"Error validating the response of the endpoint "
|
|
117
116
|
f"{openapi_request.path}: 'data' field of ResponseUnmarshalResult"
|
|
118
117
|
f"object is empty",
|
|
119
118
|
)
|
|
@@ -160,7 +159,6 @@ class OgcApiClient(RsClient):
|
|
|
160
159
|
job_id (int, str): Returns the status code of the request + the identifier
|
|
161
160
|
(or None if endpoint fails) of the running job
|
|
162
161
|
"""
|
|
163
|
-
|
|
164
162
|
# Check that the request containing the body is valid
|
|
165
163
|
request = requests.Request(
|
|
166
164
|
method="POST",
|
|
@@ -224,7 +222,7 @@ class OgcApiClient(RsClient):
|
|
|
224
222
|
job_status: dict,
|
|
225
223
|
logger=None,
|
|
226
224
|
job_name: str = "",
|
|
227
|
-
timeout: int | float = math.inf,
|
|
225
|
+
# timeout: int | float = math.inf,
|
|
228
226
|
poll_interval: int = 2,
|
|
229
227
|
) -> dict:
|
|
230
228
|
"""
|
|
@@ -234,7 +232,8 @@ class OgcApiClient(RsClient):
|
|
|
234
232
|
job_status: Returned by `_run_process`
|
|
235
233
|
logger: To show advancement in logger
|
|
236
234
|
job_name: Job name to show in the logger
|
|
237
|
-
timeout: Job completion timeout in seconds
|
|
235
|
+
timeout: Job completion timeout in seconds.
|
|
236
|
+
NOTE: This argument has been disabled, see the comment bellow !
|
|
238
237
|
poll_interval: When to check again for job completion in seconds
|
|
239
238
|
|
|
240
239
|
Returns:
|
|
@@ -243,6 +242,24 @@ class OgcApiClient(RsClient):
|
|
|
243
242
|
Raises:
|
|
244
243
|
RuntimeError in case of error
|
|
245
244
|
"""
|
|
245
|
+
# This parameter has been disabled as input, requiring the client to use the default timeout (INFINITE)
|
|
246
|
+
# for the entire staging process.
|
|
247
|
+
# The reason behind is that the rs-server staging module does not currently provide a 'cancel_job(job_id)'
|
|
248
|
+
# endpoint. The staging process only stops if at least one file fails to stage. In that case, it sends
|
|
249
|
+
# 'dask_client.cancel_all()' to terminate all the running tasks in the dask cluster, waits for them to
|
|
250
|
+
# finish, cleans up (deletes) already staged files, and marks the job status as failed. The rs-client then
|
|
251
|
+
# retrieves this failed status through 'self.get_job_info(job_identifier)' (see below in this function).
|
|
252
|
+
# Allowing 'timeout' as an input argument would be error-prone. If a client sets a timeout shorter than
|
|
253
|
+
# the actual staging duration, the rs-client would raise a timeout error even if the staging
|
|
254
|
+
# could eventually succeed.
|
|
255
|
+
# In such cases, the rs-client should ideally be able to notify the staging process to stop by
|
|
256
|
+
# calling 'cancel_job(job_id)', but this endpoint has not yet been implemented in the rs-server staging module.
|
|
257
|
+
# For now, the only option available to the rs-client is to wait until the staging process finishes, whether
|
|
258
|
+
# it ends in success or failure.
|
|
259
|
+
# If the 'cancel_job(job_id)' endpoint shall be implemented in the staging module, simply uncomment the
|
|
260
|
+
# input parameter timeout and delete / comment the following line. All the other functions that are using
|
|
261
|
+
# this function shall be matter to a change as well (uncomment the timeout parameter)
|
|
262
|
+
timeout: int | float = math.inf
|
|
246
263
|
try:
|
|
247
264
|
status_type = ""
|
|
248
265
|
job_identifier = job_status.get("jobID")
|
|
@@ -265,6 +282,8 @@ class OgcApiClient(RsClient):
|
|
|
265
282
|
|
|
266
283
|
# Or sleep n seconds and try again
|
|
267
284
|
timeout -= poll_interval
|
|
285
|
+
# This will make sense only when the 'timeout' shall be activated as input argument
|
|
286
|
+
# (see the comment) from the beginning of this function
|
|
268
287
|
if timeout <= 0:
|
|
269
288
|
raise TimeoutError(f"Timed out while waiting for {job_name} job {job_identifier!r}")
|
|
270
289
|
time.sleep(poll_interval)
|
|
@@ -15,7 +15,8 @@
|
|
|
15
15
|
"""Launch staging with rs-client-libraries"""
|
|
16
16
|
|
|
17
17
|
import json
|
|
18
|
-
|
|
18
|
+
|
|
19
|
+
# import math
|
|
19
20
|
import os
|
|
20
21
|
import os.path as osp
|
|
21
22
|
from collections import defaultdict
|
|
@@ -90,7 +91,6 @@ class StagingClient(OgcApiClient):
|
|
|
90
91
|
return {parsed.hostname: super()._run_process(RESOURCE, staging_body)}
|
|
91
92
|
|
|
92
93
|
# ----- Case 2: we directly load a STAC ItemCollection in the staging request body
|
|
93
|
-
|
|
94
94
|
# If stac_input is a file, load this file to a dictionary
|
|
95
95
|
if isinstance(stac_input, str):
|
|
96
96
|
# If the input is a valid path to a json_file, load this file
|
|
@@ -153,7 +153,7 @@ class StagingClient(OgcApiClient):
|
|
|
153
153
|
self,
|
|
154
154
|
all_job_status: dict[str, dict],
|
|
155
155
|
logger=None,
|
|
156
|
-
timeout: int | float = math.inf,
|
|
156
|
+
# timeout: int | float = math.inf,
|
|
157
157
|
poll_interval: int = 2,
|
|
158
158
|
):
|
|
159
159
|
"""
|
|
@@ -163,6 +163,7 @@ class StagingClient(OgcApiClient):
|
|
|
163
163
|
job_status: Returned by `run_staging`
|
|
164
164
|
logger: To show advancement in logger
|
|
165
165
|
timeout: Job completion timeout in seconds
|
|
166
|
+
NOTE: This argument has been disabled, see the comment in super().wait_for_job() function
|
|
166
167
|
poll_interval: When to check again for job completion in seconds
|
|
167
168
|
|
|
168
169
|
Raises:
|
|
@@ -174,6 +175,6 @@ class StagingClient(OgcApiClient):
|
|
|
174
175
|
job_status,
|
|
175
176
|
logger,
|
|
176
177
|
f"Staging from {hostname!r}",
|
|
177
|
-
timeout,
|
|
178
|
+
# timeout,
|
|
178
179
|
poll_interval,
|
|
179
180
|
)
|
|
@@ -41,6 +41,7 @@ class RsClient: # pylint: disable=too-many-instance-attributes
|
|
|
41
41
|
- rs-server-staging
|
|
42
42
|
- rs-server-cadip
|
|
43
43
|
- rs-server-auxip
|
|
44
|
+
- rs-server-prip
|
|
44
45
|
- rs-server-catalog
|
|
45
46
|
|
|
46
47
|
This class provides methods to authenticate and interact with RS-Server,
|
|
@@ -230,6 +231,16 @@ class RsClient: # pylint: disable=too-many-instance-attributes
|
|
|
230
231
|
|
|
231
232
|
return AuxipClient(self.rs_server_href, self.rs_server_api_key, self.logger, **kwargs)
|
|
232
233
|
|
|
234
|
+
def get_prip_client(self, **kwargs) -> "PripClient": # type: ignore # noqa: F821
|
|
235
|
+
"""
|
|
236
|
+
Return an instance of the child class PripClient, with the same attributes as this "self" instance.
|
|
237
|
+
"""
|
|
238
|
+
from rs_client.stac.prip_client import ( # pylint: disable=import-outside-toplevel,cyclic-import
|
|
239
|
+
PripClient,
|
|
240
|
+
)
|
|
241
|
+
|
|
242
|
+
return PripClient(self.rs_server_href, self.rs_server_api_key, self.logger, **kwargs)
|
|
243
|
+
|
|
233
244
|
def get_cadip_client(self, **kwargs) -> "CadipClient": # type: ignore # noqa: F821
|
|
234
245
|
"""
|
|
235
246
|
Return an instance of the child class CadipClient, with the same attributes as this "self" instance.
|