rs-client-libraries 0.2a14__tar.gz → 0.2a15.post17.dev0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15.post17.dev0}/PKG-INFO +8 -5
- {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15.post17.dev0}/pyproject.toml +15 -10
- {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15.post17.dev0}/rs_client/__init__.py +1 -1
- rs_client_libraries-0.2a15.post17.dev0/rs_client/ogcapi/dpr_client.py +217 -0
- {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15.post17.dev0}/rs_client/ogcapi/ogcapi_client.py +5 -13
- rs_client_libraries-0.2a15.post17.dev0/rs_client/ogcapi/staging_client.py +179 -0
- {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15.post17.dev0}/rs_client/stac/catalog_client.py +1 -1
- {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15.post17.dev0}/rs_client/stac/stac_base.py +1 -1
- {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15.post17.dev0}/rs_common/prefect_utils.py +52 -25
- {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15.post17.dev0}/rs_common/utils.py +2 -2
- {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15.post17.dev0}/rs_workflows/auxip_flow.py +5 -3
- {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15.post17.dev0}/rs_workflows/cadip_flow.py +4 -2
- {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15.post17.dev0}/rs_workflows/catalog_flow.py +2 -1
- {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15.post17.dev0}/rs_workflows/dpr_flow.py +22 -20
- {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15.post17.dev0}/rs_workflows/flow_utils.py +2 -2
- rs_client_libraries-0.2a15.post17.dev0/rs_workflows/on_demand_processing.py +257 -0
- {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15.post17.dev0}/rs_workflows/staging_flow.py +17 -31
- rs_client_libraries-0.2a14/rs_client/ogcapi/dpr_client.py +0 -80
- rs_client_libraries-0.2a14/rs_client/ogcapi/staging_client.py +0 -142
- rs_client_libraries-0.2a14/rs_workflows/on_demand_processing.py +0 -130
- {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15.post17.dev0}/LICENSE +0 -0
- {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15.post17.dev0}/README.adoc +0 -0
- {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15.post17.dev0}/rs_client/ogcapi/__init__.py +0 -0
- {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15.post17.dev0}/rs_client/rs_client.py +0 -0
- {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15.post17.dev0}/rs_client/stac/__init__.py +0 -0
- {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15.post17.dev0}/rs_client/stac/auxip_client.py +0 -0
- {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15.post17.dev0}/rs_client/stac/cadip_client.py +0 -0
- {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15.post17.dev0}/rs_common/__init__.py +0 -0
- {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15.post17.dev0}/rs_common/config.py +0 -0
- {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15.post17.dev0}/rs_common/init_opentelemetry.py +0 -0
- {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15.post17.dev0}/rs_common/logging.py +0 -0
- {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15.post17.dev0}/rs_workflows/__init__.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: rs-client-libraries
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2a15.post17.dev0
|
|
4
4
|
Summary: Copernicus Sentinel rs-client library and processing workflows.
|
|
5
5
|
License: Apache-2.0
|
|
6
6
|
Keywords: stac,copernicus,sentinel,processing
|
|
@@ -21,16 +21,19 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
|
21
21
|
Requires-Dist: cachetools (>=5.5.2,<6.0.0)
|
|
22
22
|
Requires-Dist: click (==8.1.8)
|
|
23
23
|
Requires-Dist: openapi-core (>=0.19.5,<0.20.0)
|
|
24
|
-
Requires-Dist: opentelemetry-distro (>=0.
|
|
25
|
-
Requires-Dist: opentelemetry-exporter-otlp (>=1.34.
|
|
26
|
-
Requires-Dist: opentelemetry-instrumentation-aws-lambda (>=0.
|
|
27
|
-
Requires-Dist: opentelemetry-test-utils (>=0.
|
|
24
|
+
Requires-Dist: opentelemetry-distro (>=0.55b1,<0.56)
|
|
25
|
+
Requires-Dist: opentelemetry-exporter-otlp (>=1.34.1,<2.0.0)
|
|
26
|
+
Requires-Dist: opentelemetry-instrumentation-aws-lambda (>=0.55b1,<0.56)
|
|
27
|
+
Requires-Dist: opentelemetry-test-utils (>=0.55b1,<0.56)
|
|
28
28
|
Requires-Dist: prefect[aws] (==3.2.13)
|
|
29
|
+
Requires-Dist: protobuf (>=5.29.5)
|
|
29
30
|
Requires-Dist: pystac (>=1.13.0,<2.0.0)
|
|
30
31
|
Requires-Dist: pystac-client (>=0.8.6,<0.9.0)
|
|
31
32
|
Requires-Dist: python-dateutil (>=2.9.0.post0,<3.0.0)
|
|
32
33
|
Requires-Dist: python-logging-loki (>=0.3.1,<0.4.0)
|
|
34
|
+
Requires-Dist: requests (>=2.32.4,<3.0.0)
|
|
33
35
|
Requires-Dist: stac-pydantic (>=3.1.5,<4.0.0)
|
|
36
|
+
Requires-Dist: urllib3 (>=2.5.0)
|
|
34
37
|
Project-URL: Documentation, https://home.rs-python.eu/rs-documentation/rs-client-libraries/docs/doc/
|
|
35
38
|
Project-URL: Homepage, https://home.rs-python.eu/
|
|
36
39
|
Project-URL: Repository, https://github.com/RS-PYTHON/rs-client-libraries
|
|
@@ -4,7 +4,7 @@ dynamic = ["version"] # see: https://github.com/mtkennerly/poetry-dynamic-versio
|
|
|
4
4
|
|
|
5
5
|
[tool.poetry]
|
|
6
6
|
name = "rs-client-libraries"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.2a15.post17.dev0"
|
|
8
8
|
description = "Copernicus Sentinel rs-client library and processing workflows."
|
|
9
9
|
license = "Apache-2.0"
|
|
10
10
|
authors = ["ESA under contract with Airbus and CS <ops_coprs@airbus.com>"]
|
|
@@ -37,10 +37,11 @@ include = [
|
|
|
37
37
|
|
|
38
38
|
[tool.poetry.dependencies]
|
|
39
39
|
python = "^3.11"
|
|
40
|
-
opentelemetry-distro = "^0.
|
|
41
|
-
opentelemetry-instrumentation-aws-lambda = "^0.
|
|
42
|
-
opentelemetry-test-utils = "^0.
|
|
43
|
-
opentelemetry-exporter-otlp = "^1.34.
|
|
40
|
+
opentelemetry-distro = "^0.55b1" # then run 'poetry run opentelemetry-bootstrap -a install'
|
|
41
|
+
opentelemetry-instrumentation-aws-lambda = "^0.55b1"
|
|
42
|
+
opentelemetry-test-utils = "^0.55b1"
|
|
43
|
+
opentelemetry-exporter-otlp = "^1.34.1"
|
|
44
|
+
protobuf = ">=5.29.5"
|
|
44
45
|
python-logging-loki = "^0.3.1" # to avoid otl error: No module named 'logging_loki'
|
|
45
46
|
cachetools = "^5.5.2"
|
|
46
47
|
pystac = "^1.13.0"
|
|
@@ -50,20 +51,22 @@ stac-pydantic = "^3.1.5"
|
|
|
50
51
|
openapi-core = "^0.19.5"
|
|
51
52
|
click = "8.1.8"
|
|
52
53
|
prefect = { version = "3.2.13", extras = ["aws"] }
|
|
54
|
+
requests = "^2.32.4"
|
|
55
|
+
urllib3 = ">=2.5.0"
|
|
53
56
|
|
|
54
57
|
[tool.poetry.group.dev]
|
|
55
58
|
optional = true
|
|
56
59
|
|
|
57
60
|
[tool.poetry.group.dev.dependencies]
|
|
58
61
|
pytest = "^8.3.5"
|
|
59
|
-
pytest-cov = "^6.
|
|
62
|
+
pytest-cov = "^6.2.1"
|
|
60
63
|
pytest-mock = "^3.14.0"
|
|
61
64
|
pytest-error-for-skips = "^2.0.2"
|
|
62
65
|
black = "==25.1.0"
|
|
63
|
-
mypy = "^1.
|
|
66
|
+
mypy = "^1.16.0"
|
|
64
67
|
pre-commit = "^4.1.0"
|
|
65
|
-
bandit = "^1.8.
|
|
66
|
-
safety = "^3.2
|
|
68
|
+
bandit = "^1.8.3"
|
|
69
|
+
safety = "^3.5.2"
|
|
67
70
|
pylint = "^3.3.4"
|
|
68
71
|
pytest-docker = "^3.1.2"
|
|
69
72
|
sphinx = "^8.1.3"
|
|
@@ -72,10 +75,11 @@ flake8 = "^7.1.1"
|
|
|
72
75
|
flake8-pyproject = "^1.2.3"
|
|
73
76
|
responses = "^0.25.7"
|
|
74
77
|
starlette = "^0.45.3"
|
|
78
|
+
pytest-asyncio = "^1.0.0"
|
|
75
79
|
|
|
76
80
|
[tool.poetry.group.demo.dependencies] # for hybrid mode
|
|
77
81
|
jupyterlab = "^4.2.1"
|
|
78
|
-
boto3 = "^1.
|
|
82
|
+
boto3 = "^1.38.34"
|
|
79
83
|
papermill = "^2.6.0"
|
|
80
84
|
tornado = "^6.5" # fix trivy vulnerability
|
|
81
85
|
|
|
@@ -98,6 +102,7 @@ markers = [
|
|
|
98
102
|
"unit", # mark a test as unittest
|
|
99
103
|
"integration", # mark a test as integration test
|
|
100
104
|
]
|
|
105
|
+
asyncio_default_fixture_loop_scope = "function"
|
|
101
106
|
asyncio_mode = "auto"
|
|
102
107
|
|
|
103
108
|
[tool.mypy]
|
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
# Copyright 2024 CS Group
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""Use "DPR as a service" implemented by rs-dpr-service"""
|
|
16
|
+
|
|
17
|
+
import ast
|
|
18
|
+
import os.path as osp
|
|
19
|
+
import tempfile
|
|
20
|
+
|
|
21
|
+
import yaml
|
|
22
|
+
from openapi_core import OpenAPI # Spec, validate_request, validate_response
|
|
23
|
+
|
|
24
|
+
from rs_client.ogcapi.ogcapi_client import OgcApiClient
|
|
25
|
+
from rs_common import prefect_utils
|
|
26
|
+
from rs_common.utils import get_href_service
|
|
27
|
+
|
|
28
|
+
PATH_TO_YAML_OPENAPI = osp.realpath(
|
|
29
|
+
osp.join(
|
|
30
|
+
osp.dirname(__file__),
|
|
31
|
+
"../../config",
|
|
32
|
+
"staging_templates",
|
|
33
|
+
"yaml",
|
|
34
|
+
"dpr_openapi_schema.yaml",
|
|
35
|
+
),
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class DprClient(OgcApiClient):
|
|
40
|
+
"""Implement the OGC API client for 'DPR as a service'."""
|
|
41
|
+
|
|
42
|
+
##########################################
|
|
43
|
+
# Override parent methods and attributes #
|
|
44
|
+
##########################################
|
|
45
|
+
|
|
46
|
+
# Init the OpenAPI instance from config file
|
|
47
|
+
openapi = OpenAPI.from_file_path(PATH_TO_YAML_OPENAPI)
|
|
48
|
+
|
|
49
|
+
@property
|
|
50
|
+
def endpoint_prefix(self) -> str:
|
|
51
|
+
"""Return the endpoints prefix, if any."""
|
|
52
|
+
return "dpr/"
|
|
53
|
+
|
|
54
|
+
@property
|
|
55
|
+
def href_service(self) -> str:
|
|
56
|
+
"""
|
|
57
|
+
Return the rs-dpr-service URL hostname.
|
|
58
|
+
This URL can be overwritten using the RSPY_HOST_DPR_SERVICE env variable (used e.g. for local mode).
|
|
59
|
+
Otherwise it should just be the RS-Server URL.
|
|
60
|
+
"""
|
|
61
|
+
return get_href_service(self.rs_server_href, "RSPY_HOST_DPR_SERVICE")
|
|
62
|
+
|
|
63
|
+
def run_process(
|
|
64
|
+
self,
|
|
65
|
+
process: str,
|
|
66
|
+
s3_config_dir: str,
|
|
67
|
+
payload_subpath: str,
|
|
68
|
+
s3_report_dir: str | None,
|
|
69
|
+
use_dpr_mockup: bool = False,
|
|
70
|
+
) -> dict:
|
|
71
|
+
"""Method to start the process from rs-client - Call the endpoint /processes/{process}/execution
|
|
72
|
+
|
|
73
|
+
Args:
|
|
74
|
+
process: Process name
|
|
75
|
+
s3_config_dir: S3 bucket folder that contains the payload and configuration files to pass to the processor
|
|
76
|
+
payload_subpath: Payload file path, relative to the config folder
|
|
77
|
+
s3_report_dir: S3 bucket folder were the processor report files will be written (optional). All the eopf
|
|
78
|
+
local files written in the local "./reports" directory will be pushed to this S3 bucket folder.
|
|
79
|
+
use_dpr_mockup: Use the real or the mockup DPR processor ?
|
|
80
|
+
|
|
81
|
+
Return:
|
|
82
|
+
job_id (int, str): Returns the status code of the request + the identifier
|
|
83
|
+
(or None if endpoint fails) of the running job
|
|
84
|
+
"""
|
|
85
|
+
|
|
86
|
+
# Data to pass to the real processor
|
|
87
|
+
data = {}
|
|
88
|
+
if not use_dpr_mockup:
|
|
89
|
+
data = {
|
|
90
|
+
"s3_config_dir": s3_config_dir,
|
|
91
|
+
"payload_subpath": payload_subpath,
|
|
92
|
+
"s3_report_dir": s3_report_dir,
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
# For the mockup processor, pass the payload contents.
|
|
96
|
+
# Download the payload file into a temp file.
|
|
97
|
+
else:
|
|
98
|
+
with tempfile.NamedTemporaryFile() as temp:
|
|
99
|
+
prefect_utils.s3_download_file( # type: ignore
|
|
100
|
+
osp.join(s3_config_dir, payload_subpath),
|
|
101
|
+
temp.name,
|
|
102
|
+
_sync=True, # type: ignore
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
# Read it as a yaml file
|
|
106
|
+
with open(temp.name, encoding="utf-8") as opened:
|
|
107
|
+
data = yaml.safe_load(opened)
|
|
108
|
+
|
|
109
|
+
# Add extra info
|
|
110
|
+
data.update({"use_mockup": use_dpr_mockup})
|
|
111
|
+
|
|
112
|
+
# Call the parent method
|
|
113
|
+
return super()._run_process(process, data)
|
|
114
|
+
|
|
115
|
+
def wait_for_job(self, *args, **kwargs) -> list[dict]: # type: ignore
|
|
116
|
+
"""
|
|
117
|
+
Wait for job to finish.
|
|
118
|
+
|
|
119
|
+
Returns:
|
|
120
|
+
EOPF results
|
|
121
|
+
"""
|
|
122
|
+
# Call parent method and parse results
|
|
123
|
+
job_status = super().wait_for_job(*args, **kwargs)
|
|
124
|
+
return ast.literal_eval(job_status["message"])
|
|
125
|
+
|
|
126
|
+
######################################################
|
|
127
|
+
# These endpoints are not implemented by the service #
|
|
128
|
+
######################################################
|
|
129
|
+
|
|
130
|
+
def get_processes(self) -> dict:
|
|
131
|
+
raise NotImplementedError
|
|
132
|
+
|
|
133
|
+
def get_jobs(self) -> dict:
|
|
134
|
+
raise NotImplementedError
|
|
135
|
+
|
|
136
|
+
def delete_job(self, _: str) -> dict:
|
|
137
|
+
raise NotImplementedError
|
|
138
|
+
|
|
139
|
+
def get_job_results(self, _: str) -> dict:
|
|
140
|
+
raise NotImplementedError
|
|
141
|
+
|
|
142
|
+
####################
|
|
143
|
+
# Specific methods #
|
|
144
|
+
####################
|
|
145
|
+
|
|
146
|
+
async def update_configuration(self, local_path: str, s3_path: str, is_payload: bool = False, **kwargs):
|
|
147
|
+
"""
|
|
148
|
+
Update local configuration file depending on the environment, upload it to the s3 bucket,
|
|
149
|
+
and initialize output bucket folders.
|
|
150
|
+
|
|
151
|
+
Args:
|
|
152
|
+
local_path: Local configuration file path
|
|
153
|
+
s3_path: S3 bucket path where to upload to modified updated configuration file
|
|
154
|
+
is_payload: Specific behavior for the processor payload files
|
|
155
|
+
kwargs: Specific environment variables to expand in the configuration file
|
|
156
|
+
"""
|
|
157
|
+
|
|
158
|
+
to_expand = {}
|
|
159
|
+
|
|
160
|
+
if self.local_mode:
|
|
161
|
+
# In local mode, replace the S3_xxx_CLUSTER env vars, by the env vars from the ~/.s3cfg
|
|
162
|
+
# config file, that contains access to the "real" s3 bucket
|
|
163
|
+
to_expand = {
|
|
164
|
+
"S3_ACCESSKEY_CLUSTER": "${access_key}",
|
|
165
|
+
"S3_SECRETKEY_CLUSTER": "${secret_key}",
|
|
166
|
+
"S3_ENDPOINT_CLUSTER": "${host_bucket}",
|
|
167
|
+
"S3_REGION_CLUSTER": "${bucket_location}",
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
else:
|
|
171
|
+
# In cluster mode, just use the "real" s3 bucket
|
|
172
|
+
to_expand = {
|
|
173
|
+
"S3_ACCESSKEY_CLUSTER": "${S3_ACCESSKEY}",
|
|
174
|
+
"S3_SECRETKEY_CLUSTER": "${S3_SECRETKEY}",
|
|
175
|
+
"S3_ENDPOINT_CLUSTER": "${S3_ENDPOINT}",
|
|
176
|
+
"S3_REGION_CLUSTER": "${S3_REGION}",
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
# Also expand the user-given parameters
|
|
180
|
+
to_expand.update(kwargs)
|
|
181
|
+
|
|
182
|
+
# Open the input local file
|
|
183
|
+
with open(local_path, encoding="utf-8") as opened:
|
|
184
|
+
contents = opened.read()
|
|
185
|
+
|
|
186
|
+
# Expand the env vars as $key, ${key} or %key%
|
|
187
|
+
for key, value in to_expand.items():
|
|
188
|
+
for key2 in f"${key}", f"${{{key}}}", f"%{key}%":
|
|
189
|
+
contents = contents.replace(key2, str(value))
|
|
190
|
+
|
|
191
|
+
# Read the payload contents
|
|
192
|
+
if is_payload:
|
|
193
|
+
payload = yaml.safe_load(contents)
|
|
194
|
+
|
|
195
|
+
# We need to create the output S3 folder with a dummy file before running DPR
|
|
196
|
+
for output_product in payload["I/O"]["output_products"]:
|
|
197
|
+
s3_output_dir = output_product["path"]
|
|
198
|
+
s3_empty_file = f"{s3_output_dir}/.empty"
|
|
199
|
+
self.logger.info(f"Write empty file: {self.logger.level} {s3_empty_file!r}")
|
|
200
|
+
await prefect_utils.s3_upload_empty_file(s3_empty_file)
|
|
201
|
+
|
|
202
|
+
# Change the dask authentication for local mode
|
|
203
|
+
cluster_config = payload["dask_context"]["cluster_config"]
|
|
204
|
+
if self.local_mode:
|
|
205
|
+
cluster_config["auth"] = cluster_config["auth_local_mode"]
|
|
206
|
+
del cluster_config["auth_local_mode"]
|
|
207
|
+
|
|
208
|
+
# yaml to str conversion
|
|
209
|
+
contents = yaml.dump(payload, default_flow_style=False, sort_keys=False)
|
|
210
|
+
|
|
211
|
+
# Write the modified contents to a temp file
|
|
212
|
+
with tempfile.NamedTemporaryFile() as tmp:
|
|
213
|
+
tmp.write(contents.encode("utf-8"))
|
|
214
|
+
tmp.flush()
|
|
215
|
+
|
|
216
|
+
# Upload the temp file to the s3 bucket
|
|
217
|
+
return await prefect_utils.s3_upload_file(tmp.name, s3_path)
|
|
@@ -18,7 +18,6 @@ import json
|
|
|
18
18
|
import math
|
|
19
19
|
import time
|
|
20
20
|
from typing import Any
|
|
21
|
-
from urllib.parse import urlparse
|
|
22
21
|
|
|
23
22
|
# openapi_core libraries used for endpoints validation
|
|
24
23
|
import requests
|
|
@@ -51,14 +50,6 @@ class OgcApiClient(RsClient):
|
|
|
51
50
|
is passed as an extra argument).
|
|
52
51
|
"""
|
|
53
52
|
|
|
54
|
-
@staticmethod
|
|
55
|
-
def is_url(path):
|
|
56
|
-
"""
|
|
57
|
-
Function to check if a string is an url
|
|
58
|
-
"""
|
|
59
|
-
parsed = urlparse(path)
|
|
60
|
-
return bool(parsed.scheme and parsed.netloc)
|
|
61
|
-
|
|
62
53
|
@property
|
|
63
54
|
def endpoint_prefix(self) -> str:
|
|
64
55
|
"""Return the endpoints prefix, if any."""
|
|
@@ -158,7 +149,7 @@ class OgcApiClient(RsClient):
|
|
|
158
149
|
)
|
|
159
150
|
return self.validate_and_unmarshal_response(response)
|
|
160
151
|
|
|
161
|
-
def
|
|
152
|
+
def _run_process(self, process: str, body: dict) -> dict:
|
|
162
153
|
"""Method to start the process from rs-client - Call the endpoint /processes/{process}/execution
|
|
163
154
|
|
|
164
155
|
Args:
|
|
@@ -240,7 +231,7 @@ class OgcApiClient(RsClient):
|
|
|
240
231
|
Wait for job to finish.
|
|
241
232
|
|
|
242
233
|
Args:
|
|
243
|
-
job_status: Returned by `
|
|
234
|
+
job_status: Returned by `_run_process`
|
|
244
235
|
logger: To show advancement in logger
|
|
245
236
|
job_name: Job name to show in the logger
|
|
246
237
|
timeout: Job completion timeout in seconds
|
|
@@ -253,7 +244,8 @@ class OgcApiClient(RsClient):
|
|
|
253
244
|
RuntimeError in case of error
|
|
254
245
|
"""
|
|
255
246
|
try:
|
|
256
|
-
status_type
|
|
247
|
+
status_type = ""
|
|
248
|
+
job_identifier = job_status.get("jobID")
|
|
257
249
|
if not job_identifier:
|
|
258
250
|
raise RuntimeError("Job identifier is missing.")
|
|
259
251
|
|
|
@@ -261,7 +253,7 @@ class OgcApiClient(RsClient):
|
|
|
261
253
|
job_status = self.get_job_info(job_identifier)
|
|
262
254
|
if logger:
|
|
263
255
|
logger.info(f"job_status: {job_status}")
|
|
264
|
-
status_type = job_status.get("status")
|
|
256
|
+
status_type = job_status.get("status", "")
|
|
265
257
|
if logger:
|
|
266
258
|
logger.info(
|
|
267
259
|
f"----- {job_name} job {job_identifier!r}: {status_type.upper()} \n",
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
# Copyright 2024 CS Group
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""Launch staging with rs-client-libraries"""
|
|
16
|
+
|
|
17
|
+
import json
|
|
18
|
+
import math
|
|
19
|
+
import os
|
|
20
|
+
import os.path as osp
|
|
21
|
+
from collections import defaultdict
|
|
22
|
+
from collections.abc import Sequence
|
|
23
|
+
from typing import Any
|
|
24
|
+
from urllib.parse import urlparse
|
|
25
|
+
|
|
26
|
+
# openapi_core libraries used for endpoints validation
|
|
27
|
+
from openapi_core import OpenAPI # Spec, validate_request, validate_response
|
|
28
|
+
from stac_pydantic.api import Item, ItemCollection
|
|
29
|
+
|
|
30
|
+
from rs_client.ogcapi.ogcapi_client import OgcApiClient, OgcValidationException
|
|
31
|
+
from rs_common.utils import get_href_service
|
|
32
|
+
|
|
33
|
+
PATH_TO_YAML_OPENAPI = osp.realpath(
|
|
34
|
+
osp.join(
|
|
35
|
+
osp.dirname(__file__),
|
|
36
|
+
"../../config",
|
|
37
|
+
"staging_templates",
|
|
38
|
+
"yaml",
|
|
39
|
+
"staging_openapi_schema.yaml",
|
|
40
|
+
),
|
|
41
|
+
)
|
|
42
|
+
RESOURCE = "staging"
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class StagingClient(OgcApiClient):
|
|
46
|
+
"""Implement the OGC API client for the staging."""
|
|
47
|
+
|
|
48
|
+
# Init the OpenAPI instance from config file
|
|
49
|
+
openapi = OpenAPI.from_file_path(PATH_TO_YAML_OPENAPI)
|
|
50
|
+
|
|
51
|
+
@property
|
|
52
|
+
def href_service(self) -> str:
|
|
53
|
+
"""
|
|
54
|
+
Return the RS-Server staging URL hostname.
|
|
55
|
+
This URL can be overwritten using the RSPY_HOST_STAGING env variable (used e.g. for local mode).
|
|
56
|
+
Otherwise it should just be the RS-Server URL.
|
|
57
|
+
"""
|
|
58
|
+
return get_href_service(self.rs_server_href, "RSPY_HOST_STAGING")
|
|
59
|
+
|
|
60
|
+
def run_staging( # pylint: disable=too-many-locals
|
|
61
|
+
self,
|
|
62
|
+
stac_input: dict[Any, Any] | str,
|
|
63
|
+
out_coll_name: str,
|
|
64
|
+
) -> dict[str, dict]:
|
|
65
|
+
"""Method to start the staging process from rs-client - Call the endpoint /processes/staging/execution
|
|
66
|
+
|
|
67
|
+
Args:
|
|
68
|
+
stac_input (dict | str): it can be:
|
|
69
|
+
- A Python dictionary corresponding to a Feature or a FeatureCollection (that can be for example
|
|
70
|
+
the output of a search for Cadip or Auxip sessions)
|
|
71
|
+
- A json string corresponding to a Feature or a FeatureCollection
|
|
72
|
+
- A string corresponding to a path to a json file containing a Feature or a FeatureCollection
|
|
73
|
+
- A single link that returns a STAC itemCollection: this link should be an url to search a
|
|
74
|
+
itemCollection, for example:
|
|
75
|
+
http://localhost:8002/cadip/search?ids=S1A_20231120061537234567&collections=cadip_sentinel1
|
|
76
|
+
out_coll_name (str): name of the output collection
|
|
77
|
+
Return:
|
|
78
|
+
dict(hostname, job_id (int, str)): Returns the status codes of the staging requests (one per URL hostname) +
|
|
79
|
+
the identifiers
|
|
80
|
+
"""
|
|
81
|
+
stac_input_dict = {}
|
|
82
|
+
|
|
83
|
+
# ----- Case 1: we only load a link (that refers to a STAC itemCollection) in the staging request body
|
|
84
|
+
if isinstance(stac_input, str):
|
|
85
|
+
|
|
86
|
+
# Check that it's an url
|
|
87
|
+
parsed = urlparse(stac_input)
|
|
88
|
+
if parsed.scheme and parsed.netloc and parsed.hostname:
|
|
89
|
+
staging_body = {"inputs": {"collection": out_coll_name, "items": {"href": stac_input}}}
|
|
90
|
+
return {parsed.hostname: super()._run_process(RESOURCE, staging_body)}
|
|
91
|
+
|
|
92
|
+
# ----- Case 2: we directly load a STAC ItemCollection in the staging request body
|
|
93
|
+
|
|
94
|
+
# If stac_input is a file, load this file to a dictionary
|
|
95
|
+
if isinstance(stac_input, str):
|
|
96
|
+
# If the input is a valid path to a json_file, load this file
|
|
97
|
+
if os.path.exists(os.path.dirname(stac_input)) and stac_input.endswith(".json"):
|
|
98
|
+
# Read the yaml or json file
|
|
99
|
+
with open(stac_input, encoding="utf-8") as opened:
|
|
100
|
+
stac_file_to_dict = json.loads(opened.read())
|
|
101
|
+
stac_input_dict = stac_file_to_dict
|
|
102
|
+
# If the input string is not a path, try to convert the content of the string to a json dictionary
|
|
103
|
+
else:
|
|
104
|
+
try:
|
|
105
|
+
stac_input_dict = json.loads(stac_input)
|
|
106
|
+
except json.JSONDecodeError as e:
|
|
107
|
+
raise OgcValidationException(
|
|
108
|
+
f"""Invalid input format: {stac_input} - Input data must be either:
|
|
109
|
+
- A Python dictionary corresponding to a Feature or a FeatureCollection
|
|
110
|
+
(that can be for example the output of a search for Cadip or Auxip sessions)
|
|
111
|
+
- A json string corresponding to a Feature or a FeatureCollection
|
|
112
|
+
- A string corresponding to a path to a json file containing a Feature or a
|
|
113
|
+
FeatureCollection
|
|
114
|
+
- A single link that returns a STAC ItemCollection: this link should be an
|
|
115
|
+
url to search an ItemCollection
|
|
116
|
+
""".strip(),
|
|
117
|
+
) from e
|
|
118
|
+
else:
|
|
119
|
+
stac_input_dict = stac_input
|
|
120
|
+
|
|
121
|
+
if "type" not in stac_input_dict:
|
|
122
|
+
raise KeyError("Key 'type' is missing from the staging input data")
|
|
123
|
+
|
|
124
|
+
# Validate input data using Pydantic
|
|
125
|
+
if stac_input_dict["type"] == "Feature":
|
|
126
|
+
stac_items: Sequence[Item] = [Item(**stac_input_dict)]
|
|
127
|
+
else:
|
|
128
|
+
stac_items = ItemCollection(**stac_input_dict).features
|
|
129
|
+
|
|
130
|
+
# Order the items by station url hostname
|
|
131
|
+
hostname_items: dict[str, list[Item]] = defaultdict(list)
|
|
132
|
+
for item in stac_items:
|
|
133
|
+
assets = list(item.assets.values())
|
|
134
|
+
if assets:
|
|
135
|
+
hostname = urlparse(assets[0].href).hostname or ""
|
|
136
|
+
hostname_items[hostname].append(item)
|
|
137
|
+
|
|
138
|
+
# Trigger staging for each domain items
|
|
139
|
+
hostname_triggers: dict[str, dict] = {}
|
|
140
|
+
for hostname, items in hostname_items.items():
|
|
141
|
+
stac_item_collection = ItemCollection(type="FeatureCollection", features=items)
|
|
142
|
+
staging_body = {
|
|
143
|
+
"inputs": {
|
|
144
|
+
"collection": out_coll_name,
|
|
145
|
+
"items": {"value": stac_item_collection.model_dump(mode="json")},
|
|
146
|
+
},
|
|
147
|
+
}
|
|
148
|
+
hostname_triggers[hostname] = super()._run_process(RESOURCE, staging_body)
|
|
149
|
+
|
|
150
|
+
return hostname_triggers
|
|
151
|
+
|
|
152
|
+
def wait_for_jobs(
|
|
153
|
+
self,
|
|
154
|
+
all_job_status: dict[str, dict],
|
|
155
|
+
logger=None,
|
|
156
|
+
timeout: int | float = math.inf,
|
|
157
|
+
poll_interval: int = 2,
|
|
158
|
+
):
|
|
159
|
+
"""
|
|
160
|
+
Wait for job to finish.
|
|
161
|
+
|
|
162
|
+
Args:
|
|
163
|
+
job_status: Returned by `run_staging`
|
|
164
|
+
logger: To show advancement in logger
|
|
165
|
+
timeout: Job completion timeout in seconds
|
|
166
|
+
poll_interval: When to check again for job completion in seconds
|
|
167
|
+
|
|
168
|
+
Raises:
|
|
169
|
+
RuntimeError in case of error
|
|
170
|
+
"""
|
|
171
|
+
# Call parent method for each hostname
|
|
172
|
+
for hostname, job_status in all_job_status.items():
|
|
173
|
+
super().wait_for_job(
|
|
174
|
+
job_status,
|
|
175
|
+
logger,
|
|
176
|
+
f"Staging from {hostname!r}",
|
|
177
|
+
timeout,
|
|
178
|
+
poll_interval,
|
|
179
|
+
)
|
|
@@ -166,7 +166,7 @@ class CatalogClient(StacBase): # type: ignore # pylint: disable=too-many-ancest
|
|
|
166
166
|
def get_items(
|
|
167
167
|
self,
|
|
168
168
|
collection_id: str,
|
|
169
|
-
items_ids: str | None = None,
|
|
169
|
+
items_ids: list[str] | None = None,
|
|
170
170
|
owner_id: str | None = None,
|
|
171
171
|
) -> Iterator[Item]:
|
|
172
172
|
"""Get all items from a specific collection."""
|
{rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15.post17.dev0}/rs_client/stac/stac_base.py
RENAMED
|
@@ -186,7 +186,7 @@ class StacBase(RsClient):
|
|
|
186
186
|
return self.ps_client.get_collection(collection_id)
|
|
187
187
|
|
|
188
188
|
@handle_api_error
|
|
189
|
-
def get_items(self, collection_id: str, items_ids: str | None = None) -> Iterator["Item"]:
|
|
189
|
+
def get_items(self, collection_id: str, items_ids: list[str] | None = None) -> Iterator["Item"]:
|
|
190
190
|
"""
|
|
191
191
|
Retrieve all items or specific items from a collection.
|
|
192
192
|
|