rs-client-libraries 0.2a14__tar.gz → 0.2a15__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15}/PKG-INFO +8 -5
  2. {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15}/pyproject.toml +15 -10
  3. {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15}/rs_client/__init__.py +1 -1
  4. rs_client_libraries-0.2a15/rs_client/ogcapi/dpr_client.py +217 -0
  5. {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15}/rs_client/ogcapi/ogcapi_client.py +5 -13
  6. rs_client_libraries-0.2a15/rs_client/ogcapi/staging_client.py +179 -0
  7. {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15}/rs_client/stac/catalog_client.py +1 -1
  8. {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15}/rs_client/stac/stac_base.py +1 -1
  9. {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15}/rs_common/prefect_utils.py +52 -25
  10. {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15}/rs_common/utils.py +2 -2
  11. {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15}/rs_workflows/auxip_flow.py +4 -2
  12. {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15}/rs_workflows/cadip_flow.py +4 -2
  13. {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15}/rs_workflows/catalog_flow.py +2 -1
  14. {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15}/rs_workflows/dpr_flow.py +22 -20
  15. {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15}/rs_workflows/flow_utils.py +2 -2
  16. {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15}/rs_workflows/on_demand_processing.py +14 -10
  17. {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15}/rs_workflows/staging_flow.py +17 -31
  18. rs_client_libraries-0.2a14/rs_client/ogcapi/dpr_client.py +0 -80
  19. rs_client_libraries-0.2a14/rs_client/ogcapi/staging_client.py +0 -142
  20. {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15}/LICENSE +0 -0
  21. {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15}/README.adoc +0 -0
  22. {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15}/rs_client/ogcapi/__init__.py +0 -0
  23. {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15}/rs_client/rs_client.py +0 -0
  24. {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15}/rs_client/stac/__init__.py +0 -0
  25. {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15}/rs_client/stac/auxip_client.py +0 -0
  26. {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15}/rs_client/stac/cadip_client.py +0 -0
  27. {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15}/rs_common/__init__.py +0 -0
  28. {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15}/rs_common/config.py +0 -0
  29. {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15}/rs_common/init_opentelemetry.py +0 -0
  30. {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15}/rs_common/logging.py +0 -0
  31. {rs_client_libraries-0.2a14 → rs_client_libraries-0.2a15}/rs_workflows/__init__.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.3
2
2
  Name: rs-client-libraries
3
- Version: 0.2a14
3
+ Version: 0.2a15
4
4
  Summary: Copernicus Sentinel rs-client library and processing workflows.
5
5
  License: Apache-2.0
6
6
  Keywords: stac,copernicus,sentinel,processing
@@ -21,16 +21,19 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
21
21
  Requires-Dist: cachetools (>=5.5.2,<6.0.0)
22
22
  Requires-Dist: click (==8.1.8)
23
23
  Requires-Dist: openapi-core (>=0.19.5,<0.20.0)
24
- Requires-Dist: opentelemetry-distro (>=0.55b0,<0.56)
25
- Requires-Dist: opentelemetry-exporter-otlp (>=1.34.0,<2.0.0)
26
- Requires-Dist: opentelemetry-instrumentation-aws-lambda (>=0.55b0,<0.56)
27
- Requires-Dist: opentelemetry-test-utils (>=0.55b0,<0.56)
24
+ Requires-Dist: opentelemetry-distro (>=0.55b1,<0.56)
25
+ Requires-Dist: opentelemetry-exporter-otlp (>=1.34.1,<2.0.0)
26
+ Requires-Dist: opentelemetry-instrumentation-aws-lambda (>=0.55b1,<0.56)
27
+ Requires-Dist: opentelemetry-test-utils (>=0.55b1,<0.56)
28
28
  Requires-Dist: prefect[aws] (==3.2.13)
29
+ Requires-Dist: protobuf (>=5.29.5)
29
30
  Requires-Dist: pystac (>=1.13.0,<2.0.0)
30
31
  Requires-Dist: pystac-client (>=0.8.6,<0.9.0)
31
32
  Requires-Dist: python-dateutil (>=2.9.0.post0,<3.0.0)
32
33
  Requires-Dist: python-logging-loki (>=0.3.1,<0.4.0)
34
+ Requires-Dist: requests (>=2.32.4,<3.0.0)
33
35
  Requires-Dist: stac-pydantic (>=3.1.5,<4.0.0)
36
+ Requires-Dist: urllib3 (>=2.5.0)
34
37
  Project-URL: Documentation, https://home.rs-python.eu/rs-documentation/rs-client-libraries/docs/doc/
35
38
  Project-URL: Homepage, https://home.rs-python.eu/
36
39
  Project-URL: Repository, https://github.com/RS-PYTHON/rs-client-libraries
@@ -4,7 +4,7 @@ dynamic = ["version"] # see: https://github.com/mtkennerly/poetry-dynamic-versio
4
4
 
5
5
  [tool.poetry]
6
6
  name = "rs-client-libraries"
7
- version = "0.2a14"
7
+ version = "0.2a15"
8
8
  description = "Copernicus Sentinel rs-client library and processing workflows."
9
9
  license = "Apache-2.0"
10
10
  authors = ["ESA under contract with Airbus and CS <ops_coprs@airbus.com>"]
@@ -37,10 +37,11 @@ include = [
37
37
 
38
38
  [tool.poetry.dependencies]
39
39
  python = "^3.11"
40
- opentelemetry-distro = "^0.55b0" # then run 'poetry run opentelemetry-bootstrap -a install'
41
- opentelemetry-instrumentation-aws-lambda = "^0.55b0"
42
- opentelemetry-test-utils = "^0.55b0"
43
- opentelemetry-exporter-otlp = "^1.34.0"
40
+ opentelemetry-distro = "^0.55b1" # then run 'poetry run opentelemetry-bootstrap -a install'
41
+ opentelemetry-instrumentation-aws-lambda = "^0.55b1"
42
+ opentelemetry-test-utils = "^0.55b1"
43
+ opentelemetry-exporter-otlp = "^1.34.1"
44
+ protobuf = ">=5.29.5"
44
45
  python-logging-loki = "^0.3.1" # to avoid otl error: No module named 'logging_loki'
45
46
  cachetools = "^5.5.2"
46
47
  pystac = "^1.13.0"
@@ -50,20 +51,22 @@ stac-pydantic = "^3.1.5"
50
51
  openapi-core = "^0.19.5"
51
52
  click = "8.1.8"
52
53
  prefect = { version = "3.2.13", extras = ["aws"] }
54
+ requests = "^2.32.4"
55
+ urllib3 = ">=2.5.0"
53
56
 
54
57
  [tool.poetry.group.dev]
55
58
  optional = true
56
59
 
57
60
  [tool.poetry.group.dev.dependencies]
58
61
  pytest = "^8.3.5"
59
- pytest-cov = "^6.0.0"
62
+ pytest-cov = "^6.2.1"
60
63
  pytest-mock = "^3.14.0"
61
64
  pytest-error-for-skips = "^2.0.2"
62
65
  black = "==25.1.0"
63
- mypy = "^1.15.0"
66
+ mypy = "^1.16.0"
64
67
  pre-commit = "^4.1.0"
65
- bandit = "^1.8.2"
66
- safety = "^3.2.14"
68
+ bandit = "^1.8.3"
69
+ safety = "^3.5.2"
67
70
  pylint = "^3.3.4"
68
71
  pytest-docker = "^3.1.2"
69
72
  sphinx = "^8.1.3"
@@ -72,10 +75,11 @@ flake8 = "^7.1.1"
72
75
  flake8-pyproject = "^1.2.3"
73
76
  responses = "^0.25.7"
74
77
  starlette = "^0.45.3"
78
+ pytest-asyncio = "^1.0.0"
75
79
 
76
80
  [tool.poetry.group.demo.dependencies] # for hybrid mode
77
81
  jupyterlab = "^4.2.1"
78
- boto3 = "^1.37.11"
82
+ boto3 = "^1.38.34"
79
83
  papermill = "^2.6.0"
80
84
  tornado = "^6.5" # fix trivy vulnerability
81
85
 
@@ -98,6 +102,7 @@ markers = [
98
102
  "unit", # mark a test as unittest
99
103
  "integration", # mark a test as integration test
100
104
  ]
105
+ asyncio_default_fixture_loop_scope = "function"
101
106
  asyncio_mode = "auto"
102
107
 
103
108
  [tool.mypy]
@@ -15,4 +15,4 @@
15
15
  """rsclient main module."""
16
16
 
17
17
  # Set automatically by running `poetry dynamic-versioning`
18
- __version__ = "0.2a14"
18
+ __version__ = "0.2a15"
@@ -0,0 +1,217 @@
1
+ # Copyright 2024 CS Group
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """Use "DPR as a service" implemented by rs-dpr-service"""
16
+
17
+ import ast
18
+ import os.path as osp
19
+ import tempfile
20
+
21
+ import yaml
22
+ from openapi_core import OpenAPI # Spec, validate_request, validate_response
23
+
24
+ from rs_client.ogcapi.ogcapi_client import OgcApiClient
25
+ from rs_common import prefect_utils
26
+ from rs_common.utils import get_href_service
27
+
28
+ PATH_TO_YAML_OPENAPI = osp.realpath(
29
+ osp.join(
30
+ osp.dirname(__file__),
31
+ "../../config",
32
+ "staging_templates",
33
+ "yaml",
34
+ "dpr_openapi_schema.yaml",
35
+ ),
36
+ )
37
+
38
+
39
+ class DprClient(OgcApiClient):
40
+ """Implement the OGC API client for 'DPR as a service'."""
41
+
42
+ ##########################################
43
+ # Override parent methods and attributes #
44
+ ##########################################
45
+
46
+ # Init the OpenAPI instance from config file
47
+ openapi = OpenAPI.from_file_path(PATH_TO_YAML_OPENAPI)
48
+
49
+ @property
50
+ def endpoint_prefix(self) -> str:
51
+ """Return the endpoints prefix, if any."""
52
+ return "dpr/"
53
+
54
+ @property
55
+ def href_service(self) -> str:
56
+ """
57
+ Return the rs-dpr-service URL hostname.
58
+ This URL can be overwritten using the RSPY_HOST_DPR_SERVICE env variable (used e.g. for local mode).
59
+ Otherwise it should just be the RS-Server URL.
60
+ """
61
+ return get_href_service(self.rs_server_href, "RSPY_HOST_DPR_SERVICE")
62
+
63
+ def run_process(
64
+ self,
65
+ process: str,
66
+ s3_config_dir: str,
67
+ payload_subpath: str,
68
+ s3_report_dir: str | None,
69
+ use_dpr_mockup: bool = False,
70
+ ) -> dict:
71
+ """Method to start the process from rs-client - Call the endpoint /processes/{process}/execution
72
+
73
+ Args:
74
+ process: Process name
75
+ s3_config_dir: S3 bucket folder that contains the payload and configuration files to pass to the processor
76
+ payload_subpath: Payload file path, relative to the config folder
77
+ s3_report_dir: S3 bucket folder were the processor report files will be written (optional). All the eopf
78
+ local files written in the local "./reports" directory will be pushed to this S3 bucket folder.
79
+ use_dpr_mockup: Use the real or the mockup DPR processor ?
80
+
81
+ Return:
82
+ job_id (int, str): Returns the status code of the request + the identifier
83
+ (or None if endpoint fails) of the running job
84
+ """
85
+
86
+ # Data to pass to the real processor
87
+ data = {}
88
+ if not use_dpr_mockup:
89
+ data = {
90
+ "s3_config_dir": s3_config_dir,
91
+ "payload_subpath": payload_subpath,
92
+ "s3_report_dir": s3_report_dir,
93
+ }
94
+
95
+ # For the mockup processor, pass the payload contents.
96
+ # Download the payload file into a temp file.
97
+ else:
98
+ with tempfile.NamedTemporaryFile() as temp:
99
+ prefect_utils.s3_download_file( # type: ignore
100
+ osp.join(s3_config_dir, payload_subpath),
101
+ temp.name,
102
+ _sync=True, # type: ignore
103
+ )
104
+
105
+ # Read it as a yaml file
106
+ with open(temp.name, encoding="utf-8") as opened:
107
+ data = yaml.safe_load(opened)
108
+
109
+ # Add extra info
110
+ data.update({"use_mockup": use_dpr_mockup})
111
+
112
+ # Call the parent method
113
+ return super()._run_process(process, data)
114
+
115
+ def wait_for_job(self, *args, **kwargs) -> list[dict]: # type: ignore
116
+ """
117
+ Wait for job to finish.
118
+
119
+ Returns:
120
+ EOPF results
121
+ """
122
+ # Call parent method and parse results
123
+ job_status = super().wait_for_job(*args, **kwargs)
124
+ return ast.literal_eval(job_status["message"])
125
+
126
+ ######################################################
127
+ # These endpoints are not implemented by the service #
128
+ ######################################################
129
+
130
+ def get_processes(self) -> dict:
131
+ raise NotImplementedError
132
+
133
+ def get_jobs(self) -> dict:
134
+ raise NotImplementedError
135
+
136
+ def delete_job(self, _: str) -> dict:
137
+ raise NotImplementedError
138
+
139
+ def get_job_results(self, _: str) -> dict:
140
+ raise NotImplementedError
141
+
142
+ ####################
143
+ # Specific methods #
144
+ ####################
145
+
146
+ async def update_configuration(self, local_path: str, s3_path: str, is_payload: bool = False, **kwargs):
147
+ """
148
+ Update local configuration file depending on the environment, upload it to the s3 bucket,
149
+ and initialize output bucket folders.
150
+
151
+ Args:
152
+ local_path: Local configuration file path
153
+ s3_path: S3 bucket path where to upload to modified updated configuration file
154
+ is_payload: Specific behavior for the processor payload files
155
+ kwargs: Specific environment variables to expand in the configuration file
156
+ """
157
+
158
+ to_expand = {}
159
+
160
+ if self.local_mode:
161
+ # In local mode, replace the S3_xxx_CLUSTER env vars, by the env vars from the ~/.s3cfg
162
+ # config file, that contains access to the "real" s3 bucket
163
+ to_expand = {
164
+ "S3_ACCESSKEY_CLUSTER": "${access_key}",
165
+ "S3_SECRETKEY_CLUSTER": "${secret_key}",
166
+ "S3_ENDPOINT_CLUSTER": "${host_bucket}",
167
+ "S3_REGION_CLUSTER": "${bucket_location}",
168
+ }
169
+
170
+ else:
171
+ # In cluster mode, just use the "real" s3 bucket
172
+ to_expand = {
173
+ "S3_ACCESSKEY_CLUSTER": "${S3_ACCESSKEY}",
174
+ "S3_SECRETKEY_CLUSTER": "${S3_SECRETKEY}",
175
+ "S3_ENDPOINT_CLUSTER": "${S3_ENDPOINT}",
176
+ "S3_REGION_CLUSTER": "${S3_REGION}",
177
+ }
178
+
179
+ # Also expand the user-given parameters
180
+ to_expand.update(kwargs)
181
+
182
+ # Open the input local file
183
+ with open(local_path, encoding="utf-8") as opened:
184
+ contents = opened.read()
185
+
186
+ # Expand the env vars as $key, ${key} or %key%
187
+ for key, value in to_expand.items():
188
+ for key2 in f"${key}", f"${{{key}}}", f"%{key}%":
189
+ contents = contents.replace(key2, str(value))
190
+
191
+ # Read the payload contents
192
+ if is_payload:
193
+ payload = yaml.safe_load(contents)
194
+
195
+ # We need to create the output S3 folder with a dummy file before running DPR
196
+ for output_product in payload["I/O"]["output_products"]:
197
+ s3_output_dir = output_product["path"]
198
+ s3_empty_file = f"{s3_output_dir}/.empty"
199
+ self.logger.info(f"Write empty file: {self.logger.level} {s3_empty_file!r}")
200
+ await prefect_utils.s3_upload_empty_file(s3_empty_file)
201
+
202
+ # Change the dask authentication for local mode
203
+ cluster_config = payload["dask_context"]["cluster_config"]
204
+ if self.local_mode:
205
+ cluster_config["auth"] = cluster_config["auth_local_mode"]
206
+ del cluster_config["auth_local_mode"]
207
+
208
+ # yaml to str conversion
209
+ contents = yaml.dump(payload, default_flow_style=False, sort_keys=False)
210
+
211
+ # Write the modified contents to a temp file
212
+ with tempfile.NamedTemporaryFile() as tmp:
213
+ tmp.write(contents.encode("utf-8"))
214
+ tmp.flush()
215
+
216
+ # Upload the temp file to the s3 bucket
217
+ return await prefect_utils.s3_upload_file(tmp.name, s3_path)
@@ -18,7 +18,6 @@ import json
18
18
  import math
19
19
  import time
20
20
  from typing import Any
21
- from urllib.parse import urlparse
22
21
 
23
22
  # openapi_core libraries used for endpoints validation
24
23
  import requests
@@ -51,14 +50,6 @@ class OgcApiClient(RsClient):
51
50
  is passed as an extra argument).
52
51
  """
53
52
 
54
- @staticmethod
55
- def is_url(path):
56
- """
57
- Function to check if a string is an url
58
- """
59
- parsed = urlparse(path)
60
- return bool(parsed.scheme and parsed.netloc)
61
-
62
53
  @property
63
54
  def endpoint_prefix(self) -> str:
64
55
  """Return the endpoints prefix, if any."""
@@ -158,7 +149,7 @@ class OgcApiClient(RsClient):
158
149
  )
159
150
  return self.validate_and_unmarshal_response(response)
160
151
 
161
- def run_process(self, process: str, body: dict) -> dict:
152
+ def _run_process(self, process: str, body: dict) -> dict:
162
153
  """Method to start the process from rs-client - Call the endpoint /processes/{process}/execution
163
154
 
164
155
  Args:
@@ -240,7 +231,7 @@ class OgcApiClient(RsClient):
240
231
  Wait for job to finish.
241
232
 
242
233
  Args:
243
- job_status: Returned by `run_process`
234
+ job_status: Returned by `_run_process`
244
235
  logger: To show advancement in logger
245
236
  job_name: Job name to show in the logger
246
237
  timeout: Job completion timeout in seconds
@@ -253,7 +244,8 @@ class OgcApiClient(RsClient):
253
244
  RuntimeError in case of error
254
245
  """
255
246
  try:
256
- status_type, job_identifier = job_status["status"], job_status["jobID"]
247
+ status_type = ""
248
+ job_identifier = job_status.get("jobID")
257
249
  if not job_identifier:
258
250
  raise RuntimeError("Job identifier is missing.")
259
251
 
@@ -261,7 +253,7 @@ class OgcApiClient(RsClient):
261
253
  job_status = self.get_job_info(job_identifier)
262
254
  if logger:
263
255
  logger.info(f"job_status: {job_status}")
264
- status_type = job_status.get("status")
256
+ status_type = job_status.get("status", "")
265
257
  if logger:
266
258
  logger.info(
267
259
  f"----- {job_name} job {job_identifier!r}: {status_type.upper()} \n",
@@ -0,0 +1,179 @@
1
+ # Copyright 2024 CS Group
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """Launch staging with rs-client-libraries"""
16
+
17
+ import json
18
+ import math
19
+ import os
20
+ import os.path as osp
21
+ from collections import defaultdict
22
+ from collections.abc import Sequence
23
+ from typing import Any
24
+ from urllib.parse import urlparse
25
+
26
+ # openapi_core libraries used for endpoints validation
27
+ from openapi_core import OpenAPI # Spec, validate_request, validate_response
28
+ from stac_pydantic.api import Item, ItemCollection
29
+
30
+ from rs_client.ogcapi.ogcapi_client import OgcApiClient, OgcValidationException
31
+ from rs_common.utils import get_href_service
32
+
33
+ PATH_TO_YAML_OPENAPI = osp.realpath(
34
+ osp.join(
35
+ osp.dirname(__file__),
36
+ "../../config",
37
+ "staging_templates",
38
+ "yaml",
39
+ "staging_openapi_schema.yaml",
40
+ ),
41
+ )
42
+ RESOURCE = "staging"
43
+
44
+
45
+ class StagingClient(OgcApiClient):
46
+ """Implement the OGC API client for the staging."""
47
+
48
+ # Init the OpenAPI instance from config file
49
+ openapi = OpenAPI.from_file_path(PATH_TO_YAML_OPENAPI)
50
+
51
+ @property
52
+ def href_service(self) -> str:
53
+ """
54
+ Return the RS-Server staging URL hostname.
55
+ This URL can be overwritten using the RSPY_HOST_STAGING env variable (used e.g. for local mode).
56
+ Otherwise it should just be the RS-Server URL.
57
+ """
58
+ return get_href_service(self.rs_server_href, "RSPY_HOST_STAGING")
59
+
60
+ def run_staging( # pylint: disable=too-many-locals
61
+ self,
62
+ stac_input: dict[Any, Any] | str,
63
+ out_coll_name: str,
64
+ ) -> dict[str, dict]:
65
+ """Method to start the staging process from rs-client - Call the endpoint /processes/staging/execution
66
+
67
+ Args:
68
+ stac_input (dict | str): it can be:
69
+ - A Python dictionary corresponding to a Feature or a FeatureCollection (that can be for example
70
+ the output of a search for Cadip or Auxip sessions)
71
+ - A json string corresponding to a Feature or a FeatureCollection
72
+ - A string corresponding to a path to a json file containing a Feature or a FeatureCollection
73
+ - A single link that returns a STAC itemCollection: this link should be an url to search a
74
+ itemCollection, for example:
75
+ http://localhost:8002/cadip/search?ids=S1A_20231120061537234567&collections=cadip_sentinel1
76
+ out_coll_name (str): name of the output collection
77
+ Return:
78
+ dict(hostname, job_id (int, str)): Returns the status codes of the staging requests (one per URL hostname) +
79
+ the identifiers
80
+ """
81
+ stac_input_dict = {}
82
+
83
+ # ----- Case 1: we only load a link (that refers to a STAC itemCollection) in the staging request body
84
+ if isinstance(stac_input, str):
85
+
86
+ # Check that it's an url
87
+ parsed = urlparse(stac_input)
88
+ if parsed.scheme and parsed.netloc and parsed.hostname:
89
+ staging_body = {"inputs": {"collection": out_coll_name, "items": {"href": stac_input}}}
90
+ return {parsed.hostname: super()._run_process(RESOURCE, staging_body)}
91
+
92
+ # ----- Case 2: we directly load a STAC ItemCollection in the staging request body
93
+
94
+ # If stac_input is a file, load this file to a dictionary
95
+ if isinstance(stac_input, str):
96
+ # If the input is a valid path to a json_file, load this file
97
+ if os.path.exists(os.path.dirname(stac_input)) and stac_input.endswith(".json"):
98
+ # Read the yaml or json file
99
+ with open(stac_input, encoding="utf-8") as opened:
100
+ stac_file_to_dict = json.loads(opened.read())
101
+ stac_input_dict = stac_file_to_dict
102
+ # If the input string is not a path, try to convert the content of the string to a json dictionary
103
+ else:
104
+ try:
105
+ stac_input_dict = json.loads(stac_input)
106
+ except json.JSONDecodeError as e:
107
+ raise OgcValidationException(
108
+ f"""Invalid input format: {stac_input} - Input data must be either:
109
+ - A Python dictionary corresponding to a Feature or a FeatureCollection
110
+ (that can be for example the output of a search for Cadip or Auxip sessions)
111
+ - A json string corresponding to a Feature or a FeatureCollection
112
+ - A string corresponding to a path to a json file containing a Feature or a
113
+ FeatureCollection
114
+ - A single link that returns a STAC ItemCollection: this link should be an
115
+ url to search an ItemCollection
116
+ """.strip(),
117
+ ) from e
118
+ else:
119
+ stac_input_dict = stac_input
120
+
121
+ if "type" not in stac_input_dict:
122
+ raise KeyError("Key 'type' is missing from the staging input data")
123
+
124
+ # Validate input data using Pydantic
125
+ if stac_input_dict["type"] == "Feature":
126
+ stac_items: Sequence[Item] = [Item(**stac_input_dict)]
127
+ else:
128
+ stac_items = ItemCollection(**stac_input_dict).features
129
+
130
+ # Order the items by station url hostname
131
+ hostname_items: dict[str, list[Item]] = defaultdict(list)
132
+ for item in stac_items:
133
+ assets = list(item.assets.values())
134
+ if assets:
135
+ hostname = urlparse(assets[0].href).hostname or ""
136
+ hostname_items[hostname].append(item)
137
+
138
+ # Trigger staging for each domain items
139
+ hostname_triggers: dict[str, dict] = {}
140
+ for hostname, items in hostname_items.items():
141
+ stac_item_collection = ItemCollection(type="FeatureCollection", features=items)
142
+ staging_body = {
143
+ "inputs": {
144
+ "collection": out_coll_name,
145
+ "items": {"value": stac_item_collection.model_dump(mode="json")},
146
+ },
147
+ }
148
+ hostname_triggers[hostname] = super()._run_process(RESOURCE, staging_body)
149
+
150
+ return hostname_triggers
151
+
152
+ def wait_for_jobs(
153
+ self,
154
+ all_job_status: dict[str, dict],
155
+ logger=None,
156
+ timeout: int | float = math.inf,
157
+ poll_interval: int = 2,
158
+ ):
159
+ """
160
+ Wait for job to finish.
161
+
162
+ Args:
163
+ job_status: Returned by `run_staging`
164
+ logger: To show advancement in logger
165
+ timeout: Job completion timeout in seconds
166
+ poll_interval: When to check again for job completion in seconds
167
+
168
+ Raises:
169
+ RuntimeError in case of error
170
+ """
171
+ # Call parent method for each hostname
172
+ for hostname, job_status in all_job_status.items():
173
+ super().wait_for_job(
174
+ job_status,
175
+ logger,
176
+ f"Staging from {hostname!r}",
177
+ timeout,
178
+ poll_interval,
179
+ )
@@ -166,7 +166,7 @@ class CatalogClient(StacBase): # type: ignore # pylint: disable=too-many-ancest
166
166
  def get_items(
167
167
  self,
168
168
  collection_id: str,
169
- items_ids: str | None = None,
169
+ items_ids: list[str] | None = None,
170
170
  owner_id: str | None = None,
171
171
  ) -> Iterator[Item]:
172
172
  """Get all items from a specific collection."""
@@ -186,7 +186,7 @@ class StacBase(RsClient):
186
186
  return self.ps_client.get_collection(collection_id)
187
187
 
188
188
  @handle_api_error
189
- def get_items(self, collection_id: str, items_ids: str | None = None) -> Iterator["Item"]:
189
+ def get_items(self, collection_id: str, items_ids: list[str] | None = None) -> Iterator["Item"]:
190
190
  """
191
191
  Retrieve all items or specific items from a collection.
192
192