rs-client-libraries 1.0a7__tar.gz → 1.0a8.post16.dev0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/PKG-INFO +6 -7
- {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/pyproject.toml +7 -8
- {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/__init__.py +1 -1
- {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/ogcapi/ogcapi_client.py +15 -2
- {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/rs_client.py +0 -11
- {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/stac/stac_base.py +2 -2
- {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_common/init_opentelemetry.py +43 -29
- {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/cadip_flow.py +51 -0
- {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/catalog_flow.py +73 -58
- {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/dpr_flow.py +78 -54
- {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/flow_utils.py +153 -88
- rs_client_libraries-1.0a8.post16.dev0/rs_workflows/on_demand/common/__init__.py +18 -0
- rs_client_libraries-1.0a8.post16.dev0/rs_workflows/on_demand/common/l0.py +107 -0
- rs_client_libraries-1.0a8.post16.dev0/rs_workflows/on_demand/common/l0_last_steps.py +102 -0
- rs_client_libraries-1.0a7/rs_workflows/on_demand/stage_last_sessions.py → rs_client_libraries-1.0a8.post16.dev0/rs_workflows/on_demand/common/staging.py +77 -112
- rs_client_libraries-1.0a8.post16.dev0/rs_workflows/on_demand/common/types.py +167 -0
- rs_client_libraries-1.0a8.post16.dev0/rs_workflows/on_demand/sentinel1/__init__.py +18 -0
- rs_client_libraries-1.0a8.post16.dev0/rs_workflows/on_demand/sentinel1/s1_l0.py +53 -0
- rs_client_libraries-1.0a8.post16.dev0/rs_workflows/on_demand/sentinel3/__init__.py +18 -0
- rs_client_libraries-1.0a8.post16.dev0/rs_workflows/on_demand/sentinel3/s3_l0.py +53 -0
- {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/on_demand_processing.py +70 -113
- {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/payload_generator.py +42 -14
- {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/payload_template.py +4 -4
- rs_client_libraries-1.0a8.post16.dev0/rs_workflows/prip_flow.py +131 -0
- {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/record_performance.py +5 -4
- rs_client_libraries-1.0a8.post16.dev0/rs_workflows/utils/cadip.py +144 -0
- rs_client_libraries-1.0a8.post16.dev0/rs_workflows/utils/catalog.py +90 -0
- rs_client_libraries-1.0a8.post16.dev0/rs_workflows/utils/dask.py +76 -0
- rs_client_libraries-1.0a8.post16.dev0/rs_workflows/utils/dpr.py +100 -0
- rs_client_libraries-1.0a7/rs_client/stac/edrs_client.py +0 -70
- rs_client_libraries-1.0a7/rs_workflows/prip_flow.py +0 -69
- {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/LICENSE +0 -0
- {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/README.md +0 -0
- {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/ogcapi/__init__.py +0 -0
- {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/ogcapi/dpr_client.py +0 -0
- {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/ogcapi/staging_client.py +0 -0
- {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/osam_client.py +0 -0
- {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/stac/__init__.py +0 -0
- {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/stac/auxip_client.py +0 -0
- {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/stac/cadip_client.py +0 -0
- {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/stac/catalog_client.py +0 -0
- {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/stac/prip_client.py +0 -0
- {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_common/__init__.py +0 -0
- {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_common/config.py +0 -0
- {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_common/logging.py +0 -0
- {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_common/prefect_utils.py +0 -0
- {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_common/utils.py +0 -0
- {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/__init__.py +0 -0
- {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/auxip_flow.py +0 -0
- {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/init_pi_db_flow.py +0 -0
- {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/on_demand/__init__.py +0 -0
- {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/operation/__init__.py +0 -0
- {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/operation/init_quota_monitoring_flow.py +0 -0
- {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/operation/osam_flows.py +0 -0
- {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/operation/quota_monitoring_db_models.py +0 -0
- {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/operation/quota_monitoring_flow.py +0 -0
- {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/payload_builder.py +0 -0
- {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/pi_db_models.py +0 -0
- {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/staging_flow.py +0 -0
- {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/storage_configuration.py +0 -0
- {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/utils/__init__.py +0 -0
- {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/utils/artifact_verbose.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: rs-client-libraries
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.0a8.post16.dev0
|
|
4
4
|
Summary: Copernicus Sentinel rs-client library and processing workflows.
|
|
5
5
|
License: Apache-2.0
|
|
6
6
|
License-File: LICENSE
|
|
@@ -23,13 +23,12 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
|
23
23
|
Requires-Dist: SQLAlchemy (>=2.0.37,<3.0.0)
|
|
24
24
|
Requires-Dist: cachetools (>=5.5.2)
|
|
25
25
|
Requires-Dist: click (==8.3.1)
|
|
26
|
+
Requires-Dist: dask-gateway (==2025.4.0)
|
|
26
27
|
Requires-Dist: openapi-core (==0.22.0)
|
|
27
|
-
Requires-Dist: opentelemetry-distro (>=0.
|
|
28
|
-
Requires-Dist: opentelemetry-exporter-otlp (>=1.
|
|
29
|
-
Requires-Dist:
|
|
30
|
-
Requires-Dist:
|
|
31
|
-
Requires-Dist: prefect-aws (>=0.7.4)
|
|
32
|
-
Requires-Dist: prefect[aws] (==3.6.12)
|
|
28
|
+
Requires-Dist: opentelemetry-distro (>=0.61b0,<0.62)
|
|
29
|
+
Requires-Dist: opentelemetry-exporter-otlp (>=1.40.0,<2.0.0)
|
|
30
|
+
Requires-Dist: prefect-aws (>=0.7.5)
|
|
31
|
+
Requires-Dist: prefect[aws] (==3.6.20)
|
|
33
32
|
Requires-Dist: protobuf (>=5.29.5)
|
|
34
33
|
Requires-Dist: psycopg2-binary (>=2.9.10)
|
|
35
34
|
Requires-Dist: pystac (>=1.13.0,<2.0.0)
|
|
@@ -18,7 +18,7 @@ dynamic = ["version"] # see: https://github.com/mtkennerly/poetry-dynamic-versio
|
|
|
18
18
|
|
|
19
19
|
[tool.poetry]
|
|
20
20
|
name = "rs-client-libraries"
|
|
21
|
-
version = "1.
|
|
21
|
+
version = "1.0a8.post16.dev0"
|
|
22
22
|
description = "Copernicus Sentinel rs-client library and processing workflows."
|
|
23
23
|
license = "Apache-2.0"
|
|
24
24
|
authors = ["ESA under contract with Airbus and CS <ops_coprs@airbus.com>"]
|
|
@@ -51,10 +51,8 @@ include = [
|
|
|
51
51
|
|
|
52
52
|
[tool.poetry.dependencies]
|
|
53
53
|
python = ">=3.11,<3.15"
|
|
54
|
-
opentelemetry-distro = "^0.
|
|
55
|
-
opentelemetry-
|
|
56
|
-
opentelemetry-test-utils = "^0.60b1"
|
|
57
|
-
opentelemetry-exporter-otlp = "^1.39.1"
|
|
54
|
+
opentelemetry-distro = "^0.61b0" # then run 'poetry run opentelemetry-bootstrap -a install'
|
|
55
|
+
opentelemetry-exporter-otlp = "^1.40.0"
|
|
58
56
|
protobuf = ">=5.29.5"
|
|
59
57
|
cachetools = ">=5.5.2"
|
|
60
58
|
pystac = "^1.13.0"
|
|
@@ -63,11 +61,12 @@ python-dateutil = "^2.9.0.post0"
|
|
|
63
61
|
stac-pydantic = "^3.4.0"
|
|
64
62
|
openapi-core = "0.22.0"
|
|
65
63
|
click = "8.3.1"
|
|
66
|
-
prefect = { version = "3.6.
|
|
67
|
-
prefect-aws = ">=0.7.
|
|
64
|
+
prefect = { version = "3.6.20", extras = ["aws"] }
|
|
65
|
+
prefect-aws = ">=0.7.5"
|
|
68
66
|
requests = "^2.32.4"
|
|
69
67
|
urllib3 = ">=2.6.3"
|
|
70
68
|
SQLAlchemy = "^2.0.37"
|
|
69
|
+
dask-gateway = "2025.4.0"
|
|
71
70
|
uv = ">=0.9.7"
|
|
72
71
|
psycopg2-binary = ">=2.9.10"
|
|
73
72
|
werkzeug = ">=3.1.6"
|
|
@@ -82,7 +81,7 @@ pytest-cov = "^7.0.0"
|
|
|
82
81
|
pytest-docker = "^3.2.3"
|
|
83
82
|
pytest-error-for-skips = "^2.0.2"
|
|
84
83
|
pytest-mock = "^3.15.1"
|
|
85
|
-
black = "
|
|
84
|
+
black = ">=26.3.1"
|
|
86
85
|
mypy = "^1.16.0"
|
|
87
86
|
pre-commit = "^4.1.0"
|
|
88
87
|
bandit = "^1.8.3"
|
|
@@ -32,6 +32,8 @@ from requests.models import PreparedRequest
|
|
|
32
32
|
from rs_client.rs_client import TIMEOUT, RsClient
|
|
33
33
|
from rs_common.utils import get_href_service
|
|
34
34
|
|
|
35
|
+
MAX_RETRIES_NUMBER_FOR_GETTING_JOB_STATUS = 20
|
|
36
|
+
|
|
35
37
|
|
|
36
38
|
class OgcValidationException(Exception):
|
|
37
39
|
"""
|
|
@@ -279,9 +281,20 @@ class OgcApiClient(RsClient):
|
|
|
279
281
|
previous_status: dict[str, str] = {}
|
|
280
282
|
# We use previous_status to remove log when the status remains unchanged.
|
|
281
283
|
# This way we reduce the number of useless information
|
|
282
|
-
|
|
284
|
+
retries_number = MAX_RETRIES_NUMBER_FOR_GETTING_JOB_STATUS
|
|
283
285
|
while True:
|
|
284
|
-
|
|
286
|
+
try:
|
|
287
|
+
job_status = self.get_job_info(job_identifier)
|
|
288
|
+
except TimeoutError:
|
|
289
|
+
retries_number -= 1
|
|
290
|
+
if retries_number <= 0:
|
|
291
|
+
raise
|
|
292
|
+
time.sleep(2)
|
|
293
|
+
logger.warning(
|
|
294
|
+
f"Timeout while waiting for {job_name} job {job_identifier!r} status. "
|
|
295
|
+
f"Retries left: {retries_number}",
|
|
296
|
+
)
|
|
297
|
+
continue
|
|
285
298
|
to_be_logged = previous_status != job_status
|
|
286
299
|
previous_status = job_status
|
|
287
300
|
|
|
@@ -43,7 +43,6 @@ class RsClient: # pylint: disable=too-many-instance-attributes
|
|
|
43
43
|
- rs-server-cadip
|
|
44
44
|
- rs-server-auxip
|
|
45
45
|
- rs-server-prip
|
|
46
|
-
- rs-server-edrs
|
|
47
46
|
- rs-server-catalog
|
|
48
47
|
|
|
49
48
|
This class provides methods to authenticate and interact with RS-Server,
|
|
@@ -262,16 +261,6 @@ class RsClient: # pylint: disable=too-many-instance-attributes
|
|
|
262
261
|
|
|
263
262
|
return CadipClient(self.rs_server_href, self.rs_server_api_key, self.logger, **kwargs)
|
|
264
263
|
|
|
265
|
-
def get_edrs_client(self, **kwargs) -> "EdrsClient": # type: ignore # noqa: F821
|
|
266
|
-
"""
|
|
267
|
-
Return an instance of the child class EdrsClient, with the same attributes as this "self" instance.
|
|
268
|
-
"""
|
|
269
|
-
from rs_client.stac.edrs_client import ( # pylint: disable=import-outside-toplevel,cyclic-import
|
|
270
|
-
EdrsClient,
|
|
271
|
-
)
|
|
272
|
-
|
|
273
|
-
return EdrsClient(self.rs_server_href, self.rs_server_api_key, self.logger, **kwargs)
|
|
274
|
-
|
|
275
264
|
def get_catalog_client(self, **kwargs) -> "CatalogClient": # type: ignore # noqa: F821
|
|
276
265
|
"""
|
|
277
266
|
Return an instance of the child class CatalogClient, with the same attributes as this "self" instance.
|
{rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/stac/stac_base.py
RENAMED
|
@@ -252,8 +252,8 @@ class StacBase(RsClient):
|
|
|
252
252
|
try:
|
|
253
253
|
return collection.get_items(*items_ids)
|
|
254
254
|
|
|
255
|
-
# Avoid pystac-client fallback through /search
|
|
256
|
-
# collection.get_items(*ids) internally triggers a search request
|
|
255
|
+
# Avoid pystac-client fallback through /search ; fetch items one by one.
|
|
256
|
+
# collection.get_items(*ids) internally triggers a search request.
|
|
257
257
|
except (APIError, requests.HTTPError, STACError, ValueError, TypeError) as exc:
|
|
258
258
|
self.logger.debug("Direct retrieval failed, fallback to per-item: %s", exc)
|
|
259
259
|
|
{rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_common/init_opentelemetry.py
RENAMED
|
@@ -23,10 +23,10 @@ from collections.abc import Iterator
|
|
|
23
23
|
from threading import Lock
|
|
24
24
|
|
|
25
25
|
import opentelemetry.instrumentation
|
|
26
|
+
import requests
|
|
26
27
|
from opentelemetry import trace
|
|
27
28
|
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
|
|
28
29
|
from opentelemetry.instrumentation.asyncio import AsyncioInstrumentor
|
|
29
|
-
from opentelemetry.instrumentation.aws_lambda import AwsLambdaInstrumentor
|
|
30
30
|
from opentelemetry.instrumentation.instrumentor import BaseInstrumentor # type: ignore
|
|
31
31
|
from opentelemetry.instrumentation.requests import RequestsInstrumentor
|
|
32
32
|
from opentelemetry.sdk.resources import Resource
|
|
@@ -81,43 +81,51 @@ def parse_data(data) -> str:
|
|
|
81
81
|
except Exception: # pylint: disable=broad-exception-caught # nosec
|
|
82
82
|
pass
|
|
83
83
|
|
|
84
|
-
# If we have a dict
|
|
84
|
+
# If we have a dict
|
|
85
85
|
if isinstance(data, dict):
|
|
86
|
+
|
|
87
|
+
# Decode bytes
|
|
88
|
+
data = {
|
|
89
|
+
key.decode("utf-8") if isinstance(key, bytes) else key: (
|
|
90
|
+
value.decode("utf-8") if isinstance(value, bytes) else value
|
|
91
|
+
)
|
|
92
|
+
for key, value in data.items()
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
# Convert to strings
|
|
96
|
+
data = {str(key): str(value) for key, value in data.items()}
|
|
97
|
+
|
|
98
|
+
# Apply json formatting
|
|
86
99
|
data = json.dumps(data, indent=2)
|
|
87
100
|
|
|
88
101
|
return data or ""
|
|
89
102
|
|
|
90
103
|
|
|
91
|
-
def
|
|
104
|
+
def requests_hook(span: Span, request: requests.PreparedRequest, response: requests.Response | None = None):
|
|
92
105
|
"""
|
|
93
|
-
|
|
106
|
+
Callback function invoked by RequestsInstrumentor. It implements the hooks:
|
|
107
|
+
|
|
108
|
+
- request_hook: invoked right after a span is created.
|
|
109
|
+
- response_hook: invoked right before the span has finished processing a response.
|
|
110
|
+
|
|
111
|
+
See: https://opentelemetry-python-contrib.readthedocs.io/en/latest/instrumentation/requests/requests.html
|
|
94
112
|
"""
|
|
95
|
-
if not span:
|
|
113
|
+
if not (span and span.is_recording()):
|
|
96
114
|
return
|
|
97
115
|
|
|
98
|
-
# Copy
|
|
99
|
-
# top in the grafana UI, it's more readable
|
|
100
|
-
span.set_attribute("_url", span.attributes.get("http.url"))
|
|
116
|
+
# Copy this attribute by adding a '_' prefix to the name,
|
|
117
|
+
# so it appears at the top in the grafana UI, it's more readable
|
|
118
|
+
span.set_attribute("_url", span.attributes.get("http.url")) # type: ignore
|
|
101
119
|
|
|
102
120
|
if trace_headers():
|
|
103
121
|
span.set_attribute("http.request.headers", parse_data(request.headers))
|
|
122
|
+
if response:
|
|
123
|
+
span.set_attribute("http.response.headers", parse_data(response.headers))
|
|
104
124
|
|
|
105
125
|
if trace_body():
|
|
106
126
|
span.set_attribute("http.request.body", parse_data(request.body))
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
def response_hook(span, request, response): # pylint: disable=W0613
|
|
110
|
-
"""
|
|
111
|
-
HTTP responses intrumentation
|
|
112
|
-
"""
|
|
113
|
-
if not span:
|
|
114
|
-
return
|
|
115
|
-
|
|
116
|
-
if trace_headers():
|
|
117
|
-
span.set_attribute("http.response.headers", parse_data(response.headers))
|
|
118
|
-
|
|
119
|
-
if trace_body():
|
|
120
|
-
span.set_attribute("http.response.content", parse_data(response.content))
|
|
127
|
+
if response:
|
|
128
|
+
span.set_attribute("http.response.content", parse_data(response.content))
|
|
121
129
|
|
|
122
130
|
|
|
123
131
|
def init_traces(service_name: str, logger=None):
|
|
@@ -128,6 +136,7 @@ def init_traces(service_name: str, logger=None):
|
|
|
128
136
|
service_name (str): service name
|
|
129
137
|
logger: non-default logger to user
|
|
130
138
|
"""
|
|
139
|
+
|
|
131
140
|
# See: https://github.com/softwarebloat/python-tracing-demo/tree/main
|
|
132
141
|
|
|
133
142
|
# No concurrent threads
|
|
@@ -191,24 +200,29 @@ def init_traces(service_name: str, logger=None):
|
|
|
191
200
|
classes.add(_class)
|
|
192
201
|
|
|
193
202
|
# Don't instrument these classes, they have errors, maybe we should see why
|
|
194
|
-
if _class in [AsyncioInstrumentor,
|
|
203
|
+
if _class in [AsyncioInstrumentor, BaseInstrumentor]:
|
|
195
204
|
continue
|
|
196
205
|
|
|
197
206
|
# If the "instrument" method exists, call it
|
|
198
207
|
_instrument = getattr(_class, "instrument", None)
|
|
199
208
|
if callable(_instrument):
|
|
200
|
-
|
|
201
209
|
_class_instance = _class()
|
|
210
|
+
if _class_instance.is_instrumented_by_opentelemetry:
|
|
211
|
+
continue
|
|
212
|
+
# name = f"{module_str}.{_class.__name__}".removeprefix(prefix)
|
|
213
|
+
# logger.debug(f"OpenTelemetry instrumentation of {name!r}")
|
|
214
|
+
|
|
215
|
+
# Handle specific hooks
|
|
202
216
|
if _class == RequestsInstrumentor and (trace_headers() or trace_body()):
|
|
203
217
|
_class_instance.instrument(
|
|
204
218
|
tracer_provider=otel_tracer,
|
|
205
|
-
request_hook=
|
|
206
|
-
response_hook=
|
|
219
|
+
request_hook=requests_hook,
|
|
220
|
+
response_hook=requests_hook,
|
|
207
221
|
)
|
|
208
|
-
|
|
222
|
+
|
|
223
|
+
# General case (no hooks)
|
|
224
|
+
else:
|
|
209
225
|
_class_instance.instrument(tracer_provider=otel_tracer)
|
|
210
|
-
# name = f"{module_str}.{_class.__name__}".removeprefix(prefix)
|
|
211
|
-
# logger.debug(f"OpenTelemetry instrumentation of {name!r}")
|
|
212
226
|
|
|
213
227
|
|
|
214
228
|
@_agnosticcontextmanager
|
{rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/cadip_flow.py
RENAMED
|
@@ -19,6 +19,7 @@ from pystac import ItemCollection
|
|
|
19
19
|
|
|
20
20
|
from rs_client.stac.cadip_client import CadipClient
|
|
21
21
|
from rs_workflows.flow_utils import FlowEnv, FlowEnvArgs
|
|
22
|
+
from rs_workflows.staging_flow import staging_task
|
|
22
23
|
|
|
23
24
|
|
|
24
25
|
@flow(name="Cadip search")
|
|
@@ -58,6 +59,56 @@ async def search(
|
|
|
58
59
|
return found
|
|
59
60
|
|
|
60
61
|
|
|
62
|
+
@flow(name="On-demand Cadip staging")
|
|
63
|
+
async def on_demand_cadip_staging(
|
|
64
|
+
env: FlowEnvArgs,
|
|
65
|
+
cadip_collection_identifier: str,
|
|
66
|
+
session_identifier: str,
|
|
67
|
+
catalog_collection_identifier: str,
|
|
68
|
+
staging_retries: int = 3,
|
|
69
|
+
staging_retry_delay: int = 60,
|
|
70
|
+
):
|
|
71
|
+
"""
|
|
72
|
+
Flow to retrieve a session, stage it and add the STAC item into the catalog.
|
|
73
|
+
|
|
74
|
+
Args:
|
|
75
|
+
env: Prefect flow environment
|
|
76
|
+
cadip_collection_identifier: CADIP collection identifier that contains the mission and station
|
|
77
|
+
(e.g. s1_ins for Sentinel-1 sessions from the Inuvik station)
|
|
78
|
+
session_identifier: Session identifier
|
|
79
|
+
catalog_collection_identifier: Catalog collection identifier where CADIP sessions and AUX data are staged
|
|
80
|
+
"""
|
|
81
|
+
|
|
82
|
+
# Init flow environment and opentelemetry span
|
|
83
|
+
flow_env = FlowEnv(env)
|
|
84
|
+
with flow_env.start_span(__name__, "on-demand-cadip-staging"):
|
|
85
|
+
|
|
86
|
+
# Search Cadip sessions
|
|
87
|
+
cadip_items = search_task.with_options(
|
|
88
|
+
retries=3,
|
|
89
|
+
retry_delay_seconds=60,
|
|
90
|
+
).submit(
|
|
91
|
+
flow_env.serialize(),
|
|
92
|
+
cadip_collection_identifier,
|
|
93
|
+
session_identifier,
|
|
94
|
+
error_if_empty=True,
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
# Stage Cadip items.
|
|
98
|
+
staged = staging_task.with_options(
|
|
99
|
+
retries=staging_retries,
|
|
100
|
+
retry_delay_seconds=staging_retry_delay,
|
|
101
|
+
).submit(
|
|
102
|
+
flow_env.serialize(),
|
|
103
|
+
cadip_items,
|
|
104
|
+
catalog_collection_identifier,
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
# Wait for last task to end.
|
|
108
|
+
# NOTE: use .result() and not .wait() to unwrap and propagate exceptions, if any.
|
|
109
|
+
staged.result() # type: ignore[unused-coroutine]
|
|
110
|
+
|
|
111
|
+
|
|
61
112
|
###########################
|
|
62
113
|
# Call the flows as tasks #
|
|
63
114
|
###########################
|
{rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/catalog_flow.py
RENAMED
|
@@ -17,10 +17,15 @@
|
|
|
17
17
|
import json
|
|
18
18
|
|
|
19
19
|
from prefect import flow, get_run_logger, task
|
|
20
|
-
from pystac import
|
|
20
|
+
from pystac import ItemCollection
|
|
21
21
|
|
|
22
22
|
from rs_client.stac.catalog_client import CatalogClient
|
|
23
|
-
from rs_workflows.flow_utils import
|
|
23
|
+
from rs_workflows.flow_utils import (
|
|
24
|
+
DprProcessedItemMetadata,
|
|
25
|
+
FlowEnv,
|
|
26
|
+
FlowEnvArgs,
|
|
27
|
+
FlowGeneratedProduct,
|
|
28
|
+
)
|
|
24
29
|
|
|
25
30
|
#################
|
|
26
31
|
# Catalog flows #
|
|
@@ -71,8 +76,8 @@ async def catalog_search(
|
|
|
71
76
|
@task(name="Publish to catalog")
|
|
72
77
|
async def publish(
|
|
73
78
|
env: FlowEnvArgs,
|
|
74
|
-
|
|
75
|
-
|
|
79
|
+
generated_product_to_collection_identifier: list[FlowGeneratedProduct],
|
|
80
|
+
items_metadata: list[DprProcessedItemMetadata],
|
|
76
81
|
):
|
|
77
82
|
"""
|
|
78
83
|
Publish items to the catalog
|
|
@@ -80,7 +85,7 @@ async def publish(
|
|
|
80
85
|
Args:
|
|
81
86
|
env: Prefect flow environment
|
|
82
87
|
collection: Catalog collection identifier where the items are published
|
|
83
|
-
|
|
88
|
+
items_metadata: List of DprProcessedItemMetadata containing items to publish
|
|
84
89
|
"""
|
|
85
90
|
logger = get_run_logger()
|
|
86
91
|
flow_env = FlowEnv(env)
|
|
@@ -88,29 +93,30 @@ async def publish(
|
|
|
88
93
|
catalog_client: CatalogClient = flow_env.rs_client.get_catalog_client()
|
|
89
94
|
|
|
90
95
|
with flow_env.start_span(__name__, "publish-to-catalog"):
|
|
91
|
-
for
|
|
96
|
+
for item_metadata in items_metadata:
|
|
97
|
+
item = item_metadata.stac_item
|
|
92
98
|
try:
|
|
93
|
-
#
|
|
94
|
-
|
|
95
|
-
|
|
99
|
+
# Resolve destination collection using metadata from STAC item
|
|
100
|
+
target_collection = resolve_collection(
|
|
101
|
+
item_metadata,
|
|
102
|
+
generated_product_to_collection_identifier,
|
|
96
103
|
)
|
|
97
|
-
# Resolve destination collection
|
|
98
|
-
target_collection = resolve_collection(product_type, target_collections)
|
|
99
104
|
# TEMPFIX: Items generated by DPR with [] as geometry
|
|
100
105
|
# Also, remove some fields that s1 l0 processor is not compliant with
|
|
101
106
|
# regarding the catalog ingestion
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
107
|
+
# TEMPFIX: Items generated by DPR may contain empty geometry list
|
|
108
|
+
if not item.geometry:
|
|
109
|
+
item.geometry = None
|
|
110
|
+
# TEMPFIX: Items generated by DPR may contain empty bbox list
|
|
111
|
+
if not item.bbox:
|
|
112
|
+
item.bbox = None
|
|
113
|
+
instruments = item.properties.get("instruments")
|
|
114
|
+
if instruments is not None and not isinstance(instruments, list):
|
|
115
|
+
item.properties["instruments"] = [instruments]
|
|
110
116
|
# TEMPFIX END
|
|
111
117
|
logger.info(
|
|
112
118
|
"Writing product %s to %s. This may take a while...",
|
|
113
|
-
item.id
|
|
119
|
+
item.id,
|
|
114
120
|
target_collection,
|
|
115
121
|
)
|
|
116
122
|
# Publish item to catalog
|
|
@@ -121,7 +127,7 @@ async def publish(
|
|
|
121
127
|
response = catalog_client.add_item(target_collection, item, timeout=21600)
|
|
122
128
|
logger.info(
|
|
123
129
|
"Writing product %s to %s done. Response %s with message: %s",
|
|
124
|
-
item.id
|
|
130
|
+
item.id,
|
|
125
131
|
target_collection,
|
|
126
132
|
response.status_code,
|
|
127
133
|
response.text,
|
|
@@ -129,9 +135,8 @@ async def publish(
|
|
|
129
135
|
|
|
130
136
|
except Exception as e:
|
|
131
137
|
# Re-raise with full item context for easier debugging
|
|
132
|
-
item = item.to_dict() if hasattr(item, "to_dict") else item
|
|
133
138
|
raise RuntimeError(
|
|
134
|
-
f"Exception while publishing item: {json.dumps(item, indent=2)}",
|
|
139
|
+
f"Exception while publishing item: {json.dumps(item.to_dict(), indent=2)}",
|
|
135
140
|
) from e
|
|
136
141
|
|
|
137
142
|
# list collections for logging
|
|
@@ -163,51 +168,61 @@ async def get_item(
|
|
|
163
168
|
return catalog_client.get_item(target_collection, item)
|
|
164
169
|
|
|
165
170
|
|
|
166
|
-
def resolve_collection(
|
|
167
|
-
|
|
171
|
+
def resolve_collection(
|
|
172
|
+
item_metadata: DprProcessedItemMetadata,
|
|
173
|
+
generated_product_to_collection_identifier: list[FlowGeneratedProduct],
|
|
174
|
+
) -> str:
|
|
175
|
+
"""Resolve the target collection for a given product metadata.
|
|
168
176
|
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
Args:
|
|
173
|
-
product_type: STAC product type.
|
|
174
|
-
target_collections: Dict or list of dicts with target collections.
|
|
177
|
+
The matching logic uses both output_product_id and product_type from the
|
|
178
|
+
DprProcessedItemMetadata to find the corresponding FlowGeneratedProduct.
|
|
179
|
+
Wildcards ('*') are supported for product_type in FlowGeneratedProduct.
|
|
175
180
|
|
|
176
181
|
Returns:
|
|
177
|
-
The resolved target collection.
|
|
182
|
+
The resolved target collection name.
|
|
178
183
|
|
|
179
184
|
Raises:
|
|
180
|
-
ValueError: If the product
|
|
185
|
+
ValueError: If the product cannot be resolved to any collection.
|
|
181
186
|
"""
|
|
182
|
-
collections = {}
|
|
183
187
|
logger = get_run_logger()
|
|
184
188
|
logger.info(
|
|
185
|
-
f"Resolving target collection for
|
|
186
|
-
f"with
|
|
189
|
+
f"Resolving target collection for item metadata: {item_metadata}"
|
|
190
|
+
f" with generated_product_to_collection_identifier: {generated_product_to_collection_identifier}",
|
|
187
191
|
)
|
|
188
192
|
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
#
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
193
|
+
# We prioritize matches in the following order:
|
|
194
|
+
# 1. Exact Name, Exact Type
|
|
195
|
+
# 2. Exact Name, Wildcard Type (least specific)
|
|
196
|
+
# wildcards for name are NOT supported, they don't make sense in this context as tasktable don't support them
|
|
197
|
+
|
|
198
|
+
target_collection = None
|
|
199
|
+
for gen_prod in generated_product_to_collection_identifier:
|
|
200
|
+
if gen_prod.name != item_metadata.output_product_id:
|
|
201
|
+
continue
|
|
202
|
+
# 1. Exact Name, Exact Type
|
|
203
|
+
if item_metadata.product_type == gen_prod.product_type:
|
|
204
|
+
target_collection = gen_prod.collection_name or gen_prod.product_type
|
|
205
|
+
logger.info(f"Exact match (Name & Type) found: {gen_prod}, resolved collection: {target_collection}")
|
|
206
|
+
break
|
|
207
|
+
# 2. Exact Name, Wildcard Type
|
|
208
|
+
if gen_prod.product_type == "*" and target_collection is None:
|
|
209
|
+
target_collection = gen_prod.collection_name or gen_prod.product_type
|
|
210
|
+
# This protection is also set in build_output_products from payload_generator.py,
|
|
211
|
+
# but it's good to have it here as well for safety or a future change in logic.
|
|
212
|
+
if target_collection == "*":
|
|
213
|
+
raise RuntimeError(
|
|
214
|
+
f"The product type in generated_product_to_collection_identifier cannot be '*' "
|
|
215
|
+
f"if the collection name is not specified for product '{gen_prod.name}'",
|
|
216
|
+
)
|
|
217
|
+
logger.info(
|
|
218
|
+
f"Match with Exact Name & Wildcard Type: {gen_prod}, resolved collection: {target_collection}",
|
|
219
|
+
)
|
|
220
|
+
|
|
210
221
|
if not target_collection:
|
|
211
|
-
raise ValueError(
|
|
222
|
+
raise ValueError(
|
|
223
|
+
f"No match for {item_metadata.output_product_id} in "
|
|
224
|
+
f"{generated_product_to_collection_identifier}. "
|
|
225
|
+
f"Could not find a collection to publish the stac_item from: {item_metadata}",
|
|
226
|
+
)
|
|
212
227
|
|
|
213
228
|
return target_collection
|