rs-client-libraries 1.0a7__tar.gz → 1.0a8.post16.dev0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/PKG-INFO +6 -7
  2. {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/pyproject.toml +7 -8
  3. {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/__init__.py +1 -1
  4. {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/ogcapi/ogcapi_client.py +15 -2
  5. {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/rs_client.py +0 -11
  6. {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/stac/stac_base.py +2 -2
  7. {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_common/init_opentelemetry.py +43 -29
  8. {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/cadip_flow.py +51 -0
  9. {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/catalog_flow.py +73 -58
  10. {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/dpr_flow.py +78 -54
  11. {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/flow_utils.py +153 -88
  12. rs_client_libraries-1.0a8.post16.dev0/rs_workflows/on_demand/common/__init__.py +18 -0
  13. rs_client_libraries-1.0a8.post16.dev0/rs_workflows/on_demand/common/l0.py +107 -0
  14. rs_client_libraries-1.0a8.post16.dev0/rs_workflows/on_demand/common/l0_last_steps.py +102 -0
  15. rs_client_libraries-1.0a7/rs_workflows/on_demand/stage_last_sessions.py → rs_client_libraries-1.0a8.post16.dev0/rs_workflows/on_demand/common/staging.py +77 -112
  16. rs_client_libraries-1.0a8.post16.dev0/rs_workflows/on_demand/common/types.py +167 -0
  17. rs_client_libraries-1.0a8.post16.dev0/rs_workflows/on_demand/sentinel1/__init__.py +18 -0
  18. rs_client_libraries-1.0a8.post16.dev0/rs_workflows/on_demand/sentinel1/s1_l0.py +53 -0
  19. rs_client_libraries-1.0a8.post16.dev0/rs_workflows/on_demand/sentinel3/__init__.py +18 -0
  20. rs_client_libraries-1.0a8.post16.dev0/rs_workflows/on_demand/sentinel3/s3_l0.py +53 -0
  21. {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/on_demand_processing.py +70 -113
  22. {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/payload_generator.py +42 -14
  23. {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/payload_template.py +4 -4
  24. rs_client_libraries-1.0a8.post16.dev0/rs_workflows/prip_flow.py +131 -0
  25. {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/record_performance.py +5 -4
  26. rs_client_libraries-1.0a8.post16.dev0/rs_workflows/utils/cadip.py +144 -0
  27. rs_client_libraries-1.0a8.post16.dev0/rs_workflows/utils/catalog.py +90 -0
  28. rs_client_libraries-1.0a8.post16.dev0/rs_workflows/utils/dask.py +76 -0
  29. rs_client_libraries-1.0a8.post16.dev0/rs_workflows/utils/dpr.py +100 -0
  30. rs_client_libraries-1.0a7/rs_client/stac/edrs_client.py +0 -70
  31. rs_client_libraries-1.0a7/rs_workflows/prip_flow.py +0 -69
  32. {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/LICENSE +0 -0
  33. {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/README.md +0 -0
  34. {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/ogcapi/__init__.py +0 -0
  35. {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/ogcapi/dpr_client.py +0 -0
  36. {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/ogcapi/staging_client.py +0 -0
  37. {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/osam_client.py +0 -0
  38. {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/stac/__init__.py +0 -0
  39. {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/stac/auxip_client.py +0 -0
  40. {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/stac/cadip_client.py +0 -0
  41. {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/stac/catalog_client.py +0 -0
  42. {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/stac/prip_client.py +0 -0
  43. {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_common/__init__.py +0 -0
  44. {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_common/config.py +0 -0
  45. {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_common/logging.py +0 -0
  46. {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_common/prefect_utils.py +0 -0
  47. {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_common/utils.py +0 -0
  48. {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/__init__.py +0 -0
  49. {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/auxip_flow.py +0 -0
  50. {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/init_pi_db_flow.py +0 -0
  51. {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/on_demand/__init__.py +0 -0
  52. {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/operation/__init__.py +0 -0
  53. {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/operation/init_quota_monitoring_flow.py +0 -0
  54. {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/operation/osam_flows.py +0 -0
  55. {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/operation/quota_monitoring_db_models.py +0 -0
  56. {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/operation/quota_monitoring_flow.py +0 -0
  57. {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/payload_builder.py +0 -0
  58. {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/pi_db_models.py +0 -0
  59. {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/staging_flow.py +0 -0
  60. {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/storage_configuration.py +0 -0
  61. {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/utils/__init__.py +0 -0
  62. {rs_client_libraries-1.0a7 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/utils/artifact_verbose.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: rs-client-libraries
3
- Version: 1.0a7
3
+ Version: 1.0a8.post16.dev0
4
4
  Summary: Copernicus Sentinel rs-client library and processing workflows.
5
5
  License: Apache-2.0
6
6
  License-File: LICENSE
@@ -23,13 +23,12 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
23
23
  Requires-Dist: SQLAlchemy (>=2.0.37,<3.0.0)
24
24
  Requires-Dist: cachetools (>=5.5.2)
25
25
  Requires-Dist: click (==8.3.1)
26
+ Requires-Dist: dask-gateway (==2025.4.0)
26
27
  Requires-Dist: openapi-core (==0.22.0)
27
- Requires-Dist: opentelemetry-distro (>=0.60b1,<0.61)
28
- Requires-Dist: opentelemetry-exporter-otlp (>=1.39.1,<2.0.0)
29
- Requires-Dist: opentelemetry-instrumentation-aws-lambda (>=0.60b1,<0.61)
30
- Requires-Dist: opentelemetry-test-utils (>=0.60b1,<0.61)
31
- Requires-Dist: prefect-aws (>=0.7.4)
32
- Requires-Dist: prefect[aws] (==3.6.12)
28
+ Requires-Dist: opentelemetry-distro (>=0.61b0,<0.62)
29
+ Requires-Dist: opentelemetry-exporter-otlp (>=1.40.0,<2.0.0)
30
+ Requires-Dist: prefect-aws (>=0.7.5)
31
+ Requires-Dist: prefect[aws] (==3.6.20)
33
32
  Requires-Dist: protobuf (>=5.29.5)
34
33
  Requires-Dist: psycopg2-binary (>=2.9.10)
35
34
  Requires-Dist: pystac (>=1.13.0,<2.0.0)
@@ -18,7 +18,7 @@ dynamic = ["version"] # see: https://github.com/mtkennerly/poetry-dynamic-versio
18
18
 
19
19
  [tool.poetry]
20
20
  name = "rs-client-libraries"
21
- version = "1.0a7"
21
+ version = "1.0a8.post16.dev0"
22
22
  description = "Copernicus Sentinel rs-client library and processing workflows."
23
23
  license = "Apache-2.0"
24
24
  authors = ["ESA under contract with Airbus and CS <ops_coprs@airbus.com>"]
@@ -51,10 +51,8 @@ include = [
51
51
 
52
52
  [tool.poetry.dependencies]
53
53
  python = ">=3.11,<3.15"
54
- opentelemetry-distro = "^0.60b1" # then run 'poetry run opentelemetry-bootstrap -a install'
55
- opentelemetry-instrumentation-aws-lambda = "^0.60b1"
56
- opentelemetry-test-utils = "^0.60b1"
57
- opentelemetry-exporter-otlp = "^1.39.1"
54
+ opentelemetry-distro = "^0.61b0" # then run 'poetry run opentelemetry-bootstrap -a install'
55
+ opentelemetry-exporter-otlp = "^1.40.0"
58
56
  protobuf = ">=5.29.5"
59
57
  cachetools = ">=5.5.2"
60
58
  pystac = "^1.13.0"
@@ -63,11 +61,12 @@ python-dateutil = "^2.9.0.post0"
63
61
  stac-pydantic = "^3.4.0"
64
62
  openapi-core = "0.22.0"
65
63
  click = "8.3.1"
66
- prefect = { version = "3.6.12", extras = ["aws"] }
67
- prefect-aws = ">=0.7.4"
64
+ prefect = { version = "3.6.20", extras = ["aws"] }
65
+ prefect-aws = ">=0.7.5"
68
66
  requests = "^2.32.4"
69
67
  urllib3 = ">=2.6.3"
70
68
  SQLAlchemy = "^2.0.37"
69
+ dask-gateway = "2025.4.0"
71
70
  uv = ">=0.9.7"
72
71
  psycopg2-binary = ">=2.9.10"
73
72
  werkzeug = ">=3.1.6"
@@ -82,7 +81,7 @@ pytest-cov = "^7.0.0"
82
81
  pytest-docker = "^3.2.3"
83
82
  pytest-error-for-skips = "^2.0.2"
84
83
  pytest-mock = "^3.15.1"
85
- black = "==25.1.0"
84
+ black = ">=26.3.1"
86
85
  mypy = "^1.16.0"
87
86
  pre-commit = "^4.1.0"
88
87
  bandit = "^1.8.3"
@@ -15,4 +15,4 @@
15
15
  """rsclient main module."""
16
16
 
17
17
  # Set automatically by running `poetry dynamic-versioning`
18
- __version__ = "1.0a7"
18
+ __version__ = "1.0a8.post16.dev0"
@@ -32,6 +32,8 @@ from requests.models import PreparedRequest
32
32
  from rs_client.rs_client import TIMEOUT, RsClient
33
33
  from rs_common.utils import get_href_service
34
34
 
35
+ MAX_RETRIES_NUMBER_FOR_GETTING_JOB_STATUS = 20
36
+
35
37
 
36
38
  class OgcValidationException(Exception):
37
39
  """
@@ -279,9 +281,20 @@ class OgcApiClient(RsClient):
279
281
  previous_status: dict[str, str] = {}
280
282
  # We use previous_status to remove log when the status remains unchanged.
281
283
  # This way we reduce the number of useless information
282
-
284
+ retries_number = MAX_RETRIES_NUMBER_FOR_GETTING_JOB_STATUS
283
285
  while True:
284
- job_status = self.get_job_info(job_identifier)
286
+ try:
287
+ job_status = self.get_job_info(job_identifier)
288
+ except TimeoutError:
289
+ retries_number -= 1
290
+ if retries_number <= 0:
291
+ raise
292
+ time.sleep(2)
293
+ logger.warning(
294
+ f"Timeout while waiting for {job_name} job {job_identifier!r} status. "
295
+ f"Retries left: {retries_number}",
296
+ )
297
+ continue
285
298
  to_be_logged = previous_status != job_status
286
299
  previous_status = job_status
287
300
 
@@ -43,7 +43,6 @@ class RsClient: # pylint: disable=too-many-instance-attributes
43
43
  - rs-server-cadip
44
44
  - rs-server-auxip
45
45
  - rs-server-prip
46
- - rs-server-edrs
47
46
  - rs-server-catalog
48
47
 
49
48
  This class provides methods to authenticate and interact with RS-Server,
@@ -262,16 +261,6 @@ class RsClient: # pylint: disable=too-many-instance-attributes
262
261
 
263
262
  return CadipClient(self.rs_server_href, self.rs_server_api_key, self.logger, **kwargs)
264
263
 
265
- def get_edrs_client(self, **kwargs) -> "EdrsClient": # type: ignore # noqa: F821
266
- """
267
- Return an instance of the child class EdrsClient, with the same attributes as this "self" instance.
268
- """
269
- from rs_client.stac.edrs_client import ( # pylint: disable=import-outside-toplevel,cyclic-import
270
- EdrsClient,
271
- )
272
-
273
- return EdrsClient(self.rs_server_href, self.rs_server_api_key, self.logger, **kwargs)
274
-
275
264
  def get_catalog_client(self, **kwargs) -> "CatalogClient": # type: ignore # noqa: F821
276
265
  """
277
266
  Return an instance of the child class CatalogClient, with the same attributes as this "self" instance.
@@ -252,8 +252,8 @@ class StacBase(RsClient):
252
252
  try:
253
253
  return collection.get_items(*items_ids)
254
254
 
255
- # Avoid pystac-client fallback through /search (EDRS has no /search); fetch items one by one.
256
- # collection.get_items(*ids) internally triggers a search request, which fails on EDRS.
255
+ # Avoid pystac-client fallback through /search ; fetch items one by one.
256
+ # collection.get_items(*ids) internally triggers a search request.
257
257
  except (APIError, requests.HTTPError, STACError, ValueError, TypeError) as exc:
258
258
  self.logger.debug("Direct retrieval failed, fallback to per-item: %s", exc)
259
259
 
@@ -23,10 +23,10 @@ from collections.abc import Iterator
23
23
  from threading import Lock
24
24
 
25
25
  import opentelemetry.instrumentation
26
+ import requests
26
27
  from opentelemetry import trace
27
28
  from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
28
29
  from opentelemetry.instrumentation.asyncio import AsyncioInstrumentor
29
- from opentelemetry.instrumentation.aws_lambda import AwsLambdaInstrumentor
30
30
  from opentelemetry.instrumentation.instrumentor import BaseInstrumentor # type: ignore
31
31
  from opentelemetry.instrumentation.requests import RequestsInstrumentor
32
32
  from opentelemetry.sdk.resources import Resource
@@ -81,43 +81,51 @@ def parse_data(data) -> str:
81
81
  except Exception: # pylint: disable=broad-exception-caught # nosec
82
82
  pass
83
83
 
84
- # If we have a dict, try to format it as json
84
+ # If we have a dict
85
85
  if isinstance(data, dict):
86
+
87
+ # Decode bytes
88
+ data = {
89
+ key.decode("utf-8") if isinstance(key, bytes) else key: (
90
+ value.decode("utf-8") if isinstance(value, bytes) else value
91
+ )
92
+ for key, value in data.items()
93
+ }
94
+
95
+ # Convert to strings
96
+ data = {str(key): str(value) for key, value in data.items()}
97
+
98
+ # Apply json formatting
86
99
  data = json.dumps(data, indent=2)
87
100
 
88
101
  return data or ""
89
102
 
90
103
 
91
- def request_hook(span, request):
104
+ def requests_hook(span: Span, request: requests.PreparedRequest, response: requests.Response | None = None):
92
105
  """
93
- HTTP requests intrumentation
106
+ Callback function invoked by RequestsInstrumentor. It implements the hooks:
107
+
108
+ - request_hook: invoked right after a span is created.
109
+ - response_hook: invoked right before the span has finished processing a response.
110
+
111
+ See: https://opentelemetry-python-contrib.readthedocs.io/en/latest/instrumentation/requests/requests.html
94
112
  """
95
- if not span:
113
+ if not (span and span.is_recording()):
96
114
  return
97
115
 
98
- # Copy the http.url attribute into _url so it appears at the
99
- # top in the grafana UI, it's more readable
100
- span.set_attribute("_url", span.attributes.get("http.url"))
116
+ # Copy this attribute by adding a '_' prefix to the name,
117
+ # so it appears at the top in the grafana UI, it's more readable
118
+ span.set_attribute("_url", span.attributes.get("http.url")) # type: ignore
101
119
 
102
120
  if trace_headers():
103
121
  span.set_attribute("http.request.headers", parse_data(request.headers))
122
+ if response:
123
+ span.set_attribute("http.response.headers", parse_data(response.headers))
104
124
 
105
125
  if trace_body():
106
126
  span.set_attribute("http.request.body", parse_data(request.body))
107
-
108
-
109
- def response_hook(span, request, response): # pylint: disable=W0613
110
- """
111
- HTTP responses intrumentation
112
- """
113
- if not span:
114
- return
115
-
116
- if trace_headers():
117
- span.set_attribute("http.response.headers", parse_data(response.headers))
118
-
119
- if trace_body():
120
- span.set_attribute("http.response.content", parse_data(response.content))
127
+ if response:
128
+ span.set_attribute("http.response.content", parse_data(response.content))
121
129
 
122
130
 
123
131
  def init_traces(service_name: str, logger=None):
@@ -128,6 +136,7 @@ def init_traces(service_name: str, logger=None):
128
136
  service_name (str): service name
129
137
  logger: non-default logger to user
130
138
  """
139
+
131
140
  # See: https://github.com/softwarebloat/python-tracing-demo/tree/main
132
141
 
133
142
  # No concurrent threads
@@ -191,24 +200,29 @@ def init_traces(service_name: str, logger=None):
191
200
  classes.add(_class)
192
201
 
193
202
  # Don't instrument these classes, they have errors, maybe we should see why
194
- if _class in [AsyncioInstrumentor, AwsLambdaInstrumentor, BaseInstrumentor]:
203
+ if _class in [AsyncioInstrumentor, BaseInstrumentor]:
195
204
  continue
196
205
 
197
206
  # If the "instrument" method exists, call it
198
207
  _instrument = getattr(_class, "instrument", None)
199
208
  if callable(_instrument):
200
-
201
209
  _class_instance = _class()
210
+ if _class_instance.is_instrumented_by_opentelemetry:
211
+ continue
212
+ # name = f"{module_str}.{_class.__name__}".removeprefix(prefix)
213
+ # logger.debug(f"OpenTelemetry instrumentation of {name!r}")
214
+
215
+ # Handle specific hooks
202
216
  if _class == RequestsInstrumentor and (trace_headers() or trace_body()):
203
217
  _class_instance.instrument(
204
218
  tracer_provider=otel_tracer,
205
- request_hook=request_hook,
206
- response_hook=response_hook,
219
+ request_hook=requests_hook,
220
+ response_hook=requests_hook,
207
221
  )
208
- elif not _class_instance.is_instrumented_by_opentelemetry:
222
+
223
+ # General case (no hooks)
224
+ else:
209
225
  _class_instance.instrument(tracer_provider=otel_tracer)
210
- # name = f"{module_str}.{_class.__name__}".removeprefix(prefix)
211
- # logger.debug(f"OpenTelemetry instrumentation of {name!r}")
212
226
 
213
227
 
214
228
  @_agnosticcontextmanager
@@ -19,6 +19,7 @@ from pystac import ItemCollection
19
19
 
20
20
  from rs_client.stac.cadip_client import CadipClient
21
21
  from rs_workflows.flow_utils import FlowEnv, FlowEnvArgs
22
+ from rs_workflows.staging_flow import staging_task
22
23
 
23
24
 
24
25
  @flow(name="Cadip search")
@@ -58,6 +59,56 @@ async def search(
58
59
  return found
59
60
 
60
61
 
62
+ @flow(name="On-demand Cadip staging")
63
+ async def on_demand_cadip_staging(
64
+ env: FlowEnvArgs,
65
+ cadip_collection_identifier: str,
66
+ session_identifier: str,
67
+ catalog_collection_identifier: str,
68
+ staging_retries: int = 3,
69
+ staging_retry_delay: int = 60,
70
+ ):
71
+ """
72
+ Flow to retrieve a session, stage it and add the STAC item into the catalog.
73
+
74
+ Args:
75
+ env: Prefect flow environment
76
+ cadip_collection_identifier: CADIP collection identifier that contains the mission and station
77
+ (e.g. s1_ins for Sentinel-1 sessions from the Inuvik station)
78
+ session_identifier: Session identifier
79
+ catalog_collection_identifier: Catalog collection identifier where CADIP sessions and AUX data are staged
80
+ """
81
+
82
+ # Init flow environment and opentelemetry span
83
+ flow_env = FlowEnv(env)
84
+ with flow_env.start_span(__name__, "on-demand-cadip-staging"):
85
+
86
+ # Search Cadip sessions
87
+ cadip_items = search_task.with_options(
88
+ retries=3,
89
+ retry_delay_seconds=60,
90
+ ).submit(
91
+ flow_env.serialize(),
92
+ cadip_collection_identifier,
93
+ session_identifier,
94
+ error_if_empty=True,
95
+ )
96
+
97
+ # Stage Cadip items.
98
+ staged = staging_task.with_options(
99
+ retries=staging_retries,
100
+ retry_delay_seconds=staging_retry_delay,
101
+ ).submit(
102
+ flow_env.serialize(),
103
+ cadip_items,
104
+ catalog_collection_identifier,
105
+ )
106
+
107
+ # Wait for last task to end.
108
+ # NOTE: use .result() and not .wait() to unwrap and propagate exceptions, if any.
109
+ staged.result() # type: ignore[unused-coroutine]
110
+
111
+
61
112
  ###########################
62
113
  # Call the flows as tasks #
63
114
  ###########################
@@ -17,10 +17,15 @@
17
17
  import json
18
18
 
19
19
  from prefect import flow, get_run_logger, task
20
- from pystac import Item, ItemCollection
20
+ from pystac import ItemCollection
21
21
 
22
22
  from rs_client.stac.catalog_client import CatalogClient
23
- from rs_workflows.flow_utils import FlowEnv, FlowEnvArgs
23
+ from rs_workflows.flow_utils import (
24
+ DprProcessedItemMetadata,
25
+ FlowEnv,
26
+ FlowEnvArgs,
27
+ FlowGeneratedProduct,
28
+ )
24
29
 
25
30
  #################
26
31
  # Catalog flows #
@@ -71,8 +76,8 @@ async def catalog_search(
71
76
  @task(name="Publish to catalog")
72
77
  async def publish(
73
78
  env: FlowEnvArgs,
74
- target_collections,
75
- items,
79
+ generated_product_to_collection_identifier: list[FlowGeneratedProduct],
80
+ items_metadata: list[DprProcessedItemMetadata],
76
81
  ):
77
82
  """
78
83
  Publish items to the catalog
@@ -80,7 +85,7 @@ async def publish(
80
85
  Args:
81
86
  env: Prefect flow environment
82
87
  collection: Catalog collection identifier where the items are published
83
- items: Items to publish, as STAC dicts or pystac.Items
88
+ items_metadata: List of DprProcessedItemMetadata containing items to publish
84
89
  """
85
90
  logger = get_run_logger()
86
91
  flow_env = FlowEnv(env)
@@ -88,29 +93,30 @@ async def publish(
88
93
  catalog_client: CatalogClient = flow_env.rs_client.get_catalog_client()
89
94
 
90
95
  with flow_env.start_span(__name__, "publish-to-catalog"):
91
- for item in items:
96
+ for item_metadata in items_metadata:
97
+ item = item_metadata.stac_item
92
98
  try:
93
- # Extract product type from STAC item
94
- product_type = (
95
- item.properties["product:type"] if isinstance(item, Item) else item["properties"]["product:type"]
99
+ # Resolve destination collection using metadata from STAC item
100
+ target_collection = resolve_collection(
101
+ item_metadata,
102
+ generated_product_to_collection_identifier,
96
103
  )
97
- # Resolve destination collection
98
- target_collection = resolve_collection(product_type, target_collections)
99
104
  # TEMPFIX: Items generated by DPR with [] as geometry
100
105
  # Also, remove some fields that s1 l0 processor is not compliant with
101
106
  # regarding the catalog ingestion
102
- if isinstance(item, Item):
103
- # TEMPFIX: Items generated by DPR may contain empty geometry lists
104
- if not item.geometry:
105
- item.geometry = None
106
-
107
- instruments = item.properties.get("instruments")
108
- if instruments is not None and not isinstance(instruments, list):
109
- item.properties["instruments"] = [instruments]
107
+ # TEMPFIX: Items generated by DPR may contain empty geometry list
108
+ if not item.geometry:
109
+ item.geometry = None
110
+ # TEMPFIX: Items generated by DPR may contain empty bbox list
111
+ if not item.bbox:
112
+ item.bbox = None
113
+ instruments = item.properties.get("instruments")
114
+ if instruments is not None and not isinstance(instruments, list):
115
+ item.properties["instruments"] = [instruments]
110
116
  # TEMPFIX END
111
117
  logger.info(
112
118
  "Writing product %s to %s. This may take a while...",
113
- item.id if isinstance(item, Item) else item["id"],
119
+ item.id,
114
120
  target_collection,
115
121
  )
116
122
  # Publish item to catalog
@@ -121,7 +127,7 @@ async def publish(
121
127
  response = catalog_client.add_item(target_collection, item, timeout=21600)
122
128
  logger.info(
123
129
  "Writing product %s to %s done. Response %s with message: %s",
124
- item.id if isinstance(item, Item) else item["id"],
130
+ item.id,
125
131
  target_collection,
126
132
  response.status_code,
127
133
  response.text,
@@ -129,9 +135,8 @@ async def publish(
129
135
 
130
136
  except Exception as e:
131
137
  # Re-raise with full item context for easier debugging
132
- item = item.to_dict() if hasattr(item, "to_dict") else item
133
138
  raise RuntimeError(
134
- f"Exception while publishing item: {json.dumps(item, indent=2)}",
139
+ f"Exception while publishing item: {json.dumps(item.to_dict(), indent=2)}",
135
140
  ) from e
136
141
 
137
142
  # list collections for logging
@@ -163,51 +168,61 @@ async def get_item(
163
168
  return catalog_client.get_item(target_collection, item)
164
169
 
165
170
 
166
- def resolve_collection(product_type: str, target_collections) -> str:
167
- """Resolve the target collection for a given product type.
171
+ def resolve_collection(
172
+ item_metadata: DprProcessedItemMetadata,
173
+ generated_product_to_collection_identifier: list[FlowGeneratedProduct],
174
+ ) -> str:
175
+ """Resolve the target collection for a given product metadata.
168
176
 
169
- Supports input values as strings or tuples (product_type, collection),
170
- and input as dict or list of dicts.
171
-
172
- Args:
173
- product_type: STAC product type.
174
- target_collections: Dict or list of dicts with target collections.
177
+ The matching logic uses both output_product_id and product_type from the
178
+ DprProcessedItemMetadata to find the corresponding FlowGeneratedProduct.
179
+ Wildcards ('*') are supported for product_type in FlowGeneratedProduct.
175
180
 
176
181
  Returns:
177
- The resolved target collection.
182
+ The resolved target collection name.
178
183
 
179
184
  Raises:
180
- ValueError: If the product type cannot be resolved.
185
+ ValueError: If the product cannot be resolved to any collection.
181
186
  """
182
- collections = {}
183
187
  logger = get_run_logger()
184
188
  logger.info(
185
- f"Resolving target collection for product type '{product_type}' "
186
- f"with target_collections: {target_collections}",
189
+ f"Resolving target collection for item metadata: {item_metadata}"
190
+ f" with generated_product_to_collection_identifier: {generated_product_to_collection_identifier}",
187
191
  )
188
192
 
189
- if isinstance(target_collections, dict):
190
- # normalize dict input
191
- for value in target_collections.values():
192
- if isinstance(value, tuple):
193
- key, collection = value
194
- collections[key] = collection
195
- else: # string case
196
- collections[value] = value
197
- else:
198
- # normalize list of dicts input
199
- for d in target_collections:
200
- for value in d.values():
201
- if isinstance(value, tuple):
202
- key, collection = value
203
- collections[key] = collection
204
- else: # string case
205
- collections[value] = value
206
-
207
- # lookup product_type
208
- logger.info(f"Collections found: {collections}")
209
- target_collection = collections.get(product_type, collections.get("*"))
193
+ # We prioritize matches in the following order:
194
+ # 1. Exact Name, Exact Type
195
+ # 2. Exact Name, Wildcard Type (least specific)
196
+ # wildcards for name are NOT supported, they don't make sense in this context as tasktable don't support them
197
+
198
+ target_collection = None
199
+ for gen_prod in generated_product_to_collection_identifier:
200
+ if gen_prod.name != item_metadata.output_product_id:
201
+ continue
202
+ # 1. Exact Name, Exact Type
203
+ if item_metadata.product_type == gen_prod.product_type:
204
+ target_collection = gen_prod.collection_name or gen_prod.product_type
205
+ logger.info(f"Exact match (Name & Type) found: {gen_prod}, resolved collection: {target_collection}")
206
+ break
207
+ # 2. Exact Name, Wildcard Type
208
+ if gen_prod.product_type == "*" and target_collection is None:
209
+ target_collection = gen_prod.collection_name or gen_prod.product_type
210
+ # This protection is also set in build_output_products from payload_generator.py,
211
+ # but it's good to have it here as well for safety or a future change in logic.
212
+ if target_collection == "*":
213
+ raise RuntimeError(
214
+ f"The product type in generated_product_to_collection_identifier cannot be '*' "
215
+ f"if the collection name is not specified for product '{gen_prod.name}'",
216
+ )
217
+ logger.info(
218
+ f"Match with Exact Name & Wildcard Type: {gen_prod}, resolved collection: {target_collection}",
219
+ )
220
+
210
221
  if not target_collection:
211
- raise ValueError(f"Product type unknown: {product_type}")
222
+ raise ValueError(
223
+ f"No match for {item_metadata.output_product_id} in "
224
+ f"{generated_product_to_collection_identifier}. "
225
+ f"Could not find a collection to publish the stac_item from: {item_metadata}",
226
+ )
212
227
 
213
228
  return target_collection