rs-client-libraries 1.0a8__tar.gz → 1.0a8.post16.dev0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/PKG-INFO +1 -1
  2. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/pyproject.toml +1 -1
  3. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/__init__.py +1 -1
  4. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/cadip_flow.py +51 -0
  5. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/catalog_flow.py +62 -25
  6. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/dpr_flow.py +38 -24
  7. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/flow_utils.py +9 -0
  8. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/on_demand_processing.py +2 -112
  9. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/payload_generator.py +8 -2
  10. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/payload_template.py +4 -4
  11. rs_client_libraries-1.0a8.post16.dev0/rs_workflows/prip_flow.py +131 -0
  12. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/record_performance.py +4 -3
  13. rs_client_libraries-1.0a8/rs_workflows/prip_flow.py +0 -69
  14. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/LICENSE +0 -0
  15. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/README.md +0 -0
  16. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/ogcapi/__init__.py +0 -0
  17. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/ogcapi/dpr_client.py +0 -0
  18. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/ogcapi/ogcapi_client.py +0 -0
  19. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/ogcapi/staging_client.py +0 -0
  20. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/osam_client.py +0 -0
  21. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/rs_client.py +0 -0
  22. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/stac/__init__.py +0 -0
  23. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/stac/auxip_client.py +0 -0
  24. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/stac/cadip_client.py +0 -0
  25. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/stac/catalog_client.py +0 -0
  26. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/stac/prip_client.py +0 -0
  27. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/stac/stac_base.py +0 -0
  28. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_common/__init__.py +0 -0
  29. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_common/config.py +0 -0
  30. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_common/init_opentelemetry.py +0 -0
  31. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_common/logging.py +0 -0
  32. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_common/prefect_utils.py +0 -0
  33. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_common/utils.py +0 -0
  34. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/__init__.py +0 -0
  35. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/auxip_flow.py +0 -0
  36. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/init_pi_db_flow.py +0 -0
  37. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/on_demand/__init__.py +0 -0
  38. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/on_demand/common/__init__.py +0 -0
  39. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/on_demand/common/l0.py +0 -0
  40. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/on_demand/common/l0_last_steps.py +0 -0
  41. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/on_demand/common/staging.py +0 -0
  42. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/on_demand/common/types.py +0 -0
  43. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/on_demand/sentinel1/__init__.py +0 -0
  44. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/on_demand/sentinel1/s1_l0.py +0 -0
  45. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/on_demand/sentinel3/__init__.py +0 -0
  46. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/on_demand/sentinel3/s3_l0.py +0 -0
  47. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/operation/__init__.py +0 -0
  48. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/operation/init_quota_monitoring_flow.py +0 -0
  49. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/operation/osam_flows.py +0 -0
  50. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/operation/quota_monitoring_db_models.py +0 -0
  51. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/operation/quota_monitoring_flow.py +0 -0
  52. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/payload_builder.py +0 -0
  53. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/pi_db_models.py +0 -0
  54. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/staging_flow.py +0 -0
  55. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/storage_configuration.py +0 -0
  56. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/utils/__init__.py +0 -0
  57. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/utils/artifact_verbose.py +0 -0
  58. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/utils/cadip.py +0 -0
  59. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/utils/catalog.py +0 -0
  60. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/utils/dask.py +0 -0
  61. {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/utils/dpr.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: rs-client-libraries
3
- Version: 1.0a8
3
+ Version: 1.0a8.post16.dev0
4
4
  Summary: Copernicus Sentinel rs-client library and processing workflows.
5
5
  License: Apache-2.0
6
6
  License-File: LICENSE
@@ -18,7 +18,7 @@ dynamic = ["version"] # see: https://github.com/mtkennerly/poetry-dynamic-versio
18
18
 
19
19
  [tool.poetry]
20
20
  name = "rs-client-libraries"
21
- version = "1.0a8"
21
+ version = "1.0a8.post16.dev0"
22
22
  description = "Copernicus Sentinel rs-client library and processing workflows."
23
23
  license = "Apache-2.0"
24
24
  authors = ["ESA under contract with Airbus and CS <ops_coprs@airbus.com>"]
@@ -15,4 +15,4 @@
15
15
  """rsclient main module."""
16
16
 
17
17
  # Set automatically by running `poetry dynamic-versioning`
18
- __version__ = "1.0a8"
18
+ __version__ = "1.0a8.post16.dev0"
@@ -19,6 +19,7 @@ from pystac import ItemCollection
19
19
 
20
20
  from rs_client.stac.cadip_client import CadipClient
21
21
  from rs_workflows.flow_utils import FlowEnv, FlowEnvArgs
22
+ from rs_workflows.staging_flow import staging_task
22
23
 
23
24
 
24
25
  @flow(name="Cadip search")
@@ -58,6 +59,56 @@ async def search(
58
59
  return found
59
60
 
60
61
 
62
+ @flow(name="On-demand Cadip staging")
63
+ async def on_demand_cadip_staging(
64
+ env: FlowEnvArgs,
65
+ cadip_collection_identifier: str,
66
+ session_identifier: str,
67
+ catalog_collection_identifier: str,
68
+ staging_retries: int = 3,
69
+ staging_retry_delay: int = 60,
70
+ ):
71
+ """
72
+ Flow to retrieve a session, stage it and add the STAC item into the catalog.
73
+
74
+ Args:
75
+ env: Prefect flow environment
76
+ cadip_collection_identifier: CADIP collection identifier that contains the mission and station
77
+ (e.g. s1_ins for Sentinel-1 sessions from the Inuvik station)
78
+ session_identifier: Session identifier
79
+ catalog_collection_identifier: Catalog collection identifier where CADIP sessions and AUX data are staged
80
+ """
81
+
82
+ # Init flow environment and opentelemetry span
83
+ flow_env = FlowEnv(env)
84
+ with flow_env.start_span(__name__, "on-demand-cadip-staging"):
85
+
86
+ # Search Cadip sessions
87
+ cadip_items = search_task.with_options(
88
+ retries=3,
89
+ retry_delay_seconds=60,
90
+ ).submit(
91
+ flow_env.serialize(),
92
+ cadip_collection_identifier,
93
+ session_identifier,
94
+ error_if_empty=True,
95
+ )
96
+
97
+ # Stage Cadip items.
98
+ staged = staging_task.with_options(
99
+ retries=staging_retries,
100
+ retry_delay_seconds=staging_retry_delay,
101
+ ).submit(
102
+ flow_env.serialize(),
103
+ cadip_items,
104
+ catalog_collection_identifier,
105
+ )
106
+
107
+ # Wait for last task to end.
108
+ # NOTE: use .result() and not .wait() to unwrap and propagate exceptions, if any.
109
+ staged.result() # type: ignore[unused-coroutine]
110
+
111
+
61
112
  ###########################
62
113
  # Call the flows as tasks #
63
114
  ###########################
@@ -17,10 +17,15 @@
17
17
  import json
18
18
 
19
19
  from prefect import flow, get_run_logger, task
20
- from pystac import Item, ItemCollection
20
+ from pystac import ItemCollection
21
21
 
22
22
  from rs_client.stac.catalog_client import CatalogClient
23
- from rs_workflows.flow_utils import FlowEnv, FlowEnvArgs, FlowGeneratedProduct
23
+ from rs_workflows.flow_utils import (
24
+ DprProcessedItemMetadata,
25
+ FlowEnv,
26
+ FlowEnvArgs,
27
+ FlowGeneratedProduct,
28
+ )
24
29
 
25
30
  #################
26
31
  # Catalog flows #
@@ -71,8 +76,8 @@ async def catalog_search(
71
76
  @task(name="Publish to catalog")
72
77
  async def publish(
73
78
  env: FlowEnvArgs,
74
- target_collections,
75
- items: list[Item],
79
+ generated_product_to_collection_identifier: list[FlowGeneratedProduct],
80
+ items_metadata: list[DprProcessedItemMetadata],
76
81
  ):
77
82
  """
78
83
  Publish items to the catalog
@@ -80,7 +85,7 @@ async def publish(
80
85
  Args:
81
86
  env: Prefect flow environment
82
87
  collection: Catalog collection identifier where the items are published
83
- items: Items to publish, as STAC dicts or pystac.Items
88
+ items_metadata: List of DprProcessedItemMetadata containing items to publish
84
89
  """
85
90
  logger = get_run_logger()
86
91
  flow_env = FlowEnv(env)
@@ -88,10 +93,14 @@ async def publish(
88
93
  catalog_client: CatalogClient = flow_env.rs_client.get_catalog_client()
89
94
 
90
95
  with flow_env.start_span(__name__, "publish-to-catalog"):
91
- for item in items:
96
+ for item_metadata in items_metadata:
97
+ item = item_metadata.stac_item
92
98
  try:
93
- # Resolve destination collection using product type from STAC item
94
- target_collection = resolve_collection(item.properties["product:type"], target_collections)
99
+ # Resolve destination collection using metadata from STAC item
100
+ target_collection = resolve_collection(
101
+ item_metadata,
102
+ generated_product_to_collection_identifier,
103
+ )
95
104
  # TEMPFIX: Items generated by DPR with [] as geometry
96
105
  # Also, remove some fields that s1 l0 processor is not compliant with
97
106
  # regarding the catalog ingestion
@@ -107,7 +116,7 @@ async def publish(
107
116
  # TEMPFIX END
108
117
  logger.info(
109
118
  "Writing product %s to %s. This may take a while...",
110
- item.id if isinstance(item, Item) else item["id"],
119
+ item.id,
111
120
  target_collection,
112
121
  )
113
122
  # Publish item to catalog
@@ -118,7 +127,7 @@ async def publish(
118
127
  response = catalog_client.add_item(target_collection, item, timeout=21600)
119
128
  logger.info(
120
129
  "Writing product %s to %s done. Response %s with message: %s",
121
- item.id if isinstance(item, Item) else item["id"],
130
+ item.id,
122
131
  target_collection,
123
132
  response.status_code,
124
133
  response.text,
@@ -159,33 +168,61 @@ async def get_item(
159
168
  return catalog_client.get_item(target_collection, item)
160
169
 
161
170
 
162
- def resolve_collection(product_type: str, target_collections: list[FlowGeneratedProduct]) -> str:
163
- """Resolve the target collection for a given product type.
171
+ def resolve_collection(
172
+ item_metadata: DprProcessedItemMetadata,
173
+ generated_product_to_collection_identifier: list[FlowGeneratedProduct],
174
+ ) -> str:
175
+ """Resolve the target collection for a given product metadata.
164
176
 
165
- Supports:
166
- - iterable of FlowGeneratedProduct (new)
177
+ The matching logic uses both output_product_id and product_type from the
178
+ DprProcessedItemMetadata to find the corresponding FlowGeneratedProduct.
179
+ Wildcards ('*') are supported for product_type in FlowGeneratedProduct.
167
180
 
168
181
  Returns:
169
182
  The resolved target collection name.
170
183
 
171
184
  Raises:
172
- ValueError: If the product type cannot be resolved.
185
+ ValueError: If the product cannot be resolved to any collection.
173
186
  """
174
187
  logger = get_run_logger()
175
188
  logger.info(
176
- f"Resolving target collection for product type '{product_type}' "
177
- f"with target_collections: {target_collections}",
189
+ f"Resolving target collection for item metadata: {item_metadata}"
190
+ f" with generated_product_to_collection_identifier: {generated_product_to_collection_identifier}",
178
191
  )
179
192
 
180
- collections = {}
181
-
182
- for item in target_collections:
183
- if (ptype := item.product_type) and (collection := item.collection_name):
184
- collections[ptype] = collection
185
-
186
- target_collection = collections.get(product_type) or collections.get("*")
193
+ # We prioritize matches in the following order:
194
+ # 1. Exact Name, Exact Type
195
+ # 2. Exact Name, Wildcard Type (least specific)
196
+ # wildcards for name are NOT supported, they don't make sense in this context as tasktable don't support them
197
+
198
+ target_collection = None
199
+ for gen_prod in generated_product_to_collection_identifier:
200
+ if gen_prod.name != item_metadata.output_product_id:
201
+ continue
202
+ # 1. Exact Name, Exact Type
203
+ if item_metadata.product_type == gen_prod.product_type:
204
+ target_collection = gen_prod.collection_name or gen_prod.product_type
205
+ logger.info(f"Exact match (Name & Type) found: {gen_prod}, resolved collection: {target_collection}")
206
+ break
207
+ # 2. Exact Name, Wildcard Type
208
+ if gen_prod.product_type == "*" and target_collection is None:
209
+ target_collection = gen_prod.collection_name or gen_prod.product_type
210
+ # This protection is also set in build_output_products from payload_generator.py,
211
+ # but it's good to have it here as well for safety or a future change in logic.
212
+ if target_collection == "*":
213
+ raise RuntimeError(
214
+ f"The product type in generated_product_to_collection_identifier cannot be '*' "
215
+ f"if the collection name is not specified for product '{gen_prod.name}'",
216
+ )
217
+ logger.info(
218
+ f"Match with Exact Name & Wildcard Type: {gen_prod}, resolved collection: {target_collection}",
219
+ )
187
220
 
188
221
  if not target_collection:
189
- raise ValueError(f"Product type unknown: {product_type}")
222
+ raise ValueError(
223
+ f"No match for {item_metadata.output_product_id} in "
224
+ f"{generated_product_to_collection_identifier}. "
225
+ f"Could not find a collection to publish the stac_item from: {item_metadata}",
226
+ )
190
227
 
191
228
  return target_collection
@@ -32,7 +32,7 @@ from pystac import Asset, Item
32
32
  from rs_client.ogcapi.dpr_client import ClusterInfo, DprClient, DprProcessor
33
33
  from rs_common import prefect_utils
34
34
  from rs_workflows import catalog_flow
35
- from rs_workflows.flow_utils import FlowEnv, FlowEnvArgs
35
+ from rs_workflows.flow_utils import DprProcessedItemMetadata, FlowEnv, FlowEnvArgs
36
36
  from rs_workflows.payload_template import PayloadSchema
37
37
  from rs_workflows.record_performance import record_performance_indicators
38
38
 
@@ -83,11 +83,11 @@ def extract_products_and_zattrs(files: list[str], base_path: str):
83
83
  if len(parts) != 2:
84
84
  continue
85
85
 
86
- product_name = parts[0]
86
+ product_dirname = parts[0]
87
87
 
88
88
  # 1: base_path/product/.zattrs
89
89
  if parts[1] == ".zattrs":
90
- dirs_and_attrs.append((product_name, file))
90
+ dirs_and_attrs.append((product_dirname, file))
91
91
 
92
92
  return dirs_and_attrs
93
93
 
@@ -248,7 +248,7 @@ def update_eopf_assets(
248
248
  input_products: list[dict],
249
249
  payload: PayloadSchema,
250
250
  dpr_processor: str,
251
- ) -> tuple[list[Item], list[str]]:
251
+ ) -> list[DprProcessedItemMetadata]:
252
252
  """Update EOPF assets by extracting metadata and creating STAC items.
253
253
 
254
254
  This Prefect task processes output products from a DPR (Data Processing Request)
@@ -266,15 +266,15 @@ def update_eopf_assets(
266
266
  env: Environment configuration object containing runtime settings.
267
267
  input_products: List of dictionaries representing input product metadata.
268
268
  payload: PayloadSchema object containing I/O configuration, including
269
- output product paths to scan for .zattrs files.
269
+ output_product paths to scan for .zattrs files.
270
270
  dpr_processor: str
271
271
  DPR processor name
272
272
 
273
273
  Returns:
274
- A tuple containing:
275
- - stac_items (list): List of STAC items created from EOPF metadata.
276
- - eopf_types (list): List of product types extracted from the .zattrs
277
- files (corresponds to stac_items by index).
274
+ A list of DprProcessedItemMetadata objects, each containing:
275
+ - stac_item (Item): STAC item created from EOPF metadata.
276
+ - product_type (str): Product type extracted from the .zattrs file.
277
+ - output_product_id (str): The ID of the original output product.
278
278
 
279
279
  Raises:
280
280
  RuntimeError: If any .zattrs file cannot be read or does not contain
@@ -288,7 +288,7 @@ def update_eopf_assets(
288
288
  logger = get_run_logger()
289
289
  logger.info("Starting EOPF asset update.")
290
290
  logger.info(f"Payload received: {payload}")
291
- logger.info("Input products: %s", input_products)
291
+ logger.info(f"Input products: {input_products}")
292
292
 
293
293
  if payload.io is None:
294
294
  raise RuntimeError("Payload I/O configuration is missing.")
@@ -296,15 +296,20 @@ def update_eopf_assets(
296
296
  zattrs_list = []
297
297
  for prod in payload.io.output_products:
298
298
  path = prod.path
299
- zattrs_list.extend(extract_products_and_zattrs(s3_list(path), path))
300
- logger.info(f"Product {prod.id} has been added to the list and will be published to the catalog.")
299
+ # Keep track of which output product ID each .zattrs belongs to
300
+ new_zattrs = extract_products_and_zattrs(s3_list(path), path)
301
+ for name, loc in new_zattrs:
302
+ zattrs_list.append((name, loc, prod.id))
303
+ logger.info(
304
+ f"Product {prod.id} has been added to the list and will be published to the catalog."
305
+ f" The path is {path}",
306
+ )
301
307
 
302
308
  # List & extract
303
- logger.info(f"Found {len(zattrs_list)} .zattrs files under path. The list: {zattrs_list}")
309
+ logger.info(f"Found {len(zattrs_list)} .zattrs files. The list: {zattrs_list}")
304
310
 
305
- stac_items = []
306
- product_types = []
307
- for product_name, zattrs_s3_location in zattrs_list:
311
+ items_metadata = []
312
+ for product_name, zattrs_s3_location, output_product_id in zattrs_list:
308
313
  logger.info(f"Product = {product_name} | zattrs = {zattrs_s3_location}")
309
314
  # Read metadata
310
315
  zattrs_data = read_zattrs_sync(zattrs_s3_location)
@@ -320,18 +325,24 @@ def update_eopf_assets(
320
325
 
321
326
  product_type = zattrs_data["stac_discovery"]["properties"].get("product:type", None)
322
327
  logger.info(f"Extracted EOPF product type: {product_type}")
323
- product_types.append(product_type)
324
328
 
325
329
  eopf_item = zattrs_data["stac_discovery"]
326
330
  logger.debug(f"EOPF discovery metadata extracted: {eopf_item}")
327
331
 
328
332
  # Build STAC items
329
- stac_items.append(
330
- create_stac_item(env, input_products, eopf_item, zattrs_s3_location, product_name, dpr_processor),
333
+ stac_item = create_stac_item(env, input_products, eopf_item, zattrs_s3_location, product_name, dpr_processor)
334
+
335
+ items_metadata.append(
336
+ DprProcessedItemMetadata(
337
+ stac_item=stac_item,
338
+ product_type=product_type,
339
+ output_product_id=output_product_id,
340
+ ),
331
341
  )
332
- logger.info(f"Added one stac item to the already existing list. Length: {len(stac_items)}.")
342
+ logger.info(f"Added one stac item metadata to the already existing list. Length: {len(items_metadata)}.")
333
343
 
334
- return stac_items, product_types
344
+ logger.info(f"List with stac_items to be published: {items_metadata}.")
345
+ return items_metadata
335
346
 
336
347
 
337
348
  def compute_eopf_origin_datetime(env, input_products) -> str:
@@ -412,7 +423,7 @@ async def run_processor(
412
423
  cluster_info: ClusterInfo,
413
424
  s3_payload_run: str,
414
425
  input_products: list[dict],
415
- ) -> list[Item]:
426
+ ) -> list[DprProcessedItemMetadata]:
416
427
  """
417
428
  Run the DPR processor.
418
429
 
@@ -490,7 +501,10 @@ async def run_processor(
490
501
  except FileNotFoundError:
491
502
  logger.info(f"No processor log file was uploaded under: {s3_payload_dir!r}")
492
503
 
493
- eopf_stac_items, eopf_types = update_eopf_assets(flow_env, input_products, payload, processor)
504
+ items_metadata = update_eopf_assets(flow_env, input_products, payload, processor)
505
+ eopf_stac_items = [asset.stac_item for asset in items_metadata]
506
+ eopf_types = [asset.product_type for asset in items_metadata]
507
+
494
508
  # Wait for the job to finish
495
509
  record_performance_indicators( # type: ignore
496
510
  stop_date=datetime.datetime.now(),
@@ -500,4 +514,4 @@ async def run_processor(
500
514
  dpr_processor_name=processor,
501
515
  eopf_types=eopf_types,
502
516
  )
503
- return eopf_stac_items
517
+ return items_metadata
@@ -367,6 +367,15 @@ class DprProcessOut:
367
367
  product_identifier: list[Item] = field(default_factory=list)
368
368
 
369
369
 
370
+ @dataclass
371
+ class DprProcessedItemMetadata:
372
+ """Metadata for a DPR processed item."""
373
+
374
+ output_product_id: str
375
+ product_type: str | None
376
+ stac_item: Item
377
+
378
+
370
379
  class RetryConfig(BaseModel):
371
380
  """
372
381
  Args:
@@ -28,13 +28,11 @@ from prefect.artifacts import acreate_markdown_artifact
28
28
 
29
29
  from rs_client.ogcapi.dpr_client import ClusterInfo
30
30
  from rs_common import prefect_utils
31
- from rs_common.utils import create_valcover_filter
32
- from rs_workflows import auxip_flow, cadip_flow, catalog_flow, prip_flow
31
+ from rs_workflows import auxip_flow, catalog_flow
33
32
  from rs_workflows.dpr_flow import run_processor
34
- from rs_workflows.flow_utils import DprProcessIn, FlowEnv, FlowEnvArgs, RetryConfig
33
+ from rs_workflows.flow_utils import DprProcessIn, FlowEnv, RetryConfig
35
34
  from rs_workflows.payload_builder import build_cql2_json, build_unit_list
36
35
  from rs_workflows.payload_generator import generate_payload
37
- from rs_workflows.staging_flow import staging_task
38
36
 
39
37
 
40
38
  def build_dask_dashboard_url_message(cluster_instance: str | None) -> str:
@@ -268,111 +266,3 @@ async def dpr_processing(
268
266
  published.result() # type: ignore[unused-coroutine]
269
267
 
270
268
  return
271
-
272
-
273
- @flow(name="On-demand Cadip staging")
274
- async def on_demand_cadip_staging(
275
- env: FlowEnvArgs,
276
- cadip_collection_identifier: str,
277
- session_identifier: str,
278
- catalog_collection_identifier: str,
279
- staging_retries: int = 3,
280
- staging_retry_delay: int = 60,
281
- ):
282
- """
283
- Flow to retrieve a session, stage it and add the STAC item into the catalog.
284
-
285
- Args:
286
- env: Prefect flow environment
287
- cadip_collection_identifier: CADIP collection identifier that contains the mission and station
288
- (e.g. s1_ins for Sentinel-1 sessions from the Inuvik station)
289
- session_identifier: Session identifier
290
- catalog_collection_identifier: Catalog collection identifier where CADIP sessions and AUX data are staged
291
- """
292
-
293
- # Init flow environment and opentelemetry span
294
- flow_env = FlowEnv(env)
295
- with flow_env.start_span(__name__, "on-demand-cadip-staging"):
296
-
297
- # Search Cadip sessions
298
- cadip_items = cadip_flow.search_task.with_options(
299
- retries=3,
300
- retry_delay_seconds=60,
301
- ).submit(
302
- flow_env.serialize(),
303
- cadip_collection_identifier,
304
- session_identifier,
305
- error_if_empty=True,
306
- )
307
-
308
- # Stage Cadip items.
309
- staged = staging_task.with_options(
310
- retries=staging_retries,
311
- retry_delay_seconds=staging_retry_delay,
312
- ).submit(
313
- flow_env.serialize(),
314
- cadip_items,
315
- catalog_collection_identifier,
316
- )
317
-
318
- # Wait for last task to end.
319
- # NOTE: use .result() and not .wait() to unwrap and propagate exceptions, if any.
320
- staged.result() # type: ignore[unused-coroutine]
321
-
322
-
323
- @flow(name="On-demand Prip staging")
324
- async def on_demand_prip_staging(
325
- env: FlowEnvArgs,
326
- start_datetime: datetime.datetime | str,
327
- end_datetime: datetime.datetime | str,
328
- product_type: str,
329
- prip_collection: str,
330
- catalog_collection_identifier: str,
331
- retry_config: RetryConfig = RetryConfig(), # type: ignore
332
- ):
333
- """
334
- Flow to retrieve Prip files with the given time interval defined by
335
- start_datetime and end_datetime, select only the type of files wanted,
336
- stage the files and add STAC items into the catalog.
337
-
338
- Args:
339
- env: Prefect flow environment
340
- start_datetime: Start datetime for the time interval used to filter the files
341
- (date or timestamp, e.g. "2025-08-07T11:51:12.509000Z")
342
- end_datetime: End datetime for the time interval used to filter the files
343
- (date or timestamp, e.g. "2025-08-10T14:00:00.509000Z")
344
- product_type: Prip product type wanted
345
- prip_collection: PRIP collection identifier (station)
346
- catalog_collection_identifier: Catalog collection identifier where PRIP data are staged
347
- """
348
-
349
- # Init flow environment and opentelemetry span
350
- flow_env = FlowEnv(env)
351
- with flow_env.start_span(__name__, "on-demand-prip-staging"):
352
-
353
- # CQL2 filter: filter on product type and time interval
354
- cql2_filter = create_valcover_filter(start_datetime, end_datetime, product_type)
355
-
356
- # Search Prip products
357
- prip_items = prip_flow.search_task.with_options(
358
- retries=retry_config.staging_retries,
359
- retry_delay_seconds=retry_config.staging_retry_delay,
360
- ).submit(
361
- flow_env.serialize(),
362
- prip_cql2={"filter": cql2_filter},
363
- prip_collection=prip_collection,
364
- error_if_empty=False,
365
- )
366
-
367
- # Stage Prip items
368
- staged = staging_task.with_options(
369
- retries=retry_config.staging_retries,
370
- retry_delay_seconds=retry_config.staging_retry_delay,
371
- ).submit(
372
- flow_env.serialize(),
373
- prip_items,
374
- catalog_collection_identifier,
375
- )
376
-
377
- # Wait for last task to end (unwrap exceptions if any)
378
- staged.result() # type: ignore[unused-coroutine]
@@ -240,6 +240,13 @@ def find_s3_output_bucket(
240
240
  if fallback_bucket is None:
241
241
  fallback_bucket = bucket
242
242
  logger.info(f"Configuration bucket: fallback_bucket: {fallback_bucket}")
243
+ else:
244
+ logger.warning(
245
+ "Multiple default configurations found in the configuration map "
246
+ "(rs-catalog-staging-configmap), expected only one. Using the first "
247
+ f"one found ({fallback_bucket}), but please check your configuration table to avoid "
248
+ "unexpected behaviors. There should be only one entry with the three first columns set to '*'",
249
+ )
243
250
 
244
251
  if fallback_bucket:
245
252
  logger.info(f"Configuration bucket: Return fallback_bucket: {fallback_bucket}")
@@ -383,7 +390,6 @@ def build_output_products(
383
390
 
384
391
  outputs = []
385
392
  processed_products = set()
386
- common_uuid = str(uuid4())
387
393
 
388
394
  for output_product in dpr_process_in.generated_product_to_collection_identifier:
389
395
  product_name = output_product.name
@@ -405,7 +411,7 @@ def build_output_products(
405
411
  )
406
412
 
407
413
  bucket_name = find_s3_output_bucket(bucket_configuration, owner_id, output_collection, product_type)
408
- output_path = os.path.join("s3://", bucket_name, owner_id, output_collection, common_uuid)
414
+ output_path = os.path.join("s3://", bucket_name, owner_id, output_collection, str(uuid4()))
409
415
  # cf story 871/Set S3 configuration in payload.yaml
410
416
  store_name = storage_configuration.get_storage_for_specific_product(product_name)
411
417
  if not store_name:
@@ -281,10 +281,10 @@ class OutputProduct(BasePayloadModel):
281
281
  opening_mode: str | None = Field(default="CREATE")
282
282
  apply_eoqc: bool | None = Field(default=False)
283
283
  # Excluded from serialization by default
284
- # This is because the concept of "final product" is present in tasktable only, and
285
- # it is unknown for the processor. The processor simply fails if it finds an unknown
286
- # field in the payload. So we need to exclude it from the payload, but we want to
287
- # keep it in the model for internal use in run_processor task.
284
+ # This is because the "final product" concept exists only in the tasktable and
285
+ # is is not recognized by the processor. The processor simply fails if it finds an unknown
286
+ # field in the payload. So we need to exclude this field from the payload, but we want to
287
+ # keep it in the model for internal use in the run_processor task.
288
288
  final_product: bool | None = Field(default=True, exclude=True)
289
289
 
290
290
 
@@ -0,0 +1,131 @@
1
+ # Copyright 2023-2026 Airbus, CS Group
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """Prip flow implementation"""
16
+
17
+ import datetime
18
+
19
+ from prefect import flow, get_run_logger, task
20
+ from pystac import ItemCollection
21
+
22
+ from rs_client.stac.prip_client import PripClient
23
+ from rs_common.utils import create_valcover_filter
24
+ from rs_workflows.flow_utils import FlowEnv, FlowEnvArgs, RetryConfig
25
+ from rs_workflows.staging_flow import staging_task
26
+
27
+
28
+ @flow(name="Prip search")
29
+ async def search(
30
+ env: FlowEnvArgs,
31
+ prip_cql2: dict,
32
+ prip_collection: str = "",
33
+ error_if_empty: bool = False,
34
+ ) -> ItemCollection | None:
35
+ """
36
+ Search Prip products.
37
+
38
+ Args:
39
+ env: Prefect flow environment (at least the owner_id is required)
40
+ prip_cql2: PRIP CQL2 filter.
41
+ prip_collection: PRIP ollection identifier (to know the station)
42
+ error_if_empty: Raise a ValueError if the results are empty.
43
+ """
44
+ logger = get_run_logger()
45
+
46
+ # Init flow environment and opentelemetry span
47
+ flow_env = FlowEnv(env)
48
+ with flow_env.start_span(__name__, "prip-search"):
49
+
50
+ logger.info("Start PRIP search")
51
+ prip_client: PripClient = flow_env.rs_client.get_prip_client()
52
+ found = prip_client.search(
53
+ method="POST",
54
+ stac_filter=prip_cql2.get("filter"),
55
+ max_items=prip_cql2.get("limit", 10),
56
+ sortby=prip_cql2.get("sortby", "-created"),
57
+ collections=[prip_collection],
58
+ )
59
+ if (not found) and error_if_empty:
60
+ raise ValueError("No PRIP products found")
61
+ logger.info(f"PRIP search found {len(found)} results: {found}")
62
+ return found
63
+
64
+
65
+ @flow(name="On-demand Prip staging")
66
+ async def on_demand_prip_staging(
67
+ env: FlowEnvArgs,
68
+ start_datetime: datetime.datetime | str,
69
+ end_datetime: datetime.datetime | str,
70
+ product_type: str,
71
+ prip_collection: str,
72
+ catalog_collection_identifier: str,
73
+ retry_config: RetryConfig = RetryConfig(), # type: ignore
74
+ ):
75
+ """
76
+ Flow to retrieve Prip files with the given time interval defined by
77
+ start_datetime and end_datetime, select only the type of files wanted,
78
+ stage the files and add STAC items into the catalog.
79
+
80
+ Args:
81
+ env: Prefect flow environment
82
+ start_datetime: Start datetime for the time interval used to filter the files
83
+ (date or timestamp, e.g. "2025-08-07T11:51:12.509000Z")
84
+ end_datetime: End datetime for the time interval used to filter the files
85
+ (date or timestamp, e.g. "2025-08-10T14:00:00.509000Z")
86
+ product_type: Prip product type wanted
87
+ prip_collection: PRIP collection identifier (station)
88
+ catalog_collection_identifier: Catalog collection identifier where PRIP data are staged
89
+ """
90
+
91
+ # Init flow environment and opentelemetry span
92
+ flow_env = FlowEnv(env)
93
+ with flow_env.start_span(__name__, "on-demand-prip-staging"):
94
+
95
+ # CQL2 filter: filter on product type and time interval
96
+ cql2_filter = create_valcover_filter(start_datetime, end_datetime, product_type)
97
+
98
+ # Search Prip products
99
+ prip_items = search_task.with_options(
100
+ retries=retry_config.staging_retries,
101
+ retry_delay_seconds=retry_config.staging_retry_delay,
102
+ ).submit(
103
+ flow_env.serialize(),
104
+ prip_cql2={"filter": cql2_filter},
105
+ prip_collection=prip_collection,
106
+ error_if_empty=False,
107
+ )
108
+
109
+ # Stage Prip items
110
+ staged = staging_task.with_options(
111
+ retries=retry_config.staging_retries,
112
+ retry_delay_seconds=retry_config.staging_retry_delay,
113
+ ).submit(
114
+ flow_env.serialize(),
115
+ prip_items,
116
+ catalog_collection_identifier,
117
+ )
118
+
119
+ # Wait for last task to end (unwrap exceptions if any)
120
+ staged.result() # type: ignore[unused-coroutine]
121
+
122
+
123
+ ###########################
124
+ # Call the flows as tasks #
125
+ ###########################
126
+
127
+
128
+ @task(name="PRIP search")
129
+ async def search_task(*args, **kwargs) -> ItemCollection | None:
130
+ """See: search"""
131
+ return await search.fn(*args, **kwargs)
@@ -388,11 +388,12 @@ def record_product_expected(flow_run_id: str, dpr_processor_name, payload, eopf_
388
388
  ("S03MWRL0_", 3, 3),
389
389
  ("S03OLCCR0", 0, 1),
390
390
  ("S03OLCCR1", 0, 1),
391
- ("S03OLCL0_", 23, 23),
391
+ ("S03OLCL0_", 22, 24),
392
392
  ("S03SLSL0_", 15, 22),
393
- ("S03ALTL0_", 12, 13),
393
+ ("S03ALTL0_", 10, 13),
394
394
  ("S03SRCRL0", 0, 1),
395
- ("S03HKML0_", 2, 2),
395
+ ("S03HKML0_", 1, 1),
396
+ ("S03HKM2L0", 1, 1),
396
397
  ("S03NATL0_", 2, 2),
397
398
  ]
398
399
  else:
@@ -1,69 +0,0 @@
1
- # Copyright 2023-2026 Airbus, CS Group
2
- #
3
- # Licensed under the Apache License, Version 2.0 (the "License");
4
- # you may not use this file except in compliance with the License.
5
- # You may obtain a copy of the License at
6
- #
7
- # http://www.apache.org/licenses/LICENSE-2.0
8
- #
9
- # Unless required by applicable law or agreed to in writing, software
10
- # distributed under the License is distributed on an "AS IS" BASIS,
11
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
- # See the License for the specific language governing permissions and
13
- # limitations under the License.
14
-
15
- """Prip flow implementation"""
16
-
17
- from prefect import flow, get_run_logger, task
18
- from pystac import ItemCollection
19
-
20
- from rs_client.stac.prip_client import PripClient
21
- from rs_workflows.flow_utils import FlowEnv, FlowEnvArgs
22
-
23
-
24
- @flow(name="Prip search")
25
- async def search(
26
- env: FlowEnvArgs,
27
- prip_cql2: dict,
28
- prip_collection: str = "",
29
- error_if_empty: bool = False,
30
- ) -> ItemCollection | None:
31
- """
32
- Search Prip products.
33
-
34
- Args:
35
- env: Prefect flow environment (at least the owner_id is required)
36
- prip_cql2: PRIP CQL2 filter.
37
- prip_collection: PRIP ollection identifier (to know the station)
38
- error_if_empty: Raise a ValueError if the results are empty.
39
- """
40
- logger = get_run_logger()
41
-
42
- # Init flow environment and opentelemetry span
43
- flow_env = FlowEnv(env)
44
- with flow_env.start_span(__name__, "prip-search"):
45
-
46
- logger.info("Start PRIP search")
47
- prip_client: PripClient = flow_env.rs_client.get_prip_client()
48
- found = prip_client.search(
49
- method="POST",
50
- stac_filter=prip_cql2.get("filter"),
51
- max_items=prip_cql2.get("limit", 10),
52
- sortby=prip_cql2.get("sortby", "-created"),
53
- collections=[prip_collection],
54
- )
55
- if (not found) and error_if_empty:
56
- raise ValueError("No PRIP products found")
57
- logger.info(f"PRIP search found {len(found)} results: {found}")
58
- return found
59
-
60
-
61
- ###########################
62
- # Call the flows as tasks #
63
- ###########################
64
-
65
-
66
- @task(name="PRIP search")
67
- async def search_task(*args, **kwargs) -> ItemCollection | None:
68
- """See: search"""
69
- return await search.fn(*args, **kwargs)