rs-client-libraries 1.0a3__tar.gz → 1.0a4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. {rs_client_libraries-1.0a3 → rs_client_libraries-1.0a4}/PKG-INFO +2 -1
  2. {rs_client_libraries-1.0a3 → rs_client_libraries-1.0a4}/pyproject.toml +3 -2
  3. {rs_client_libraries-1.0a3 → rs_client_libraries-1.0a4}/rs_client/__init__.py +1 -1
  4. {rs_client_libraries-1.0a3 → rs_client_libraries-1.0a4}/rs_client/stac/catalog_client.py +4 -0
  5. {rs_client_libraries-1.0a3 → rs_client_libraries-1.0a4}/rs_workflows/auxip_flow.py +13 -3
  6. {rs_client_libraries-1.0a3 → rs_client_libraries-1.0a4}/rs_workflows/record_performance.py +203 -7
  7. {rs_client_libraries-1.0a3 → rs_client_libraries-1.0a4}/LICENSE +0 -0
  8. {rs_client_libraries-1.0a3 → rs_client_libraries-1.0a4}/README.adoc +0 -0
  9. {rs_client_libraries-1.0a3 → rs_client_libraries-1.0a4}/rs_client/ogcapi/__init__.py +0 -0
  10. {rs_client_libraries-1.0a3 → rs_client_libraries-1.0a4}/rs_client/ogcapi/dpr_client.py +0 -0
  11. {rs_client_libraries-1.0a3 → rs_client_libraries-1.0a4}/rs_client/ogcapi/ogcapi_client.py +0 -0
  12. {rs_client_libraries-1.0a3 → rs_client_libraries-1.0a4}/rs_client/ogcapi/staging_client.py +0 -0
  13. {rs_client_libraries-1.0a3 → rs_client_libraries-1.0a4}/rs_client/rs_client.py +0 -0
  14. {rs_client_libraries-1.0a3 → rs_client_libraries-1.0a4}/rs_client/stac/__init__.py +0 -0
  15. {rs_client_libraries-1.0a3 → rs_client_libraries-1.0a4}/rs_client/stac/auxip_client.py +0 -0
  16. {rs_client_libraries-1.0a3 → rs_client_libraries-1.0a4}/rs_client/stac/cadip_client.py +0 -0
  17. {rs_client_libraries-1.0a3 → rs_client_libraries-1.0a4}/rs_client/stac/prip_client.py +0 -0
  18. {rs_client_libraries-1.0a3 → rs_client_libraries-1.0a4}/rs_client/stac/stac_base.py +0 -0
  19. {rs_client_libraries-1.0a3 → rs_client_libraries-1.0a4}/rs_common/__init__.py +0 -0
  20. {rs_client_libraries-1.0a3 → rs_client_libraries-1.0a4}/rs_common/config.py +0 -0
  21. {rs_client_libraries-1.0a3 → rs_client_libraries-1.0a4}/rs_common/init_opentelemetry.py +0 -0
  22. {rs_client_libraries-1.0a3 → rs_client_libraries-1.0a4}/rs_common/logging.py +0 -0
  23. {rs_client_libraries-1.0a3 → rs_client_libraries-1.0a4}/rs_common/prefect_utils.py +0 -0
  24. {rs_client_libraries-1.0a3 → rs_client_libraries-1.0a4}/rs_common/utils.py +0 -0
  25. {rs_client_libraries-1.0a3 → rs_client_libraries-1.0a4}/rs_workflows/__init__.py +0 -0
  26. {rs_client_libraries-1.0a3 → rs_client_libraries-1.0a4}/rs_workflows/cadip_flow.py +0 -0
  27. {rs_client_libraries-1.0a3 → rs_client_libraries-1.0a4}/rs_workflows/catalog_flow.py +0 -0
  28. {rs_client_libraries-1.0a3 → rs_client_libraries-1.0a4}/rs_workflows/dpr_flow.py +0 -0
  29. {rs_client_libraries-1.0a3 → rs_client_libraries-1.0a4}/rs_workflows/flow_utils.py +0 -0
  30. {rs_client_libraries-1.0a3 → rs_client_libraries-1.0a4}/rs_workflows/init_pi_db_flow.py +0 -0
  31. {rs_client_libraries-1.0a3 → rs_client_libraries-1.0a4}/rs_workflows/on_demand_processing.py +0 -0
  32. {rs_client_libraries-1.0a3 → rs_client_libraries-1.0a4}/rs_workflows/payload_builder.py +0 -0
  33. {rs_client_libraries-1.0a3 → rs_client_libraries-1.0a4}/rs_workflows/pi_db_models.py +0 -0
  34. {rs_client_libraries-1.0a3 → rs_client_libraries-1.0a4}/rs_workflows/prip_flow.py +0 -0
  35. {rs_client_libraries-1.0a3 → rs_client_libraries-1.0a4}/rs_workflows/staging_flow.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: rs-client-libraries
3
- Version: 1.0a3
3
+ Version: 1.0a4
4
4
  Summary: Copernicus Sentinel rs-client library and processing workflows.
5
5
  License: Apache-2.0
6
6
  License-File: LICENSE
@@ -37,6 +37,7 @@ Requires-Dist: python-logging-loki (>=0.3.1,<0.4.0)
37
37
  Requires-Dist: requests (>=2.32.4,<3.0.0)
38
38
  Requires-Dist: stac-pydantic (>=3.4.0,<4.0.0)
39
39
  Requires-Dist: urllib3 (>=2.5.0)
40
+ Requires-Dist: uv (>=0.9.7)
40
41
  Project-URL: Documentation, https://home.rs-python.eu/rs-documentation/rs-client-libraries/docs/doc/
41
42
  Project-URL: Homepage, https://home.rs-python.eu/
42
43
  Project-URL: Repository, https://github.com/RS-PYTHON/rs-client-libraries
@@ -18,7 +18,7 @@ dynamic = ["version"] # see: https://github.com/mtkennerly/poetry-dynamic-versio
18
18
 
19
19
  [tool.poetry]
20
20
  name = "rs-client-libraries"
21
- version = "1.0a3"
21
+ version = "1.0a4"
22
22
  description = "Copernicus Sentinel rs-client library and processing workflows."
23
23
  license = "Apache-2.0"
24
24
  authors = ["ESA under contract with Airbus and CS <ops_coprs@airbus.com>"]
@@ -68,6 +68,7 @@ prefect = { version = "3.2.13", extras = ["aws"] }
68
68
  requests = "^2.32.4"
69
69
  urllib3 = ">=2.5.0"
70
70
  SQLAlchemy = "^2.0.37"
71
+ uv = ">=0.9.7"
71
72
 
72
73
  [tool.poetry.group.dev]
73
74
  optional = true
@@ -90,7 +91,7 @@ sphinx-rtd-theme = "^3.0.2"
90
91
  flake8 = "^7.1.1"
91
92
  flake8-pyproject = "^1.2.3"
92
93
  responses = "^0.25.7"
93
- starlette = "^0.48.0"
94
+ starlette = "^0.49.0"
94
95
 
95
96
  [tool.poetry.group.demo.dependencies] # for hybrid mode
96
97
  jupyterlab = "^4.2.1"
@@ -15,4 +15,4 @@
15
15
  """rsclient main module."""
16
16
 
17
17
  # Set automatically by running `poetry dynamic-versioning`
18
- __version__ = "1.0a3"
18
+ __version__ = "1.0a4"
@@ -273,6 +273,10 @@ class CatalogClient(StacBase): # type: ignore # pylint: disable=too-many-ancest
273
273
  full_collection_id = self.full_collection_id(owner_id, short_collection_id)
274
274
  collection.id = full_collection_id
275
275
 
276
+ # Default title (NOTE: this is how the collection is displayed in the Stac Browser)
277
+ if not collection.title:
278
+ collection.title = full_collection_id
279
+
276
280
  # Default description
277
281
  if not collection.description:
278
282
  collection.description = f"This is the collection {short_collection_id} from user {full_owner_id}."
@@ -22,6 +22,7 @@ from prefect.artifacts import acreate_markdown_artifact
22
22
  from pystac import ItemCollection
23
23
 
24
24
  from rs_client.stac.auxip_client import AuxipClient
25
+ from rs_client.stac.catalog_client import CatalogClient
25
26
  from rs_common.utils import create_valcover_filter
26
27
  from rs_workflows.flow_utils import FlowEnv, FlowEnvArgs
27
28
  from rs_workflows.staging_flow import staging_task
@@ -87,7 +88,7 @@ async def auxip_staging(
87
88
 
88
89
  Returns:
89
90
  bool: Return status: False if staging failed, True otherwise
90
- ItemCollection: List of Items retrieved from the Auxip search and staged to the catalog
91
+ ItemCollection: List of catalog Items staged from Auxip station
91
92
  """
92
93
  logger = get_run_logger()
93
94
 
@@ -134,16 +135,25 @@ async def auxip_staging(
134
135
  logger.debug({job_name: job_result})
135
136
  return_status = False
136
137
 
138
+ # Get staged items from catalog (to have the correct href)
139
+ catalog_client: CatalogClient = flow_env.rs_client.get_catalog_client()
140
+ catalog_items = ItemCollection(
141
+ catalog_client.get_items(
142
+ collection_id=catalog_collection_identifier,
143
+ items_ids=[item.id for item in auxip_items],
144
+ ),
145
+ )
146
+
137
147
  # Create artifact if all jobs succeeded
138
148
  if return_status:
139
149
  logger.info("Staging successful, creating artifact with a list of staged items.")
140
150
  await acreate_markdown_artifact(
141
- markdown=f"{json.dumps(auxip_items.to_dict(), indent=2)}",
151
+ markdown=f"{json.dumps(catalog_items.to_dict(), indent=2)}",
142
152
  key="auxiliary-files",
143
153
  description="Auxiliary files added to catalog.",
144
154
  )
145
155
 
146
- return return_status, auxip_items
156
+ return return_status, catalog_items
147
157
 
148
158
 
149
159
  @flow(name="On-demand Auxip staging")
@@ -142,6 +142,7 @@ def get_pi_category_id(eopf_type: str) -> int | None:
142
142
  logger.info("DB session closed")
143
143
 
144
144
 
145
+ @task(name="Record Flow Run")
145
146
  def record_flow_run(
146
147
  start_date: datetime | str | None = None,
147
148
  stop_date: datetime | str | None = None,
@@ -220,8 +221,41 @@ def record_flow_run(
220
221
  return flow_run_id
221
222
 
222
223
 
224
+ @task(name="Record Products Realised")
223
225
  def record_product_realised(flow_run_id, stac_items):
224
- """Insert records in product_realised table"""
226
+ """
227
+ Insert STAC product metadata into the `product_realised` database table.
228
+
229
+ This task records all products discovered during a flow run by inserting
230
+ one row per STAC item into the `product_realised` table. Each inserted record
231
+ includes product metadata (EOPF type, sensing time, origin date, etc.) and
232
+ default validation flags, which will later be updated by the validation step.
233
+
234
+ Behavior:
235
+ - Skips insertion if no STAC items are provided.
236
+ - Inserts new rows only (no upsert logic).
237
+ - Marks all timing and validation flags (`unexpected`, `on_time_X_day`) as False initially.
238
+ - Rolls back the transaction on any error to avoid partial inserts.
239
+
240
+ Args:
241
+ flow_run_id (str | UUID): Identifier of the current flow run, used to link inserted records.
242
+ stac_items (list[dict]): List of STAC item dictionaries generated by DPR or discovery steps.
243
+
244
+ Raises:
245
+ KeyError: If a required field (e.g., `product:type`) is missing from a STAC item.
246
+ Exception: For any other unexpected database or runtime errors.
247
+
248
+ Logging:
249
+ - Logs when no STAC items are provided.
250
+ - Logs each successful insert operation.
251
+ - Logs detailed errors on failure.
252
+
253
+ Notes:
254
+ - The `sensing_start_datetime` and `origin_date` fields are extracted from
255
+ the STAC item's properties. If missing, the current timestamp (`datetime.now()`)
256
+ is used as a fallback.
257
+ - The `pi_category_id` is derived from the product type using `get_pi_category_id()`.
258
+ """
225
259
  logger = get_run_logger()
226
260
  metadata = MetaData()
227
261
  db, engine = get_db_session()
@@ -295,8 +329,43 @@ def extract_min_datetime(list_items):
295
329
  return earliest
296
330
 
297
331
 
332
+ @task(name="Record Products Expected")
298
333
  def record_product_expected(flow_run_id: str, dpr_processor_name, payload):
299
- """Insert hardcoded records in product_expected table by flow_run_id."""
334
+ """
335
+ Insert expected product definitions into the `product_expected` table for a given flow run.
336
+
337
+ This task records all product types that are expected to be generated by a specific
338
+ DPR processor (currently only `s3_l0`). It defines each expected product type with
339
+ its minimum and maximum expected counts and stores them in the `product_expected` table.
340
+
341
+ Behavior:
342
+ - Inserts one record per expected product type, if it does not already exist
343
+ for the given `flow_run_id`.
344
+ - Uses a hardcoded mapping of expected product types and their min/max count.
345
+ - Derives the sensing start time from the input STAC items in the payload.
346
+
347
+ Args:
348
+ flow_run_id (str): Unique identifier of the current flow run.
349
+ dpr_processor_name (str): Name of the DPR processor (e.g., `"s3_l0"`).
350
+ payload (dict): JSON-like payload containing workflow metadata and STAC inputs/outputs.
351
+
352
+ Raises:
353
+ KeyError: If an expected EOPF type from the payload is not found in the lookup mapping.
354
+ Exception: For any unexpected errors during lookup or database operations.
355
+
356
+ Logging:
357
+ - Logs skipped execution for unsupported processors.
358
+ - Logs each successful product insertion.
359
+ - Logs lookup failures or unexpected database errors.
360
+
361
+ Notes:
362
+ - The hardcoded mapping (`eopf_type_dict`) defines the expected product types
363
+ and their respective min/max expected counts.
364
+ - The sensing start datetime (`sensing_start_datetime`) is derived from
365
+ workflow inputs using `extract_min_datetime()`.
366
+ - The `pi_category_id` field is determined dynamically from `get_pi_category_id()`.
367
+ - Duplicate entries for the same `(flow_run_id, eopf_type)` are not reinserted.
368
+ """
300
369
 
301
370
  logger = get_run_logger()
302
371
  metadata = MetaData()
@@ -304,8 +373,8 @@ def record_product_expected(flow_run_id: str, dpr_processor_name, payload):
304
373
  product_expected = Table("product_expected", metadata, autoload_with=engine)
305
374
 
306
375
  eopf_type_dict = []
307
- # products expected for S3 Level 0
308
- if dpr_processor_name == "s3_l0":
376
+
377
+ if dpr_processor_name in ["s3_l0", "mockup"]:
309
378
  eopf_type_dict = [
310
379
  ("S03DORDOP", 1, 1),
311
380
  ("S03DORNAV", 1, 1),
@@ -371,9 +440,46 @@ def record_product_expected(flow_run_id: str, dpr_processor_name, payload):
371
440
  db.close()
372
441
 
373
442
 
443
+ @task(name="Validate Products")
374
444
  def validate_products(flow_run_id: str):
375
- """Validate realised vs expected products for a given flow_run_id.
376
- : running multiple times won't duplicate inserts/updates.
445
+ """
446
+ Validate realised products against expected definitions for a given flow run.
447
+
448
+ This task ensures that the products generated during a flow run (`product_realised`)
449
+ match the products defined as expected (`product_expected`). It performs a
450
+ consistency check between expected and realised product types and counts, updating
451
+ the database accordingly.
452
+
453
+ Behavior:
454
+ - For each expected product type, counts the number of realised instances.
455
+ - Marks missing or excess products based on defined min/max thresholds.
456
+ - Inserts missing product records into the `product_missing` table if they are
457
+ not already present.
458
+ - Flags unexpected or surplus realised products in the `product_realised` table
459
+ by setting `unexpected=True`.
460
+ - Detects any realised product types that were not listed in the expectations
461
+ and marks them as unexpected.
462
+
463
+ Args:
464
+ flow_run_id (str): Unique identifier of the current flow run.
465
+
466
+ Raises:
467
+ Exception: For any unexpected errors during validation or database operations.
468
+
469
+ Logging:
470
+ - Logs counts and validation results for each product type.
471
+ - Logs creation of missing-product records.
472
+ - Logs when products are marked as unexpected or already processed.
473
+ - Logs warnings when metadata for missing products cannot be inferred.
474
+
475
+ Notes:
476
+ - Running this validation multiple times is idempotent; it will not duplicate
477
+ inserts or updates.
478
+ - The logic relies on existing rows in `product_expected` and `product_realised`.
479
+ - `product_missing` entries are created only when a deficit is detected.
480
+ - Products exceeding the allowed maximum or with unexpected types are flagged
481
+ instead of deleted.
482
+ - All changes are committed at the end; a rollback occurs on any exception.
377
483
  """
378
484
 
379
485
  logger = get_run_logger()
@@ -511,7 +617,95 @@ def validate_products(flow_run_id: str):
511
617
  db.close()
512
618
 
513
619
 
514
- @task
620
+ @task(name="Update Timeliness Fields")
621
+ def update_timeliness_fields(flow_run_id):
622
+ """
623
+ Compute and update timeliness metrics for all realised products in a given flow run.
624
+
625
+ This task evaluates how promptly each product was stored in the catalog after its
626
+ origin time and updates corresponding boolean flags (`on_time_X_day`) in the
627
+ `product_realised` table. These flags indicate whether each product met its
628
+ expected timeliness threshold based on the category-specific maximum delay.
629
+
630
+ Behavior:
631
+ - Retrieves all `product_realised` records for the specified `flow_run_id`.
632
+ - For each product, calculates the delay between `origin_date` and
633
+ `catalog_stored_datetime`.
634
+ - Compares the delay against the maximum allowed delay from `pi_category`
635
+ to determine timeliness flags at 0, 1, 2, 3, and 7-day thresholds.
636
+ - Updates the relevant fields in the database for each product.
637
+ - Skips execution if no products are found for the given `flow_run_id`.
638
+
639
+ Args:
640
+ flow_run_id (str | UUID): Unique identifier of the flow run whose products should be updated.
641
+
642
+ Raises:
643
+ Exception: For any unexpected database or runtime errors during the update process.
644
+
645
+ Logging:
646
+ - Logs the computed delay for each product.
647
+ - Logs when updates are applied or when no products are found.
648
+ - Logs detailed errors on exceptions.
649
+
650
+ Notes:
651
+ - The `max_delay_seconds` value is retrieved from the associated `pi_category` entry.
652
+ - The timeliness thresholds (0, 1, 2, 3, and 7 days) are evaluated relative
653
+ to the product’s origin time.
654
+ - All updates are committed at once after processing all records; a rollback
655
+ occurs on any error.
656
+ - Running this task multiple times simply recalculates and updates fields,
657
+ maintaining idempotence.
658
+ """
659
+
660
+ logger = get_run_logger()
661
+ metadata = MetaData()
662
+ db, engine = get_db_session()
663
+ pi_category = Table("pi_category", metadata, autoload_with=engine)
664
+ product_realised = Table("product_realised", metadata, autoload_with=engine)
665
+
666
+ try:
667
+
668
+ products = db.execute(select(product_realised).where(product_realised.c.flow_run_id == flow_run_id)).fetchall()
669
+
670
+ if not products:
671
+ logger.info("No records provided — skipping updating the timeliness in product_realised.")
672
+ return
673
+
674
+ for prod in products:
675
+ catalog_stored_datetime = prod.catalog_stored_datetime
676
+ origin_datetime = prod.origin_date
677
+
678
+ # Get the allowed max delay (in seconds)
679
+ max_delay_seconds = db.execute(
680
+ select(pi_category.c.max_delay_seconds).where(pi_category.c.id == prod.pi_category_id),
681
+ ).scalar()
682
+
683
+ delay = (catalog_stored_datetime - origin_datetime).total_seconds()
684
+ logger.info(f"For product {prod.id} delay is {delay} seconds.")
685
+
686
+ updates = {
687
+ "on_time_0_day": delay <= max_delay_seconds,
688
+ "on_time_1_day": delay <= max_delay_seconds + 1 * 24 * 3600,
689
+ "on_time_2_day": delay <= max_delay_seconds + 2 * 24 * 3600,
690
+ "on_time_3_day": delay <= max_delay_seconds + 3 * 24 * 3600,
691
+ "on_time_7_day": delay <= max_delay_seconds + 7 * 24 * 3600,
692
+ }
693
+
694
+ db.execute(update(product_realised).where(product_realised.c.id == prod.id).values(**updates))
695
+
696
+ db.commit()
697
+ logger.info(f"Updated timeliness fields for flow_run_id={flow_run_id}")
698
+
699
+ except Exception as e:
700
+ db.rollback()
701
+ logger.error(f"Unexpected error in update_timeliness_fields: {e}")
702
+ raise
703
+
704
+ finally:
705
+ db.close()
706
+
707
+
708
+ @task(name="Record Performance Indicators")
515
709
  def record_performance_indicators(
516
710
  # flow_run params
517
711
  start_date: datetime | str | None = None,
@@ -551,6 +745,8 @@ def record_performance_indicators(
551
745
  record_product_realised(flow_run_id, stac_items)
552
746
 
553
747
  validate_products(flow_run_id)
748
+
749
+ update_timeliness_fields(flow_run_id)
554
750
  logger.info("Transaction committed successfully!")
555
751
 
556
752
  except Exception as e: