rs-client-libraries 0.2a15.post17.dev0__tar.gz → 1.0a1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a1}/PKG-INFO +8 -7
  2. {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a1}/pyproject.toml +24 -9
  3. {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a1}/rs_client/__init__.py +1 -1
  4. {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a1}/rs_client/ogcapi/dpr_client.py +25 -3
  5. {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a1}/rs_client/ogcapi/ogcapi_client.py +10 -11
  6. {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a1}/rs_client/ogcapi/staging_client.py +0 -1
  7. {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a1}/rs_client/stac/catalog_client.py +164 -32
  8. {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a1}/rs_common/prefect_utils.py +7 -15
  9. {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a1}/rs_common/utils.py +5 -5
  10. {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a1}/rs_workflows/dpr_flow.py +2 -2
  11. rs_client_libraries-1.0a1/rs_workflows/init_pi_db_flow.py +140 -0
  12. {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a1}/rs_workflows/on_demand_processing.py +12 -26
  13. rs_client_libraries-1.0a1/rs_workflows/pi_db_models.py +156 -0
  14. rs_client_libraries-1.0a1/rs_workflows/record_flow_run.py +102 -0
  15. {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a1}/rs_workflows/staging_flow.py +1 -1
  16. {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a1}/LICENSE +0 -0
  17. {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a1}/README.adoc +0 -0
  18. {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a1}/rs_client/ogcapi/__init__.py +0 -0
  19. {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a1}/rs_client/rs_client.py +0 -0
  20. {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a1}/rs_client/stac/__init__.py +0 -0
  21. {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a1}/rs_client/stac/auxip_client.py +0 -0
  22. {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a1}/rs_client/stac/cadip_client.py +0 -0
  23. {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a1}/rs_client/stac/stac_base.py +0 -0
  24. {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a1}/rs_common/__init__.py +0 -0
  25. {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a1}/rs_common/config.py +0 -0
  26. {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a1}/rs_common/init_opentelemetry.py +0 -0
  27. {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a1}/rs_common/logging.py +0 -0
  28. {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a1}/rs_workflows/__init__.py +0 -0
  29. {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a1}/rs_workflows/auxip_flow.py +0 -0
  30. {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a1}/rs_workflows/cadip_flow.py +0 -0
  31. {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a1}/rs_workflows/catalog_flow.py +0 -0
  32. {rs_client_libraries-0.2a15.post17.dev0 → rs_client_libraries-1.0a1}/rs_workflows/flow_utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.3
2
2
  Name: rs-client-libraries
3
- Version: 0.2a15.post17.dev0
3
+ Version: 1.0a1
4
4
  Summary: Copernicus Sentinel rs-client library and processing workflows.
5
5
  License: Apache-2.0
6
6
  Keywords: stac,copernicus,sentinel,processing
@@ -18,21 +18,22 @@ Classifier: Programming Language :: Python :: 3.12
18
18
  Classifier: Programming Language :: Python :: 3.13
19
19
  Classifier: Topic :: Scientific/Engineering :: Image Processing
20
20
  Classifier: Topic :: Software Development :: Libraries :: Python Modules
21
+ Requires-Dist: SQLAlchemy (>=2.0.37,<3.0.0)
21
22
  Requires-Dist: cachetools (>=5.5.2,<6.0.0)
22
23
  Requires-Dist: click (==8.1.8)
23
24
  Requires-Dist: openapi-core (>=0.19.5,<0.20.0)
24
- Requires-Dist: opentelemetry-distro (>=0.55b1,<0.56)
25
- Requires-Dist: opentelemetry-exporter-otlp (>=1.34.1,<2.0.0)
26
- Requires-Dist: opentelemetry-instrumentation-aws-lambda (>=0.55b1,<0.56)
27
- Requires-Dist: opentelemetry-test-utils (>=0.55b1,<0.56)
25
+ Requires-Dist: opentelemetry-distro (>=0.57b0,<0.58)
26
+ Requires-Dist: opentelemetry-exporter-otlp (>=1.36.0,<2.0.0)
27
+ Requires-Dist: opentelemetry-instrumentation-aws-lambda (>=0.57b0,<0.58)
28
+ Requires-Dist: opentelemetry-test-utils (>=0.57b0,<0.58)
28
29
  Requires-Dist: prefect[aws] (==3.2.13)
29
30
  Requires-Dist: protobuf (>=5.29.5)
30
31
  Requires-Dist: pystac (>=1.13.0,<2.0.0)
31
- Requires-Dist: pystac-client (>=0.8.6,<0.9.0)
32
+ Requires-Dist: pystac-client (>=0.9.0,<0.10.0)
32
33
  Requires-Dist: python-dateutil (>=2.9.0.post0,<3.0.0)
33
34
  Requires-Dist: python-logging-loki (>=0.3.1,<0.4.0)
34
35
  Requires-Dist: requests (>=2.32.4,<3.0.0)
35
- Requires-Dist: stac-pydantic (>=3.1.5,<4.0.0)
36
+ Requires-Dist: stac-pydantic (>=3.4.0,<4.0.0)
36
37
  Requires-Dist: urllib3 (>=2.5.0)
37
38
  Project-URL: Documentation, https://home.rs-python.eu/rs-documentation/rs-client-libraries/docs/doc/
38
39
  Project-URL: Homepage, https://home.rs-python.eu/
@@ -1,10 +1,24 @@
1
+ # Copyright 2025 CS Group
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
1
15
  [project]
2
16
  name = "rs-client-libraries"
3
17
  dynamic = ["version"] # see: https://github.com/mtkennerly/poetry-dynamic-versioning
4
18
 
5
19
  [tool.poetry]
6
20
  name = "rs-client-libraries"
7
- version = "0.2a15.post17.dev0"
21
+ version = "1.0a1"
8
22
  description = "Copernicus Sentinel rs-client library and processing workflows."
9
23
  license = "Apache-2.0"
10
24
  authors = ["ESA under contract with Airbus and CS <ops_coprs@airbus.com>"]
@@ -37,22 +51,23 @@ include = [
37
51
 
38
52
  [tool.poetry.dependencies]
39
53
  python = "^3.11"
40
- opentelemetry-distro = "^0.55b1" # then run 'poetry run opentelemetry-bootstrap -a install'
41
- opentelemetry-instrumentation-aws-lambda = "^0.55b1"
42
- opentelemetry-test-utils = "^0.55b1"
43
- opentelemetry-exporter-otlp = "^1.34.1"
54
+ opentelemetry-distro = "^0.57b0" # then run 'poetry run opentelemetry-bootstrap -a install'
55
+ opentelemetry-instrumentation-aws-lambda = "^0.57b0"
56
+ opentelemetry-test-utils = "^0.57b0"
57
+ opentelemetry-exporter-otlp = "^1.36.0"
44
58
  protobuf = ">=5.29.5"
45
59
  python-logging-loki = "^0.3.1" # to avoid otl error: No module named 'logging_loki'
46
60
  cachetools = "^5.5.2"
47
61
  pystac = "^1.13.0"
48
- pystac-client = "^0.8.6"
62
+ pystac-client = "^0.9.0"
49
63
  python-dateutil = "^2.9.0.post0"
50
- stac-pydantic = "^3.1.5"
64
+ stac-pydantic = "^3.4.0"
51
65
  openapi-core = "^0.19.5"
52
66
  click = "8.1.8"
53
67
  prefect = { version = "3.2.13", extras = ["aws"] }
54
68
  requests = "^2.32.4"
55
69
  urllib3 = ">=2.5.0"
70
+ SQLAlchemy = "^2.0.37"
56
71
 
57
72
  [tool.poetry.group.dev]
58
73
  optional = true
@@ -74,8 +89,8 @@ sphinx-rtd-theme = "^3.0.2"
74
89
  flake8 = "^7.1.1"
75
90
  flake8-pyproject = "^1.2.3"
76
91
  responses = "^0.25.7"
77
- starlette = "^0.45.3"
78
- pytest-asyncio = "^1.0.0"
92
+ starlette = "^0.47.2"
93
+ pytest-asyncio = "^1.1.0"
79
94
 
80
95
  [tool.poetry.group.demo.dependencies] # for hybrid mode
81
96
  jupyterlab = "^4.2.1"
@@ -15,4 +15,4 @@
15
15
  """rsclient main module."""
16
16
 
17
17
  # Set automatically by running `poetry dynamic-versioning`
18
- __version__ = "0.2a15.post17.dev0"
18
+ __version__ = "1.0a1"
@@ -67,6 +67,7 @@ class DprClient(OgcApiClient):
67
67
  payload_subpath: str,
68
68
  s3_report_dir: str | None,
69
69
  use_dpr_mockup: bool = False,
70
+ extra_data: dict | None = None,
70
71
  ) -> dict:
71
72
  """Method to start the process from rs-client - Call the endpoint /processes/{process}/execution
72
73
 
@@ -77,6 +78,7 @@ class DprClient(OgcApiClient):
77
78
  s3_report_dir: S3 bucket folder were the processor report files will be written (optional). All the eopf
78
79
  local files written in the local "./reports" directory will be pushed to this S3 bucket folder.
79
80
  use_dpr_mockup: Use the real or the mockup DPR processor ?
81
+ extra_data: Extra data to pass to the processor.
80
82
 
81
83
  Return:
82
84
  job_id (int, str): Returns the status code of the request + the identifier
@@ -90,7 +92,7 @@ class DprClient(OgcApiClient):
90
92
  "s3_config_dir": s3_config_dir,
91
93
  "payload_subpath": payload_subpath,
92
94
  "s3_report_dir": s3_report_dir,
93
- }
95
+ } | (extra_data or {})
94
96
 
95
97
  # For the mockup processor, pass the payload contents.
96
98
  # Download the payload file into a temp file.
@@ -112,6 +114,20 @@ class DprClient(OgcApiClient):
112
114
  # Call the parent method
113
115
  return super()._run_process(process, data)
114
116
 
117
+ def run_conv_safe_zarr(self, payload: dict):
118
+ """Method to start the safe to zarr conversion process from rs-client -
119
+ Call the endpoint /processes/conv_safe_zarr/execution
120
+
121
+ Args:
122
+ payload: Dictionary to pass to the processor,
123
+ containing input_safe_path - the s3 path of legacy product and
124
+ output_zarr_dir_path - the s3 path for the new zarr
125
+ Return:
126
+ job_id (int, str): Returns the status code of the request + the identifier
127
+ (or None if endpoint fails) of the running job
128
+ """
129
+ return super()._run_process("conv_safe_zarr", payload)
130
+
115
131
  def wait_for_job(self, *args, **kwargs) -> list[dict]: # type: ignore
116
132
  """
117
133
  Wait for job to finish.
@@ -128,10 +144,16 @@ class DprClient(OgcApiClient):
128
144
  ######################################################
129
145
 
130
146
  def get_processes(self) -> dict:
131
- raise NotImplementedError
147
+ """Get all defined processes with logging."""
148
+ processes = super().get_processes()
149
+ self.logger.debug("Fetched %d processes", len(processes))
150
+ return processes
132
151
 
133
152
  def get_jobs(self) -> dict:
134
- raise NotImplementedError
153
+ """Get all registered jobs with logging."""
154
+ jobs = super().get_jobs()
155
+ self.logger.debug("Fetched %d jobs", len(jobs))
156
+ return jobs
135
157
 
136
158
  def delete_job(self, _: str) -> dict:
137
159
  raise NotImplementedError
@@ -67,23 +67,23 @@ class OgcApiClient(RsClient):
67
67
  request (Request): endpoint request
68
68
 
69
69
  Returns:
70
- ResponseUnmarshalResult.data: data validated by the openapi_core
70
+ RequestUnmarshalResult.data: data validated by the openapi_core
71
71
  unmarshal_response method
72
72
  """
73
73
  openapi_request = RequestsOpenAPIRequest(request)
74
74
 
75
- # validate_request(request, spec=Spec.from_file_path(PATH_TO_YAML_OPENAPI))
75
+ # validate request
76
76
  result = self.get_openapi().unmarshal_request(openapi_request)
77
77
 
78
78
  if result.errors:
79
79
  raise OgcValidationException(
80
- f"Error validating the request of the enpoint "
81
- f"{openapi_request.path}: {str(result.errors[0])}", # type: ignore
80
+ f"Error validating the request of the endpoint "
81
+ f"{openapi_request.path}: {', '.join(str(x) for x in result.errors)}",
82
82
  )
83
83
  if not result.body:
84
84
  raise OgcValidationException(
85
- f"Error validating the request of the enpoint "
86
- f"{openapi_request.path}: 'data' field of ResponseUnmarshalResult"
85
+ f"Error validating the request of the endpoint "
86
+ f"{openapi_request.path}: 'data' field of RequestUnmarshalResult"
87
87
  f"object is empty",
88
88
  )
89
89
  return result.body
@@ -102,18 +102,17 @@ class OgcApiClient(RsClient):
102
102
  openapi_request = RequestsOpenAPIRequest(response.request)
103
103
  openapi_response = RequestsOpenAPIResponse(response)
104
104
 
105
- # Alternative method to validate the response
106
- # validate_response(response=response, spec= Spec.from_file_path(PATH_TO_YAML_OPENAPI), request=request)
105
+ # validate response
107
106
  result = self.get_openapi().unmarshal_response(openapi_request, openapi_response) # type: ignore
108
107
  if result.errors:
109
108
  raise OgcValidationException( # type: ignore
110
- f"Error validating the response of the enpoint {openapi_request.path} - "
109
+ f"Error validating the response of the endpoint {openapi_request.path} - "
111
110
  f"Server response content: {response.json()} - "
112
- f"Validation error of the server response: {str(result.errors[0])}", # type: ignore
111
+ f"Validation error of the server response: {', '.join(str(x) for x in result.errors)}",
113
112
  )
114
113
  if not result.data:
115
114
  raise OgcValidationException(
116
- f"Error validating the response of the enpoint "
115
+ f"Error validating the response of the endpoint "
117
116
  f"{openapi_request.path}: 'data' field of ResponseUnmarshalResult"
118
117
  f"object is empty",
119
118
  )
@@ -90,7 +90,6 @@ class StagingClient(OgcApiClient):
90
90
  return {parsed.hostname: super()._run_process(RESOURCE, staging_body)}
91
91
 
92
92
  # ----- Case 2: we directly load a STAC ItemCollection in the staging request body
93
-
94
93
  # If stac_input is a file, load this file to a dictionary
95
94
  if isinstance(stac_input, str):
96
95
  # If the input is a valid path to a json_file, load this file
@@ -21,14 +21,15 @@ import logging
21
21
  import re
22
22
  from collections.abc import Iterator
23
23
 
24
- import pystac
25
24
  from pystac import Collection, Item, Link, RelType
25
+ from pystac_client import Client
26
26
  from pystac_client.collection_client import CollectionClient
27
27
  from pystac_client.item_search import ItemSearch
28
- from requests import Response
28
+ from requests import HTTPError, Response
29
29
 
30
30
  from rs_client.rs_client import TIMEOUT
31
31
  from rs_client.stac.stac_base import StacBase
32
+ from rs_common import utils
32
33
  from rs_common.utils import get_href_service
33
34
 
34
35
 
@@ -108,6 +109,7 @@ class CatalogClient(StacBase): # type: ignore # pylint: disable=too-many-ancest
108
109
  ##############
109
110
  # Properties #
110
111
  ##############
112
+
111
113
  @property
112
114
  def href_service(self) -> str:
113
115
  """
@@ -149,6 +151,48 @@ class CatalogClient(StacBase): # type: ignore # pylint: disable=too-many-ancest
149
151
  concat_char = ":"
150
152
  return f"{owner_id or self.owner_id}{concat_char}{collection_id}"
151
153
 
154
+ #####################
155
+ # Utility functions #
156
+ #####################
157
+
158
+ def raise_for_status(self, response: Response, ignore: list[int] | None = None):
159
+ """
160
+ Raises :class:`HTTPError`, if one occurred.
161
+
162
+ Args:
163
+ response: HTTP response
164
+ ignore: ignore error its status code is in this list
165
+ """
166
+ if ignore and (response.status_code in ignore):
167
+ return
168
+ try:
169
+ response.raise_for_status()
170
+ except HTTPError as error:
171
+ message = f"{error.args[0]}\nDetail: {utils.read_response_error(response)}"
172
+ raise HTTPError(message, response=response) # pylint: disable=raise-missing-from
173
+
174
+ def clear_collection_cache(self):
175
+ """Clear the lru_caches because they still contains the old collection."""
176
+ StacBase.get_collection.cache_clear() # pylint: disable=no-member
177
+ Client.get_collection.cache_clear() # pylint: disable=no-member
178
+
179
+ def process_response(self, response: Response, raise_for_status: bool, ignore: list[int] | None = None) -> Response:
180
+ """
181
+ Process the HTTP response.
182
+
183
+ Args:
184
+ response: The HTTP response to process.
185
+ raise_for_status: If True, raise an error for HTTP errors.
186
+ ignore: A list of status codes to ignore.
187
+
188
+ Returns:
189
+ The processed HTTP response.
190
+ """
191
+ if raise_for_status:
192
+ self.raise_for_status(response, ignore)
193
+ self.clear_collection_cache()
194
+ return response
195
+
152
196
  ################################
153
197
  # Specific STAC implementation #
154
198
  ################################
@@ -182,9 +226,10 @@ class CatalogClient(StacBase): # type: ignore # pylint: disable=too-many-ancest
182
226
  ) -> ItemSearch | None:
183
227
  """Search items inside a specific collection."""
184
228
 
185
- kwargs["collections"] = [
186
- self.full_collection_id(kwargs["owner_id"], collection, "_") for collection in kwargs["collections"]
187
- ] # type: ignore
229
+ if "collections" in kwargs:
230
+ kwargs["collections"] = [
231
+ self.full_collection_id(kwargs.get("owner_id"), collection, "_") for collection in kwargs["collections"]
232
+ ] # type: ignore
188
233
  return super().search(**kwargs) # type: ignore
189
234
 
190
235
  # end of STAC read opperations
@@ -192,8 +237,10 @@ class CatalogClient(StacBase): # type: ignore # pylint: disable=too-many-ancest
192
237
  # STAC write opperations. These can't be done with pystac_client
193
238
  # - add_collection
194
239
  # - remove_collection
240
+ # - update_collection
195
241
  # - add_item
196
242
  # - remove_item
243
+ # - update_item
197
244
 
198
245
  def add_collection(
199
246
  self,
@@ -201,6 +248,7 @@ class CatalogClient(StacBase): # type: ignore # pylint: disable=too-many-ancest
201
248
  add_public_license: bool = True,
202
249
  owner_id: str | None = None,
203
250
  timeout: int = TIMEOUT,
251
+ raise_for_status: bool = True,
204
252
  ) -> Response:
205
253
  """Update the collection links, then post the collection into the catalog.
206
254
 
@@ -209,9 +257,13 @@ class CatalogClient(StacBase): # type: ignore # pylint: disable=too-many-ancest
209
257
  add_public_license (bool): If True, add a public domain license field and link.
210
258
  owner_id (str, optional): Collection owner ID. If missing, we use self.owner_id.
211
259
  timeout (int): The timeout duration for the HTTP request.
260
+ raise_for_status (bool): If True, raise HTTPError in case of server error.
212
261
 
213
262
  Returns:
214
263
  JSONResponse (json): The response of the request.
264
+
265
+ Raises:
266
+ HTTPError in case of server error.
215
267
  """
216
268
 
217
269
  full_owner_id = owner_id or self.owner_id
@@ -240,9 +292,6 @@ class CatalogClient(StacBase): # type: ignore # pylint: disable=too-many-ancest
240
292
  ),
241
293
  )
242
294
 
243
- # Update the links
244
- self.ps_client.add_child(collection)
245
-
246
295
  # Restore the short collection_id at the root of the collection
247
296
  collection.id = short_collection_id
248
297
 
@@ -250,18 +299,20 @@ class CatalogClient(StacBase): # type: ignore # pylint: disable=too-many-ancest
250
299
  collection.validate_all()
251
300
 
252
301
  # Post the collection to the catalog
253
- return self.http_session.post(
302
+ response = self.http_session.post(
254
303
  f"{self.href_service}/catalog/collections",
255
304
  json=collection.to_dict(),
256
305
  **self.apikey_headers,
257
306
  timeout=timeout,
258
307
  )
308
+ return self.process_response(response, raise_for_status)
259
309
 
260
310
  def remove_collection(
261
311
  self,
262
312
  collection_id: str,
263
313
  owner_id: str | None = None,
264
314
  timeout: int = TIMEOUT,
315
+ raise_for_status: bool = True,
265
316
  ) -> Response:
266
317
  """Remove/delete a collection from the catalog.
267
318
 
@@ -269,31 +320,65 @@ class CatalogClient(StacBase): # type: ignore # pylint: disable=too-many-ancest
269
320
  collection_id (str): The collection id.
270
321
  owner_id (str, optional): Collection owner ID. If missing, we use self.owner_id.
271
322
  timeout (int): The timeout duration for the HTTP request.
323
+ raise_for_status (bool): If True, raise HTTPError in case of server error.
272
324
 
273
325
  Returns:
274
- JSONResponse: The response of the request.
326
+ JSONResponse (json): The response of the request.
327
+
328
+ Raises:
329
+ HTTPError in case of server error.
275
330
  """
276
331
  # owner_id:collection_id
277
332
  full_collection_id = self.full_collection_id(owner_id, collection_id)
278
333
 
279
- # Remove the collection from the "child" links of the local catalog instance
280
- collection_link = f"{self.ps_client.self_href.rstrip('/')}/collections/{full_collection_id}"
281
- self.ps_client.links = [
282
- link
283
- for link in self.ps_client.links
284
- if not ((link.rel == pystac.RelType.CHILD) and (link.href == collection_link))
285
- ]
334
+ # Remove the collection from the server catalog
335
+ response = self.http_session.delete(
336
+ f"{self.href_service}/catalog/collections/{full_collection_id}",
337
+ **self.apikey_headers,
338
+ timeout=timeout,
339
+ )
340
+ return self.process_response(response, raise_for_status, ignore=[404])
341
+
342
+ def update_collection(
343
+ self,
344
+ collection: Collection | CollectionClient | dict,
345
+ timeout: int = TIMEOUT,
346
+ raise_for_status: bool = True,
347
+ ) -> Response:
348
+ """Put/update a collection in the catalog.
349
+
350
+ Args:
351
+ collection: The collection contents.
352
+ timeout (int): The timeout duration for the HTTP request.
353
+ raise_for_status (bool): If True, raise HTTPError in case of server error.
286
354
 
287
- # We need to clear the cache for this and parent "get_collection" methods
288
- # because their returned value must be updated.
289
- self.ps_client.get_collection.cache_clear()
355
+ Returns:
356
+ JSONResponse (json): The response of the request.
290
357
 
291
- # Remove the collection from the server catalog
292
- return self.http_session.delete(
358
+ Raises:
359
+ HTTPError in case of server error.
360
+ """
361
+
362
+ # Convert to dict
363
+ col_dict: dict = collection if isinstance(collection, dict) else collection.to_dict()
364
+
365
+ # Get the collection owner_id and remove it from the collection id
366
+ # which is like <owner_id>_<short_collection_id>
367
+ owner_id = col_dict["owner"]
368
+ collection_id = col_dict["id"].removeprefix(f"{owner_id}_")
369
+ col_dict["id"] = collection_id
370
+
371
+ # owner_id:collection_id
372
+ full_collection_id = self.full_collection_id(owner_id, collection_id)
373
+
374
+ # Update the collection in the server catalog
375
+ response = self.http_session.put(
293
376
  f"{self.href_service}/catalog/collections/{full_collection_id}",
377
+ json=col_dict,
294
378
  **self.apikey_headers,
295
379
  timeout=timeout,
296
380
  )
381
+ return self.process_response(response, raise_for_status)
297
382
 
298
383
  def add_item( # type: ignore # pylint: disable=arguments-renamed
299
384
  self,
@@ -301,6 +386,7 @@ class CatalogClient(StacBase): # type: ignore # pylint: disable=too-many-ancest
301
386
  item: Item,
302
387
  owner_id: str | None = None,
303
388
  timeout: int = TIMEOUT,
389
+ raise_for_status: bool = True,
304
390
  ) -> Response:
305
391
  """Update the item links, then post the item into the catalog.
306
392
 
@@ -309,9 +395,13 @@ class CatalogClient(StacBase): # type: ignore # pylint: disable=too-many-ancest
309
395
  item (Item): STAC item to update and post
310
396
  owner_id (str, optional): Collection owner ID. If missing, we use self.owner_id.
311
397
  timeout (int): The timeout duration for the HTTP request.
398
+ raise_for_status (bool): If True, raise HTTPError in case of server error.
312
399
 
313
400
  Returns:
314
- JSONResponse: The response of the request.
401
+ JSONResponse (json): The response of the request.
402
+
403
+ Raises:
404
+ HTTPError in case of server error.
315
405
  """
316
406
  # owner_id:collection_id
317
407
  full_collection_id = self.full_collection_id(owner_id, collection_id)
@@ -319,19 +409,14 @@ class CatalogClient(StacBase): # type: ignore # pylint: disable=too-many-ancest
319
409
  # Check that the item is compliant to STAC
320
410
  item.validate()
321
411
 
322
- # Get the collection from the catalog
323
- collection = self.get_collection(collection_id, owner_id)
324
-
325
- # Update the item contents
326
- collection.add_item(item) # type: ignore
327
-
328
412
  # Post the item to the catalog
329
- return self.http_session.post(
413
+ response = self.http_session.post(
330
414
  f"{self.href_service}/catalog/collections/{full_collection_id}/items",
331
415
  json=item.to_dict(),
332
416
  **self.apikey_headers,
333
417
  timeout=timeout,
334
418
  )
419
+ return self.process_response(response, raise_for_status)
335
420
 
336
421
  def remove_item( # type: ignore # pylint: disable=arguments-differ
337
422
  self,
@@ -339,6 +424,7 @@ class CatalogClient(StacBase): # type: ignore # pylint: disable=too-many-ancest
339
424
  item_id: str,
340
425
  owner_id: str | None = None,
341
426
  timeout: int = TIMEOUT,
427
+ raise_for_status: bool = True,
342
428
  ) -> Response:
343
429
  """Remove/delete an item from a collection.
344
430
 
@@ -347,18 +433,64 @@ class CatalogClient(StacBase): # type: ignore # pylint: disable=too-many-ancest
347
433
  item_id (str): The item id.
348
434
  owner_id (str, optional): Collection owner ID. If missing, we use self.owner_id.
349
435
  timeout (int): The timeout duration for the HTTP request.
436
+ raise_for_status (bool): If True, raise HTTPError in case of server error.
350
437
 
351
438
  Returns:
352
- JSONResponse: The response of the request.
439
+ JSONResponse (json): The response of the request.
440
+
441
+ Raises:
442
+ HTTPError in case of server error.
353
443
  """
354
444
  # owner_id:collection_id
355
445
  full_collection_id = self.full_collection_id(owner_id, collection_id)
356
446
 
357
447
  # Remove the collection from the server catalog
358
- return self.http_session.delete(
448
+ response = self.http_session.delete(
359
449
  f"{self.href_service}/catalog/collections/{full_collection_id}/items/{item_id}",
360
450
  **self.apikey_headers,
361
451
  timeout=timeout,
362
452
  )
453
+ return self.process_response(response, raise_for_status)
454
+
455
+ def update_item(
456
+ self,
457
+ item: Item | dict,
458
+ timeout: int = TIMEOUT,
459
+ raise_for_status: bool = True,
460
+ ) -> Response:
461
+ """Put/update an item in the catalog.
462
+
463
+ Args:
464
+ item: The item contents.
465
+ timeout (int): The timeout duration for the HTTP request.
466
+ raise_for_status (bool): If True, raise HTTPError in case of server error.
467
+
468
+ Returns:
469
+ JSONResponse (json): The response of the request.
470
+
471
+ Raises:
472
+ HTTPError in case of server error.
473
+ """
474
+
475
+ # Convert to dict
476
+ item_dict: dict = item if isinstance(item, dict) else item.to_dict()
477
+
478
+ # Get the collection owner_id and remove it from the collection id
479
+ # which is like <owner_id>_<short_collection_id>
480
+ owner_id = item_dict["properties"]["owner"]
481
+ collection_id = item_dict["collection"].removeprefix(f"{owner_id}_")
482
+ item_dict["collection"] = collection_id
483
+
484
+ # owner_id:collection_id
485
+ full_collection_id = self.full_collection_id(owner_id, collection_id)
486
+
487
+ # Update the item in the server catalog
488
+ response = self.http_session.put(
489
+ f"{self.href_service}/catalog/collections/{full_collection_id}/items/{item_dict['id']}",
490
+ json=item_dict,
491
+ **self.apikey_headers,
492
+ timeout=timeout,
493
+ )
494
+ return self.process_response(response, raise_for_status)
363
495
 
364
496
  # end of STAC write opperations
@@ -28,7 +28,6 @@ from collections.abc import Callable
28
28
  from pathlib import Path
29
29
  from typing import Any
30
30
 
31
- from botocore.utils import calculate_md5
32
31
  from fastapi.concurrency import run_in_threadpool
33
32
  from prefect.blocks.system import Secret
34
33
  from prefect.client.orchestration import get_client
@@ -145,6 +144,11 @@ async def init_prefect_blocks():
145
144
  "S3_ENDPOINT": os.environ["S3_ENDPOINT"],
146
145
  "LOCAL_DASK_USERNAME": os.environ["LOCAL_DASK_USERNAME"],
147
146
  "LOCAL_DASK_PASSWORD": os.environ["LOCAL_DASK_PASSWORD"],
147
+ "POSTGRES_USER": os.environ["POSTGRES_USER"],
148
+ "POSTGRES_PASSWORD": os.environ["POSTGRES_PASSWORD"],
149
+ "POSTGRES_PORT": os.environ["POSTGRES_PORT"],
150
+ "POSTGRES_PI_DB": os.environ["POSTGRES_PI_DB"],
151
+ "POSTGRES_HOST": os.environ["POSTGRES_HOST"],
148
152
  },
149
153
  ).save(BLOCK_NAME_ENV_GLOBAL, overwrite=True)
150
154
 
@@ -373,7 +377,7 @@ async def s3_download_dir(
373
377
 
374
378
 
375
379
  def s3_delete(s3_prefix: str, log: bool = False):
376
- """Remove all files from S3 bucket with the given prefix, using low-level client and Content-MD5 header."""
380
+ """Remove all files from S3 bucket with the given prefix, using low-level client."""
377
381
  s3_bucket, prefix = get_s3_bucket(s3_prefix)
378
382
  objects = s3_bucket._get_bucket_resource().objects # pylint: disable=protected-access
379
383
 
@@ -391,19 +395,7 @@ def s3_delete(s3_prefix: str, log: bool = False):
391
395
  return
392
396
 
393
397
  s3_client = s3_bucket._get_s3_client() # pylint: disable=protected-access
394
-
395
- # Not for the local minio bucket
396
398
  endpoint_url = s3_bucket.credentials.aws_client_parameters.endpoint_url
397
- if not any(provider in endpoint_url for provider in ["localhost", "minio"]):
398
-
399
- # Hook to compute Content-MD5 from actual serialized body
400
- def inject_md5_on_real_payload(request, **_):
401
- request.headers["Content-MD5"] = calculate_md5(request.body)
402
-
403
- s3_client.meta.events.register(
404
- "before-sign.s3.DeleteObjects",
405
- inject_md5_on_real_payload,
406
- )
407
399
 
408
400
  if log:
409
401
  keys = [f"s3://{s3_bucket.bucket_name}/{o.get('Key')}" for o in objects_to_delete]
@@ -412,7 +404,7 @@ def s3_delete(s3_prefix: str, log: bool = False):
412
404
  )
413
405
 
414
406
  # Split the list of objects to delete
415
- chunk_size = 100
407
+ chunk_size = 1000
416
408
  for i in range(0, len(objects_to_delete), chunk_size):
417
409
  s3_client.delete_objects(
418
410
  Bucket=s3_bucket.bucket_name,
@@ -32,19 +32,19 @@ class AuthInfo:
32
32
  apikey_config: dict
33
33
 
34
34
 
35
- def read_response_error(response):
35
+ def read_response_error(response) -> str:
36
36
  """Read and return an HTTP response error detail."""
37
37
 
38
38
  # Try to read the response detail or error
39
39
  try:
40
40
  json = response.json()
41
- detail = json.get("detail") or json.get("description") or json["error"]
41
+ if isinstance(json, str):
42
+ return json
43
+ return json.get("detail") or json.get("description") or json["error"]
42
44
 
43
45
  # If this fail, get the full response content
44
46
  except Exception: # pylint: disable=broad-exception-caught
45
- detail = response.content.decode("utf-8", errors="ignore")
46
-
47
- return detail
47
+ return response.content.decode("utf-8", errors="ignore")
48
48
 
49
49
 
50
50
  def get_href_service(rs_server_href, env_var) -> str:
@@ -177,8 +177,8 @@ async def write_payload(
177
177
  raise ValueError(f"No assets in item {item.id}")
178
178
 
179
179
  first_asset = list(item.assets.items())[0][1]
180
- logger.info(f"first_asset = {first_asset}")
181
- full_s3_href = first_asset.extra_fields.get("alternate", {}).get("s3", {}).get("href")
180
+ logger.debug(f"rs-client-libs: first_asset = {first_asset}")
181
+ full_s3_href = first_asset.href
182
182
  if not full_s3_href:
183
183
  raise ValueError(f"S3 HREF not found in extra fields for item {item.id}")
184
184
 
@@ -0,0 +1,140 @@
1
+ # Copyright 2025 CS Group
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """Create the database used in performance indicator"""
16
+
17
+ import os
18
+
19
+ from prefect import flow, get_run_logger, task
20
+ from sqlalchemy import create_engine
21
+ from sqlalchemy.orm import sessionmaker
22
+
23
+ from rs_workflows.flow_utils import FlowEnv, FlowEnvArgs
24
+ from rs_workflows.pi_db_models import Base, PiCategory
25
+
26
+ PI_CATEGORY_DATA = [
27
+ ("S1", "L0-SEG-NRT", "Segments EW, IW, SM with timeliness NRT & PT", 1800),
28
+ ("S1", "L0-HKGP", "HKTM & GPS", 1800),
29
+ ("S1", "L0-WV", "Slices L0 WV", 3600),
30
+ ("S1", "L0-NRT", "Slices L0 EW, IW, SM with timeliness NRT & PT", 5400),
31
+ ("S1", "L12-NRT", "Slices L1 and L2 EW, IW, SM with timeliness NRT & PT", 10800),
32
+ ("S1", "L2-WV", "Slices L2 Wave", 10800),
33
+ ("S1", "L12-NTC", "Slices L1 and L2 EW, IW, SM with timeliness NTC", 21600),
34
+ ("S2", "L0-HKAN", "HKTM & SADATA", 1800),
35
+ ("S2", "L1C", "Level-1C Datastrip, Tile, and True Colour Image", 4500),
36
+ ("S2", "L2A", "Level-2A Datastrip, Tile, and True Colour Image", 7200),
37
+ ("S3", "HKAN", "TM_0_NAT___, TM_0_HKM___, TM_0_HKM2__", 1800),
38
+ ("S3", "NRT", "All NRT products", 3600),
39
+ ("S3", "STC1", "STC for SY_2_SYN and SY_2_VGP", 72000),
40
+ ("S3", "STC2", "STC for SY_2_VG1, SY_2_VG10 and SRAL/MWR", 158400),
41
+ ("S3", "NTC1", "NTC for all optical instruments : OLCI and SLSTR", 252000),
42
+ ("S3", "NTC2", "NTC for SRAL and MWR", 2505600),
43
+ ("S1", "None", "S1 product outside any PI", 0),
44
+ ("S2", "None", "S2 product outside any PI", 0),
45
+ ("S3", "None", "S3 product outside any PI", 0),
46
+ ]
47
+
48
+
49
+ @task
50
+ def create_schema(db_url: str):
51
+ """
52
+ Creates all database tables defined in the pi_db_models.
53
+
54
+ This task initializes the database schema for the Performance Indicator (PI) database
55
+ using the provided SQLAlchemy engine.
56
+
57
+ Args:
58
+ engine (sqlalchemy.engine.Engine): SQLAlchemy database engine connected to the target database.
59
+ """
60
+ logger = get_run_logger()
61
+ logger.info(f"Received: {db_url}")
62
+ engine = create_engine(db_url)
63
+ logger.info("Call the create_all")
64
+ Base.metadata.create_all(engine)
65
+
66
+
67
+ @task
68
+ def insert_pi_categories(db_url: str):
69
+ """
70
+ Inserts default Performance Indicator (PI) categories into the database if none exist.
71
+
72
+ This task checks whether the `pi_category` table is empty and, if so, inserts predefined
73
+ categories from `PI_CATEGORY_DATA`.
74
+
75
+ Args:
76
+ engine (sqlalchemy.engine.Engine): SQLAlchemy database engine connected to the target database.
77
+
78
+ Notes:
79
+ - If categories already exist, no action is taken.
80
+ - Commits the transaction only if new data is inserted.
81
+ """
82
+ engine = create_engine(db_url)
83
+ own_session_maker = sessionmaker(bind=engine)
84
+ session = own_session_maker()
85
+ try:
86
+ if session.query(PiCategory).count() == 0:
87
+ for mission, name, desc, max_delay in PI_CATEGORY_DATA:
88
+ session.add(PiCategory(mission=mission, name=name, description=desc, max_delay_seconds=max_delay))
89
+ session.commit()
90
+ finally:
91
+ session.close()
92
+
93
+
94
+ @flow(name="PI db init")
95
+ async def init_pi_database(env: FlowEnvArgs):
96
+ """
97
+ Initializes the Performance Indicator (PI) database (named `performance`) schema and populates default categories.
98
+
99
+ This Prefect flow:
100
+ - Creates all required tables for the PI database.
101
+ - Inserts default PI categories if none exist.
102
+
103
+ Args:
104
+ env (FlowEnvArgs): Prefect flow environment configuration, including runtime context variables.
105
+
106
+ Environment Variables Required:
107
+ POSTGRES_USER (str): PostgreSQL username.
108
+ POSTGRES_PASSWORD (str): PostgreSQL password.
109
+ POSTGRES_HOST (str): PostgreSQL host address.
110
+ POSTGRES_PORT (str): PostgreSQL port.
111
+ POSTGRES_PI_DB (str): Name of the Performance Indicator database.
112
+
113
+ Flow Steps:
114
+ 1. Initialize flow environment and tracing span.
115
+ 2. Build database connection URL.
116
+ 3. Create database schema via `create_schema` task.
117
+ 4. Insert default PI categories via `insert_pi_categories` task.
118
+ """
119
+ logger = get_run_logger()
120
+
121
+ # Init flow environment and opentelemetry span
122
+ flow_env = FlowEnv(env)
123
+ with flow_env.start_span(__name__, "init-pi-database"):
124
+
125
+ logger.info("Starting the initialization of the tables for the performance indicator database...")
126
+
127
+ db_url = (
128
+ f"postgresql+psycopg2://{os.environ['POSTGRES_USER']}:"
129
+ f"{os.environ['POSTGRES_PASSWORD']}@{os.environ['POSTGRES_HOST']}:"
130
+ f"{os.environ['POSTGRES_PORT']}/{os.environ['POSTGRES_PI_DB']}"
131
+ )
132
+ # Prefect tasks attempt to serialize inputs (for caching, retries, mapping, etc.) and
133
+ # also generate a cache key by hashing them. SQLAlchemy Engine contains locks
134
+ # and connection pools (thread.RLock, weakref.ReferenceType, etc.), which are not serializable.
135
+ # That's why instead of passing the engine object, pass only the db_url, a string, which is serializable.
136
+ # Each task can then create its own engine locally.
137
+ create_schema(db_url)
138
+ insert_pi_categories(db_url)
139
+
140
+ logger.info("The initialization of the tables for the performance indicator database finished")
@@ -17,8 +17,7 @@
17
17
  import datetime
18
18
  from pathlib import Path
19
19
 
20
- from prefect import flow, get_run_logger, task
21
- from pystac import ItemCollection
20
+ from prefect import flow
22
21
 
23
22
  from rs_workflows import auxip_flow, cadip_flow, catalog_flow
24
23
  from rs_workflows.dpr_flow import (
@@ -210,13 +209,18 @@ async def on_demand_auxip_staging(
210
209
  if isinstance(end_datetime, datetime.datetime):
211
210
  end_datetime = end_datetime.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
212
211
 
213
- # CQL2 ValCover filter. The product:type filter is not added here
214
- # because for some reason composite filters with "and" don't work
212
+ # CQL2 filter: we use a filter combining a ValCover filter and a product type filter
215
213
  cql2_filter = {
216
- "op": "t_contains",
214
+ "op": "and",
217
215
  "args": [
218
- {"interval": [{"property": "start_datetime"}, {"property": "end_datetime"}]},
219
- {"interval": [start_datetime, end_datetime]},
216
+ {"op": "=", "args": [{"property": "product:type"}, product_type]},
217
+ {
218
+ "op": "t_contains",
219
+ "args": [
220
+ {"interval": [{"property": "start_datetime"}, {"property": "end_datetime"}]},
221
+ {"interval": [start_datetime, end_datetime]},
222
+ ],
223
+ },
220
224
  ],
221
225
  }
222
226
 
@@ -227,31 +231,13 @@ async def on_demand_auxip_staging(
227
231
  error_if_empty=False,
228
232
  )
229
233
 
230
- # Filtering Auxip items to only keep the ones with the correct product type
231
- items_to_stage = filter_product_type.submit(auxip_items, product_type)
232
-
233
234
  # Stage Auxip items.
234
235
  staged = staging_task_auxip.submit(
235
236
  flow_env.serialize(),
236
- items_to_stage,
237
+ auxip_items,
237
238
  catalog_collection_identifier,
238
239
  )
239
240
 
240
241
  # Wait for last task to end.
241
242
  # NOTE: use .result() and not .wait() to unwrap and propagate exceptions, if any.
242
243
  staged.result() # type: ignore[unused-coroutine]
243
-
244
-
245
- @task(name="Filter product type")
246
- async def filter_product_type(auxip_items: ItemCollection | None, product_type: str) -> ItemCollection:
247
- """Filter Auxip items to only keep the ones with the correct product type"""
248
- logger = get_run_logger()
249
- items_to_stage = []
250
-
251
- if auxip_items:
252
- for item in auxip_items: # type: ignore[attr-defined]
253
- if "product:type" in item.properties.keys() and item.properties["product:type"] == product_type:
254
- items_to_stage.append(item)
255
-
256
- logger.info(f"Filtering search results: {len(items_to_stage)} item(s) found of product type {product_type}")
257
- return ItemCollection(items_to_stage)
@@ -0,0 +1,156 @@
1
+ # Copyright 2025 CS Group
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """Tables for the performance indicator database"""
16
+
17
+ from sqlalchemy import (
18
+ TIMESTAMP,
19
+ BigInteger,
20
+ Boolean,
21
+ CheckConstraint,
22
+ Column,
23
+ ForeignKey,
24
+ Integer,
25
+ Text,
26
+ )
27
+ from sqlalchemy.dialects.postgresql import JSONB, UUID
28
+ from sqlalchemy.orm import DeclarativeBase, relationship
29
+
30
+
31
+ class Base(DeclarativeBase): # pylint: disable=too-few-public-methods
32
+ """
33
+ Set Base as a type, thus mypy understands it. Otherwise, error from mypy
34
+ """
35
+
36
+ pass # pylint: disable=unnecessary-pass
37
+
38
+
39
+ class FlowRun(Base): # pylint: disable=too-few-public-methods
40
+ """
41
+ The table flow_run brings together Prefect flow general information.
42
+ """
43
+
44
+ __tablename__ = "flow_run"
45
+
46
+ id = Column(BigInteger, primary_key=True, autoincrement=True)
47
+ flow_type = Column(Text)
48
+ mission = Column(Text)
49
+ prefect_flow_id = Column(UUID(as_uuid=True))
50
+ prefect_flow_parent_id = Column(UUID(as_uuid=True))
51
+ dask_version = Column(Text)
52
+ python_version = Column(Text)
53
+ dpr_processor_name = Column(Text)
54
+ dpr_processor_version = Column(Text)
55
+ dpr_processor_unit = Column(Text)
56
+ dpr_processing_input_stac_items = Column(JSONB)
57
+ dpr_processing_start_datetime = Column(TIMESTAMP)
58
+ dpr_processing_stop_datetime = Column(TIMESTAMP)
59
+ dpr_processing_status = Column(Text)
60
+ excluded_from_pi = Column(Boolean, default=False)
61
+
62
+ products_expected = relationship("ProductExpected", back_populates="flow_run", cascade="all, delete")
63
+ products_realised = relationship("ProductRealised", back_populates="flow_run", cascade="all, delete")
64
+ products_missing = relationship("ProductMissing", back_populates="flow_run", cascade="all, delete")
65
+
66
+
67
+ class PiCategory(Base): # pylint: disable=too-few-public-methods
68
+ """
69
+ The table pi_category provides the list of PI per mission.
70
+ """
71
+
72
+ __tablename__ = "pi_category"
73
+
74
+ id = Column(Integer, primary_key=True, autoincrement=True)
75
+ mission = Column(Text)
76
+ name = Column(Text)
77
+ description = Column(Text)
78
+ max_delay_seconds = Column(BigInteger)
79
+
80
+ products_expected = relationship("ProductExpected", back_populates="pi_category")
81
+ products_realised = relationship("ProductRealised", back_populates="pi_category")
82
+ products_missing = relationship("ProductMissing", back_populates="pi_category")
83
+
84
+
85
+ class ProductExpected(Base): # pylint: disable=too-few-public-methods
86
+ """
87
+ This table provides all expected product from a single flow_run.
88
+ This is the difficult part of the PI computing: estimate the production.
89
+ The computing depends on mission, level and more generally on every STAC ITEM
90
+ provided as input to the DPR processor.
91
+ """
92
+
93
+ __tablename__ = "product_expected"
94
+
95
+ id = Column(BigInteger, primary_key=True, autoincrement=True)
96
+ flow_run_id = Column(BigInteger, ForeignKey("flow_run.id", ondelete="CASCADE", onupdate="CASCADE"))
97
+ pi_category_id = Column(Integer, ForeignKey("pi_category.id"))
98
+ eopf_type = Column(Text)
99
+ sensing_start_datetime = Column(TIMESTAMP)
100
+ min_count = Column(Integer)
101
+ max_count = Column(Integer)
102
+
103
+ __table_args__ = (
104
+ CheckConstraint("min_count >= 0"),
105
+ CheckConstraint("max_count >= min_count"),
106
+ )
107
+
108
+ flow_run = relationship("FlowRun", back_populates="products_expected")
109
+ pi_category = relationship("PiCategory", back_populates="products_expected")
110
+
111
+
112
+ class ProductRealised(Base): # pylint: disable=too-few-public-methods
113
+ """
114
+ The table product_realised provides the list of products generated by the DPR processor.
115
+ """
116
+
117
+ __tablename__ = "product_realised"
118
+
119
+ id = Column(BigInteger, primary_key=True, autoincrement=True)
120
+ flow_run_id = Column(BigInteger, ForeignKey("flow_run.id", ondelete="CASCADE", onupdate="CASCADE"))
121
+ pi_category_id = Column(Integer, ForeignKey("pi_category.id"))
122
+ eopf_type = Column(Text)
123
+ stac_item = Column(JSONB)
124
+ sensing_start_datetime = Column(TIMESTAMP)
125
+ origin_date = Column(TIMESTAMP)
126
+ catalog_stored_datetime = Column(TIMESTAMP)
127
+ unexpected = Column(Boolean, default=False)
128
+ on_time_0_day = Column(Boolean, default=False)
129
+ on_time_1_day = Column(Boolean, default=False)
130
+ on_time_2_day = Column(Boolean, default=False)
131
+ on_time_3_day = Column(Boolean, default=False)
132
+ on_time_7_day = Column(Boolean, default=False)
133
+
134
+ flow_run = relationship("FlowRun", back_populates="products_realised")
135
+ pi_category = relationship("PiCategory", back_populates="products_realised")
136
+
137
+
138
+ class ProductMissing(Base): # pylint: disable=too-few-public-methods
139
+ """
140
+ The table product_missing provides the list of missing_products.
141
+ The list is based on what has been estimated during the construction of the table product_estimated.
142
+ """
143
+
144
+ __tablename__ = "product_missing"
145
+
146
+ id = Column(BigInteger, primary_key=True, autoincrement=True)
147
+ flow_run_id = Column(BigInteger, ForeignKey("flow_run.id", ondelete="CASCADE", onupdate="CASCADE"))
148
+ pi_category_id = Column(Integer, ForeignKey("pi_category.id"))
149
+ eopf_type = Column(Text)
150
+ sensing_start_datetime = Column(TIMESTAMP)
151
+ count = Column(Integer)
152
+
153
+ __table_args__ = (CheckConstraint("count >= 0"),)
154
+
155
+ flow_run = relationship("FlowRun", back_populates="products_missing")
156
+ pi_category = relationship("PiCategory", back_populates="products_missing")
@@ -0,0 +1,102 @@
1
+ # Copyright 2025 CS Group
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """Module with task used to insert or update flow run table."""
16
+
17
+ import os
18
+ import sys
19
+ from importlib.metadata import version
20
+
21
+ from prefect import get_run_logger, runtime, task
22
+ from sqlalchemy import MetaData, Table, create_engine, select, update
23
+ from sqlalchemy.orm import sessionmaker
24
+
25
+
26
+ @task
27
+ def record_flow_run(start_date=None, stop_date=None, status=None):
28
+ """
29
+ start_date: UTC date and time of the DPR processing starts.
30
+ stop_date: UTC date and time of the DPR processing ends.
31
+ status: Can be : NULL, OK, NOK. Set to 'NULL' by default.
32
+ """
33
+ logger = get_run_logger()
34
+ logger.info("Inserting or updating a record in flow_run table")
35
+
36
+ db_url = (
37
+ f"postgresql+psycopg2://{os.environ['POSTGRES_USER']}:"
38
+ f"{os.environ['POSTGRES_PASSWORD']}@{os.environ['POSTGRES_HOST']}:"
39
+ f"{os.environ['POSTGRES_PORT']}/{os.environ['POSTGRES_PI_DB']}"
40
+ )
41
+ engine = create_engine(db_url, pool_pre_ping=True)
42
+ db = sessionmaker(bind=engine, autoflush=False, autocommit=False)()
43
+
44
+ try:
45
+ metadata = MetaData()
46
+ flow_run = Table("flow_run", metadata, autoload_with=engine)
47
+
48
+ prefect_flow_id = runtime.flow_run.id
49
+
50
+ # Check if record exists
51
+ existing = db.execute(select(flow_run.c.id).where(flow_run.c.prefect_flow_id == prefect_flow_id)).fetchone()
52
+
53
+ if not existing:
54
+ # Insert new record
55
+ values = {
56
+ "flow_type": runtime.flow_run.parameters.get("flow_run_type", "systematic"),
57
+ "mission": runtime.flow_run.parameters.get("mission", "null"),
58
+ "prefect_flow_id": prefect_flow_id,
59
+ "prefect_flow_parent_id": runtime.flow_run.parent_flow_run_id,
60
+ "dask_version": version("dask"),
61
+ "python_version": sys.version.split()[0],
62
+ "dpr_processor_name": runtime.flow_run.parameters.get("dpr_processor_name", "dpr_processor"),
63
+ "dpr_processor_version": runtime.flow_run.parameters.get(
64
+ "dpr_processor_version",
65
+ "dpr_processor_version",
66
+ ),
67
+ "dpr_processor_unit": runtime.flow_run.parameters.get("dpr_processor_unit", "dpr_processor_unit"),
68
+ "dpr_processing_input_stac_items": runtime.flow_run.parameters.get(
69
+ "dpr_processing_input_stac_items",
70
+ "dpr_processing_input_stac_items",
71
+ ),
72
+ "dpr_processing_start_datetime": start_date,
73
+ "dpr_processing_stop_datetime": stop_date,
74
+ "dpr_processing_status": status,
75
+ "excluded_from_pi": False,
76
+ }
77
+ db.execute(flow_run.insert().values(**values))
78
+ logger.info("Inserted new flow_run record")
79
+ else:
80
+ # Update only selected fields if provided
81
+ update_values = {}
82
+ if start_date is not None:
83
+ update_values["dpr_processing_start_datetime"] = start_date
84
+ if stop_date is not None:
85
+ update_values["dpr_processing_stop_datetime"] = stop_date
86
+ if status is not None:
87
+ update_values["dpr_processing_status"] = status
88
+
89
+ if update_values:
90
+ stmt = update(flow_run).where(flow_run.c.prefect_flow_id == prefect_flow_id).values(**update_values)
91
+ db.execute(stmt)
92
+ logger.info(f"Updated flow_run {prefect_flow_id} with {update_values}")
93
+
94
+ db.commit()
95
+ logger.info("Successfully committed transaction to flow_run!")
96
+
97
+ except Exception as e:
98
+ db.rollback()
99
+ logger.error(f"Failed to insert/update flow_run in task: {e}")
100
+ raise
101
+ finally:
102
+ db.close()
@@ -26,7 +26,7 @@ async def staging(
26
26
  env: FlowEnvArgs,
27
27
  stac_input: str | ItemCollection | dict, # warning: dict as last choice for prefect ui
28
28
  catalog_collection_identifier: str,
29
- timeout: int = 120,
29
+ timeout: int = 1200,
30
30
  poll_interval: int = 2,
31
31
  ):
32
32
  """