rs-client-libraries 1.0a2__tar.gz → 1.0a3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/PKG-INFO +6 -6
  2. {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/pyproject.toml +7 -7
  3. {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_client/__init__.py +1 -1
  4. {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_client/ogcapi/dpr_client.py +52 -10
  5. {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_client/ogcapi/ogcapi_client.py +2 -1
  6. {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_client/ogcapi/staging_client.py +7 -2
  7. {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_client/rs_client.py +7 -2
  8. {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_client/stac/catalog_client.py +2 -3
  9. {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_common/prefect_utils.py +49 -21
  10. {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_common/utils.py +43 -0
  11. rs_client_libraries-1.0a3/rs_workflows/auxip_flow.py +202 -0
  12. {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_workflows/catalog_flow.py +53 -2
  13. {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_workflows/dpr_flow.py +15 -9
  14. {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_workflows/flow_utils.py +91 -11
  15. rs_client_libraries-1.0a3/rs_workflows/on_demand_processing.py +242 -0
  16. rs_client_libraries-1.0a3/rs_workflows/payload_builder.py +310 -0
  17. {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_workflows/record_performance.py +259 -25
  18. {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_workflows/staging_flow.py +9 -16
  19. rs_client_libraries-1.0a2/rs_workflows/auxip_flow.py +0 -70
  20. rs_client_libraries-1.0a2/rs_workflows/on_demand_processing.py +0 -316
  21. {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/LICENSE +0 -0
  22. {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/README.adoc +0 -0
  23. {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_client/ogcapi/__init__.py +0 -0
  24. {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_client/stac/__init__.py +0 -0
  25. {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_client/stac/auxip_client.py +0 -0
  26. {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_client/stac/cadip_client.py +0 -0
  27. {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_client/stac/prip_client.py +0 -0
  28. {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_client/stac/stac_base.py +0 -0
  29. {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_common/__init__.py +0 -0
  30. {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_common/config.py +0 -0
  31. {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_common/init_opentelemetry.py +0 -0
  32. {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_common/logging.py +0 -0
  33. {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_workflows/__init__.py +0 -0
  34. {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_workflows/cadip_flow.py +0 -0
  35. {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_workflows/init_pi_db_flow.py +0 -0
  36. {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_workflows/pi_db_models.py +0 -0
  37. {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_workflows/prip_flow.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: rs-client-libraries
3
- Version: 1.0a2
3
+ Version: 1.0a3
4
4
  Summary: Copernicus Sentinel rs-client library and processing workflows.
5
5
  License: Apache-2.0
6
6
  License-File: LICENSE
@@ -21,13 +21,13 @@ Classifier: Programming Language :: Python :: 3.14
21
21
  Classifier: Topic :: Scientific/Engineering :: Image Processing
22
22
  Classifier: Topic :: Software Development :: Libraries :: Python Modules
23
23
  Requires-Dist: SQLAlchemy (>=2.0.37,<3.0.0)
24
- Requires-Dist: cachetools (>=5.5.2,<6.0.0)
24
+ Requires-Dist: cachetools (>=5.5.2)
25
25
  Requires-Dist: click (==8.1.8)
26
26
  Requires-Dist: openapi-core (>=0.19.5,<0.20.0)
27
- Requires-Dist: opentelemetry-distro (>=0.58b0,<0.59)
28
- Requires-Dist: opentelemetry-exporter-otlp (>=1.37.0,<2.0.0)
29
- Requires-Dist: opentelemetry-instrumentation-aws-lambda (>=0.58b0,<0.59)
30
- Requires-Dist: opentelemetry-test-utils (>=0.58b0,<0.59)
27
+ Requires-Dist: opentelemetry-distro (>=0.59b0,<0.60)
28
+ Requires-Dist: opentelemetry-exporter-otlp (>=1.38.0,<2.0.0)
29
+ Requires-Dist: opentelemetry-instrumentation-aws-lambda (>=0.59b0,<0.60)
30
+ Requires-Dist: opentelemetry-test-utils (>=0.59b0,<0.60)
31
31
  Requires-Dist: prefect[aws] (==3.2.13)
32
32
  Requires-Dist: protobuf (>=5.29.5)
33
33
  Requires-Dist: pystac (>=1.13.0,<2.0.0)
@@ -18,7 +18,7 @@ dynamic = ["version"] # see: https://github.com/mtkennerly/poetry-dynamic-versio
18
18
 
19
19
  [tool.poetry]
20
20
  name = "rs-client-libraries"
21
- version = "1.0a2"
21
+ version = "1.0a3"
22
22
  description = "Copernicus Sentinel rs-client library and processing workflows."
23
23
  license = "Apache-2.0"
24
24
  authors = ["ESA under contract with Airbus and CS <ops_coprs@airbus.com>"]
@@ -51,13 +51,13 @@ include = [
51
51
 
52
52
  [tool.poetry.dependencies]
53
53
  python = "^3.11"
54
- opentelemetry-distro = "^0.58b0" # then run 'poetry run opentelemetry-bootstrap -a install'
55
- opentelemetry-instrumentation-aws-lambda = "^0.58b0"
56
- opentelemetry-test-utils = "^0.58b0"
57
- opentelemetry-exporter-otlp = "^1.37.0"
54
+ opentelemetry-distro = "^0.59b0" # then run 'poetry run opentelemetry-bootstrap -a install'
55
+ opentelemetry-instrumentation-aws-lambda = "^0.59b0"
56
+ opentelemetry-test-utils = "^0.59b0"
57
+ opentelemetry-exporter-otlp = "^1.38.0"
58
58
  protobuf = ">=5.29.5"
59
59
  python-logging-loki = "^0.3.1" # to avoid otl error: No module named 'logging_loki'
60
- cachetools = "^5.5.2"
60
+ cachetools = ">=5.5.2"
61
61
  pystac = "^1.13.0"
62
62
  pystac-client = "^0.9.0"
63
63
  python-dateutil = "^2.9.0.post0"
@@ -90,7 +90,7 @@ sphinx-rtd-theme = "^3.0.2"
90
90
  flake8 = "^7.1.1"
91
91
  flake8-pyproject = "^1.2.3"
92
92
  responses = "^0.25.7"
93
- starlette = "^0.47.2"
93
+ starlette = "^0.48.0"
94
94
 
95
95
  [tool.poetry.group.demo.dependencies] # for hybrid mode
96
96
  jupyterlab = "^4.2.1"
@@ -15,4 +15,4 @@
15
15
  """rsclient main module."""
16
16
 
17
17
  # Set automatically by running `poetry dynamic-versioning`
18
- __version__ = "1.0a2"
18
+ __version__ = "1.0a3"
@@ -17,6 +17,7 @@
17
17
  import ast
18
18
  import os.path as osp
19
19
  import tempfile
20
+ from dataclasses import asdict, dataclass
20
21
  from enum import Enum
21
22
  from pathlib import Path
22
23
 
@@ -39,17 +40,33 @@ PATH_TO_YAML_OPENAPI = osp.realpath(
39
40
  )
40
41
 
41
42
 
42
- class DprProcess(str, Enum):
43
+ class DprProcessor(str, Enum):
43
44
  """DPR processor name"""
44
45
 
45
46
  # String value = resource name in the rs-dpr-service
46
- CONV_SAFE_ZARR = "conv_safe_zarr"
47
47
  MOCKUP = "mockup"
48
48
  S1L0 = "s1_l0"
49
49
  S3L0 = "s3_l0"
50
50
  S1ARD = "s1_ard"
51
51
 
52
52
 
53
+ @dataclass
54
+ class ClusterInfo:
55
+ """
56
+ Information to connect to a DPR Dask cluster.
57
+
58
+ Attributes:
59
+ jupyter_token: JupyterHub API token. Only used in cluster mode, not local mode.
60
+ cluster_label: Dask cluster label e.g. "dask-l0"
61
+ cluster_instance: Dask cluster instance ID (something like "dask-gateway.17e196069443463495547eb97f532834").
62
+ If instance is empty, the DPR processor will use the first cluster with the given label.
63
+ """
64
+
65
+ jupyter_token: str
66
+ cluster_label: str
67
+ cluster_instance: str | None = ""
68
+
69
+
53
70
  class DprClient(OgcApiClient):
54
71
  """Implement the OGC API client for 'DPR as a service'."""
55
72
 
@@ -74,9 +91,25 @@ class DprClient(OgcApiClient):
74
91
  """
75
92
  return get_href_service(self.rs_server_href, "RSPY_HOST_DPR_SERVICE")
76
93
 
94
+ def get_process( # type: ignore # pylint: disable=arguments-differ
95
+ self,
96
+ process_id: str,
97
+ cluster_info: ClusterInfo,
98
+ **kwargs,
99
+ ) -> dict:
100
+ """
101
+ Call parent method with additional HTTP Get parameters.
102
+
103
+ Args:
104
+ process_id (str): name of the resource
105
+ cluster_info: Information to connect to a DPR Dask cluster
106
+ """
107
+ return super().get_process(process_id, params=asdict(cluster_info), **kwargs)
108
+
77
109
  def run_process(
78
110
  self,
79
- process: DprProcess,
111
+ process: DprProcessor,
112
+ cluster_info: ClusterInfo,
80
113
  s3_config_dir: str,
81
114
  payload_subpath: str,
82
115
  s3_report_dir: str | None,
@@ -86,6 +119,7 @@ class DprClient(OgcApiClient):
86
119
 
87
120
  Args:
88
121
  process: DPR process
122
+ cluster_info: Information to connect to a DPR Dask cluster
89
123
  s3_config_dir: S3 bucket folder that contains the payload and configuration files to pass to the processor
90
124
  payload_subpath: Payload file path, relative to the config folder
91
125
  s3_report_dir: S3 bucket folder were the processor report files will be written (optional). All the eopf
@@ -97,7 +131,7 @@ class DprClient(OgcApiClient):
97
131
  (or None if endpoint fails) of the running job
98
132
  """
99
133
 
100
- use_mockup = process == DprProcess.MOCKUP
134
+ use_mockup = process == DprProcessor.MOCKUP
101
135
 
102
136
  # Data to pass to the real processor
103
137
  data = {}
@@ -125,21 +159,26 @@ class DprClient(OgcApiClient):
125
159
  # Add extra info
126
160
  data.update({"use_mockup": use_mockup})
127
161
 
162
+ # Add the cluster info
163
+ data.update(asdict(cluster_info))
164
+
128
165
  # Call the parent method
129
166
  return super()._run_process(process.value, data)
130
167
 
131
- def run_conv_safe_zarr(self, payload: dict):
168
+ def run_conv_safe_zarr(self, payload: dict, cluster_info: ClusterInfo):
132
169
  """Method to start the safe to zarr conversion process from rs-client -
133
170
  Call the endpoint /processes/conv_safe_zarr/execution
134
171
 
135
172
  Args:
136
173
  payload: Dictionary to pass to the processor,
174
+ cluster_info: Information to connect to a DPR Dask cluster
137
175
  containing input_safe_path - the s3 path of legacy product and
138
176
  output_zarr_dir_path - the s3 path for the new zarr
139
177
  Return:
140
178
  job_id (int, str): Returns the status code of the request + the identifier
141
179
  (or None if endpoint fails) of the running job
142
180
  """
181
+ payload.update(asdict(cluster_info)) # Add the cluster info to the payload
143
182
  return super()._run_process("conv_safe_zarr", payload)
144
183
 
145
184
  def wait_for_job(self, *args, **kwargs) -> list[dict]: # type: ignore
@@ -241,11 +280,14 @@ class DprClient(OgcApiClient):
241
280
  self.logger.info(f"Write empty file: {self.logger.level} {s3_empty_file!r}")
242
281
  await prefect_utils.s3_upload_empty_file(s3_empty_file)
243
282
 
244
- # Change the dask authentication for local mode
245
- cluster_config = payload["dask_context"]["cluster_config"]
246
- if self.local_mode:
247
- cluster_config["auth"] = cluster_config["auth_local_mode"]
248
- del cluster_config["auth_local_mode"]
283
+ # Change the dask authentication for local mode (used in old demos, could be removed)
284
+ try:
285
+ cluster_config = payload["dask_context"]["cluster_config"]
286
+ if self.local_mode:
287
+ cluster_config["auth"] = cluster_config["auth_local_mode"]
288
+ del cluster_config["auth_local_mode"]
289
+ except KeyError:
290
+ pass
249
291
 
250
292
  # yaml to str conversion
251
293
  contents = yaml.dump(payload, default_flow_style=False, sort_keys=False)
@@ -135,7 +135,7 @@ class OgcApiClient(RsClient):
135
135
  )
136
136
  return self.validate_and_unmarshal_response(response)
137
137
 
138
- def get_process(self, process_id: str) -> dict:
138
+ def get_process(self, process_id: str, **kwargs) -> dict:
139
139
  """
140
140
  Wrapper to get a specific process
141
141
  Args:
@@ -145,6 +145,7 @@ class OgcApiClient(RsClient):
145
145
  url=f"{self.href_service}/{self.endpoint_prefix}processes/{process_id}",
146
146
  timeout=TIMEOUT,
147
147
  **self.apikey_headers,
148
+ **kwargs,
148
149
  )
149
150
  return self.validate_and_unmarshal_response(response)
150
151
 
@@ -155,7 +155,7 @@ class StagingClient(OgcApiClient):
155
155
  logger=None,
156
156
  # timeout: int | float = math.inf,
157
157
  poll_interval: int = 2,
158
- ):
158
+ ) -> dict[str, dict]:
159
159
  """
160
160
  Wait for job to finish.
161
161
 
@@ -166,15 +166,20 @@ class StagingClient(OgcApiClient):
166
166
  NOTE: This argument has been disabled, see the comment in super().wait_for_job() function
167
167
  poll_interval: When to check again for job completion in seconds
168
168
 
169
+ Returns:
170
+ dict[str, dict]: Completed job status
171
+
169
172
  Raises:
170
173
  RuntimeError in case of error
171
174
  """
175
+ job_results: dict[str, dict] = {}
172
176
  # Call parent method for each hostname
173
177
  for hostname, job_status in all_job_status.items():
174
- super().wait_for_job(
178
+ job_results[hostname] = super().wait_for_job(
175
179
  job_status,
176
180
  logger,
177
181
  f"Staging from {hostname!r}",
178
182
  # timeout,
179
183
  poll_interval,
180
184
  )
185
+ return job_results
@@ -20,6 +20,7 @@ import sys
20
20
 
21
21
  import requests
22
22
  from cachetools import TTLCache, cached
23
+ from pystac_client.stac_api_io import Timeout
23
24
 
24
25
  from rs_common import utils
25
26
  from rs_common.logging import Logging
@@ -27,7 +28,7 @@ from rs_common.utils import AuthInfo
27
28
 
28
29
  APIKEY_HEADER = "x-api-key"
29
30
 
30
- # Timeout in seconds
31
+ # Default timeout in seconds
31
32
  TIMEOUT = 30
32
33
 
33
34
  # API Key Manager URL used to get an API Key information.
@@ -57,6 +58,7 @@ class RsClient: # pylint: disable=too-many-instance-attributes
57
58
  local_mode (bool): Indicates whether the client is running in local mode.
58
59
  apikey_headers (dict): API key headers for HTTP requests.
59
60
  http_session (requests.Session): HTTP session for handling requests.
61
+ timeout (Timeout): timeout in seconds when contacting API key manager. Default to 30s.
60
62
  """
61
63
 
62
64
  def __init__( # pylint: disable=too-many-branches, too-many-arguments, too-many-positional-arguments
@@ -65,6 +67,7 @@ class RsClient: # pylint: disable=too-many-instance-attributes
65
67
  rs_server_api_key: str | None = None,
66
68
  owner_id: str | None = None,
67
69
  logger: logging.Logger | None = None,
70
+ timeout: Timeout = TIMEOUT,
68
71
  ):
69
72
  """
70
73
  Initializes an RsClient instance.
@@ -109,6 +112,8 @@ class RsClient: # pylint: disable=too-many-instance-attributes
109
112
  if self.rs_server_oauth2_cookie:
110
113
  self.http_session.cookies.set("session", self.rs_server_oauth2_cookie)
111
114
 
115
+ self.timeout = timeout
116
+
112
117
  def log_and_raise(self, message: str, original: Exception):
113
118
  """
114
119
  Logs an error message and raises a RuntimeError.
@@ -167,7 +172,7 @@ class RsClient: # pylint: disable=too-many-instance-attributes
167
172
 
168
173
  # Request the API key manager, pass user-defined api key in http header
169
174
  self.logger.debug("Call the API key manager")
170
- response = self.http_session.get(RSPY_UAC_CHECK_URL, **self.apikey_headers, timeout=TIMEOUT)
175
+ response = self.http_session.get(RSPY_UAC_CHECK_URL, **self.apikey_headers, timeout=self.timeout)
171
176
  if not response.ok:
172
177
  raise RuntimeError(
173
178
  f"API key manager status code {response.status_code}: {utils.read_response_error(response)}",
@@ -21,10 +21,9 @@ import logging
21
21
  import re
22
22
  from collections.abc import Iterator
23
23
 
24
- from pystac import Collection, Item, Link, RelType
24
+ from pystac import Collection, Item, ItemCollection, Link, RelType
25
25
  from pystac_client import Client
26
26
  from pystac_client.collection_client import CollectionClient
27
- from pystac_client.item_search import ItemSearch
28
27
  from requests import HTTPError, Response
29
28
 
30
29
  from rs_client.rs_client import TIMEOUT
@@ -223,7 +222,7 @@ class CatalogClient(StacBase): # type: ignore # pylint: disable=too-many-ancest
223
222
  def search( # type: ignore # pylint: disable=too-many-arguments, arguments-differ
224
223
  self,
225
224
  **kwargs,
226
- ) -> ItemSearch | None:
225
+ ) -> ItemCollection | None:
227
226
  """Search items inside a specific collection."""
228
227
 
229
228
  if "collections" in kwargs:
@@ -35,6 +35,7 @@ from prefect.exceptions import ObjectNotFound
35
35
  from prefect.utilities.asyncutils import sync_compatible
36
36
  from prefect_aws import AwsCredentials, S3Bucket
37
37
 
38
+ from rs_common.logging import Logging
38
39
  from rs_common.utils import env_bool
39
40
 
40
41
  # In local mode, all your services are running locally.
@@ -49,11 +50,14 @@ owner_id: str | None = ""
49
50
  # Prefect block names
50
51
  BLOCK_NAME_ENV_GLOBAL: str = "env-vars"
51
52
  BLOCK_NAME_ENV_USER: str = "env-vars-{0}" # env variables for the user/owner_id
52
- BLOCK_NAME_SHARE_BUCKET: str = "share-bucket"
53
+ BLOCK_NAME_SHARE_BUCKET_GLOBAL: str = "share-bucket"
54
+ BLOCK_NAME_SHARE_BUCKET_USER: str = "share-bucket-{0}" # share bucket for the user/owner_id
53
55
 
54
56
  # S3 bucket object for each bucket name.
55
57
  S3_BUCKETS: dict[str, S3Bucket] = {}
56
58
 
59
+ logger = Logging.default(__name__)
60
+
57
61
 
58
62
  def init_global_env(new_owner_id: str | None = None):
59
63
  """
@@ -84,14 +88,14 @@ def get_ip_address() -> str:
84
88
  return socket.gethostbyname(socket.gethostname())
85
89
 
86
90
 
87
- def format_env_user(any_owner_id: str):
91
+ def format_env_user(block_name: str, any_owner_id: str):
88
92
  """Format the Prefect secret block for the current user"""
89
- name = BLOCK_NAME_ENV_USER.format(any_owner_id).lower()
93
+ name = block_name.format(any_owner_id).lower()
90
94
  return re.sub("[^a-zA-Z0-9]", "-", name) # replace special characters by dash
91
95
 
92
96
 
93
97
  @sync_compatible
94
- async def read_apikey(optional: bool = True, save_to_env: bool = True) -> None:
98
+ async def read_apikey(optional: bool = False, save_to_env: bool = True) -> None:
95
99
  """
96
100
  Read the API key, either from the environment variable or from an interactive input form.
97
101
 
@@ -106,8 +110,11 @@ async def read_apikey(optional: bool = True, save_to_env: bool = True) -> None:
106
110
  # If the API is saved as an env var in the ~/.env file, then it has already
107
111
  # been read automatically by rs-infra-core/.github/jupyter/resources/00-read-env.py
108
112
  apikey = os.getenv("RSPY_APIKEY")
109
- if (not apikey) and (not optional):
110
-
113
+ if apikey:
114
+ logger.debug(f"Use API key (probably from '~/.env'): '{apikey[:4]}***'")
115
+ elif optional:
116
+ logger.debug("Don't use any API key")
117
+ else:
111
118
  # Else read it from user input
112
119
  apikey = getpass.getpass("Enter your API key:")
113
120
 
@@ -119,7 +126,7 @@ async def read_apikey(optional: bool = True, save_to_env: bool = True) -> None:
119
126
  if save_to_env:
120
127
  with open(os.path.expanduser("~/.env"), "a", encoding="utf-8") as env_file:
121
128
  env_file.write(f"\nRSPY_APIKEY={apikey}\n")
122
- print("API key saved to ~/.env.")
129
+ logger.debug("API key saved to '~/.env'")
123
130
 
124
131
 
125
132
  @sync_compatible
@@ -182,7 +189,7 @@ async def init_prefect_blocks():
182
189
  env_vars[key] = value
183
190
 
184
191
  # Save env vars in a secret block for the current user
185
- await Secret(value=env_vars).save(format_env_user(owner_id), overwrite=True) # type: ignore
192
+ await Secret(value=env_vars).save(format_env_user(BLOCK_NAME_ENV_USER, owner_id), overwrite=True) # type: ignore
186
193
 
187
194
  # Now read back the blocks so we are sure our env vars are up-to-date
188
195
  await read_prefect_blocks(owner_id)
@@ -202,7 +209,7 @@ async def read_prefect_blocks(any_owner_id: str | None = None):
202
209
 
203
210
  # Read the env vars for the given user
204
211
  if any_owner_id:
205
- os.environ.update((await Secret.load(format_env_user(any_owner_id))).get())
212
+ os.environ.update((await Secret.load(format_env_user(BLOCK_NAME_ENV_USER, any_owner_id))).get())
206
213
 
207
214
  # Init the env of the current module from the env vars we have just read
208
215
  init_global_env(any_owner_id)
@@ -222,13 +229,13 @@ async def wait_for_deployment(name: str, wait: int | float = 1, max_retry: int =
222
229
  while True:
223
230
  try:
224
231
  await client.read_deployment_by_name(name)
225
- print(f"Finished deploying prefect flow: {name!r}")
232
+ logger.info(f"Finished deploying prefect flow: {name!r}")
226
233
  return
227
234
  except ObjectNotFound:
228
235
  retry += 1
229
236
  if retry >= max_retry:
230
237
  raise
231
- print(f"Wait for deployment of prefect flow: {name!r} ...")
238
+ logger.info(f"Wait for deployment of prefect flow: {name!r} ...")
232
239
  await asyncio.sleep(wait)
233
240
 
234
241
 
@@ -277,21 +284,42 @@ def get_s3_bucket(s3_path: str) -> tuple[S3Bucket, str]:
277
284
 
278
285
 
279
286
  @sync_compatible
280
- async def get_share_bucket() -> S3Bucket:
281
- """Get the prefect share bucket folder"""
287
+ async def get_share_bucket(sub_folder: str = "") -> tuple[S3Bucket, str]:
288
+ """
289
+ Get the prefect share bucket folder.
282
290
 
283
- # Try to read it from the prefect block
284
- try:
285
- return await S3Bucket.load(BLOCK_NAME_SHARE_BUCKET)
291
+ Args:
292
+ sub_folder: subfolder to use in the bucket. If empty, use the default folder and secret block.
293
+ Else use a specific secret block. WARNING: this only works in a single-threaded environment
294
+ else these specific secret blocks may overwrite each other.
286
295
 
287
- # If it doesn't exist yet, create and return it
288
- except ValueError:
289
- pass
296
+ Returns:
297
+ The prefect block for the share bucket and the block name.
298
+ """
299
+
300
+ # Use the default secret block for the default folder
301
+ if not sub_folder:
302
+ block_name = BLOCK_NAME_SHARE_BUCKET_GLOBAL
303
+
304
+ # Try to read and return the prefect block, if it already exists.
305
+ try:
306
+ return await S3Bucket.load(block_name), block_name
307
+
308
+ # If it doesn't exist yet, create and return it
309
+ except ValueError:
310
+ pass
311
+
312
+ # Use a specific block for the current user. Always overwrite it, because the subfolder may have changed.
313
+ else:
314
+ block_name = format_env_user(BLOCK_NAME_SHARE_BUCKET_USER, owner_id) # type: ignore
290
315
 
291
316
  # Read the prefect blocks that contain the S3 authentication
292
317
  bucket_name = os.environ["PREFECT_BUCKET_NAME"]
293
318
  bucket_folder = os.environ["PREFECT_BUCKET_FOLDER"]
294
319
 
320
+ if sub_folder:
321
+ bucket_folder = os.path.join(bucket_folder, sub_folder)
322
+
295
323
  # Get a s3 bucket object from its name
296
324
  generic_bucket, _ = get_s3_bucket(bucket_name)
297
325
 
@@ -303,8 +331,8 @@ async def get_share_bucket() -> S3Bucket:
303
331
  )
304
332
 
305
333
  # Save it as a prefect block and return it
306
- await share_bucket.save(BLOCK_NAME_SHARE_BUCKET, overwrite=True)
307
- return share_bucket
334
+ await share_bucket.save(block_name, overwrite=True)
335
+ return share_bucket, block_name
308
336
 
309
337
 
310
338
  @sync_compatible
@@ -16,6 +16,7 @@
16
16
 
17
17
  import os
18
18
  from dataclasses import dataclass
19
+ from datetime import datetime
19
20
 
20
21
 
21
22
  @dataclass
@@ -68,3 +69,45 @@ def env_bool(var: str, default: bool) -> bool:
68
69
  if val in ("n", "no", "f", "false", "off", "0"):
69
70
  return False
70
71
  return default
72
+
73
+
74
+ def strftime_millis(date: datetime):
75
+ """Format datetime with milliseconds precision"""
76
+ return date.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
77
+
78
+
79
+ def create_valcover_filter(
80
+ start_datetime: datetime | str,
81
+ end_datetime: datetime | str,
82
+ product_type: str,
83
+ ) -> dict:
84
+ """Creates a ValCover filter from the input values to be used in flows
85
+
86
+ Args:
87
+ start_datetime: Start datetime for the time interval used to filter the files
88
+ end_datetime: End datetime for the time interval used to filter the files
89
+ product_type: Auxiliary file type wanted
90
+
91
+ Returns:
92
+ dict: ValCover filter
93
+ """
94
+ # Convert datetime inputs to str
95
+ if isinstance(start_datetime, datetime):
96
+ start_datetime = strftime_millis(start_datetime)
97
+ if isinstance(end_datetime, datetime):
98
+ end_datetime = strftime_millis(end_datetime)
99
+
100
+ # CQL2 filter: we use a filter combining a ValCover filter and a product type filter
101
+ return {
102
+ "op": "and",
103
+ "args": [
104
+ {"op": "=", "args": [{"property": "product:type"}, product_type]},
105
+ {
106
+ "op": "t_contains",
107
+ "args": [
108
+ {"interval": [{"property": "start_datetime"}, {"property": "end_datetime"}]},
109
+ {"interval": [start_datetime, end_datetime]},
110
+ ],
111
+ },
112
+ ],
113
+ }