rs-client-libraries 1.0a2__tar.gz → 1.0a3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/PKG-INFO +6 -6
- {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/pyproject.toml +7 -7
- {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_client/__init__.py +1 -1
- {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_client/ogcapi/dpr_client.py +52 -10
- {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_client/ogcapi/ogcapi_client.py +2 -1
- {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_client/ogcapi/staging_client.py +7 -2
- {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_client/rs_client.py +7 -2
- {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_client/stac/catalog_client.py +2 -3
- {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_common/prefect_utils.py +49 -21
- {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_common/utils.py +43 -0
- rs_client_libraries-1.0a3/rs_workflows/auxip_flow.py +202 -0
- {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_workflows/catalog_flow.py +53 -2
- {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_workflows/dpr_flow.py +15 -9
- {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_workflows/flow_utils.py +91 -11
- rs_client_libraries-1.0a3/rs_workflows/on_demand_processing.py +242 -0
- rs_client_libraries-1.0a3/rs_workflows/payload_builder.py +310 -0
- {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_workflows/record_performance.py +259 -25
- {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_workflows/staging_flow.py +9 -16
- rs_client_libraries-1.0a2/rs_workflows/auxip_flow.py +0 -70
- rs_client_libraries-1.0a2/rs_workflows/on_demand_processing.py +0 -316
- {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/LICENSE +0 -0
- {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/README.adoc +0 -0
- {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_client/ogcapi/__init__.py +0 -0
- {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_client/stac/__init__.py +0 -0
- {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_client/stac/auxip_client.py +0 -0
- {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_client/stac/cadip_client.py +0 -0
- {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_client/stac/prip_client.py +0 -0
- {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_client/stac/stac_base.py +0 -0
- {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_common/__init__.py +0 -0
- {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_common/config.py +0 -0
- {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_common/init_opentelemetry.py +0 -0
- {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_common/logging.py +0 -0
- {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_workflows/__init__.py +0 -0
- {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_workflows/cadip_flow.py +0 -0
- {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_workflows/init_pi_db_flow.py +0 -0
- {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_workflows/pi_db_models.py +0 -0
- {rs_client_libraries-1.0a2 → rs_client_libraries-1.0a3}/rs_workflows/prip_flow.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: rs-client-libraries
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.0a3
|
|
4
4
|
Summary: Copernicus Sentinel rs-client library and processing workflows.
|
|
5
5
|
License: Apache-2.0
|
|
6
6
|
License-File: LICENSE
|
|
@@ -21,13 +21,13 @@ Classifier: Programming Language :: Python :: 3.14
|
|
|
21
21
|
Classifier: Topic :: Scientific/Engineering :: Image Processing
|
|
22
22
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
23
23
|
Requires-Dist: SQLAlchemy (>=2.0.37,<3.0.0)
|
|
24
|
-
Requires-Dist: cachetools (>=5.5.2
|
|
24
|
+
Requires-Dist: cachetools (>=5.5.2)
|
|
25
25
|
Requires-Dist: click (==8.1.8)
|
|
26
26
|
Requires-Dist: openapi-core (>=0.19.5,<0.20.0)
|
|
27
|
-
Requires-Dist: opentelemetry-distro (>=0.
|
|
28
|
-
Requires-Dist: opentelemetry-exporter-otlp (>=1.
|
|
29
|
-
Requires-Dist: opentelemetry-instrumentation-aws-lambda (>=0.
|
|
30
|
-
Requires-Dist: opentelemetry-test-utils (>=0.
|
|
27
|
+
Requires-Dist: opentelemetry-distro (>=0.59b0,<0.60)
|
|
28
|
+
Requires-Dist: opentelemetry-exporter-otlp (>=1.38.0,<2.0.0)
|
|
29
|
+
Requires-Dist: opentelemetry-instrumentation-aws-lambda (>=0.59b0,<0.60)
|
|
30
|
+
Requires-Dist: opentelemetry-test-utils (>=0.59b0,<0.60)
|
|
31
31
|
Requires-Dist: prefect[aws] (==3.2.13)
|
|
32
32
|
Requires-Dist: protobuf (>=5.29.5)
|
|
33
33
|
Requires-Dist: pystac (>=1.13.0,<2.0.0)
|
|
@@ -18,7 +18,7 @@ dynamic = ["version"] # see: https://github.com/mtkennerly/poetry-dynamic-versio
|
|
|
18
18
|
|
|
19
19
|
[tool.poetry]
|
|
20
20
|
name = "rs-client-libraries"
|
|
21
|
-
version = "1.
|
|
21
|
+
version = "1.0a3"
|
|
22
22
|
description = "Copernicus Sentinel rs-client library and processing workflows."
|
|
23
23
|
license = "Apache-2.0"
|
|
24
24
|
authors = ["ESA under contract with Airbus and CS <ops_coprs@airbus.com>"]
|
|
@@ -51,13 +51,13 @@ include = [
|
|
|
51
51
|
|
|
52
52
|
[tool.poetry.dependencies]
|
|
53
53
|
python = "^3.11"
|
|
54
|
-
opentelemetry-distro = "^0.
|
|
55
|
-
opentelemetry-instrumentation-aws-lambda = "^0.
|
|
56
|
-
opentelemetry-test-utils = "^0.
|
|
57
|
-
opentelemetry-exporter-otlp = "^1.
|
|
54
|
+
opentelemetry-distro = "^0.59b0" # then run 'poetry run opentelemetry-bootstrap -a install'
|
|
55
|
+
opentelemetry-instrumentation-aws-lambda = "^0.59b0"
|
|
56
|
+
opentelemetry-test-utils = "^0.59b0"
|
|
57
|
+
opentelemetry-exporter-otlp = "^1.38.0"
|
|
58
58
|
protobuf = ">=5.29.5"
|
|
59
59
|
python-logging-loki = "^0.3.1" # to avoid otl error: No module named 'logging_loki'
|
|
60
|
-
cachetools = "
|
|
60
|
+
cachetools = ">=5.5.2"
|
|
61
61
|
pystac = "^1.13.0"
|
|
62
62
|
pystac-client = "^0.9.0"
|
|
63
63
|
python-dateutil = "^2.9.0.post0"
|
|
@@ -90,7 +90,7 @@ sphinx-rtd-theme = "^3.0.2"
|
|
|
90
90
|
flake8 = "^7.1.1"
|
|
91
91
|
flake8-pyproject = "^1.2.3"
|
|
92
92
|
responses = "^0.25.7"
|
|
93
|
-
starlette = "^0.
|
|
93
|
+
starlette = "^0.48.0"
|
|
94
94
|
|
|
95
95
|
[tool.poetry.group.demo.dependencies] # for hybrid mode
|
|
96
96
|
jupyterlab = "^4.2.1"
|
|
@@ -17,6 +17,7 @@
|
|
|
17
17
|
import ast
|
|
18
18
|
import os.path as osp
|
|
19
19
|
import tempfile
|
|
20
|
+
from dataclasses import asdict, dataclass
|
|
20
21
|
from enum import Enum
|
|
21
22
|
from pathlib import Path
|
|
22
23
|
|
|
@@ -39,17 +40,33 @@ PATH_TO_YAML_OPENAPI = osp.realpath(
|
|
|
39
40
|
)
|
|
40
41
|
|
|
41
42
|
|
|
42
|
-
class
|
|
43
|
+
class DprProcessor(str, Enum):
|
|
43
44
|
"""DPR processor name"""
|
|
44
45
|
|
|
45
46
|
# String value = resource name in the rs-dpr-service
|
|
46
|
-
CONV_SAFE_ZARR = "conv_safe_zarr"
|
|
47
47
|
MOCKUP = "mockup"
|
|
48
48
|
S1L0 = "s1_l0"
|
|
49
49
|
S3L0 = "s3_l0"
|
|
50
50
|
S1ARD = "s1_ard"
|
|
51
51
|
|
|
52
52
|
|
|
53
|
+
@dataclass
|
|
54
|
+
class ClusterInfo:
|
|
55
|
+
"""
|
|
56
|
+
Information to connect to a DPR Dask cluster.
|
|
57
|
+
|
|
58
|
+
Attributes:
|
|
59
|
+
jupyter_token: JupyterHub API token. Only used in cluster mode, not local mode.
|
|
60
|
+
cluster_label: Dask cluster label e.g. "dask-l0"
|
|
61
|
+
cluster_instance: Dask cluster instance ID (something like "dask-gateway.17e196069443463495547eb97f532834").
|
|
62
|
+
If instance is empty, the DPR processor will use the first cluster with the given label.
|
|
63
|
+
"""
|
|
64
|
+
|
|
65
|
+
jupyter_token: str
|
|
66
|
+
cluster_label: str
|
|
67
|
+
cluster_instance: str | None = ""
|
|
68
|
+
|
|
69
|
+
|
|
53
70
|
class DprClient(OgcApiClient):
|
|
54
71
|
"""Implement the OGC API client for 'DPR as a service'."""
|
|
55
72
|
|
|
@@ -74,9 +91,25 @@ class DprClient(OgcApiClient):
|
|
|
74
91
|
"""
|
|
75
92
|
return get_href_service(self.rs_server_href, "RSPY_HOST_DPR_SERVICE")
|
|
76
93
|
|
|
94
|
+
def get_process( # type: ignore # pylint: disable=arguments-differ
|
|
95
|
+
self,
|
|
96
|
+
process_id: str,
|
|
97
|
+
cluster_info: ClusterInfo,
|
|
98
|
+
**kwargs,
|
|
99
|
+
) -> dict:
|
|
100
|
+
"""
|
|
101
|
+
Call parent method with additional HTTP Get parameters.
|
|
102
|
+
|
|
103
|
+
Args:
|
|
104
|
+
process_id (str): name of the resource
|
|
105
|
+
cluster_info: Information to connect to a DPR Dask cluster
|
|
106
|
+
"""
|
|
107
|
+
return super().get_process(process_id, params=asdict(cluster_info), **kwargs)
|
|
108
|
+
|
|
77
109
|
def run_process(
|
|
78
110
|
self,
|
|
79
|
-
process:
|
|
111
|
+
process: DprProcessor,
|
|
112
|
+
cluster_info: ClusterInfo,
|
|
80
113
|
s3_config_dir: str,
|
|
81
114
|
payload_subpath: str,
|
|
82
115
|
s3_report_dir: str | None,
|
|
@@ -86,6 +119,7 @@ class DprClient(OgcApiClient):
|
|
|
86
119
|
|
|
87
120
|
Args:
|
|
88
121
|
process: DPR process
|
|
122
|
+
cluster_info: Information to connect to a DPR Dask cluster
|
|
89
123
|
s3_config_dir: S3 bucket folder that contains the payload and configuration files to pass to the processor
|
|
90
124
|
payload_subpath: Payload file path, relative to the config folder
|
|
91
125
|
s3_report_dir: S3 bucket folder were the processor report files will be written (optional). All the eopf
|
|
@@ -97,7 +131,7 @@ class DprClient(OgcApiClient):
|
|
|
97
131
|
(or None if endpoint fails) of the running job
|
|
98
132
|
"""
|
|
99
133
|
|
|
100
|
-
use_mockup = process ==
|
|
134
|
+
use_mockup = process == DprProcessor.MOCKUP
|
|
101
135
|
|
|
102
136
|
# Data to pass to the real processor
|
|
103
137
|
data = {}
|
|
@@ -125,21 +159,26 @@ class DprClient(OgcApiClient):
|
|
|
125
159
|
# Add extra info
|
|
126
160
|
data.update({"use_mockup": use_mockup})
|
|
127
161
|
|
|
162
|
+
# Add the cluster info
|
|
163
|
+
data.update(asdict(cluster_info))
|
|
164
|
+
|
|
128
165
|
# Call the parent method
|
|
129
166
|
return super()._run_process(process.value, data)
|
|
130
167
|
|
|
131
|
-
def run_conv_safe_zarr(self, payload: dict):
|
|
168
|
+
def run_conv_safe_zarr(self, payload: dict, cluster_info: ClusterInfo):
|
|
132
169
|
"""Method to start the safe to zarr conversion process from rs-client -
|
|
133
170
|
Call the endpoint /processes/conv_safe_zarr/execution
|
|
134
171
|
|
|
135
172
|
Args:
|
|
136
173
|
payload: Dictionary to pass to the processor,
|
|
174
|
+
cluster_info: Information to connect to a DPR Dask cluster
|
|
137
175
|
containing input_safe_path - the s3 path of legacy product and
|
|
138
176
|
output_zarr_dir_path - the s3 path for the new zarr
|
|
139
177
|
Return:
|
|
140
178
|
job_id (int, str): Returns the status code of the request + the identifier
|
|
141
179
|
(or None if endpoint fails) of the running job
|
|
142
180
|
"""
|
|
181
|
+
payload.update(asdict(cluster_info)) # Add the cluster info to the payload
|
|
143
182
|
return super()._run_process("conv_safe_zarr", payload)
|
|
144
183
|
|
|
145
184
|
def wait_for_job(self, *args, **kwargs) -> list[dict]: # type: ignore
|
|
@@ -241,11 +280,14 @@ class DprClient(OgcApiClient):
|
|
|
241
280
|
self.logger.info(f"Write empty file: {self.logger.level} {s3_empty_file!r}")
|
|
242
281
|
await prefect_utils.s3_upload_empty_file(s3_empty_file)
|
|
243
282
|
|
|
244
|
-
# Change the dask authentication for local mode
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
283
|
+
# Change the dask authentication for local mode (used in old demos, could be removed)
|
|
284
|
+
try:
|
|
285
|
+
cluster_config = payload["dask_context"]["cluster_config"]
|
|
286
|
+
if self.local_mode:
|
|
287
|
+
cluster_config["auth"] = cluster_config["auth_local_mode"]
|
|
288
|
+
del cluster_config["auth_local_mode"]
|
|
289
|
+
except KeyError:
|
|
290
|
+
pass
|
|
249
291
|
|
|
250
292
|
# yaml to str conversion
|
|
251
293
|
contents = yaml.dump(payload, default_flow_style=False, sort_keys=False)
|
|
@@ -135,7 +135,7 @@ class OgcApiClient(RsClient):
|
|
|
135
135
|
)
|
|
136
136
|
return self.validate_and_unmarshal_response(response)
|
|
137
137
|
|
|
138
|
-
def get_process(self, process_id: str) -> dict:
|
|
138
|
+
def get_process(self, process_id: str, **kwargs) -> dict:
|
|
139
139
|
"""
|
|
140
140
|
Wrapper to get a specific process
|
|
141
141
|
Args:
|
|
@@ -145,6 +145,7 @@ class OgcApiClient(RsClient):
|
|
|
145
145
|
url=f"{self.href_service}/{self.endpoint_prefix}processes/{process_id}",
|
|
146
146
|
timeout=TIMEOUT,
|
|
147
147
|
**self.apikey_headers,
|
|
148
|
+
**kwargs,
|
|
148
149
|
)
|
|
149
150
|
return self.validate_and_unmarshal_response(response)
|
|
150
151
|
|
|
@@ -155,7 +155,7 @@ class StagingClient(OgcApiClient):
|
|
|
155
155
|
logger=None,
|
|
156
156
|
# timeout: int | float = math.inf,
|
|
157
157
|
poll_interval: int = 2,
|
|
158
|
-
):
|
|
158
|
+
) -> dict[str, dict]:
|
|
159
159
|
"""
|
|
160
160
|
Wait for job to finish.
|
|
161
161
|
|
|
@@ -166,15 +166,20 @@ class StagingClient(OgcApiClient):
|
|
|
166
166
|
NOTE: This argument has been disabled, see the comment in super().wait_for_job() function
|
|
167
167
|
poll_interval: When to check again for job completion in seconds
|
|
168
168
|
|
|
169
|
+
Returns:
|
|
170
|
+
dict[str, dict]: Completed job status
|
|
171
|
+
|
|
169
172
|
Raises:
|
|
170
173
|
RuntimeError in case of error
|
|
171
174
|
"""
|
|
175
|
+
job_results: dict[str, dict] = {}
|
|
172
176
|
# Call parent method for each hostname
|
|
173
177
|
for hostname, job_status in all_job_status.items():
|
|
174
|
-
super().wait_for_job(
|
|
178
|
+
job_results[hostname] = super().wait_for_job(
|
|
175
179
|
job_status,
|
|
176
180
|
logger,
|
|
177
181
|
f"Staging from {hostname!r}",
|
|
178
182
|
# timeout,
|
|
179
183
|
poll_interval,
|
|
180
184
|
)
|
|
185
|
+
return job_results
|
|
@@ -20,6 +20,7 @@ import sys
|
|
|
20
20
|
|
|
21
21
|
import requests
|
|
22
22
|
from cachetools import TTLCache, cached
|
|
23
|
+
from pystac_client.stac_api_io import Timeout
|
|
23
24
|
|
|
24
25
|
from rs_common import utils
|
|
25
26
|
from rs_common.logging import Logging
|
|
@@ -27,7 +28,7 @@ from rs_common.utils import AuthInfo
|
|
|
27
28
|
|
|
28
29
|
APIKEY_HEADER = "x-api-key"
|
|
29
30
|
|
|
30
|
-
#
|
|
31
|
+
# Default timeout in seconds
|
|
31
32
|
TIMEOUT = 30
|
|
32
33
|
|
|
33
34
|
# API Key Manager URL used to get an API Key information.
|
|
@@ -57,6 +58,7 @@ class RsClient: # pylint: disable=too-many-instance-attributes
|
|
|
57
58
|
local_mode (bool): Indicates whether the client is running in local mode.
|
|
58
59
|
apikey_headers (dict): API key headers for HTTP requests.
|
|
59
60
|
http_session (requests.Session): HTTP session for handling requests.
|
|
61
|
+
timeout (Timeout): timeout in seconds when contacting API key manager. Default to 30s.
|
|
60
62
|
"""
|
|
61
63
|
|
|
62
64
|
def __init__( # pylint: disable=too-many-branches, too-many-arguments, too-many-positional-arguments
|
|
@@ -65,6 +67,7 @@ class RsClient: # pylint: disable=too-many-instance-attributes
|
|
|
65
67
|
rs_server_api_key: str | None = None,
|
|
66
68
|
owner_id: str | None = None,
|
|
67
69
|
logger: logging.Logger | None = None,
|
|
70
|
+
timeout: Timeout = TIMEOUT,
|
|
68
71
|
):
|
|
69
72
|
"""
|
|
70
73
|
Initializes an RsClient instance.
|
|
@@ -109,6 +112,8 @@ class RsClient: # pylint: disable=too-many-instance-attributes
|
|
|
109
112
|
if self.rs_server_oauth2_cookie:
|
|
110
113
|
self.http_session.cookies.set("session", self.rs_server_oauth2_cookie)
|
|
111
114
|
|
|
115
|
+
self.timeout = timeout
|
|
116
|
+
|
|
112
117
|
def log_and_raise(self, message: str, original: Exception):
|
|
113
118
|
"""
|
|
114
119
|
Logs an error message and raises a RuntimeError.
|
|
@@ -167,7 +172,7 @@ class RsClient: # pylint: disable=too-many-instance-attributes
|
|
|
167
172
|
|
|
168
173
|
# Request the API key manager, pass user-defined api key in http header
|
|
169
174
|
self.logger.debug("Call the API key manager")
|
|
170
|
-
response = self.http_session.get(RSPY_UAC_CHECK_URL, **self.apikey_headers, timeout=
|
|
175
|
+
response = self.http_session.get(RSPY_UAC_CHECK_URL, **self.apikey_headers, timeout=self.timeout)
|
|
171
176
|
if not response.ok:
|
|
172
177
|
raise RuntimeError(
|
|
173
178
|
f"API key manager status code {response.status_code}: {utils.read_response_error(response)}",
|
|
@@ -21,10 +21,9 @@ import logging
|
|
|
21
21
|
import re
|
|
22
22
|
from collections.abc import Iterator
|
|
23
23
|
|
|
24
|
-
from pystac import Collection, Item, Link, RelType
|
|
24
|
+
from pystac import Collection, Item, ItemCollection, Link, RelType
|
|
25
25
|
from pystac_client import Client
|
|
26
26
|
from pystac_client.collection_client import CollectionClient
|
|
27
|
-
from pystac_client.item_search import ItemSearch
|
|
28
27
|
from requests import HTTPError, Response
|
|
29
28
|
|
|
30
29
|
from rs_client.rs_client import TIMEOUT
|
|
@@ -223,7 +222,7 @@ class CatalogClient(StacBase): # type: ignore # pylint: disable=too-many-ancest
|
|
|
223
222
|
def search( # type: ignore # pylint: disable=too-many-arguments, arguments-differ
|
|
224
223
|
self,
|
|
225
224
|
**kwargs,
|
|
226
|
-
) ->
|
|
225
|
+
) -> ItemCollection | None:
|
|
227
226
|
"""Search items inside a specific collection."""
|
|
228
227
|
|
|
229
228
|
if "collections" in kwargs:
|
|
@@ -35,6 +35,7 @@ from prefect.exceptions import ObjectNotFound
|
|
|
35
35
|
from prefect.utilities.asyncutils import sync_compatible
|
|
36
36
|
from prefect_aws import AwsCredentials, S3Bucket
|
|
37
37
|
|
|
38
|
+
from rs_common.logging import Logging
|
|
38
39
|
from rs_common.utils import env_bool
|
|
39
40
|
|
|
40
41
|
# In local mode, all your services are running locally.
|
|
@@ -49,11 +50,14 @@ owner_id: str | None = ""
|
|
|
49
50
|
# Prefect block names
|
|
50
51
|
BLOCK_NAME_ENV_GLOBAL: str = "env-vars"
|
|
51
52
|
BLOCK_NAME_ENV_USER: str = "env-vars-{0}" # env variables for the user/owner_id
|
|
52
|
-
|
|
53
|
+
BLOCK_NAME_SHARE_BUCKET_GLOBAL: str = "share-bucket"
|
|
54
|
+
BLOCK_NAME_SHARE_BUCKET_USER: str = "share-bucket-{0}" # share bucket for the user/owner_id
|
|
53
55
|
|
|
54
56
|
# S3 bucket object for each bucket name.
|
|
55
57
|
S3_BUCKETS: dict[str, S3Bucket] = {}
|
|
56
58
|
|
|
59
|
+
logger = Logging.default(__name__)
|
|
60
|
+
|
|
57
61
|
|
|
58
62
|
def init_global_env(new_owner_id: str | None = None):
|
|
59
63
|
"""
|
|
@@ -84,14 +88,14 @@ def get_ip_address() -> str:
|
|
|
84
88
|
return socket.gethostbyname(socket.gethostname())
|
|
85
89
|
|
|
86
90
|
|
|
87
|
-
def format_env_user(any_owner_id: str):
|
|
91
|
+
def format_env_user(block_name: str, any_owner_id: str):
|
|
88
92
|
"""Format the Prefect secret block for the current user"""
|
|
89
|
-
name =
|
|
93
|
+
name = block_name.format(any_owner_id).lower()
|
|
90
94
|
return re.sub("[^a-zA-Z0-9]", "-", name) # replace special characters by dash
|
|
91
95
|
|
|
92
96
|
|
|
93
97
|
@sync_compatible
|
|
94
|
-
async def read_apikey(optional: bool =
|
|
98
|
+
async def read_apikey(optional: bool = False, save_to_env: bool = True) -> None:
|
|
95
99
|
"""
|
|
96
100
|
Read the API key, either from the environment variable or from an interactive input form.
|
|
97
101
|
|
|
@@ -106,8 +110,11 @@ async def read_apikey(optional: bool = True, save_to_env: bool = True) -> None:
|
|
|
106
110
|
# If the API is saved as an env var in the ~/.env file, then it has already
|
|
107
111
|
# been read automatically by rs-infra-core/.github/jupyter/resources/00-read-env.py
|
|
108
112
|
apikey = os.getenv("RSPY_APIKEY")
|
|
109
|
-
if
|
|
110
|
-
|
|
113
|
+
if apikey:
|
|
114
|
+
logger.debug(f"Use API key (probably from '~/.env'): '{apikey[:4]}***'")
|
|
115
|
+
elif optional:
|
|
116
|
+
logger.debug("Don't use any API key")
|
|
117
|
+
else:
|
|
111
118
|
# Else read it from user input
|
|
112
119
|
apikey = getpass.getpass("Enter your API key:")
|
|
113
120
|
|
|
@@ -119,7 +126,7 @@ async def read_apikey(optional: bool = True, save_to_env: bool = True) -> None:
|
|
|
119
126
|
if save_to_env:
|
|
120
127
|
with open(os.path.expanduser("~/.env"), "a", encoding="utf-8") as env_file:
|
|
121
128
|
env_file.write(f"\nRSPY_APIKEY={apikey}\n")
|
|
122
|
-
|
|
129
|
+
logger.debug("API key saved to '~/.env'")
|
|
123
130
|
|
|
124
131
|
|
|
125
132
|
@sync_compatible
|
|
@@ -182,7 +189,7 @@ async def init_prefect_blocks():
|
|
|
182
189
|
env_vars[key] = value
|
|
183
190
|
|
|
184
191
|
# Save env vars in a secret block for the current user
|
|
185
|
-
await Secret(value=env_vars).save(format_env_user(owner_id), overwrite=True) # type: ignore
|
|
192
|
+
await Secret(value=env_vars).save(format_env_user(BLOCK_NAME_ENV_USER, owner_id), overwrite=True) # type: ignore
|
|
186
193
|
|
|
187
194
|
# Now read back the blocks so we are sure our env vars are up-to-date
|
|
188
195
|
await read_prefect_blocks(owner_id)
|
|
@@ -202,7 +209,7 @@ async def read_prefect_blocks(any_owner_id: str | None = None):
|
|
|
202
209
|
|
|
203
210
|
# Read the env vars for the given user
|
|
204
211
|
if any_owner_id:
|
|
205
|
-
os.environ.update((await Secret.load(format_env_user(any_owner_id))).get())
|
|
212
|
+
os.environ.update((await Secret.load(format_env_user(BLOCK_NAME_ENV_USER, any_owner_id))).get())
|
|
206
213
|
|
|
207
214
|
# Init the env of the current module from the env vars we have just read
|
|
208
215
|
init_global_env(any_owner_id)
|
|
@@ -222,13 +229,13 @@ async def wait_for_deployment(name: str, wait: int | float = 1, max_retry: int =
|
|
|
222
229
|
while True:
|
|
223
230
|
try:
|
|
224
231
|
await client.read_deployment_by_name(name)
|
|
225
|
-
|
|
232
|
+
logger.info(f"Finished deploying prefect flow: {name!r}")
|
|
226
233
|
return
|
|
227
234
|
except ObjectNotFound:
|
|
228
235
|
retry += 1
|
|
229
236
|
if retry >= max_retry:
|
|
230
237
|
raise
|
|
231
|
-
|
|
238
|
+
logger.info(f"Wait for deployment of prefect flow: {name!r} ...")
|
|
232
239
|
await asyncio.sleep(wait)
|
|
233
240
|
|
|
234
241
|
|
|
@@ -277,21 +284,42 @@ def get_s3_bucket(s3_path: str) -> tuple[S3Bucket, str]:
|
|
|
277
284
|
|
|
278
285
|
|
|
279
286
|
@sync_compatible
|
|
280
|
-
async def get_share_bucket() -> S3Bucket:
|
|
281
|
-
"""
|
|
287
|
+
async def get_share_bucket(sub_folder: str = "") -> tuple[S3Bucket, str]:
|
|
288
|
+
"""
|
|
289
|
+
Get the prefect share bucket folder.
|
|
282
290
|
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
291
|
+
Args:
|
|
292
|
+
sub_folder: subfolder to use in the bucket. If empty, use the default folder and secret block.
|
|
293
|
+
Else use a specific secret block. WARNING: this only works in a single-threaded environment
|
|
294
|
+
else these specific secret blocks may overwrite each other.
|
|
286
295
|
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
296
|
+
Returns:
|
|
297
|
+
The prefect block for the share bucket and the block name.
|
|
298
|
+
"""
|
|
299
|
+
|
|
300
|
+
# Use the default secret block for the default folder
|
|
301
|
+
if not sub_folder:
|
|
302
|
+
block_name = BLOCK_NAME_SHARE_BUCKET_GLOBAL
|
|
303
|
+
|
|
304
|
+
# Try to read and return the prefect block, if it already exists.
|
|
305
|
+
try:
|
|
306
|
+
return await S3Bucket.load(block_name), block_name
|
|
307
|
+
|
|
308
|
+
# If it doesn't exist yet, create and return it
|
|
309
|
+
except ValueError:
|
|
310
|
+
pass
|
|
311
|
+
|
|
312
|
+
# Use a specific block for the current user. Always overwrite it, because the subfolder may have changed.
|
|
313
|
+
else:
|
|
314
|
+
block_name = format_env_user(BLOCK_NAME_SHARE_BUCKET_USER, owner_id) # type: ignore
|
|
290
315
|
|
|
291
316
|
# Read the prefect blocks that contain the S3 authentication
|
|
292
317
|
bucket_name = os.environ["PREFECT_BUCKET_NAME"]
|
|
293
318
|
bucket_folder = os.environ["PREFECT_BUCKET_FOLDER"]
|
|
294
319
|
|
|
320
|
+
if sub_folder:
|
|
321
|
+
bucket_folder = os.path.join(bucket_folder, sub_folder)
|
|
322
|
+
|
|
295
323
|
# Get a s3 bucket object from its name
|
|
296
324
|
generic_bucket, _ = get_s3_bucket(bucket_name)
|
|
297
325
|
|
|
@@ -303,8 +331,8 @@ async def get_share_bucket() -> S3Bucket:
|
|
|
303
331
|
)
|
|
304
332
|
|
|
305
333
|
# Save it as a prefect block and return it
|
|
306
|
-
await share_bucket.save(
|
|
307
|
-
return share_bucket
|
|
334
|
+
await share_bucket.save(block_name, overwrite=True)
|
|
335
|
+
return share_bucket, block_name
|
|
308
336
|
|
|
309
337
|
|
|
310
338
|
@sync_compatible
|
|
@@ -16,6 +16,7 @@
|
|
|
16
16
|
|
|
17
17
|
import os
|
|
18
18
|
from dataclasses import dataclass
|
|
19
|
+
from datetime import datetime
|
|
19
20
|
|
|
20
21
|
|
|
21
22
|
@dataclass
|
|
@@ -68,3 +69,45 @@ def env_bool(var: str, default: bool) -> bool:
|
|
|
68
69
|
if val in ("n", "no", "f", "false", "off", "0"):
|
|
69
70
|
return False
|
|
70
71
|
return default
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def strftime_millis(date: datetime):
|
|
75
|
+
"""Format datetime with milliseconds precision"""
|
|
76
|
+
return date.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def create_valcover_filter(
|
|
80
|
+
start_datetime: datetime | str,
|
|
81
|
+
end_datetime: datetime | str,
|
|
82
|
+
product_type: str,
|
|
83
|
+
) -> dict:
|
|
84
|
+
"""Creates a ValCover filter from the input values to be used in flows
|
|
85
|
+
|
|
86
|
+
Args:
|
|
87
|
+
start_datetime: Start datetime for the time interval used to filter the files
|
|
88
|
+
end_datetime: End datetime for the time interval used to filter the files
|
|
89
|
+
product_type: Auxiliary file type wanted
|
|
90
|
+
|
|
91
|
+
Returns:
|
|
92
|
+
dict: ValCover filter
|
|
93
|
+
"""
|
|
94
|
+
# Convert datetime inputs to str
|
|
95
|
+
if isinstance(start_datetime, datetime):
|
|
96
|
+
start_datetime = strftime_millis(start_datetime)
|
|
97
|
+
if isinstance(end_datetime, datetime):
|
|
98
|
+
end_datetime = strftime_millis(end_datetime)
|
|
99
|
+
|
|
100
|
+
# CQL2 filter: we use a filter combining a ValCover filter and a product type filter
|
|
101
|
+
return {
|
|
102
|
+
"op": "and",
|
|
103
|
+
"args": [
|
|
104
|
+
{"op": "=", "args": [{"property": "product:type"}, product_type]},
|
|
105
|
+
{
|
|
106
|
+
"op": "t_contains",
|
|
107
|
+
"args": [
|
|
108
|
+
{"interval": [{"property": "start_datetime"}, {"property": "end_datetime"}]},
|
|
109
|
+
{"interval": [start_datetime, end_datetime]},
|
|
110
|
+
],
|
|
111
|
+
},
|
|
112
|
+
],
|
|
113
|
+
}
|