rs-client-libraries 1.0a8.post16.dev0__tar.gz → 1.0a9__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/PKG-INFO +3 -3
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/pyproject.toml +9 -7
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_client/__init__.py +1 -1
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_client/ogcapi/dpr_client.py +4 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_client/ogcapi/ogcapi_client.py +1 -4
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_client/ogcapi/staging_client.py +22 -13
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_common/prefect_utils.py +16 -16
- rs_client_libraries-1.0a9/rs_common/utils.py +290 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/auxip_flow.py +149 -5
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/cadip_flow.py +4 -1
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/catalog_flow.py +20 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/dpr_flow.py +8 -3
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/flow_utils.py +2 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/on_demand/common/l0_last_steps.py +1 -1
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/on_demand/sentinel3/s3_l0.py +1 -1
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/on_demand_processing.py +215 -48
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/payload_builder.py +17 -5
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/payload_generator.py +36 -16
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/record_performance.py +12 -5
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/staging_flow.py +5 -7
- rs_client_libraries-1.0a8.post16.dev0/rs_common/utils.py +0 -122
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/LICENSE +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/README.md +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_client/ogcapi/__init__.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_client/osam_client.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_client/rs_client.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_client/stac/__init__.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_client/stac/auxip_client.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_client/stac/cadip_client.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_client/stac/catalog_client.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_client/stac/prip_client.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_client/stac/stac_base.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_common/__init__.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_common/config.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_common/init_opentelemetry.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_common/logging.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/__init__.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/init_pi_db_flow.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/on_demand/__init__.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/on_demand/common/__init__.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/on_demand/common/l0.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/on_demand/common/staging.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/on_demand/common/types.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/on_demand/sentinel1/__init__.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/on_demand/sentinel1/s1_l0.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/on_demand/sentinel3/__init__.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/operation/__init__.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/operation/init_quota_monitoring_flow.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/operation/osam_flows.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/operation/quota_monitoring_db_models.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/operation/quota_monitoring_flow.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/payload_template.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/pi_db_models.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/prip_flow.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/storage_configuration.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/utils/__init__.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/utils/artifact_verbose.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/utils/cadip.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/utils/catalog.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/utils/dask.py +0 -0
- {rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_workflows/utils/dpr.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: rs-client-libraries
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.0a9
|
|
4
4
|
Summary: Copernicus Sentinel rs-client library and processing workflows.
|
|
5
5
|
License: Apache-2.0
|
|
6
6
|
License-File: LICENSE
|
|
@@ -25,8 +25,8 @@ Requires-Dist: cachetools (>=5.5.2)
|
|
|
25
25
|
Requires-Dist: click (==8.3.1)
|
|
26
26
|
Requires-Dist: dask-gateway (==2025.4.0)
|
|
27
27
|
Requires-Dist: openapi-core (==0.22.0)
|
|
28
|
-
Requires-Dist: opentelemetry-distro (>=0.
|
|
29
|
-
Requires-Dist: opentelemetry-exporter-otlp (>=1.
|
|
28
|
+
Requires-Dist: opentelemetry-distro (>=0.62b0,<0.63)
|
|
29
|
+
Requires-Dist: opentelemetry-exporter-otlp (>=1.41.0,<2.0.0)
|
|
30
30
|
Requires-Dist: prefect-aws (>=0.7.5)
|
|
31
31
|
Requires-Dist: prefect[aws] (==3.6.20)
|
|
32
32
|
Requires-Dist: protobuf (>=5.29.5)
|
|
@@ -18,7 +18,7 @@ dynamic = ["version"] # see: https://github.com/mtkennerly/poetry-dynamic-versio
|
|
|
18
18
|
|
|
19
19
|
[tool.poetry]
|
|
20
20
|
name = "rs-client-libraries"
|
|
21
|
-
version = "1.
|
|
21
|
+
version = "1.0a9"
|
|
22
22
|
description = "Copernicus Sentinel rs-client library and processing workflows."
|
|
23
23
|
license = "Apache-2.0"
|
|
24
24
|
authors = ["ESA under contract with Airbus and CS <ops_coprs@airbus.com>"]
|
|
@@ -51,8 +51,8 @@ include = [
|
|
|
51
51
|
|
|
52
52
|
[tool.poetry.dependencies]
|
|
53
53
|
python = ">=3.11,<3.15"
|
|
54
|
-
opentelemetry-distro = "^0.
|
|
55
|
-
opentelemetry-exporter-otlp = "^1.
|
|
54
|
+
opentelemetry-distro = "^0.62b0" # then run 'poetry run opentelemetry-bootstrap -a install'
|
|
55
|
+
opentelemetry-exporter-otlp = "^1.41.0"
|
|
56
56
|
protobuf = ">=5.29.5"
|
|
57
57
|
cachetools = ">=5.5.2"
|
|
58
58
|
pystac = "^1.13.0"
|
|
@@ -140,13 +140,15 @@ max-line-length = 120
|
|
|
140
140
|
[tool.pylint.messages_control]
|
|
141
141
|
max-line-length=120
|
|
142
142
|
disable = [
|
|
143
|
-
"logging-fstring-interpolation",
|
|
144
143
|
"global-statement",
|
|
145
|
-
"
|
|
146
|
-
"
|
|
144
|
+
"import-error",
|
|
145
|
+
"logging-fstring-interpolation",
|
|
147
146
|
"not-context-manager",
|
|
148
|
-
"too-
|
|
147
|
+
"too-few-public-methods",
|
|
148
|
+
"too-many-arguments",
|
|
149
149
|
"too-many-branches",
|
|
150
|
+
"too-many-locals",
|
|
151
|
+
"too-many-positional-arguments",
|
|
150
152
|
"too-many-statements",
|
|
151
153
|
]
|
|
152
154
|
|
{rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_client/ogcapi/dpr_client.py
RENAMED
|
@@ -47,6 +47,8 @@ class DprProcessor(str, Enum):
|
|
|
47
47
|
S1L0 = "s1_l0"
|
|
48
48
|
S3L0 = "s3_l0"
|
|
49
49
|
S1ARD = "s1_ard"
|
|
50
|
+
S3L1OLCI = "s3_l1olci"
|
|
51
|
+
# Add S3L2OLCI as "s3_l2olci" when implementation needed (already exists in rs-dpr-service)
|
|
50
52
|
|
|
51
53
|
|
|
52
54
|
class DprPipeline(str, Enum):
|
|
@@ -56,6 +58,8 @@ class DprPipeline(str, Enum):
|
|
|
56
58
|
S1L0FULL = "s1_l0_full"
|
|
57
59
|
S3L0FULL = "s3_l0_full"
|
|
58
60
|
S1ARDFULL = "s1_ard_full"
|
|
61
|
+
S3L1OLCIFULL = "ol1_eo"
|
|
62
|
+
# Add S3L2OLCIFULL as "ol2_fr" when implementation needed (already exists in rs-dpr-service)
|
|
59
63
|
|
|
60
64
|
|
|
61
65
|
@dataclass
|
|
@@ -80,7 +80,7 @@ class OgcApiClient(RsClient):
|
|
|
80
80
|
|
|
81
81
|
if result.errors:
|
|
82
82
|
raise OgcValidationException(
|
|
83
|
-
f"Error validating the request of the endpoint "
|
|
83
|
+
f"Error validating the request body {request.body!r} of the endpoint "
|
|
84
84
|
f"{openapi_request.path}: {', '.join(str(x) for x in result.errors)}",
|
|
85
85
|
)
|
|
86
86
|
if not result.body:
|
|
@@ -226,7 +226,6 @@ class OgcApiClient(RsClient):
|
|
|
226
226
|
job_status: dict,
|
|
227
227
|
logger=None,
|
|
228
228
|
job_name: str = "",
|
|
229
|
-
# timeout: int | float = math.inf,
|
|
230
229
|
poll_interval: int = 2,
|
|
231
230
|
) -> dict:
|
|
232
231
|
"""
|
|
@@ -236,8 +235,6 @@ class OgcApiClient(RsClient):
|
|
|
236
235
|
job_status: Returned by `_run_process`
|
|
237
236
|
logger: To show advancement in logger
|
|
238
237
|
job_name: Job name to show in the logger
|
|
239
|
-
timeout: Job completion timeout in seconds.
|
|
240
|
-
NOTE: This argument has been disabled, see the comment bellow !
|
|
241
238
|
poll_interval: When to check again for job completion in seconds
|
|
242
239
|
|
|
243
240
|
Returns:
|
|
@@ -62,6 +62,7 @@ class StagingClient(OgcApiClient):
|
|
|
62
62
|
self,
|
|
63
63
|
stac_input: dict[Any, Any] | str,
|
|
64
64
|
out_coll_name: str,
|
|
65
|
+
asset_names: set[str] | None = None,
|
|
65
66
|
) -> dict[str, dict]:
|
|
66
67
|
"""Method to start the staging process from rs-client - Call the endpoint /processes/staging/execution
|
|
67
68
|
|
|
@@ -74,7 +75,9 @@ class StagingClient(OgcApiClient):
|
|
|
74
75
|
- A single link that returns a STAC itemCollection: this link should be an url to search a
|
|
75
76
|
itemCollection, for example:
|
|
76
77
|
http://localhost:8002/cadip/search?ids=S1A_20231120061537234567&collections=cadip_sentinel1
|
|
77
|
-
|
|
78
|
+
out_coll_name (str): name of the output collection
|
|
79
|
+
asset_names (set[str]): An optional set to keep only selected asset names.
|
|
80
|
+
If not provided, all assets are staged.
|
|
78
81
|
Return:
|
|
79
82
|
dict(hostname, job_id (int, str)): Returns the status codes of the staging requests (one per URL hostname) +
|
|
80
83
|
the identifiers
|
|
@@ -83,11 +86,10 @@ class StagingClient(OgcApiClient):
|
|
|
83
86
|
|
|
84
87
|
# ----- Case 1: we only load a link (that refers to a STAC itemCollection) in the staging request body
|
|
85
88
|
if isinstance(stac_input, str):
|
|
86
|
-
|
|
87
89
|
# Check that it's an url
|
|
88
90
|
parsed = urlparse(stac_input)
|
|
89
91
|
if parsed.scheme and parsed.netloc and parsed.hostname:
|
|
90
|
-
staging_body =
|
|
92
|
+
staging_body = self.staging_body(out_coll_name, {"href": stac_input}, asset_names)
|
|
91
93
|
return {parsed.hostname: super()._run_process(RESOURCE, staging_body)}
|
|
92
94
|
|
|
93
95
|
# ----- Case 2: we directly load a STAC ItemCollection in the staging request body
|
|
@@ -139,21 +141,31 @@ class StagingClient(OgcApiClient):
|
|
|
139
141
|
hostname_triggers: dict[str, dict] = {}
|
|
140
142
|
for hostname, items in hostname_items.items():
|
|
141
143
|
stac_item_collection = ItemCollection(type="FeatureCollection", features=items)
|
|
142
|
-
staging_body =
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
}
|
|
144
|
+
staging_body = self.staging_body(
|
|
145
|
+
out_coll_name,
|
|
146
|
+
{"value": stac_item_collection.model_dump(mode="json")},
|
|
147
|
+
asset_names,
|
|
148
|
+
)
|
|
148
149
|
hostname_triggers[hostname] = super()._run_process(RESOURCE, staging_body)
|
|
149
150
|
|
|
150
151
|
return hostname_triggers
|
|
151
152
|
|
|
153
|
+
def staging_body(
|
|
154
|
+
self,
|
|
155
|
+
collection: str,
|
|
156
|
+
items: dict[str, Any],
|
|
157
|
+
asset_names: set[str] | None = None,
|
|
158
|
+
) -> dict[str, Any]:
|
|
159
|
+
"""Build the staging request body with provided arguments"""
|
|
160
|
+
inputs = {"collection": collection, "items": items}
|
|
161
|
+
if asset_names:
|
|
162
|
+
inputs["asset_names"] = list(asset_names)
|
|
163
|
+
return {"inputs": inputs}
|
|
164
|
+
|
|
152
165
|
def wait_for_jobs(
|
|
153
166
|
self,
|
|
154
167
|
all_job_status: dict[str, dict],
|
|
155
168
|
logger=None,
|
|
156
|
-
# timeout: int | float = math.inf,
|
|
157
169
|
poll_interval: int = 2,
|
|
158
170
|
) -> dict[str, dict]:
|
|
159
171
|
"""
|
|
@@ -162,8 +174,6 @@ class StagingClient(OgcApiClient):
|
|
|
162
174
|
Args:
|
|
163
175
|
job_status: Returned by `run_staging`
|
|
164
176
|
logger: To show advancement in logger
|
|
165
|
-
timeout: Job completion timeout in seconds
|
|
166
|
-
NOTE: This argument has been disabled, see the comment in super().wait_for_job() function
|
|
167
177
|
poll_interval: When to check again for job completion in seconds
|
|
168
178
|
|
|
169
179
|
Returns:
|
|
@@ -179,7 +189,6 @@ class StagingClient(OgcApiClient):
|
|
|
179
189
|
job_status,
|
|
180
190
|
logger,
|
|
181
191
|
f"Staging from {hostname!r}",
|
|
182
|
-
# timeout,
|
|
183
192
|
poll_interval,
|
|
184
193
|
)
|
|
185
194
|
return job_results
|
{rs_client_libraries-1.0a8.post16.dev0 → rs_client_libraries-1.0a9}/rs_common/prefect_utils.py
RENAMED
|
@@ -45,11 +45,11 @@ from rs_common.utils import env_bool
|
|
|
45
45
|
# In local mode, all your services are running locally.
|
|
46
46
|
# In cluster mode, we use the services deployed on the RS-Server website.
|
|
47
47
|
# This configuration is set in an environment variable.
|
|
48
|
-
|
|
49
|
-
|
|
48
|
+
LOCAL_MODE: bool = False
|
|
49
|
+
CLUSTER_MODE: bool = not LOCAL_MODE
|
|
50
50
|
|
|
51
51
|
# Current user
|
|
52
|
-
|
|
52
|
+
OWNER_ID: str = ""
|
|
53
53
|
|
|
54
54
|
# Prefect block names
|
|
55
55
|
BLOCK_NAME_ENV_GLOBAL: str = "env-vars"
|
|
@@ -73,16 +73,16 @@ def init_global_env(new_owner_id: str | None = None):
|
|
|
73
73
|
Args:
|
|
74
74
|
new_owner_id: new ower/user id, if known.
|
|
75
75
|
"""
|
|
76
|
-
global
|
|
76
|
+
global LOCAL_MODE, CLUSTER_MODE, OWNER_ID
|
|
77
77
|
|
|
78
|
-
|
|
79
|
-
|
|
78
|
+
LOCAL_MODE = env_bool("RSPY_LOCAL_MODE", default=False)
|
|
79
|
+
CLUSTER_MODE = not LOCAL_MODE
|
|
80
80
|
|
|
81
81
|
# Override the owner id or read it from the env vars
|
|
82
82
|
if new_owner_id:
|
|
83
|
-
|
|
83
|
+
OWNER_ID = new_owner_id
|
|
84
84
|
else:
|
|
85
|
-
|
|
85
|
+
OWNER_ID = os.getenv("JUPYTERHUB_USER", "") if CLUSTER_MODE else os.getenv("RSPY_HOST_USER", "")
|
|
86
86
|
|
|
87
87
|
|
|
88
88
|
# Init the global variables
|
|
@@ -110,7 +110,7 @@ async def read_apikey(optional: bool = False, save_to_env: bool = True) -> None:
|
|
|
110
110
|
save_to_env (bool): If True, saves the API key to the ~/.env file.
|
|
111
111
|
"""
|
|
112
112
|
# No API key in local mode
|
|
113
|
-
if
|
|
113
|
+
if LOCAL_MODE:
|
|
114
114
|
return
|
|
115
115
|
|
|
116
116
|
# If the API is saved as an env var in the ~/.env file, then it has already
|
|
@@ -177,7 +177,7 @@ async def init_prefect_blocks():
|
|
|
177
177
|
|
|
178
178
|
# In local mode, create the block that contains the environment variables for all users.
|
|
179
179
|
# In cluster mode, this block has already been created by an admin.
|
|
180
|
-
if
|
|
180
|
+
if LOCAL_MODE:
|
|
181
181
|
|
|
182
182
|
# Get all env var names that start with DASK_GATEWAY_
|
|
183
183
|
regex = re.compile("^DASK_GATEWAY_.*")
|
|
@@ -207,7 +207,7 @@ async def init_prefect_blocks():
|
|
|
207
207
|
# Env vars for current user/owner_id
|
|
208
208
|
|
|
209
209
|
# In cluster mode, read the API key
|
|
210
|
-
if
|
|
210
|
+
if CLUSTER_MODE:
|
|
211
211
|
await read_apikey()
|
|
212
212
|
|
|
213
213
|
# Read env vars that are available from the client env in both local and cluster mode.
|
|
@@ -227,10 +227,10 @@ async def init_prefect_blocks():
|
|
|
227
227
|
user_env_vars[key] = "1"
|
|
228
228
|
|
|
229
229
|
# Save env vars in a secret block for the current user
|
|
230
|
-
await update_prefect_block(format_env_user(BLOCK_NAME_ENV_USER,
|
|
230
|
+
await update_prefect_block(format_env_user(BLOCK_NAME_ENV_USER, OWNER_ID), user_env_vars)
|
|
231
231
|
|
|
232
232
|
# Now read back the blocks so we are sure our env vars are up-to-date
|
|
233
|
-
await read_prefect_blocks(
|
|
233
|
+
await read_prefect_blocks(OWNER_ID)
|
|
234
234
|
|
|
235
235
|
|
|
236
236
|
@sync_compatible
|
|
@@ -277,7 +277,7 @@ async def save_bucket_credentials(
|
|
|
277
277
|
)
|
|
278
278
|
|
|
279
279
|
# Update the prefect block
|
|
280
|
-
await update_prefect_block(format_env_user(BLOCK_NAME_ENV_USER,
|
|
280
|
+
await update_prefect_block(format_env_user(BLOCK_NAME_ENV_USER, OWNER_ID), env_vars)
|
|
281
281
|
|
|
282
282
|
# Save the env vars
|
|
283
283
|
os.environ.update(env_vars)
|
|
@@ -303,7 +303,7 @@ async def remove_bucket_credentials(obs_ids: list[str] | str):
|
|
|
303
303
|
env_vars += [f"S3_{key}_ACCESSKEY", f"S3_{key}_SECRETKEY", f"S3_{key}_ENDPOINT", f"S3_{key}_REGION"]
|
|
304
304
|
|
|
305
305
|
# Update the prefect block
|
|
306
|
-
await update_prefect_block(format_env_user(BLOCK_NAME_ENV_USER,
|
|
306
|
+
await update_prefect_block(format_env_user(BLOCK_NAME_ENV_USER, OWNER_ID), remove_keys=env_vars)
|
|
307
307
|
|
|
308
308
|
# NOTE: I'm not sure we should remove the env vars from os.environ, it's risky because if the env vars
|
|
309
309
|
# are also defined in other blocks, then maybe we don't want to remove them from os.environ.
|
|
@@ -425,7 +425,7 @@ async def get_share_bucket(sub_folder: str = "") -> tuple[S3Bucket, str]:
|
|
|
425
425
|
|
|
426
426
|
# Use a specific block for the current user. Always overwrite it, because the subfolder may have changed.
|
|
427
427
|
else:
|
|
428
|
-
block_name = format_env_user(BLOCK_NAME_SHARE_BUCKET_USER,
|
|
428
|
+
block_name = format_env_user(BLOCK_NAME_SHARE_BUCKET_USER, OWNER_ID) # type: ignore
|
|
429
429
|
|
|
430
430
|
# Read the prefect blocks that contain the S3 authentication
|
|
431
431
|
bucket_name = os.environ["PREFECT_BUCKET_NAME"]
|
|
@@ -0,0 +1,290 @@
|
|
|
1
|
+
# Copyright 2023-2026 Airbus, CS Group
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""This module is used to share common functions between apis"""
|
|
16
|
+
|
|
17
|
+
import logging
|
|
18
|
+
import os
|
|
19
|
+
import shutil
|
|
20
|
+
import tarfile
|
|
21
|
+
import zipfile
|
|
22
|
+
from dataclasses import dataclass
|
|
23
|
+
from datetime import datetime
|
|
24
|
+
from pathlib import Path
|
|
25
|
+
from typing import Any
|
|
26
|
+
|
|
27
|
+
from prefect import get_run_logger
|
|
28
|
+
from prefect.exceptions import MissingContextError
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass
|
|
32
|
+
class AuthInfo:
|
|
33
|
+
"""User authentication information in Keycloak."""
|
|
34
|
+
|
|
35
|
+
# User login (preferred username)
|
|
36
|
+
user_login: str
|
|
37
|
+
|
|
38
|
+
# IAM roles
|
|
39
|
+
iam_roles: list[str]
|
|
40
|
+
|
|
41
|
+
# Oauth2 attributes and/or custom `config` associated to the API key
|
|
42
|
+
attributes: dict[str, Any]
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def read_response_error(response) -> str:
|
|
46
|
+
"""Read and return an HTTP response error detail."""
|
|
47
|
+
|
|
48
|
+
# Try to read the response detail or error
|
|
49
|
+
try:
|
|
50
|
+
json = response.json()
|
|
51
|
+
if isinstance(json, str):
|
|
52
|
+
return json
|
|
53
|
+
return json.get("detail") or json.get("description") or json["error"]
|
|
54
|
+
|
|
55
|
+
# If this fail, get the full response content
|
|
56
|
+
except Exception: # pylint: disable=broad-exception-caught
|
|
57
|
+
return response.content.decode("utf-8", errors="ignore")
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def get_href_service(rs_server_href, env_var) -> str:
|
|
61
|
+
"""Get specific href link."""
|
|
62
|
+
if from_env := os.getenv(env_var, None):
|
|
63
|
+
return from_env.rstrip("/")
|
|
64
|
+
if not rs_server_href:
|
|
65
|
+
raise RuntimeError("RS-Server URL is undefined")
|
|
66
|
+
return rs_server_href.rstrip("/")
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def env_bool(var: str, default: bool) -> bool:
|
|
70
|
+
"""
|
|
71
|
+
Return True if an environemnt variable is set to 1, true or yes (case insensitive).
|
|
72
|
+
Return False if set to 0, false or no (case insensitive).
|
|
73
|
+
Return the default value if not set or set to a different value.
|
|
74
|
+
"""
|
|
75
|
+
val = os.getenv(var, str(default)).lower()
|
|
76
|
+
if val in ("y", "yes", "t", "true", "on", "1"):
|
|
77
|
+
return True
|
|
78
|
+
if val in ("n", "no", "f", "false", "off", "0"):
|
|
79
|
+
return False
|
|
80
|
+
return default
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def strftime_millis(date: datetime):
|
|
84
|
+
"""Format datetime with milliseconds precision"""
|
|
85
|
+
return date.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def create_valcover_filter(
|
|
89
|
+
start_datetime: datetime | str,
|
|
90
|
+
end_datetime: datetime | str,
|
|
91
|
+
product_type: str,
|
|
92
|
+
) -> dict:
|
|
93
|
+
"""Creates a ValCover filter from the input values to be used in flows
|
|
94
|
+
|
|
95
|
+
Args:
|
|
96
|
+
start_datetime: Start datetime for the time interval used to filter the files
|
|
97
|
+
end_datetime: End datetime for the time interval used to filter the files
|
|
98
|
+
product_type: Auxiliary file type wanted
|
|
99
|
+
|
|
100
|
+
Returns:
|
|
101
|
+
dict: ValCover filter
|
|
102
|
+
"""
|
|
103
|
+
# Convert datetime inputs to str
|
|
104
|
+
if isinstance(start_datetime, datetime):
|
|
105
|
+
start_datetime = strftime_millis(start_datetime)
|
|
106
|
+
if isinstance(end_datetime, datetime):
|
|
107
|
+
end_datetime = strftime_millis(end_datetime)
|
|
108
|
+
|
|
109
|
+
# CQL2 filter: we use a filter combining a ValCover filter and a product type filter
|
|
110
|
+
return {
|
|
111
|
+
"op": "and",
|
|
112
|
+
"args": [
|
|
113
|
+
{"op": "=", "args": [{"property": "product:type"}, product_type]},
|
|
114
|
+
{
|
|
115
|
+
"op": "t_contains",
|
|
116
|
+
"args": [
|
|
117
|
+
{"interval": [{"property": "start_datetime"}, {"property": "end_datetime"}]},
|
|
118
|
+
{"interval": [start_datetime, end_datetime]},
|
|
119
|
+
],
|
|
120
|
+
},
|
|
121
|
+
],
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
# True if the 'RSPY_LOCAL_MODE' environemnt variable is set to 1, true or yes (case insensitive).
|
|
126
|
+
# By default: if not set or set to a different value, return False.
|
|
127
|
+
LOCAL_MODE: bool = env_bool("RSPY_LOCAL_MODE", default=False)
|
|
128
|
+
|
|
129
|
+
# Cluster mode is the opposite of local mode
|
|
130
|
+
CLUSTER_MODE: bool = not LOCAL_MODE
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _get_logger():
|
|
134
|
+
"""Return the Prefect run logger when available, otherwise a module logger."""
|
|
135
|
+
try:
|
|
136
|
+
return get_run_logger()
|
|
137
|
+
except MissingContextError:
|
|
138
|
+
return logging.getLogger(__name__)
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _is_safe_extract_path(extract_to: Path, member_name: str) -> bool:
|
|
142
|
+
"""Return whether an archive member stays within the target directory."""
|
|
143
|
+
member_path = Path(member_name.replace("\\", "/"))
|
|
144
|
+
if member_path.is_absolute():
|
|
145
|
+
return False
|
|
146
|
+
try:
|
|
147
|
+
(extract_to / member_path).resolve().relative_to(extract_to.resolve())
|
|
148
|
+
except ValueError:
|
|
149
|
+
return False
|
|
150
|
+
return True
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def extract_zip(zip_path: Path, extract_to: Path):
|
|
154
|
+
"""Extract a ZIP archive into the target directory."""
|
|
155
|
+
logger = get_run_logger()
|
|
156
|
+
logger.info(f"Extracting ZIP: {zip_path} -> {extract_to}")
|
|
157
|
+
|
|
158
|
+
with zipfile.ZipFile(zip_path, "r") as zip_ref:
|
|
159
|
+
count = 0
|
|
160
|
+
for name in zip_ref.namelist():
|
|
161
|
+
if not _is_safe_extract_path(extract_to, name):
|
|
162
|
+
logger.warning(f"Skipping unsafe ZIP member: {name}")
|
|
163
|
+
continue
|
|
164
|
+
destination = (extract_to / name.replace("\\", "/")).resolve()
|
|
165
|
+
if name.endswith("/"):
|
|
166
|
+
destination.mkdir(parents=True, exist_ok=True)
|
|
167
|
+
else:
|
|
168
|
+
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
169
|
+
with zip_ref.open(name) as src, destination.open("wb") as dst:
|
|
170
|
+
shutil.copyfileobj(src, dst)
|
|
171
|
+
count += 1
|
|
172
|
+
logger.info(f"ZIP contains {count} safe entries")
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def extract_tar(file_path: Path, extract_to: Path) -> int:
|
|
176
|
+
"""Extract a TAR-compatible archive into the target directory."""
|
|
177
|
+
logger = get_run_logger()
|
|
178
|
+
logger.info(f"Extracting TAR archive: {file_path} -> {extract_to}")
|
|
179
|
+
|
|
180
|
+
with tarfile.open(file_path, "r:*") as tar:
|
|
181
|
+
count = 0
|
|
182
|
+
for member in tar.getmembers():
|
|
183
|
+
if member.issym() or member.islnk() or member.isdev():
|
|
184
|
+
logger.warning(f"Skipping unsafe TAR member: {member.name}")
|
|
185
|
+
continue
|
|
186
|
+
if not _is_safe_extract_path(extract_to, member.name):
|
|
187
|
+
logger.warning(f"Skipping unsafe TAR member: {member.name}")
|
|
188
|
+
continue
|
|
189
|
+
destination = (extract_to / member.name).resolve()
|
|
190
|
+
if member.isdir():
|
|
191
|
+
destination.mkdir(parents=True, exist_ok=True)
|
|
192
|
+
else:
|
|
193
|
+
extracted = tar.extractfile(member)
|
|
194
|
+
if extracted is None:
|
|
195
|
+
continue
|
|
196
|
+
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
197
|
+
with extracted, destination.open("wb") as dst:
|
|
198
|
+
shutil.copyfileobj(extracted, dst)
|
|
199
|
+
count += 1
|
|
200
|
+
logger.info(f"TAR archive contains {count} safe entries")
|
|
201
|
+
return count
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def strip_archive_suffix(name: str) -> str:
|
|
205
|
+
"""Return the asset name without its supported archive suffix."""
|
|
206
|
+
normalized_name = name.lower()
|
|
207
|
+
for suffix in (".tar.gz", ".tgz", ".zip", ".tar"):
|
|
208
|
+
if normalized_name.endswith(suffix):
|
|
209
|
+
return name[: -len(suffix)]
|
|
210
|
+
return name
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def get_upload_prefix(asset_href: str, asset_name: str) -> str:
|
|
214
|
+
"""Return the S3 prefix where extracted content should be uploaded."""
|
|
215
|
+
logger = _get_logger()
|
|
216
|
+
parent_prefix = asset_href.rsplit("/", 1)[0]
|
|
217
|
+
|
|
218
|
+
if asset_name.lower().endswith(".zip"):
|
|
219
|
+
result = parent_prefix + "/"
|
|
220
|
+
logger.info(f"get_upload_prefix returning ZIP prefix={result}")
|
|
221
|
+
return result
|
|
222
|
+
|
|
223
|
+
last_segment = parent_prefix.rsplit("/", 1)[-1]
|
|
224
|
+
normalized_segment = strip_archive_suffix(last_segment)
|
|
225
|
+
|
|
226
|
+
if normalized_segment != last_segment:
|
|
227
|
+
base_prefix = parent_prefix.rsplit("/", 1)[0]
|
|
228
|
+
result = base_prefix + "/" + normalized_segment + "/"
|
|
229
|
+
logger.info(f"get_upload_prefix returning normalized prefix={result}")
|
|
230
|
+
return result
|
|
231
|
+
|
|
232
|
+
result = parent_prefix + "/"
|
|
233
|
+
logger.info(f"get_upload_prefix returning parent prefix={result}")
|
|
234
|
+
return result
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def _extract_nested_archive(full_path: Path) -> bool:
|
|
238
|
+
"""Extract a nested TAR-compatible archive and delete it on success."""
|
|
239
|
+
logger = get_run_logger()
|
|
240
|
+
logger.info(f"Found nested archive: {full_path}")
|
|
241
|
+
extracted_members = extract_tar(full_path, full_path.parent)
|
|
242
|
+
if not extracted_members:
|
|
243
|
+
logger.warning("Skipping removal of nested archive because no members were extracted: " f"{full_path}")
|
|
244
|
+
return False
|
|
245
|
+
full_path.unlink()
|
|
246
|
+
return True
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def recursive_extract(folder: Path) -> int:
|
|
250
|
+
"""Extract nested TAR-compatible archives found anywhere under ``folder``."""
|
|
251
|
+
logger = get_run_logger()
|
|
252
|
+
extracted_count = 0
|
|
253
|
+
extracted = True
|
|
254
|
+
|
|
255
|
+
while extracted:
|
|
256
|
+
extracted = False
|
|
257
|
+
|
|
258
|
+
for root, _, files in os.walk(folder):
|
|
259
|
+
for file in files:
|
|
260
|
+
full_path = Path(root) / file
|
|
261
|
+
|
|
262
|
+
if file.lower().endswith((".tar", ".tgz", ".tar.gz")) and _extract_nested_archive(full_path):
|
|
263
|
+
extracted = True
|
|
264
|
+
extracted_count += 1
|
|
265
|
+
|
|
266
|
+
logger.info(f"Processed {extracted_count} nested archive(s)")
|
|
267
|
+
return extracted_count
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def normalize_extract_dir(extract_dir: Path) -> Path:
|
|
271
|
+
"""
|
|
272
|
+
Return the directory that should be used as the upload root.
|
|
273
|
+
|
|
274
|
+
If the extraction produced a single top-level directory, descend into it to
|
|
275
|
+
avoid creating an unnecessary extra folder level in S3.
|
|
276
|
+
"""
|
|
277
|
+
logger = get_run_logger()
|
|
278
|
+
root_items = list(extract_dir.iterdir())
|
|
279
|
+
|
|
280
|
+
if len(root_items) != 1:
|
|
281
|
+
logger.info("No normalization needed, multiple root items found")
|
|
282
|
+
return extract_dir
|
|
283
|
+
|
|
284
|
+
root_item = root_items[0]
|
|
285
|
+
if not root_item.is_dir():
|
|
286
|
+
logger.info("No normalization needed, single root item is not a directory")
|
|
287
|
+
return extract_dir
|
|
288
|
+
|
|
289
|
+
logger.info(f"Normalizing folder structure: entering {root_item}")
|
|
290
|
+
return root_item
|