rs-client-libraries 1.0a8__tar.gz → 1.0a8.post16.dev0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/PKG-INFO +1 -1
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/pyproject.toml +1 -1
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/__init__.py +1 -1
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/cadip_flow.py +51 -0
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/catalog_flow.py +62 -25
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/dpr_flow.py +38 -24
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/flow_utils.py +9 -0
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/on_demand_processing.py +2 -112
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/payload_generator.py +8 -2
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/payload_template.py +4 -4
- rs_client_libraries-1.0a8.post16.dev0/rs_workflows/prip_flow.py +131 -0
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/record_performance.py +4 -3
- rs_client_libraries-1.0a8/rs_workflows/prip_flow.py +0 -69
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/LICENSE +0 -0
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/README.md +0 -0
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/ogcapi/__init__.py +0 -0
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/ogcapi/dpr_client.py +0 -0
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/ogcapi/ogcapi_client.py +0 -0
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/ogcapi/staging_client.py +0 -0
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/osam_client.py +0 -0
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/rs_client.py +0 -0
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/stac/__init__.py +0 -0
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/stac/auxip_client.py +0 -0
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/stac/cadip_client.py +0 -0
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/stac/catalog_client.py +0 -0
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/stac/prip_client.py +0 -0
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/stac/stac_base.py +0 -0
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_common/__init__.py +0 -0
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_common/config.py +0 -0
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_common/init_opentelemetry.py +0 -0
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_common/logging.py +0 -0
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_common/prefect_utils.py +0 -0
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_common/utils.py +0 -0
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/__init__.py +0 -0
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/auxip_flow.py +0 -0
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/init_pi_db_flow.py +0 -0
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/on_demand/__init__.py +0 -0
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/on_demand/common/__init__.py +0 -0
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/on_demand/common/l0.py +0 -0
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/on_demand/common/l0_last_steps.py +0 -0
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/on_demand/common/staging.py +0 -0
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/on_demand/common/types.py +0 -0
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/on_demand/sentinel1/__init__.py +0 -0
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/on_demand/sentinel1/s1_l0.py +0 -0
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/on_demand/sentinel3/__init__.py +0 -0
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/on_demand/sentinel3/s3_l0.py +0 -0
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/operation/__init__.py +0 -0
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/operation/init_quota_monitoring_flow.py +0 -0
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/operation/osam_flows.py +0 -0
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/operation/quota_monitoring_db_models.py +0 -0
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/operation/quota_monitoring_flow.py +0 -0
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/payload_builder.py +0 -0
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/pi_db_models.py +0 -0
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/staging_flow.py +0 -0
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/storage_configuration.py +0 -0
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/utils/__init__.py +0 -0
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/utils/artifact_verbose.py +0 -0
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/utils/cadip.py +0 -0
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/utils/catalog.py +0 -0
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/utils/dask.py +0 -0
- {rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/utils/dpr.py +0 -0
|
@@ -18,7 +18,7 @@ dynamic = ["version"] # see: https://github.com/mtkennerly/poetry-dynamic-versio
|
|
|
18
18
|
|
|
19
19
|
[tool.poetry]
|
|
20
20
|
name = "rs-client-libraries"
|
|
21
|
-
version = "1.0a8"
|
|
21
|
+
version = "1.0a8.post16.dev0"
|
|
22
22
|
description = "Copernicus Sentinel rs-client library and processing workflows."
|
|
23
23
|
license = "Apache-2.0"
|
|
24
24
|
authors = ["ESA under contract with Airbus and CS <ops_coprs@airbus.com>"]
|
{rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/cadip_flow.py
RENAMED
|
@@ -19,6 +19,7 @@ from pystac import ItemCollection
|
|
|
19
19
|
|
|
20
20
|
from rs_client.stac.cadip_client import CadipClient
|
|
21
21
|
from rs_workflows.flow_utils import FlowEnv, FlowEnvArgs
|
|
22
|
+
from rs_workflows.staging_flow import staging_task
|
|
22
23
|
|
|
23
24
|
|
|
24
25
|
@flow(name="Cadip search")
|
|
@@ -58,6 +59,56 @@ async def search(
|
|
|
58
59
|
return found
|
|
59
60
|
|
|
60
61
|
|
|
62
|
+
@flow(name="On-demand Cadip staging")
|
|
63
|
+
async def on_demand_cadip_staging(
|
|
64
|
+
env: FlowEnvArgs,
|
|
65
|
+
cadip_collection_identifier: str,
|
|
66
|
+
session_identifier: str,
|
|
67
|
+
catalog_collection_identifier: str,
|
|
68
|
+
staging_retries: int = 3,
|
|
69
|
+
staging_retry_delay: int = 60,
|
|
70
|
+
):
|
|
71
|
+
"""
|
|
72
|
+
Flow to retrieve a session, stage it and add the STAC item into the catalog.
|
|
73
|
+
|
|
74
|
+
Args:
|
|
75
|
+
env: Prefect flow environment
|
|
76
|
+
cadip_collection_identifier: CADIP collection identifier that contains the mission and station
|
|
77
|
+
(e.g. s1_ins for Sentinel-1 sessions from the Inuvik station)
|
|
78
|
+
session_identifier: Session identifier
|
|
79
|
+
catalog_collection_identifier: Catalog collection identifier where CADIP sessions and AUX data are staged
|
|
80
|
+
"""
|
|
81
|
+
|
|
82
|
+
# Init flow environment and opentelemetry span
|
|
83
|
+
flow_env = FlowEnv(env)
|
|
84
|
+
with flow_env.start_span(__name__, "on-demand-cadip-staging"):
|
|
85
|
+
|
|
86
|
+
# Search Cadip sessions
|
|
87
|
+
cadip_items = search_task.with_options(
|
|
88
|
+
retries=3,
|
|
89
|
+
retry_delay_seconds=60,
|
|
90
|
+
).submit(
|
|
91
|
+
flow_env.serialize(),
|
|
92
|
+
cadip_collection_identifier,
|
|
93
|
+
session_identifier,
|
|
94
|
+
error_if_empty=True,
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
# Stage Cadip items.
|
|
98
|
+
staged = staging_task.with_options(
|
|
99
|
+
retries=staging_retries,
|
|
100
|
+
retry_delay_seconds=staging_retry_delay,
|
|
101
|
+
).submit(
|
|
102
|
+
flow_env.serialize(),
|
|
103
|
+
cadip_items,
|
|
104
|
+
catalog_collection_identifier,
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
# Wait for last task to end.
|
|
108
|
+
# NOTE: use .result() and not .wait() to unwrap and propagate exceptions, if any.
|
|
109
|
+
staged.result() # type: ignore[unused-coroutine]
|
|
110
|
+
|
|
111
|
+
|
|
61
112
|
###########################
|
|
62
113
|
# Call the flows as tasks #
|
|
63
114
|
###########################
|
{rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/catalog_flow.py
RENAMED
|
@@ -17,10 +17,15 @@
|
|
|
17
17
|
import json
|
|
18
18
|
|
|
19
19
|
from prefect import flow, get_run_logger, task
|
|
20
|
-
from pystac import
|
|
20
|
+
from pystac import ItemCollection
|
|
21
21
|
|
|
22
22
|
from rs_client.stac.catalog_client import CatalogClient
|
|
23
|
-
from rs_workflows.flow_utils import
|
|
23
|
+
from rs_workflows.flow_utils import (
|
|
24
|
+
DprProcessedItemMetadata,
|
|
25
|
+
FlowEnv,
|
|
26
|
+
FlowEnvArgs,
|
|
27
|
+
FlowGeneratedProduct,
|
|
28
|
+
)
|
|
24
29
|
|
|
25
30
|
#################
|
|
26
31
|
# Catalog flows #
|
|
@@ -71,8 +76,8 @@ async def catalog_search(
|
|
|
71
76
|
@task(name="Publish to catalog")
|
|
72
77
|
async def publish(
|
|
73
78
|
env: FlowEnvArgs,
|
|
74
|
-
|
|
75
|
-
|
|
79
|
+
generated_product_to_collection_identifier: list[FlowGeneratedProduct],
|
|
80
|
+
items_metadata: list[DprProcessedItemMetadata],
|
|
76
81
|
):
|
|
77
82
|
"""
|
|
78
83
|
Publish items to the catalog
|
|
@@ -80,7 +85,7 @@ async def publish(
|
|
|
80
85
|
Args:
|
|
81
86
|
env: Prefect flow environment
|
|
82
87
|
collection: Catalog collection identifier where the items are published
|
|
83
|
-
|
|
88
|
+
items_metadata: List of DprProcessedItemMetadata containing items to publish
|
|
84
89
|
"""
|
|
85
90
|
logger = get_run_logger()
|
|
86
91
|
flow_env = FlowEnv(env)
|
|
@@ -88,10 +93,14 @@ async def publish(
|
|
|
88
93
|
catalog_client: CatalogClient = flow_env.rs_client.get_catalog_client()
|
|
89
94
|
|
|
90
95
|
with flow_env.start_span(__name__, "publish-to-catalog"):
|
|
91
|
-
for
|
|
96
|
+
for item_metadata in items_metadata:
|
|
97
|
+
item = item_metadata.stac_item
|
|
92
98
|
try:
|
|
93
|
-
# Resolve destination collection using
|
|
94
|
-
target_collection = resolve_collection(
|
|
99
|
+
# Resolve destination collection using metadata from STAC item
|
|
100
|
+
target_collection = resolve_collection(
|
|
101
|
+
item_metadata,
|
|
102
|
+
generated_product_to_collection_identifier,
|
|
103
|
+
)
|
|
95
104
|
# TEMPFIX: Items generated by DPR with [] as geometry
|
|
96
105
|
# Also, remove some fields that s1 l0 processor is not compliant with
|
|
97
106
|
# regarding the catalog ingestion
|
|
@@ -107,7 +116,7 @@ async def publish(
|
|
|
107
116
|
# TEMPFIX END
|
|
108
117
|
logger.info(
|
|
109
118
|
"Writing product %s to %s. This may take a while...",
|
|
110
|
-
item.id
|
|
119
|
+
item.id,
|
|
111
120
|
target_collection,
|
|
112
121
|
)
|
|
113
122
|
# Publish item to catalog
|
|
@@ -118,7 +127,7 @@ async def publish(
|
|
|
118
127
|
response = catalog_client.add_item(target_collection, item, timeout=21600)
|
|
119
128
|
logger.info(
|
|
120
129
|
"Writing product %s to %s done. Response %s with message: %s",
|
|
121
|
-
item.id
|
|
130
|
+
item.id,
|
|
122
131
|
target_collection,
|
|
123
132
|
response.status_code,
|
|
124
133
|
response.text,
|
|
@@ -159,33 +168,61 @@ async def get_item(
|
|
|
159
168
|
return catalog_client.get_item(target_collection, item)
|
|
160
169
|
|
|
161
170
|
|
|
162
|
-
def resolve_collection(
|
|
163
|
-
|
|
171
|
+
def resolve_collection(
|
|
172
|
+
item_metadata: DprProcessedItemMetadata,
|
|
173
|
+
generated_product_to_collection_identifier: list[FlowGeneratedProduct],
|
|
174
|
+
) -> str:
|
|
175
|
+
"""Resolve the target collection for a given product metadata.
|
|
164
176
|
|
|
165
|
-
|
|
166
|
-
|
|
177
|
+
The matching logic uses both output_product_id and product_type from the
|
|
178
|
+
DprProcessedItemMetadata to find the corresponding FlowGeneratedProduct.
|
|
179
|
+
Wildcards ('*') are supported for product_type in FlowGeneratedProduct.
|
|
167
180
|
|
|
168
181
|
Returns:
|
|
169
182
|
The resolved target collection name.
|
|
170
183
|
|
|
171
184
|
Raises:
|
|
172
|
-
ValueError: If the product
|
|
185
|
+
ValueError: If the product cannot be resolved to any collection.
|
|
173
186
|
"""
|
|
174
187
|
logger = get_run_logger()
|
|
175
188
|
logger.info(
|
|
176
|
-
f"Resolving target collection for
|
|
177
|
-
f"with
|
|
189
|
+
f"Resolving target collection for item metadata: {item_metadata}"
|
|
190
|
+
f" with generated_product_to_collection_identifier: {generated_product_to_collection_identifier}",
|
|
178
191
|
)
|
|
179
192
|
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
193
|
+
# We prioritize matches in the following order:
|
|
194
|
+
# 1. Exact Name, Exact Type
|
|
195
|
+
# 2. Exact Name, Wildcard Type (least specific)
|
|
196
|
+
# wildcards for name are NOT supported, they don't make sense in this context as tasktable don't support them
|
|
197
|
+
|
|
198
|
+
target_collection = None
|
|
199
|
+
for gen_prod in generated_product_to_collection_identifier:
|
|
200
|
+
if gen_prod.name != item_metadata.output_product_id:
|
|
201
|
+
continue
|
|
202
|
+
# 1. Exact Name, Exact Type
|
|
203
|
+
if item_metadata.product_type == gen_prod.product_type:
|
|
204
|
+
target_collection = gen_prod.collection_name or gen_prod.product_type
|
|
205
|
+
logger.info(f"Exact match (Name & Type) found: {gen_prod}, resolved collection: {target_collection}")
|
|
206
|
+
break
|
|
207
|
+
# 2. Exact Name, Wildcard Type
|
|
208
|
+
if gen_prod.product_type == "*" and target_collection is None:
|
|
209
|
+
target_collection = gen_prod.collection_name or gen_prod.product_type
|
|
210
|
+
# This protection is also set in build_output_products from payload_generator.py,
|
|
211
|
+
# but it's good to have it here as well for safety or a future change in logic.
|
|
212
|
+
if target_collection == "*":
|
|
213
|
+
raise RuntimeError(
|
|
214
|
+
f"The product type in generated_product_to_collection_identifier cannot be '*' "
|
|
215
|
+
f"if the collection name is not specified for product '{gen_prod.name}'",
|
|
216
|
+
)
|
|
217
|
+
logger.info(
|
|
218
|
+
f"Match with Exact Name & Wildcard Type: {gen_prod}, resolved collection: {target_collection}",
|
|
219
|
+
)
|
|
187
220
|
|
|
188
221
|
if not target_collection:
|
|
189
|
-
raise ValueError(
|
|
222
|
+
raise ValueError(
|
|
223
|
+
f"No match for {item_metadata.output_product_id} in "
|
|
224
|
+
f"{generated_product_to_collection_identifier}. "
|
|
225
|
+
f"Could not find a collection to publish the stac_item from: {item_metadata}",
|
|
226
|
+
)
|
|
190
227
|
|
|
191
228
|
return target_collection
|
{rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/dpr_flow.py
RENAMED
|
@@ -32,7 +32,7 @@ from pystac import Asset, Item
|
|
|
32
32
|
from rs_client.ogcapi.dpr_client import ClusterInfo, DprClient, DprProcessor
|
|
33
33
|
from rs_common import prefect_utils
|
|
34
34
|
from rs_workflows import catalog_flow
|
|
35
|
-
from rs_workflows.flow_utils import FlowEnv, FlowEnvArgs
|
|
35
|
+
from rs_workflows.flow_utils import DprProcessedItemMetadata, FlowEnv, FlowEnvArgs
|
|
36
36
|
from rs_workflows.payload_template import PayloadSchema
|
|
37
37
|
from rs_workflows.record_performance import record_performance_indicators
|
|
38
38
|
|
|
@@ -83,11 +83,11 @@ def extract_products_and_zattrs(files: list[str], base_path: str):
|
|
|
83
83
|
if len(parts) != 2:
|
|
84
84
|
continue
|
|
85
85
|
|
|
86
|
-
|
|
86
|
+
product_dirname = parts[0]
|
|
87
87
|
|
|
88
88
|
# 1: base_path/product/.zattrs
|
|
89
89
|
if parts[1] == ".zattrs":
|
|
90
|
-
dirs_and_attrs.append((
|
|
90
|
+
dirs_and_attrs.append((product_dirname, file))
|
|
91
91
|
|
|
92
92
|
return dirs_and_attrs
|
|
93
93
|
|
|
@@ -248,7 +248,7 @@ def update_eopf_assets(
|
|
|
248
248
|
input_products: list[dict],
|
|
249
249
|
payload: PayloadSchema,
|
|
250
250
|
dpr_processor: str,
|
|
251
|
-
) ->
|
|
251
|
+
) -> list[DprProcessedItemMetadata]:
|
|
252
252
|
"""Update EOPF assets by extracting metadata and creating STAC items.
|
|
253
253
|
|
|
254
254
|
This Prefect task processes output products from a DPR (Data Processing Request)
|
|
@@ -266,15 +266,15 @@ def update_eopf_assets(
|
|
|
266
266
|
env: Environment configuration object containing runtime settings.
|
|
267
267
|
input_products: List of dictionaries representing input product metadata.
|
|
268
268
|
payload: PayloadSchema object containing I/O configuration, including
|
|
269
|
-
|
|
269
|
+
output_product paths to scan for .zattrs files.
|
|
270
270
|
dpr_processor: str
|
|
271
271
|
DPR processor name
|
|
272
272
|
|
|
273
273
|
Returns:
|
|
274
|
-
A
|
|
275
|
-
-
|
|
276
|
-
-
|
|
277
|
-
|
|
274
|
+
A list of DprProcessedItemMetadata objects, each containing:
|
|
275
|
+
- stac_item (Item): STAC item created from EOPF metadata.
|
|
276
|
+
- product_type (str): Product type extracted from the .zattrs file.
|
|
277
|
+
- output_product_id (str): The ID of the original output product.
|
|
278
278
|
|
|
279
279
|
Raises:
|
|
280
280
|
RuntimeError: If any .zattrs file cannot be read or does not contain
|
|
@@ -288,7 +288,7 @@ def update_eopf_assets(
|
|
|
288
288
|
logger = get_run_logger()
|
|
289
289
|
logger.info("Starting EOPF asset update.")
|
|
290
290
|
logger.info(f"Payload received: {payload}")
|
|
291
|
-
logger.info("Input products:
|
|
291
|
+
logger.info(f"Input products: {input_products}")
|
|
292
292
|
|
|
293
293
|
if payload.io is None:
|
|
294
294
|
raise RuntimeError("Payload I/O configuration is missing.")
|
|
@@ -296,15 +296,20 @@ def update_eopf_assets(
|
|
|
296
296
|
zattrs_list = []
|
|
297
297
|
for prod in payload.io.output_products:
|
|
298
298
|
path = prod.path
|
|
299
|
-
|
|
300
|
-
|
|
299
|
+
# Keep track of which output product ID each .zattrs belongs to
|
|
300
|
+
new_zattrs = extract_products_and_zattrs(s3_list(path), path)
|
|
301
|
+
for name, loc in new_zattrs:
|
|
302
|
+
zattrs_list.append((name, loc, prod.id))
|
|
303
|
+
logger.info(
|
|
304
|
+
f"Product {prod.id} has been added to the list and will be published to the catalog."
|
|
305
|
+
f" The path is {path}",
|
|
306
|
+
)
|
|
301
307
|
|
|
302
308
|
# List & extract
|
|
303
|
-
logger.info(f"Found {len(zattrs_list)} .zattrs files
|
|
309
|
+
logger.info(f"Found {len(zattrs_list)} .zattrs files. The list: {zattrs_list}")
|
|
304
310
|
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
for product_name, zattrs_s3_location in zattrs_list:
|
|
311
|
+
items_metadata = []
|
|
312
|
+
for product_name, zattrs_s3_location, output_product_id in zattrs_list:
|
|
308
313
|
logger.info(f"Product = {product_name} | zattrs = {zattrs_s3_location}")
|
|
309
314
|
# Read metadata
|
|
310
315
|
zattrs_data = read_zattrs_sync(zattrs_s3_location)
|
|
@@ -320,18 +325,24 @@ def update_eopf_assets(
|
|
|
320
325
|
|
|
321
326
|
product_type = zattrs_data["stac_discovery"]["properties"].get("product:type", None)
|
|
322
327
|
logger.info(f"Extracted EOPF product type: {product_type}")
|
|
323
|
-
product_types.append(product_type)
|
|
324
328
|
|
|
325
329
|
eopf_item = zattrs_data["stac_discovery"]
|
|
326
330
|
logger.debug(f"EOPF discovery metadata extracted: {eopf_item}")
|
|
327
331
|
|
|
328
332
|
# Build STAC items
|
|
329
|
-
|
|
330
|
-
|
|
333
|
+
stac_item = create_stac_item(env, input_products, eopf_item, zattrs_s3_location, product_name, dpr_processor)
|
|
334
|
+
|
|
335
|
+
items_metadata.append(
|
|
336
|
+
DprProcessedItemMetadata(
|
|
337
|
+
stac_item=stac_item,
|
|
338
|
+
product_type=product_type,
|
|
339
|
+
output_product_id=output_product_id,
|
|
340
|
+
),
|
|
331
341
|
)
|
|
332
|
-
logger.info(f"Added one stac item to the already existing list. Length: {len(
|
|
342
|
+
logger.info(f"Added one stac item metadata to the already existing list. Length: {len(items_metadata)}.")
|
|
333
343
|
|
|
334
|
-
|
|
344
|
+
logger.info(f"List with stac_items to be published: {items_metadata}.")
|
|
345
|
+
return items_metadata
|
|
335
346
|
|
|
336
347
|
|
|
337
348
|
def compute_eopf_origin_datetime(env, input_products) -> str:
|
|
@@ -412,7 +423,7 @@ async def run_processor(
|
|
|
412
423
|
cluster_info: ClusterInfo,
|
|
413
424
|
s3_payload_run: str,
|
|
414
425
|
input_products: list[dict],
|
|
415
|
-
) -> list[
|
|
426
|
+
) -> list[DprProcessedItemMetadata]:
|
|
416
427
|
"""
|
|
417
428
|
Run the DPR processor.
|
|
418
429
|
|
|
@@ -490,7 +501,10 @@ async def run_processor(
|
|
|
490
501
|
except FileNotFoundError:
|
|
491
502
|
logger.info(f"No processor log file was uploaded under: {s3_payload_dir!r}")
|
|
492
503
|
|
|
493
|
-
|
|
504
|
+
items_metadata = update_eopf_assets(flow_env, input_products, payload, processor)
|
|
505
|
+
eopf_stac_items = [asset.stac_item for asset in items_metadata]
|
|
506
|
+
eopf_types = [asset.product_type for asset in items_metadata]
|
|
507
|
+
|
|
494
508
|
# Wait for the job to finish
|
|
495
509
|
record_performance_indicators( # type: ignore
|
|
496
510
|
stop_date=datetime.datetime.now(),
|
|
@@ -500,4 +514,4 @@ async def run_processor(
|
|
|
500
514
|
dpr_processor_name=processor,
|
|
501
515
|
eopf_types=eopf_types,
|
|
502
516
|
)
|
|
503
|
-
return
|
|
517
|
+
return items_metadata
|
{rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/flow_utils.py
RENAMED
|
@@ -367,6 +367,15 @@ class DprProcessOut:
|
|
|
367
367
|
product_identifier: list[Item] = field(default_factory=list)
|
|
368
368
|
|
|
369
369
|
|
|
370
|
+
@dataclass
|
|
371
|
+
class DprProcessedItemMetadata:
|
|
372
|
+
"""Metadata for a DPR processed item."""
|
|
373
|
+
|
|
374
|
+
output_product_id: str
|
|
375
|
+
product_type: str | None
|
|
376
|
+
stac_item: Item
|
|
377
|
+
|
|
378
|
+
|
|
370
379
|
class RetryConfig(BaseModel):
|
|
371
380
|
"""
|
|
372
381
|
Args:
|
|
@@ -28,13 +28,11 @@ from prefect.artifacts import acreate_markdown_artifact
|
|
|
28
28
|
|
|
29
29
|
from rs_client.ogcapi.dpr_client import ClusterInfo
|
|
30
30
|
from rs_common import prefect_utils
|
|
31
|
-
from
|
|
32
|
-
from rs_workflows import auxip_flow, cadip_flow, catalog_flow, prip_flow
|
|
31
|
+
from rs_workflows import auxip_flow, catalog_flow
|
|
33
32
|
from rs_workflows.dpr_flow import run_processor
|
|
34
|
-
from rs_workflows.flow_utils import DprProcessIn, FlowEnv,
|
|
33
|
+
from rs_workflows.flow_utils import DprProcessIn, FlowEnv, RetryConfig
|
|
35
34
|
from rs_workflows.payload_builder import build_cql2_json, build_unit_list
|
|
36
35
|
from rs_workflows.payload_generator import generate_payload
|
|
37
|
-
from rs_workflows.staging_flow import staging_task
|
|
38
36
|
|
|
39
37
|
|
|
40
38
|
def build_dask_dashboard_url_message(cluster_instance: str | None) -> str:
|
|
@@ -268,111 +266,3 @@ async def dpr_processing(
|
|
|
268
266
|
published.result() # type: ignore[unused-coroutine]
|
|
269
267
|
|
|
270
268
|
return
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
@flow(name="On-demand Cadip staging")
|
|
274
|
-
async def on_demand_cadip_staging(
|
|
275
|
-
env: FlowEnvArgs,
|
|
276
|
-
cadip_collection_identifier: str,
|
|
277
|
-
session_identifier: str,
|
|
278
|
-
catalog_collection_identifier: str,
|
|
279
|
-
staging_retries: int = 3,
|
|
280
|
-
staging_retry_delay: int = 60,
|
|
281
|
-
):
|
|
282
|
-
"""
|
|
283
|
-
Flow to retrieve a session, stage it and add the STAC item into the catalog.
|
|
284
|
-
|
|
285
|
-
Args:
|
|
286
|
-
env: Prefect flow environment
|
|
287
|
-
cadip_collection_identifier: CADIP collection identifier that contains the mission and station
|
|
288
|
-
(e.g. s1_ins for Sentinel-1 sessions from the Inuvik station)
|
|
289
|
-
session_identifier: Session identifier
|
|
290
|
-
catalog_collection_identifier: Catalog collection identifier where CADIP sessions and AUX data are staged
|
|
291
|
-
"""
|
|
292
|
-
|
|
293
|
-
# Init flow environment and opentelemetry span
|
|
294
|
-
flow_env = FlowEnv(env)
|
|
295
|
-
with flow_env.start_span(__name__, "on-demand-cadip-staging"):
|
|
296
|
-
|
|
297
|
-
# Search Cadip sessions
|
|
298
|
-
cadip_items = cadip_flow.search_task.with_options(
|
|
299
|
-
retries=3,
|
|
300
|
-
retry_delay_seconds=60,
|
|
301
|
-
).submit(
|
|
302
|
-
flow_env.serialize(),
|
|
303
|
-
cadip_collection_identifier,
|
|
304
|
-
session_identifier,
|
|
305
|
-
error_if_empty=True,
|
|
306
|
-
)
|
|
307
|
-
|
|
308
|
-
# Stage Cadip items.
|
|
309
|
-
staged = staging_task.with_options(
|
|
310
|
-
retries=staging_retries,
|
|
311
|
-
retry_delay_seconds=staging_retry_delay,
|
|
312
|
-
).submit(
|
|
313
|
-
flow_env.serialize(),
|
|
314
|
-
cadip_items,
|
|
315
|
-
catalog_collection_identifier,
|
|
316
|
-
)
|
|
317
|
-
|
|
318
|
-
# Wait for last task to end.
|
|
319
|
-
# NOTE: use .result() and not .wait() to unwrap and propagate exceptions, if any.
|
|
320
|
-
staged.result() # type: ignore[unused-coroutine]
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
@flow(name="On-demand Prip staging")
|
|
324
|
-
async def on_demand_prip_staging(
|
|
325
|
-
env: FlowEnvArgs,
|
|
326
|
-
start_datetime: datetime.datetime | str,
|
|
327
|
-
end_datetime: datetime.datetime | str,
|
|
328
|
-
product_type: str,
|
|
329
|
-
prip_collection: str,
|
|
330
|
-
catalog_collection_identifier: str,
|
|
331
|
-
retry_config: RetryConfig = RetryConfig(), # type: ignore
|
|
332
|
-
):
|
|
333
|
-
"""
|
|
334
|
-
Flow to retrieve Prip files with the given time interval defined by
|
|
335
|
-
start_datetime and end_datetime, select only the type of files wanted,
|
|
336
|
-
stage the files and add STAC items into the catalog.
|
|
337
|
-
|
|
338
|
-
Args:
|
|
339
|
-
env: Prefect flow environment
|
|
340
|
-
start_datetime: Start datetime for the time interval used to filter the files
|
|
341
|
-
(date or timestamp, e.g. "2025-08-07T11:51:12.509000Z")
|
|
342
|
-
end_datetime: End datetime for the time interval used to filter the files
|
|
343
|
-
(date or timestamp, e.g. "2025-08-10T14:00:00.509000Z")
|
|
344
|
-
product_type: Prip product type wanted
|
|
345
|
-
prip_collection: PRIP collection identifier (station)
|
|
346
|
-
catalog_collection_identifier: Catalog collection identifier where PRIP data are staged
|
|
347
|
-
"""
|
|
348
|
-
|
|
349
|
-
# Init flow environment and opentelemetry span
|
|
350
|
-
flow_env = FlowEnv(env)
|
|
351
|
-
with flow_env.start_span(__name__, "on-demand-prip-staging"):
|
|
352
|
-
|
|
353
|
-
# CQL2 filter: filter on product type and time interval
|
|
354
|
-
cql2_filter = create_valcover_filter(start_datetime, end_datetime, product_type)
|
|
355
|
-
|
|
356
|
-
# Search Prip products
|
|
357
|
-
prip_items = prip_flow.search_task.with_options(
|
|
358
|
-
retries=retry_config.staging_retries,
|
|
359
|
-
retry_delay_seconds=retry_config.staging_retry_delay,
|
|
360
|
-
).submit(
|
|
361
|
-
flow_env.serialize(),
|
|
362
|
-
prip_cql2={"filter": cql2_filter},
|
|
363
|
-
prip_collection=prip_collection,
|
|
364
|
-
error_if_empty=False,
|
|
365
|
-
)
|
|
366
|
-
|
|
367
|
-
# Stage Prip items
|
|
368
|
-
staged = staging_task.with_options(
|
|
369
|
-
retries=retry_config.staging_retries,
|
|
370
|
-
retry_delay_seconds=retry_config.staging_retry_delay,
|
|
371
|
-
).submit(
|
|
372
|
-
flow_env.serialize(),
|
|
373
|
-
prip_items,
|
|
374
|
-
catalog_collection_identifier,
|
|
375
|
-
)
|
|
376
|
-
|
|
377
|
-
# Wait for last task to end (unwrap exceptions if any)
|
|
378
|
-
staged.result() # type: ignore[unused-coroutine]
|
|
@@ -240,6 +240,13 @@ def find_s3_output_bucket(
|
|
|
240
240
|
if fallback_bucket is None:
|
|
241
241
|
fallback_bucket = bucket
|
|
242
242
|
logger.info(f"Configuration bucket: fallback_bucket: {fallback_bucket}")
|
|
243
|
+
else:
|
|
244
|
+
logger.warning(
|
|
245
|
+
"Multiple default configurations found in the configuration map "
|
|
246
|
+
"(rs-catalog-staging-configmap), expected only one. Using the first "
|
|
247
|
+
f"one found ({fallback_bucket}), but please check your configuration table to avoid "
|
|
248
|
+
"unexpected behaviors. There should be only one entry with the three first columns set to '*'",
|
|
249
|
+
)
|
|
243
250
|
|
|
244
251
|
if fallback_bucket:
|
|
245
252
|
logger.info(f"Configuration bucket: Return fallback_bucket: {fallback_bucket}")
|
|
@@ -383,7 +390,6 @@ def build_output_products(
|
|
|
383
390
|
|
|
384
391
|
outputs = []
|
|
385
392
|
processed_products = set()
|
|
386
|
-
common_uuid = str(uuid4())
|
|
387
393
|
|
|
388
394
|
for output_product in dpr_process_in.generated_product_to_collection_identifier:
|
|
389
395
|
product_name = output_product.name
|
|
@@ -405,7 +411,7 @@ def build_output_products(
|
|
|
405
411
|
)
|
|
406
412
|
|
|
407
413
|
bucket_name = find_s3_output_bucket(bucket_configuration, owner_id, output_collection, product_type)
|
|
408
|
-
output_path = os.path.join("s3://", bucket_name, owner_id, output_collection,
|
|
414
|
+
output_path = os.path.join("s3://", bucket_name, owner_id, output_collection, str(uuid4()))
|
|
409
415
|
# cf story 871/Set S3 configuration in payload.yaml
|
|
410
416
|
store_name = storage_configuration.get_storage_for_specific_product(product_name)
|
|
411
417
|
if not store_name:
|
{rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/payload_template.py
RENAMED
|
@@ -281,10 +281,10 @@ class OutputProduct(BasePayloadModel):
|
|
|
281
281
|
opening_mode: str | None = Field(default="CREATE")
|
|
282
282
|
apply_eoqc: bool | None = Field(default=False)
|
|
283
283
|
# Excluded from serialization by default
|
|
284
|
-
# This is because the
|
|
285
|
-
#
|
|
286
|
-
# field in the payload. So we need to exclude
|
|
287
|
-
# keep it in the model for internal use in run_processor task.
|
|
284
|
+
# This is because the "final product" concept exists only in the tasktable and
|
|
285
|
+
# is is not recognized by the processor. The processor simply fails if it finds an unknown
|
|
286
|
+
# field in the payload. So we need to exclude this field from the payload, but we want to
|
|
287
|
+
# keep it in the model for internal use in the run_processor task.
|
|
288
288
|
final_product: bool | None = Field(default=True, exclude=True)
|
|
289
289
|
|
|
290
290
|
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
# Copyright 2023-2026 Airbus, CS Group
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""Prip flow implementation"""
|
|
16
|
+
|
|
17
|
+
import datetime
|
|
18
|
+
|
|
19
|
+
from prefect import flow, get_run_logger, task
|
|
20
|
+
from pystac import ItemCollection
|
|
21
|
+
|
|
22
|
+
from rs_client.stac.prip_client import PripClient
|
|
23
|
+
from rs_common.utils import create_valcover_filter
|
|
24
|
+
from rs_workflows.flow_utils import FlowEnv, FlowEnvArgs, RetryConfig
|
|
25
|
+
from rs_workflows.staging_flow import staging_task
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@flow(name="Prip search")
|
|
29
|
+
async def search(
|
|
30
|
+
env: FlowEnvArgs,
|
|
31
|
+
prip_cql2: dict,
|
|
32
|
+
prip_collection: str = "",
|
|
33
|
+
error_if_empty: bool = False,
|
|
34
|
+
) -> ItemCollection | None:
|
|
35
|
+
"""
|
|
36
|
+
Search Prip products.
|
|
37
|
+
|
|
38
|
+
Args:
|
|
39
|
+
env: Prefect flow environment (at least the owner_id is required)
|
|
40
|
+
prip_cql2: PRIP CQL2 filter.
|
|
41
|
+
prip_collection: PRIP ollection identifier (to know the station)
|
|
42
|
+
error_if_empty: Raise a ValueError if the results are empty.
|
|
43
|
+
"""
|
|
44
|
+
logger = get_run_logger()
|
|
45
|
+
|
|
46
|
+
# Init flow environment and opentelemetry span
|
|
47
|
+
flow_env = FlowEnv(env)
|
|
48
|
+
with flow_env.start_span(__name__, "prip-search"):
|
|
49
|
+
|
|
50
|
+
logger.info("Start PRIP search")
|
|
51
|
+
prip_client: PripClient = flow_env.rs_client.get_prip_client()
|
|
52
|
+
found = prip_client.search(
|
|
53
|
+
method="POST",
|
|
54
|
+
stac_filter=prip_cql2.get("filter"),
|
|
55
|
+
max_items=prip_cql2.get("limit", 10),
|
|
56
|
+
sortby=prip_cql2.get("sortby", "-created"),
|
|
57
|
+
collections=[prip_collection],
|
|
58
|
+
)
|
|
59
|
+
if (not found) and error_if_empty:
|
|
60
|
+
raise ValueError("No PRIP products found")
|
|
61
|
+
logger.info(f"PRIP search found {len(found)} results: {found}")
|
|
62
|
+
return found
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@flow(name="On-demand Prip staging")
|
|
66
|
+
async def on_demand_prip_staging(
|
|
67
|
+
env: FlowEnvArgs,
|
|
68
|
+
start_datetime: datetime.datetime | str,
|
|
69
|
+
end_datetime: datetime.datetime | str,
|
|
70
|
+
product_type: str,
|
|
71
|
+
prip_collection: str,
|
|
72
|
+
catalog_collection_identifier: str,
|
|
73
|
+
retry_config: RetryConfig = RetryConfig(), # type: ignore
|
|
74
|
+
):
|
|
75
|
+
"""
|
|
76
|
+
Flow to retrieve Prip files with the given time interval defined by
|
|
77
|
+
start_datetime and end_datetime, select only the type of files wanted,
|
|
78
|
+
stage the files and add STAC items into the catalog.
|
|
79
|
+
|
|
80
|
+
Args:
|
|
81
|
+
env: Prefect flow environment
|
|
82
|
+
start_datetime: Start datetime for the time interval used to filter the files
|
|
83
|
+
(date or timestamp, e.g. "2025-08-07T11:51:12.509000Z")
|
|
84
|
+
end_datetime: End datetime for the time interval used to filter the files
|
|
85
|
+
(date or timestamp, e.g. "2025-08-10T14:00:00.509000Z")
|
|
86
|
+
product_type: Prip product type wanted
|
|
87
|
+
prip_collection: PRIP collection identifier (station)
|
|
88
|
+
catalog_collection_identifier: Catalog collection identifier where PRIP data are staged
|
|
89
|
+
"""
|
|
90
|
+
|
|
91
|
+
# Init flow environment and opentelemetry span
|
|
92
|
+
flow_env = FlowEnv(env)
|
|
93
|
+
with flow_env.start_span(__name__, "on-demand-prip-staging"):
|
|
94
|
+
|
|
95
|
+
# CQL2 filter: filter on product type and time interval
|
|
96
|
+
cql2_filter = create_valcover_filter(start_datetime, end_datetime, product_type)
|
|
97
|
+
|
|
98
|
+
# Search Prip products
|
|
99
|
+
prip_items = search_task.with_options(
|
|
100
|
+
retries=retry_config.staging_retries,
|
|
101
|
+
retry_delay_seconds=retry_config.staging_retry_delay,
|
|
102
|
+
).submit(
|
|
103
|
+
flow_env.serialize(),
|
|
104
|
+
prip_cql2={"filter": cql2_filter},
|
|
105
|
+
prip_collection=prip_collection,
|
|
106
|
+
error_if_empty=False,
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
# Stage Prip items
|
|
110
|
+
staged = staging_task.with_options(
|
|
111
|
+
retries=retry_config.staging_retries,
|
|
112
|
+
retry_delay_seconds=retry_config.staging_retry_delay,
|
|
113
|
+
).submit(
|
|
114
|
+
flow_env.serialize(),
|
|
115
|
+
prip_items,
|
|
116
|
+
catalog_collection_identifier,
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
# Wait for last task to end (unwrap exceptions if any)
|
|
120
|
+
staged.result() # type: ignore[unused-coroutine]
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
###########################
|
|
124
|
+
# Call the flows as tasks #
|
|
125
|
+
###########################
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
@task(name="PRIP search")
|
|
129
|
+
async def search_task(*args, **kwargs) -> ItemCollection | None:
|
|
130
|
+
"""See: search"""
|
|
131
|
+
return await search.fn(*args, **kwargs)
|
|
@@ -388,11 +388,12 @@ def record_product_expected(flow_run_id: str, dpr_processor_name, payload, eopf_
|
|
|
388
388
|
("S03MWRL0_", 3, 3),
|
|
389
389
|
("S03OLCCR0", 0, 1),
|
|
390
390
|
("S03OLCCR1", 0, 1),
|
|
391
|
-
("S03OLCL0_",
|
|
391
|
+
("S03OLCL0_", 22, 24),
|
|
392
392
|
("S03SLSL0_", 15, 22),
|
|
393
|
-
("S03ALTL0_",
|
|
393
|
+
("S03ALTL0_", 10, 13),
|
|
394
394
|
("S03SRCRL0", 0, 1),
|
|
395
|
-
("S03HKML0_",
|
|
395
|
+
("S03HKML0_", 1, 1),
|
|
396
|
+
("S03HKM2L0", 1, 1),
|
|
396
397
|
("S03NATL0_", 2, 2),
|
|
397
398
|
]
|
|
398
399
|
else:
|
|
@@ -1,69 +0,0 @@
|
|
|
1
|
-
# Copyright 2023-2026 Airbus, CS Group
|
|
2
|
-
#
|
|
3
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
-
# you may not use this file except in compliance with the License.
|
|
5
|
-
# You may obtain a copy of the License at
|
|
6
|
-
#
|
|
7
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
-
#
|
|
9
|
-
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
-
# See the License for the specific language governing permissions and
|
|
13
|
-
# limitations under the License.
|
|
14
|
-
|
|
15
|
-
"""Prip flow implementation"""
|
|
16
|
-
|
|
17
|
-
from prefect import flow, get_run_logger, task
|
|
18
|
-
from pystac import ItemCollection
|
|
19
|
-
|
|
20
|
-
from rs_client.stac.prip_client import PripClient
|
|
21
|
-
from rs_workflows.flow_utils import FlowEnv, FlowEnvArgs
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
@flow(name="Prip search")
|
|
25
|
-
async def search(
|
|
26
|
-
env: FlowEnvArgs,
|
|
27
|
-
prip_cql2: dict,
|
|
28
|
-
prip_collection: str = "",
|
|
29
|
-
error_if_empty: bool = False,
|
|
30
|
-
) -> ItemCollection | None:
|
|
31
|
-
"""
|
|
32
|
-
Search Prip products.
|
|
33
|
-
|
|
34
|
-
Args:
|
|
35
|
-
env: Prefect flow environment (at least the owner_id is required)
|
|
36
|
-
prip_cql2: PRIP CQL2 filter.
|
|
37
|
-
prip_collection: PRIP ollection identifier (to know the station)
|
|
38
|
-
error_if_empty: Raise a ValueError if the results are empty.
|
|
39
|
-
"""
|
|
40
|
-
logger = get_run_logger()
|
|
41
|
-
|
|
42
|
-
# Init flow environment and opentelemetry span
|
|
43
|
-
flow_env = FlowEnv(env)
|
|
44
|
-
with flow_env.start_span(__name__, "prip-search"):
|
|
45
|
-
|
|
46
|
-
logger.info("Start PRIP search")
|
|
47
|
-
prip_client: PripClient = flow_env.rs_client.get_prip_client()
|
|
48
|
-
found = prip_client.search(
|
|
49
|
-
method="POST",
|
|
50
|
-
stac_filter=prip_cql2.get("filter"),
|
|
51
|
-
max_items=prip_cql2.get("limit", 10),
|
|
52
|
-
sortby=prip_cql2.get("sortby", "-created"),
|
|
53
|
-
collections=[prip_collection],
|
|
54
|
-
)
|
|
55
|
-
if (not found) and error_if_empty:
|
|
56
|
-
raise ValueError("No PRIP products found")
|
|
57
|
-
logger.info(f"PRIP search found {len(found)} results: {found}")
|
|
58
|
-
return found
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
###########################
|
|
62
|
-
# Call the flows as tasks #
|
|
63
|
-
###########################
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
@task(name="PRIP search")
|
|
67
|
-
async def search_task(*args, **kwargs) -> ItemCollection | None:
|
|
68
|
-
"""See: search"""
|
|
69
|
-
return await search.fn(*args, **kwargs)
|
|
File without changes
|
|
File without changes
|
{rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/ogcapi/__init__.py
RENAMED
|
File without changes
|
{rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/ogcapi/dpr_client.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/osam_client.py
RENAMED
|
File without changes
|
|
File without changes
|
{rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/stac/__init__.py
RENAMED
|
File without changes
|
{rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/stac/auxip_client.py
RENAMED
|
File without changes
|
{rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/stac/cadip_client.py
RENAMED
|
File without changes
|
{rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/stac/catalog_client.py
RENAMED
|
File without changes
|
{rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/stac/prip_client.py
RENAMED
|
File without changes
|
{rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_client/stac/stac_base.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_common/init_opentelemetry.py
RENAMED
|
File without changes
|
|
File without changes
|
{rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_common/prefect_utils.py
RENAMED
|
File without changes
|
|
File without changes
|
{rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/__init__.py
RENAMED
|
File without changes
|
{rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/auxip_flow.py
RENAMED
|
File without changes
|
{rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/init_pi_db_flow.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/payload_builder.py
RENAMED
|
File without changes
|
{rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/pi_db_models.py
RENAMED
|
File without changes
|
{rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/staging_flow.py
RENAMED
|
File without changes
|
|
File without changes
|
{rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/utils/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
{rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/utils/cadip.py
RENAMED
|
File without changes
|
{rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/utils/catalog.py
RENAMED
|
File without changes
|
{rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/utils/dask.py
RENAMED
|
File without changes
|
{rs_client_libraries-1.0a8 → rs_client_libraries-1.0a8.post16.dev0}/rs_workflows/utils/dpr.py
RENAMED
|
File without changes
|