rs-client-libraries 0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- config/staging_templates/README.md +17 -0
- config/staging_templates/yaml/additionalParameter.yaml +30 -0
- config/staging_templates/yaml/bbox.yaml +34 -0
- config/staging_templates/yaml/binaryInputValue.yaml +15 -0
- config/staging_templates/yaml/confClasses.yaml +22 -0
- config/staging_templates/yaml/descriptionType.yaml +36 -0
- config/staging_templates/yaml/dpr_openapi_schema.yaml +223 -0
- config/staging_templates/yaml/exception.yaml +31 -0
- config/staging_templates/yaml/execute.yaml +34 -0
- config/staging_templates/yaml/format.yaml +25 -0
- config/staging_templates/yaml/inlineOrRefData.yaml +18 -0
- config/staging_templates/yaml/inputDescription.yaml +32 -0
- config/staging_templates/yaml/inputValue.yaml +17 -0
- config/staging_templates/yaml/inputValueNoObject.yaml +22 -0
- config/staging_templates/yaml/jobControlOptions.yaml +19 -0
- config/staging_templates/yaml/jobList.yaml +27 -0
- config/staging_templates/yaml/landingPage.yaml +28 -0
- config/staging_templates/yaml/link.yaml +31 -0
- config/staging_templates/yaml/metadata.yaml +22 -0
- config/staging_templates/yaml/output.yaml +20 -0
- config/staging_templates/yaml/outputDescription.yaml +22 -0
- config/staging_templates/yaml/process.yaml +23 -0
- config/staging_templates/yaml/processList.yaml +27 -0
- config/staging_templates/yaml/processSummary.yaml +37 -0
- config/staging_templates/yaml/qualifiedInputValue.yaml +22 -0
- config/staging_templates/yaml/reference.yaml +21 -0
- config/staging_templates/yaml/results.yaml +16 -0
- config/staging_templates/yaml/schema.yaml +144 -0
- config/staging_templates/yaml/staging_openapi_schema.yaml +223 -0
- config/staging_templates/yaml/statusCode.yaml +22 -0
- config/staging_templates/yaml/statusInfo.yaml +52 -0
- config/staging_templates/yaml/subscriber.yaml +33 -0
- config/staging_templates/yaml/transmissionMode.yaml +19 -0
- rs_client/__init__.py +18 -0
- rs_client/ogcapi/__init__.py +15 -0
- rs_client/ogcapi/dpr_client.py +231 -0
- rs_client/ogcapi/ogcapi_client.py +282 -0
- rs_client/ogcapi/staging_client.py +178 -0
- rs_client/rs_client.py +277 -0
- rs_client/stac/__init__.py +15 -0
- rs_client/stac/auxip_client.py +70 -0
- rs_client/stac/cadip_client.py +70 -0
- rs_client/stac/catalog_client.py +479 -0
- rs_client/stac/stac_base.py +313 -0
- rs_client_libraries-0.2.dist-info/LICENSE +201 -0
- rs_client_libraries-0.2.dist-info/METADATA +44 -0
- rs_client_libraries-0.2.dist-info/RECORD +62 -0
- rs_client_libraries-0.2.dist-info/WHEEL +4 -0
- rs_common/__init__.py +20 -0
- rs_common/config.py +69 -0
- rs_common/init_opentelemetry.py +250 -0
- rs_common/logging.py +107 -0
- rs_common/prefect_utils.py +407 -0
- rs_common/utils.py +70 -0
- rs_workflows/__init__.py +15 -0
- rs_workflows/auxip_flow.py +70 -0
- rs_workflows/cadip_flow.py +69 -0
- rs_workflows/catalog_flow.py +71 -0
- rs_workflows/dpr_flow.py +272 -0
- rs_workflows/flow_utils.py +127 -0
- rs_workflows/on_demand_processing.py +243 -0
- rs_workflows/staging_flow.py +78 -0
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
# Copyright 2025 CS Group
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""Prefect flows and tasks for on-demand processing"""
|
|
16
|
+
|
|
17
|
+
import datetime
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
|
|
20
|
+
from prefect import flow
|
|
21
|
+
|
|
22
|
+
from rs_workflows import auxip_flow, cadip_flow, catalog_flow
|
|
23
|
+
from rs_workflows.dpr_flow import (
|
|
24
|
+
read_payload_values,
|
|
25
|
+
read_tasktable,
|
|
26
|
+
run_processor,
|
|
27
|
+
write_payload,
|
|
28
|
+
)
|
|
29
|
+
from rs_workflows.flow_utils import FlowEnv, FlowEnvArgs, ProcessorEnum
|
|
30
|
+
from rs_workflows.staging_flow import staging_task_auxip, staging_task_cadip
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@flow(name="On-demand processing")
|
|
34
|
+
async def on_demand_processing(
|
|
35
|
+
env: FlowEnvArgs,
|
|
36
|
+
processor: ProcessorEnum,
|
|
37
|
+
cadip_collection_identifier: str,
|
|
38
|
+
session_identifier: str,
|
|
39
|
+
catalog_collection_identifier: str,
|
|
40
|
+
s3_payload_template: str,
|
|
41
|
+
s3_output_data: str,
|
|
42
|
+
use_dpr_mockup: bool = False,
|
|
43
|
+
):
|
|
44
|
+
"""
|
|
45
|
+
Prefect flow for on-demand processing.
|
|
46
|
+
|
|
47
|
+
Args:
|
|
48
|
+
env: Prefect flow environment
|
|
49
|
+
processor: DPR processor name
|
|
50
|
+
cadip_collection_identifier: CADIP collection identifier that contains the mission and station
|
|
51
|
+
(e.g. s1_ins for Sentinel-1 sessions from the Inuvik station)
|
|
52
|
+
session_identifier: Session identifier
|
|
53
|
+
catalog_collection_identifier: Catalog collection identifier where CADIP sessions and AUX data are staged
|
|
54
|
+
s3_payload_template: S3 bucket location of the DPR payload file template.
|
|
55
|
+
s3_output_data: S3 bucket location of the output processed products. They will then be copied to the
|
|
56
|
+
catalog bucket.
|
|
57
|
+
use_dpr_mockup: Use the real or the mockup DPR processor ?
|
|
58
|
+
"""
|
|
59
|
+
# logger = get_run_logger()
|
|
60
|
+
|
|
61
|
+
# Init flow environment and opentelemetry span
|
|
62
|
+
flow_env = FlowEnv(env)
|
|
63
|
+
with flow_env.start_span(__name__, "on-demand-processing"):
|
|
64
|
+
|
|
65
|
+
# Read values from the payload file
|
|
66
|
+
payload_values = read_payload_values.submit(s3_payload_template)
|
|
67
|
+
|
|
68
|
+
# Search Cadip sessions
|
|
69
|
+
cadip_items = cadip_flow.search_task.submit(
|
|
70
|
+
flow_env.serialize(),
|
|
71
|
+
cadip_collection_identifier,
|
|
72
|
+
session_identifier,
|
|
73
|
+
error_if_empty=True,
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
# Read Auxip CQL2 filter from the processor tasktable.
|
|
77
|
+
auxip_cql2 = read_tasktable.submit(flow_env.serialize(), processor, payload_values, cadip_items)
|
|
78
|
+
|
|
79
|
+
# Search Auxip products
|
|
80
|
+
auxip_items = auxip_flow.search_task.submit(flow_env.serialize(), auxip_cql2, error_if_empty=True)
|
|
81
|
+
|
|
82
|
+
# Auxip and Cadip item ids
|
|
83
|
+
item_ids = []
|
|
84
|
+
for items in [cadip_items.result(), auxip_items.result()]:
|
|
85
|
+
for item in items or []: # type: ignore[union-attr]
|
|
86
|
+
item_ids.append(item.id)
|
|
87
|
+
|
|
88
|
+
# Stage Auxip and Cadip items.
|
|
89
|
+
# Note: the only difference between staging_task_auxip and
|
|
90
|
+
# staging_task_cadip is the task name in the prefect dashboard.
|
|
91
|
+
staged = [
|
|
92
|
+
staging_task_auxip.submit(
|
|
93
|
+
flow_env.serialize(),
|
|
94
|
+
auxip_items,
|
|
95
|
+
catalog_collection_identifier,
|
|
96
|
+
),
|
|
97
|
+
staging_task_cadip.submit(
|
|
98
|
+
flow_env.serialize(),
|
|
99
|
+
cadip_items,
|
|
100
|
+
catalog_collection_identifier,
|
|
101
|
+
),
|
|
102
|
+
]
|
|
103
|
+
|
|
104
|
+
# Write the final payload file from its template version and staged items.
|
|
105
|
+
# It will be uploaded in the same s3 dir than the template file.
|
|
106
|
+
s3_payload_run = s3_payload_template + ".run" + Path(s3_payload_template).suffix
|
|
107
|
+
written = write_payload.submit(
|
|
108
|
+
flow_env.serialize(),
|
|
109
|
+
s3_payload_template,
|
|
110
|
+
item_ids,
|
|
111
|
+
catalog_collection_identifier,
|
|
112
|
+
s3_output_data,
|
|
113
|
+
s3_payload_run,
|
|
114
|
+
wait_for=staged, # wait for items to be staged in the catalog
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
# Run the DPR processor
|
|
118
|
+
processed_items = run_processor.submit(
|
|
119
|
+
flow_env.serialize(),
|
|
120
|
+
processor,
|
|
121
|
+
s3_payload_run,
|
|
122
|
+
use_dpr_mockup,
|
|
123
|
+
wait_for=written,
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
# Publish processed items to the catalog
|
|
127
|
+
published = catalog_flow.publish.submit(
|
|
128
|
+
flow_env.serialize(),
|
|
129
|
+
catalog_collection_identifier,
|
|
130
|
+
processed_items,
|
|
131
|
+
s3_output_data,
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
# Wait for last task to end.
|
|
135
|
+
# NOTE: use .result() and not .wait() to unwrap and propagate exceptions, if any.
|
|
136
|
+
published.result() # type: ignore[unused-coroutine]
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
@flow(name="On-demand Cadip staging")
|
|
140
|
+
async def on_demand_cadip_staging(
|
|
141
|
+
env: FlowEnvArgs,
|
|
142
|
+
cadip_collection_identifier: str,
|
|
143
|
+
session_identifier: str,
|
|
144
|
+
catalog_collection_identifier: str,
|
|
145
|
+
):
|
|
146
|
+
"""
|
|
147
|
+
Flow to retrieve a session, stage it and add the STAC item into the catalog.
|
|
148
|
+
|
|
149
|
+
Args:
|
|
150
|
+
env: Prefect flow environment
|
|
151
|
+
cadip_collection_identifier: CADIP collection identifier that contains the mission and station
|
|
152
|
+
(e.g. s1_ins for Sentinel-1 sessions from the Inuvik station)
|
|
153
|
+
session_identifier: Session identifier
|
|
154
|
+
catalog_collection_identifier: Catalog collection identifier where CADIP sessions and AUX data are staged
|
|
155
|
+
"""
|
|
156
|
+
|
|
157
|
+
# Init flow environment and opentelemetry span
|
|
158
|
+
flow_env = FlowEnv(env)
|
|
159
|
+
with flow_env.start_span(__name__, "on-demand-cadip-staging"):
|
|
160
|
+
|
|
161
|
+
# Search Cadip sessions
|
|
162
|
+
cadip_items = cadip_flow.search_task.submit(
|
|
163
|
+
flow_env.serialize(),
|
|
164
|
+
cadip_collection_identifier,
|
|
165
|
+
session_identifier,
|
|
166
|
+
error_if_empty=True,
|
|
167
|
+
)
|
|
168
|
+
|
|
169
|
+
# Stage Cadip items.
|
|
170
|
+
staged = staging_task_cadip.submit(flow_env.serialize(), cadip_items, catalog_collection_identifier)
|
|
171
|
+
|
|
172
|
+
# Wait for last task to end.
|
|
173
|
+
# NOTE: use .result() and not .wait() to unwrap and propagate exceptions, if any.
|
|
174
|
+
staged.result() # type: ignore[unused-coroutine]
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
@flow(name="On-demand Auxip staging")
|
|
178
|
+
async def on_demand_auxip_staging(
|
|
179
|
+
env: FlowEnvArgs,
|
|
180
|
+
start_datetime: datetime.datetime | str,
|
|
181
|
+
end_datetime: datetime.datetime | str,
|
|
182
|
+
product_type: str,
|
|
183
|
+
catalog_collection_identifier: str,
|
|
184
|
+
):
|
|
185
|
+
"""
|
|
186
|
+
Flow to retrieve Auxip files using a ValCover filter with the given time interval defined by
|
|
187
|
+
start_datetime and end_datetime, select only the type of files wanted if eopf_type is given, stage
|
|
188
|
+
the files and add STAC items into the catalog.
|
|
189
|
+
Informations on ValCover filter:
|
|
190
|
+
https://pforge-exchange2.astrium.eads.net/confluence/display/COPRS/4.+External+data+selection+policies
|
|
191
|
+
|
|
192
|
+
Args:
|
|
193
|
+
env: Prefect flow environment
|
|
194
|
+
start_datetime: Start datetime for the time interval used to filter the files
|
|
195
|
+
(select a date or directly enter a timestamp, e.g. "2025-08-07T11:51:12.509000Z")
|
|
196
|
+
end_datetime: End datetime for the time interval used to filter the files
|
|
197
|
+
(select a date or directly enter a timestamp, e.g. "2025-08-10T14:00:00.509000Z")
|
|
198
|
+
product_type: Auxiliary file type wanted
|
|
199
|
+
catalog_collection_identifier: Catalog collection identifier where CADIP sessions and AUX data are staged
|
|
200
|
+
"""
|
|
201
|
+
|
|
202
|
+
# Init flow environment and opentelemetry span
|
|
203
|
+
flow_env = FlowEnv(env)
|
|
204
|
+
with flow_env.start_span(__name__, "on-demand-auxip-staging"):
|
|
205
|
+
|
|
206
|
+
# Convert datetime inputs to str
|
|
207
|
+
if isinstance(start_datetime, datetime.datetime):
|
|
208
|
+
start_datetime = start_datetime.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
|
|
209
|
+
if isinstance(end_datetime, datetime.datetime):
|
|
210
|
+
end_datetime = end_datetime.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
|
|
211
|
+
|
|
212
|
+
# CQL2 filter: we use a filter combining a ValCover filter and a product type filter
|
|
213
|
+
cql2_filter = {
|
|
214
|
+
"op": "and",
|
|
215
|
+
"args": [
|
|
216
|
+
{"op": "=", "args": [{"property": "product:type"}, product_type]},
|
|
217
|
+
{
|
|
218
|
+
"op": "t_contains",
|
|
219
|
+
"args": [
|
|
220
|
+
{"interval": [{"property": "start_datetime"}, {"property": "end_datetime"}]},
|
|
221
|
+
{"interval": [start_datetime, end_datetime]},
|
|
222
|
+
],
|
|
223
|
+
},
|
|
224
|
+
],
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
# Search Auxip products
|
|
228
|
+
auxip_items = auxip_flow.search_task.submit(
|
|
229
|
+
flow_env.serialize(),
|
|
230
|
+
auxip_cql2={"filter": cql2_filter},
|
|
231
|
+
error_if_empty=False,
|
|
232
|
+
)
|
|
233
|
+
|
|
234
|
+
# Stage Auxip items.
|
|
235
|
+
staged = staging_task_auxip.submit(
|
|
236
|
+
flow_env.serialize(),
|
|
237
|
+
auxip_items,
|
|
238
|
+
catalog_collection_identifier,
|
|
239
|
+
)
|
|
240
|
+
|
|
241
|
+
# Wait for last task to end.
|
|
242
|
+
# NOTE: use .result() and not .wait() to unwrap and propagate exceptions, if any.
|
|
243
|
+
staged.result() # type: ignore[unused-coroutine]
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
# Copyright 2025 CS Group
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""Staging flow implementation"""
|
|
16
|
+
|
|
17
|
+
from prefect import flow, get_run_logger, task
|
|
18
|
+
from pystac import ItemCollection
|
|
19
|
+
|
|
20
|
+
from rs_client.ogcapi.staging_client import StagingClient
|
|
21
|
+
from rs_workflows.flow_utils import FlowEnv, FlowEnvArgs
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@flow(name="Staging")
|
|
25
|
+
async def staging(
|
|
26
|
+
env: FlowEnvArgs,
|
|
27
|
+
stac_input: str | ItemCollection | dict, # warning: dict as last choice for prefect ui
|
|
28
|
+
catalog_collection_identifier: str,
|
|
29
|
+
timeout: int = 1200,
|
|
30
|
+
poll_interval: int = 2,
|
|
31
|
+
):
|
|
32
|
+
"""
|
|
33
|
+
Stage STAC items.
|
|
34
|
+
|
|
35
|
+
Args:
|
|
36
|
+
env: Prefect flow environment (at least the owner_id is required)
|
|
37
|
+
stac_input (dict | str): it can be:<br>
|
|
38
|
+
- A Python dictionary corresponding to a Feature or a FeatureCollection (that can be for example
|
|
39
|
+
the output of a search for Cadip or Auxip sessions)<br>
|
|
40
|
+
- A json string corresponding to a Feature or a FeatureCollection<br>
|
|
41
|
+
- A string corresponding to a path to a json file containing a Feature or a FeatureCollection<br>
|
|
42
|
+
- A single link that returns a STAC ItemCollection: this link should be an url to search an ItemCollection
|
|
43
|
+
catalog_collection_identifier: Catalog collection identifier where items are staged
|
|
44
|
+
timeout: Job completion timeout in seconds
|
|
45
|
+
poll_interval: When to check again for job completion in seconds
|
|
46
|
+
"""
|
|
47
|
+
logger = get_run_logger()
|
|
48
|
+
|
|
49
|
+
# Init flow environment and opentelemetry span
|
|
50
|
+
flow_env = FlowEnv(env)
|
|
51
|
+
with flow_env.start_span(__name__, "staging"):
|
|
52
|
+
|
|
53
|
+
staging_client: StagingClient = flow_env.rs_client.get_staging_client()
|
|
54
|
+
|
|
55
|
+
# Convert pystac object into dict
|
|
56
|
+
if isinstance(stac_input, ItemCollection):
|
|
57
|
+
stac_input = stac_input.to_dict()
|
|
58
|
+
|
|
59
|
+
# Trigger the staging and wait for jobs to finish
|
|
60
|
+
job_status = staging_client.run_staging(stac_input, catalog_collection_identifier)
|
|
61
|
+
staging_client.wait_for_jobs(job_status, logger, timeout, poll_interval)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
###########################
|
|
65
|
+
# Call the flows as tasks #
|
|
66
|
+
###########################
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
@task(name="Auxip staging")
|
|
70
|
+
async def staging_task_auxip(*args, **kwargs):
|
|
71
|
+
"""See: staging"""
|
|
72
|
+
return await staging.fn(*args, **kwargs)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
@task(name="Cadip staging")
|
|
76
|
+
async def staging_task_cadip(*args, **kwargs):
|
|
77
|
+
"""See: staging"""
|
|
78
|
+
return await staging.fn(*args, **kwargs)
|