rs-client-libraries 0.2a15__tar.gz → 0.2a15.post17.dev0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. {rs_client_libraries-0.2a15 → rs_client_libraries-0.2a15.post17.dev0}/PKG-INFO +1 -1
  2. {rs_client_libraries-0.2a15 → rs_client_libraries-0.2a15.post17.dev0}/pyproject.toml +1 -1
  3. {rs_client_libraries-0.2a15 → rs_client_libraries-0.2a15.post17.dev0}/rs_client/__init__.py +1 -1
  4. {rs_client_libraries-0.2a15 → rs_client_libraries-0.2a15.post17.dev0}/rs_workflows/auxip_flow.py +1 -1
  5. rs_client_libraries-0.2a15.post17.dev0/rs_workflows/on_demand_processing.py +257 -0
  6. rs_client_libraries-0.2a15/rs_workflows/on_demand_processing.py +0 -134
  7. {rs_client_libraries-0.2a15 → rs_client_libraries-0.2a15.post17.dev0}/LICENSE +0 -0
  8. {rs_client_libraries-0.2a15 → rs_client_libraries-0.2a15.post17.dev0}/README.adoc +0 -0
  9. {rs_client_libraries-0.2a15 → rs_client_libraries-0.2a15.post17.dev0}/rs_client/ogcapi/__init__.py +0 -0
  10. {rs_client_libraries-0.2a15 → rs_client_libraries-0.2a15.post17.dev0}/rs_client/ogcapi/dpr_client.py +0 -0
  11. {rs_client_libraries-0.2a15 → rs_client_libraries-0.2a15.post17.dev0}/rs_client/ogcapi/ogcapi_client.py +0 -0
  12. {rs_client_libraries-0.2a15 → rs_client_libraries-0.2a15.post17.dev0}/rs_client/ogcapi/staging_client.py +0 -0
  13. {rs_client_libraries-0.2a15 → rs_client_libraries-0.2a15.post17.dev0}/rs_client/rs_client.py +0 -0
  14. {rs_client_libraries-0.2a15 → rs_client_libraries-0.2a15.post17.dev0}/rs_client/stac/__init__.py +0 -0
  15. {rs_client_libraries-0.2a15 → rs_client_libraries-0.2a15.post17.dev0}/rs_client/stac/auxip_client.py +0 -0
  16. {rs_client_libraries-0.2a15 → rs_client_libraries-0.2a15.post17.dev0}/rs_client/stac/cadip_client.py +0 -0
  17. {rs_client_libraries-0.2a15 → rs_client_libraries-0.2a15.post17.dev0}/rs_client/stac/catalog_client.py +0 -0
  18. {rs_client_libraries-0.2a15 → rs_client_libraries-0.2a15.post17.dev0}/rs_client/stac/stac_base.py +0 -0
  19. {rs_client_libraries-0.2a15 → rs_client_libraries-0.2a15.post17.dev0}/rs_common/__init__.py +0 -0
  20. {rs_client_libraries-0.2a15 → rs_client_libraries-0.2a15.post17.dev0}/rs_common/config.py +0 -0
  21. {rs_client_libraries-0.2a15 → rs_client_libraries-0.2a15.post17.dev0}/rs_common/init_opentelemetry.py +0 -0
  22. {rs_client_libraries-0.2a15 → rs_client_libraries-0.2a15.post17.dev0}/rs_common/logging.py +0 -0
  23. {rs_client_libraries-0.2a15 → rs_client_libraries-0.2a15.post17.dev0}/rs_common/prefect_utils.py +0 -0
  24. {rs_client_libraries-0.2a15 → rs_client_libraries-0.2a15.post17.dev0}/rs_common/utils.py +0 -0
  25. {rs_client_libraries-0.2a15 → rs_client_libraries-0.2a15.post17.dev0}/rs_workflows/__init__.py +0 -0
  26. {rs_client_libraries-0.2a15 → rs_client_libraries-0.2a15.post17.dev0}/rs_workflows/cadip_flow.py +0 -0
  27. {rs_client_libraries-0.2a15 → rs_client_libraries-0.2a15.post17.dev0}/rs_workflows/catalog_flow.py +0 -0
  28. {rs_client_libraries-0.2a15 → rs_client_libraries-0.2a15.post17.dev0}/rs_workflows/dpr_flow.py +0 -0
  29. {rs_client_libraries-0.2a15 → rs_client_libraries-0.2a15.post17.dev0}/rs_workflows/flow_utils.py +0 -0
  30. {rs_client_libraries-0.2a15 → rs_client_libraries-0.2a15.post17.dev0}/rs_workflows/staging_flow.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.3
2
2
  Name: rs-client-libraries
3
- Version: 0.2a15
3
+ Version: 0.2a15.post17.dev0
4
4
  Summary: Copernicus Sentinel rs-client library and processing workflows.
5
5
  License: Apache-2.0
6
6
  Keywords: stac,copernicus,sentinel,processing
@@ -4,7 +4,7 @@ dynamic = ["version"] # see: https://github.com/mtkennerly/poetry-dynamic-versio
4
4
 
5
5
  [tool.poetry]
6
6
  name = "rs-client-libraries"
7
- version = "0.2a15"
7
+ version = "0.2a15.post17.dev0"
8
8
  description = "Copernicus Sentinel rs-client library and processing workflows."
9
9
  license = "Apache-2.0"
10
10
  authors = ["ESA under contract with Airbus and CS <ops_coprs@airbus.com>"]
@@ -15,4 +15,4 @@
15
15
  """rsclient main module."""
16
16
 
17
17
  # Set automatically by running `poetry dynamic-versioning`
18
- __version__ = "0.2a15"
18
+ __version__ = "0.2a15.post17.dev0"
@@ -53,7 +53,7 @@ async def search(
53
53
  )
54
54
  if (not found) and error_if_empty:
55
55
  raise ValueError(
56
- f"No Auxip session found for CQL2 filter: {json.dumps(auxip_cql2, indent=2)}",
56
+ f"No Auxip product found for CQL2 filter: {json.dumps(auxip_cql2, indent=2)}",
57
57
  )
58
58
  logger.info(f"Auxip search found {len(found)} results: {found}")
59
59
  return found
@@ -0,0 +1,257 @@
1
+ # Copyright 2025 CS Group
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """Prefect flows and tasks for on-demand processing"""
16
+
17
+ import datetime
18
+ from pathlib import Path
19
+
20
+ from prefect import flow, get_run_logger, task
21
+ from pystac import ItemCollection
22
+
23
+ from rs_workflows import auxip_flow, cadip_flow, catalog_flow
24
+ from rs_workflows.dpr_flow import (
25
+ read_payload_values,
26
+ read_tasktable,
27
+ run_processor,
28
+ write_payload,
29
+ )
30
+ from rs_workflows.flow_utils import FlowEnv, FlowEnvArgs, ProcessorEnum
31
+ from rs_workflows.staging_flow import staging_task_auxip, staging_task_cadip
32
+
33
+
34
+ @flow(name="On-demand processing")
35
+ async def on_demand_processing(
36
+ env: FlowEnvArgs,
37
+ processor: ProcessorEnum,
38
+ cadip_collection_identifier: str,
39
+ session_identifier: str,
40
+ catalog_collection_identifier: str,
41
+ s3_payload_template: str,
42
+ s3_output_data: str,
43
+ use_dpr_mockup: bool = False,
44
+ ):
45
+ """
46
+ Prefect flow for on-demand processing.
47
+
48
+ Args:
49
+ env: Prefect flow environment
50
+ processor: DPR processor name
51
+ cadip_collection_identifier: CADIP collection identifier that contains the mission and station
52
+ (e.g. s1_ins for Sentinel-1 sessions from the Inuvik station)
53
+ session_identifier: Session identifier
54
+ catalog_collection_identifier: Catalog collection identifier where CADIP sessions and AUX data are staged
55
+ s3_payload_template: S3 bucket location of the DPR payload file template.
56
+ s3_output_data: S3 bucket location of the output processed products. They will then be copied to the
57
+ catalog bucket.
58
+ use_dpr_mockup: Use the real or the mockup DPR processor ?
59
+ """
60
+ # logger = get_run_logger()
61
+
62
+ # Init flow environment and opentelemetry span
63
+ flow_env = FlowEnv(env)
64
+ with flow_env.start_span(__name__, "on-demand-processing"):
65
+
66
+ # Read values from the payload file
67
+ payload_values = read_payload_values.submit(s3_payload_template)
68
+
69
+ # Search Cadip sessions
70
+ cadip_items = cadip_flow.search_task.submit(
71
+ flow_env.serialize(),
72
+ cadip_collection_identifier,
73
+ session_identifier,
74
+ error_if_empty=True,
75
+ )
76
+
77
+ # Read Auxip CQL2 filter from the processor tasktable.
78
+ auxip_cql2 = read_tasktable.submit(flow_env.serialize(), processor, payload_values, cadip_items)
79
+
80
+ # Search Auxip products
81
+ auxip_items = auxip_flow.search_task.submit(flow_env.serialize(), auxip_cql2, error_if_empty=True)
82
+
83
+ # Auxip and Cadip item ids
84
+ item_ids = []
85
+ for items in [cadip_items.result(), auxip_items.result()]:
86
+ for item in items or []: # type: ignore[union-attr]
87
+ item_ids.append(item.id)
88
+
89
+ # Stage Auxip and Cadip items.
90
+ # Note: the only difference between staging_task_auxip and
91
+ # staging_task_cadip is the task name in the prefect dashboard.
92
+ staged = [
93
+ staging_task_auxip.submit(
94
+ flow_env.serialize(),
95
+ auxip_items,
96
+ catalog_collection_identifier,
97
+ ),
98
+ staging_task_cadip.submit(
99
+ flow_env.serialize(),
100
+ cadip_items,
101
+ catalog_collection_identifier,
102
+ ),
103
+ ]
104
+
105
+ # Write the final payload file from its template version and staged items.
106
+ # It will be uploaded in the same s3 dir than the template file.
107
+ s3_payload_run = s3_payload_template + ".run" + Path(s3_payload_template).suffix
108
+ written = write_payload.submit(
109
+ flow_env.serialize(),
110
+ s3_payload_template,
111
+ item_ids,
112
+ catalog_collection_identifier,
113
+ s3_output_data,
114
+ s3_payload_run,
115
+ wait_for=staged, # wait for items to be staged in the catalog
116
+ )
117
+
118
+ # Run the DPR processor
119
+ processed_items = run_processor.submit(
120
+ flow_env.serialize(),
121
+ processor,
122
+ s3_payload_run,
123
+ use_dpr_mockup,
124
+ wait_for=written,
125
+ )
126
+
127
+ # Publish processed items to the catalog
128
+ published = catalog_flow.publish.submit(
129
+ flow_env.serialize(),
130
+ catalog_collection_identifier,
131
+ processed_items,
132
+ s3_output_data,
133
+ )
134
+
135
+ # Wait for last task to end.
136
+ # NOTE: use .result() and not .wait() to unwrap and propagate exceptions, if any.
137
+ published.result() # type: ignore[unused-coroutine]
138
+
139
+
140
+ @flow(name="On-demand Cadip staging")
141
+ async def on_demand_cadip_staging(
142
+ env: FlowEnvArgs,
143
+ cadip_collection_identifier: str,
144
+ session_identifier: str,
145
+ catalog_collection_identifier: str,
146
+ ):
147
+ """
148
+ Flow to retrieve a session, stage it and add the STAC item into the catalog.
149
+
150
+ Args:
151
+ env: Prefect flow environment
152
+ cadip_collection_identifier: CADIP collection identifier that contains the mission and station
153
+ (e.g. s1_ins for Sentinel-1 sessions from the Inuvik station)
154
+ session_identifier: Session identifier
155
+ catalog_collection_identifier: Catalog collection identifier where CADIP sessions and AUX data are staged
156
+ """
157
+
158
+ # Init flow environment and opentelemetry span
159
+ flow_env = FlowEnv(env)
160
+ with flow_env.start_span(__name__, "on-demand-cadip-staging"):
161
+
162
+ # Search Cadip sessions
163
+ cadip_items = cadip_flow.search_task.submit(
164
+ flow_env.serialize(),
165
+ cadip_collection_identifier,
166
+ session_identifier,
167
+ error_if_empty=True,
168
+ )
169
+
170
+ # Stage Cadip items.
171
+ staged = staging_task_cadip.submit(flow_env.serialize(), cadip_items, catalog_collection_identifier)
172
+
173
+ # Wait for last task to end.
174
+ # NOTE: use .result() and not .wait() to unwrap and propagate exceptions, if any.
175
+ staged.result() # type: ignore[unused-coroutine]
176
+
177
+
178
+ @flow(name="On-demand Auxip staging")
179
+ async def on_demand_auxip_staging(
180
+ env: FlowEnvArgs,
181
+ start_datetime: datetime.datetime | str,
182
+ end_datetime: datetime.datetime | str,
183
+ product_type: str,
184
+ catalog_collection_identifier: str,
185
+ ):
186
+ """
187
+ Flow to retrieve Auxip files using a ValCover filter with the given time interval defined by
188
+ start_datetime and end_datetime, select only the type of files wanted if eopf_type is given, stage
189
+ the files and add STAC items into the catalog.
190
+ Informations on ValCover filter:
191
+ https://pforge-exchange2.astrium.eads.net/confluence/display/COPRS/4.+External+data+selection+policies
192
+
193
+ Args:
194
+ env: Prefect flow environment
195
+ start_datetime: Start datetime for the time interval used to filter the files
196
+ (select a date or directly enter a timestamp, e.g. "2025-08-07T11:51:12.509000Z")
197
+ end_datetime: End datetime for the time interval used to filter the files
198
+ (select a date or directly enter a timestamp, e.g. "2025-08-10T14:00:00.509000Z")
199
+ product_type: Auxiliary file type wanted
200
+ catalog_collection_identifier: Catalog collection identifier where CADIP sessions and AUX data are staged
201
+ """
202
+
203
+ # Init flow environment and opentelemetry span
204
+ flow_env = FlowEnv(env)
205
+ with flow_env.start_span(__name__, "on-demand-auxip-staging"):
206
+
207
+ # Convert datetime inputs to str
208
+ if isinstance(start_datetime, datetime.datetime):
209
+ start_datetime = start_datetime.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
210
+ if isinstance(end_datetime, datetime.datetime):
211
+ end_datetime = end_datetime.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
212
+
213
+ # CQL2 ValCover filter. The product:type filter is not added here
214
+ # because for some reason composite filters with "and" don't work
215
+ cql2_filter = {
216
+ "op": "t_contains",
217
+ "args": [
218
+ {"interval": [{"property": "start_datetime"}, {"property": "end_datetime"}]},
219
+ {"interval": [start_datetime, end_datetime]},
220
+ ],
221
+ }
222
+
223
+ # Search Auxip products
224
+ auxip_items = auxip_flow.search_task.submit(
225
+ flow_env.serialize(),
226
+ auxip_cql2={"filter": cql2_filter},
227
+ error_if_empty=False,
228
+ )
229
+
230
+ # Filtering Auxip items to only keep the ones with the correct product type
231
+ items_to_stage = filter_product_type.submit(auxip_items, product_type)
232
+
233
+ # Stage Auxip items.
234
+ staged = staging_task_auxip.submit(
235
+ flow_env.serialize(),
236
+ items_to_stage,
237
+ catalog_collection_identifier,
238
+ )
239
+
240
+ # Wait for last task to end.
241
+ # NOTE: use .result() and not .wait() to unwrap and propagate exceptions, if any.
242
+ staged.result() # type: ignore[unused-coroutine]
243
+
244
+
245
+ @task(name="Filter product type")
246
+ async def filter_product_type(auxip_items: ItemCollection | None, product_type: str) -> ItemCollection:
247
+ """Filter Auxip items to only keep the ones with the correct product type"""
248
+ logger = get_run_logger()
249
+ items_to_stage = []
250
+
251
+ if auxip_items:
252
+ for item in auxip_items: # type: ignore[attr-defined]
253
+ if "product:type" in item.properties.keys() and item.properties["product:type"] == product_type:
254
+ items_to_stage.append(item)
255
+
256
+ logger.info(f"Filtering search results: {len(items_to_stage)} item(s) found of product type {product_type}")
257
+ return ItemCollection(items_to_stage)
@@ -1,134 +0,0 @@
1
- # Copyright 2025 CS Group
2
- #
3
- # Licensed under the Apache License, Version 2.0 (the "License");
4
- # you may not use this file except in compliance with the License.
5
- # You may obtain a copy of the License at
6
- #
7
- # http://www.apache.org/licenses/LICENSE-2.0
8
- #
9
- # Unless required by applicable law or agreed to in writing, software
10
- # distributed under the License is distributed on an "AS IS" BASIS,
11
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
- # See the License for the specific language governing permissions and
13
- # limitations under the License.
14
-
15
- """Prefect flows and tasks for on-demand processing"""
16
-
17
- from pathlib import Path
18
-
19
- from prefect import flow
20
-
21
- from rs_workflows import auxip_flow, cadip_flow, catalog_flow
22
- from rs_workflows.dpr_flow import (
23
- read_payload_values,
24
- read_tasktable,
25
- run_processor,
26
- write_payload,
27
- )
28
- from rs_workflows.flow_utils import FlowEnv, FlowEnvArgs, ProcessorEnum
29
- from rs_workflows.staging_flow import staging_task_auxip, staging_task_cadip
30
-
31
-
32
- @flow(name="On-demand processing")
33
- async def on_demand_processing(
34
- env: FlowEnvArgs,
35
- processor: ProcessorEnum,
36
- cadip_collection_identifier: str,
37
- session_identifier: str,
38
- catalog_collection_identifier: str,
39
- s3_payload_template: str,
40
- s3_output_data: str,
41
- use_dpr_mockup: bool = False,
42
- ):
43
- """
44
- Prefect flow for on-demand processing.
45
-
46
- Args:
47
- env: Prefect flow environment
48
- processor: DPR processor name
49
- cadip_collection_identifier: CADIP collection identifier (to know the station)
50
- session_identifier: Session identifier
51
- catalog_collection_identifier: Catalog collection identifier where CADIP sessions and AUX data are staged
52
- s3_payload_template: S3 bucket location of the DPR payload file template.
53
- s3_output_data: S3 bucket location of the output processed products. They will then be copied to the
54
- catalog bucket.
55
- use_dpr_mockup: Use the real or the mockup DPR processor ?
56
- """
57
- # logger = get_run_logger()
58
-
59
- # Init flow environment and opentelemetry span
60
- flow_env = FlowEnv(env)
61
- with flow_env.start_span(__name__, "on-demand-processing"):
62
-
63
- # Read values from the payload file
64
- payload_values = read_payload_values.submit(s3_payload_template)
65
-
66
- # Search Cadip sessions
67
- cadip_items = cadip_flow.search_task.submit(
68
- flow_env.serialize(),
69
- cadip_collection_identifier,
70
- session_identifier,
71
- error_if_empty=True,
72
- )
73
-
74
- # Read Auxip CQL2 filter from the processor tasktable.
75
- auxip_cql2 = read_tasktable.submit(flow_env.serialize(), processor, payload_values, cadip_items)
76
-
77
- # Search Auxip products
78
- auxip_items = auxip_flow.search_task.submit(flow_env.serialize(), auxip_cql2, error_if_empty=True)
79
-
80
- # Auxip and Cadip item ids
81
- item_ids = []
82
- for items in [cadip_items.result(), auxip_items.result()]:
83
- for item in items or []: # type: ignore[union-attr]
84
- item_ids.append(item.id)
85
-
86
- # Stage Auxip and Cadip items.
87
- # Note: the only difference between staging_task_auxip and
88
- # staging_task_cadip is the task name in the prefect dashboard.
89
- staged = [
90
- staging_task_auxip.submit(
91
- flow_env.serialize(),
92
- auxip_items,
93
- catalog_collection_identifier,
94
- ),
95
- staging_task_cadip.submit(
96
- flow_env.serialize(),
97
- cadip_items,
98
- catalog_collection_identifier,
99
- ),
100
- ]
101
-
102
- # Write the final payload file from its template version and staged items.
103
- # It will be uploaded in the same s3 dir than the template file.
104
- s3_payload_run = s3_payload_template + ".run" + Path(s3_payload_template).suffix
105
- written = write_payload.submit(
106
- flow_env.serialize(),
107
- s3_payload_template,
108
- item_ids,
109
- catalog_collection_identifier,
110
- s3_output_data,
111
- s3_payload_run,
112
- wait_for=staged, # wait for items to be staged in the catalog
113
- )
114
-
115
- # Run the DPR processor
116
- processed_items = run_processor.submit(
117
- flow_env.serialize(),
118
- processor,
119
- s3_payload_run,
120
- use_dpr_mockup,
121
- wait_for=written,
122
- )
123
-
124
- # Publish processed items to the catalog
125
- published = catalog_flow.publish.submit(
126
- flow_env.serialize(),
127
- catalog_collection_identifier,
128
- processed_items,
129
- s3_output_data,
130
- )
131
-
132
- # Wait for last task to end.
133
- # NOTE: use .result() and not .wait() to unwrap and propagate exceptions, if any.
134
- published.result() # type: ignore[unused-coroutine]