rs-client-libraries 0.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. config/staging_templates/README.md +17 -0
  2. config/staging_templates/yaml/additionalParameter.yaml +30 -0
  3. config/staging_templates/yaml/bbox.yaml +34 -0
  4. config/staging_templates/yaml/binaryInputValue.yaml +15 -0
  5. config/staging_templates/yaml/confClasses.yaml +22 -0
  6. config/staging_templates/yaml/descriptionType.yaml +36 -0
  7. config/staging_templates/yaml/dpr_openapi_schema.yaml +223 -0
  8. config/staging_templates/yaml/exception.yaml +31 -0
  9. config/staging_templates/yaml/execute.yaml +34 -0
  10. config/staging_templates/yaml/format.yaml +25 -0
  11. config/staging_templates/yaml/inlineOrRefData.yaml +18 -0
  12. config/staging_templates/yaml/inputDescription.yaml +32 -0
  13. config/staging_templates/yaml/inputValue.yaml +17 -0
  14. config/staging_templates/yaml/inputValueNoObject.yaml +22 -0
  15. config/staging_templates/yaml/jobControlOptions.yaml +19 -0
  16. config/staging_templates/yaml/jobList.yaml +27 -0
  17. config/staging_templates/yaml/landingPage.yaml +28 -0
  18. config/staging_templates/yaml/link.yaml +31 -0
  19. config/staging_templates/yaml/metadata.yaml +22 -0
  20. config/staging_templates/yaml/output.yaml +20 -0
  21. config/staging_templates/yaml/outputDescription.yaml +22 -0
  22. config/staging_templates/yaml/process.yaml +23 -0
  23. config/staging_templates/yaml/processList.yaml +27 -0
  24. config/staging_templates/yaml/processSummary.yaml +37 -0
  25. config/staging_templates/yaml/qualifiedInputValue.yaml +22 -0
  26. config/staging_templates/yaml/reference.yaml +21 -0
  27. config/staging_templates/yaml/results.yaml +16 -0
  28. config/staging_templates/yaml/schema.yaml +144 -0
  29. config/staging_templates/yaml/staging_openapi_schema.yaml +223 -0
  30. config/staging_templates/yaml/statusCode.yaml +22 -0
  31. config/staging_templates/yaml/statusInfo.yaml +52 -0
  32. config/staging_templates/yaml/subscriber.yaml +33 -0
  33. config/staging_templates/yaml/transmissionMode.yaml +19 -0
  34. rs_client/__init__.py +18 -0
  35. rs_client/ogcapi/__init__.py +15 -0
  36. rs_client/ogcapi/dpr_client.py +231 -0
  37. rs_client/ogcapi/ogcapi_client.py +282 -0
  38. rs_client/ogcapi/staging_client.py +178 -0
  39. rs_client/rs_client.py +277 -0
  40. rs_client/stac/__init__.py +15 -0
  41. rs_client/stac/auxip_client.py +70 -0
  42. rs_client/stac/cadip_client.py +70 -0
  43. rs_client/stac/catalog_client.py +479 -0
  44. rs_client/stac/stac_base.py +313 -0
  45. rs_client_libraries-0.2.dist-info/LICENSE +201 -0
  46. rs_client_libraries-0.2.dist-info/METADATA +44 -0
  47. rs_client_libraries-0.2.dist-info/RECORD +62 -0
  48. rs_client_libraries-0.2.dist-info/WHEEL +4 -0
  49. rs_common/__init__.py +20 -0
  50. rs_common/config.py +69 -0
  51. rs_common/init_opentelemetry.py +250 -0
  52. rs_common/logging.py +107 -0
  53. rs_common/prefect_utils.py +407 -0
  54. rs_common/utils.py +70 -0
  55. rs_workflows/__init__.py +15 -0
  56. rs_workflows/auxip_flow.py +70 -0
  57. rs_workflows/cadip_flow.py +69 -0
  58. rs_workflows/catalog_flow.py +71 -0
  59. rs_workflows/dpr_flow.py +272 -0
  60. rs_workflows/flow_utils.py +127 -0
  61. rs_workflows/on_demand_processing.py +243 -0
  62. rs_workflows/staging_flow.py +78 -0
@@ -0,0 +1,243 @@
1
+ # Copyright 2025 CS Group
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """Prefect flows and tasks for on-demand processing"""
16
+
17
+ import datetime
18
+ from pathlib import Path
19
+
20
+ from prefect import flow
21
+
22
+ from rs_workflows import auxip_flow, cadip_flow, catalog_flow
23
+ from rs_workflows.dpr_flow import (
24
+ read_payload_values,
25
+ read_tasktable,
26
+ run_processor,
27
+ write_payload,
28
+ )
29
+ from rs_workflows.flow_utils import FlowEnv, FlowEnvArgs, ProcessorEnum
30
+ from rs_workflows.staging_flow import staging_task_auxip, staging_task_cadip
31
+
32
+
33
+ @flow(name="On-demand processing")
34
+ async def on_demand_processing(
35
+ env: FlowEnvArgs,
36
+ processor: ProcessorEnum,
37
+ cadip_collection_identifier: str,
38
+ session_identifier: str,
39
+ catalog_collection_identifier: str,
40
+ s3_payload_template: str,
41
+ s3_output_data: str,
42
+ use_dpr_mockup: bool = False,
43
+ ):
44
+ """
45
+ Prefect flow for on-demand processing.
46
+
47
+ Args:
48
+ env: Prefect flow environment
49
+ processor: DPR processor name
50
+ cadip_collection_identifier: CADIP collection identifier that contains the mission and station
51
+ (e.g. s1_ins for Sentinel-1 sessions from the Inuvik station)
52
+ session_identifier: Session identifier
53
+ catalog_collection_identifier: Catalog collection identifier where CADIP sessions and AUX data are staged
54
+ s3_payload_template: S3 bucket location of the DPR payload file template.
55
+ s3_output_data: S3 bucket location of the output processed products. They will then be copied to the
56
+ catalog bucket.
57
+ use_dpr_mockup: Use the real or the mockup DPR processor ?
58
+ """
59
+ # logger = get_run_logger()
60
+
61
+ # Init flow environment and opentelemetry span
62
+ flow_env = FlowEnv(env)
63
+ with flow_env.start_span(__name__, "on-demand-processing"):
64
+
65
+ # Read values from the payload file
66
+ payload_values = read_payload_values.submit(s3_payload_template)
67
+
68
+ # Search Cadip sessions
69
+ cadip_items = cadip_flow.search_task.submit(
70
+ flow_env.serialize(),
71
+ cadip_collection_identifier,
72
+ session_identifier,
73
+ error_if_empty=True,
74
+ )
75
+
76
+ # Read Auxip CQL2 filter from the processor tasktable.
77
+ auxip_cql2 = read_tasktable.submit(flow_env.serialize(), processor, payload_values, cadip_items)
78
+
79
+ # Search Auxip products
80
+ auxip_items = auxip_flow.search_task.submit(flow_env.serialize(), auxip_cql2, error_if_empty=True)
81
+
82
+ # Auxip and Cadip item ids
83
+ item_ids = []
84
+ for items in [cadip_items.result(), auxip_items.result()]:
85
+ for item in items or []: # type: ignore[union-attr]
86
+ item_ids.append(item.id)
87
+
88
+ # Stage Auxip and Cadip items.
89
+ # Note: the only difference between staging_task_auxip and
90
+ # staging_task_cadip is the task name in the prefect dashboard.
91
+ staged = [
92
+ staging_task_auxip.submit(
93
+ flow_env.serialize(),
94
+ auxip_items,
95
+ catalog_collection_identifier,
96
+ ),
97
+ staging_task_cadip.submit(
98
+ flow_env.serialize(),
99
+ cadip_items,
100
+ catalog_collection_identifier,
101
+ ),
102
+ ]
103
+
104
+ # Write the final payload file from its template version and staged items.
105
+ # It will be uploaded in the same s3 dir than the template file.
106
+ s3_payload_run = s3_payload_template + ".run" + Path(s3_payload_template).suffix
107
+ written = write_payload.submit(
108
+ flow_env.serialize(),
109
+ s3_payload_template,
110
+ item_ids,
111
+ catalog_collection_identifier,
112
+ s3_output_data,
113
+ s3_payload_run,
114
+ wait_for=staged, # wait for items to be staged in the catalog
115
+ )
116
+
117
+ # Run the DPR processor
118
+ processed_items = run_processor.submit(
119
+ flow_env.serialize(),
120
+ processor,
121
+ s3_payload_run,
122
+ use_dpr_mockup,
123
+ wait_for=written,
124
+ )
125
+
126
+ # Publish processed items to the catalog
127
+ published = catalog_flow.publish.submit(
128
+ flow_env.serialize(),
129
+ catalog_collection_identifier,
130
+ processed_items,
131
+ s3_output_data,
132
+ )
133
+
134
+ # Wait for last task to end.
135
+ # NOTE: use .result() and not .wait() to unwrap and propagate exceptions, if any.
136
+ published.result() # type: ignore[unused-coroutine]
137
+
138
+
139
+ @flow(name="On-demand Cadip staging")
140
+ async def on_demand_cadip_staging(
141
+ env: FlowEnvArgs,
142
+ cadip_collection_identifier: str,
143
+ session_identifier: str,
144
+ catalog_collection_identifier: str,
145
+ ):
146
+ """
147
+ Flow to retrieve a session, stage it and add the STAC item into the catalog.
148
+
149
+ Args:
150
+ env: Prefect flow environment
151
+ cadip_collection_identifier: CADIP collection identifier that contains the mission and station
152
+ (e.g. s1_ins for Sentinel-1 sessions from the Inuvik station)
153
+ session_identifier: Session identifier
154
+ catalog_collection_identifier: Catalog collection identifier where CADIP sessions and AUX data are staged
155
+ """
156
+
157
+ # Init flow environment and opentelemetry span
158
+ flow_env = FlowEnv(env)
159
+ with flow_env.start_span(__name__, "on-demand-cadip-staging"):
160
+
161
+ # Search Cadip sessions
162
+ cadip_items = cadip_flow.search_task.submit(
163
+ flow_env.serialize(),
164
+ cadip_collection_identifier,
165
+ session_identifier,
166
+ error_if_empty=True,
167
+ )
168
+
169
+ # Stage Cadip items.
170
+ staged = staging_task_cadip.submit(flow_env.serialize(), cadip_items, catalog_collection_identifier)
171
+
172
+ # Wait for last task to end.
173
+ # NOTE: use .result() and not .wait() to unwrap and propagate exceptions, if any.
174
+ staged.result() # type: ignore[unused-coroutine]
175
+
176
+
177
+ @flow(name="On-demand Auxip staging")
178
+ async def on_demand_auxip_staging(
179
+ env: FlowEnvArgs,
180
+ start_datetime: datetime.datetime | str,
181
+ end_datetime: datetime.datetime | str,
182
+ product_type: str,
183
+ catalog_collection_identifier: str,
184
+ ):
185
+ """
186
+ Flow to retrieve Auxip files using a ValCover filter with the given time interval defined by
187
+ start_datetime and end_datetime, select only the type of files wanted if eopf_type is given, stage
188
+ the files and add STAC items into the catalog.
189
+ Informations on ValCover filter:
190
+ https://pforge-exchange2.astrium.eads.net/confluence/display/COPRS/4.+External+data+selection+policies
191
+
192
+ Args:
193
+ env: Prefect flow environment
194
+ start_datetime: Start datetime for the time interval used to filter the files
195
+ (select a date or directly enter a timestamp, e.g. "2025-08-07T11:51:12.509000Z")
196
+ end_datetime: End datetime for the time interval used to filter the files
197
+ (select a date or directly enter a timestamp, e.g. "2025-08-10T14:00:00.509000Z")
198
+ product_type: Auxiliary file type wanted
199
+ catalog_collection_identifier: Catalog collection identifier where CADIP sessions and AUX data are staged
200
+ """
201
+
202
+ # Init flow environment and opentelemetry span
203
+ flow_env = FlowEnv(env)
204
+ with flow_env.start_span(__name__, "on-demand-auxip-staging"):
205
+
206
+ # Convert datetime inputs to str
207
+ if isinstance(start_datetime, datetime.datetime):
208
+ start_datetime = start_datetime.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
209
+ if isinstance(end_datetime, datetime.datetime):
210
+ end_datetime = end_datetime.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
211
+
212
+ # CQL2 filter: we use a filter combining a ValCover filter and a product type filter
213
+ cql2_filter = {
214
+ "op": "and",
215
+ "args": [
216
+ {"op": "=", "args": [{"property": "product:type"}, product_type]},
217
+ {
218
+ "op": "t_contains",
219
+ "args": [
220
+ {"interval": [{"property": "start_datetime"}, {"property": "end_datetime"}]},
221
+ {"interval": [start_datetime, end_datetime]},
222
+ ],
223
+ },
224
+ ],
225
+ }
226
+
227
+ # Search Auxip products
228
+ auxip_items = auxip_flow.search_task.submit(
229
+ flow_env.serialize(),
230
+ auxip_cql2={"filter": cql2_filter},
231
+ error_if_empty=False,
232
+ )
233
+
234
+ # Stage Auxip items.
235
+ staged = staging_task_auxip.submit(
236
+ flow_env.serialize(),
237
+ auxip_items,
238
+ catalog_collection_identifier,
239
+ )
240
+
241
+ # Wait for last task to end.
242
+ # NOTE: use .result() and not .wait() to unwrap and propagate exceptions, if any.
243
+ staged.result() # type: ignore[unused-coroutine]
@@ -0,0 +1,78 @@
1
+ # Copyright 2025 CS Group
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """Staging flow implementation"""
16
+
17
+ from prefect import flow, get_run_logger, task
18
+ from pystac import ItemCollection
19
+
20
+ from rs_client.ogcapi.staging_client import StagingClient
21
+ from rs_workflows.flow_utils import FlowEnv, FlowEnvArgs
22
+
23
+
24
+ @flow(name="Staging")
25
+ async def staging(
26
+ env: FlowEnvArgs,
27
+ stac_input: str | ItemCollection | dict, # warning: dict as last choice for prefect ui
28
+ catalog_collection_identifier: str,
29
+ timeout: int = 1200,
30
+ poll_interval: int = 2,
31
+ ):
32
+ """
33
+ Stage STAC items.
34
+
35
+ Args:
36
+ env: Prefect flow environment (at least the owner_id is required)
37
+ stac_input (dict | str): it can be:<br>
38
+ - A Python dictionary corresponding to a Feature or a FeatureCollection (that can be for example
39
+ the output of a search for Cadip or Auxip sessions)<br>
40
+ - A json string corresponding to a Feature or a FeatureCollection<br>
41
+ - A string corresponding to a path to a json file containing a Feature or a FeatureCollection<br>
42
+ - A single link that returns a STAC ItemCollection: this link should be an url to search an ItemCollection
43
+ catalog_collection_identifier: Catalog collection identifier where items are staged
44
+ timeout: Job completion timeout in seconds
45
+ poll_interval: When to check again for job completion in seconds
46
+ """
47
+ logger = get_run_logger()
48
+
49
+ # Init flow environment and opentelemetry span
50
+ flow_env = FlowEnv(env)
51
+ with flow_env.start_span(__name__, "staging"):
52
+
53
+ staging_client: StagingClient = flow_env.rs_client.get_staging_client()
54
+
55
+ # Convert pystac object into dict
56
+ if isinstance(stac_input, ItemCollection):
57
+ stac_input = stac_input.to_dict()
58
+
59
+ # Trigger the staging and wait for jobs to finish
60
+ job_status = staging_client.run_staging(stac_input, catalog_collection_identifier)
61
+ staging_client.wait_for_jobs(job_status, logger, timeout, poll_interval)
62
+
63
+
64
+ ###########################
65
+ # Call the flows as tasks #
66
+ ###########################
67
+
68
+
69
+ @task(name="Auxip staging")
70
+ async def staging_task_auxip(*args, **kwargs):
71
+ """See: staging"""
72
+ return await staging.fn(*args, **kwargs)
73
+
74
+
75
+ @task(name="Cadip staging")
76
+ async def staging_task_cadip(*args, **kwargs):
77
+ """See: staging"""
78
+ return await staging.fn(*args, **kwargs)