rs-client-libraries 0.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. config/staging_templates/README.md +17 -0
  2. config/staging_templates/yaml/additionalParameter.yaml +30 -0
  3. config/staging_templates/yaml/bbox.yaml +34 -0
  4. config/staging_templates/yaml/binaryInputValue.yaml +15 -0
  5. config/staging_templates/yaml/confClasses.yaml +22 -0
  6. config/staging_templates/yaml/descriptionType.yaml +36 -0
  7. config/staging_templates/yaml/dpr_openapi_schema.yaml +223 -0
  8. config/staging_templates/yaml/exception.yaml +31 -0
  9. config/staging_templates/yaml/execute.yaml +34 -0
  10. config/staging_templates/yaml/format.yaml +25 -0
  11. config/staging_templates/yaml/inlineOrRefData.yaml +18 -0
  12. config/staging_templates/yaml/inputDescription.yaml +32 -0
  13. config/staging_templates/yaml/inputValue.yaml +17 -0
  14. config/staging_templates/yaml/inputValueNoObject.yaml +22 -0
  15. config/staging_templates/yaml/jobControlOptions.yaml +19 -0
  16. config/staging_templates/yaml/jobList.yaml +27 -0
  17. config/staging_templates/yaml/landingPage.yaml +28 -0
  18. config/staging_templates/yaml/link.yaml +31 -0
  19. config/staging_templates/yaml/metadata.yaml +22 -0
  20. config/staging_templates/yaml/output.yaml +20 -0
  21. config/staging_templates/yaml/outputDescription.yaml +22 -0
  22. config/staging_templates/yaml/process.yaml +23 -0
  23. config/staging_templates/yaml/processList.yaml +27 -0
  24. config/staging_templates/yaml/processSummary.yaml +37 -0
  25. config/staging_templates/yaml/qualifiedInputValue.yaml +22 -0
  26. config/staging_templates/yaml/reference.yaml +21 -0
  27. config/staging_templates/yaml/results.yaml +16 -0
  28. config/staging_templates/yaml/schema.yaml +144 -0
  29. config/staging_templates/yaml/staging_openapi_schema.yaml +223 -0
  30. config/staging_templates/yaml/statusCode.yaml +22 -0
  31. config/staging_templates/yaml/statusInfo.yaml +52 -0
  32. config/staging_templates/yaml/subscriber.yaml +33 -0
  33. config/staging_templates/yaml/transmissionMode.yaml +19 -0
  34. rs_client/__init__.py +18 -0
  35. rs_client/ogcapi/__init__.py +15 -0
  36. rs_client/ogcapi/dpr_client.py +231 -0
  37. rs_client/ogcapi/ogcapi_client.py +282 -0
  38. rs_client/ogcapi/staging_client.py +178 -0
  39. rs_client/rs_client.py +277 -0
  40. rs_client/stac/__init__.py +15 -0
  41. rs_client/stac/auxip_client.py +70 -0
  42. rs_client/stac/cadip_client.py +70 -0
  43. rs_client/stac/catalog_client.py +479 -0
  44. rs_client/stac/stac_base.py +313 -0
  45. rs_client_libraries-0.2.dist-info/LICENSE +201 -0
  46. rs_client_libraries-0.2.dist-info/METADATA +44 -0
  47. rs_client_libraries-0.2.dist-info/RECORD +62 -0
  48. rs_client_libraries-0.2.dist-info/WHEEL +4 -0
  49. rs_common/__init__.py +20 -0
  50. rs_common/config.py +69 -0
  51. rs_common/init_opentelemetry.py +250 -0
  52. rs_common/logging.py +107 -0
  53. rs_common/prefect_utils.py +407 -0
  54. rs_common/utils.py +70 -0
  55. rs_workflows/__init__.py +15 -0
  56. rs_workflows/auxip_flow.py +70 -0
  57. rs_workflows/cadip_flow.py +69 -0
  58. rs_workflows/catalog_flow.py +71 -0
  59. rs_workflows/dpr_flow.py +272 -0
  60. rs_workflows/flow_utils.py +127 -0
  61. rs_workflows/on_demand_processing.py +243 -0
  62. rs_workflows/staging_flow.py +78 -0
@@ -0,0 +1,71 @@
1
+ # Copyright 2025 CS Group
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """Catalog flow implementation"""
16
+
17
+ import json
18
+ from datetime import datetime
19
+
20
+ from prefect import get_run_logger, task
21
+ from pystac import Asset, Item
22
+
23
+ from rs_client.stac.catalog_client import CatalogClient
24
+ from rs_workflows.flow_utils import FlowEnv, FlowEnvArgs
25
+
26
+
27
+ @task(name="Publish to catalog")
28
+ async def publish(
29
+ env: FlowEnvArgs,
30
+ catalog_collection_identifier: str,
31
+ items: list[dict],
32
+ s3_output_data: str,
33
+ ):
34
+ """
35
+ Publish items to the catalog
36
+
37
+ Args:
38
+ env: Prefect flow environment
39
+ catalog_collection_identifier: Catalog collection identifier where the items are staged
40
+ items: Items to publish, as STAC dicts
41
+ s3_output_data: S3 bucket location of the output processed products.
42
+ """
43
+ logger = get_run_logger()
44
+
45
+ # Init flow environment and opentelemetry span
46
+ flow_env = FlowEnv(env)
47
+ with flow_env.start_span(__name__, "publish-to-catalog"):
48
+ catalog_client: CatalogClient = flow_env.rs_client.get_catalog_client()
49
+ for feature_dict in items:
50
+ try:
51
+ item = Item(
52
+ id=feature_dict["stac_discovery"]["id"],
53
+ geometry=feature_dict["stac_discovery"]["geometry"],
54
+ bbox=feature_dict["stac_discovery"]["bbox"],
55
+ datetime=datetime.fromisoformat(
56
+ feature_dict["stac_discovery"]["properties"]["datetime"],
57
+ ),
58
+ properties=feature_dict["stac_discovery"]["properties"],
59
+ )
60
+ asset = Asset(href=f"{s3_output_data}/{item.id}.zarr.zip")
61
+ item.assets = {f"{item.id}.zarr.zip": asset}
62
+ catalog_client.add_item(catalog_collection_identifier, item)
63
+ except Exception as e:
64
+ raise RuntimeError(f"Exception while publishing: {json.dumps(feature_dict, indent=2)}") from e
65
+
66
+ collections = catalog_client.get_collections()
67
+ logger.info("\nCollections response:")
68
+ for collection in collections:
69
+ logger.info(f"ID: {collection.id}, Title: {collection.title}")
70
+
71
+ logger.info("End catalog publishing")
@@ -0,0 +1,272 @@
1
+ # Copyright 2025 CS Group
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """DPR flow implementation"""
16
+
17
+ import copy
18
+ import tempfile
19
+ from os import path as osp
20
+
21
+ import yaml
22
+ from prefect import get_run_logger, task
23
+ from pydantic import BaseModel
24
+ from pystac import ItemCollection
25
+
26
+ from rs_client.ogcapi.dpr_client import DprClient
27
+ from rs_client.stac.catalog_client import CatalogClient
28
+ from rs_common import prefect_utils
29
+ from rs_workflows.flow_utils import FlowEnv, FlowEnvArgs, ProcessorEnum
30
+
31
+
32
+ class PayloadValues(BaseModel):
33
+ """Values read from the payload file."""
34
+
35
+ module: str
36
+ processing_unit: str
37
+
38
+
39
+ @task(name="Read payload values")
40
+ async def read_payload_values(s3_payload: str) -> PayloadValues:
41
+ """
42
+ Read values from the payload file.
43
+
44
+ Args:
45
+ s3_payload: S3 bucket location of the DPR payload file.
46
+ """
47
+ logger = get_run_logger()
48
+
49
+ # Download the payload file into a temp file
50
+ with tempfile.NamedTemporaryFile() as temp:
51
+ await prefect_utils.s3_download_file(s3_payload, temp.name)
52
+
53
+ # Read it as a yaml file
54
+ with open(temp.name, encoding="utf-8") as opened:
55
+ payload = yaml.safe_load(opened)
56
+
57
+ workflow = payload.get("workflow", [])
58
+ for step in workflow:
59
+ if "name" not in step:
60
+ continue
61
+ module = step.get("module")
62
+ processing_unit = step.get("processing_unit")
63
+ if not module:
64
+ raise ValueError(f"Missing 'module' in processor payload configuration: {step['name']}")
65
+ if not processing_unit:
66
+ raise ValueError(f"Missing 'processing_unit' in processor payload configuration: {step['name']}")
67
+ logger.info(f"For {step['name']} found module: {module} and processing_unit: {processing_unit}")
68
+ return PayloadValues(module=module, processing_unit=processing_unit)
69
+
70
+ raise ValueError(f"No processor defined in the workflow of payload file {s3_payload}")
71
+
72
+
73
+ @task(name="Read TaskTable")
74
+ async def read_tasktable(
75
+ env: FlowEnvArgs,
76
+ processor: ProcessorEnum, # pylint: disable=unused-argument
77
+ payload_values: PayloadValues, # pylint: disable=unused-argument
78
+ cadip_items: ItemCollection, # pylint: disable=unused-argument
79
+ ) -> dict:
80
+ """
81
+ Read Auxip CQL2 filter from the processor tasktable. See:
82
+ https://gitlab.eopf.copernicus.eu/cpm/eopf-cpm/-/blob/main/docs/source/processor-orchestration-guide/tasktables.rst
83
+
84
+ Args:
85
+ env: Prefect flow environment
86
+ processor: DPR processor name
87
+ payload_values: Values read from the payload file
88
+ cadip_items: Results of the Cadip search
89
+ """
90
+ # logger = get_run_logger()
91
+
92
+ # Init flow environment and opentelemetry span
93
+ flow_env = FlowEnv(env)
94
+ with flow_env.start_span(__name__, "read-tasktable"):
95
+
96
+ # TODO 2: for now the DPR endpoint returns an empty dict which raises a validation error.
97
+ # So just return an empty dict for now.
98
+ # auxip_cql2 = flow_env.rs_client.get_dpr_client().get_process(processor.value)
99
+ return {}
100
+
101
+
102
+ @task(name="Write payload file")
103
+ async def write_payload(
104
+ env: FlowEnvArgs,
105
+ s3_payload_template: str,
106
+ item_ids: list[str],
107
+ catalog_collection_identifier: str,
108
+ s3_output_data: str,
109
+ s3_payload_run: str,
110
+ ):
111
+ """
112
+ Write the final payload file from its template version and staged items.
113
+
114
+ Args:
115
+ env: Prefect flow environment
116
+ s3_payload_template: S3 bucket location of the input template DPR payload file.
117
+ item_ids: Item IDs to be processed by the processor.
118
+ catalog_collection_identifier: Catalog collection identifier where the items are staged
119
+ s3_output_data: S3 bucket location of the output processed products.
120
+ s3_payload_run: S3 bucket location of the output final DPR payload file.
121
+ """
122
+
123
+ # TODO: should be moved to dpr_client.py and it should call dpr_client.py::update_configuration
124
+
125
+ logger = get_run_logger()
126
+
127
+ # Init flow environment and opentelemetry span
128
+ flow_env = FlowEnv(env)
129
+ with flow_env.start_span(__name__, "write-payload"):
130
+
131
+ # Download the payload file into a temp file
132
+ with tempfile.NamedTemporaryFile() as temp:
133
+ await prefect_utils.s3_download_file(s3_payload_template, temp.name)
134
+
135
+ # Read it as a yaml file
136
+ with open(temp.name, encoding="utf-8") as opened:
137
+ payload = yaml.safe_load(opened)
138
+
139
+ # Create mandatory keys if missing
140
+ payload.setdefault("I/O", {})
141
+
142
+ # Retrieve a default template from the existing input_products if available;
143
+ # otherwise, use a minimal default.
144
+ if payload["I/O"].get("input_products"):
145
+ default_template = copy.deepcopy(payload["I/O"]["input_products"][0])
146
+ else:
147
+ default_template = {"id": None, "path": None}
148
+
149
+ # Retrieve a default template from the existing output_products if available;
150
+ # otherwise, use a minimal default.
151
+ if payload["I/O"].get("output_products"):
152
+ default_template_out = copy.deepcopy(payload["I/O"]["output_products"][0])
153
+ else:
154
+ default_template_out = {"id": None, "path": None}
155
+ output_ids = {}
156
+
157
+ for wf in payload.get("workflow", []):
158
+ outputs = wf.get("outputs", {})
159
+ if outputs:
160
+ # Merge outputs from all workflows; keys are not used here, only the values (output IDs)
161
+ output_ids.update(outputs)
162
+ else:
163
+ raise ValueError("At least on output should be in outputs workflow")
164
+
165
+ new_input_products = []
166
+ workflow_inputs = {}
167
+ cadip_index = 1
168
+ auxip_index = 1
169
+
170
+ # Get the staged items from the catalog
171
+ catalog_client: CatalogClient = flow_env.rs_client.get_catalog_client()
172
+ catalog_items = catalog_client.get_items(catalog_collection_identifier, item_ids)
173
+
174
+ # For each staged item
175
+ for item in catalog_items:
176
+ if len(item.assets) == 0:
177
+ raise ValueError(f"No assets in item {item.id}")
178
+
179
+ first_asset = list(item.assets.items())[0][1]
180
+ logger.debug(f"rs-client-libs: first_asset = {first_asset}")
181
+ full_s3_href = first_asset.href
182
+ if not full_s3_href:
183
+ raise ValueError(f"S3 HREF not found in extra fields for item {item.id}")
184
+
185
+ session_s3_href = "/".join(full_s3_href.split("/")[:-1])
186
+ logger.info(f"Session {item.id} has S3_HREF: {session_s3_href}")
187
+
188
+ # Create a new input product using the default template.
189
+ new_product = copy.deepcopy(default_template)
190
+ new_product["id"] = item.id
191
+ new_product["path"] = session_s3_href
192
+
193
+ # these are not accepted by the real processor
194
+ # if "cadip:id" in item.properties:
195
+ # new_product["store_type"] = "cadu"
196
+ # if "auxip:id" in item.properties:
197
+ # new_product["store_type"] = "aux"
198
+
199
+ new_product["store_type"] = "safe"
200
+
201
+ new_input_products.append(new_product)
202
+
203
+ if "cadip:id" in item.properties:
204
+ workflow_inputs[f"CADU{cadip_index}"] = item.id
205
+ cadip_index += 1
206
+ if "auxip:id" in item.properties:
207
+ workflow_inputs[f"AUX{auxip_index}"] = item.id
208
+ auxip_index += 1
209
+
210
+ # Build a new output_products list using the default template for each output
211
+ new_output_products = []
212
+ for _, output_id in output_ids.items():
213
+ # Create a new entry from the default template
214
+ product_entry = copy.deepcopy(default_template_out)
215
+ # Update the id and path accordingly
216
+ product_entry["id"] = output_id
217
+ product_entry["path"] = f"{s3_output_data}" # with /{output_id} ?
218
+ new_output_products.append(product_entry)
219
+
220
+ # Update payload with new input products and workflow inputs
221
+ payload["I/O"]["input_products"] = new_input_products
222
+ for wf in payload.get("workflow", []):
223
+ wf["inputs"] = workflow_inputs
224
+ payload["I/O"]["output_products"] = new_output_products
225
+
226
+ # Write back the payload contents to a temp file
227
+ with tempfile.NamedTemporaryFile() as temp:
228
+ with open(temp.name, "w", encoding="utf-8") as opened:
229
+ yaml.dump(payload, opened, default_flow_style=False, sort_keys=False)
230
+
231
+ with open(temp.name, encoding="utf-8") as opened:
232
+ payload_after = opened.read()
233
+ logger.info(f"Payload file after 'write_payload':\n {payload_after}")
234
+
235
+ # Upload the new config payload file back to S3
236
+ await prefect_utils.s3_upload_file(temp.name, s3_payload_run)
237
+
238
+
239
+ @task(name="Run DPR processor")
240
+ async def run_processor(
241
+ env: FlowEnvArgs,
242
+ processor: ProcessorEnum,
243
+ s3_payload_run: str,
244
+ use_dpr_mockup: bool = False,
245
+ ) -> list[dict]:
246
+ """
247
+ Run the DPR processor.
248
+
249
+ Args:
250
+ env: Prefect flow environment
251
+ processor: DPR processor name
252
+ s3_payload_run: S3 bucket location of the output final DPR payload file.
253
+ use_dpr_mockup: Use the real or the mockup DPR processor ?
254
+ """
255
+ logger = get_run_logger()
256
+
257
+ # Init flow environment and opentelemetry span
258
+ flow_env = FlowEnv(env)
259
+ with flow_env.start_span(__name__, "run-processor"):
260
+
261
+ # Trigger the processor run from the dpr service
262
+ dpr_client: DprClient = flow_env.rs_client.get_dpr_client()
263
+ job_status = dpr_client.run_process(
264
+ process=processor.value,
265
+ s3_config_dir=osp.dirname(s3_payload_run),
266
+ payload_subpath=osp.basename(s3_payload_run),
267
+ s3_report_dir=None,
268
+ use_dpr_mockup=use_dpr_mockup,
269
+ )
270
+
271
+ # Wait for the job to finish
272
+ return dpr_client.wait_for_job(job_status, logger, f"{processor.value!r} processor")
@@ -0,0 +1,127 @@
1
+ # Copyright 2025 CS Group
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """Utility module for the Prefect flows."""
16
+
17
+ import os
18
+ from collections.abc import Iterator
19
+ from enum import Enum
20
+
21
+ from opentelemetry import trace
22
+ from opentelemetry.trace import Span, SpanContext
23
+ from opentelemetry.util._decorator import _agnosticcontextmanager
24
+ from prefect import get_run_logger
25
+ from pydantic import BaseModel, Field
26
+
27
+ from rs_client.rs_client import RsClient
28
+ from rs_common import init_opentelemetry, prefect_utils
29
+
30
+
31
+ class ProcessorEnum(str, Enum):
32
+ """DPR processor name"""
33
+
34
+ # String value = resource name in the rs-dpr-service
35
+ S1L0 = "s1_l0"
36
+ S3L0 = "s3_l0"
37
+
38
+
39
+ class FlowEnvArgs(BaseModel):
40
+ """
41
+ Prefect flow environment, as a Pydantic serializable object.
42
+
43
+ Attributes:
44
+ owner_id: User/owner ID (necessary to retrieve the user info: API key and OAuth2 cookie)
45
+ from the right Prefect block. NOTE: may be useless after each user has their own prefect
46
+ server because there will be only one block.
47
+ calling_span (tuple): Serialized OpenTelemetry span of the calling flow, if any.
48
+ """
49
+
50
+ owner_id: str = Field(description="User/owner ID")
51
+ calling_span: tuple[int, int, bool] | None = Field(description="OpenTelemetry info", default=None)
52
+
53
+
54
+ class FlowEnv:
55
+ """
56
+ Prefect flow environment and reusable objects.
57
+
58
+ Attributes:
59
+ owner_id (str): User/owner ID
60
+ calling_span (SpanContext | None): OpenTelemetry span of the calling flow, if any.
61
+ this_span (SpanContext | None): Current OpenTelemetry span.
62
+ rs_client (RsClient): RsClient instance
63
+ """
64
+
65
+ def __init__(self, args: FlowEnvArgs):
66
+ """Constructor."""
67
+ self.owner_id: str = args.owner_id
68
+ self.calling_span: SpanContext | None = None
69
+ self.this_span: SpanContext | None = None
70
+
71
+ # Deserialize the calling span, if any
72
+ if args.calling_span:
73
+ self.calling_span = SpanContext(*args.calling_span)
74
+
75
+ # Read prefect blocks into env vars
76
+ prefect_utils.read_prefect_blocks(self.owner_id, _sync=True) # type: ignore
77
+
78
+ # Init opentelemetry traces
79
+ init_opentelemetry.init_traces("rs.client")
80
+
81
+ # Init the RsClient instance from the env vars
82
+ self.rs_client = RsClient(
83
+ rs_server_href=os.getenv("RSPY_WEBSITE"),
84
+ rs_server_api_key=os.getenv("RSPY_APIKEY"),
85
+ owner_id=self.owner_id,
86
+ logger=get_run_logger(), # type: ignore
87
+ )
88
+
89
+ def serialize(self) -> FlowEnvArgs:
90
+ """Serialize this object with Pydantic."""
91
+
92
+ # The serialized object will be used by a new opentelemetry span.
93
+ # Its calling span will be either the current span, or the current calling span.
94
+ new_calling_span = self.this_span or self.calling_span
95
+ if new_calling_span:
96
+ # Only keep the first n attributes, the other need custom serialization
97
+ serialized_span = tuple(new_calling_span)[:3]
98
+ else:
99
+ serialized_span = None
100
+
101
+ return FlowEnvArgs(owner_id=self.owner_id, calling_span=serialized_span) # type: ignore
102
+
103
+ @_agnosticcontextmanager
104
+ def start_span(
105
+ self,
106
+ instrumenting_module_name: str,
107
+ name: str,
108
+ ) -> Iterator[Span]:
109
+ """
110
+ Context manager for creating a new main or child OpenTelemetry span and set it
111
+ as the current span in this tracer's context.
112
+
113
+ Args:
114
+ instrumenting_module_name: Caller module name, just pass __name__
115
+ name: The name of the span to be created (use a custom name)
116
+
117
+ Yields:
118
+ The newly-created span.
119
+ """
120
+ # Create new span and save it
121
+ with init_opentelemetry.start_span( # pylint: disable=contextmanager-generator-missing-cleanup
122
+ instrumenting_module_name,
123
+ name,
124
+ self.calling_span,
125
+ ) as span:
126
+ self.this_span = trace.get_current_span().get_span_context()
127
+ yield span