rs-client-libraries 0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- config/staging_templates/README.md +17 -0
- config/staging_templates/yaml/additionalParameter.yaml +30 -0
- config/staging_templates/yaml/bbox.yaml +34 -0
- config/staging_templates/yaml/binaryInputValue.yaml +15 -0
- config/staging_templates/yaml/confClasses.yaml +22 -0
- config/staging_templates/yaml/descriptionType.yaml +36 -0
- config/staging_templates/yaml/dpr_openapi_schema.yaml +223 -0
- config/staging_templates/yaml/exception.yaml +31 -0
- config/staging_templates/yaml/execute.yaml +34 -0
- config/staging_templates/yaml/format.yaml +25 -0
- config/staging_templates/yaml/inlineOrRefData.yaml +18 -0
- config/staging_templates/yaml/inputDescription.yaml +32 -0
- config/staging_templates/yaml/inputValue.yaml +17 -0
- config/staging_templates/yaml/inputValueNoObject.yaml +22 -0
- config/staging_templates/yaml/jobControlOptions.yaml +19 -0
- config/staging_templates/yaml/jobList.yaml +27 -0
- config/staging_templates/yaml/landingPage.yaml +28 -0
- config/staging_templates/yaml/link.yaml +31 -0
- config/staging_templates/yaml/metadata.yaml +22 -0
- config/staging_templates/yaml/output.yaml +20 -0
- config/staging_templates/yaml/outputDescription.yaml +22 -0
- config/staging_templates/yaml/process.yaml +23 -0
- config/staging_templates/yaml/processList.yaml +27 -0
- config/staging_templates/yaml/processSummary.yaml +37 -0
- config/staging_templates/yaml/qualifiedInputValue.yaml +22 -0
- config/staging_templates/yaml/reference.yaml +21 -0
- config/staging_templates/yaml/results.yaml +16 -0
- config/staging_templates/yaml/schema.yaml +144 -0
- config/staging_templates/yaml/staging_openapi_schema.yaml +223 -0
- config/staging_templates/yaml/statusCode.yaml +22 -0
- config/staging_templates/yaml/statusInfo.yaml +52 -0
- config/staging_templates/yaml/subscriber.yaml +33 -0
- config/staging_templates/yaml/transmissionMode.yaml +19 -0
- rs_client/__init__.py +18 -0
- rs_client/ogcapi/__init__.py +15 -0
- rs_client/ogcapi/dpr_client.py +231 -0
- rs_client/ogcapi/ogcapi_client.py +282 -0
- rs_client/ogcapi/staging_client.py +178 -0
- rs_client/rs_client.py +277 -0
- rs_client/stac/__init__.py +15 -0
- rs_client/stac/auxip_client.py +70 -0
- rs_client/stac/cadip_client.py +70 -0
- rs_client/stac/catalog_client.py +479 -0
- rs_client/stac/stac_base.py +313 -0
- rs_client_libraries-0.2.dist-info/LICENSE +201 -0
- rs_client_libraries-0.2.dist-info/METADATA +44 -0
- rs_client_libraries-0.2.dist-info/RECORD +62 -0
- rs_client_libraries-0.2.dist-info/WHEEL +4 -0
- rs_common/__init__.py +20 -0
- rs_common/config.py +69 -0
- rs_common/init_opentelemetry.py +250 -0
- rs_common/logging.py +107 -0
- rs_common/prefect_utils.py +407 -0
- rs_common/utils.py +70 -0
- rs_workflows/__init__.py +15 -0
- rs_workflows/auxip_flow.py +70 -0
- rs_workflows/cadip_flow.py +69 -0
- rs_workflows/catalog_flow.py +71 -0
- rs_workflows/dpr_flow.py +272 -0
- rs_workflows/flow_utils.py +127 -0
- rs_workflows/on_demand_processing.py +243 -0
- rs_workflows/staging_flow.py +78 -0
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
# Copyright 2025 CS Group
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""Catalog flow implementation"""
|
|
16
|
+
|
|
17
|
+
import json
|
|
18
|
+
from datetime import datetime
|
|
19
|
+
|
|
20
|
+
from prefect import get_run_logger, task
|
|
21
|
+
from pystac import Asset, Item
|
|
22
|
+
|
|
23
|
+
from rs_client.stac.catalog_client import CatalogClient
|
|
24
|
+
from rs_workflows.flow_utils import FlowEnv, FlowEnvArgs
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@task(name="Publish to catalog")
|
|
28
|
+
async def publish(
|
|
29
|
+
env: FlowEnvArgs,
|
|
30
|
+
catalog_collection_identifier: str,
|
|
31
|
+
items: list[dict],
|
|
32
|
+
s3_output_data: str,
|
|
33
|
+
):
|
|
34
|
+
"""
|
|
35
|
+
Publish items to the catalog
|
|
36
|
+
|
|
37
|
+
Args:
|
|
38
|
+
env: Prefect flow environment
|
|
39
|
+
catalog_collection_identifier: Catalog collection identifier where the items are staged
|
|
40
|
+
items: Items to publish, as STAC dicts
|
|
41
|
+
s3_output_data: S3 bucket location of the output processed products.
|
|
42
|
+
"""
|
|
43
|
+
logger = get_run_logger()
|
|
44
|
+
|
|
45
|
+
# Init flow environment and opentelemetry span
|
|
46
|
+
flow_env = FlowEnv(env)
|
|
47
|
+
with flow_env.start_span(__name__, "publish-to-catalog"):
|
|
48
|
+
catalog_client: CatalogClient = flow_env.rs_client.get_catalog_client()
|
|
49
|
+
for feature_dict in items:
|
|
50
|
+
try:
|
|
51
|
+
item = Item(
|
|
52
|
+
id=feature_dict["stac_discovery"]["id"],
|
|
53
|
+
geometry=feature_dict["stac_discovery"]["geometry"],
|
|
54
|
+
bbox=feature_dict["stac_discovery"]["bbox"],
|
|
55
|
+
datetime=datetime.fromisoformat(
|
|
56
|
+
feature_dict["stac_discovery"]["properties"]["datetime"],
|
|
57
|
+
),
|
|
58
|
+
properties=feature_dict["stac_discovery"]["properties"],
|
|
59
|
+
)
|
|
60
|
+
asset = Asset(href=f"{s3_output_data}/{item.id}.zarr.zip")
|
|
61
|
+
item.assets = {f"{item.id}.zarr.zip": asset}
|
|
62
|
+
catalog_client.add_item(catalog_collection_identifier, item)
|
|
63
|
+
except Exception as e:
|
|
64
|
+
raise RuntimeError(f"Exception while publishing: {json.dumps(feature_dict, indent=2)}") from e
|
|
65
|
+
|
|
66
|
+
collections = catalog_client.get_collections()
|
|
67
|
+
logger.info("\nCollections response:")
|
|
68
|
+
for collection in collections:
|
|
69
|
+
logger.info(f"ID: {collection.id}, Title: {collection.title}")
|
|
70
|
+
|
|
71
|
+
logger.info("End catalog publishing")
|
rs_workflows/dpr_flow.py
ADDED
|
@@ -0,0 +1,272 @@
|
|
|
1
|
+
# Copyright 2025 CS Group
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""DPR flow implementation"""
|
|
16
|
+
|
|
17
|
+
import copy
|
|
18
|
+
import tempfile
|
|
19
|
+
from os import path as osp
|
|
20
|
+
|
|
21
|
+
import yaml
|
|
22
|
+
from prefect import get_run_logger, task
|
|
23
|
+
from pydantic import BaseModel
|
|
24
|
+
from pystac import ItemCollection
|
|
25
|
+
|
|
26
|
+
from rs_client.ogcapi.dpr_client import DprClient
|
|
27
|
+
from rs_client.stac.catalog_client import CatalogClient
|
|
28
|
+
from rs_common import prefect_utils
|
|
29
|
+
from rs_workflows.flow_utils import FlowEnv, FlowEnvArgs, ProcessorEnum
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class PayloadValues(BaseModel):
|
|
33
|
+
"""Values read from the payload file."""
|
|
34
|
+
|
|
35
|
+
module: str
|
|
36
|
+
processing_unit: str
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@task(name="Read payload values")
|
|
40
|
+
async def read_payload_values(s3_payload: str) -> PayloadValues:
|
|
41
|
+
"""
|
|
42
|
+
Read values from the payload file.
|
|
43
|
+
|
|
44
|
+
Args:
|
|
45
|
+
s3_payload: S3 bucket location of the DPR payload file.
|
|
46
|
+
"""
|
|
47
|
+
logger = get_run_logger()
|
|
48
|
+
|
|
49
|
+
# Download the payload file into a temp file
|
|
50
|
+
with tempfile.NamedTemporaryFile() as temp:
|
|
51
|
+
await prefect_utils.s3_download_file(s3_payload, temp.name)
|
|
52
|
+
|
|
53
|
+
# Read it as a yaml file
|
|
54
|
+
with open(temp.name, encoding="utf-8") as opened:
|
|
55
|
+
payload = yaml.safe_load(opened)
|
|
56
|
+
|
|
57
|
+
workflow = payload.get("workflow", [])
|
|
58
|
+
for step in workflow:
|
|
59
|
+
if "name" not in step:
|
|
60
|
+
continue
|
|
61
|
+
module = step.get("module")
|
|
62
|
+
processing_unit = step.get("processing_unit")
|
|
63
|
+
if not module:
|
|
64
|
+
raise ValueError(f"Missing 'module' in processor payload configuration: {step['name']}")
|
|
65
|
+
if not processing_unit:
|
|
66
|
+
raise ValueError(f"Missing 'processing_unit' in processor payload configuration: {step['name']}")
|
|
67
|
+
logger.info(f"For {step['name']} found module: {module} and processing_unit: {processing_unit}")
|
|
68
|
+
return PayloadValues(module=module, processing_unit=processing_unit)
|
|
69
|
+
|
|
70
|
+
raise ValueError(f"No processor defined in the workflow of payload file {s3_payload}")
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
@task(name="Read TaskTable")
|
|
74
|
+
async def read_tasktable(
|
|
75
|
+
env: FlowEnvArgs,
|
|
76
|
+
processor: ProcessorEnum, # pylint: disable=unused-argument
|
|
77
|
+
payload_values: PayloadValues, # pylint: disable=unused-argument
|
|
78
|
+
cadip_items: ItemCollection, # pylint: disable=unused-argument
|
|
79
|
+
) -> dict:
|
|
80
|
+
"""
|
|
81
|
+
Read Auxip CQL2 filter from the processor tasktable. See:
|
|
82
|
+
https://gitlab.eopf.copernicus.eu/cpm/eopf-cpm/-/blob/main/docs/source/processor-orchestration-guide/tasktables.rst
|
|
83
|
+
|
|
84
|
+
Args:
|
|
85
|
+
env: Prefect flow environment
|
|
86
|
+
processor: DPR processor name
|
|
87
|
+
payload_values: Values read from the payload file
|
|
88
|
+
cadip_items: Results of the Cadip search
|
|
89
|
+
"""
|
|
90
|
+
# logger = get_run_logger()
|
|
91
|
+
|
|
92
|
+
# Init flow environment and opentelemetry span
|
|
93
|
+
flow_env = FlowEnv(env)
|
|
94
|
+
with flow_env.start_span(__name__, "read-tasktable"):
|
|
95
|
+
|
|
96
|
+
# TODO 2: for now the DPR endpoint returns an empty dict which raises a validation error.
|
|
97
|
+
# So just return an empty dict for now.
|
|
98
|
+
# auxip_cql2 = flow_env.rs_client.get_dpr_client().get_process(processor.value)
|
|
99
|
+
return {}
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
@task(name="Write payload file")
|
|
103
|
+
async def write_payload(
|
|
104
|
+
env: FlowEnvArgs,
|
|
105
|
+
s3_payload_template: str,
|
|
106
|
+
item_ids: list[str],
|
|
107
|
+
catalog_collection_identifier: str,
|
|
108
|
+
s3_output_data: str,
|
|
109
|
+
s3_payload_run: str,
|
|
110
|
+
):
|
|
111
|
+
"""
|
|
112
|
+
Write the final payload file from its template version and staged items.
|
|
113
|
+
|
|
114
|
+
Args:
|
|
115
|
+
env: Prefect flow environment
|
|
116
|
+
s3_payload_template: S3 bucket location of the input template DPR payload file.
|
|
117
|
+
item_ids: Item IDs to be processed by the processor.
|
|
118
|
+
catalog_collection_identifier: Catalog collection identifier where the items are staged
|
|
119
|
+
s3_output_data: S3 bucket location of the output processed products.
|
|
120
|
+
s3_payload_run: S3 bucket location of the output final DPR payload file.
|
|
121
|
+
"""
|
|
122
|
+
|
|
123
|
+
# TODO: should be moved to dpr_client.py and it should call dpr_client.py::update_configuration
|
|
124
|
+
|
|
125
|
+
logger = get_run_logger()
|
|
126
|
+
|
|
127
|
+
# Init flow environment and opentelemetry span
|
|
128
|
+
flow_env = FlowEnv(env)
|
|
129
|
+
with flow_env.start_span(__name__, "write-payload"):
|
|
130
|
+
|
|
131
|
+
# Download the payload file into a temp file
|
|
132
|
+
with tempfile.NamedTemporaryFile() as temp:
|
|
133
|
+
await prefect_utils.s3_download_file(s3_payload_template, temp.name)
|
|
134
|
+
|
|
135
|
+
# Read it as a yaml file
|
|
136
|
+
with open(temp.name, encoding="utf-8") as opened:
|
|
137
|
+
payload = yaml.safe_load(opened)
|
|
138
|
+
|
|
139
|
+
# Create mandatory keys if missing
|
|
140
|
+
payload.setdefault("I/O", {})
|
|
141
|
+
|
|
142
|
+
# Retrieve a default template from the existing input_products if available;
|
|
143
|
+
# otherwise, use a minimal default.
|
|
144
|
+
if payload["I/O"].get("input_products"):
|
|
145
|
+
default_template = copy.deepcopy(payload["I/O"]["input_products"][0])
|
|
146
|
+
else:
|
|
147
|
+
default_template = {"id": None, "path": None}
|
|
148
|
+
|
|
149
|
+
# Retrieve a default template from the existing output_products if available;
|
|
150
|
+
# otherwise, use a minimal default.
|
|
151
|
+
if payload["I/O"].get("output_products"):
|
|
152
|
+
default_template_out = copy.deepcopy(payload["I/O"]["output_products"][0])
|
|
153
|
+
else:
|
|
154
|
+
default_template_out = {"id": None, "path": None}
|
|
155
|
+
output_ids = {}
|
|
156
|
+
|
|
157
|
+
for wf in payload.get("workflow", []):
|
|
158
|
+
outputs = wf.get("outputs", {})
|
|
159
|
+
if outputs:
|
|
160
|
+
# Merge outputs from all workflows; keys are not used here, only the values (output IDs)
|
|
161
|
+
output_ids.update(outputs)
|
|
162
|
+
else:
|
|
163
|
+
raise ValueError("At least on output should be in outputs workflow")
|
|
164
|
+
|
|
165
|
+
new_input_products = []
|
|
166
|
+
workflow_inputs = {}
|
|
167
|
+
cadip_index = 1
|
|
168
|
+
auxip_index = 1
|
|
169
|
+
|
|
170
|
+
# Get the staged items from the catalog
|
|
171
|
+
catalog_client: CatalogClient = flow_env.rs_client.get_catalog_client()
|
|
172
|
+
catalog_items = catalog_client.get_items(catalog_collection_identifier, item_ids)
|
|
173
|
+
|
|
174
|
+
# For each staged item
|
|
175
|
+
for item in catalog_items:
|
|
176
|
+
if len(item.assets) == 0:
|
|
177
|
+
raise ValueError(f"No assets in item {item.id}")
|
|
178
|
+
|
|
179
|
+
first_asset = list(item.assets.items())[0][1]
|
|
180
|
+
logger.debug(f"rs-client-libs: first_asset = {first_asset}")
|
|
181
|
+
full_s3_href = first_asset.href
|
|
182
|
+
if not full_s3_href:
|
|
183
|
+
raise ValueError(f"S3 HREF not found in extra fields for item {item.id}")
|
|
184
|
+
|
|
185
|
+
session_s3_href = "/".join(full_s3_href.split("/")[:-1])
|
|
186
|
+
logger.info(f"Session {item.id} has S3_HREF: {session_s3_href}")
|
|
187
|
+
|
|
188
|
+
# Create a new input product using the default template.
|
|
189
|
+
new_product = copy.deepcopy(default_template)
|
|
190
|
+
new_product["id"] = item.id
|
|
191
|
+
new_product["path"] = session_s3_href
|
|
192
|
+
|
|
193
|
+
# these are not accepted by the real processor
|
|
194
|
+
# if "cadip:id" in item.properties:
|
|
195
|
+
# new_product["store_type"] = "cadu"
|
|
196
|
+
# if "auxip:id" in item.properties:
|
|
197
|
+
# new_product["store_type"] = "aux"
|
|
198
|
+
|
|
199
|
+
new_product["store_type"] = "safe"
|
|
200
|
+
|
|
201
|
+
new_input_products.append(new_product)
|
|
202
|
+
|
|
203
|
+
if "cadip:id" in item.properties:
|
|
204
|
+
workflow_inputs[f"CADU{cadip_index}"] = item.id
|
|
205
|
+
cadip_index += 1
|
|
206
|
+
if "auxip:id" in item.properties:
|
|
207
|
+
workflow_inputs[f"AUX{auxip_index}"] = item.id
|
|
208
|
+
auxip_index += 1
|
|
209
|
+
|
|
210
|
+
# Build a new output_products list using the default template for each output
|
|
211
|
+
new_output_products = []
|
|
212
|
+
for _, output_id in output_ids.items():
|
|
213
|
+
# Create a new entry from the default template
|
|
214
|
+
product_entry = copy.deepcopy(default_template_out)
|
|
215
|
+
# Update the id and path accordingly
|
|
216
|
+
product_entry["id"] = output_id
|
|
217
|
+
product_entry["path"] = f"{s3_output_data}" # with /{output_id} ?
|
|
218
|
+
new_output_products.append(product_entry)
|
|
219
|
+
|
|
220
|
+
# Update payload with new input products and workflow inputs
|
|
221
|
+
payload["I/O"]["input_products"] = new_input_products
|
|
222
|
+
for wf in payload.get("workflow", []):
|
|
223
|
+
wf["inputs"] = workflow_inputs
|
|
224
|
+
payload["I/O"]["output_products"] = new_output_products
|
|
225
|
+
|
|
226
|
+
# Write back the payload contents to a temp file
|
|
227
|
+
with tempfile.NamedTemporaryFile() as temp:
|
|
228
|
+
with open(temp.name, "w", encoding="utf-8") as opened:
|
|
229
|
+
yaml.dump(payload, opened, default_flow_style=False, sort_keys=False)
|
|
230
|
+
|
|
231
|
+
with open(temp.name, encoding="utf-8") as opened:
|
|
232
|
+
payload_after = opened.read()
|
|
233
|
+
logger.info(f"Payload file after 'write_payload':\n {payload_after}")
|
|
234
|
+
|
|
235
|
+
# Upload the new config payload file back to S3
|
|
236
|
+
await prefect_utils.s3_upload_file(temp.name, s3_payload_run)
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
@task(name="Run DPR processor")
|
|
240
|
+
async def run_processor(
|
|
241
|
+
env: FlowEnvArgs,
|
|
242
|
+
processor: ProcessorEnum,
|
|
243
|
+
s3_payload_run: str,
|
|
244
|
+
use_dpr_mockup: bool = False,
|
|
245
|
+
) -> list[dict]:
|
|
246
|
+
"""
|
|
247
|
+
Run the DPR processor.
|
|
248
|
+
|
|
249
|
+
Args:
|
|
250
|
+
env: Prefect flow environment
|
|
251
|
+
processor: DPR processor name
|
|
252
|
+
s3_payload_run: S3 bucket location of the output final DPR payload file.
|
|
253
|
+
use_dpr_mockup: Use the real or the mockup DPR processor ?
|
|
254
|
+
"""
|
|
255
|
+
logger = get_run_logger()
|
|
256
|
+
|
|
257
|
+
# Init flow environment and opentelemetry span
|
|
258
|
+
flow_env = FlowEnv(env)
|
|
259
|
+
with flow_env.start_span(__name__, "run-processor"):
|
|
260
|
+
|
|
261
|
+
# Trigger the processor run from the dpr service
|
|
262
|
+
dpr_client: DprClient = flow_env.rs_client.get_dpr_client()
|
|
263
|
+
job_status = dpr_client.run_process(
|
|
264
|
+
process=processor.value,
|
|
265
|
+
s3_config_dir=osp.dirname(s3_payload_run),
|
|
266
|
+
payload_subpath=osp.basename(s3_payload_run),
|
|
267
|
+
s3_report_dir=None,
|
|
268
|
+
use_dpr_mockup=use_dpr_mockup,
|
|
269
|
+
)
|
|
270
|
+
|
|
271
|
+
# Wait for the job to finish
|
|
272
|
+
return dpr_client.wait_for_job(job_status, logger, f"{processor.value!r} processor")
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
# Copyright 2025 CS Group
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""Utility module for the Prefect flows."""
|
|
16
|
+
|
|
17
|
+
import os
|
|
18
|
+
from collections.abc import Iterator
|
|
19
|
+
from enum import Enum
|
|
20
|
+
|
|
21
|
+
from opentelemetry import trace
|
|
22
|
+
from opentelemetry.trace import Span, SpanContext
|
|
23
|
+
from opentelemetry.util._decorator import _agnosticcontextmanager
|
|
24
|
+
from prefect import get_run_logger
|
|
25
|
+
from pydantic import BaseModel, Field
|
|
26
|
+
|
|
27
|
+
from rs_client.rs_client import RsClient
|
|
28
|
+
from rs_common import init_opentelemetry, prefect_utils
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class ProcessorEnum(str, Enum):
|
|
32
|
+
"""DPR processor name"""
|
|
33
|
+
|
|
34
|
+
# String value = resource name in the rs-dpr-service
|
|
35
|
+
S1L0 = "s1_l0"
|
|
36
|
+
S3L0 = "s3_l0"
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class FlowEnvArgs(BaseModel):
|
|
40
|
+
"""
|
|
41
|
+
Prefect flow environment, as a Pydantic serializable object.
|
|
42
|
+
|
|
43
|
+
Attributes:
|
|
44
|
+
owner_id: User/owner ID (necessary to retrieve the user info: API key and OAuth2 cookie)
|
|
45
|
+
from the right Prefect block. NOTE: may be useless after each user has their own prefect
|
|
46
|
+
server because there will be only one block.
|
|
47
|
+
calling_span (tuple): Serialized OpenTelemetry span of the calling flow, if any.
|
|
48
|
+
"""
|
|
49
|
+
|
|
50
|
+
owner_id: str = Field(description="User/owner ID")
|
|
51
|
+
calling_span: tuple[int, int, bool] | None = Field(description="OpenTelemetry info", default=None)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class FlowEnv:
|
|
55
|
+
"""
|
|
56
|
+
Prefect flow environment and reusable objects.
|
|
57
|
+
|
|
58
|
+
Attributes:
|
|
59
|
+
owner_id (str): User/owner ID
|
|
60
|
+
calling_span (SpanContext | None): OpenTelemetry span of the calling flow, if any.
|
|
61
|
+
this_span (SpanContext | None): Current OpenTelemetry span.
|
|
62
|
+
rs_client (RsClient): RsClient instance
|
|
63
|
+
"""
|
|
64
|
+
|
|
65
|
+
def __init__(self, args: FlowEnvArgs):
|
|
66
|
+
"""Constructor."""
|
|
67
|
+
self.owner_id: str = args.owner_id
|
|
68
|
+
self.calling_span: SpanContext | None = None
|
|
69
|
+
self.this_span: SpanContext | None = None
|
|
70
|
+
|
|
71
|
+
# Deserialize the calling span, if any
|
|
72
|
+
if args.calling_span:
|
|
73
|
+
self.calling_span = SpanContext(*args.calling_span)
|
|
74
|
+
|
|
75
|
+
# Read prefect blocks into env vars
|
|
76
|
+
prefect_utils.read_prefect_blocks(self.owner_id, _sync=True) # type: ignore
|
|
77
|
+
|
|
78
|
+
# Init opentelemetry traces
|
|
79
|
+
init_opentelemetry.init_traces("rs.client")
|
|
80
|
+
|
|
81
|
+
# Init the RsClient instance from the env vars
|
|
82
|
+
self.rs_client = RsClient(
|
|
83
|
+
rs_server_href=os.getenv("RSPY_WEBSITE"),
|
|
84
|
+
rs_server_api_key=os.getenv("RSPY_APIKEY"),
|
|
85
|
+
owner_id=self.owner_id,
|
|
86
|
+
logger=get_run_logger(), # type: ignore
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
def serialize(self) -> FlowEnvArgs:
|
|
90
|
+
"""Serialize this object with Pydantic."""
|
|
91
|
+
|
|
92
|
+
# The serialized object will be used by a new opentelemetry span.
|
|
93
|
+
# Its calling span will be either the current span, or the current calling span.
|
|
94
|
+
new_calling_span = self.this_span or self.calling_span
|
|
95
|
+
if new_calling_span:
|
|
96
|
+
# Only keep the first n attributes, the other need custom serialization
|
|
97
|
+
serialized_span = tuple(new_calling_span)[:3]
|
|
98
|
+
else:
|
|
99
|
+
serialized_span = None
|
|
100
|
+
|
|
101
|
+
return FlowEnvArgs(owner_id=self.owner_id, calling_span=serialized_span) # type: ignore
|
|
102
|
+
|
|
103
|
+
@_agnosticcontextmanager
|
|
104
|
+
def start_span(
|
|
105
|
+
self,
|
|
106
|
+
instrumenting_module_name: str,
|
|
107
|
+
name: str,
|
|
108
|
+
) -> Iterator[Span]:
|
|
109
|
+
"""
|
|
110
|
+
Context manager for creating a new main or child OpenTelemetry span and set it
|
|
111
|
+
as the current span in this tracer's context.
|
|
112
|
+
|
|
113
|
+
Args:
|
|
114
|
+
instrumenting_module_name: Caller module name, just pass __name__
|
|
115
|
+
name: The name of the span to be created (use a custom name)
|
|
116
|
+
|
|
117
|
+
Yields:
|
|
118
|
+
The newly-created span.
|
|
119
|
+
"""
|
|
120
|
+
# Create new span and save it
|
|
121
|
+
with init_opentelemetry.start_span( # pylint: disable=contextmanager-generator-missing-cleanup
|
|
122
|
+
instrumenting_module_name,
|
|
123
|
+
name,
|
|
124
|
+
self.calling_span,
|
|
125
|
+
) as span:
|
|
126
|
+
self.this_span = trace.get_current_span().get_span_context()
|
|
127
|
+
yield span
|