deltafi 2.0rc1705080991758__py3-none-any.whl → 2.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of deltafi might be problematic. Click here for more details.
- deltafi/__init__.py +1 -1
- deltafi/action.py +36 -20
- deltafi/actioneventqueue.py +1 -1
- deltafi/actiontype.py +3 -1
- deltafi/domain.py +73 -63
- deltafi/exception.py +1 -11
- deltafi/genericmodel.py +4 -2
- deltafi/input.py +1 -1
- deltafi/logger.py +4 -4
- deltafi/metric.py +2 -2
- deltafi/plugin.py +198 -50
- deltafi/result.py +95 -31
- deltafi/storage.py +6 -1
- deltafi/test_kit/__init__.py +1 -1
- deltafi/test_kit/assertions.py +10 -2
- deltafi/test_kit/compare_helpers.py +251 -8
- deltafi/test_kit/constants.py +1 -1
- deltafi/test_kit/egress.py +54 -0
- deltafi/test_kit/framework.py +146 -112
- deltafi/test_kit/timed_ingress.py +101 -0
- deltafi/test_kit/transform.py +45 -15
- {deltafi-2.0rc1705080991758.dist-info → deltafi-2.4.0.dist-info}/METADATA +13 -12
- deltafi-2.4.0.dist-info/RECORD +24 -0
- {deltafi-2.0rc1705080991758.dist-info → deltafi-2.4.0.dist-info}/WHEEL +1 -1
- deltafi-2.0rc1705080991758.dist-info/RECORD +0 -22
deltafi/test_kit/framework.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
#
|
|
2
2
|
# DeltaFi - Data transformation and enrichment platform
|
|
3
3
|
#
|
|
4
|
-
# Copyright 2021-
|
|
4
|
+
# Copyright 2021-2025 DeltaFi Contributors <deltafi@deltafi.org>
|
|
5
5
|
#
|
|
6
6
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
7
7
|
# you may not use this file except in compliance with the License.
|
|
@@ -23,9 +23,9 @@ from typing import List
|
|
|
23
23
|
|
|
24
24
|
from deltafi.domain import DeltaFileMessage, Event, Content, Context
|
|
25
25
|
from deltafi.logger import get_logger
|
|
26
|
+
from deltafi.metric import Metric
|
|
26
27
|
from deltafi.result import ErrorResult, FilterResult
|
|
27
28
|
from deltafi.storage import Segment
|
|
28
|
-
|
|
29
29
|
from .assertions import *
|
|
30
30
|
from .compare_helpers import GenericCompareHelper, CompareHelper
|
|
31
31
|
from .constants import *
|
|
@@ -36,16 +36,20 @@ class IOContent:
|
|
|
36
36
|
The IOContent class holds the details for loading input or output
|
|
37
37
|
content into the test framework.
|
|
38
38
|
Attributes:
|
|
39
|
-
file_name (str): The name of file in test/data.
|
|
40
|
-
content_name (str): The name of the content.
|
|
41
|
-
content_type (str): The media type of the content
|
|
42
|
-
offset (int): Offset to use in Segment
|
|
43
|
-
content_bytes (str):
|
|
39
|
+
file_name (str) : The name of file in test/data.
|
|
40
|
+
content_name (str) : The name of the content.
|
|
41
|
+
content_type (str) : The media type of the content
|
|
42
|
+
offset (int) : Offset to use in Segment
|
|
43
|
+
content_bytes (str): Optional. If set to a String of length greater than zero, indicates to consumers of this
|
|
44
|
+
IOContent that they should bypass file read and use these bytes for content.
|
|
45
|
+
no_content (bool) : Optional. If 'True', then consumers should not attempt to interpret content but should
|
|
46
|
+
apply other aspects of this IOContent. When 'True', 'content_bytes' should be ignored and
|
|
47
|
+
loaded content, if any, should be interpreted as empty String or otherwise as documented by
|
|
48
|
+
the consumer.
|
|
44
49
|
"""
|
|
45
50
|
|
|
46
|
-
def __init__(self, file_name: str, content_name: str = None,
|
|
47
|
-
|
|
48
|
-
content_bytes: str = ""):
|
|
51
|
+
def __init__(self, file_name: str, content_name: str = None, content_type: str = None, offset: int = 0,
|
|
52
|
+
content_bytes: str = "", no_content: bool = False):
|
|
49
53
|
self.file_name = file_name
|
|
50
54
|
if content_name is None:
|
|
51
55
|
self.content_name = file_name
|
|
@@ -56,7 +60,12 @@ class IOContent:
|
|
|
56
60
|
else:
|
|
57
61
|
self.content_type = content_type
|
|
58
62
|
self.offset = offset
|
|
59
|
-
self.
|
|
63
|
+
self.no_content = no_content
|
|
64
|
+
if no_content:
|
|
65
|
+
self.content_bytes = None
|
|
66
|
+
else:
|
|
67
|
+
self.content_bytes = content_bytes
|
|
68
|
+
self.segment_uuid = uuid.uuid4()
|
|
60
69
|
|
|
61
70
|
@classmethod
|
|
62
71
|
def file_type(cls, name: str):
|
|
@@ -78,13 +87,12 @@ class LoadedContent:
|
|
|
78
87
|
if data is not None:
|
|
79
88
|
self.data = data
|
|
80
89
|
else:
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
"did": did
|
|
87
|
-
})
|
|
90
|
+
if ioc.no_content:
|
|
91
|
+
self.data = ""
|
|
92
|
+
else:
|
|
93
|
+
self.data = ioc.content_bytes
|
|
94
|
+
self.segment = Segment.from_dict(
|
|
95
|
+
{"uuid": str(ioc.segment_uuid), "offset": self.offset, "size": len(self.data), "did": did})
|
|
88
96
|
|
|
89
97
|
|
|
90
98
|
class InternalContentService:
|
|
@@ -97,10 +105,7 @@ class InternalContentService:
|
|
|
97
105
|
self.loaded_content[c.segment.uuid] = c
|
|
98
106
|
|
|
99
107
|
def put_str(self, did: str, string_data: str):
|
|
100
|
-
segment = Segment(uuid=str(uuid.uuid4()),
|
|
101
|
-
offset=0,
|
|
102
|
-
size=len(string_data),
|
|
103
|
-
did=did)
|
|
108
|
+
segment = Segment(uuid=str(uuid.uuid4()), offset=0, size=len(string_data), did=did)
|
|
104
109
|
self.outputs[segment.uuid] = string_data
|
|
105
110
|
return segment
|
|
106
111
|
|
|
@@ -109,8 +114,17 @@ class InternalContentService:
|
|
|
109
114
|
seg_id = segments[0].uuid
|
|
110
115
|
return self.loaded_content[seg_id].data
|
|
111
116
|
|
|
117
|
+
def get_bytes(self, segments: List[Segment]):
|
|
118
|
+
seg_id = segments[0].uuid
|
|
119
|
+
return self.loaded_content[seg_id].data.encode('utf-8')
|
|
120
|
+
|
|
112
121
|
def get_output(self, seg_id: str):
|
|
113
|
-
|
|
122
|
+
if seg_id in self.outputs:
|
|
123
|
+
return self.outputs[seg_id]
|
|
124
|
+
elif seg_id in self.loaded_content:
|
|
125
|
+
return self.loaded_content[seg_id].data
|
|
126
|
+
else:
|
|
127
|
+
return None
|
|
114
128
|
|
|
115
129
|
|
|
116
130
|
class TestCaseBase(ABC):
|
|
@@ -123,9 +137,11 @@ class TestCaseBase(ABC):
|
|
|
123
137
|
- compare_tool: (optional) CompareHelper instanced for comparing output content
|
|
124
138
|
- inputs: (optional) List[IOContent]: input content to action
|
|
125
139
|
- parameters: (optional) Dict: map of action input parameters
|
|
140
|
+
- in_memo: (optional) str: Input 'memo' value for a TimedIngress context
|
|
126
141
|
- in_meta: (optional) Dict: map of metadata as input to action
|
|
127
|
-
-
|
|
128
|
-
|
|
142
|
+
- join_meta: (optional): List[Dict]: When a List is provided, this enables the JOIN portion of an action.
|
|
143
|
+
When using JOIN, join_meta must match the size of inputs, though the Dict can be empty
|
|
144
|
+
- did: (optional): str: overrides random DID
|
|
129
145
|
"""
|
|
130
146
|
if "action" in data:
|
|
131
147
|
self.action = data["action"]
|
|
@@ -144,34 +160,43 @@ class TestCaseBase(ABC):
|
|
|
144
160
|
|
|
145
161
|
self.inputs = data["inputs"] if "inputs" in data else []
|
|
146
162
|
self.file_name = data["file_name"] if "file_name" in data else "filename"
|
|
147
|
-
self.outputs = data["outputs"] if "outputs" in data else []
|
|
148
163
|
self.parameters = data["parameters"] if "parameters" in data else {}
|
|
149
164
|
self.in_meta = data["in_meta"] if "in_meta" in data else {}
|
|
150
|
-
self.
|
|
151
|
-
self.
|
|
165
|
+
self.in_memo = data["in_memo"] if "in_memo" in data else None
|
|
166
|
+
self.use_did = data["did"] if "did" in data else None
|
|
152
167
|
self.expected_result_type = None
|
|
153
|
-
self.
|
|
154
|
-
self.
|
|
168
|
+
self.err_or_filt_cause = None
|
|
169
|
+
self.err_or_filt_context = None
|
|
170
|
+
self.err_or_filt_annotations = None
|
|
171
|
+
self.join_meta = data["join_meta"] if "join_meta" in data else None
|
|
172
|
+
self.expected_metrics = []
|
|
173
|
+
|
|
174
|
+
def add_metric(self, metric: Metric):
|
|
175
|
+
self.expected_metrics.append(metric)
|
|
155
176
|
|
|
156
|
-
def expect_error_result(self, cause: str, context: str):
|
|
177
|
+
def expect_error_result(self, cause: str, context: str, annotations: Dict = None):
|
|
157
178
|
"""
|
|
158
179
|
A Sets the expected output of the action to an Error Result
|
|
159
180
|
:param cause: the expected error cause
|
|
160
181
|
:param context: the expected error context
|
|
182
|
+
:param annotations: Dict: (Optional) the expected annotations
|
|
161
183
|
"""
|
|
162
184
|
self.expected_result_type = ErrorResult
|
|
163
|
-
self.
|
|
164
|
-
self.
|
|
185
|
+
self.err_or_filt_cause = cause
|
|
186
|
+
self.err_or_filt_context = context
|
|
187
|
+
self.err_or_filt_annotations = annotations
|
|
165
188
|
|
|
166
|
-
def expect_filter_result(self, cause: str, context: str=None):
|
|
189
|
+
def expect_filter_result(self, cause: str, context: str = None, annotations: Dict = None):
|
|
167
190
|
"""
|
|
168
191
|
A Sets the expected output of the action to a Filter Result
|
|
169
192
|
:param cause: the expected filter cause (message)
|
|
170
|
-
:param context: the expected
|
|
193
|
+
:param context: (Optional) the expected filter context
|
|
194
|
+
:param annotations: Dict: (Optional) the expected annotations
|
|
171
195
|
"""
|
|
172
196
|
self.expected_result_type = FilterResult
|
|
173
|
-
self.
|
|
174
|
-
self.
|
|
197
|
+
self.err_or_filt_cause = cause
|
|
198
|
+
self.err_or_filt_context = context
|
|
199
|
+
self.err_or_filt_annotations = annotations
|
|
175
200
|
|
|
176
201
|
|
|
177
202
|
class ActionTest(ABC):
|
|
@@ -183,17 +208,20 @@ class ActionTest(ABC):
|
|
|
183
208
|
"""
|
|
184
209
|
self.content_service = InternalContentService()
|
|
185
210
|
self.did = ""
|
|
186
|
-
self.expected_outputs = []
|
|
187
211
|
self.loaded_inputs = []
|
|
188
212
|
self.package_name = package_name
|
|
189
213
|
self.res_path = ""
|
|
214
|
+
self.context = None
|
|
190
215
|
|
|
191
|
-
def __reset__(self):
|
|
216
|
+
def __reset__(self, did: str):
|
|
192
217
|
self.content_service = InternalContentService()
|
|
193
|
-
|
|
194
|
-
|
|
218
|
+
if did is None:
|
|
219
|
+
self.did = str(uuid.uuid4())
|
|
220
|
+
else:
|
|
221
|
+
self.did = did
|
|
195
222
|
self.loaded_inputs = []
|
|
196
223
|
self.res_path = ""
|
|
224
|
+
self.context = None
|
|
197
225
|
|
|
198
226
|
def load_file(self, ioc: IOContent):
|
|
199
227
|
file_res = self.res_path.joinpath(ioc.file_name)
|
|
@@ -207,68 +235,67 @@ class ActionTest(ABC):
|
|
|
207
235
|
|
|
208
236
|
# Load inputs
|
|
209
237
|
for input_ioc in test_case.inputs:
|
|
210
|
-
if len(input_ioc.content_bytes) == 0:
|
|
238
|
+
if not input_ioc.no_content and len(input_ioc.content_bytes) == 0:
|
|
211
239
|
self.loaded_inputs.append(LoadedContent(self.did, input_ioc, self.load_file(input_ioc)))
|
|
212
240
|
else:
|
|
213
241
|
self.loaded_inputs.append(LoadedContent(self.did, input_ioc, None))
|
|
214
242
|
|
|
215
|
-
|
|
216
|
-
for output_ioc in test_case.outputs:
|
|
217
|
-
if len(output_ioc.content_bytes) == 0:
|
|
218
|
-
self.expected_outputs.append(LoadedContent(self.did, output_ioc, self.load_file(output_ioc)))
|
|
219
|
-
else:
|
|
220
|
-
self.expected_outputs.append(LoadedContent(self.did, output_ioc, None))
|
|
221
|
-
|
|
222
|
-
def make_content_list(self, test_case: TestCaseBase):
|
|
243
|
+
def make_content_list(self):
|
|
223
244
|
content_list = []
|
|
224
245
|
for loaded_input in self.loaded_inputs:
|
|
225
|
-
c = Content(name=loaded_input.name,
|
|
226
|
-
segments=[loaded_input.segment],
|
|
227
|
-
media_type=loaded_input.content_type,
|
|
246
|
+
c = Content(name=loaded_input.name, segments=[loaded_input.segment], media_type=loaded_input.content_type,
|
|
228
247
|
content_service=self.content_service)
|
|
229
248
|
content_list.append(c)
|
|
230
249
|
loaded_input.content = c
|
|
231
250
|
|
|
232
251
|
return content_list
|
|
233
252
|
|
|
234
|
-
def
|
|
235
|
-
content_list = self.make_content_list(
|
|
253
|
+
def make_df_msgs(self, test_case: TestCaseBase):
|
|
254
|
+
content_list = self.make_content_list()
|
|
236
255
|
self.content_service.load(self.loaded_inputs)
|
|
237
256
|
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
257
|
+
delta_file_messages = []
|
|
258
|
+
|
|
259
|
+
if test_case.join_meta is None:
|
|
260
|
+
delta_file_messages.append(DeltaFileMessage(metadata=test_case.in_meta, content_list=content_list))
|
|
261
|
+
else:
|
|
262
|
+
for index, content in enumerate(content_list):
|
|
263
|
+
delta_file_messages.append(DeltaFileMessage(
|
|
264
|
+
metadata=test_case.join_meta[index],
|
|
265
|
+
content_list=[content]))
|
|
266
|
+
|
|
267
|
+
return delta_file_messages
|
|
242
268
|
|
|
243
269
|
def make_context(self, test_case: TestCaseBase):
|
|
244
270
|
action_name = INGRESS_FLOW + "." + test_case.action.__class__.__name__
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
271
|
+
join = {} if test_case.join_meta else None
|
|
272
|
+
self.context = Context(
|
|
273
|
+
did=self.did,
|
|
274
|
+
delta_file_name=test_case.file_name,
|
|
275
|
+
data_source="DATASRC",
|
|
276
|
+
flow_name=INGRESS_FLOW,
|
|
277
|
+
flow_id="FLOWID",
|
|
278
|
+
action_name=action_name,
|
|
279
|
+
action_version="1.0",
|
|
280
|
+
hostname=HOSTNAME,
|
|
281
|
+
system_name=SYSTEM,
|
|
282
|
+
content_service=self.content_service,
|
|
283
|
+
saved_content=[],
|
|
284
|
+
join=join,
|
|
285
|
+
memo=test_case.in_memo,
|
|
286
|
+
logger=get_logger())
|
|
287
|
+
return self.context
|
|
257
288
|
|
|
258
289
|
def make_event(self, test_case: TestCaseBase):
|
|
259
|
-
return Event(
|
|
260
|
-
|
|
261
|
-
context=self.make_context(test_case),
|
|
262
|
-
params=test_case.parameters,
|
|
263
|
-
queue_name="",
|
|
264
|
-
return_address="")
|
|
290
|
+
return Event(delta_file_messages=self.make_df_msgs(test_case), context=self.make_context(test_case),
|
|
291
|
+
params=test_case.parameters, queue_name="", return_address="")
|
|
265
292
|
|
|
266
293
|
def call_action(self, test_case: TestCaseBase):
|
|
267
294
|
self.get_contents(test_case)
|
|
268
295
|
return test_case.action.execute_action(self.make_event(test_case))
|
|
269
296
|
|
|
270
297
|
def run_and_check_result_type(self, test_case: TestCaseBase, result_type):
|
|
271
|
-
self.__reset__()
|
|
298
|
+
self.__reset__(test_case.use_did)
|
|
272
299
|
result = self.call_action(test_case)
|
|
273
300
|
|
|
274
301
|
if not isinstance(result, result_type):
|
|
@@ -279,53 +306,60 @@ class ActionTest(ABC):
|
|
|
279
306
|
def execute_error(self, test_case: TestCaseBase):
|
|
280
307
|
result = self.run_and_check_result_type(test_case, ErrorResult)
|
|
281
308
|
resp = result.response()
|
|
282
|
-
|
|
283
|
-
|
|
309
|
+
assert_equal_with_label(test_case.err_or_filt_cause, resp['cause'], "error cause")
|
|
310
|
+
if test_case.err_or_filt_context is not None:
|
|
311
|
+
assert_equal_with_label(test_case.err_or_filt_context, resp['context'], "error context")
|
|
312
|
+
if test_case.err_or_filt_annotations is not None:
|
|
313
|
+
assert_keys_and_values(test_case.err_or_filt_annotations, result.annotations)
|
|
284
314
|
|
|
285
|
-
def execute_filter(self, test_case):
|
|
315
|
+
def execute_filter(self, test_case: TestCaseBase):
|
|
286
316
|
result = self.run_and_check_result_type(test_case, FilterResult)
|
|
287
317
|
resp = result.response()
|
|
288
|
-
|
|
318
|
+
assert_equal_with_label(test_case.err_or_filt_cause, resp['message'], "filter cause")
|
|
319
|
+
if test_case.err_or_filt_context is not None:
|
|
320
|
+
assert_equal_with_label(test_case.err_or_filt_context, resp['context'], "filter context")
|
|
321
|
+
if test_case.err_or_filt_annotations is not None:
|
|
322
|
+
assert_keys_and_values(test_case.err_or_filt_annotations, result.annotations)
|
|
289
323
|
|
|
290
324
|
def execute(self, test_case: TestCaseBase):
|
|
291
|
-
if
|
|
325
|
+
if test_case.expected_result_type == ErrorResult:
|
|
292
326
|
self.execute_error(test_case)
|
|
293
|
-
elif
|
|
327
|
+
elif test_case.expected_result_type == FilterResult:
|
|
294
328
|
self.execute_filter(test_case)
|
|
295
329
|
else:
|
|
296
330
|
raise ValueError(f"unknown type: {test_case.expected_result_type}")
|
|
297
331
|
|
|
298
|
-
|
|
332
|
+
@staticmethod
|
|
333
|
+
def compare_content_details(expected: LoadedContent, actual: Content):
|
|
299
334
|
assert_equal(expected.content_type, actual.media_type)
|
|
300
335
|
assert_equal(expected.name, actual.name)
|
|
301
336
|
|
|
302
|
-
def compare_one_content(self,
|
|
337
|
+
def compare_one_content(self, comparator: CompareHelper, expected: LoadedContent, actual, index):
|
|
303
338
|
self.compare_content_details(expected, actual)
|
|
304
339
|
seg_id = actual.segments[0].uuid
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
)
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
assert_equal_len(self.expected_outputs, content)
|
|
313
|
-
for index, expected in enumerate(self.expected_outputs):
|
|
314
|
-
self.compare_one_content(comparitor, expected, content[index], index)
|
|
315
|
-
|
|
316
|
-
def compare_domains(self, comparitor: CompareHelper, expected_items: List[Dict], results: List[Dict]):
|
|
317
|
-
assert_equal_len(expected_items, results)
|
|
318
|
-
for index, expected in enumerate(expected_items):
|
|
319
|
-
actual = results[index]
|
|
320
|
-
assert_equal(expected['name'], actual['name'])
|
|
321
|
-
assert_equal(expected['mediaType'], actual['mediaType'])
|
|
322
|
-
|
|
323
|
-
expected_value = expected['value']
|
|
324
|
-
if type(expected_value) == str:
|
|
325
|
-
comparitor.compare(expected_value, actual['value'], f"Domain[{index}]")
|
|
326
|
-
elif type(expected_value) == IOContent:
|
|
327
|
-
expected_data = self.load_file(expected_value)
|
|
328
|
-
comparitor.compare(expected_data, actual['value'], f"Domain[{index}]")
|
|
340
|
+
comparator.compare(expected.data, self.content_service.get_output(seg_id), f"Content[{index}]")
|
|
341
|
+
|
|
342
|
+
def compare_content_list(self, comparator: CompareHelper, expected_outputs: List[IOContent], content: List):
|
|
343
|
+
assert_equal_len(expected_outputs, content)
|
|
344
|
+
for index, expected_ioc in enumerate(expected_outputs):
|
|
345
|
+
if not expected_ioc.no_content and len(expected_ioc.content_bytes) == 0:
|
|
346
|
+
expected = LoadedContent(self.did, expected_ioc, self.load_file(expected_ioc))
|
|
329
347
|
else:
|
|
330
|
-
|
|
331
|
-
|
|
348
|
+
expected = LoadedContent(self.did, expected_ioc, None)
|
|
349
|
+
self.compare_one_content(comparator, expected, content[index], index)
|
|
350
|
+
|
|
351
|
+
@staticmethod
|
|
352
|
+
def compare_one_metric(expected: Metric, result: Metric):
|
|
353
|
+
assert_equal_short(expected.name, result.name, "invalid metric name")
|
|
354
|
+
assert_equal_short(expected.value, result.value, f"invalid metric value for {expected.name}")
|
|
355
|
+
assert_keys_and_values(expected.tags, result.tags)
|
|
356
|
+
|
|
357
|
+
def compare_metrics(self, expected_metrics: List[Metric], results: List[Metric]):
|
|
358
|
+
if len(expected_metrics) > 0:
|
|
359
|
+
assert_equal_len_with_label(expected_metrics, results, "invalid metrics count")
|
|
360
|
+
for index, expected in enumerate(expected_metrics):
|
|
361
|
+
self.compare_one_metric(expected, results[index])
|
|
362
|
+
|
|
363
|
+
def has_saved_content__size(self, count: int):
|
|
364
|
+
assert_equal_with_label(
|
|
365
|
+
count, len(self.context.saved_content), "savedContent")
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
#
|
|
2
|
+
# DeltaFi - Data transformation and enrichment platform
|
|
3
|
+
#
|
|
4
|
+
# Copyright 2021-2025 DeltaFi Contributors <deltafi@deltafi.org>
|
|
5
|
+
#
|
|
6
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
7
|
+
# you may not use this file except in compliance with the License.
|
|
8
|
+
# You may obtain a copy of the License at
|
|
9
|
+
#
|
|
10
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
11
|
+
#
|
|
12
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
13
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
14
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
15
|
+
# See the License for the specific language governing permissions and
|
|
16
|
+
# limitations under the License.
|
|
17
|
+
#
|
|
18
|
+
|
|
19
|
+
from typing import List
|
|
20
|
+
|
|
21
|
+
from deltafi.result import IngressResult, IngressResultItem, IngressStatusEnum
|
|
22
|
+
from .assertions import *
|
|
23
|
+
from .framework import TestCaseBase, ActionTest, IOContent
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class TimedIngressTestCase(TestCaseBase):
|
|
27
|
+
def __init__(self, fields: Dict):
|
|
28
|
+
super().__init__(fields)
|
|
29
|
+
self.memo = None
|
|
30
|
+
self.results = []
|
|
31
|
+
self.execute_immediate = False
|
|
32
|
+
self.status = IngressStatusEnum.HEALTHY
|
|
33
|
+
self.status_message = None
|
|
34
|
+
|
|
35
|
+
def expect_ingress_result(self,
|
|
36
|
+
memo: str = None,
|
|
37
|
+
exec_immed: bool = False,
|
|
38
|
+
status: IngressStatusEnum = IngressStatusEnum.HEALTHY,
|
|
39
|
+
status_message: str = None):
|
|
40
|
+
self.expected_result_type = IngressResult
|
|
41
|
+
self.memo = memo
|
|
42
|
+
self.execute_immediate = exec_immed
|
|
43
|
+
self.status = status
|
|
44
|
+
self.status_message = status_message
|
|
45
|
+
|
|
46
|
+
def add_ingress_result_item(self, content: List[IOContent], metadata: Dict, name: str = None, annotations: Dict = None):
|
|
47
|
+
if annotations is None:
|
|
48
|
+
annotations = {}
|
|
49
|
+
self.results.append(
|
|
50
|
+
{
|
|
51
|
+
'content': content,
|
|
52
|
+
'metadata': metadata,
|
|
53
|
+
'name': name,
|
|
54
|
+
'annotations': annotations
|
|
55
|
+
}
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class TimedIngressActionTest(ActionTest):
|
|
60
|
+
def __init__(self, package_name: str):
|
|
61
|
+
"""
|
|
62
|
+
Provides structure for testing DeltaFi TimedIngress action
|
|
63
|
+
Args:
|
|
64
|
+
package_name: name of the actions package for finding resources
|
|
65
|
+
"""
|
|
66
|
+
super().__init__(package_name)
|
|
67
|
+
|
|
68
|
+
def ingress(self, test_case: TimedIngressTestCase):
|
|
69
|
+
if test_case.expected_result_type == IngressResult:
|
|
70
|
+
self.expect_ingress_result(test_case)
|
|
71
|
+
else:
|
|
72
|
+
super().execute(test_case)
|
|
73
|
+
|
|
74
|
+
def expect_ingress_result(self, test_case: TimedIngressTestCase):
|
|
75
|
+
result = super().run_and_check_result_type(test_case, IngressResult)
|
|
76
|
+
self.assert_ingress_result(test_case, result)
|
|
77
|
+
|
|
78
|
+
def assert_ingress_result(self, test_case: TimedIngressTestCase, result: IngressResult):
|
|
79
|
+
assert_equal_short(test_case.memo, result.memo, "invalid memo")
|
|
80
|
+
assert_equal_short(test_case.execute_immediate, result.execute_immediate, "invalid execute_immediate")
|
|
81
|
+
assert_equal_short(test_case.status, result.status, "invalid status")
|
|
82
|
+
assert_equal_with_label(test_case.status_message, result.status_message, "invalid status_message")
|
|
83
|
+
|
|
84
|
+
assert_equal_len_with_label(test_case.results, result.ingress_result_items, "item count mismatch")
|
|
85
|
+
for index, ingress_item in enumerate(result.ingress_result_items):
|
|
86
|
+
self.compare_one_ingress_item(test_case, ingress_item, index)
|
|
87
|
+
expected = test_case.results[index]
|
|
88
|
+
if 'name' in expected:
|
|
89
|
+
assert_equal_with_label(expected["name"], ingress_item.delta_file_name, f"name[{index}]")
|
|
90
|
+
|
|
91
|
+
def compare_one_ingress_item(self, test_case: TimedIngressTestCase, result: IngressResultItem, index: int):
|
|
92
|
+
expected = test_case.results[index]
|
|
93
|
+
|
|
94
|
+
# Check output
|
|
95
|
+
self.compare_content_list(test_case.compare_tool, expected['content'], result.content)
|
|
96
|
+
|
|
97
|
+
# Check metadata
|
|
98
|
+
assert_keys_and_values(expected['metadata'], result.metadata)
|
|
99
|
+
|
|
100
|
+
# Check annotations
|
|
101
|
+
assert_keys_and_values(expected['annotations'], result.annotations)
|
deltafi/test_kit/transform.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
#
|
|
2
2
|
# DeltaFi - Data transformation and enrichment platform
|
|
3
3
|
#
|
|
4
|
-
# Copyright 2021-
|
|
4
|
+
# Copyright 2021-2025 DeltaFi Contributors <deltafi@deltafi.org>
|
|
5
5
|
#
|
|
6
6
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
7
7
|
# you may not use this file except in compliance with the License.
|
|
@@ -18,24 +18,33 @@
|
|
|
18
18
|
|
|
19
19
|
from typing import List
|
|
20
20
|
|
|
21
|
-
from deltafi.result import TransformResult
|
|
22
|
-
|
|
21
|
+
from deltafi.result import TransformResult, TransformResults
|
|
23
22
|
from .assertions import *
|
|
24
|
-
from .framework import TestCaseBase, ActionTest
|
|
23
|
+
from .framework import TestCaseBase, ActionTest, IOContent
|
|
25
24
|
|
|
26
25
|
|
|
27
26
|
class TransformTestCase(TestCaseBase):
|
|
28
27
|
def __init__(self, fields: Dict):
|
|
29
28
|
super().__init__(fields)
|
|
30
|
-
self.
|
|
31
|
-
self.delete_metadata_keys = []
|
|
32
|
-
self.annotations = {}
|
|
29
|
+
self.results = []
|
|
33
30
|
|
|
34
|
-
def expect_transform_result(self
|
|
31
|
+
def expect_transform_result(self):
|
|
35
32
|
self.expected_result_type = TransformResult
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
self.
|
|
33
|
+
|
|
34
|
+
def expect_transform_results(self):
|
|
35
|
+
self.expected_result_type = TransformResults
|
|
36
|
+
|
|
37
|
+
def add_transform_result(self, content: List[IOContent], metadata: Dict, delete_metadata_keys: List[str],
|
|
38
|
+
annotations: Dict, name: str = None):
|
|
39
|
+
self.results.append(
|
|
40
|
+
{
|
|
41
|
+
'content': content,
|
|
42
|
+
'metadata': metadata,
|
|
43
|
+
'delete_metadata_keys': delete_metadata_keys,
|
|
44
|
+
'annotations': annotations,
|
|
45
|
+
'name': name
|
|
46
|
+
}
|
|
47
|
+
)
|
|
39
48
|
|
|
40
49
|
|
|
41
50
|
class TransformActionTest(ActionTest):
|
|
@@ -50,6 +59,8 @@ class TransformActionTest(ActionTest):
|
|
|
50
59
|
def transform(self, test_case: TransformTestCase):
|
|
51
60
|
if test_case.expected_result_type == TransformResult:
|
|
52
61
|
self.expect_transform_result(test_case)
|
|
62
|
+
elif test_case.expected_result_type == TransformResults:
|
|
63
|
+
self.expect_transform_results(test_case)
|
|
53
64
|
else:
|
|
54
65
|
super().execute(test_case)
|
|
55
66
|
|
|
@@ -57,16 +68,35 @@ class TransformActionTest(ActionTest):
|
|
|
57
68
|
result = super().run_and_check_result_type(test_case, TransformResult)
|
|
58
69
|
self.assert_transform_result(test_case, result)
|
|
59
70
|
|
|
71
|
+
def expect_transform_results(self, test_case: TransformTestCase):
|
|
72
|
+
result = super().run_and_check_result_type(test_case, TransformResults)
|
|
73
|
+
self.assert_transform_results(test_case, result)
|
|
74
|
+
|
|
75
|
+
def assert_transform_results(self, test_case: TransformTestCase, result: TransformResults):
|
|
76
|
+
assert_equal_len_with_label(test_case.results, result.child_results, "invalid child count")
|
|
77
|
+
for index, child_result in enumerate(result.child_results):
|
|
78
|
+
self.compare_one_transform_result(test_case, child_result, index)
|
|
79
|
+
expected = test_case.results[index]
|
|
80
|
+
if 'name' in expected:
|
|
81
|
+
assert_equal_with_label(expected["name"], child_result.delta_file_name, f"name[{index}]")
|
|
82
|
+
|
|
60
83
|
def assert_transform_result(self, test_case: TransformTestCase, result: TransformResult):
|
|
84
|
+
# Check metrics
|
|
85
|
+
self.compare_metrics(test_case.expected_metrics, result.metrics)
|
|
86
|
+
self.compare_one_transform_result(test_case, result, 0)
|
|
87
|
+
|
|
88
|
+
def compare_one_transform_result(self, test_case: TransformTestCase, result: TransformResult, index: int):
|
|
89
|
+
expected = test_case.results[index]
|
|
90
|
+
|
|
61
91
|
# Check output
|
|
62
|
-
self.
|
|
92
|
+
self.compare_content_list(test_case.compare_tool, expected['content'], result.content)
|
|
63
93
|
|
|
64
94
|
# Check metadata
|
|
65
|
-
assert_keys_and_values(
|
|
95
|
+
assert_keys_and_values(expected['metadata'], result.metadata)
|
|
66
96
|
|
|
67
97
|
# Check deleted metadata
|
|
68
|
-
for key in
|
|
98
|
+
for key in expected['delete_metadata_keys']:
|
|
69
99
|
assert_key_in(key, result.delete_metadata_keys)
|
|
70
100
|
|
|
71
101
|
# Check annotations
|
|
72
|
-
assert_keys_and_values(
|
|
102
|
+
assert_keys_and_values(expected['annotations'], result.annotations)
|
|
@@ -1,31 +1,32 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
2
|
Name: deltafi
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.4.0
|
|
4
4
|
Summary: SDK for DeltaFi plugins and actions
|
|
5
5
|
License: Apache License, Version 2.0
|
|
6
6
|
Keywords: deltafi
|
|
7
7
|
Author: DeltaFi
|
|
8
8
|
Author-email: deltafi@systolic.com
|
|
9
|
-
Requires-Python: >=3.
|
|
9
|
+
Requires-Python: >=3.9,<4.0
|
|
10
10
|
Classifier: Development Status :: 4 - Beta
|
|
11
11
|
Classifier: Intended Audience :: Developers
|
|
12
12
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
13
13
|
Classifier: License :: Other/Proprietary License
|
|
14
14
|
Classifier: Operating System :: OS Independent
|
|
15
15
|
Classifier: Programming Language :: Python :: 3
|
|
16
|
-
Classifier: Programming Language :: Python :: 3.7
|
|
17
|
-
Classifier: Programming Language :: Python :: 3.8
|
|
18
16
|
Classifier: Programming Language :: Python :: 3.9
|
|
19
17
|
Classifier: Programming Language :: Python :: 3.10
|
|
20
18
|
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
21
21
|
Classifier: Topic :: Software Development
|
|
22
|
-
Requires-Dist:
|
|
23
|
-
Requires-Dist:
|
|
24
|
-
Requires-Dist:
|
|
25
|
-
Requires-Dist:
|
|
26
|
-
Requires-Dist:
|
|
27
|
-
Requires-Dist:
|
|
28
|
-
Requires-Dist:
|
|
22
|
+
Requires-Dist: PyYAML (==6.0.2)
|
|
23
|
+
Requires-Dist: deepdiff (==6.7.1)
|
|
24
|
+
Requires-Dist: json-logging (==1.3.0)
|
|
25
|
+
Requires-Dist: minio (==7.2.10)
|
|
26
|
+
Requires-Dist: pydantic (==2.9.2)
|
|
27
|
+
Requires-Dist: redis (==5.2.0)
|
|
28
|
+
Requires-Dist: requests (==2.32.3)
|
|
29
|
+
Requires-Dist: urllib3 (==2.2.3)
|
|
29
30
|
Project-URL: Bug Reports, https://chat.deltafi.org/deltafi/channels/bug-reports
|
|
30
31
|
Project-URL: Documentation, https://docs.deltafi.org/#/
|
|
31
32
|
Project-URL: Source Code, https://gitlab.com/deltafi/deltafi
|