deltafi 1.2.19__py3-none-any.whl → 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of deltafi might be problematic. Click here for more details.
- deltafi/__init__.py +1 -1
- deltafi/action.py +40 -130
- deltafi/actioneventqueue.py +1 -1
- deltafi/actiontype.py +3 -7
- deltafi/domain.py +73 -63
- deltafi/exception.py +1 -11
- deltafi/genericmodel.py +4 -2
- deltafi/input.py +2 -166
- deltafi/logger.py +4 -4
- deltafi/metric.py +2 -2
- deltafi/plugin.py +112 -43
- deltafi/result.py +74 -261
- deltafi/storage.py +6 -1
- deltafi/test_kit/__init__.py +1 -1
- deltafi/test_kit/assertions.py +1 -1
- deltafi/test_kit/compare_helpers.py +1 -1
- deltafi/test_kit/constants.py +1 -1
- deltafi/test_kit/egress.py +1 -1
- deltafi/test_kit/framework.py +78 -69
- deltafi/test_kit/transform.py +42 -14
- {deltafi-1.2.19.dist-info → deltafi-2.0.0.dist-info}/METADATA +8 -6
- deltafi-2.0.0.dist-info/RECORD +23 -0
- {deltafi-1.2.19.dist-info → deltafi-2.0.0.dist-info}/WHEEL +1 -1
- deltafi/test_kit/domain.py +0 -59
- deltafi/test_kit/enrich.py +0 -70
- deltafi/test_kit/format.py +0 -105
- deltafi/test_kit/load.py +0 -128
- deltafi/test_kit/validate.py +0 -54
- deltafi-1.2.19.dist-info/RECORD +0 -28
deltafi/test_kit/framework.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
#
|
|
2
2
|
# DeltaFi - Data transformation and enrichment platform
|
|
3
3
|
#
|
|
4
|
-
# Copyright 2021-
|
|
4
|
+
# Copyright 2021-2024 DeltaFi Contributors <deltafi@deltafi.org>
|
|
5
5
|
#
|
|
6
6
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
7
7
|
# you may not use this file except in compliance with the License.
|
|
@@ -26,7 +26,6 @@ from deltafi.logger import get_logger
|
|
|
26
26
|
from deltafi.metric import Metric
|
|
27
27
|
from deltafi.result import ErrorResult, FilterResult
|
|
28
28
|
from deltafi.storage import Segment
|
|
29
|
-
|
|
30
29
|
from .assertions import *
|
|
31
30
|
from .compare_helpers import GenericCompareHelper, CompareHelper
|
|
32
31
|
from .constants import *
|
|
@@ -37,15 +36,20 @@ class IOContent:
|
|
|
37
36
|
The IOContent class holds the details for loading input or output
|
|
38
37
|
content into the test framework.
|
|
39
38
|
Attributes:
|
|
40
|
-
file_name (str): The name of file in test/data.
|
|
41
|
-
content_name (str): The name of the content.
|
|
42
|
-
content_type (str): The media type of the content
|
|
43
|
-
offset (int): Offset to use in Segment
|
|
44
|
-
content_bytes (str):
|
|
39
|
+
file_name (str) : The name of file in test/data.
|
|
40
|
+
content_name (str) : The name of the content.
|
|
41
|
+
content_type (str) : The media type of the content
|
|
42
|
+
offset (int) : Offset to use in Segment
|
|
43
|
+
content_bytes (str): Optional. If set to a String of length greater than zero, indicates to consumers of this
|
|
44
|
+
IOContent that they should bypass file read and use these bytes for content.
|
|
45
|
+
no_content (bool) : Optional. If 'True', then consumers should not attempt to interpret content but should
|
|
46
|
+
apply other aspects of this IOContent. When 'True', 'content_bytes' should be ignored and
|
|
47
|
+
loaded content, if any, should be interpreted as empty String or otherwise as documented by
|
|
48
|
+
the consumer.
|
|
45
49
|
"""
|
|
46
50
|
|
|
47
51
|
def __init__(self, file_name: str, content_name: str = None, content_type: str = None, offset: int = 0,
|
|
48
|
-
content_bytes: str = ""):
|
|
52
|
+
content_bytes: str = "", no_content: bool = False):
|
|
49
53
|
self.file_name = file_name
|
|
50
54
|
if content_name is None:
|
|
51
55
|
self.content_name = file_name
|
|
@@ -56,7 +60,11 @@ class IOContent:
|
|
|
56
60
|
else:
|
|
57
61
|
self.content_type = content_type
|
|
58
62
|
self.offset = offset
|
|
59
|
-
self.
|
|
63
|
+
self.no_content = no_content
|
|
64
|
+
if no_content:
|
|
65
|
+
self.content_bytes = None
|
|
66
|
+
else:
|
|
67
|
+
self.content_bytes = content_bytes
|
|
60
68
|
self.segment_uuid = uuid.uuid4()
|
|
61
69
|
|
|
62
70
|
@classmethod
|
|
@@ -79,7 +87,10 @@ class LoadedContent:
|
|
|
79
87
|
if data is not None:
|
|
80
88
|
self.data = data
|
|
81
89
|
else:
|
|
82
|
-
|
|
90
|
+
if ioc.no_content:
|
|
91
|
+
self.data = ""
|
|
92
|
+
else:
|
|
93
|
+
self.data = ioc.content_bytes
|
|
83
94
|
self.segment = Segment.from_dict(
|
|
84
95
|
{"uuid": str(ioc.segment_uuid), "offset": self.offset, "size": len(self.data), "did": did})
|
|
85
96
|
|
|
@@ -103,6 +114,10 @@ class InternalContentService:
|
|
|
103
114
|
seg_id = segments[0].uuid
|
|
104
115
|
return self.loaded_content[seg_id].data
|
|
105
116
|
|
|
117
|
+
def get_bytes(self, segments: List[Segment]):
|
|
118
|
+
seg_id = segments[0].uuid
|
|
119
|
+
return self.loaded_content[seg_id].data.encode('utf-8')
|
|
120
|
+
|
|
106
121
|
def get_output(self, seg_id: str):
|
|
107
122
|
if seg_id in self.outputs:
|
|
108
123
|
return self.outputs[seg_id]
|
|
@@ -123,8 +138,8 @@ class TestCaseBase(ABC):
|
|
|
123
138
|
- inputs: (optional) List[IOContent]: input content to action
|
|
124
139
|
- parameters: (optional) Dict: map of action input parameters
|
|
125
140
|
- in_meta: (optional) Dict: map of metadata as input to action
|
|
126
|
-
-
|
|
127
|
-
|
|
141
|
+
- join_meta: (optional): List[Dict]: When a List is provided, this enables the JOIN portion of an action.
|
|
142
|
+
When using JOIN, join_meta must match the size of inputs, though the Dict can be empty
|
|
128
143
|
- did: (optional): str: overrides random DID
|
|
129
144
|
"""
|
|
130
145
|
if "action" in data:
|
|
@@ -144,16 +159,14 @@ class TestCaseBase(ABC):
|
|
|
144
159
|
|
|
145
160
|
self.inputs = data["inputs"] if "inputs" in data else []
|
|
146
161
|
self.file_name = data["file_name"] if "file_name" in data else "filename"
|
|
147
|
-
self.outputs = data["outputs"] if "outputs" in data else []
|
|
148
162
|
self.parameters = data["parameters"] if "parameters" in data else {}
|
|
149
163
|
self.in_meta = data["in_meta"] if "in_meta" in data else {}
|
|
150
|
-
self.in_domains = data["in_domains"] if "in_domains" in data else []
|
|
151
|
-
self.in_enrichments = data["in_enrichments"] if "in_enrichments" in data else []
|
|
152
164
|
self.use_did = data["did"] if "did" in data else None
|
|
153
165
|
self.expected_result_type = None
|
|
154
166
|
self.err_or_filt_cause = None
|
|
155
167
|
self.err_or_filt_context = None
|
|
156
168
|
self.err_or_filt_annotations = None
|
|
169
|
+
self.join_meta = data["join_meta"] if "join_meta" in data else None
|
|
157
170
|
self.expected_metrics = []
|
|
158
171
|
|
|
159
172
|
def add_metric(self, metric: Metric):
|
|
@@ -164,7 +177,7 @@ class TestCaseBase(ABC):
|
|
|
164
177
|
A Sets the expected output of the action to an Error Result
|
|
165
178
|
:param cause: the expected error cause
|
|
166
179
|
:param context: the expected error context
|
|
167
|
-
:param annotations (Optional)
|
|
180
|
+
:param annotations: Dict: (Optional) the expected annotations
|
|
168
181
|
"""
|
|
169
182
|
self.expected_result_type = ErrorResult
|
|
170
183
|
self.err_or_filt_cause = cause
|
|
@@ -175,8 +188,8 @@ class TestCaseBase(ABC):
|
|
|
175
188
|
"""
|
|
176
189
|
A Sets the expected output of the action to a Filter Result
|
|
177
190
|
:param cause: the expected filter cause (message)
|
|
178
|
-
:param context (Optional)
|
|
179
|
-
:param annotations (Optional)
|
|
191
|
+
:param context: (Optional) the expected filter context
|
|
192
|
+
:param annotations: Dict: (Optional) the expected annotations
|
|
180
193
|
"""
|
|
181
194
|
self.expected_result_type = FilterResult
|
|
182
195
|
self.err_or_filt_cause = cause
|
|
@@ -193,10 +206,10 @@ class ActionTest(ABC):
|
|
|
193
206
|
"""
|
|
194
207
|
self.content_service = InternalContentService()
|
|
195
208
|
self.did = ""
|
|
196
|
-
self.expected_outputs = []
|
|
197
209
|
self.loaded_inputs = []
|
|
198
210
|
self.package_name = package_name
|
|
199
211
|
self.res_path = ""
|
|
212
|
+
self.context = None
|
|
200
213
|
|
|
201
214
|
def __reset__(self, did: str):
|
|
202
215
|
self.content_service = InternalContentService()
|
|
@@ -204,9 +217,9 @@ class ActionTest(ABC):
|
|
|
204
217
|
self.did = str(uuid.uuid4())
|
|
205
218
|
else:
|
|
206
219
|
self.did = did
|
|
207
|
-
self.expected_outputs = []
|
|
208
220
|
self.loaded_inputs = []
|
|
209
221
|
self.res_path = ""
|
|
222
|
+
self.context = None
|
|
210
223
|
|
|
211
224
|
def load_file(self, ioc: IOContent):
|
|
212
225
|
file_res = self.res_path.joinpath(ioc.file_name)
|
|
@@ -220,19 +233,12 @@ class ActionTest(ABC):
|
|
|
220
233
|
|
|
221
234
|
# Load inputs
|
|
222
235
|
for input_ioc in test_case.inputs:
|
|
223
|
-
if len(input_ioc.content_bytes) == 0:
|
|
236
|
+
if not input_ioc.no_content and len(input_ioc.content_bytes) == 0:
|
|
224
237
|
self.loaded_inputs.append(LoadedContent(self.did, input_ioc, self.load_file(input_ioc)))
|
|
225
238
|
else:
|
|
226
239
|
self.loaded_inputs.append(LoadedContent(self.did, input_ioc, None))
|
|
227
240
|
|
|
228
|
-
|
|
229
|
-
for output_ioc in test_case.outputs:
|
|
230
|
-
if len(output_ioc.content_bytes) == 0:
|
|
231
|
-
self.expected_outputs.append(LoadedContent(self.did, output_ioc, self.load_file(output_ioc)))
|
|
232
|
-
else:
|
|
233
|
-
self.expected_outputs.append(LoadedContent(self.did, output_ioc, None))
|
|
234
|
-
|
|
235
|
-
def make_content_list(self, test_case: TestCaseBase):
|
|
241
|
+
def make_content_list(self):
|
|
236
242
|
content_list = []
|
|
237
243
|
for loaded_input in self.loaded_inputs:
|
|
238
244
|
c = Content(name=loaded_input.name, segments=[loaded_input.segment], media_type=loaded_input.content_type,
|
|
@@ -242,34 +248,43 @@ class ActionTest(ABC):
|
|
|
242
248
|
|
|
243
249
|
return content_list
|
|
244
250
|
|
|
245
|
-
def
|
|
246
|
-
content_list = self.make_content_list(
|
|
251
|
+
def make_df_msgs(self, test_case: TestCaseBase):
|
|
252
|
+
content_list = self.make_content_list()
|
|
247
253
|
self.content_service.load(self.loaded_inputs)
|
|
248
254
|
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
255
|
+
delta_file_messages = []
|
|
256
|
+
|
|
257
|
+
if test_case.join_meta is None:
|
|
258
|
+
delta_file_messages.append(DeltaFileMessage(metadata=test_case.in_meta, content_list=content_list))
|
|
259
|
+
else:
|
|
260
|
+
for index, content in enumerate(content_list):
|
|
261
|
+
delta_file_messages.append(DeltaFileMessage(
|
|
262
|
+
metadata=test_case.join_meta[index],
|
|
263
|
+
content_list=[content]))
|
|
264
|
+
|
|
265
|
+
return delta_file_messages
|
|
254
266
|
|
|
255
267
|
def make_context(self, test_case: TestCaseBase):
|
|
256
268
|
action_name = INGRESS_FLOW + "." + test_case.action.__class__.__name__
|
|
257
|
-
|
|
269
|
+
join = {} if test_case.join_meta else None
|
|
270
|
+
self.context = Context(
|
|
258
271
|
did=self.did,
|
|
259
|
-
|
|
272
|
+
delta_file_name=test_case.file_name,
|
|
273
|
+
data_source="DATASRC",
|
|
274
|
+
flow_name=INGRESS_FLOW,
|
|
275
|
+
flow_id="FLOWID",
|
|
260
276
|
action_name=action_name,
|
|
261
|
-
|
|
262
|
-
ingress_flow=INGRESS_FLOW,
|
|
263
|
-
egress_flow=EGRESS_FLOW,
|
|
264
|
-
system=SYSTEM,
|
|
277
|
+
action_version="1.0",
|
|
265
278
|
hostname=HOSTNAME,
|
|
279
|
+
system_name=SYSTEM,
|
|
266
280
|
content_service=self.content_service,
|
|
267
|
-
|
|
268
|
-
|
|
281
|
+
saved_content=[],
|
|
282
|
+
join=join,
|
|
269
283
|
logger=get_logger())
|
|
284
|
+
return self.context
|
|
270
285
|
|
|
271
286
|
def make_event(self, test_case: TestCaseBase):
|
|
272
|
-
return Event(delta_file_messages=
|
|
287
|
+
return Event(delta_file_messages=self.make_df_msgs(test_case), context=self.make_context(test_case),
|
|
273
288
|
params=test_case.parameters, queue_name="", return_address="")
|
|
274
289
|
|
|
275
290
|
def call_action(self, test_case: TestCaseBase):
|
|
@@ -311,37 +326,27 @@ class ActionTest(ABC):
|
|
|
311
326
|
else:
|
|
312
327
|
raise ValueError(f"unknown type: {test_case.expected_result_type}")
|
|
313
328
|
|
|
314
|
-
|
|
329
|
+
@staticmethod
|
|
330
|
+
def compare_content_details(expected: LoadedContent, actual: Content):
|
|
315
331
|
assert_equal(expected.content_type, actual.media_type)
|
|
316
332
|
assert_equal(expected.name, actual.name)
|
|
317
333
|
|
|
318
|
-
def compare_one_content(self,
|
|
334
|
+
def compare_one_content(self, comparator: CompareHelper, expected: LoadedContent, actual, index):
|
|
319
335
|
self.compare_content_details(expected, actual)
|
|
320
336
|
seg_id = actual.segments[0].uuid
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
def
|
|
324
|
-
assert_equal_len(
|
|
325
|
-
for index,
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
def compare_domains(self, comparitor: CompareHelper, expected_items: List[Dict], results: List[Dict]):
|
|
329
|
-
assert_equal_len(expected_items, results)
|
|
330
|
-
for index, expected in enumerate(expected_items):
|
|
331
|
-
actual = results[index]
|
|
332
|
-
assert_equal(expected['name'], actual['name'])
|
|
333
|
-
assert_equal(expected['mediaType'], actual['mediaType'])
|
|
334
|
-
|
|
335
|
-
expected_value = expected['value']
|
|
336
|
-
if type(expected_value) == str:
|
|
337
|
-
comparitor.compare(expected_value, actual['value'], f"Domain[{index}]")
|
|
338
|
-
elif type(expected_value) == IOContent:
|
|
339
|
-
expected_data = self.load_file(expected_value)
|
|
340
|
-
comparitor.compare(expected_data, actual['value'], f"Domain[{index}]")
|
|
337
|
+
comparator.compare(expected.data, self.content_service.get_output(seg_id), f"Content[{index}]")
|
|
338
|
+
|
|
339
|
+
def compare_content_list(self, comparator: CompareHelper, expected_outputs: List[IOContent], content: List):
|
|
340
|
+
assert_equal_len(expected_outputs, content)
|
|
341
|
+
for index, expected_ioc in enumerate(expected_outputs):
|
|
342
|
+
if not expected_ioc.no_content and len(expected_ioc.content_bytes) == 0:
|
|
343
|
+
expected = LoadedContent(self.did, expected_ioc, self.load_file(expected_ioc))
|
|
341
344
|
else:
|
|
342
|
-
|
|
345
|
+
expected = LoadedContent(self.did, expected_ioc, None)
|
|
346
|
+
self.compare_one_content(comparator, expected, content[index], index)
|
|
343
347
|
|
|
344
|
-
|
|
348
|
+
@staticmethod
|
|
349
|
+
def compare_one_metric(expected: Metric, result: Metric):
|
|
345
350
|
assert expected.name == result.name
|
|
346
351
|
assert_equal_with_label(expected.value, result.value, expected.name)
|
|
347
352
|
assert_keys_and_values(expected.tags, result.tags)
|
|
@@ -351,3 +356,7 @@ class ActionTest(ABC):
|
|
|
351
356
|
assert_equal_len(expected_metrics, results)
|
|
352
357
|
for index, expected in enumerate(expected_metrics):
|
|
353
358
|
self.compare_one_metric(expected, results[index])
|
|
359
|
+
|
|
360
|
+
def has_saved_content__size(self, count: int):
|
|
361
|
+
assert_equal_with_label(
|
|
362
|
+
count, len(self.context.saved_content), "savedContent")
|
deltafi/test_kit/transform.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
#
|
|
2
2
|
# DeltaFi - Data transformation and enrichment platform
|
|
3
3
|
#
|
|
4
|
-
# Copyright 2021-
|
|
4
|
+
# Copyright 2021-2024 DeltaFi Contributors <deltafi@deltafi.org>
|
|
5
5
|
#
|
|
6
6
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
7
7
|
# you may not use this file except in compliance with the License.
|
|
@@ -18,24 +18,34 @@
|
|
|
18
18
|
|
|
19
19
|
from typing import List
|
|
20
20
|
|
|
21
|
-
from deltafi.result import TransformResult
|
|
21
|
+
from deltafi.result import TransformResult, TransformResults
|
|
22
22
|
|
|
23
23
|
from .assertions import *
|
|
24
|
-
from .framework import TestCaseBase, ActionTest
|
|
24
|
+
from .framework import TestCaseBase, ActionTest, IOContent
|
|
25
25
|
|
|
26
26
|
|
|
27
27
|
class TransformTestCase(TestCaseBase):
|
|
28
28
|
def __init__(self, fields: Dict):
|
|
29
29
|
super().__init__(fields)
|
|
30
|
-
self.
|
|
31
|
-
self.delete_metadata_keys = []
|
|
32
|
-
self.annotations = {}
|
|
30
|
+
self.results = []
|
|
33
31
|
|
|
34
|
-
def expect_transform_result(self
|
|
32
|
+
def expect_transform_result(self):
|
|
35
33
|
self.expected_result_type = TransformResult
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
self.
|
|
34
|
+
|
|
35
|
+
def expect_transform_results(self):
|
|
36
|
+
self.expected_result_type = TransformResults
|
|
37
|
+
|
|
38
|
+
def add_transform_result(self, content: List[IOContent], metadata: Dict, delete_metadata_keys: List[str],
|
|
39
|
+
annotations: Dict, name: str = None):
|
|
40
|
+
self.results.append(
|
|
41
|
+
{
|
|
42
|
+
'content': content,
|
|
43
|
+
'metadata': metadata,
|
|
44
|
+
'delete_metadata_keys': delete_metadata_keys,
|
|
45
|
+
'annotations': annotations,
|
|
46
|
+
'name': name
|
|
47
|
+
}
|
|
48
|
+
)
|
|
39
49
|
|
|
40
50
|
|
|
41
51
|
class TransformActionTest(ActionTest):
|
|
@@ -50,6 +60,8 @@ class TransformActionTest(ActionTest):
|
|
|
50
60
|
def transform(self, test_case: TransformTestCase):
|
|
51
61
|
if test_case.expected_result_type == TransformResult:
|
|
52
62
|
self.expect_transform_result(test_case)
|
|
63
|
+
elif test_case.expected_result_type == TransformResults:
|
|
64
|
+
self.expect_transform_results(test_case)
|
|
53
65
|
else:
|
|
54
66
|
super().execute(test_case)
|
|
55
67
|
|
|
@@ -57,19 +69,35 @@ class TransformActionTest(ActionTest):
|
|
|
57
69
|
result = super().run_and_check_result_type(test_case, TransformResult)
|
|
58
70
|
self.assert_transform_result(test_case, result)
|
|
59
71
|
|
|
72
|
+
def expect_transform_results(self, test_case: TransformTestCase):
|
|
73
|
+
result = super().run_and_check_result_type(test_case, TransformResults)
|
|
74
|
+
self.assert_transform_results(test_case, result)
|
|
75
|
+
|
|
76
|
+
def assert_transform_results(self, test_case: TransformTestCase, result: TransformResults):
|
|
77
|
+
assert_equal_len(test_case.results, result.child_results)
|
|
78
|
+
for index, child_result in enumerate(result.child_results):
|
|
79
|
+
self.compare_one_transform_result(test_case, child_result, index)
|
|
80
|
+
expected = test_case.results[index]
|
|
81
|
+
if 'name' in expected:
|
|
82
|
+
assert_equal_with_label(expected["name"], child_result.delta_file_name, f"name[{index}]")
|
|
83
|
+
|
|
60
84
|
def assert_transform_result(self, test_case: TransformTestCase, result: TransformResult):
|
|
61
85
|
# Check metrics
|
|
62
86
|
self.compare_metrics(test_case.expected_metrics, result.metrics)
|
|
87
|
+
self.compare_one_transform_result(test_case, result, 0)
|
|
88
|
+
|
|
89
|
+
def compare_one_transform_result(self, test_case: TransformTestCase, result: TransformResult, index: int):
|
|
90
|
+
expected = test_case.results[index]
|
|
63
91
|
|
|
64
92
|
# Check output
|
|
65
|
-
self.
|
|
93
|
+
self.compare_content_list(test_case.compare_tool, expected['content'], result.content)
|
|
66
94
|
|
|
67
95
|
# Check metadata
|
|
68
|
-
assert_keys_and_values(
|
|
96
|
+
assert_keys_and_values(expected['metadata'], result.metadata)
|
|
69
97
|
|
|
70
98
|
# Check deleted metadata
|
|
71
|
-
for key in
|
|
99
|
+
for key in expected['delete_metadata_keys']:
|
|
72
100
|
assert_key_in(key, result.delete_metadata_keys)
|
|
73
101
|
|
|
74
102
|
# Check annotations
|
|
75
|
-
assert_keys_and_values(
|
|
103
|
+
assert_keys_and_values(expected['annotations'], result.annotations)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: deltafi
|
|
3
|
-
Version:
|
|
3
|
+
Version: 2.0.0
|
|
4
4
|
Summary: SDK for DeltaFi plugins and actions
|
|
5
5
|
License: Apache License, Version 2.0
|
|
6
6
|
Keywords: deltafi
|
|
@@ -17,14 +17,16 @@ Classifier: Programming Language :: Python :: 3.9
|
|
|
17
17
|
Classifier: Programming Language :: Python :: 3.10
|
|
18
18
|
Classifier: Programming Language :: Python :: 3.11
|
|
19
19
|
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
21
|
Classifier: Topic :: Software Development
|
|
22
|
+
Requires-Dist: PyYAML (==6.0.2)
|
|
21
23
|
Requires-Dist: deepdiff (==6.7.1)
|
|
22
24
|
Requires-Dist: json-logging (==1.3.0)
|
|
23
|
-
Requires-Dist: minio (==7.2.
|
|
24
|
-
Requires-Dist: pydantic (==2.
|
|
25
|
-
Requires-Dist: redis (==5.0
|
|
26
|
-
Requires-Dist: requests (==2.
|
|
27
|
-
Requires-Dist: urllib3 (==2.2.
|
|
25
|
+
Requires-Dist: minio (==7.2.10)
|
|
26
|
+
Requires-Dist: pydantic (==2.9.2)
|
|
27
|
+
Requires-Dist: redis (==5.2.0)
|
|
28
|
+
Requires-Dist: requests (==2.32.3)
|
|
29
|
+
Requires-Dist: urllib3 (==2.2.3)
|
|
28
30
|
Project-URL: Bug Reports, https://chat.deltafi.org/deltafi/channels/bug-reports
|
|
29
31
|
Project-URL: Documentation, https://docs.deltafi.org/#/
|
|
30
32
|
Project-URL: Source Code, https://gitlab.com/deltafi/deltafi
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
deltafi/__init__.py,sha256=sSGWjpvzcCzgLywoEg55z_0satt0g_LPTwNIWtylgF4,709
|
|
2
|
+
deltafi/action.py,sha256=kuDY7-qcJsiRTB3ZigiYg-XOcNeahnDU3NGSZBy10vM,5753
|
|
3
|
+
deltafi/actioneventqueue.py,sha256=_05aDeqVpUbGWG3jFf0AalOsVfAvOTn0oBxjpQIQxIs,2847
|
|
4
|
+
deltafi/actiontype.py,sha256=CpVI0wk9C-eu44e0jYvqBzE6yIX_zfRrW5RCf4ohu4Y,913
|
|
5
|
+
deltafi/domain.py,sha256=wCWqCjWbQ02thAMHYXTuUMHY58QZyyTRkYNZFd3kPPo,12182
|
|
6
|
+
deltafi/exception.py,sha256=LNNFqc_lA89I-RvnVsbTQ2vnWQ9WZXdQqLwiXwNgc_w,943
|
|
7
|
+
deltafi/genericmodel.py,sha256=WU8zfqEO_n84CZ0KpH9FhgTsL9jyU0EXSuhL0IdLWFw,1152
|
|
8
|
+
deltafi/input.py,sha256=ydAhuw68N9qMeacc9YE4U79zRqxWiinZNkm1AxoFBEk,1656
|
|
9
|
+
deltafi/logger.py,sha256=mKfJTnuupf3sto6hV-SIXajtcP-xTSSdJ2Ufd-lJPTo,2140
|
|
10
|
+
deltafi/metric.py,sha256=79Gb2C1qYXeQYshgFPu2BcMT5oEb0LhmNfwyulSqsIc,972
|
|
11
|
+
deltafi/plugin.py,sha256=7e4wblHAG41UCOZ1RcWjkw-0RJF4_dAksx5b4XCNIU4,15521
|
|
12
|
+
deltafi/result.py,sha256=2mefttQAdvYhsw6IjvouY4L5W1n3s6tP7oJUhLtyyVk,9047
|
|
13
|
+
deltafi/storage.py,sha256=vCE29Yzk5s7ub4O1PdaxXBKPBQy4OHs7j46z3O5MezE,2979
|
|
14
|
+
deltafi/test_kit/__init__.py,sha256=sSGWjpvzcCzgLywoEg55z_0satt0g_LPTwNIWtylgF4,709
|
|
15
|
+
deltafi/test_kit/assertions.py,sha256=MdUXENLn0aDvknMtsnSAb-DwvpzXlMkaYQ6-RRkWG8s,1378
|
|
16
|
+
deltafi/test_kit/compare_helpers.py,sha256=qRINvCQqBUaalT1DG2oz2egSkUjiSOfqKF5U7WyeT_g,12957
|
|
17
|
+
deltafi/test_kit/constants.py,sha256=Suygx9CEob2Skw4UyzzMCibQ8hRhGHC_d_Xab8AJMFE,833
|
|
18
|
+
deltafi/test_kit/egress.py,sha256=53SBeJJmphDl-jZdqkC1dKYWZ4ATWlzY02TUt0Sj1zs,1899
|
|
19
|
+
deltafi/test_kit/framework.py,sha256=g7NRl_KxUFio8aZ1xo1gEBKqL9qquYx9qAyFpPLHsmM,15150
|
|
20
|
+
deltafi/test_kit/transform.py,sha256=Qj9LIyZ8RYj0tCeq7agrDlEkkgtBR86TNinV9oncTnM,4089
|
|
21
|
+
deltafi-2.0.0.dist-info/METADATA,sha256=S8ORZcGFlq_9FZ3C38gUYNA7A7qr-ufZTDVbaAbW1UI,1517
|
|
22
|
+
deltafi-2.0.0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
|
|
23
|
+
deltafi-2.0.0.dist-info/RECORD,,
|
deltafi/test_kit/domain.py
DELETED
|
@@ -1,59 +0,0 @@
|
|
|
1
|
-
#
|
|
2
|
-
# DeltaFi - Data transformation and enrichment platform
|
|
3
|
-
#
|
|
4
|
-
# Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
|
|
5
|
-
#
|
|
6
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
7
|
-
# you may not use this file except in compliance with the License.
|
|
8
|
-
# You may obtain a copy of the License at
|
|
9
|
-
#
|
|
10
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
11
|
-
#
|
|
12
|
-
# Unless required by applicable law or agreed to in writing, software
|
|
13
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
14
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
15
|
-
# See the License for the specific language governing permissions and
|
|
16
|
-
# limitations under the License.
|
|
17
|
-
#
|
|
18
|
-
|
|
19
|
-
from deltafi.result import DomainResult
|
|
20
|
-
|
|
21
|
-
from .assertions import *
|
|
22
|
-
from .framework import TestCaseBase, ActionTest
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
class DomainTestCase(TestCaseBase):
|
|
26
|
-
def __init__(self, fields: Dict):
|
|
27
|
-
super().__init__(fields)
|
|
28
|
-
self.annotations = {}
|
|
29
|
-
|
|
30
|
-
def expect_domain_result(self, annotations: Dict):
|
|
31
|
-
self.expected_result_type = DomainResult
|
|
32
|
-
self.annotations = annotations
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
class DomainActionTest(ActionTest):
|
|
36
|
-
def __init__(self, package_name: str):
|
|
37
|
-
"""
|
|
38
|
-
Provides structure for testing DeltaFi Domain action
|
|
39
|
-
Args:
|
|
40
|
-
package_name: name of the actions package for finding resources
|
|
41
|
-
"""
|
|
42
|
-
super().__init__(package_name)
|
|
43
|
-
|
|
44
|
-
def domain(self, test_case: DomainTestCase):
|
|
45
|
-
if test_case.expected_result_type == DomainResult:
|
|
46
|
-
self.expect_domain_result(test_case)
|
|
47
|
-
else:
|
|
48
|
-
super().execute(test_case)
|
|
49
|
-
|
|
50
|
-
def expect_domain_result(self, test_case: DomainTestCase):
|
|
51
|
-
result = super().run_and_check_result_type(test_case, DomainResult)
|
|
52
|
-
self.assert_domain_result(test_case, result)
|
|
53
|
-
|
|
54
|
-
def assert_domain_result(self, test_case: DomainTestCase, result: DomainResult):
|
|
55
|
-
# Check metrics
|
|
56
|
-
self.compare_metrics(test_case.expected_metrics, result.metrics)
|
|
57
|
-
|
|
58
|
-
# Check annotations
|
|
59
|
-
assert_keys_and_values(test_case.annotations, result.annotations)
|
deltafi/test_kit/enrich.py
DELETED
|
@@ -1,70 +0,0 @@
|
|
|
1
|
-
#
|
|
2
|
-
# DeltaFi - Data transformation and enrichment platform
|
|
3
|
-
#
|
|
4
|
-
# Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
|
|
5
|
-
#
|
|
6
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
7
|
-
# you may not use this file except in compliance with the License.
|
|
8
|
-
# You may obtain a copy of the License at
|
|
9
|
-
#
|
|
10
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
11
|
-
#
|
|
12
|
-
# Unless required by applicable law or agreed to in writing, software
|
|
13
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
14
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
15
|
-
# See the License for the specific language governing permissions and
|
|
16
|
-
# limitations under the License.
|
|
17
|
-
#
|
|
18
|
-
|
|
19
|
-
from deltafi.result import EnrichResult
|
|
20
|
-
|
|
21
|
-
from .assertions import *
|
|
22
|
-
from .framework import TestCaseBase, ActionTest
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
class EnrichTestCase(TestCaseBase):
|
|
26
|
-
def __init__(self, fields: Dict):
|
|
27
|
-
super().__init__(fields)
|
|
28
|
-
self.enrichments = []
|
|
29
|
-
self.annotations = {}
|
|
30
|
-
|
|
31
|
-
def expect_enrich_result(self, annotations: Dict):
|
|
32
|
-
self.expected_result_type = EnrichResult
|
|
33
|
-
self.annotations = annotations
|
|
34
|
-
|
|
35
|
-
def add_enrichment(self, name: str, value: str, media_type: str):
|
|
36
|
-
self.enrichments.append({'name': name, 'value': value, 'mediaType': media_type})
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
class EnrichActionTest(ActionTest):
|
|
40
|
-
def __init__(self, package_name: str):
|
|
41
|
-
"""
|
|
42
|
-
Provides structure for testing DeltaFi Enrich action
|
|
43
|
-
Args:
|
|
44
|
-
package_name: name of the actions package for finding resources
|
|
45
|
-
"""
|
|
46
|
-
super().__init__(package_name)
|
|
47
|
-
|
|
48
|
-
def enrich(self, test_case: EnrichTestCase):
|
|
49
|
-
if test_case.expected_result_type == EnrichResult:
|
|
50
|
-
self.expect_enrich_result(test_case)
|
|
51
|
-
else:
|
|
52
|
-
super().execute(test_case)
|
|
53
|
-
|
|
54
|
-
def expect_enrich_result(self, test_case: EnrichTestCase):
|
|
55
|
-
result = super().run_and_check_result_type(test_case, EnrichResult)
|
|
56
|
-
self.assert_enrich_result(test_case, result)
|
|
57
|
-
|
|
58
|
-
def assert_enrich_result(self, test_case: EnrichTestCase, result: EnrichResult):
|
|
59
|
-
# Check metrics
|
|
60
|
-
self.compare_metrics(test_case.expected_metrics, result.metrics)
|
|
61
|
-
|
|
62
|
-
# Check annotations
|
|
63
|
-
assert_keys_and_values(test_case.annotations, result.annotations)
|
|
64
|
-
|
|
65
|
-
# Check enrichments
|
|
66
|
-
assert_equal_len(test_case.enrichments, result.enrichments)
|
|
67
|
-
if len(test_case.enrichments) > 0:
|
|
68
|
-
for index, expected in enumerate(test_case.enrichments):
|
|
69
|
-
actual = result.enrichments[index]
|
|
70
|
-
assert_equal(expected, actual)
|