deltafi 1.2.20__py3-none-any.whl → 2.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of deltafi might be problematic. Click here for more details.

@@ -1,7 +1,7 @@
1
1
  #
2
2
  # DeltaFi - Data transformation and enrichment platform
3
3
  #
4
- # Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
4
+ # Copyright 2021-2024 DeltaFi Contributors <deltafi@deltafi.org>
5
5
  #
6
6
  # Licensed under the Apache License, Version 2.0 (the "License");
7
7
  # you may not use this file except in compliance with the License.
@@ -26,7 +26,6 @@ from deltafi.logger import get_logger
26
26
  from deltafi.metric import Metric
27
27
  from deltafi.result import ErrorResult, FilterResult
28
28
  from deltafi.storage import Segment
29
-
30
29
  from .assertions import *
31
30
  from .compare_helpers import GenericCompareHelper, CompareHelper
32
31
  from .constants import *
@@ -37,15 +36,20 @@ class IOContent:
37
36
  The IOContent class holds the details for loading input or output
38
37
  content into the test framework.
39
38
  Attributes:
40
- file_name (str): The name of file in test/data.
41
- content_name (str): The name of the content.
42
- content_type (str): The media type of the content
43
- offset (int): Offset to use in Segment
44
- content_bytes (str): Bypass file read, and uses these bytes for content
39
+ file_name (str) : The name of file in test/data.
40
+ content_name (str) : The name of the content.
41
+ content_type (str) : The media type of the content
42
+ offset (int) : Offset to use in Segment
43
+ content_bytes (str): Optional. If set to a String of length greater than zero, indicates to consumers of this
44
+ IOContent that they should bypass file read and use these bytes for content.
45
+ no_content (bool) : Optional. If 'True', then consumers should not attempt to interpret content but should
46
+ apply other aspects of this IOContent. When 'True', 'content_bytes' should be ignored and
47
+ loaded content, if any, should be interpreted as empty String or otherwise as documented by
48
+ the consumer.
45
49
  """
46
50
 
47
51
  def __init__(self, file_name: str, content_name: str = None, content_type: str = None, offset: int = 0,
48
- content_bytes: str = ""):
52
+ content_bytes: str = "", no_content: bool = False):
49
53
  self.file_name = file_name
50
54
  if content_name is None:
51
55
  self.content_name = file_name
@@ -56,7 +60,11 @@ class IOContent:
56
60
  else:
57
61
  self.content_type = content_type
58
62
  self.offset = offset
59
- self.content_bytes = content_bytes
63
+ self.no_content = no_content
64
+ if no_content:
65
+ self.content_bytes = None
66
+ else:
67
+ self.content_bytes = content_bytes
60
68
  self.segment_uuid = uuid.uuid4()
61
69
 
62
70
  @classmethod
@@ -79,7 +87,10 @@ class LoadedContent:
79
87
  if data is not None:
80
88
  self.data = data
81
89
  else:
82
- self.data = ioc.content_bytes
90
+ if ioc.no_content:
91
+ self.data = ""
92
+ else:
93
+ self.data = ioc.content_bytes
83
94
  self.segment = Segment.from_dict(
84
95
  {"uuid": str(ioc.segment_uuid), "offset": self.offset, "size": len(self.data), "did": did})
85
96
 
@@ -103,6 +114,10 @@ class InternalContentService:
103
114
  seg_id = segments[0].uuid
104
115
  return self.loaded_content[seg_id].data
105
116
 
117
+ def get_bytes(self, segments: List[Segment]):
118
+ seg_id = segments[0].uuid
119
+ return self.loaded_content[seg_id].data.encode('utf-8')
120
+
106
121
  def get_output(self, seg_id: str):
107
122
  if seg_id in self.outputs:
108
123
  return self.outputs[seg_id]
@@ -123,8 +138,8 @@ class TestCaseBase(ABC):
123
138
  - inputs: (optional) List[IOContent]: input content to action
124
139
  - parameters: (optional) Dict: map of action input parameters
125
140
  - in_meta: (optional) Dict: map of metadata as input to action
126
- - in_domains: (optional) List[Domain]: list of domains as input to action
127
- - in_enrichments: (optional) List[Domain]: list of enrichments as input to action
141
+ - join_meta: (optional): List[Dict]: When a List is provided, this enables the JOIN portion of an action.
142
+ When using JOIN, join_meta must match the size of inputs, though the Dict can be empty
128
143
  - did: (optional): str: overrides random DID
129
144
  """
130
145
  if "action" in data:
@@ -144,16 +159,14 @@ class TestCaseBase(ABC):
144
159
 
145
160
  self.inputs = data["inputs"] if "inputs" in data else []
146
161
  self.file_name = data["file_name"] if "file_name" in data else "filename"
147
- self.outputs = data["outputs"] if "outputs" in data else []
148
162
  self.parameters = data["parameters"] if "parameters" in data else {}
149
163
  self.in_meta = data["in_meta"] if "in_meta" in data else {}
150
- self.in_domains = data["in_domains"] if "in_domains" in data else []
151
- self.in_enrichments = data["in_enrichments"] if "in_enrichments" in data else []
152
164
  self.use_did = data["did"] if "did" in data else None
153
165
  self.expected_result_type = None
154
166
  self.err_or_filt_cause = None
155
167
  self.err_or_filt_context = None
156
168
  self.err_or_filt_annotations = None
169
+ self.join_meta = data["join_meta"] if "join_meta" in data else None
157
170
  self.expected_metrics = []
158
171
 
159
172
  def add_metric(self, metric: Metric):
@@ -164,7 +177,7 @@ class TestCaseBase(ABC):
164
177
  A Sets the expected output of the action to an Error Result
165
178
  :param cause: the expected error cause
166
179
  :param context: the expected error context
167
- :param annotations (Optional): Dict: the expected annotations
180
+ :param annotations: Dict: (Optional) the expected annotations
168
181
  """
169
182
  self.expected_result_type = ErrorResult
170
183
  self.err_or_filt_cause = cause
@@ -175,8 +188,8 @@ class TestCaseBase(ABC):
175
188
  """
176
189
  A Sets the expected output of the action to a Filter Result
177
190
  :param cause: the expected filter cause (message)
178
- :param context (Optional): the expected filter context
179
- :param annotations (Optional): Dict: the expected annotations
191
+ :param context: (Optional) the expected filter context
192
+ :param annotations: Dict: (Optional) the expected annotations
180
193
  """
181
194
  self.expected_result_type = FilterResult
182
195
  self.err_or_filt_cause = cause
@@ -193,10 +206,10 @@ class ActionTest(ABC):
193
206
  """
194
207
  self.content_service = InternalContentService()
195
208
  self.did = ""
196
- self.expected_outputs = []
197
209
  self.loaded_inputs = []
198
210
  self.package_name = package_name
199
211
  self.res_path = ""
212
+ self.context = None
200
213
 
201
214
  def __reset__(self, did: str):
202
215
  self.content_service = InternalContentService()
@@ -204,9 +217,9 @@ class ActionTest(ABC):
204
217
  self.did = str(uuid.uuid4())
205
218
  else:
206
219
  self.did = did
207
- self.expected_outputs = []
208
220
  self.loaded_inputs = []
209
221
  self.res_path = ""
222
+ self.context = None
210
223
 
211
224
  def load_file(self, ioc: IOContent):
212
225
  file_res = self.res_path.joinpath(ioc.file_name)
@@ -220,19 +233,12 @@ class ActionTest(ABC):
220
233
 
221
234
  # Load inputs
222
235
  for input_ioc in test_case.inputs:
223
- if len(input_ioc.content_bytes) == 0:
236
+ if not input_ioc.no_content and len(input_ioc.content_bytes) == 0:
224
237
  self.loaded_inputs.append(LoadedContent(self.did, input_ioc, self.load_file(input_ioc)))
225
238
  else:
226
239
  self.loaded_inputs.append(LoadedContent(self.did, input_ioc, None))
227
240
 
228
- # Load expected outputs
229
- for output_ioc in test_case.outputs:
230
- if len(output_ioc.content_bytes) == 0:
231
- self.expected_outputs.append(LoadedContent(self.did, output_ioc, self.load_file(output_ioc)))
232
- else:
233
- self.expected_outputs.append(LoadedContent(self.did, output_ioc, None))
234
-
235
- def make_content_list(self, test_case: TestCaseBase):
241
+ def make_content_list(self):
236
242
  content_list = []
237
243
  for loaded_input in self.loaded_inputs:
238
244
  c = Content(name=loaded_input.name, segments=[loaded_input.segment], media_type=loaded_input.content_type,
@@ -242,34 +248,43 @@ class ActionTest(ABC):
242
248
 
243
249
  return content_list
244
250
 
245
- def make_df_msg(self, test_case: TestCaseBase):
246
- content_list = self.make_content_list(test_case)
251
+ def make_df_msgs(self, test_case: TestCaseBase):
252
+ content_list = self.make_content_list()
247
253
  self.content_service.load(self.loaded_inputs)
248
254
 
249
- return DeltaFileMessage(
250
- metadata=test_case.in_meta,
251
- content_list=content_list,
252
- domains=test_case.in_domains,
253
- enrichments=test_case.in_enrichments)
255
+ delta_file_messages = []
256
+
257
+ if test_case.join_meta is None:
258
+ delta_file_messages.append(DeltaFileMessage(metadata=test_case.in_meta, content_list=content_list))
259
+ else:
260
+ for index, content in enumerate(content_list):
261
+ delta_file_messages.append(DeltaFileMessage(
262
+ metadata=test_case.join_meta[index],
263
+ content_list=[content]))
264
+
265
+ return delta_file_messages
254
266
 
255
267
  def make_context(self, test_case: TestCaseBase):
256
268
  action_name = INGRESS_FLOW + "." + test_case.action.__class__.__name__
257
- return Context(
269
+ join = {} if test_case.join_meta else None
270
+ self.context = Context(
258
271
  did=self.did,
259
- action_flow=INGRESS_FLOW,
272
+ delta_file_name=test_case.file_name,
273
+ data_source="DATASRC",
274
+ flow_name=INGRESS_FLOW,
275
+ flow_id="FLOWID",
260
276
  action_name=action_name,
261
- source_filename=test_case.file_name,
262
- ingress_flow=INGRESS_FLOW,
263
- egress_flow=EGRESS_FLOW,
264
- system=SYSTEM,
277
+ action_version="1.0",
265
278
  hostname=HOSTNAME,
279
+ system_name=SYSTEM,
266
280
  content_service=self.content_service,
267
- collect=None,
268
- collected_dids=None,
281
+ saved_content=[],
282
+ join=join,
269
283
  logger=get_logger())
284
+ return self.context
270
285
 
271
286
  def make_event(self, test_case: TestCaseBase):
272
- return Event(delta_file_messages=[self.make_df_msg(test_case)], context=self.make_context(test_case),
287
+ return Event(delta_file_messages=self.make_df_msgs(test_case), context=self.make_context(test_case),
273
288
  params=test_case.parameters, queue_name="", return_address="")
274
289
 
275
290
  def call_action(self, test_case: TestCaseBase):
@@ -311,37 +326,27 @@ class ActionTest(ABC):
311
326
  else:
312
327
  raise ValueError(f"unknown type: {test_case.expected_result_type}")
313
328
 
314
- def compare_content_details(self, expected: LoadedContent, actual: Content):
329
+ @staticmethod
330
+ def compare_content_details(expected: LoadedContent, actual: Content):
315
331
  assert_equal(expected.content_type, actual.media_type)
316
332
  assert_equal(expected.name, actual.name)
317
333
 
318
- def compare_one_content(self, comparitor: CompareHelper, expected: LoadedContent, actual, index):
334
+ def compare_one_content(self, comparator: CompareHelper, expected: LoadedContent, actual, index):
319
335
  self.compare_content_details(expected, actual)
320
336
  seg_id = actual.segments[0].uuid
321
- comparitor.compare(expected.data, self.content_service.get_output(seg_id), f"Content[{index}]")
322
-
323
- def compare_all_output(self, comparitor: CompareHelper, content: List):
324
- assert_equal_len(self.expected_outputs, content)
325
- for index, expected in enumerate(self.expected_outputs):
326
- self.compare_one_content(comparitor, expected, content[index], index)
327
-
328
- def compare_domains(self, comparitor: CompareHelper, expected_items: List[Dict], results: List[Dict]):
329
- assert_equal_len(expected_items, results)
330
- for index, expected in enumerate(expected_items):
331
- actual = results[index]
332
- assert_equal(expected['name'], actual['name'])
333
- assert_equal(expected['mediaType'], actual['mediaType'])
334
-
335
- expected_value = expected['value']
336
- if type(expected_value) == str:
337
- comparitor.compare(expected_value, actual['value'], f"Domain[{index}]")
338
- elif type(expected_value) == IOContent:
339
- expected_data = self.load_file(expected_value)
340
- comparitor.compare(expected_data, actual['value'], f"Domain[{index}]")
337
+ comparator.compare(expected.data, self.content_service.get_output(seg_id), f"Content[{index}]")
338
+
339
+ def compare_content_list(self, comparator: CompareHelper, expected_outputs: List[IOContent], content: List):
340
+ assert_equal_len(expected_outputs, content)
341
+ for index, expected_ioc in enumerate(expected_outputs):
342
+ if not expected_ioc.no_content and len(expected_ioc.content_bytes) == 0:
343
+ expected = LoadedContent(self.did, expected_ioc, self.load_file(expected_ioc))
341
344
  else:
342
- raise ValueError(f"unknown expected_value type: {type(expected_value)}")
345
+ expected = LoadedContent(self.did, expected_ioc, None)
346
+ self.compare_one_content(comparator, expected, content[index], index)
343
347
 
344
- def compare_one_metric(self, expected: Metric, result: Metric):
348
+ @staticmethod
349
+ def compare_one_metric(expected: Metric, result: Metric):
345
350
  assert expected.name == result.name
346
351
  assert_equal_with_label(expected.value, result.value, expected.name)
347
352
  assert_keys_and_values(expected.tags, result.tags)
@@ -351,3 +356,7 @@ class ActionTest(ABC):
351
356
  assert_equal_len(expected_metrics, results)
352
357
  for index, expected in enumerate(expected_metrics):
353
358
  self.compare_one_metric(expected, results[index])
359
+
360
+ def has_saved_content__size(self, count: int):
361
+ assert_equal_with_label(
362
+ count, len(self.context.saved_content), "savedContent")
@@ -1,7 +1,7 @@
1
1
  #
2
2
  # DeltaFi - Data transformation and enrichment platform
3
3
  #
4
- # Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
4
+ # Copyright 2021-2024 DeltaFi Contributors <deltafi@deltafi.org>
5
5
  #
6
6
  # Licensed under the Apache License, Version 2.0 (the "License");
7
7
  # you may not use this file except in compliance with the License.
@@ -18,24 +18,34 @@
18
18
 
19
19
  from typing import List
20
20
 
21
- from deltafi.result import TransformResult
21
+ from deltafi.result import TransformResult, TransformResults
22
22
 
23
23
  from .assertions import *
24
- from .framework import TestCaseBase, ActionTest
24
+ from .framework import TestCaseBase, ActionTest, IOContent
25
25
 
26
26
 
27
27
  class TransformTestCase(TestCaseBase):
28
28
  def __init__(self, fields: Dict):
29
29
  super().__init__(fields)
30
- self.metadata = {}
31
- self.delete_metadata_keys = []
32
- self.annotations = {}
30
+ self.results = []
33
31
 
34
- def expect_transform_result(self, metadata: Dict, delete_metadata_keys: List[str], annotations: Dict):
32
+ def expect_transform_result(self):
35
33
  self.expected_result_type = TransformResult
36
- self.metadata = metadata
37
- self.delete_metadata_keys = delete_metadata_keys
38
- self.annotations = annotations
34
+
35
+ def expect_transform_results(self):
36
+ self.expected_result_type = TransformResults
37
+
38
+ def add_transform_result(self, content: List[IOContent], metadata: Dict, delete_metadata_keys: List[str],
39
+ annotations: Dict, name: str = None):
40
+ self.results.append(
41
+ {
42
+ 'content': content,
43
+ 'metadata': metadata,
44
+ 'delete_metadata_keys': delete_metadata_keys,
45
+ 'annotations': annotations,
46
+ 'name': name
47
+ }
48
+ )
39
49
 
40
50
 
41
51
  class TransformActionTest(ActionTest):
@@ -50,6 +60,8 @@ class TransformActionTest(ActionTest):
50
60
  def transform(self, test_case: TransformTestCase):
51
61
  if test_case.expected_result_type == TransformResult:
52
62
  self.expect_transform_result(test_case)
63
+ elif test_case.expected_result_type == TransformResults:
64
+ self.expect_transform_results(test_case)
53
65
  else:
54
66
  super().execute(test_case)
55
67
 
@@ -57,19 +69,35 @@ class TransformActionTest(ActionTest):
57
69
  result = super().run_and_check_result_type(test_case, TransformResult)
58
70
  self.assert_transform_result(test_case, result)
59
71
 
72
+ def expect_transform_results(self, test_case: TransformTestCase):
73
+ result = super().run_and_check_result_type(test_case, TransformResults)
74
+ self.assert_transform_results(test_case, result)
75
+
76
+ def assert_transform_results(self, test_case: TransformTestCase, result: TransformResults):
77
+ assert_equal_len(test_case.results, result.child_results)
78
+ for index, child_result in enumerate(result.child_results):
79
+ self.compare_one_transform_result(test_case, child_result, index)
80
+ expected = test_case.results[index]
81
+ if 'name' in expected:
82
+ assert_equal_with_label(expected["name"], child_result.delta_file_name, f"name[{index}]")
83
+
60
84
  def assert_transform_result(self, test_case: TransformTestCase, result: TransformResult):
61
85
  # Check metrics
62
86
  self.compare_metrics(test_case.expected_metrics, result.metrics)
87
+ self.compare_one_transform_result(test_case, result, 0)
88
+
89
+ def compare_one_transform_result(self, test_case: TransformTestCase, result: TransformResult, index: int):
90
+ expected = test_case.results[index]
63
91
 
64
92
  # Check output
65
- self.compare_all_output(test_case.compare_tool, result.content)
93
+ self.compare_content_list(test_case.compare_tool, expected['content'], result.content)
66
94
 
67
95
  # Check metadata
68
- assert_keys_and_values(test_case.metadata, result.metadata)
96
+ assert_keys_and_values(expected['metadata'], result.metadata)
69
97
 
70
98
  # Check deleted metadata
71
- for key in test_case.delete_metadata_keys:
99
+ for key in expected['delete_metadata_keys']:
72
100
  assert_key_in(key, result.delete_metadata_keys)
73
101
 
74
102
  # Check annotations
75
- assert_keys_and_values(test_case.annotations, result.annotations)
103
+ assert_keys_and_values(expected['annotations'], result.annotations)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: deltafi
3
- Version: 1.2.20
3
+ Version: 2.0.0
4
4
  Summary: SDK for DeltaFi plugins and actions
5
5
  License: Apache License, Version 2.0
6
6
  Keywords: deltafi
@@ -17,14 +17,16 @@ Classifier: Programming Language :: Python :: 3.9
17
17
  Classifier: Programming Language :: Python :: 3.10
18
18
  Classifier: Programming Language :: Python :: 3.11
19
19
  Classifier: Programming Language :: Python :: 3.12
20
+ Classifier: Programming Language :: Python :: 3.13
20
21
  Classifier: Topic :: Software Development
22
+ Requires-Dist: PyYAML (==6.0.2)
21
23
  Requires-Dist: deepdiff (==6.7.1)
22
24
  Requires-Dist: json-logging (==1.3.0)
23
- Requires-Dist: minio (==7.2.5)
24
- Requires-Dist: pydantic (==2.7.1)
25
- Requires-Dist: redis (==5.0.4)
26
- Requires-Dist: requests (==2.31.0)
27
- Requires-Dist: urllib3 (==2.2.1)
25
+ Requires-Dist: minio (==7.2.10)
26
+ Requires-Dist: pydantic (==2.9.2)
27
+ Requires-Dist: redis (==5.2.0)
28
+ Requires-Dist: requests (==2.32.3)
29
+ Requires-Dist: urllib3 (==2.2.3)
28
30
  Project-URL: Bug Reports, https://chat.deltafi.org/deltafi/channels/bug-reports
29
31
  Project-URL: Documentation, https://docs.deltafi.org/#/
30
32
  Project-URL: Source Code, https://gitlab.com/deltafi/deltafi
@@ -0,0 +1,23 @@
1
+ deltafi/__init__.py,sha256=sSGWjpvzcCzgLywoEg55z_0satt0g_LPTwNIWtylgF4,709
2
+ deltafi/action.py,sha256=kuDY7-qcJsiRTB3ZigiYg-XOcNeahnDU3NGSZBy10vM,5753
3
+ deltafi/actioneventqueue.py,sha256=_05aDeqVpUbGWG3jFf0AalOsVfAvOTn0oBxjpQIQxIs,2847
4
+ deltafi/actiontype.py,sha256=CpVI0wk9C-eu44e0jYvqBzE6yIX_zfRrW5RCf4ohu4Y,913
5
+ deltafi/domain.py,sha256=wCWqCjWbQ02thAMHYXTuUMHY58QZyyTRkYNZFd3kPPo,12182
6
+ deltafi/exception.py,sha256=LNNFqc_lA89I-RvnVsbTQ2vnWQ9WZXdQqLwiXwNgc_w,943
7
+ deltafi/genericmodel.py,sha256=WU8zfqEO_n84CZ0KpH9FhgTsL9jyU0EXSuhL0IdLWFw,1152
8
+ deltafi/input.py,sha256=ydAhuw68N9qMeacc9YE4U79zRqxWiinZNkm1AxoFBEk,1656
9
+ deltafi/logger.py,sha256=mKfJTnuupf3sto6hV-SIXajtcP-xTSSdJ2Ufd-lJPTo,2140
10
+ deltafi/metric.py,sha256=79Gb2C1qYXeQYshgFPu2BcMT5oEb0LhmNfwyulSqsIc,972
11
+ deltafi/plugin.py,sha256=7e4wblHAG41UCOZ1RcWjkw-0RJF4_dAksx5b4XCNIU4,15521
12
+ deltafi/result.py,sha256=2mefttQAdvYhsw6IjvouY4L5W1n3s6tP7oJUhLtyyVk,9047
13
+ deltafi/storage.py,sha256=vCE29Yzk5s7ub4O1PdaxXBKPBQy4OHs7j46z3O5MezE,2979
14
+ deltafi/test_kit/__init__.py,sha256=sSGWjpvzcCzgLywoEg55z_0satt0g_LPTwNIWtylgF4,709
15
+ deltafi/test_kit/assertions.py,sha256=MdUXENLn0aDvknMtsnSAb-DwvpzXlMkaYQ6-RRkWG8s,1378
16
+ deltafi/test_kit/compare_helpers.py,sha256=qRINvCQqBUaalT1DG2oz2egSkUjiSOfqKF5U7WyeT_g,12957
17
+ deltafi/test_kit/constants.py,sha256=Suygx9CEob2Skw4UyzzMCibQ8hRhGHC_d_Xab8AJMFE,833
18
+ deltafi/test_kit/egress.py,sha256=53SBeJJmphDl-jZdqkC1dKYWZ4ATWlzY02TUt0Sj1zs,1899
19
+ deltafi/test_kit/framework.py,sha256=g7NRl_KxUFio8aZ1xo1gEBKqL9qquYx9qAyFpPLHsmM,15150
20
+ deltafi/test_kit/transform.py,sha256=Qj9LIyZ8RYj0tCeq7agrDlEkkgtBR86TNinV9oncTnM,4089
21
+ deltafi-2.0.0.dist-info/METADATA,sha256=S8ORZcGFlq_9FZ3C38gUYNA7A7qr-ufZTDVbaAbW1UI,1517
22
+ deltafi-2.0.0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
23
+ deltafi-2.0.0.dist-info/RECORD,,
@@ -1,4 +1,4 @@
1
1
  Wheel-Version: 1.0
2
- Generator: poetry-core 1.9.0
2
+ Generator: poetry-core 1.9.1
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any
@@ -1,59 +0,0 @@
1
- #
2
- # DeltaFi - Data transformation and enrichment platform
3
- #
4
- # Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
5
- #
6
- # Licensed under the Apache License, Version 2.0 (the "License");
7
- # you may not use this file except in compliance with the License.
8
- # You may obtain a copy of the License at
9
- #
10
- # http://www.apache.org/licenses/LICENSE-2.0
11
- #
12
- # Unless required by applicable law or agreed to in writing, software
13
- # distributed under the License is distributed on an "AS IS" BASIS,
14
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
- # See the License for the specific language governing permissions and
16
- # limitations under the License.
17
- #
18
-
19
- from deltafi.result import DomainResult
20
-
21
- from .assertions import *
22
- from .framework import TestCaseBase, ActionTest
23
-
24
-
25
- class DomainTestCase(TestCaseBase):
26
- def __init__(self, fields: Dict):
27
- super().__init__(fields)
28
- self.annotations = {}
29
-
30
- def expect_domain_result(self, annotations: Dict):
31
- self.expected_result_type = DomainResult
32
- self.annotations = annotations
33
-
34
-
35
- class DomainActionTest(ActionTest):
36
- def __init__(self, package_name: str):
37
- """
38
- Provides structure for testing DeltaFi Domain action
39
- Args:
40
- package_name: name of the actions package for finding resources
41
- """
42
- super().__init__(package_name)
43
-
44
- def domain(self, test_case: DomainTestCase):
45
- if test_case.expected_result_type == DomainResult:
46
- self.expect_domain_result(test_case)
47
- else:
48
- super().execute(test_case)
49
-
50
- def expect_domain_result(self, test_case: DomainTestCase):
51
- result = super().run_and_check_result_type(test_case, DomainResult)
52
- self.assert_domain_result(test_case, result)
53
-
54
- def assert_domain_result(self, test_case: DomainTestCase, result: DomainResult):
55
- # Check metrics
56
- self.compare_metrics(test_case.expected_metrics, result.metrics)
57
-
58
- # Check annotations
59
- assert_keys_and_values(test_case.annotations, result.annotations)
@@ -1,70 +0,0 @@
1
- #
2
- # DeltaFi - Data transformation and enrichment platform
3
- #
4
- # Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
5
- #
6
- # Licensed under the Apache License, Version 2.0 (the "License");
7
- # you may not use this file except in compliance with the License.
8
- # You may obtain a copy of the License at
9
- #
10
- # http://www.apache.org/licenses/LICENSE-2.0
11
- #
12
- # Unless required by applicable law or agreed to in writing, software
13
- # distributed under the License is distributed on an "AS IS" BASIS,
14
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
- # See the License for the specific language governing permissions and
16
- # limitations under the License.
17
- #
18
-
19
- from deltafi.result import EnrichResult
20
-
21
- from .assertions import *
22
- from .framework import TestCaseBase, ActionTest
23
-
24
-
25
- class EnrichTestCase(TestCaseBase):
26
- def __init__(self, fields: Dict):
27
- super().__init__(fields)
28
- self.enrichments = []
29
- self.annotations = {}
30
-
31
- def expect_enrich_result(self, annotations: Dict):
32
- self.expected_result_type = EnrichResult
33
- self.annotations = annotations
34
-
35
- def add_enrichment(self, name: str, value: str, media_type: str):
36
- self.enrichments.append({'name': name, 'value': value, 'mediaType': media_type})
37
-
38
-
39
- class EnrichActionTest(ActionTest):
40
- def __init__(self, package_name: str):
41
- """
42
- Provides structure for testing DeltaFi Enrich action
43
- Args:
44
- package_name: name of the actions package for finding resources
45
- """
46
- super().__init__(package_name)
47
-
48
- def enrich(self, test_case: EnrichTestCase):
49
- if test_case.expected_result_type == EnrichResult:
50
- self.expect_enrich_result(test_case)
51
- else:
52
- super().execute(test_case)
53
-
54
- def expect_enrich_result(self, test_case: EnrichTestCase):
55
- result = super().run_and_check_result_type(test_case, EnrichResult)
56
- self.assert_enrich_result(test_case, result)
57
-
58
- def assert_enrich_result(self, test_case: EnrichTestCase, result: EnrichResult):
59
- # Check metrics
60
- self.compare_metrics(test_case.expected_metrics, result.metrics)
61
-
62
- # Check annotations
63
- assert_keys_and_values(test_case.annotations, result.annotations)
64
-
65
- # Check enrichments
66
- assert_equal_len(test_case.enrichments, result.enrichments)
67
- if len(test_case.enrichments) > 0:
68
- for index, expected in enumerate(test_case.enrichments):
69
- actual = result.enrichments[index]
70
- assert_equal(expected, actual)