deltafi 2.0rc1705080991758__py3-none-any.whl → 2.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of deltafi might be problematic. Click here for more details.

@@ -1,7 +1,7 @@
1
1
  #
2
2
  # DeltaFi - Data transformation and enrichment platform
3
3
  #
4
- # Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
4
+ # Copyright 2021-2025 DeltaFi Contributors <deltafi@deltafi.org>
5
5
  #
6
6
  # Licensed under the Apache License, Version 2.0 (the "License");
7
7
  # you may not use this file except in compliance with the License.
@@ -23,9 +23,9 @@ from typing import List
23
23
 
24
24
  from deltafi.domain import DeltaFileMessage, Event, Content, Context
25
25
  from deltafi.logger import get_logger
26
+ from deltafi.metric import Metric
26
27
  from deltafi.result import ErrorResult, FilterResult
27
28
  from deltafi.storage import Segment
28
-
29
29
  from .assertions import *
30
30
  from .compare_helpers import GenericCompareHelper, CompareHelper
31
31
  from .constants import *
@@ -36,16 +36,20 @@ class IOContent:
36
36
  The IOContent class holds the details for loading input or output
37
37
  content into the test framework.
38
38
  Attributes:
39
- file_name (str): The name of file in test/data.
40
- content_name (str): The name of the content.
41
- content_type (str): The media type of the content
42
- offset (int): Offset to use in Segment
43
- content_bytes (str): Bypass file read, and uses these bytes for content
39
+ file_name (str) : The name of file in test/data.
40
+ content_name (str) : The name of the content.
41
+ content_type (str) : The media type of the content
42
+ offset (int) : Offset to use in Segment
43
+ content_bytes (str): Optional. If set to a String of length greater than zero, indicates to consumers of this
44
+ IOContent that they should bypass file read and use these bytes for content.
45
+ no_content (bool) : Optional. If 'True', then consumers should not attempt to interpret content but should
46
+ apply other aspects of this IOContent. When 'True', 'content_bytes' should be ignored and
47
+ loaded content, if any, should be interpreted as empty String or otherwise as documented by
48
+ the consumer.
44
49
  """
45
50
 
46
- def __init__(self, file_name: str, content_name: str = None,
47
- content_type: str = None, offset: int = 0,
48
- content_bytes: str = ""):
51
+ def __init__(self, file_name: str, content_name: str = None, content_type: str = None, offset: int = 0,
52
+ content_bytes: str = "", no_content: bool = False):
49
53
  self.file_name = file_name
50
54
  if content_name is None:
51
55
  self.content_name = file_name
@@ -56,7 +60,12 @@ class IOContent:
56
60
  else:
57
61
  self.content_type = content_type
58
62
  self.offset = offset
59
- self.content_bytes = content_bytes
63
+ self.no_content = no_content
64
+ if no_content:
65
+ self.content_bytes = None
66
+ else:
67
+ self.content_bytes = content_bytes
68
+ self.segment_uuid = uuid.uuid4()
60
69
 
61
70
  @classmethod
62
71
  def file_type(cls, name: str):
@@ -78,13 +87,12 @@ class LoadedContent:
78
87
  if data is not None:
79
88
  self.data = data
80
89
  else:
81
- self.data = ioc.content_bytes
82
- self.segment = Segment.from_dict({
83
- "uuid": str(uuid.uuid4()),
84
- "offset": self.offset,
85
- "size": len(self.data),
86
- "did": did
87
- })
90
+ if ioc.no_content:
91
+ self.data = ""
92
+ else:
93
+ self.data = ioc.content_bytes
94
+ self.segment = Segment.from_dict(
95
+ {"uuid": str(ioc.segment_uuid), "offset": self.offset, "size": len(self.data), "did": did})
88
96
 
89
97
 
90
98
  class InternalContentService:
@@ -97,10 +105,7 @@ class InternalContentService:
97
105
  self.loaded_content[c.segment.uuid] = c
98
106
 
99
107
  def put_str(self, did: str, string_data: str):
100
- segment = Segment(uuid=str(uuid.uuid4()),
101
- offset=0,
102
- size=len(string_data),
103
- did=did)
108
+ segment = Segment(uuid=str(uuid.uuid4()), offset=0, size=len(string_data), did=did)
104
109
  self.outputs[segment.uuid] = string_data
105
110
  return segment
106
111
 
@@ -109,8 +114,17 @@ class InternalContentService:
109
114
  seg_id = segments[0].uuid
110
115
  return self.loaded_content[seg_id].data
111
116
 
117
+ def get_bytes(self, segments: List[Segment]):
118
+ seg_id = segments[0].uuid
119
+ return self.loaded_content[seg_id].data.encode('utf-8')
120
+
112
121
  def get_output(self, seg_id: str):
113
- return self.outputs[seg_id]
122
+ if seg_id in self.outputs:
123
+ return self.outputs[seg_id]
124
+ elif seg_id in self.loaded_content:
125
+ return self.loaded_content[seg_id].data
126
+ else:
127
+ return None
114
128
 
115
129
 
116
130
  class TestCaseBase(ABC):
@@ -123,9 +137,11 @@ class TestCaseBase(ABC):
123
137
  - compare_tool: (optional) CompareHelper instanced for comparing output content
124
138
  - inputs: (optional) List[IOContent]: input content to action
125
139
  - parameters: (optional) Dict: map of action input parameters
140
+ - in_memo: (optional) str: Input 'memo' value for a TimedIngress context
126
141
  - in_meta: (optional) Dict: map of metadata as input to action
127
- - in_domains: (optional) List[Domain]: list of domains as input to action
128
- - in_enrichments: (optional) List[Domain]: list of enrichments as input to action
142
+ - join_meta: (optional): List[Dict]: When a List is provided, this enables the JOIN portion of an action.
143
+ When using JOIN, join_meta must match the size of inputs, though the Dict can be empty
144
+ - did: (optional): str: overrides random DID
129
145
  """
130
146
  if "action" in data:
131
147
  self.action = data["action"]
@@ -144,34 +160,43 @@ class TestCaseBase(ABC):
144
160
 
145
161
  self.inputs = data["inputs"] if "inputs" in data else []
146
162
  self.file_name = data["file_name"] if "file_name" in data else "filename"
147
- self.outputs = data["outputs"] if "outputs" in data else []
148
163
  self.parameters = data["parameters"] if "parameters" in data else {}
149
164
  self.in_meta = data["in_meta"] if "in_meta" in data else {}
150
- self.in_domains = data["in_domains"] if "in_domains" in data else []
151
- self.in_enrichments = data["in_enrichments"] if "in_enrichments" in data else []
165
+ self.in_memo = data["in_memo"] if "in_memo" in data else None
166
+ self.use_did = data["did"] if "did" in data else None
152
167
  self.expected_result_type = None
153
- self.cause_regex = None
154
- self.context_regex = None
168
+ self.err_or_filt_cause = None
169
+ self.err_or_filt_context = None
170
+ self.err_or_filt_annotations = None
171
+ self.join_meta = data["join_meta"] if "join_meta" in data else None
172
+ self.expected_metrics = []
173
+
174
+ def add_metric(self, metric: Metric):
175
+ self.expected_metrics.append(metric)
155
176
 
156
- def expect_error_result(self, cause: str, context: str):
177
+ def expect_error_result(self, cause: str, context: str, annotations: Dict = None):
157
178
  """
158
179
  A Sets the expected output of the action to an Error Result
159
180
  :param cause: the expected error cause
160
181
  :param context: the expected error context
182
+ :param annotations: Dict: (Optional) the expected annotations
161
183
  """
162
184
  self.expected_result_type = ErrorResult
163
- self.cause_regex = cause
164
- self.context_regex = context
185
+ self.err_or_filt_cause = cause
186
+ self.err_or_filt_context = context
187
+ self.err_or_filt_annotations = annotations
165
188
 
166
- def expect_filter_result(self, cause: str, context: str=None):
189
+ def expect_filter_result(self, cause: str, context: str = None, annotations: Dict = None):
167
190
  """
168
191
  A Sets the expected output of the action to a Filter Result
169
192
  :param cause: the expected filter cause (message)
170
- :param context: the expected error context (optional)
193
+ :param context: (Optional) the expected filter context
194
+ :param annotations: Dict: (Optional) the expected annotations
171
195
  """
172
196
  self.expected_result_type = FilterResult
173
- self.cause_regex = cause
174
- self.context_regex = context
197
+ self.err_or_filt_cause = cause
198
+ self.err_or_filt_context = context
199
+ self.err_or_filt_annotations = annotations
175
200
 
176
201
 
177
202
  class ActionTest(ABC):
@@ -183,17 +208,20 @@ class ActionTest(ABC):
183
208
  """
184
209
  self.content_service = InternalContentService()
185
210
  self.did = ""
186
- self.expected_outputs = []
187
211
  self.loaded_inputs = []
188
212
  self.package_name = package_name
189
213
  self.res_path = ""
214
+ self.context = None
190
215
 
191
- def __reset__(self):
216
+ def __reset__(self, did: str):
192
217
  self.content_service = InternalContentService()
193
- self.did = str(uuid.uuid4())
194
- self.expected_outputs = []
218
+ if did is None:
219
+ self.did = str(uuid.uuid4())
220
+ else:
221
+ self.did = did
195
222
  self.loaded_inputs = []
196
223
  self.res_path = ""
224
+ self.context = None
197
225
 
198
226
  def load_file(self, ioc: IOContent):
199
227
  file_res = self.res_path.joinpath(ioc.file_name)
@@ -207,68 +235,67 @@ class ActionTest(ABC):
207
235
 
208
236
  # Load inputs
209
237
  for input_ioc in test_case.inputs:
210
- if len(input_ioc.content_bytes) == 0:
238
+ if not input_ioc.no_content and len(input_ioc.content_bytes) == 0:
211
239
  self.loaded_inputs.append(LoadedContent(self.did, input_ioc, self.load_file(input_ioc)))
212
240
  else:
213
241
  self.loaded_inputs.append(LoadedContent(self.did, input_ioc, None))
214
242
 
215
- # Load expected outputs
216
- for output_ioc in test_case.outputs:
217
- if len(output_ioc.content_bytes) == 0:
218
- self.expected_outputs.append(LoadedContent(self.did, output_ioc, self.load_file(output_ioc)))
219
- else:
220
- self.expected_outputs.append(LoadedContent(self.did, output_ioc, None))
221
-
222
- def make_content_list(self, test_case: TestCaseBase):
243
+ def make_content_list(self):
223
244
  content_list = []
224
245
  for loaded_input in self.loaded_inputs:
225
- c = Content(name=loaded_input.name,
226
- segments=[loaded_input.segment],
227
- media_type=loaded_input.content_type,
246
+ c = Content(name=loaded_input.name, segments=[loaded_input.segment], media_type=loaded_input.content_type,
228
247
  content_service=self.content_service)
229
248
  content_list.append(c)
230
249
  loaded_input.content = c
231
250
 
232
251
  return content_list
233
252
 
234
- def make_df_msg(self, test_case: TestCaseBase):
235
- content_list = self.make_content_list(test_case)
253
+ def make_df_msgs(self, test_case: TestCaseBase):
254
+ content_list = self.make_content_list()
236
255
  self.content_service.load(self.loaded_inputs)
237
256
 
238
- return DeltaFileMessage(metadata=test_case.in_meta,
239
- content_list=content_list,
240
- domains=test_case.in_domains,
241
- enrichments=test_case.in_enrichments)
257
+ delta_file_messages = []
258
+
259
+ if test_case.join_meta is None:
260
+ delta_file_messages.append(DeltaFileMessage(metadata=test_case.in_meta, content_list=content_list))
261
+ else:
262
+ for index, content in enumerate(content_list):
263
+ delta_file_messages.append(DeltaFileMessage(
264
+ metadata=test_case.join_meta[index],
265
+ content_list=[content]))
266
+
267
+ return delta_file_messages
242
268
 
243
269
  def make_context(self, test_case: TestCaseBase):
244
270
  action_name = INGRESS_FLOW + "." + test_case.action.__class__.__name__
245
- return Context(did=self.did,
246
- action_flow=INGRESS_FLOW,
247
- action_name=action_name,
248
- source_filename=test_case.file_name,
249
- ingress_flow=INGRESS_FLOW,
250
- egress_flow=EGRESS_FLOW,
251
- system=SYSTEM,
252
- hostname=HOSTNAME,
253
- content_service=self.content_service,
254
- collect=None,
255
- collected_dids=None,
256
- logger=get_logger())
271
+ join = {} if test_case.join_meta else None
272
+ self.context = Context(
273
+ did=self.did,
274
+ delta_file_name=test_case.file_name,
275
+ data_source="DATASRC",
276
+ flow_name=INGRESS_FLOW,
277
+ flow_id="FLOWID",
278
+ action_name=action_name,
279
+ action_version="1.0",
280
+ hostname=HOSTNAME,
281
+ system_name=SYSTEM,
282
+ content_service=self.content_service,
283
+ saved_content=[],
284
+ join=join,
285
+ memo=test_case.in_memo,
286
+ logger=get_logger())
287
+ return self.context
257
288
 
258
289
  def make_event(self, test_case: TestCaseBase):
259
- return Event(
260
- delta_file_messages=[self.make_df_msg(test_case)],
261
- context=self.make_context(test_case),
262
- params=test_case.parameters,
263
- queue_name="",
264
- return_address="")
290
+ return Event(delta_file_messages=self.make_df_msgs(test_case), context=self.make_context(test_case),
291
+ params=test_case.parameters, queue_name="", return_address="")
265
292
 
266
293
  def call_action(self, test_case: TestCaseBase):
267
294
  self.get_contents(test_case)
268
295
  return test_case.action.execute_action(self.make_event(test_case))
269
296
 
270
297
  def run_and_check_result_type(self, test_case: TestCaseBase, result_type):
271
- self.__reset__()
298
+ self.__reset__(test_case.use_did)
272
299
  result = self.call_action(test_case)
273
300
 
274
301
  if not isinstance(result, result_type):
@@ -279,53 +306,60 @@ class ActionTest(ABC):
279
306
  def execute_error(self, test_case: TestCaseBase):
280
307
  result = self.run_and_check_result_type(test_case, ErrorResult)
281
308
  resp = result.response()
282
- assert resp['cause'] == test_case.cause_regex
283
- assert resp['context'] == test_case.context_regex
309
+ assert_equal_with_label(test_case.err_or_filt_cause, resp['cause'], "error cause")
310
+ if test_case.err_or_filt_context is not None:
311
+ assert_equal_with_label(test_case.err_or_filt_context, resp['context'], "error context")
312
+ if test_case.err_or_filt_annotations is not None:
313
+ assert_keys_and_values(test_case.err_or_filt_annotations, result.annotations)
284
314
 
285
- def execute_filter(self, test_case):
315
+ def execute_filter(self, test_case: TestCaseBase):
286
316
  result = self.run_and_check_result_type(test_case, FilterResult)
287
317
  resp = result.response()
288
- assert resp['message'] == test_case.cause_regex
318
+ assert_equal_with_label(test_case.err_or_filt_cause, resp['message'], "filter cause")
319
+ if test_case.err_or_filt_context is not None:
320
+ assert_equal_with_label(test_case.err_or_filt_context, resp['context'], "filter context")
321
+ if test_case.err_or_filt_annotations is not None:
322
+ assert_keys_and_values(test_case.err_or_filt_annotations, result.annotations)
289
323
 
290
324
  def execute(self, test_case: TestCaseBase):
291
- if isinstance(test_case.expected_result_type, ErrorResult.__class__):
325
+ if test_case.expected_result_type == ErrorResult:
292
326
  self.execute_error(test_case)
293
- elif isinstance(test_case.expected_result_type, FilterResult.__class__):
327
+ elif test_case.expected_result_type == FilterResult:
294
328
  self.execute_filter(test_case)
295
329
  else:
296
330
  raise ValueError(f"unknown type: {test_case.expected_result_type}")
297
331
 
298
- def compare_content_details(self, expected: LoadedContent, actual: Content):
332
+ @staticmethod
333
+ def compare_content_details(expected: LoadedContent, actual: Content):
299
334
  assert_equal(expected.content_type, actual.media_type)
300
335
  assert_equal(expected.name, actual.name)
301
336
 
302
- def compare_one_content(self, comparitor: CompareHelper, expected: LoadedContent, actual, index):
337
+ def compare_one_content(self, comparator: CompareHelper, expected: LoadedContent, actual, index):
303
338
  self.compare_content_details(expected, actual)
304
339
  seg_id = actual.segments[0].uuid
305
- comparitor.compare(
306
- expected.data,
307
- self.content_service.get_output(seg_id),
308
- f"Content[{index}]"
309
- )
310
-
311
- def compare_all_output(self, comparitor: CompareHelper, content: List):
312
- assert_equal_len(self.expected_outputs, content)
313
- for index, expected in enumerate(self.expected_outputs):
314
- self.compare_one_content(comparitor, expected, content[index], index)
315
-
316
- def compare_domains(self, comparitor: CompareHelper, expected_items: List[Dict], results: List[Dict]):
317
- assert_equal_len(expected_items, results)
318
- for index, expected in enumerate(expected_items):
319
- actual = results[index]
320
- assert_equal(expected['name'], actual['name'])
321
- assert_equal(expected['mediaType'], actual['mediaType'])
322
-
323
- expected_value = expected['value']
324
- if type(expected_value) == str:
325
- comparitor.compare(expected_value, actual['value'], f"Domain[{index}]")
326
- elif type(expected_value) == IOContent:
327
- expected_data = self.load_file(expected_value)
328
- comparitor.compare(expected_data, actual['value'], f"Domain[{index}]")
340
+ comparator.compare(expected.data, self.content_service.get_output(seg_id), f"Content[{index}]")
341
+
342
+ def compare_content_list(self, comparator: CompareHelper, expected_outputs: List[IOContent], content: List):
343
+ assert_equal_len(expected_outputs, content)
344
+ for index, expected_ioc in enumerate(expected_outputs):
345
+ if not expected_ioc.no_content and len(expected_ioc.content_bytes) == 0:
346
+ expected = LoadedContent(self.did, expected_ioc, self.load_file(expected_ioc))
329
347
  else:
330
- raise ValueError(
331
- f"unknown expected_value type: {type(expected_value)}")
348
+ expected = LoadedContent(self.did, expected_ioc, None)
349
+ self.compare_one_content(comparator, expected, content[index], index)
350
+
351
+ @staticmethod
352
+ def compare_one_metric(expected: Metric, result: Metric):
353
+ assert_equal_short(expected.name, result.name, "invalid metric name")
354
+ assert_equal_short(expected.value, result.value, f"invalid metric value for {expected.name}")
355
+ assert_keys_and_values(expected.tags, result.tags)
356
+
357
+ def compare_metrics(self, expected_metrics: List[Metric], results: List[Metric]):
358
+ if len(expected_metrics) > 0:
359
+ assert_equal_len_with_label(expected_metrics, results, "invalid metrics count")
360
+ for index, expected in enumerate(expected_metrics):
361
+ self.compare_one_metric(expected, results[index])
362
+
363
+ def has_saved_content__size(self, count: int):
364
+ assert_equal_with_label(
365
+ count, len(self.context.saved_content), "savedContent")
@@ -0,0 +1,101 @@
1
+ #
2
+ # DeltaFi - Data transformation and enrichment platform
3
+ #
4
+ # Copyright 2021-2025 DeltaFi Contributors <deltafi@deltafi.org>
5
+ #
6
+ # Licensed under the Apache License, Version 2.0 (the "License");
7
+ # you may not use this file except in compliance with the License.
8
+ # You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS,
14
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ # See the License for the specific language governing permissions and
16
+ # limitations under the License.
17
+ #
18
+
19
+ from typing import List
20
+
21
+ from deltafi.result import IngressResult, IngressResultItem, IngressStatusEnum
22
+ from .assertions import *
23
+ from .framework import TestCaseBase, ActionTest, IOContent
24
+
25
+
26
+ class TimedIngressTestCase(TestCaseBase):
27
+ def __init__(self, fields: Dict):
28
+ super().__init__(fields)
29
+ self.memo = None
30
+ self.results = []
31
+ self.execute_immediate = False
32
+ self.status = IngressStatusEnum.HEALTHY
33
+ self.status_message = None
34
+
35
+ def expect_ingress_result(self,
36
+ memo: str = None,
37
+ exec_immed: bool = False,
38
+ status: IngressStatusEnum = IngressStatusEnum.HEALTHY,
39
+ status_message: str = None):
40
+ self.expected_result_type = IngressResult
41
+ self.memo = memo
42
+ self.execute_immediate = exec_immed
43
+ self.status = status
44
+ self.status_message = status_message
45
+
46
+ def add_ingress_result_item(self, content: List[IOContent], metadata: Dict, name: str = None, annotations: Dict = None):
47
+ if annotations is None:
48
+ annotations = {}
49
+ self.results.append(
50
+ {
51
+ 'content': content,
52
+ 'metadata': metadata,
53
+ 'name': name,
54
+ 'annotations': annotations
55
+ }
56
+ )
57
+
58
+
59
+ class TimedIngressActionTest(ActionTest):
60
+ def __init__(self, package_name: str):
61
+ """
62
+ Provides structure for testing DeltaFi TimedIngress action
63
+ Args:
64
+ package_name: name of the actions package for finding resources
65
+ """
66
+ super().__init__(package_name)
67
+
68
+ def ingress(self, test_case: TimedIngressTestCase):
69
+ if test_case.expected_result_type == IngressResult:
70
+ self.expect_ingress_result(test_case)
71
+ else:
72
+ super().execute(test_case)
73
+
74
+ def expect_ingress_result(self, test_case: TimedIngressTestCase):
75
+ result = super().run_and_check_result_type(test_case, IngressResult)
76
+ self.assert_ingress_result(test_case, result)
77
+
78
+ def assert_ingress_result(self, test_case: TimedIngressTestCase, result: IngressResult):
79
+ assert_equal_short(test_case.memo, result.memo, "invalid memo")
80
+ assert_equal_short(test_case.execute_immediate, result.execute_immediate, "invalid execute_immediate")
81
+ assert_equal_short(test_case.status, result.status, "invalid status")
82
+ assert_equal_with_label(test_case.status_message, result.status_message, "invalid status_message")
83
+
84
+ assert_equal_len_with_label(test_case.results, result.ingress_result_items, "item count mismatch")
85
+ for index, ingress_item in enumerate(result.ingress_result_items):
86
+ self.compare_one_ingress_item(test_case, ingress_item, index)
87
+ expected = test_case.results[index]
88
+ if 'name' in expected:
89
+ assert_equal_with_label(expected["name"], ingress_item.delta_file_name, f"name[{index}]")
90
+
91
+ def compare_one_ingress_item(self, test_case: TimedIngressTestCase, result: IngressResultItem, index: int):
92
+ expected = test_case.results[index]
93
+
94
+ # Check output
95
+ self.compare_content_list(test_case.compare_tool, expected['content'], result.content)
96
+
97
+ # Check metadata
98
+ assert_keys_and_values(expected['metadata'], result.metadata)
99
+
100
+ # Check annotations
101
+ assert_keys_and_values(expected['annotations'], result.annotations)
@@ -1,7 +1,7 @@
1
1
  #
2
2
  # DeltaFi - Data transformation and enrichment platform
3
3
  #
4
- # Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
4
+ # Copyright 2021-2025 DeltaFi Contributors <deltafi@deltafi.org>
5
5
  #
6
6
  # Licensed under the Apache License, Version 2.0 (the "License");
7
7
  # you may not use this file except in compliance with the License.
@@ -18,24 +18,33 @@
18
18
 
19
19
  from typing import List
20
20
 
21
- from deltafi.result import TransformResult
22
-
21
+ from deltafi.result import TransformResult, TransformResults
23
22
  from .assertions import *
24
- from .framework import TestCaseBase, ActionTest
23
+ from .framework import TestCaseBase, ActionTest, IOContent
25
24
 
26
25
 
27
26
  class TransformTestCase(TestCaseBase):
28
27
  def __init__(self, fields: Dict):
29
28
  super().__init__(fields)
30
- self.metadata = {}
31
- self.delete_metadata_keys = []
32
- self.annotations = {}
29
+ self.results = []
33
30
 
34
- def expect_transform_result(self, metadata: Dict, delete_metadata_keys: List[str], annotations: Dict):
31
+ def expect_transform_result(self):
35
32
  self.expected_result_type = TransformResult
36
- self.metadata = metadata
37
- self.delete_metadata_keys = delete_metadata_keys
38
- self.annotations = annotations
33
+
34
+ def expect_transform_results(self):
35
+ self.expected_result_type = TransformResults
36
+
37
+ def add_transform_result(self, content: List[IOContent], metadata: Dict, delete_metadata_keys: List[str],
38
+ annotations: Dict, name: str = None):
39
+ self.results.append(
40
+ {
41
+ 'content': content,
42
+ 'metadata': metadata,
43
+ 'delete_metadata_keys': delete_metadata_keys,
44
+ 'annotations': annotations,
45
+ 'name': name
46
+ }
47
+ )
39
48
 
40
49
 
41
50
  class TransformActionTest(ActionTest):
@@ -50,6 +59,8 @@ class TransformActionTest(ActionTest):
50
59
  def transform(self, test_case: TransformTestCase):
51
60
  if test_case.expected_result_type == TransformResult:
52
61
  self.expect_transform_result(test_case)
62
+ elif test_case.expected_result_type == TransformResults:
63
+ self.expect_transform_results(test_case)
53
64
  else:
54
65
  super().execute(test_case)
55
66
 
@@ -57,16 +68,35 @@ class TransformActionTest(ActionTest):
57
68
  result = super().run_and_check_result_type(test_case, TransformResult)
58
69
  self.assert_transform_result(test_case, result)
59
70
 
71
+ def expect_transform_results(self, test_case: TransformTestCase):
72
+ result = super().run_and_check_result_type(test_case, TransformResults)
73
+ self.assert_transform_results(test_case, result)
74
+
75
+ def assert_transform_results(self, test_case: TransformTestCase, result: TransformResults):
76
+ assert_equal_len_with_label(test_case.results, result.child_results, "invalid child count")
77
+ for index, child_result in enumerate(result.child_results):
78
+ self.compare_one_transform_result(test_case, child_result, index)
79
+ expected = test_case.results[index]
80
+ if 'name' in expected:
81
+ assert_equal_with_label(expected["name"], child_result.delta_file_name, f"name[{index}]")
82
+
60
83
  def assert_transform_result(self, test_case: TransformTestCase, result: TransformResult):
84
+ # Check metrics
85
+ self.compare_metrics(test_case.expected_metrics, result.metrics)
86
+ self.compare_one_transform_result(test_case, result, 0)
87
+
88
+ def compare_one_transform_result(self, test_case: TransformTestCase, result: TransformResult, index: int):
89
+ expected = test_case.results[index]
90
+
61
91
  # Check output
62
- self.compare_all_output(test_case.compare_tool, result.content)
92
+ self.compare_content_list(test_case.compare_tool, expected['content'], result.content)
63
93
 
64
94
  # Check metadata
65
- assert_keys_and_values(test_case.metadata, result.metadata)
95
+ assert_keys_and_values(expected['metadata'], result.metadata)
66
96
 
67
97
  # Check deleted metadata
68
- for key in test_case.delete_metadata_keys:
98
+ for key in expected['delete_metadata_keys']:
69
99
  assert_key_in(key, result.delete_metadata_keys)
70
100
 
71
101
  # Check annotations
72
- assert_keys_and_values(test_case.annotations, result.annotations)
102
+ assert_keys_and_values(expected['annotations'], result.annotations)
@@ -1,31 +1,32 @@
1
- Metadata-Version: 2.1
1
+ Metadata-Version: 2.3
2
2
  Name: deltafi
3
- Version: 2.0rc1705080991758
3
+ Version: 2.4.0
4
4
  Summary: SDK for DeltaFi plugins and actions
5
5
  License: Apache License, Version 2.0
6
6
  Keywords: deltafi
7
7
  Author: DeltaFi
8
8
  Author-email: deltafi@systolic.com
9
- Requires-Python: >=3.7,<4.0
9
+ Requires-Python: >=3.9,<4.0
10
10
  Classifier: Development Status :: 4 - Beta
11
11
  Classifier: Intended Audience :: Developers
12
12
  Classifier: License :: OSI Approved :: Apache Software License
13
13
  Classifier: License :: Other/Proprietary License
14
14
  Classifier: Operating System :: OS Independent
15
15
  Classifier: Programming Language :: Python :: 3
16
- Classifier: Programming Language :: Python :: 3.7
17
- Classifier: Programming Language :: Python :: 3.8
18
16
  Classifier: Programming Language :: Python :: 3.9
19
17
  Classifier: Programming Language :: Python :: 3.10
20
18
  Classifier: Programming Language :: Python :: 3.11
19
+ Classifier: Programming Language :: Python :: 3.12
20
+ Classifier: Programming Language :: Python :: 3.13
21
21
  Classifier: Topic :: Software Development
22
- Requires-Dist: deepdiff (>=6.3.1)
23
- Requires-Dist: json-logging (>=1.3.0)
24
- Requires-Dist: minio (>=7.1.17)
25
- Requires-Dist: pydantic (>=2.4.2)
26
- Requires-Dist: redis (>=5.0.1)
27
- Requires-Dist: requests (>=2.31.0)
28
- Requires-Dist: urllib3 (>=2.0.6)
22
+ Requires-Dist: PyYAML (==6.0.2)
23
+ Requires-Dist: deepdiff (==6.7.1)
24
+ Requires-Dist: json-logging (==1.3.0)
25
+ Requires-Dist: minio (==7.2.10)
26
+ Requires-Dist: pydantic (==2.9.2)
27
+ Requires-Dist: redis (==5.2.0)
28
+ Requires-Dist: requests (==2.32.3)
29
+ Requires-Dist: urllib3 (==2.2.3)
29
30
  Project-URL: Bug Reports, https://chat.deltafi.org/deltafi/channels/bug-reports
30
31
  Project-URL: Documentation, https://docs.deltafi.org/#/
31
32
  Project-URL: Source Code, https://gitlab.com/deltafi/deltafi