deltafi 2.0rc1705080991758__py3-none-any.whl → 2.0rc1713530263087__py3-none-any.whl

deltafi/action.py

@@ -110,7 +110,7 @@ class TimedIngressAction(Action, ABC):
 
 class TransformAction(Action, ABC):
     def __init__(self, description: str):
-        super().__init__(ActionType.TRANSFORM, description, (TransformResult, ErrorResult, FilterResult))
+        super().__init__(ActionType.TRANSFORM, description, (TransformResult, TransformResults, ErrorResult, FilterResult))
 
     def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
         return TransformInput(content=delta_file_message.content_list, metadata=delta_file_message.metadata)

deltafi/domain.py

@@ -296,41 +296,17 @@ class Content:
                        content_service=content_service)
 
 
-class Domain(NamedTuple):
-    name: str
-    value: str
-    media_type: str
-
-    @classmethod
-    def from_dict(cls, domain: dict):
-        name = domain['name']
-        if 'value' in domain:
-            value = domain['value']
-        else:
-            value = None
-        media_type = domain['mediaType']
-        return Domain(name=name,
-                      value=value,
-                      media_type=media_type)
-
-
 class DeltaFileMessage(NamedTuple):
     metadata: Dict[str, str]
     content_list: List[Content]
-    domains: List[Domain]
-    enrichments: List[Domain]
 
     @classmethod
     def from_dict(cls, delta_file_message: dict, content_service: ContentService):
         metadata = delta_file_message['metadata']
         content_list = [Content.from_dict(content, content_service) for content in delta_file_message['contentList']]
-        domains = [Domain.from_dict(domain) for domain in delta_file_message['domains']] if 'domains' in delta_file_message else []
-        enrichments = [Domain.from_dict(domain) for domain in delta_file_message['enrichments']] if 'enrichments' in delta_file_message else []
 
         return DeltaFileMessage(metadata=metadata,
-                                content_list=content_list,
-                                domains=domains,
-                                enrichments=enrichments)
+                                content_list=content_list)
 
 
 class Event(NamedTuple):

deltafi/exception.py

@@ -23,16 +23,6 @@ class ExpectedContentException(RuntimeError):
         self.size = size
 
 
-class MissingDomainException(RuntimeError):
-    def __init__(self, name):
-        self.name = name
-
-
-class MissingEnrichmentException(RuntimeError):
-    def __init__(self, name):
-        self.name = name
-
-
 class MissingMetadataException(RuntimeError):
     def __init__(self, key):
         self.key = key

deltafi/plugin.py

@@ -34,8 +34,7 @@ from importlib import metadata
 import requests
 from deltafi.actioneventqueue import ActionEventQueue
 from deltafi.domain import Event, ActionExecution
-from deltafi.exception import ExpectedContentException, MissingDomainException, MissingEnrichmentException, \
-    MissingMetadataException
+from deltafi.exception import ExpectedContentException, MissingMetadataException
 from deltafi.logger import get_logger
 from deltafi.result import ErrorResult
 from deltafi.storage import ContentService
@@ -275,14 +274,6 @@ class Plugin(object):
                                          f"Action attempted to look up element {e.index + 1} (index {e.index}) from "
                                          f"content list of size {e.size}",
                                          f"{str(e)}\n{traceback.format_exc()}")
-                except MissingDomainException as e:
-                    result = ErrorResult(event.context,
-                                         f"Action attempted to access domain {e.name}, which does not exist",
-                                         f"{str(e)}\n{traceback.format_exc()}")
-                except MissingEnrichmentException as e:
-                    result = ErrorResult(event.context,
-                                         f"Action attempted to access enrichment {e.name}, which does not exist",
-                                         f"{str(e)}\n{traceback.format_exc()}")
                 except MissingMetadataException as e:
                     result = ErrorResult(event.context,
                                          f"Missing metadata with key {e.key}",

deltafi/result.py

@@ -17,9 +17,9 @@
 #
 
 import abc
-from enum import Enum
 import uuid
-from typing import Dict, List
+from enum import Enum
+from typing import NamedTuple
 
 from deltafi.domain import Content, Context
 from deltafi.metric import Metric
@@ -44,6 +44,7 @@ class Result:
 
     def add_metric(self, metric: Metric):
         self.metrics.append(metric)
+        return self
 
 
 class EgressResult(Result):
@@ -76,7 +77,7 @@ class ErrorResult(Result):
 
 
 class FilterResult(Result):
-    def __init__(self, context: Context, filtered_cause: str, filtered_context: str=None):
+    def __init__(self, context: Context, filtered_cause: str, filtered_context: str = None):
         super().__init__('filter', 'FILTER', context)
         self.filtered_cause = filtered_cause
         self.filtered_context = filtered_context
@@ -178,8 +179,8 @@ class TransformResult(Result):
     def __init__(self, context: Context):
         super().__init__('transform', 'TRANSFORM', context)
         self.content = []
-        self.metadata = {}
         self.annotations = {}
+        self.metadata = {}
         self.delete_metadata_keys = []
 
     # content can be a single Content or a List[Content]
@@ -220,10 +221,41 @@ class TransformResult(Result):
         self.delete_metadata_keys.append(key)
         return self
 
-    def response(self):
+    def json(self):
         return {
             'content': [content.json() for content in self.content],
-            'metadata': self.metadata,
             'annotations': self.annotations,
+            'metadata': self.metadata,
             'deleteMetadataKeys': self.delete_metadata_keys
         }
+
+    def response(self):
+        return [self.json()]
+
+
+class NamedTransformResult(NamedTuple):
+    result: TransformResult
+    name: str
+
+    def json(self):
+        j = self.result.json()
+        if self.name is not None:
+            j['name'] = self.name
+        return j
+
+
+class TransformResults(Result):
+    def __init__(self, context: Context):
+        super().__init__('transform', 'TRANSFORM', context)
+        self.named_results = []
+
+    def add_result(self, result: TransformResult, name: str = None):
+        self.named_results.append(NamedTransformResult(result, name))
+        return self
+
+    def response(self):
+        transform_events = []
+        for named_result in self.named_results:
+            json_dict = named_result.json()
+            transform_events.append(json_dict)
+        return transform_events

deltafi/test_kit/domain.py

@@ -0,0 +1,59 @@
+#
+#    DeltaFi - Data transformation and enrichment platform
+#
+#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+from deltafi.result import DomainResult
+
+from .assertions import *
+from .framework import TestCaseBase, ActionTest
+
+
+class DomainTestCase(TestCaseBase):
+    def __init__(self, fields: Dict):
+        super().__init__(fields)
+        self.annotations = {}
+
+    def expect_domain_result(self, annotations: Dict):
+        self.expected_result_type = DomainResult
+        self.annotations = annotations
+
+
+class DomainActionTest(ActionTest):
+    def __init__(self, package_name: str):
+        """
+        Provides structure for testing DeltaFi Domain action
+        Args:
+            package_name: name of the actions package for finding resources
+        """
+        super().__init__(package_name)
+
+    def domain(self, test_case: DomainTestCase):
+        if test_case.expected_result_type == DomainResult:
+            self.expect_domain_result(test_case)
+        else:
+            super().execute(test_case)
+
+    def expect_domain_result(self, test_case: DomainTestCase):
+        result = super().run_and_check_result_type(test_case, DomainResult)
+        self.assert_domain_result(test_case, result)
+
+    def assert_domain_result(self, test_case: DomainTestCase, result: DomainResult):
+        # Check metrics
+        self.compare_metrics(test_case.expected_metrics, result.metrics)
+
+        # Check annotations
+        assert_keys_and_values(test_case.annotations, result.annotations)

deltafi/test_kit/egress.py

@@ -0,0 +1,54 @@
+#
+#    DeltaFi - Data transformation and enrichment platform
+#
+#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+from deltafi.result import EgressResult
+
+from .assertions import *
+from .framework import TestCaseBase, ActionTest
+
+
+class EgressTestCase(TestCaseBase):
+    def __init__(self, fields: Dict):
+        super().__init__(fields)
+
+    def expect_egress_result(self):
+        self.expected_result_type = EgressResult
+
+
+class EgressActionTest(ActionTest):
+    def __init__(self, package_name: str):
+        """
+        Provides structure for testing DeltaFi Egress action
+        Args:
+            package_name: name of the actions package for finding resources
+        """
+        super().__init__(package_name)
+
+    def egress(self, test_case: EgressTestCase):
+        if test_case.expected_result_type == EgressResult:
+            self.expect_egress_result(test_case)
+        else:
+            super().execute(test_case)
+
+    def expect_egress_result(self, test_case: EgressTestCase):
+        result = super().run_and_check_result_type(test_case, EgressResult)
+        self.assert_egress_result(test_case, result)
+
+    def assert_egress_result(self, test_case: EgressTestCase, result: EgressResult):
+        # Check metrics
+        self.compare_metrics(test_case.expected_metrics, result.metrics)

deltafi/test_kit/enrich.py

@@ -0,0 +1,70 @@
+#
+#    DeltaFi - Data transformation and enrichment platform
+#
+#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+from deltafi.result import EnrichResult
+
+from .assertions import *
+from .framework import TestCaseBase, ActionTest
+
+
+class EnrichTestCase(TestCaseBase):
+    def __init__(self, fields: Dict):
+        super().__init__(fields)
+        self.enrichments = []
+        self.annotations = {}
+
+    def expect_enrich_result(self, annotations: Dict):
+        self.expected_result_type = EnrichResult
+        self.annotations = annotations
+
+    def add_enrichment(self, name: str, value: str, media_type: str):
+        self.enrichments.append({'name': name, 'value': value, 'mediaType': media_type})
+
+
+class EnrichActionTest(ActionTest):
+    def __init__(self, package_name: str):
+        """
+        Provides structure for testing DeltaFi Enrich action
+        Args:
+            package_name: name of the actions package for finding resources
+        """
+        super().__init__(package_name)
+
+    def enrich(self, test_case: EnrichTestCase):
+        if test_case.expected_result_type == EnrichResult:
+            self.expect_enrich_result(test_case)
+        else:
+            super().execute(test_case)
+
+    def expect_enrich_result(self, test_case: EnrichTestCase):
+        result = super().run_and_check_result_type(test_case, EnrichResult)
+        self.assert_enrich_result(test_case, result)
+
+    def assert_enrich_result(self, test_case: EnrichTestCase, result: EnrichResult):
+        # Check metrics
+        self.compare_metrics(test_case.expected_metrics, result.metrics)
+
+        # Check annotations
+        assert_keys_and_values(test_case.annotations, result.annotations)
+
+        # Check enrichments
+        assert_equal_len(test_case.enrichments, result.enrichments)
+        if len(test_case.enrichments) > 0:
+            for index, expected in enumerate(test_case.enrichments):
+                actual = result.enrichments[index]
+                assert_equal(expected, actual)

deltafi/test_kit/framework.py

@@ -23,6 +23,7 @@ from typing import List
 
 from deltafi.domain import DeltaFileMessage, Event, Content, Context
 from deltafi.logger import get_logger
+from deltafi.metric import Metric
 from deltafi.result import ErrorResult, FilterResult
 from deltafi.storage import Segment
 
@@ -43,8 +44,7 @@ class IOContent:
         content_bytes (str): Bypass file read, and uses these bytes for content
     """
 
-    def __init__(self, file_name: str, content_name: str = None,
-                 content_type: str = None, offset: int = 0,
+    def __init__(self, file_name: str, content_name: str = None, content_type: str = None, offset: int = 0,
                  content_bytes: str = ""):
         self.file_name = file_name
         if content_name is None:
@@ -79,12 +79,8 @@ class LoadedContent:
             self.data = data
         else:
             self.data = ioc.content_bytes
-        self.segment = Segment.from_dict({
-            "uuid": str(uuid.uuid4()),
-            "offset": self.offset,
-            "size": len(self.data),
-            "did": did
-        })
+        self.segment = Segment.from_dict(
+            {"uuid": str(uuid.uuid4()), "offset": self.offset, "size": len(self.data), "did": did})
 
 
 class InternalContentService:
@@ -97,10 +93,7 @@ class InternalContentService:
             self.loaded_content[c.segment.uuid] = c
 
     def put_str(self, did: str, string_data: str):
-        segment = Segment(uuid=str(uuid.uuid4()),
-                          offset=0,
-                          size=len(string_data),
-                          did=did)
+        segment = Segment(uuid=str(uuid.uuid4()), offset=0, size=len(string_data), did=did)
         self.outputs[segment.uuid] = string_data
         return segment
 
@@ -124,8 +117,7 @@ class TestCaseBase(ABC):
         - inputs: (optional) List[IOContent]: input content to action
         - parameters: (optional) Dict: map of action input parameters
         - in_meta: (optional) Dict: map of metadata as input to action
-        - in_domains: (optional) List[Domain]: list of domains as input to action
-        - in_enrichments: (optional) List[Domain]: list of enrichments as input to action
+        - did: (optional): str: overrides random DID
         """
         if "action" in data:
             self.action = data["action"]
@@ -144,34 +136,41 @@ class TestCaseBase(ABC):
 
         self.inputs = data["inputs"] if "inputs" in data else []
         self.file_name = data["file_name"] if "file_name" in data else "filename"
-        self.outputs = data["outputs"] if "outputs" in data else []
         self.parameters = data["parameters"] if "parameters" in data else {}
         self.in_meta = data["in_meta"] if "in_meta" in data else {}
-        self.in_domains = data["in_domains"] if "in_domains" in data else []
-        self.in_enrichments = data["in_enrichments"] if "in_enrichments" in data else []
+        self.use_did = data["did"] if "did" in data else None
         self.expected_result_type = None
-        self.cause_regex = None
-        self.context_regex = None
+        self.err_or_filt_cause = None
+        self.err_or_filt_context = None
+        self.err_or_filt_annotations = None
+        self.expected_metrics = []
 
-    def expect_error_result(self, cause: str, context: str):
+    def add_metric(self, metric: Metric):
+        self.expected_metrics.append(metric)
+
+    def expect_error_result(self, cause: str, context: str, annotations: Dict = None):
         """
         A Sets the expected output of the action to an Error Result
         :param cause: the expected error cause
         :param context: the expected error context
+        :param annotations (Optional): Dict: the expected annotations
         """
         self.expected_result_type = ErrorResult
-        self.cause_regex = cause
-        self.context_regex = context
+        self.err_or_filt_cause = cause
+        self.err_or_filt_context = context
+        self.err_or_filt_annotations = annotations
 
-    def expect_filter_result(self, cause: str, context: str=None):
+    def expect_filter_result(self, cause: str, context: str = None, annotations: Dict = None):
         """
         A Sets the expected output of the action to a Filter Result
         :param cause: the expected filter cause (message)
-        :param context: the expected error context (optional)
+        :param context (Optional): the expected filter context
+        :param annotations (Optional): Dict: the expected annotations
         """
         self.expected_result_type = FilterResult
-        self.cause_regex = cause
-        self.context_regex = context
+        self.err_or_filt_cause = cause
+        self.err_or_filt_context = context
+        self.err_or_filt_annotations = annotations
 
 
 class ActionTest(ABC):
@@ -183,15 +182,27 @@ class ActionTest(ABC):
         """
         self.content_service = InternalContentService()
         self.did = ""
-        self.expected_outputs = []
         self.loaded_inputs = []
         self.package_name = package_name
         self.res_path = ""
 
-    def __reset__(self):
+    def __reset__(self, did: str):
         self.content_service = InternalContentService()
+<<<<<<< HEAD
+        if did is None:
+            self.did = str(uuid.uuid4())
+        else:
+            self.did = did
+        self.expected_outputs = []
+||||||| parent of 831733c7 (2.0 Refactor)
         self.did = str(uuid.uuid4())
         self.expected_outputs = []
+=======
+        if did is None:
+            self.did = str(uuid.uuid4())
+        else:
+            self.did = did
+>>>>>>> 831733c7 (2.0 Refactor)
         self.loaded_inputs = []
         self.res_path = ""
 
@@ -212,19 +223,10 @@ class ActionTest(ABC):
             else:
                 self.loaded_inputs.append(LoadedContent(self.did, input_ioc, None))
 
-        # Load expected outputs
-        for output_ioc in test_case.outputs:
-            if len(output_ioc.content_bytes) == 0:
-                self.expected_outputs.append(LoadedContent(self.did, output_ioc, self.load_file(output_ioc)))
-            else:
-                self.expected_outputs.append(LoadedContent(self.did, output_ioc, None))
-
     def make_content_list(self, test_case: TestCaseBase):
         content_list = []
         for loaded_input in self.loaded_inputs:
-            c = Content(name=loaded_input.name,
-                        segments=[loaded_input.segment],
-                        media_type=loaded_input.content_type,
+            c = Content(name=loaded_input.name, segments=[loaded_input.segment], media_type=loaded_input.content_type,
                         content_service=self.content_service)
             content_list.append(c)
             loaded_input.content = c
@@ -235,40 +237,48 @@ class ActionTest(ABC):
         content_list = self.make_content_list(test_case)
         self.content_service.load(self.loaded_inputs)
 
+<<<<<<< HEAD
+        return DeltaFileMessage(
+            metadata=test_case.in_meta,
+            content_list=content_list,
+            domains=test_case.in_domains,
+            enrichments=test_case.in_enrichments)
+||||||| parent of 831733c7 (2.0 Refactor)
         return DeltaFileMessage(metadata=test_case.in_meta,
                                 content_list=content_list,
                                 domains=test_case.in_domains,
                                 enrichments=test_case.in_enrichments)
+=======
+        return DeltaFileMessage(metadata=test_case.in_meta,
+                                content_list=content_list)
+>>>>>>> 831733c7 (2.0 Refactor)
 
     def make_context(self, test_case: TestCaseBase):
         action_name = INGRESS_FLOW + "." + test_case.action.__class__.__name__
-        return Context(did=self.did,
-                       action_flow=INGRESS_FLOW,
-                       action_name=action_name,
-                       source_filename=test_case.file_name,
-                       ingress_flow=INGRESS_FLOW,
-                       egress_flow=EGRESS_FLOW,
-                       system=SYSTEM,
-                       hostname=HOSTNAME,
-                       content_service=self.content_service,
-                       collect=None,
-                       collected_dids=None,
-                       logger=get_logger())
+        return Context(
+            did=self.did,
+            action_flow=INGRESS_FLOW,
+            action_name=action_name,
+            source_filename=test_case.file_name,
+            ingress_flow=INGRESS_FLOW,
+            egress_flow=EGRESS_FLOW,
+            system=SYSTEM,
+            hostname=HOSTNAME,
+            content_service=self.content_service,
+            collect=None,
+            collected_dids=None,
+            logger=get_logger())
 
     def make_event(self, test_case: TestCaseBase):
-        return Event(
-            delta_file_messages=[self.make_df_msg(test_case)],
-            context=self.make_context(test_case),
-            params=test_case.parameters,
-            queue_name="",
-            return_address="")
+        return Event(delta_file_messages=[self.make_df_msg(test_case)], context=self.make_context(test_case),
+                     params=test_case.parameters, queue_name="", return_address="")
 
     def call_action(self, test_case: TestCaseBase):
         self.get_contents(test_case)
         return test_case.action.execute_action(self.make_event(test_case))
 
     def run_and_check_result_type(self, test_case: TestCaseBase, result_type):
-        self.__reset__()
+        self.__reset__(test_case.use_did)
         result = self.call_action(test_case)
 
         if not isinstance(result, result_type):
@@ -279,18 +289,25 @@ class ActionTest(ABC):
     def execute_error(self, test_case: TestCaseBase):
         result = self.run_and_check_result_type(test_case, ErrorResult)
         resp = result.response()
-        assert resp['cause'] == test_case.cause_regex
-        assert resp['context'] == test_case.context_regex
+        assert_equal_with_label(test_case.err_or_filt_cause, resp['cause'], "error cause")
+        if test_case.err_or_filt_context is not None:
+            assert_equal_with_label(test_case.err_or_filt_context, resp['context'], "error context")
+        if test_case.err_or_filt_annotations is not None:
+            assert_keys_and_values(test_case.err_or_filt_annotations, result.annotations)
 
-    def execute_filter(self, test_case):
+    def execute_filter(self, test_case: TestCaseBase):
         result = self.run_and_check_result_type(test_case, FilterResult)
         resp = result.response()
-        assert resp['message'] == test_case.cause_regex
+        assert_equal_with_label(test_case.err_or_filt_cause, resp['message'], "filter cause")
+        if test_case.err_or_filt_context is not None:
+            assert_equal_with_label(test_case.err_or_filt_context, resp['context'], "filter context")
+        if test_case.err_or_filt_annotations is not None:
+            assert_keys_and_values(test_case.err_or_filt_annotations, result.annotations)
 
     def execute(self, test_case: TestCaseBase):
-        if isinstance(test_case.expected_result_type, ErrorResult.__class__):
+        if test_case.expected_result_type == ErrorResult:
             self.execute_error(test_case)
-        elif isinstance(test_case.expected_result_type, FilterResult.__class__):
+        elif test_case.expected_result_type == FilterResult:
             self.execute_filter(test_case)
         else:
             raise ValueError(f"unknown type: {test_case.expected_result_type}")
@@ -302,23 +319,21 @@ class ActionTest(ABC):
     def compare_one_content(self, comparitor: CompareHelper, expected: LoadedContent, actual, index):
         self.compare_content_details(expected, actual)
         seg_id = actual.segments[0].uuid
-        comparitor.compare(
-            expected.data,
-            self.content_service.get_output(seg_id),
-            f"Content[{index}]"
-        )
-
-    def compare_all_output(self, comparitor: CompareHelper, content: List):
-        assert_equal_len(self.expected_outputs, content)
-        for index, expected in enumerate(self.expected_outputs):
+        comparitor.compare(expected.data, self.content_service.get_output(seg_id), f"Content[{index}]")
+
+    def compare_content_list(self, comparitor: CompareHelper, expected_outputs: List[IOContent], content: List):
+        assert_equal_len(expected_outputs, content)
+        for index, expected_ioc in enumerate(expected_outputs):
+            if len(expected_ioc.content_bytes) == 0:
+                expected = LoadedContent(self.did, expected_ioc, self.load_file(output_ioc))
+            else:
+                expected = LoadedContent(self.did, expected_ioc, None)
             self.compare_one_content(comparitor, expected, content[index], index)
 
-    def compare_domains(self, comparitor: CompareHelper, expected_items: List[Dict], results: List[Dict]):
-        assert_equal_len(expected_items, results)
-        for index, expected in enumerate(expected_items):
-            actual = results[index]
-            assert_equal(expected['name'], actual['name'])
-            assert_equal(expected['mediaType'], actual['mediaType'])
+    def compare_one_metric(self, expected: Metric, result: Metric):
+        assert expected.name == result.name
+        assert_equal_with_label(expected.value, result.value, expected.name)
+        assert_keys_and_values(expected.tags, result.tags)
 
             expected_value = expected['value']
             if type(expected_value) == str:
@@ -327,5 +342,15 @@ class ActionTest(ABC):
                 expected_data = self.load_file(expected_value)
                 comparitor.compare(expected_data, actual['value'], f"Domain[{index}]")
             else:
-                raise ValueError(
-                    f"unknown expected_value type: {type(expected_value)}")
+                raise ValueError(f"unknown expected_value type: {type(expected_value)}")
+
+    def compare_one_metric(self, expected: Metric, result: Metric):
+        assert expected.name == result.name
+        assert_equal_with_label(expected.value, result.value, expected.name)
+        assert_keys_and_values(expected.tags, result.tags)
+
+    def compare_metrics(self, expected_metrics: List[Metric], results: List[Metric]):
+        if len(expected_metrics) > 0:
+            assert_equal_len(expected_metrics, results)
+            for index, expected in enumerate(expected_metrics):
+                self.compare_one_metric(expected, results[index])

deltafi/test_kit/transform.py

@@ -18,24 +18,34 @@
 
 from typing import List
 
-from deltafi.result import TransformResult
+from deltafi.result import TransformResult, TransformResults
 
 from .assertions import *
-from .framework import TestCaseBase, ActionTest
+from .framework import TestCaseBase, ActionTest, IOContent
 
 
 class TransformTestCase(TestCaseBase):
     def __init__(self, fields: Dict):
         super().__init__(fields)
-        self.metadata = {}
-        self.delete_metadata_keys = []
-        self.annotations = {}
+        self.results = []
 
-    def expect_transform_result(self, metadata: Dict, delete_metadata_keys: List[str], annotations: Dict):
+    def expect_transform_result(self):
         self.expected_result_type = TransformResult
-        self.metadata = metadata
-        self.delete_metadata_keys = delete_metadata_keys
-        self.annotations = annotations
+
+    def expect_transform_results(self):
+        self.expected_result_type = TransformResults
+
+    def add_transform_result(self, content: List[IOContent], metadata: Dict, delete_metadata_keys: List[str],
+                             annotations: Dict, name: str = None):
+        self.results.append(
+            {
+                'content': content,
+                'metadata': metadata,
+                'delete_metadata_keys': delete_metadata_keys,
+                'annotations': annotations,
+                'name': name
+            }
+        )
 
 
 class TransformActionTest(ActionTest):
@@ -50,6 +60,8 @@ class TransformActionTest(ActionTest):
     def transform(self, test_case: TransformTestCase):
         if test_case.expected_result_type == TransformResult:
             self.expect_transform_result(test_case)
+        elif test_case.expected_result_type == TransformResults:
+            self.expect_transform_results(test_case)
         else:
             super().execute(test_case)
 
@@ -57,16 +69,35 @@ class TransformActionTest(ActionTest):
         result = super().run_and_check_result_type(test_case, TransformResult)
         self.assert_transform_result(test_case, result)
 
+    def expect_transform_results(self, test_case: TransformTestCase):
+        result = super().run_and_check_result_type(test_case, TransformResults)
+        self.assert_transform_results(test_case, result)
+
+    def assert_transform_results(self, test_case: TransformTestCase, result: TransformResults):
+        assert_equal_len(test_case.results, result.named_results)
+        for index, named_result in enumerate(result.named_results):
+            self.compare_one_transform_result(test_case, named_result.result, index)
+            expected = test_case.results[index]
+            if 'name' in expected:
+                assert_equal_with_label(expected["name"], named_result.name, f"name[{index}]")
+
     def assert_transform_result(self, test_case: TransformTestCase, result: TransformResult):
+        # Check metrics
+        self.compare_metrics(test_case.expected_metrics, result.metrics)
+        self.compare_one_transform_result(test_case, result, 0)
+
+    def compare_one_transform_result(self, test_case: TransformTestCase, result: TransformResult, index: int):
+        expected = test_case.results[index]
+
         # Check output
-        self.compare_all_output(test_case.compare_tool, result.content)
+        self.compare_content_list(test_case.compare_tool, expected['content'], result.content)
 
         # Check metadata
-        assert_keys_and_values(test_case.metadata, result.metadata)
+        assert_keys_and_values(expected['metadata'], result.metadata)
 
         # Check deleted metadata
-        for key in test_case.delete_metadata_keys:
+        for key in expected['delete_metadata_keys']:
             assert_key_in(key, result.delete_metadata_keys)
 
         # Check annotations
-        assert_keys_and_values(test_case.annotations, result.annotations)
+        assert_keys_and_values(expected['annotations'], result.annotations)

deltafi/test_kit/validate.py

@@ -0,0 +1,54 @@
+#
+#    DeltaFi - Data transformation and enrichment platform
+#
+#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+from deltafi.result import ValidateResult
+
+from .assertions import *
+from .framework import TestCaseBase, ActionTest
+
+
+class ValidateTestCase(TestCaseBase):
+    def __init__(self, fields: Dict):
+        super().__init__(fields)
+
+    def expect_validate_result(self):
+        self.expected_result_type = ValidateResult
+
+
+class ValidateActionTest(ActionTest):
+    def __init__(self, package_name: str):
+        """
+        Provides structure for testing DeltaFi Validate action
+        Args:
+            package_name: name of the actions package for finding resources
+        """
+        super().__init__(package_name)
+
+    def validate(self, test_case: ValidateTestCase):
+        if test_case.expected_result_type == ValidateResult:
+            self.expect_validate_result(test_case)
+        else:
+            super().execute(test_case)
+
+    def expect_validate_result(self, test_case: ValidateTestCase):
+        result = super().run_and_check_result_type(test_case, ValidateResult)
+        self.assert_validate_result(test_case, result)
+
+    def assert_validate_result(self, test_case: ValidateTestCase, result: ValidateResult):
+        # Check metrics
+        self.compare_metrics(test_case.expected_metrics, result.metrics)

{deltafi-2.0rc1705080991758.dist-info → deltafi-2.0rc1713530263087.dist-info}/METADATA

@@ -1,31 +1,30 @@
 Metadata-Version: 2.1
 Name: deltafi
-Version: 2.0rc1705080991758
+Version: 2.0rc1713530263087
 Summary: SDK for DeltaFi plugins and actions
 License: Apache License, Version 2.0
 Keywords: deltafi
 Author: DeltaFi
 Author-email: deltafi@systolic.com
-Requires-Python: >=3.7,<4.0
+Requires-Python: >=3.9,<4.0
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: License :: Other/Proprietary License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.7
-Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Software Development
-Requires-Dist: deepdiff (>=6.3.1)
+Requires-Dist: deepdiff (>=6.7.1)
 Requires-Dist: json-logging (>=1.3.0)
-Requires-Dist: minio (>=7.1.17)
-Requires-Dist: pydantic (>=2.4.2)
+Requires-Dist: minio (>=7.2.3)
+Requires-Dist: pydantic (>=2.5.3)
 Requires-Dist: redis (>=5.0.1)
 Requires-Dist: requests (>=2.31.0)
-Requires-Dist: urllib3 (>=2.0.6)
+Requires-Dist: urllib3 (>=2.1.0)
 Project-URL: Bug Reports, https://chat.deltafi.org/deltafi/channels/bug-reports
 Project-URL: Documentation, https://docs.deltafi.org/#/
 Project-URL: Source Code, https://gitlab.com/deltafi/deltafi

deltafi-2.0rc1713530263087.dist-info/RECORD

@@ -0,0 +1,26 @@
+deltafi/__init__.py,sha256=qv6y7PpBG0ekTN9EuD9Lj8JYOLVqZA6tvHwgjplncAM,709
+deltafi/action.py,sha256=xxDQD-gye6gW9Q13Ylm51qiVDgqD36WMK191Gi7FEis,5323
+deltafi/actioneventqueue.py,sha256=mCRE1PFXy_KvaeezTeMze5N400CO1V06zEF8FFD6xZk,2847
+deltafi/actiontype.py,sha256=6OoZNEMcreD5f_QaGhTu8MEYLr3CalzcNu9XuFx40zg,865
+deltafi/domain.py,sha256=weZII1PfUOoCUpk3LfS8EXhX3ZQ73fFuoSwKgnyBX3Y,11301
+deltafi/exception.py,sha256=Z8LwxxTiTyuG3v7FAsjfz6cAqRAJFKK2q_2TbEDrIJE,943
+deltafi/genericmodel.py,sha256=8VXWgFMjIwG4o3rkcLOa3bQH0Nf-T1Kw621QtMw-4q0,1090
+deltafi/input.py,sha256=2qQcOWqz7lOXZIZipTz5XtcGESpjdnUAsZ7b9YqHw7M,1656
+deltafi/logger.py,sha256=q76R_Gn8BfcASH3Zy0V82m5ot34bjTnSELyKbjhvx3E,2136
+deltafi/metric.py,sha256=eIDjZQVO53KuAFoEtjLNFwqMrp_7BC0Td9GLD1tb6sE,967
+deltafi/plugin.py,sha256=5mx7Og6aREwjPDY4BknCnDsuWDg7mz8On6J0M8mQ16I,12294
+deltafi/result.py,sha256=rugAkh9NZdl0QDHzyxDoZbkE8dnJNAFLbqBNdIjjn_k,8177
+deltafi/storage.py,sha256=toq58EPZgzj2TfDF-YpFUmRnsWSjACA0KQAZzkM04xU,2740
+deltafi/test_kit/__init__.py,sha256=qv6y7PpBG0ekTN9EuD9Lj8JYOLVqZA6tvHwgjplncAM,709
+deltafi/test_kit/assertions.py,sha256=2eahqmbedhnHXMguiJHdaV-6sx4_jvCbxH4HSzx6PV8,1378
+deltafi/test_kit/compare_helpers.py,sha256=Oi07zsq7jlpAHxqjwHsea84acC3lAL3ooG0VC42lLAc,1581
+deltafi/test_kit/constants.py,sha256=epz0OS-qILcc8t7iIs5B3m2-wvZx8FpYpyb19ZsImbI,833
+deltafi/test_kit/domain.py,sha256=uqpDravlb6E3Ddm4QpRaJTdiVuQ2MMdZnzc4Oi9EZJA,2090
+deltafi/test_kit/egress.py,sha256=nF7YYak-_X35m7h2rSIcju86IJBf8cVaFHs7xw7-_b8,1899
+deltafi/test_kit/enrich.py,sha256=M682cqInVS4aipI0jOrR25x7m0ErrpjGpebXGZDPwxo,2587
+deltafi/test_kit/framework.py,sha256=m11V6JMZHn4oXnmB_pEgI5p7fZWCWAO7HjHLunSh5AA,14602
+deltafi/test_kit/transform.py,sha256=q9zQdZswTKwU5Om8E9Lwg0G5JI8LEXuRZjT9oam-Zoo,4085
+deltafi/test_kit/validate.py,sha256=NnXD5amOE_9a4Zr-v9DKYIv5f_HvwtmlI-IdFtVp7ko,1933
+deltafi-2.0rc1713530263087.dist-info/METADATA,sha256=M8bj2tsQIgAkHIKkYzYQ3eAT6A51colJHptMVKPJnMo,1446
+deltafi-2.0rc1713530263087.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+deltafi-2.0rc1713530263087.dist-info/RECORD,,

{deltafi-2.0rc1705080991758.dist-info → deltafi-2.0rc1713530263087.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 1.7.0
+Generator: poetry-core 1.9.0
 Root-Is-Purelib: true
 Tag: py3-none-any

deltafi-2.0rc1705080991758.dist-info/RECORD

@@ -1,22 +0,0 @@
-deltafi/__init__.py,sha256=qv6y7PpBG0ekTN9EuD9Lj8JYOLVqZA6tvHwgjplncAM,709
-deltafi/action.py,sha256=cjnRsmBNCSCosFipIS_UXMHKC5MRITVSdc6MIeYUyL4,5305
-deltafi/actioneventqueue.py,sha256=mCRE1PFXy_KvaeezTeMze5N400CO1V06zEF8FFD6xZk,2847
-deltafi/actiontype.py,sha256=6OoZNEMcreD5f_QaGhTu8MEYLr3CalzcNu9XuFx40zg,865
-deltafi/domain.py,sha256=M97_n6KsMmIxCh3h622jNj1n6c5iMEAOn-SFH36oLis,12161
-deltafi/exception.py,sha256=qQ2TY2QPtMU9t5RH8jmFo_cX98AJ-zOFWP_Wfm5U6qQ,1149
-deltafi/genericmodel.py,sha256=8VXWgFMjIwG4o3rkcLOa3bQH0Nf-T1Kw621QtMw-4q0,1090
-deltafi/input.py,sha256=2qQcOWqz7lOXZIZipTz5XtcGESpjdnUAsZ7b9YqHw7M,1656
-deltafi/logger.py,sha256=q76R_Gn8BfcASH3Zy0V82m5ot34bjTnSELyKbjhvx3E,2136
-deltafi/metric.py,sha256=eIDjZQVO53KuAFoEtjLNFwqMrp_7BC0Td9GLD1tb6sE,967
-deltafi/plugin.py,sha256=LTkAII2ur6BFylxnp80slzximkDyKr5bvHEzczR9C4U,12956
-deltafi/result.py,sha256=2r3BKNQntlMpFXZtYOm7skoLzBocDRBwtRP3oRkGrfc,7330
-deltafi/storage.py,sha256=toq58EPZgzj2TfDF-YpFUmRnsWSjACA0KQAZzkM04xU,2740
-deltafi/test_kit/__init__.py,sha256=qv6y7PpBG0ekTN9EuD9Lj8JYOLVqZA6tvHwgjplncAM,709
-deltafi/test_kit/assertions.py,sha256=2eahqmbedhnHXMguiJHdaV-6sx4_jvCbxH4HSzx6PV8,1378
-deltafi/test_kit/compare_helpers.py,sha256=Oi07zsq7jlpAHxqjwHsea84acC3lAL3ooG0VC42lLAc,1581
-deltafi/test_kit/constants.py,sha256=epz0OS-qILcc8t7iIs5B3m2-wvZx8FpYpyb19ZsImbI,833
-deltafi/test_kit/framework.py,sha256=R07AH4zxWXaJ3QNufGrtiOvdZrZKEoo2FtZLGmgAnRk,13062
-deltafi/test_kit/transform.py,sha256=x2TNTV39TmJQbVImm-sOfUlS_qaWcDRi5mfK4ZE0ONE,2608
-deltafi-2.0rc1705080991758.dist-info/METADATA,sha256=jNjBLNkWpeCzQl1GiXamnwcmszP1DjtQB8Cm8XOGfE8,1496
-deltafi-2.0rc1705080991758.dist-info/WHEEL,sha256=d2fvjOD7sXsVzChCqf0Ty0JbHKBaLYwDbGQDwQTnJ50,88
-deltafi-2.0rc1705080991758.dist-info/RECORD,,