deltafi 2.0rc1705080991758__tar.gz → 2.0rc1713530263087__tar.gz

{deltafi-2.0rc1705080991758 → deltafi-2.0rc1713530263087}/PKG-INFO

@@ -1,31 +1,30 @@
 Metadata-Version: 2.1
 Name: deltafi
-Version: 2.0rc1705080991758
+Version: 2.0rc1713530263087
 Summary: SDK for DeltaFi plugins and actions
 License: Apache License, Version 2.0
 Keywords: deltafi
 Author: DeltaFi
 Author-email: deltafi@systolic.com
-Requires-Python: >=3.7,<4.0
+Requires-Python: >=3.9,<4.0
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: License :: Other/Proprietary License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.7
-Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Software Development
-Requires-Dist: deepdiff (>=6.3.1)
+Requires-Dist: deepdiff (>=6.7.1)
 Requires-Dist: json-logging (>=1.3.0)
-Requires-Dist: minio (>=7.1.17)
-Requires-Dist: pydantic (>=2.4.2)
+Requires-Dist: minio (>=7.2.3)
+Requires-Dist: pydantic (>=2.5.3)
 Requires-Dist: redis (>=5.0.1)
 Requires-Dist: requests (>=2.31.0)
-Requires-Dist: urllib3 (>=2.0.6)
+Requires-Dist: urllib3 (>=2.1.0)
 Project-URL: Bug Reports, https://chat.deltafi.org/deltafi/channels/bug-reports
 Project-URL: Documentation, https://docs.deltafi.org/#/
 Project-URL: Source Code, https://gitlab.com/deltafi/deltafi

{deltafi-2.0rc1705080991758 → deltafi-2.0rc1713530263087}/deltafi/action.py

@@ -110,7 +110,7 @@ class TimedIngressAction(Action, ABC):
 
 class TransformAction(Action, ABC):
     def __init__(self, description: str):
-        super().__init__(ActionType.TRANSFORM, description, (TransformResult, ErrorResult, FilterResult))
+        super().__init__(ActionType.TRANSFORM, description, (TransformResult, TransformResults, ErrorResult, FilterResult))
 
     def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
         return TransformInput(content=delta_file_message.content_list, metadata=delta_file_message.metadata)

{deltafi-2.0rc1705080991758 → deltafi-2.0rc1713530263087}/deltafi/domain.py

@@ -296,41 +296,17 @@ class Content:
                        content_service=content_service)
 
 
-class Domain(NamedTuple):
-    name: str
-    value: str
-    media_type: str
-
-    @classmethod
-    def from_dict(cls, domain: dict):
-        name = domain['name']
-        if 'value' in domain:
-            value = domain['value']
-        else:
-            value = None
-        media_type = domain['mediaType']
-        return Domain(name=name,
-                      value=value,
-                      media_type=media_type)
-
-
 class DeltaFileMessage(NamedTuple):
     metadata: Dict[str, str]
     content_list: List[Content]
-    domains: List[Domain]
-    enrichments: List[Domain]
 
     @classmethod
     def from_dict(cls, delta_file_message: dict, content_service: ContentService):
         metadata = delta_file_message['metadata']
         content_list = [Content.from_dict(content, content_service) for content in delta_file_message['contentList']]
-        domains = [Domain.from_dict(domain) for domain in delta_file_message['domains']] if 'domains' in delta_file_message else []
-        enrichments = [Domain.from_dict(domain) for domain in delta_file_message['enrichments']] if 'enrichments' in delta_file_message else []
 
         return DeltaFileMessage(metadata=metadata,
-                                content_list=content_list,
-                                domains=domains,
-                                enrichments=enrichments)
+                                content_list=content_list)
 
 
 class Event(NamedTuple):

{deltafi-2.0rc1705080991758 → deltafi-2.0rc1713530263087}/deltafi/exception.py

@@ -23,16 +23,6 @@ class ExpectedContentException(RuntimeError):
         self.size = size
 
 
-class MissingDomainException(RuntimeError):
-    def __init__(self, name):
-        self.name = name
-
-
-class MissingEnrichmentException(RuntimeError):
-    def __init__(self, name):
-        self.name = name
-
-
 class MissingMetadataException(RuntimeError):
     def __init__(self, key):
         self.key = key

{deltafi-2.0rc1705080991758 → deltafi-2.0rc1713530263087}/deltafi/plugin.py

@@ -34,8 +34,7 @@ from importlib import metadata
 import requests
 from deltafi.actioneventqueue import ActionEventQueue
 from deltafi.domain import Event, ActionExecution
-from deltafi.exception import ExpectedContentException, MissingDomainException, MissingEnrichmentException, \
-    MissingMetadataException
+from deltafi.exception import ExpectedContentException, MissingMetadataException
 from deltafi.logger import get_logger
 from deltafi.result import ErrorResult
 from deltafi.storage import ContentService
@@ -275,14 +274,6 @@ class Plugin(object):
                                          f"Action attempted to look up element {e.index + 1} (index {e.index}) from "
                                          f"content list of size {e.size}",
                                          f"{str(e)}\n{traceback.format_exc()}")
-                except MissingDomainException as e:
-                    result = ErrorResult(event.context,
-                                         f"Action attempted to access domain {e.name}, which does not exist",
-                                         f"{str(e)}\n{traceback.format_exc()}")
-                except MissingEnrichmentException as e:
-                    result = ErrorResult(event.context,
-                                         f"Action attempted to access enrichment {e.name}, which does not exist",
-                                         f"{str(e)}\n{traceback.format_exc()}")
                 except MissingMetadataException as e:
                     result = ErrorResult(event.context,
                                          f"Missing metadata with key {e.key}",

{deltafi-2.0rc1705080991758 → deltafi-2.0rc1713530263087}/deltafi/result.py

@@ -17,9 +17,9 @@
 #
 
 import abc
-from enum import Enum
 import uuid
-from typing import Dict, List
+from enum import Enum
+from typing import NamedTuple
 
 from deltafi.domain import Content, Context
 from deltafi.metric import Metric
@@ -44,6 +44,7 @@ class Result:
 
     def add_metric(self, metric: Metric):
         self.metrics.append(metric)
+        return self
 
 
 class EgressResult(Result):
@@ -76,7 +77,7 @@ class ErrorResult(Result):
 
 
 class FilterResult(Result):
-    def __init__(self, context: Context, filtered_cause: str, filtered_context: str=None):
+    def __init__(self, context: Context, filtered_cause: str, filtered_context: str = None):
         super().__init__('filter', 'FILTER', context)
         self.filtered_cause = filtered_cause
         self.filtered_context = filtered_context
@@ -178,8 +179,8 @@ class TransformResult(Result):
     def __init__(self, context: Context):
         super().__init__('transform', 'TRANSFORM', context)
         self.content = []
-        self.metadata = {}
         self.annotations = {}
+        self.metadata = {}
         self.delete_metadata_keys = []
 
     # content can be a single Content or a List[Content]
@@ -220,10 +221,41 @@ class TransformResult(Result):
         self.delete_metadata_keys.append(key)
         return self
 
-    def response(self):
+    def json(self):
         return {
             'content': [content.json() for content in self.content],
-            'metadata': self.metadata,
             'annotations': self.annotations,
+            'metadata': self.metadata,
             'deleteMetadataKeys': self.delete_metadata_keys
         }
+
+    def response(self):
+        return [self.json()]
+
+
+class NamedTransformResult(NamedTuple):
+    result: TransformResult
+    name: str
+
+    def json(self):
+        j = self.result.json()
+        if self.name is not None:
+            j['name'] = self.name
+        return j
+
+
+class TransformResults(Result):
+    def __init__(self, context: Context):
+        super().__init__('transform', 'TRANSFORM', context)
+        self.named_results = []
+
+    def add_result(self, result: TransformResult, name: str = None):
+        self.named_results.append(NamedTransformResult(result, name))
+        return self
+
+    def response(self):
+        transform_events = []
+        for named_result in self.named_results:
+            json_dict = named_result.json()
+            transform_events.append(json_dict)
+        return transform_events

deltafi-2.0rc1713530263087/deltafi/test_kit/domain.py

@@ -0,0 +1,59 @@
+#
+#    DeltaFi - Data transformation and enrichment platform
+#
+#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+from deltafi.result import DomainResult
+
+from .assertions import *
+from .framework import TestCaseBase, ActionTest
+
+
+class DomainTestCase(TestCaseBase):
+    def __init__(self, fields: Dict):
+        super().__init__(fields)
+        self.annotations = {}
+
+    def expect_domain_result(self, annotations: Dict):
+        self.expected_result_type = DomainResult
+        self.annotations = annotations
+
+
+class DomainActionTest(ActionTest):
+    def __init__(self, package_name: str):
+        """
+        Provides structure for testing DeltaFi Domain action
+        Args:
+            package_name: name of the actions package for finding resources
+        """
+        super().__init__(package_name)
+
+    def domain(self, test_case: DomainTestCase):
+        if test_case.expected_result_type == DomainResult:
+            self.expect_domain_result(test_case)
+        else:
+            super().execute(test_case)
+
+    def expect_domain_result(self, test_case: DomainTestCase):
+        result = super().run_and_check_result_type(test_case, DomainResult)
+        self.assert_domain_result(test_case, result)
+
+    def assert_domain_result(self, test_case: DomainTestCase, result: DomainResult):
+        # Check metrics
+        self.compare_metrics(test_case.expected_metrics, result.metrics)
+
+        # Check annotations
+        assert_keys_and_values(test_case.annotations, result.annotations)

deltafi-2.0rc1713530263087/deltafi/test_kit/egress.py

@@ -0,0 +1,54 @@
+#
+#    DeltaFi - Data transformation and enrichment platform
+#
+#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+from deltafi.result import EgressResult
+
+from .assertions import *
+from .framework import TestCaseBase, ActionTest
+
+
+class EgressTestCase(TestCaseBase):
+    def __init__(self, fields: Dict):
+        super().__init__(fields)
+
+    def expect_egress_result(self):
+        self.expected_result_type = EgressResult
+
+
+class EgressActionTest(ActionTest):
+    def __init__(self, package_name: str):
+        """
+        Provides structure for testing DeltaFi Egress action
+        Args:
+            package_name: name of the actions package for finding resources
+        """
+        super().__init__(package_name)
+
+    def egress(self, test_case: EgressTestCase):
+        if test_case.expected_result_type == EgressResult:
+            self.expect_egress_result(test_case)
+        else:
+            super().execute(test_case)
+
+    def expect_egress_result(self, test_case: EgressTestCase):
+        result = super().run_and_check_result_type(test_case, EgressResult)
+        self.assert_egress_result(test_case, result)
+
+    def assert_egress_result(self, test_case: EgressTestCase, result: EgressResult):
+        # Check metrics
+        self.compare_metrics(test_case.expected_metrics, result.metrics)

deltafi-2.0rc1713530263087/deltafi/test_kit/enrich.py

@@ -0,0 +1,70 @@
+#
+#    DeltaFi - Data transformation and enrichment platform
+#
+#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+from deltafi.result import EnrichResult
+
+from .assertions import *
+from .framework import TestCaseBase, ActionTest
+
+
+class EnrichTestCase(TestCaseBase):
+    def __init__(self, fields: Dict):
+        super().__init__(fields)
+        self.enrichments = []
+        self.annotations = {}
+
+    def expect_enrich_result(self, annotations: Dict):
+        self.expected_result_type = EnrichResult
+        self.annotations = annotations
+
+    def add_enrichment(self, name: str, value: str, media_type: str):
+        self.enrichments.append({'name': name, 'value': value, 'mediaType': media_type})
+
+
+class EnrichActionTest(ActionTest):
+    def __init__(self, package_name: str):
+        """
+        Provides structure for testing DeltaFi Enrich action
+        Args:
+            package_name: name of the actions package for finding resources
+        """
+        super().__init__(package_name)
+
+    def enrich(self, test_case: EnrichTestCase):
+        if test_case.expected_result_type == EnrichResult:
+            self.expect_enrich_result(test_case)
+        else:
+            super().execute(test_case)
+
+    def expect_enrich_result(self, test_case: EnrichTestCase):
+        result = super().run_and_check_result_type(test_case, EnrichResult)
+        self.assert_enrich_result(test_case, result)
+
+    def assert_enrich_result(self, test_case: EnrichTestCase, result: EnrichResult):
+        # Check metrics
+        self.compare_metrics(test_case.expected_metrics, result.metrics)
+
+        # Check annotations
+        assert_keys_and_values(test_case.annotations, result.annotations)
+
+        # Check enrichments
+        assert_equal_len(test_case.enrichments, result.enrichments)
+        if len(test_case.enrichments) > 0:
+            for index, expected in enumerate(test_case.enrichments):
+                actual = result.enrichments[index]
+                assert_equal(expected, actual)

{deltafi-2.0rc1705080991758 → deltafi-2.0rc1713530263087}/deltafi/test_kit/framework.py

@@ -23,6 +23,7 @@ from typing import List
 
 from deltafi.domain import DeltaFileMessage, Event, Content, Context
 from deltafi.logger import get_logger
+from deltafi.metric import Metric
 from deltafi.result import ErrorResult, FilterResult
 from deltafi.storage import Segment
 
@@ -43,8 +44,7 @@ class IOContent:
         content_bytes (str): Bypass file read, and uses these bytes for content
     """
 
-    def __init__(self, file_name: str, content_name: str = None,
-                 content_type: str = None, offset: int = 0,
+    def __init__(self, file_name: str, content_name: str = None, content_type: str = None, offset: int = 0,
                  content_bytes: str = ""):
         self.file_name = file_name
         if content_name is None:
@@ -79,12 +79,8 @@ class LoadedContent:
             self.data = data
         else:
             self.data = ioc.content_bytes
-        self.segment = Segment.from_dict({
-            "uuid": str(uuid.uuid4()),
-            "offset": self.offset,
-            "size": len(self.data),
-            "did": did
-        })
+        self.segment = Segment.from_dict(
+            {"uuid": str(uuid.uuid4()), "offset": self.offset, "size": len(self.data), "did": did})
 
 
 class InternalContentService:
@@ -97,10 +93,7 @@ class InternalContentService:
             self.loaded_content[c.segment.uuid] = c
 
     def put_str(self, did: str, string_data: str):
-        segment = Segment(uuid=str(uuid.uuid4()),
-                          offset=0,
-                          size=len(string_data),
-                          did=did)
+        segment = Segment(uuid=str(uuid.uuid4()), offset=0, size=len(string_data), did=did)
         self.outputs[segment.uuid] = string_data
         return segment
 
@@ -124,8 +117,7 @@ class TestCaseBase(ABC):
         - inputs: (optional) List[IOContent]: input content to action
         - parameters: (optional) Dict: map of action input parameters
         - in_meta: (optional) Dict: map of metadata as input to action
-        - in_domains: (optional) List[Domain]: list of domains as input to action
-        - in_enrichments: (optional) List[Domain]: list of enrichments as input to action
+        - did: (optional): str: overrides random DID
         """
         if "action" in data:
             self.action = data["action"]
@@ -144,34 +136,41 @@ class TestCaseBase(ABC):
 
         self.inputs = data["inputs"] if "inputs" in data else []
         self.file_name = data["file_name"] if "file_name" in data else "filename"
-        self.outputs = data["outputs"] if "outputs" in data else []
         self.parameters = data["parameters"] if "parameters" in data else {}
         self.in_meta = data["in_meta"] if "in_meta" in data else {}
-        self.in_domains = data["in_domains"] if "in_domains" in data else []
-        self.in_enrichments = data["in_enrichments"] if "in_enrichments" in data else []
+        self.use_did = data["did"] if "did" in data else None
         self.expected_result_type = None
-        self.cause_regex = None
-        self.context_regex = None
+        self.err_or_filt_cause = None
+        self.err_or_filt_context = None
+        self.err_or_filt_annotations = None
+        self.expected_metrics = []
 
-    def expect_error_result(self, cause: str, context: str):
+    def add_metric(self, metric: Metric):
+        self.expected_metrics.append(metric)
+
+    def expect_error_result(self, cause: str, context: str, annotations: Dict = None):
         """
         A Sets the expected output of the action to an Error Result
         :param cause: the expected error cause
         :param context: the expected error context
+        :param annotations (Optional): Dict: the expected annotations
         """
         self.expected_result_type = ErrorResult
-        self.cause_regex = cause
-        self.context_regex = context
+        self.err_or_filt_cause = cause
+        self.err_or_filt_context = context
+        self.err_or_filt_annotations = annotations
 
-    def expect_filter_result(self, cause: str, context: str=None):
+    def expect_filter_result(self, cause: str, context: str = None, annotations: Dict = None):
         """
         A Sets the expected output of the action to a Filter Result
         :param cause: the expected filter cause (message)
-        :param context: the expected error context (optional)
+        :param context (Optional): the expected filter context
+        :param annotations (Optional): Dict: the expected annotations
         """
         self.expected_result_type = FilterResult
-        self.cause_regex = cause
-        self.context_regex = context
+        self.err_or_filt_cause = cause
+        self.err_or_filt_context = context
+        self.err_or_filt_annotations = annotations
 
 
 class ActionTest(ABC):
@@ -183,15 +182,27 @@ class ActionTest(ABC):
         """
         self.content_service = InternalContentService()
         self.did = ""
-        self.expected_outputs = []
         self.loaded_inputs = []
         self.package_name = package_name
         self.res_path = ""
 
-    def __reset__(self):
+    def __reset__(self, did: str):
         self.content_service = InternalContentService()
+<<<<<<< HEAD
+        if did is None:
+            self.did = str(uuid.uuid4())
+        else:
+            self.did = did
+        self.expected_outputs = []
+||||||| parent of 831733c7 (2.0 Refactor)
         self.did = str(uuid.uuid4())
         self.expected_outputs = []
+=======
+        if did is None:
+            self.did = str(uuid.uuid4())
+        else:
+            self.did = did
+>>>>>>> 831733c7 (2.0 Refactor)
         self.loaded_inputs = []
         self.res_path = ""
 
@@ -212,19 +223,10 @@ class ActionTest(ABC):
             else:
                 self.loaded_inputs.append(LoadedContent(self.did, input_ioc, None))
 
-        # Load expected outputs
-        for output_ioc in test_case.outputs:
-            if len(output_ioc.content_bytes) == 0:
-                self.expected_outputs.append(LoadedContent(self.did, output_ioc, self.load_file(output_ioc)))
-            else:
-                self.expected_outputs.append(LoadedContent(self.did, output_ioc, None))
-
     def make_content_list(self, test_case: TestCaseBase):
         content_list = []
         for loaded_input in self.loaded_inputs:
-            c = Content(name=loaded_input.name,
-                        segments=[loaded_input.segment],
-                        media_type=loaded_input.content_type,
+            c = Content(name=loaded_input.name, segments=[loaded_input.segment], media_type=loaded_input.content_type,
                         content_service=self.content_service)
             content_list.append(c)
             loaded_input.content = c
@@ -235,40 +237,48 @@ class ActionTest(ABC):
         content_list = self.make_content_list(test_case)
         self.content_service.load(self.loaded_inputs)
 
+<<<<<<< HEAD
+        return DeltaFileMessage(
+            metadata=test_case.in_meta,
+            content_list=content_list,
+            domains=test_case.in_domains,
+            enrichments=test_case.in_enrichments)
+||||||| parent of 831733c7 (2.0 Refactor)
         return DeltaFileMessage(metadata=test_case.in_meta,
                                 content_list=content_list,
                                 domains=test_case.in_domains,
                                 enrichments=test_case.in_enrichments)
+=======
+        return DeltaFileMessage(metadata=test_case.in_meta,
+                                content_list=content_list)
+>>>>>>> 831733c7 (2.0 Refactor)
 
     def make_context(self, test_case: TestCaseBase):
         action_name = INGRESS_FLOW + "." + test_case.action.__class__.__name__
-        return Context(did=self.did,
-                       action_flow=INGRESS_FLOW,
-                       action_name=action_name,
-                       source_filename=test_case.file_name,
-                       ingress_flow=INGRESS_FLOW,
-                       egress_flow=EGRESS_FLOW,
-                       system=SYSTEM,
-                       hostname=HOSTNAME,
-                       content_service=self.content_service,
-                       collect=None,
-                       collected_dids=None,
-                       logger=get_logger())
+        return Context(
+            did=self.did,
+            action_flow=INGRESS_FLOW,
+            action_name=action_name,
+            source_filename=test_case.file_name,
+            ingress_flow=INGRESS_FLOW,
+            egress_flow=EGRESS_FLOW,
+            system=SYSTEM,
+            hostname=HOSTNAME,
+            content_service=self.content_service,
+            collect=None,
+            collected_dids=None,
+            logger=get_logger())
 
     def make_event(self, test_case: TestCaseBase):
-        return Event(
-            delta_file_messages=[self.make_df_msg(test_case)],
-            context=self.make_context(test_case),
-            params=test_case.parameters,
-            queue_name="",
-            return_address="")
+        return Event(delta_file_messages=[self.make_df_msg(test_case)], context=self.make_context(test_case),
+                     params=test_case.parameters, queue_name="", return_address="")
 
     def call_action(self, test_case: TestCaseBase):
         self.get_contents(test_case)
         return test_case.action.execute_action(self.make_event(test_case))
 
     def run_and_check_result_type(self, test_case: TestCaseBase, result_type):
-        self.__reset__()
+        self.__reset__(test_case.use_did)
         result = self.call_action(test_case)
 
         if not isinstance(result, result_type):
@@ -279,18 +289,25 @@ class ActionTest(ABC):
     def execute_error(self, test_case: TestCaseBase):
         result = self.run_and_check_result_type(test_case, ErrorResult)
         resp = result.response()
-        assert resp['cause'] == test_case.cause_regex
-        assert resp['context'] == test_case.context_regex
+        assert_equal_with_label(test_case.err_or_filt_cause, resp['cause'], "error cause")
+        if test_case.err_or_filt_context is not None:
+            assert_equal_with_label(test_case.err_or_filt_context, resp['context'], "error context")
+        if test_case.err_or_filt_annotations is not None:
+            assert_keys_and_values(test_case.err_or_filt_annotations, result.annotations)
 
-    def execute_filter(self, test_case):
+    def execute_filter(self, test_case: TestCaseBase):
         result = self.run_and_check_result_type(test_case, FilterResult)
         resp = result.response()
-        assert resp['message'] == test_case.cause_regex
+        assert_equal_with_label(test_case.err_or_filt_cause, resp['message'], "filter cause")
+        if test_case.err_or_filt_context is not None:
+            assert_equal_with_label(test_case.err_or_filt_context, resp['context'], "filter context")
+        if test_case.err_or_filt_annotations is not None:
+            assert_keys_and_values(test_case.err_or_filt_annotations, result.annotations)
 
     def execute(self, test_case: TestCaseBase):
-        if isinstance(test_case.expected_result_type, ErrorResult.__class__):
+        if test_case.expected_result_type == ErrorResult:
             self.execute_error(test_case)
-        elif isinstance(test_case.expected_result_type, FilterResult.__class__):
+        elif test_case.expected_result_type == FilterResult:
             self.execute_filter(test_case)
         else:
             raise ValueError(f"unknown type: {test_case.expected_result_type}")
@@ -302,23 +319,21 @@ class ActionTest(ABC):
     def compare_one_content(self, comparitor: CompareHelper, expected: LoadedContent, actual, index):
         self.compare_content_details(expected, actual)
         seg_id = actual.segments[0].uuid
-        comparitor.compare(
-            expected.data,
-            self.content_service.get_output(seg_id),
-            f"Content[{index}]"
-        )
-
-    def compare_all_output(self, comparitor: CompareHelper, content: List):
-        assert_equal_len(self.expected_outputs, content)
-        for index, expected in enumerate(self.expected_outputs):
+        comparitor.compare(expected.data, self.content_service.get_output(seg_id), f"Content[{index}]")
+
+    def compare_content_list(self, comparitor: CompareHelper, expected_outputs: List[IOContent], content: List):
+        assert_equal_len(expected_outputs, content)
+        for index, expected_ioc in enumerate(expected_outputs):
+            if len(expected_ioc.content_bytes) == 0:
+                expected = LoadedContent(self.did, expected_ioc, self.load_file(output_ioc))
+            else:
+                expected = LoadedContent(self.did, expected_ioc, None)
             self.compare_one_content(comparitor, expected, content[index], index)
 
-    def compare_domains(self, comparitor: CompareHelper, expected_items: List[Dict], results: List[Dict]):
-        assert_equal_len(expected_items, results)
-        for index, expected in enumerate(expected_items):
-            actual = results[index]
-            assert_equal(expected['name'], actual['name'])
-            assert_equal(expected['mediaType'], actual['mediaType'])
+    def compare_one_metric(self, expected: Metric, result: Metric):
+        assert expected.name == result.name
+        assert_equal_with_label(expected.value, result.value, expected.name)
+        assert_keys_and_values(expected.tags, result.tags)
 
             expected_value = expected['value']
             if type(expected_value) == str:
@@ -327,5 +342,15 @@ class ActionTest(ABC):
                 expected_data = self.load_file(expected_value)
                 comparitor.compare(expected_data, actual['value'], f"Domain[{index}]")
             else:
-                raise ValueError(
-                    f"unknown expected_value type: {type(expected_value)}")
+                raise ValueError(f"unknown expected_value type: {type(expected_value)}")
+
+    def compare_one_metric(self, expected: Metric, result: Metric):
+        assert expected.name == result.name
+        assert_equal_with_label(expected.value, result.value, expected.name)
+        assert_keys_and_values(expected.tags, result.tags)
+
+    def compare_metrics(self, expected_metrics: List[Metric], results: List[Metric]):
+        if len(expected_metrics) > 0:
+            assert_equal_len(expected_metrics, results)
+            for index, expected in enumerate(expected_metrics):
+                self.compare_one_metric(expected, results[index])

deltafi-2.0rc1713530263087/deltafi/test_kit/transform.py

@@ -0,0 +1,103 @@
+#
+#    DeltaFi - Data transformation and enrichment platform
+#
+#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+from typing import List
+
+from deltafi.result import TransformResult, TransformResults
+
+from .assertions import *
+from .framework import TestCaseBase, ActionTest, IOContent
+
+
+class TransformTestCase(TestCaseBase):
+    def __init__(self, fields: Dict):
+        super().__init__(fields)
+        self.results = []
+
+    def expect_transform_result(self):
+        self.expected_result_type = TransformResult
+
+    def expect_transform_results(self):
+        self.expected_result_type = TransformResults
+
+    def add_transform_result(self, content: List[IOContent], metadata: Dict, delete_metadata_keys: List[str],
+                             annotations: Dict, name: str = None):
+        self.results.append(
+            {
+                'content': content,
+                'metadata': metadata,
+                'delete_metadata_keys': delete_metadata_keys,
+                'annotations': annotations,
+                'name': name
+            }
+        )
+
+
+class TransformActionTest(ActionTest):
+    def __init__(self, package_name: str):
+        """
+        Provides structure for testing DeltaFi Transform action
+        Args:
+            package_name: name of the actions package for finding resources
+        """
+        super().__init__(package_name)
+
+    def transform(self, test_case: TransformTestCase):
+        if test_case.expected_result_type == TransformResult:
+            self.expect_transform_result(test_case)
+        elif test_case.expected_result_type == TransformResults:
+            self.expect_transform_results(test_case)
+        else:
+            super().execute(test_case)
+
+    def expect_transform_result(self, test_case: TransformTestCase):
+        result = super().run_and_check_result_type(test_case, TransformResult)
+        self.assert_transform_result(test_case, result)
+
+    def expect_transform_results(self, test_case: TransformTestCase):
+        result = super().run_and_check_result_type(test_case, TransformResults)
+        self.assert_transform_results(test_case, result)
+
+    def assert_transform_results(self, test_case: TransformTestCase, result: TransformResults):
+        assert_equal_len(test_case.results, result.named_results)
+        for index, named_result in enumerate(result.named_results):
+            self.compare_one_transform_result(test_case, named_result.result, index)
+            expected = test_case.results[index]
+            if 'name' in expected:
+                assert_equal_with_label(expected["name"], named_result.name, f"name[{index}]")
+
+    def assert_transform_result(self, test_case: TransformTestCase, result: TransformResult):
+        # Check metrics
+        self.compare_metrics(test_case.expected_metrics, result.metrics)
+        self.compare_one_transform_result(test_case, result, 0)
+
+    def compare_one_transform_result(self, test_case: TransformTestCase, result: TransformResult, index: int):
+        expected = test_case.results[index]
+
+        # Check output
+        self.compare_content_list(test_case.compare_tool, expected['content'], result.content)
+
+        # Check metadata
+        assert_keys_and_values(expected['metadata'], result.metadata)
+
+        # Check deleted metadata
+        for key in expected['delete_metadata_keys']:
+            assert_key_in(key, result.delete_metadata_keys)
+
+        # Check annotations
+        assert_keys_and_values(expected['annotations'], result.annotations)

deltafi-2.0rc1713530263087/deltafi/test_kit/validate.py

@@ -0,0 +1,54 @@
+#
+#    DeltaFi - Data transformation and enrichment platform
+#
+#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+from deltafi.result import ValidateResult
+
+from .assertions import *
+from .framework import TestCaseBase, ActionTest
+
+
+class ValidateTestCase(TestCaseBase):
+    def __init__(self, fields: Dict):
+        super().__init__(fields)
+
+    def expect_validate_result(self):
+        self.expected_result_type = ValidateResult
+
+
+class ValidateActionTest(ActionTest):
+    def __init__(self, package_name: str):
+        """
+        Provides structure for testing DeltaFi Validate action
+        Args:
+            package_name: name of the actions package for finding resources
+        """
+        super().__init__(package_name)
+
+    def validate(self, test_case: ValidateTestCase):
+        if test_case.expected_result_type == ValidateResult:
+            self.expect_validate_result(test_case)
+        else:
+            super().execute(test_case)
+
+    def expect_validate_result(self, test_case: ValidateTestCase):
+        result = super().run_and_check_result_type(test_case, ValidateResult)
+        self.assert_validate_result(test_case, result)
+
+    def assert_validate_result(self, test_case: ValidateTestCase, result: ValidateResult):
+        # Check metrics
+        self.compare_metrics(test_case.expected_metrics, result.metrics)

{deltafi-2.0rc1705080991758 → deltafi-2.0rc1713530263087}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "deltafi"
-version = "2.0rc1705080991758"
+version = "2.0rc1713530263087"
 description = "SDK for DeltaFi plugins and actions"
 authors = ["DeltaFi <deltafi@systolic.com>"]
 license = "Apache License, Version 2.0"
@@ -13,28 +13,28 @@ classifiers = [
     "Development Status :: 4 - Beta",
     "Intended Audience :: Developers",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.7",
-    "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
-    "Programming Language :: Python :: 3.10"
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12"
 ]
 
 [tool.poetry.dependencies]
-python = "^3.7"
-deepdiff = ">=6.3.1"
+python = "^3.9"
+deepdiff = ">=6.7.1"
 json-logging = ">=1.3.0"
-minio = ">=7.1.17"
-pydantic = ">=2.4.2"
+minio = ">=7.2.3"
+pydantic = ">=2.5.3"
 redis = ">=5.0.1"
 requests = ">=2.31.0"
-urllib3 = ">=2.0.6"
+urllib3 = ">=2.1.0"
 
 [tool.poetry.group.test]
 optional = true
 
 [tool.poetry.group.test.dependencies]
-pytest = ">=7.4.2"
-pytest-mock = ">=3.11.1"
+pytest = ">=7.4.4"
+pytest-mock = ">=3.12.0"
 mockito = ">=1.4.0"
 
 [tool.poetry.urls]

deltafi-2.0rc1705080991758/deltafi/test_kit/transform.py

@@ -1,72 +0,0 @@
-#
-#    DeltaFi - Data transformation and enrichment platform
-#
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-#
-
-from typing import List
-
-from deltafi.result import TransformResult
-
-from .assertions import *
-from .framework import TestCaseBase, ActionTest
-
-
-class TransformTestCase(TestCaseBase):
-    def __init__(self, fields: Dict):
-        super().__init__(fields)
-        self.metadata = {}
-        self.delete_metadata_keys = []
-        self.annotations = {}
-
-    def expect_transform_result(self, metadata: Dict, delete_metadata_keys: List[str], annotations: Dict):
-        self.expected_result_type = TransformResult
-        self.metadata = metadata
-        self.delete_metadata_keys = delete_metadata_keys
-        self.annotations = annotations
-
-
-class TransformActionTest(ActionTest):
-    def __init__(self, package_name: str):
-        """
-        Provides structure for testing DeltaFi Transform action
-        Args:
-            package_name: name of the actions package for finding resources
-        """
-        super().__init__(package_name)
-
-    def transform(self, test_case: TransformTestCase):
-        if test_case.expected_result_type == TransformResult:
-            self.expect_transform_result(test_case)
-        else:
-            super().execute(test_case)
-
-    def expect_transform_result(self, test_case: TransformTestCase):
-        result = super().run_and_check_result_type(test_case, TransformResult)
-        self.assert_transform_result(test_case, result)
-
-    def assert_transform_result(self, test_case: TransformTestCase, result: TransformResult):
-        # Check output
-        self.compare_all_output(test_case.compare_tool, result.content)
-
-        # Check metadata
-        assert_keys_and_values(test_case.metadata, result.metadata)
-
-        # Check deleted metadata
-        for key in test_case.delete_metadata_keys:
-            assert_key_in(key, result.delete_metadata_keys)
-
-        # Check annotations
-        assert_keys_and_values(test_case.annotations, result.annotations)