deltafi 1.0.5__py3-none-any.whl → 1.0.7__py3-none-any.whl

deltafi/domain.py

@@ -25,6 +25,7 @@ from deltafi.storage import ContentService, Segment
 
 class Context(NamedTuple):
     did: str
+    action_flow: str
     action_name: str
     source_filename: str
     ingress_flow: str
@@ -37,7 +38,9 @@ class Context(NamedTuple):
     @classmethod
     def create(cls, context: dict, hostname: str, content_service: ContentService, logger: Logger):
         did = context['did']
-        action_name = context['name']
+        action_name_parts = context['name'].split(".")
+        action_flow = action_name_parts[0]
+        action_name = action_name_parts[1]
         if 'sourceFilename' in context:
             source_filename = context['sourceFilename']
         else:
@@ -49,6 +52,7 @@ class Context(NamedTuple):
             egress_flow = None
         system = context['systemName']
         return Context(did=did,
+                       action_flow=action_flow,
                        action_name=action_name,
                        source_filename=source_filename,
                        ingress_flow=ingress_flow,

deltafi/plugin.py

@@ -270,7 +270,7 @@ class Plugin(object):
 
                 response = {
                     'did': event.context.did,
-                    'action': event.context.action_name,
+                    'action': event.context.action_flow + "." + event.context.action_name,
                     'start': start_time,
                     'stop': time.time(),
                     'type': result.result_type,

deltafi/result.py

@@ -17,8 +17,8 @@
 #
 
 import abc
-from typing import Dict, List
 import uuid
+from typing import Dict, List
 
 from deltafi.domain import Content, Context
 from deltafi.metric import Metric
@@ -123,40 +123,66 @@ class FormatResult(Result):
     def __init__(self, context: Context):
         super().__init__('format', 'FORMAT', context)
         self.content = None
+        self.delete_metadata_keys = []
         self.metadata = {}
 
     def add_metadata(self, key: str, value: str):
         self.metadata[key] = value
         return self
 
+    def delete_metadata_key(self, key: str):
+        self.delete_metadata_keys.append(key)
+        return self
+
     def set_content(self, content: Content):
         self.content = content
         return self
 
     def save_string_content(self, string_data: str, name: str, media_type: str):
         segment = self.context.content_service.put_str(self.context.did, string_data)
-        self.content = Content(name=name, segments=[segment], media_type=media_type, content_service=self.context.content_service)
+        self.content = Content(name=name, segments=[segment], media_type=media_type,
+                               content_service=self.context.content_service)
         return self
 
     def save_byte_content(self, byte_data: bytes, name: str, media_type: str):
         segment = self.context.content_service.put_bytes(self.context.did, byte_data)
-        self.content = Content(name=name, segments=[segment], media_type=media_type, content_service=self.context.content_service)
+        self.content = Content(name=name, segments=[segment], media_type=media_type,
+                               content_service=self.context.content_service)
         return self
 
     def response(self):
         return {
             'content': self.content.json(),
-            'metadata': self.metadata
+            'metadata': self.metadata,
+            'deleteMetadataKeys': self.delete_metadata_keys
         }
 
 
+class ChildFormatResult:
+    def __init__(self, format_result: FormatResult = None):
+        self._did = str(uuid.uuid4())
+        self.format_result = format_result
+
+    @property
+    def did(self):
+        return self._did
+
+    def response(self):
+        res = self.format_result.response()
+        res["did"] = self._did
+        return res
+
+
 class FormatManyResult(Result):
     def __init__(self, context: Context):
         super().__init__('formatMany', 'FORMAT_MANY', context)
         self.format_results = []
 
-    def add_format_result(self, format_result: FormatResult):
-        self.format_results.append(format_result)
+    def add_format_result(self, format_result):
+        if isinstance(format_result, ChildFormatResult):
+            self.format_results.append(format_result)
+        else:
+            self.format_results.append(ChildFormatResult(format_result))
         return self
 
     def response(self):
@@ -184,12 +210,14 @@ class LoadResult(Result):
 
     def save_string_content(self, string_data: str, name: str, media_type: str):
         segment = self.context.content_service.put_str(self.context.did, string_data)
-        self.content.append(Content(name=name, segments=[segment], media_type=media_type, content_service=self.context.content_service))
+        self.content.append(
+            Content(name=name, segments=[segment], media_type=media_type, content_service=self.context.content_service))
         return self
 
     def save_byte_content(self, byte_data: bytes, name: str, media_type: str):
         segment = self.context.content_service.put_bytes(self.context.did, byte_data)
-        self.content.append(Content(name=name, segments=[segment], media_type=media_type, content_service=self.context.content_service))
+        self.content.append(
+            Content(name=name, segments=[segment], media_type=media_type, content_service=self.context.content_service))
         return self
 
     def add_metadata(self, key: str, value: str):
@@ -300,12 +328,14 @@ class TransformResult(Result):
 
     def save_string_content(self, string_data: str, name: str, media_type: str):
         segment = self.context.content_service.put_str(self.context.did, string_data)
-        self.content.append(Content(name=name, segments=[segment], media_type=media_type, content_service=self.context.content_service))
+        self.content.append(
+            Content(name=name, segments=[segment], media_type=media_type, content_service=self.context.content_service))
         return self
 
     def save_byte_content(self, byte_data: bytes, name: str, media_type: str):
         segment = self.context.content_service.put_bytes(self.context.did, byte_data)
-        self.content.append(Content(name=name, segments=[segment], media_type=media_type, content_service=self.context.content_service))
+        self.content.append(
+            Content(name=name, segments=[segment], media_type=media_type, content_service=self.context.content_service))
         return self
 
     def add_metadata(self, key: str, value: str):

deltafi/test_kit/__init__.py

@@ -0,0 +1,17 @@
+#
+#    DeltaFi - Data transformation and enrichment platform
+#
+#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#

deltafi/test_kit/assertions.py

@@ -0,0 +1,47 @@
+#
+#    DeltaFi - Data transformation and enrichment platform
+#
+#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+from typing import Dict
+
+from .constants import IGNORE_VALUE
+
+
+def assert_equal(e, a):
+    assert e == a, f"{e} != {a}"
+
+
+def assert_equal_with_label(e, a, l):
+    assert e == a, f"{l}. Expected:\n<<{e}>>\nBut was:\n<<{a}>>"
+
+
+def assert_equal_len(e, a):
+    assert len(e) == len(a), f"{len(e)} != {len(a)}"
+
+
+def assert_key_in(k, m):
+    assert k in m, f"{k} not found"
+
+
+def assert_key_not_in(k, m):
+    assert k not in m, f"{k} found, but not expected"
+
+
+def assert_keys_and_values(expected: Dict, actual: Dict):
+    for key in expected:
+        assert_key_in(key, actual)
+        if expected[key] != IGNORE_VALUE:
+            assert_equal(expected[key], actual[key])

deltafi/test_kit/compare_helpers.py

@@ -0,0 +1,49 @@
+#
+#    DeltaFi - Data transformation and enrichment platform
+#
+#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+import json
+from abc import ABC
+from abc import abstractmethod
+from typing import List
+
+from deepdiff import DeepDiff
+
+from .assertions import *
+
+
+class CompareHelper(ABC):
+    @abstractmethod
+    def compare(self, expected: str, actual: str, label: str):
+        pass
+
+
+class GenericCompareHelper(CompareHelper):
+    def compare(self, expected: str, actual: str, label: str):
+        assert_equal_with_label(expected, actual, label)
+
+
+class JsonCompareHelper(CompareHelper):
+    def __init__(self, regex_exclusion_list: List):
+        self.excludes = regex_exclusion_list
+
+    def compare(self, expected: str, actual: str, label: str):
+        exp = json.loads(expected)
+        act = json.loads(actual)
+        diffs = DeepDiff(exp, act, exclude_regex_paths=self.excludes)
+        if len(diffs) > 0:
+            raise ValueError(f"{diffs}")
+        assert len(diffs) == 0

deltafi/test_kit/constants.py

@@ -0,0 +1,23 @@
+#
+#    DeltaFi - Data transformation and enrichment platform
+#
+#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+INGRESS_FLOW = "ingress-flow"
+EGRESS_FLOW = "egress-flow"
+HOSTNAME = "HOSTNAME"
+SYSTEM = "SYSTEM"
+IGNORE_VALUE = "%%IGNORE%%"

deltafi/test_kit/format.py

@@ -0,0 +1,99 @@
+#
+#    DeltaFi - Data transformation and enrichment platform
+#
+#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+from typing import List
+
+from deltafi.result import FormatResult, FormatManyResult
+
+from .assertions import *
+from .framework import TestCaseBase, ActionTest
+
+
+class FormatTestCase(TestCaseBase):
+    def __init__(self, fields: Dict):
+        super().__init__(fields)
+        self.metadata = {}
+        self.delete_metadata_keys = []
+        self.expected_format_many_result = []
+
+    def expect_format_result(self, metadata: Dict, delete_metadata_keys: List[str]):
+        self.expected_result_type = FormatResult
+        self.metadata = metadata
+        self.delete_metadata_keys = delete_metadata_keys
+
+    def add_format_many_result(self, metadata: Dict, delete_metadata_keys: List):
+        self.expected_result_type = FormatManyResult
+        self.expected_format_many_result.append(
+            {
+                "metadata": metadata,
+                "delete_metadata_keys": delete_metadata_keys
+            }
+        )
+
+
+class FormatActionTest(ActionTest):
+    def __init__(self, package_name: str):
+        """
+        Provides structure for testing DeltaFi Format action
+        Args:
+            package_name: name of the actions package for finding resources
+        """
+        super().__init__(package_name)
+
+    def format(self, test_case: FormatTestCase):
+        if test_case.expected_result_type == FormatManyResult:
+            self.expect_format_many_result(test_case)
+        elif test_case.expected_result_type == FormatResult:
+            self.expect_format_result(test_case)
+        else:
+            super().execute(test_case)
+
+    def expect_format_result(self, test_case: FormatTestCase):
+        result = super().run_and_check_result_type(test_case, FormatResult)
+        self.assert_format_result(test_case, result)
+
+    def expect_format_many_result(self, test_case: FormatTestCase):
+        result = super().run_and_check_result_type(test_case, FormatManyResult)
+        self.assert_format_many_result(test_case, result)
+
+    def assert_format_result(self, test_case: FormatTestCase, result: FormatResult):
+        # Check output
+        if result.content is None:
+            self.compare_all_output(test_case.compare_tool, [])
+        else:
+            self.compare_all_output(test_case.compare_tool, [result.content])
+
+        # Check metadata
+        assert_keys_and_values(test_case.metadata, result.metadata)
+
+        # Check deleted metadata
+        for key in test_case.delete_metadata_keys:
+            assert_key_in(key, result.delete_metadata_keys)
+
+    def assert_format_many_result(self, test_case: FormatTestCase, actual: FormatManyResult):
+        assert_equal_len(test_case.expected_format_many_result, actual.format_results)
+        for index, expected_child_result in enumerate(test_case.expected_format_many_result):
+            actual_child = actual.format_results[index]
+            self.compare_one_content(test_case.compare_tool,
+                                     self.expected_outputs[index],
+                                     actual_child.format_result.content,
+                                     index)
+
+            assert_keys_and_values(expected_child_result['metadata'],
+                                   actual_child.format_result.metadata)
+            for key in expected_child_result['delete_metadata_keys']:
+                assert_key_in(key, actual_child.format_result.delete_metadata_keys)

deltafi/test_kit/framework.py

@@ -0,0 +1,326 @@
+#
+#    DeltaFi - Data transformation and enrichment platform
+#
+#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+import uuid
+from abc import ABC
+from importlib.resources import files
+from typing import List
+
+from deltafi.domain import DeltaFileMessage, Event, Content, Context
+from deltafi.logger import get_logger
+from deltafi.result import ErrorResult, FilterResult
+from deltafi.storage import Segment
+
+from .assertions import *
+from .compare_helpers import GenericCompareHelper, CompareHelper
+from .constants import *
+
+
+class IOContent:
+    """
+    The IOContent class holds the details for loading input or output
+     content into the test framework.
+    Attributes:
+        file_name (str): The name of file in test/data.
+        content_name (str): The name of the content.
+        content_type (str): The media type of the content
+        offset (int): Offset to use in Segment
+        content_bytes (str): Bypass file read, and uses these bytes for content
+    """
+
+    def __init__(self, file_name: str, content_name: str = None,
+                 content_type: str = None, offset: int = 0,
+                 content_bytes: str = ""):
+        self.file_name = file_name
+        if content_name is None:
+            self.content_name = file_name
+        else:
+            self.content_name = content_name
+        if content_type is None:
+            self.content_type = IOContent.file_type(file_name)
+        else:
+            self.content_type = content_type
+        self.offset = offset
+        self.content_bytes = content_bytes
+
+    @classmethod
+    def file_type(cls, name: str):
+        if name.endswith(".json"):
+            return "application/json"
+        elif name.endswith(".xml"):
+            return "application/xnl"
+        elif name.endswith(".txt"):
+            return "text/plain"
+        else:
+            return "application/octet-stream"
+
+
+class LoadedContent:
+    def __init__(self, did: str, ioc: IOContent, data: str):
+        self.name = ioc.content_name
+        self.content_type = ioc.content_type
+        self.offset = ioc.offset
+        if data is not None:
+            self.data = data
+        else:
+            self.data = ioc.content_bytes
+        self.segment = Segment.from_dict({
+            "uuid": str(uuid.uuid4()),
+            "offset": self.offset,
+            "size": len(self.data),
+            "did": did
+        })
+
+
+class InternalContentService:
+    def __init__(self):
+        self.loaded_content = {}
+        self.outputs = {}
+
+    def load(self, content_list: List[LoadedContent]):
+        for c in content_list:
+            self.loaded_content[c.segment.uuid] = c
+
+    def put_str(self, did: str, string_data: str):
+        segment = Segment(uuid=str(uuid.uuid4()),
+                          offset=0,
+                          size=len(string_data),
+                          did=did)
+        self.outputs[segment.uuid] = string_data
+        return segment
+
+    def get_str(self, segments: List[Segment]):
+        # TODO: String multiple segment ids together
+        seg_id = segments[0].uuid
+        return self.loaded_content[seg_id].data
+
+    def get_output(self, seg_id: str):
+        return self.outputs[seg_id]
+
+
+class TestCaseBase(ABC):
+    def __init__(self, data: Dict):
+        """
+        A test case for DeltaFi python actions
+        :param data: Dict of test case fields
+        - action: instance of the action being tested
+        - data_dir: str: subdirectory name (e.g., test name) for locating test data files, i.e., test/data/{data_dir)
+        - compare_tool: (optional) CompareHelper instanced for comparing output content
+        - inputs: (optional) List[IOContent]: input content to action
+        - parameters: (optional) Dict: map of action input parameters
+        - in_meta: (optional) Dict: map of metadata as input to action
+        - in_domains: (optional) List[Domain]: list of domains as input to action
+        - in_enrichments: (optional) List[Domain]: list of enrichments as input to action
+        """
+        if "action" in data:
+            self.action = data["action"]
+        else:
+            raise ValueError("action is required")
+
+        if "data_dir" in data:
+            self.data_dir = data["data_dir"]
+        else:
+            raise ValueError("data_dir is required")
+
+        if "compare_tool" in data:
+            self.compare_tool = data["compare_tool"]
+        else:
+            self.compare_tool = GenericCompareHelper()
+
+        self.inputs = data["inputs"] if "inputs" in data else []
+        self.file_name = data["file_name"] if "file_name" in data else "filename"
+        self.outputs = data["outputs"] if "outputs" in data else []
+        self.parameters = data["parameters"] if "parameters" in data else {}
+        self.in_meta = data["in_meta"] if "in_meta" in data else {}
+        self.in_domains = data["in_domains"] if "in_domains" in data else []
+        self.in_enrichments = data["in_enrichments"] if "in_enrichments" in data else []
+        self.expected_result_type = None
+        self.cause_regex = None
+        self.context_regex = None
+
+    def expect_error_result(self, cause: str, context: str):
+        """
+        A Sets the expected output of the action to an Error Result
+        :param cause: the expected error cause
+        :param context: the expected error context
+        """
+        self.expected_result_type = ErrorResult
+        self.cause_regex = cause
+        self.context_regex = context
+
+    def expect_filter_result(self, cause: str):
+        """
+        A Sets the expected output of the action to a Filter Result
+        :param cause: the expected filter cause (message)
+        """
+        self.expected_result_type = FilterResult
+        self.cause_regex = cause
+
+
+class ActionTest(ABC):
+    def __init__(self, package_name: str):
+        """
+        Provides structure for testing DeltaFi actions
+        Args:
+            package_name: name of the actions package for finding resources
+        """
+        self.content_service = InternalContentService()
+        self.did = ""
+        self.expected_outputs = []
+        self.loaded_inputs = []
+        self.package_name = package_name
+        self.res_path = ""
+
+    def __reset__(self):
+        self.content_service = InternalContentService()
+        self.did = str(uuid.uuid4())
+        self.expected_outputs = []
+        self.loaded_inputs = []
+        self.res_path = ""
+
+    def load_file(self, ioc: IOContent):
+        file_res = self.res_path.joinpath(ioc.file_name)
+        with file_res.open("r") as f:
+            contents = f.read()
+        return contents
+
+    def get_contents(self, test_case: TestCaseBase):
+        pkg_path = files(self.package_name)
+        self.res_path = pkg_path.joinpath(f"test/data/{test_case.data_dir}/")
+
+        # Load inputs
+        for input_ioc in test_case.inputs:
+            if len(input_ioc.content_bytes) == 0:
+                self.loaded_inputs.append(LoadedContent(self.did, input_ioc, self.load_file(input_ioc)))
+            else:
+                self.loaded_inputs.append(LoadedContent(self.did, input_ioc, None))
+
+        # Load expected outputs
+        for output_ioc in test_case.outputs:
+            if len(output_ioc.content_bytes) == 0:
+                self.expected_outputs.append(LoadedContent(self.did, output_ioc, self.load_file(output_ioc)))
+            else:
+                self.expected_outputs.append(LoadedContent(self.did, output_ioc, None))
+
+    def make_content_list(self, test_case: TestCaseBase):
+        content_list = []
+        for loaded_input in self.loaded_inputs:
+            c = Content(name=loaded_input.name,
+                        segments=[loaded_input.segment],
+                        media_type=loaded_input.content_type,
+                        content_service=self.content_service)
+            content_list.append(c)
+            loaded_input.content = c
+
+        return content_list
+
+    def make_df_msg(self, test_case: TestCaseBase):
+        content_list = self.make_content_list(test_case)
+        self.content_service.load(self.loaded_inputs)
+
+        return DeltaFileMessage(metadata=test_case.in_meta,
+                                content_list=content_list,
+                                domains=test_case.in_domains,
+                                enrichments=test_case.in_enrichments)
+
+    def make_context(self, test_case: TestCaseBase):
+        action_name = INGRESS_FLOW + "." + test_case.action.__class__.__name__
+        return Context(did=self.did,
+                       action_flow=INGRESS_FLOW,
+                       action_name=action_name,
+                       source_filename=test_case.file_name,
+                       ingress_flow=INGRESS_FLOW,
+                       egress_flow=EGRESS_FLOW,
+                       system=SYSTEM,
+                       hostname=HOSTNAME,
+                       content_service=self.content_service,
+                       logger=get_logger())
+
+    def make_event(self, test_case: TestCaseBase):
+        return Event(
+            delta_file_messages=[self.make_df_msg(test_case)],
+            context=self.make_context(test_case),
+            params=test_case.parameters,
+            queue_name="",
+            return_address="")
+
+    def call_action(self, test_case: TestCaseBase):
+        self.get_contents(test_case)
+        return test_case.action.execute(self.make_event(test_case))
+
+    def run_and_check_result_type(self, test_case: TestCaseBase, result_type):
+        self.__reset__()
+        result = self.call_action(test_case)
+
+        if not isinstance(result, result_type):
+            raise ValueError(f"Result type {result.__class__.__name__} does not match {result_type.__name__}")
+
+        return result
+
+    def execute_error(self, test_case: TestCaseBase):
+        result = self.run_and_check_result_type(test_case, ErrorResult)
+        resp = result.response()
+        assert resp['cause'] == test_case.cause_regex
+        assert resp['context'] == test_case.context_regex
+
+    def execute_filter(self, test_case):
+        result = self.run_and_check_result_type(test_case, FilterResult)
+        resp = result.response()
+        assert resp['message'] == test_case.cause_regex
+
+    def execute(self, test_case: TestCaseBase):
+        if isinstance(test_case.expected_result_type, ErrorResult.__class__):
+            self.execute_error(test_case)
+        elif isinstance(test_case.expected_result_type, FilterResult.__class__):
+            self.execute_filter(test_case)
+        else:
+            raise ValueError(f"unknown type: {test_case.expected_result_type}")
+
+    def compare_content_details(self, expected: LoadedContent, actual: Content):
+        assert_equal(expected.content_type, actual.media_type)
+        assert_equal(expected.name, actual.name)
+
+    def compare_one_content(self, comparitor: CompareHelper, expected: LoadedContent, actual, index):
+        self.compare_content_details(expected, actual)
+        seg_id = actual.segments[0].uuid
+        comparitor.compare(
+            expected.data,
+            self.content_service.get_output(seg_id),
+            f"Content[{index}]"
+        )
+
+    def compare_all_output(self, comparitor: CompareHelper, content: List):
+        assert_equal_len(self.expected_outputs, content)
+        for index, expected in enumerate(self.expected_outputs):
+            self.compare_one_content(comparitor, expected, content[index], index)
+
+    def compare_domains(self, comparitor: CompareHelper, expected_items: List[Dict], results: List[Dict]):
+        assert_equal_len(expected_items, results)
+        for index, expected in enumerate(expected_items):
+            actual = results[index]
+            assert_equal(expected['name'], actual['name'])
+            assert_equal(expected['mediaType'], actual['mediaType'])
+
+            expected_value = expected['value']
+            if type(expected_value) == str:
+                comparitor.compare(expected_value, actual['value'], f"Domain[{index}]")
+            elif type(expected_value) == IOContent:
+                expected_data = self.load_file(expected_value)
+                comparitor.compare(expected_data, actual['value'], f"Domain[{index}]")
+            else:
+                raise ValueError(
+                    f"unknown expected_value type: {type(expected_value)}")

deltafi/test_kit/load.py

@@ -0,0 +1,121 @@
+#
+#    DeltaFi - Data transformation and enrichment platform
+#
+#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+from typing import List
+
+from deltafi.result import LoadResult, ReinjectResult
+
+from .assertions import *
+from .compare_helpers import GenericCompareHelper, CompareHelper
+from .framework import TestCaseBase, ActionTest, IOContent
+
+
+class LoadTestCase(TestCaseBase):
+    def __init__(self, fields: Dict):
+        super().__init__(fields)
+        self.metadata = {}
+        self.delete_metadata_keys = []
+        self.annotations = {}
+        self.domains = []
+        self.domain_cmp_tool = GenericCompareHelper()
+        self.children = []
+
+    def set_domain_compare_tool(self, helper: CompareHelper):
+        self.domain_cmp_tool = helper
+
+    def expect_load_result(self, metadata: Dict, delete_metadata_keys: List[str], annotations: Dict, domains: List):
+        self.expected_result_type = LoadResult
+        self.metadata = metadata
+        self.delete_metadata_keys = delete_metadata_keys
+        self.annotations = annotations
+        self.domains = domains
+
+    def add_reinject_child(self, filename: str, flow: str, content: IOContent, metadata: Dict):
+        self.expected_result_type = ReinjectResult
+        self.children.append(
+            {
+                "filename": filename,
+                "flow": flow,
+                "content": content,
+                "metadata": metadata
+            }
+        )
+
+
+class LoadActionTest(ActionTest):
+    def __init__(self, package_name: str):
+        """
+        Provides structure for testing DeltaFi Load action
+        Args:
+            package_name: name of the actions package for finding resources
+        """
+        super().__init__(package_name)
+
+    def load(self, test_case: LoadTestCase):
+        if test_case.expected_result_type == ReinjectResult:
+            self.expect_reinject_result(test_case)
+        elif test_case.expected_result_type == LoadResult:
+            self.expect_load_result(test_case)
+        else:
+            super().execute(test_case)
+
+    def expect_load_result(self, test_case: LoadTestCase):
+        result = super().run_and_check_result_type(test_case, LoadResult)
+        self.assert_load_result(test_case, result)
+
+    def expect_reinject_result(self, test_case: LoadTestCase):
+        result = super().run_and_check_result_type(test_case, ReinjectResult)
+        self.assert_reinject_result(test_case, result)
+
+    def assert_load_result(self, test_case: LoadTestCase, result: LoadResult):
+        # Check output
+        self.compare_all_output(test_case.compare_tool, result.content)
+
+        # Check metadata
+        assert_keys_and_values(test_case.metadata, result.metadata)
+
+        # Check deleted metadata
+        for key in test_case.delete_metadata_keys:
+            assert_key_in(key, result.delete_metadata_keys)
+
+        # Check annotations
+        assert_keys_and_values(test_case.annotations, result.annotations)
+
+        # Check domains
+        self.compare_domains(test_case.domain_cmp_tool, test_case.domains, result.domains)
+
+    def assert_reinject_result(self, test_case: LoadTestCase, actual: ReinjectResult):
+        assert_equal_len(test_case.children, actual.children)
+        for index, expected in enumerate(test_case.children):
+            reinject_child = actual.children[index]
+            assert_equal(expected['filename'], reinject_child.filename)
+            assert_equal(expected['flow'], reinject_child.flow)
+            assert_keys_and_values(expected['metadata'], reinject_child.metadata)
+
+            expected_value = expected['content']
+            child_content = reinject_child.content[0]
+            seg_id = child_content.segments[0].uuid
+            actual_content = self.content_service.get_output(seg_id)
+
+            if type(expected_value) == str:
+                test_case.domain_cmp_tool.compare(expected_value, actual_content, f"RI_child[{index}]")
+            elif type(expected_value) == IOContent:
+                expected_data = self.load_file(expected_value)
+                test_case.domain_cmp_tool.compare(expected_data, actual_content, f"RI_child[{index}]")
+            else:
+                raise ValueError(
+                    f"unknown expected_value type: {type(expected_value)}")

deltafi/test_kit/transform.py

@@ -0,0 +1,71 @@
+#
+#    DeltaFi - Data transformation and enrichment platform
+#
+#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+from typing import List
+
+from deltafi.result import TransformResult
+
+from .assertions import *
+from .framework import TestCaseBase, ActionTest
+
+
+class TransformTestCase(TestCaseBase):
+    def __init__(self, fields: Dict):
+        super().__init__(fields)
+        self.metadata = {}
+        self.delete_metadata_keys = []
+        self.annotations = {}
+
+    def expect_transform_result(self, metadata: Dict, delete_metadata_keys: List[str], annotations: Dict):
+        self.expected_result_type = TransformResult
+        self.metadata = metadata
+        self.delete_metadata_keys = delete_metadata_keys
+        self.annotations = annotations
+
+
+class TransformActionTest(ActionTest):
+    def __init__(self, package_name: str):
+        """
+        Provides structure for testing DeltaFi Transform action
+        Args:
+            package_name: name of the actions package for finding resources
+        """
+        super().__init__(package_name)
+
+    def transform(self, test_case: TransformTestCase):
+        if test_case.expected_result_type == TransformResult:
+            self.expect_transform_result(test_case)
+        else:
+            super().execute(test_case)
+
+    def expect_transform_result(self, test_case: TransformTestCase):
+        result = super().run_and_check_result_type(test_case, TransformResult)
+        self.assert_transform_result(test_case, result)
+
+    def assert_transform_result(self, test_case: TransformTestCase, result: TransformResult):
+        # Check output
+        self.compare_all_output(test_case.compare_tool, result.content)
+
+        # Check metadata
+        assert_keys_and_values(test_case.metadata, result.metadata)
+
+        # Check deleted metadata
+        for key in test_case.delete_metadata_keys:
+            assert_key_in(key, result.delete_metadata_keys)
+
+        # Check annotations
+        assert_keys_and_values(test_case.annotations, result.annotations)

{deltafi-1.0.5.dist-info → deltafi-1.0.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltafi
-Version: 1.0.5
+Version: 1.0.7
 Summary: SDK for DeltaFi plugins and actions
 License: Apache License, Version 2.0
 Keywords: deltafi
@@ -24,6 +24,7 @@ Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Topic :: Software Development
+Requires-Dist: deepdiff (>=6.3.1)
 Requires-Dist: json-logging (>=1.3.0)
 Requires-Dist: minio (>=7.1.15)
 Requires-Dist: pydantic (>=1.10.9,<2.0.0)

deltafi-1.0.7.dist-info/RECORD

@@ -0,0 +1,23 @@
+deltafi/__init__.py,sha256=PNMUt1SbY5DhoTxle11dPv21Pr_bo8paVc7TVu4Cs7E,706
+deltafi/action.py,sha256=ssbSGP-Nn15mD2UdHXmbBuNnVw5vPZY88RZRCV9xqMw,7330
+deltafi/actioneventqueue.py,sha256=HR1n-Co5Nq8IarYKR7Uzydi8mkCdOZbC-yJWHuA6KwA,1981
+deltafi/actiontype.py,sha256=gt8j4PgYuxoRbif5-FJi1fu4eLSy2J4oVbQ2qceRgqM,985
+deltafi/domain.py,sha256=-0ad8dGq-JcLyTnrIXHSMr7ZgBnailUJSZn1yPwEHI8,11197
+deltafi/exception.py,sha256=qQ2TY2QPtMU9t5RH8jmFo_cX98AJ-zOFWP_Wfm5U6qQ,1149
+deltafi/input.py,sha256=DuY280ZglZxEUkhjTtyr0g9Ohu2drn16UVhASH244nA,6290
+deltafi/logger.py,sha256=q76R_Gn8BfcASH3Zy0V82m5ot34bjTnSELyKbjhvx3E,2136
+deltafi/metric.py,sha256=eIDjZQVO53KuAFoEtjLNFwqMrp_7BC0Td9GLD1tb6sE,967
+deltafi/plugin.py,sha256=dIBbhwRLbBlF2NyuRz09-Jj8BSIkvJa74JkAzsfiYBc,11700
+deltafi/result.py,sha256=fRo83BZNcTfIDYDePGBJuB84FOnpgBiR23zbedJ2mD8,11512
+deltafi/storage.py,sha256=toq58EPZgzj2TfDF-YpFUmRnsWSjACA0KQAZzkM04xU,2740
+deltafi/test_kit/__init__.py,sha256=PNMUt1SbY5DhoTxle11dPv21Pr_bo8paVc7TVu4Cs7E,706
+deltafi/test_kit/assertions.py,sha256=2yJKCLTenJW44CprFLGorqfjVC6_6z-oAfPY1Q7J__0,1377
+deltafi/test_kit/compare_helpers.py,sha256=xeP6ToXvlCxVK3LX4glf9VsDfAmOoexYjkQpltfZ2T8,1580
+deltafi/test_kit/constants.py,sha256=epz0OS-qILcc8t7iIs5B3m2-wvZx8FpYpyb19ZsImbI,833
+deltafi/test_kit/format.py,sha256=NTG7UnwvXQlzn6WqKjMG9EZg0tKTu1g3CcNDD_6AQwk,4070
+deltafi/test_kit/framework.py,sha256=fEdllkV8luBwdZYkwMZo7-BzgRO_GIcg5e6lMyTD8fI,12855
+deltafi/test_kit/load.py,sha256=0K5vCQXbXl5pos9Yg_4qcZ9bJYfdQUGUosTxncv0xmw,4854
+deltafi/test_kit/transform.py,sha256=Eeqxev_dxoiFw72eAlYJ-3MDQH-kJtuHRGSIMbaatrg,2607
+deltafi-1.0.7.dist-info/METADATA,sha256=x24guqKi-7KbkMFCqJd-XGqgbeNGf7zhldbX34xfouQ,1749
+deltafi-1.0.7.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
+deltafi-1.0.7.dist-info/RECORD,,

deltafi-1.0.5.dist-info/RECORD

@@ -1,15 +0,0 @@
-deltafi/__init__.py,sha256=PNMUt1SbY5DhoTxle11dPv21Pr_bo8paVc7TVu4Cs7E,706
-deltafi/action.py,sha256=ssbSGP-Nn15mD2UdHXmbBuNnVw5vPZY88RZRCV9xqMw,7330
-deltafi/actioneventqueue.py,sha256=HR1n-Co5Nq8IarYKR7Uzydi8mkCdOZbC-yJWHuA6KwA,1981
-deltafi/actiontype.py,sha256=gt8j4PgYuxoRbif5-FJi1fu4eLSy2J4oVbQ2qceRgqM,985
-deltafi/domain.py,sha256=x0tS5DKv9tXYJK7eda1SVkkbOhRk_4scpXMfU-FZSJM,11025
-deltafi/exception.py,sha256=qQ2TY2QPtMU9t5RH8jmFo_cX98AJ-zOFWP_Wfm5U6qQ,1149
-deltafi/input.py,sha256=DuY280ZglZxEUkhjTtyr0g9Ohu2drn16UVhASH244nA,6290
-deltafi/logger.py,sha256=q76R_Gn8BfcASH3Zy0V82m5ot34bjTnSELyKbjhvx3E,2136
-deltafi/metric.py,sha256=eIDjZQVO53KuAFoEtjLNFwqMrp_7BC0Td9GLD1tb6sE,967
-deltafi/plugin.py,sha256=7eHU-P8GIqmK8fz9Re7sOWafZ9wxOayifWmmnyKXRrw,11666
-deltafi/result.py,sha256=aIFyytP5fOQBXmV-8IQleR5cQQrKVW36DEuOJSAGCgA,10706
-deltafi/storage.py,sha256=toq58EPZgzj2TfDF-YpFUmRnsWSjACA0KQAZzkM04xU,2740
-deltafi-1.0.5.dist-info/METADATA,sha256=ihKnu4QL1beQdQr8Js6c0cWwOBU0VIth8KEzyNW2P3w,1715
-deltafi-1.0.5.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
-deltafi-1.0.5.dist-info/RECORD,,