deltafi 1.0.6__py3-none-any.whl → 1.0.7__py3-none-any.whl

deltafi/domain.py

@@ -25,6 +25,7 @@ from deltafi.storage import ContentService, Segment
 
 class Context(NamedTuple):
     did: str
+    action_flow: str
     action_name: str
     source_filename: str
     ingress_flow: str
@@ -37,7 +38,9 @@ class Context(NamedTuple):
     @classmethod
     def create(cls, context: dict, hostname: str, content_service: ContentService, logger: Logger):
         did = context['did']
-        action_name = context['name']
+        action_name_parts = context['name'].split(".")
+        action_flow = action_name_parts[0]
+        action_name = action_name_parts[1]
         if 'sourceFilename' in context:
             source_filename = context['sourceFilename']
         else:
@@ -49,6 +52,7 @@ class Context(NamedTuple):
             egress_flow = None
         system = context['systemName']
         return Context(did=did,
+                       action_flow=action_flow,
                        action_name=action_name,
                        source_filename=source_filename,
                        ingress_flow=ingress_flow,

deltafi/plugin.py

@@ -270,7 +270,7 @@ class Plugin(object):
 
                 response = {
                     'did': event.context.did,
-                    'action': event.context.action_name,
+                    'action': event.context.action_flow + "." + event.context.action_name,
                     'start': start_time,
                     'stop': time.time(),
                     'type': result.result_type,

deltafi/result.py

@@ -17,8 +17,8 @@
 #
 
 import abc
-from typing import Dict, List
 import uuid
+from typing import Dict, List
 
 from deltafi.domain import Content, Context
 from deltafi.metric import Metric
@@ -123,40 +123,66 @@ class FormatResult(Result):
     def __init__(self, context: Context):
         super().__init__('format', 'FORMAT', context)
         self.content = None
+        self.delete_metadata_keys = []
         self.metadata = {}
 
     def add_metadata(self, key: str, value: str):
         self.metadata[key] = value
         return self
 
+    def delete_metadata_key(self, key: str):
+        self.delete_metadata_keys.append(key)
+        return self
+
     def set_content(self, content: Content):
         self.content = content
         return self
 
     def save_string_content(self, string_data: str, name: str, media_type: str):
         segment = self.context.content_service.put_str(self.context.did, string_data)
-        self.content = Content(name=name, segments=[segment], media_type=media_type, content_service=self.context.content_service)
+        self.content = Content(name=name, segments=[segment], media_type=media_type,
+                               content_service=self.context.content_service)
         return self
 
     def save_byte_content(self, byte_data: bytes, name: str, media_type: str):
         segment = self.context.content_service.put_bytes(self.context.did, byte_data)
-        self.content = Content(name=name, segments=[segment], media_type=media_type, content_service=self.context.content_service)
+        self.content = Content(name=name, segments=[segment], media_type=media_type,
+                               content_service=self.context.content_service)
         return self
 
     def response(self):
         return {
             'content': self.content.json(),
-            'metadata': self.metadata
+            'metadata': self.metadata,
+            'deleteMetadataKeys': self.delete_metadata_keys
         }
 
 
+class ChildFormatResult:
+    def __init__(self, format_result: FormatResult = None):
+        self._did = str(uuid.uuid4())
+        self.format_result = format_result
+
+    @property
+    def did(self):
+        return self._did
+
+    def response(self):
+        res = self.format_result.response()
+        res["did"] = self._did
+        return res
+
+
 class FormatManyResult(Result):
     def __init__(self, context: Context):
         super().__init__('formatMany', 'FORMAT_MANY', context)
         self.format_results = []
 
-    def add_format_result(self, format_result: FormatResult):
-        self.format_results.append(format_result)
+    def add_format_result(self, format_result):
+        if isinstance(format_result, ChildFormatResult):
+            self.format_results.append(format_result)
+        else:
+            self.format_results.append(ChildFormatResult(format_result))
         return self
 
     def response(self):
@@ -184,12 +210,14 @@ class LoadResult(Result):
 
     def save_string_content(self, string_data: str, name: str, media_type: str):
         segment = self.context.content_service.put_str(self.context.did, string_data)
-        self.content.append(Content(name=name, segments=[segment], media_type=media_type, content_service=self.context.content_service))
+        self.content.append(
+            Content(name=name, segments=[segment], media_type=media_type, content_service=self.context.content_service))
         return self
 
     def save_byte_content(self, byte_data: bytes, name: str, media_type: str):
         segment = self.context.content_service.put_bytes(self.context.did, byte_data)
-        self.content.append(Content(name=name, segments=[segment], media_type=media_type, content_service=self.context.content_service))
+        self.content.append(
+            Content(name=name, segments=[segment], media_type=media_type, content_service=self.context.content_service))
         return self
 
     def add_metadata(self, key: str, value: str):
@@ -300,12 +328,14 @@ class TransformResult(Result):
 
     def save_string_content(self, string_data: str, name: str, media_type: str):
         segment = self.context.content_service.put_str(self.context.did, string_data)
-        self.content.append(Content(name=name, segments=[segment], media_type=media_type, content_service=self.context.content_service))
+        self.content.append(
+            Content(name=name, segments=[segment], media_type=media_type, content_service=self.context.content_service))
         return self
 
     def save_byte_content(self, byte_data: bytes, name: str, media_type: str):
         segment = self.context.content_service.put_bytes(self.context.did, byte_data)
-        self.content.append(Content(name=name, segments=[segment], media_type=media_type, content_service=self.context.content_service))
+        self.content.append(
+            Content(name=name, segments=[segment], media_type=media_type, content_service=self.context.content_service))
         return self
 
     def add_metadata(self, key: str, value: str):

deltafi/test_kit/format.py

@@ -0,0 +1,99 @@
+#
+#    DeltaFi - Data transformation and enrichment platform
+#
+#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+from typing import List
+
+from deltafi.result import FormatResult, FormatManyResult
+
+from .assertions import *
+from .framework import TestCaseBase, ActionTest
+
+
+class FormatTestCase(TestCaseBase):
+    def __init__(self, fields: Dict):
+        super().__init__(fields)
+        self.metadata = {}
+        self.delete_metadata_keys = []
+        self.expected_format_many_result = []
+
+    def expect_format_result(self, metadata: Dict, delete_metadata_keys: List[str]):
+        self.expected_result_type = FormatResult
+        self.metadata = metadata
+        self.delete_metadata_keys = delete_metadata_keys
+
+    def add_format_many_result(self, metadata: Dict, delete_metadata_keys: List):
+        self.expected_result_type = FormatManyResult
+        self.expected_format_many_result.append(
+            {
+                "metadata": metadata,
+                "delete_metadata_keys": delete_metadata_keys
+            }
+        )
+
+
+class FormatActionTest(ActionTest):
+    def __init__(self, package_name: str):
+        """
+        Provides structure for testing DeltaFi Format action
+        Args:
+            package_name: name of the actions package for finding resources
+        """
+        super().__init__(package_name)
+
+    def format(self, test_case: FormatTestCase):
+        if test_case.expected_result_type == FormatManyResult:
+            self.expect_format_many_result(test_case)
+        elif test_case.expected_result_type == FormatResult:
+            self.expect_format_result(test_case)
+        else:
+            super().execute(test_case)
+
+    def expect_format_result(self, test_case: FormatTestCase):
+        result = super().run_and_check_result_type(test_case, FormatResult)
+        self.assert_format_result(test_case, result)
+
+    def expect_format_many_result(self, test_case: FormatTestCase):
+        result = super().run_and_check_result_type(test_case, FormatManyResult)
+        self.assert_format_many_result(test_case, result)
+
+    def assert_format_result(self, test_case: FormatTestCase, result: FormatResult):
+        # Check output
+        if result.content is None:
+            self.compare_all_output(test_case.compare_tool, [])
+        else:
+            self.compare_all_output(test_case.compare_tool, [result.content])
+
+        # Check metadata
+        assert_keys_and_values(test_case.metadata, result.metadata)
+
+        # Check deleted metadata
+        for key in test_case.delete_metadata_keys:
+            assert_key_in(key, result.delete_metadata_keys)
+
+    def assert_format_many_result(self, test_case: FormatTestCase, actual: FormatManyResult):
+        assert_equal_len(test_case.expected_format_many_result, actual.format_results)
+        for index, expected_child_result in enumerate(test_case.expected_format_many_result):
+            actual_child = actual.format_results[index]
+            self.compare_one_content(test_case.compare_tool,
+                                     self.expected_outputs[index],
+                                     actual_child.format_result.content,
+                                     index)
+
+            assert_keys_and_values(expected_child_result['metadata'],
+                                   actual_child.format_result.metadata)
+            for key in expected_child_result['delete_metadata_keys']:
+                assert_key_in(key, actual_child.format_result.delete_metadata_keys)

deltafi/test_kit/framework.py

@@ -118,8 +118,7 @@ class TestCaseBase(ABC):
         A test case for DeltaFi python actions
         :param data: Dict of test case fields
         - action: instance of the action being tested
-        - pkg_name: str: name of the actions package for finding resources
-        - test_name: str: name of the test for finding test files, e.g. test/data/{test_name)
+        - data_dir: str: subdirectory name (e.g., test name) for locating test data files, i.e., test/data/{data_dir)
         - compare_tool: (optional) CompareHelper instanced for comparing output content
         - inputs: (optional) List[IOContent]: input content to action
         - parameters: (optional) Dict: map of action input parameters
@@ -132,15 +131,10 @@ class TestCaseBase(ABC):
         else:
             raise ValueError("action is required")
 
-        if "pkg_name" in data:
-            self.pkg_name = data["pkg_name"]
+        if "data_dir" in data:
+            self.data_dir = data["data_dir"]
         else:
-            raise ValueError("pkg_name is required")
-
-        if "test_name" in data:
-            self.test_name = data["test_name"]
-        else:
-            raise ValueError("test_name is required")
+            raise ValueError("data_dir is required")
 
         if "compare_tool" in data:
             self.compare_tool = data["compare_tool"]
@@ -178,11 +172,17 @@ class TestCaseBase(ABC):
 
 
 class ActionTest(ABC):
-    def __init__(self):
+    def __init__(self, package_name: str):
+        """
+        Provides structure for testing DeltaFi actions
+        Args:
+            package_name: name of the actions package for finding resources
+        """
         self.content_service = InternalContentService()
         self.did = ""
         self.expected_outputs = []
         self.loaded_inputs = []
+        self.package_name = package_name
         self.res_path = ""
 
     def __reset__(self):
@@ -199,8 +199,8 @@ class ActionTest(ABC):
         return contents
 
     def get_contents(self, test_case: TestCaseBase):
-        pkg_path = files(test_case.pkg_name)
-        self.res_path = pkg_path.joinpath(f"test/data/{test_case.test_name}/")
+        pkg_path = files(self.package_name)
+        self.res_path = pkg_path.joinpath(f"test/data/{test_case.data_dir}/")
 
         # Load inputs
         for input_ioc in test_case.inputs:
@@ -240,6 +240,7 @@ class ActionTest(ABC):
     def make_context(self, test_case: TestCaseBase):
         action_name = INGRESS_FLOW + "." + test_case.action.__class__.__name__
         return Context(did=self.did,
+                       action_flow=INGRESS_FLOW,
                        action_name=action_name,
                        source_filename=test_case.file_name,
                        ingress_flow=INGRESS_FLOW,
@@ -293,16 +294,19 @@ class ActionTest(ABC):
         assert_equal(expected.content_type, actual.media_type)
         assert_equal(expected.name, actual.name)
 
+    def compare_one_content(self, comparitor: CompareHelper, expected: LoadedContent, actual, index):
+        self.compare_content_details(expected, actual)
+        seg_id = actual.segments[0].uuid
+        comparitor.compare(
+            expected.data,
+            self.content_service.get_output(seg_id),
+            f"Content[{index}]"
+        )
+
     def compare_all_output(self, comparitor: CompareHelper, content: List):
         assert_equal_len(self.expected_outputs, content)
         for index, expected in enumerate(self.expected_outputs):
-            self.compare_content_details(expected, content[index])
-            seg_id = content[index].segments[0].uuid
-            comparitor.compare(
-                expected.data,
-                self.content_service.get_output(seg_id),
-                f"Content[{index}]"
-            )
+            self.compare_one_content(comparitor, expected, content[index], index)
 
     def compare_domains(self, comparitor: CompareHelper, expected_items: List[Dict], results: List[Dict]):
         assert_equal_len(expected_items, results)

deltafi/test_kit/load.py

@@ -44,14 +44,26 @@ class LoadTestCase(TestCaseBase):
         self.annotations = annotations
         self.domains = domains
 
-    def expect_reinject_result(self, children: List):
+    def add_reinject_child(self, filename: str, flow: str, content: IOContent, metadata: Dict):
         self.expected_result_type = ReinjectResult
-        self.children = children
+        self.children.append(
+            {
+                "filename": filename,
+                "flow": flow,
+                "content": content,
+                "metadata": metadata
+            }
+        )
 
 
 class LoadActionTest(ActionTest):
-    def __init__(self):
-        super().__init__()
+    def __init__(self, package_name: str):
+        """
+        Provides structure for testing DeltaFi Load action
+        Args:
+            package_name: name of the actions package for finding resources
+        """
+        super().__init__(package_name)
 
     def load(self, test_case: LoadTestCase):
         if test_case.expected_result_type == ReinjectResult:

deltafi/test_kit/transform.py

@@ -38,8 +38,13 @@ class TransformTestCase(TestCaseBase):
 
 
 class TransformActionTest(ActionTest):
-    def __init__(self):
-        super().__init__()
+    def __init__(self, package_name: str):
+        """
+        Provides structure for testing DeltaFi Transform action
+        Args:
+            package_name: name of the actions package for finding resources
+        """
+        super().__init__(package_name)
 
     def transform(self, test_case: TransformTestCase):
         if test_case.expected_result_type == TransformResult:

{deltafi-1.0.6.dist-info → deltafi-1.0.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltafi
-Version: 1.0.6
+Version: 1.0.7
 Summary: SDK for DeltaFi plugins and actions
 License: Apache License, Version 2.0
 Keywords: deltafi

{deltafi-1.0.6.dist-info → deltafi-1.0.7.dist-info}/RECORD

@@ -2,21 +2,22 @@ deltafi/__init__.py,sha256=PNMUt1SbY5DhoTxle11dPv21Pr_bo8paVc7TVu4Cs7E,706
 deltafi/action.py,sha256=ssbSGP-Nn15mD2UdHXmbBuNnVw5vPZY88RZRCV9xqMw,7330
 deltafi/actioneventqueue.py,sha256=HR1n-Co5Nq8IarYKR7Uzydi8mkCdOZbC-yJWHuA6KwA,1981
 deltafi/actiontype.py,sha256=gt8j4PgYuxoRbif5-FJi1fu4eLSy2J4oVbQ2qceRgqM,985
-deltafi/domain.py,sha256=x0tS5DKv9tXYJK7eda1SVkkbOhRk_4scpXMfU-FZSJM,11025
+deltafi/domain.py,sha256=-0ad8dGq-JcLyTnrIXHSMr7ZgBnailUJSZn1yPwEHI8,11197
 deltafi/exception.py,sha256=qQ2TY2QPtMU9t5RH8jmFo_cX98AJ-zOFWP_Wfm5U6qQ,1149
 deltafi/input.py,sha256=DuY280ZglZxEUkhjTtyr0g9Ohu2drn16UVhASH244nA,6290
 deltafi/logger.py,sha256=q76R_Gn8BfcASH3Zy0V82m5ot34bjTnSELyKbjhvx3E,2136
 deltafi/metric.py,sha256=eIDjZQVO53KuAFoEtjLNFwqMrp_7BC0Td9GLD1tb6sE,967
-deltafi/plugin.py,sha256=7eHU-P8GIqmK8fz9Re7sOWafZ9wxOayifWmmnyKXRrw,11666
-deltafi/result.py,sha256=aIFyytP5fOQBXmV-8IQleR5cQQrKVW36DEuOJSAGCgA,10706
+deltafi/plugin.py,sha256=dIBbhwRLbBlF2NyuRz09-Jj8BSIkvJa74JkAzsfiYBc,11700
+deltafi/result.py,sha256=fRo83BZNcTfIDYDePGBJuB84FOnpgBiR23zbedJ2mD8,11512
 deltafi/storage.py,sha256=toq58EPZgzj2TfDF-YpFUmRnsWSjACA0KQAZzkM04xU,2740
 deltafi/test_kit/__init__.py,sha256=PNMUt1SbY5DhoTxle11dPv21Pr_bo8paVc7TVu4Cs7E,706
 deltafi/test_kit/assertions.py,sha256=2yJKCLTenJW44CprFLGorqfjVC6_6z-oAfPY1Q7J__0,1377
 deltafi/test_kit/compare_helpers.py,sha256=xeP6ToXvlCxVK3LX4glf9VsDfAmOoexYjkQpltfZ2T8,1580
 deltafi/test_kit/constants.py,sha256=epz0OS-qILcc8t7iIs5B3m2-wvZx8FpYpyb19ZsImbI,833
-deltafi/test_kit/framework.py,sha256=T3F3owpsis7vZXmJ2bs6WvHZ8KyEuA7t3m1nHXmGJ2k,12637
-deltafi/test_kit/load.py,sha256=CNy1mIbQ24rTSQ5VedmTQu8SpkPEauxoLfEOfMea4j0,4432
-deltafi/test_kit/transform.py,sha256=jQ_gH9hFYonGgIqFvNDc0hhID_02mL4jdfoEXEdOpUc,2398
-deltafi-1.0.6.dist-info/METADATA,sha256=c1xxvw6K0zgxhBB6KFLoyZFoCN1H5qvy4alddt4SU5A,1749
-deltafi-1.0.6.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
-deltafi-1.0.6.dist-info/RECORD,,
+deltafi/test_kit/format.py,sha256=NTG7UnwvXQlzn6WqKjMG9EZg0tKTu1g3CcNDD_6AQwk,4070
+deltafi/test_kit/framework.py,sha256=fEdllkV8luBwdZYkwMZo7-BzgRO_GIcg5e6lMyTD8fI,12855
+deltafi/test_kit/load.py,sha256=0K5vCQXbXl5pos9Yg_4qcZ9bJYfdQUGUosTxncv0xmw,4854
+deltafi/test_kit/transform.py,sha256=Eeqxev_dxoiFw72eAlYJ-3MDQH-kJtuHRGSIMbaatrg,2607
+deltafi-1.0.7.dist-info/METADATA,sha256=x24guqKi-7KbkMFCqJd-XGqgbeNGf7zhldbX34xfouQ,1749
+deltafi-1.0.7.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
+deltafi-1.0.7.dist-info/RECORD,,