deltafi 2.0rc1719271450675__py3-none-any.whl → 2.0rc1720728217472__py3-none-any.whl

deltafi/test_kit/load.py

@@ -0,0 +1,128 @@
+#
+#    DeltaFi - Data transformation and enrichment platform
+#
+#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+from typing import List
+
+from deltafi.result import LoadResult, ReinjectResult
+
+from .assertions import *
+from .compare_helpers import GenericCompareHelper, CompareHelper
+from .framework import TestCaseBase, ActionTest, IOContent
+
+
+class LoadTestCase(TestCaseBase):
+    def __init__(self, fields: Dict):
+        super().__init__(fields)
+        self.metadata = {}
+        self.delete_metadata_keys = []
+        self.annotations = {}
+        self.domains = []
+        self.domain_cmp_tool = GenericCompareHelper()
+        self.children = []
+
+    def set_domain_compare_tool(self, helper: CompareHelper):
+        self.domain_cmp_tool = helper
+
+    def expect_load_result(self, metadata: Dict, delete_metadata_keys: List[str], annotations: Dict, domains: List):
+        self.expected_result_type = LoadResult
+        self.metadata = metadata
+        self.delete_metadata_keys = delete_metadata_keys
+        self.annotations = annotations
+        self.domains = domains
+
+    def add_reinject_child(self, filename: str, flow: str, content: IOContent, metadata: Dict):
+        self.expected_result_type = ReinjectResult
+        self.children.append(
+            {
+                "filename": filename,
+                "flow": flow,
+                "content": content,
+                "metadata": metadata
+            }
+        )
+
+
+class LoadActionTest(ActionTest):
+    def __init__(self, package_name: str):
+        """
+        Provides structure for testing DeltaFi Load action
+        Args:
+            package_name: name of the actions package for finding resources
+        """
+        super().__init__(package_name)
+
+    def load(self, test_case: LoadTestCase):
+        if test_case.expected_result_type == ReinjectResult:
+            self.expect_reinject_result(test_case)
+        elif test_case.expected_result_type == LoadResult:
+            self.expect_load_result(test_case)
+        else:
+            super().execute(test_case)
+
+    def expect_load_result(self, test_case: LoadTestCase):
+        result = super().run_and_check_result_type(test_case, LoadResult)
+        self.assert_load_result(test_case, result)
+
+    def expect_reinject_result(self, test_case: LoadTestCase):
+        result = super().run_and_check_result_type(test_case, ReinjectResult)
+        self.assert_reinject_result(test_case, result)
+
+    def assert_load_result(self, test_case: LoadTestCase, result: LoadResult):
+        # Check metrics
+        self.compare_metrics(test_case.expected_metrics, result.metrics)
+
+        # Check output
+        self.compare_all_output(test_case.compare_tool, result.content)
+
+        # Check metadata
+        assert_keys_and_values(test_case.metadata, result.metadata)
+
+        # Check deleted metadata
+        for key in test_case.delete_metadata_keys:
+            assert_key_in(key, result.delete_metadata_keys)
+
+        # Check annotations
+        assert_keys_and_values(test_case.annotations, result.annotations)
+
+        # Check domains
+        self.compare_domains(test_case.domain_cmp_tool, test_case.domains, result.domains)
+
+    def assert_reinject_result(self, test_case: LoadTestCase, actual: ReinjectResult):
+        # Check metrics
+        self.compare_metrics(test_case.expected_metrics, actual.metrics)
+
+        # Check reinject
+        assert_equal_len(test_case.children, actual.children)
+        for index, expected in enumerate(test_case.children):
+            reinject_child = actual.children[index]
+            assert_equal(expected['filename'], reinject_child.filename)
+            assert_equal(expected['flow'], reinject_child.flow)
+            assert_keys_and_values(expected['metadata'], reinject_child.metadata)
+
+            expected_value = expected['content']
+            child_content = reinject_child.content[0]
+            seg_id = child_content.segments[0].uuid
+            actual_content = self.content_service.get_output(seg_id)
+
+            if type(expected_value) == str:
+                test_case.domain_cmp_tool.compare(expected_value, actual_content, f"RI_child[{index}]")
+            elif type(expected_value) == IOContent:
+                expected_data = self.load_file(expected_value)
+                test_case.domain_cmp_tool.compare(expected_data, actual_content, f"RI_child[{index}]")
+            else:
+                raise ValueError(f"unknown expected_value type: {type(expected_value)}")

deltafi/test_kit/transform.py

@@ -18,34 +18,24 @@
 
 from typing import List
 
-from deltafi.result import TransformResult, TransformResults
+from deltafi.result import TransformResult
 
 from .assertions import *
-from .framework import TestCaseBase, ActionTest, IOContent
+from .framework import TestCaseBase, ActionTest
 
 
 class TransformTestCase(TestCaseBase):
     def __init__(self, fields: Dict):
         super().__init__(fields)
-        self.results = []
+        self.metadata = {}
+        self.delete_metadata_keys = []
+        self.annotations = {}
 
-    def expect_transform_result(self):
+    def expect_transform_result(self, metadata: Dict, delete_metadata_keys: List[str], annotations: Dict):
         self.expected_result_type = TransformResult
-
-    def expect_transform_results(self):
-        self.expected_result_type = TransformResults
-
-    def add_transform_result(self, content: List[IOContent], metadata: Dict, delete_metadata_keys: List[str],
-                             annotations: Dict, name: str = None):
-        self.results.append(
-            {
-                'content': content,
-                'metadata': metadata,
-                'delete_metadata_keys': delete_metadata_keys,
-                'annotations': annotations,
-                'name': name
-            }
-        )
+        self.metadata = metadata
+        self.delete_metadata_keys = delete_metadata_keys
+        self.annotations = annotations
 
 
 class TransformActionTest(ActionTest):
@@ -60,8 +50,6 @@ class TransformActionTest(ActionTest):
     def transform(self, test_case: TransformTestCase):
         if test_case.expected_result_type == TransformResult:
             self.expect_transform_result(test_case)
-        elif test_case.expected_result_type == TransformResults:
-            self.expect_transform_results(test_case)
         else:
             super().execute(test_case)
 
@@ -69,35 +57,19 @@ class TransformActionTest(ActionTest):
         result = super().run_and_check_result_type(test_case, TransformResult)
         self.assert_transform_result(test_case, result)
 
-    def expect_transform_results(self, test_case: TransformTestCase):
-        result = super().run_and_check_result_type(test_case, TransformResults)
-        self.assert_transform_results(test_case, result)
-
-    def assert_transform_results(self, test_case: TransformTestCase, result: TransformResults):
-        assert_equal_len(test_case.results, result.named_results)
-        for index, named_result in enumerate(result.named_results):
-            self.compare_one_transform_result(test_case, named_result.result, index)
-            expected = test_case.results[index]
-            if 'name' in expected:
-                assert_equal_with_label(expected["name"], named_result.name, f"name[{index}]")
-
     def assert_transform_result(self, test_case: TransformTestCase, result: TransformResult):
         # Check metrics
         self.compare_metrics(test_case.expected_metrics, result.metrics)
-        self.compare_one_transform_result(test_case, result, 0)
-
-    def compare_one_transform_result(self, test_case: TransformTestCase, result: TransformResult, index: int):
-        expected = test_case.results[index]
 
         # Check output
-        self.compare_content_list(test_case.compare_tool, expected['content'], result.content)
+        self.compare_all_output(test_case.compare_tool, result.content)
 
         # Check metadata
-        assert_keys_and_values(expected['metadata'], result.metadata)
+        assert_keys_and_values(test_case.metadata, result.metadata)
 
         # Check deleted metadata
-        for key in expected['delete_metadata_keys']:
+        for key in test_case.delete_metadata_keys:
             assert_key_in(key, result.delete_metadata_keys)
 
         # Check annotations
-        assert_keys_and_values(expected['annotations'], result.annotations)
+        assert_keys_and_values(test_case.annotations, result.annotations)

{deltafi-2.0rc1719271450675.dist-info → deltafi-2.0rc1720728217472.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltafi
-Version: 2.0rc1719271450675
+Version: 2.0rc1720728217472
 Summary: SDK for DeltaFi plugins and actions
 License: Apache License, Version 2.0
 Keywords: deltafi
@@ -20,11 +20,11 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Software Development
 Requires-Dist: deepdiff (>=6.7.1)
 Requires-Dist: json-logging (>=1.3.0)
-Requires-Dist: minio (>=7.2.5)
-Requires-Dist: pydantic (>=2.7.1)
-Requires-Dist: redis (>=5.0.4)
+Requires-Dist: minio (>=7.2.3)
+Requires-Dist: pydantic (>=2.5.3)
+Requires-Dist: redis (>=5.0.1)
 Requires-Dist: requests (>=2.31.0)
-Requires-Dist: urllib3 (>=2.2.1)
+Requires-Dist: urllib3 (>=2.1.0)
 Project-URL: Bug Reports, https://chat.deltafi.org/deltafi/channels/bug-reports
 Project-URL: Documentation, https://docs.deltafi.org/#/
 Project-URL: Source Code, https://gitlab.com/deltafi/deltafi

{deltafi-2.0rc1719271450675.dist-info → deltafi-2.0rc1720728217472.dist-info}/RECORD

@@ -1,15 +1,15 @@
 deltafi/__init__.py,sha256=qv6y7PpBG0ekTN9EuD9Lj8JYOLVqZA6tvHwgjplncAM,709
-deltafi/action.py,sha256=DGtbzrOM5BJRfn0NGAm19rLgzQyt6iiXy2_VpxHSyaw,5343
+deltafi/action.py,sha256=TM8HwfC12GgfHMBWPmO88N7J9xvwGO8BMwxKTNqwg9A,10498
 deltafi/actioneventqueue.py,sha256=mCRE1PFXy_KvaeezTeMze5N400CO1V06zEF8FFD6xZk,2847
-deltafi/actiontype.py,sha256=V7JbTgHQIaWQ2N36pG-XV7kAXhq8qvj954z71iHw_wI,913
-deltafi/domain.py,sha256=TZoawrb09TIMhCEAQKP2GVLdT4c3qBRsrl4v_F4R-5Q,12072
-deltafi/exception.py,sha256=Z8LwxxTiTyuG3v7FAsjfz6cAqRAJFKK2q_2TbEDrIJE,943
+deltafi/actiontype.py,sha256=SgCxs5Kn_6BuzuKDk5h93ZLtfdbzV0sWLcEaaB0mqlU,997
+deltafi/domain.py,sha256=M97_n6KsMmIxCh3h622jNj1n6c5iMEAOn-SFH36oLis,12161
+deltafi/exception.py,sha256=qQ2TY2QPtMU9t5RH8jmFo_cX98AJ-zOFWP_Wfm5U6qQ,1149
 deltafi/genericmodel.py,sha256=8VXWgFMjIwG4o3rkcLOa3bQH0Nf-T1Kw621QtMw-4q0,1090
-deltafi/input.py,sha256=2qQcOWqz7lOXZIZipTz5XtcGESpjdnUAsZ7b9YqHw7M,1656
+deltafi/input.py,sha256=DuY280ZglZxEUkhjTtyr0g9Ohu2drn16UVhASH244nA,6290
 deltafi/logger.py,sha256=q76R_Gn8BfcASH3Zy0V82m5ot34bjTnSELyKbjhvx3E,2136
 deltafi/metric.py,sha256=eIDjZQVO53KuAFoEtjLNFwqMrp_7BC0Td9GLD1tb6sE,967
-deltafi/plugin.py,sha256=k2VO_5OJUqTGn6yN00KY-DoOv6Jhe2QoMpy9hr9vUi4,12407
-deltafi/result.py,sha256=AMjGygjvix9PdUbQvSa_gZEkTn9BJh2p-BdirqdjJpM,8212
+deltafi/plugin.py,sha256=GkDq9oc5O0QYUZ7CdYcP1f9zRT38ufvzudZ4addHVS8,13076
+deltafi/result.py,sha256=KkFKewPxcP8_Slp0D2COJ43h5_fF5Vti9ocWtp8pjpc,14911
 deltafi/storage.py,sha256=toq58EPZgzj2TfDF-YpFUmRnsWSjACA0KQAZzkM04xU,2740
 deltafi/test_kit/__init__.py,sha256=qv6y7PpBG0ekTN9EuD9Lj8JYOLVqZA6tvHwgjplncAM,709
 deltafi/test_kit/assertions.py,sha256=2eahqmbedhnHXMguiJHdaV-6sx4_jvCbxH4HSzx6PV8,1378
@@ -18,9 +18,11 @@ deltafi/test_kit/constants.py,sha256=epz0OS-qILcc8t7iIs5B3m2-wvZx8FpYpyb19ZsImbI
 deltafi/test_kit/domain.py,sha256=uqpDravlb6E3Ddm4QpRaJTdiVuQ2MMdZnzc4Oi9EZJA,2090
 deltafi/test_kit/egress.py,sha256=nF7YYak-_X35m7h2rSIcju86IJBf8cVaFHs7xw7-_b8,1899
 deltafi/test_kit/enrich.py,sha256=M682cqInVS4aipI0jOrR25x7m0ErrpjGpebXGZDPwxo,2587
-deltafi/test_kit/framework.py,sha256=KCnLVURkbnRZvsCvHDvAaXRrZACzA2ZcHJn803PatD8,13400
-deltafi/test_kit/transform.py,sha256=q9zQdZswTKwU5Om8E9Lwg0G5JI8LEXuRZjT9oam-Zoo,4085
+deltafi/test_kit/format.py,sha256=j5spoimv4fIB7wrPSUS-aLcYvDJS0ntu4oSJjQ1INsw,4170
+deltafi/test_kit/framework.py,sha256=5PTWoKu8eMU5GaVLmyoghoOrp6G65efXwqZKlUlyEaM,14733
+deltafi/test_kit/load.py,sha256=dzXZDjb5BKxM9F5E2B06ArnGy0HvE1kwPizL8VybsOQ,5055
+deltafi/test_kit/transform.py,sha256=AZ5oamhaOm4yaR07_gCoTlDiHkd4KwiAWWNj3mnYp8Q,2706
 deltafi/test_kit/validate.py,sha256=NnXD5amOE_9a4Zr-v9DKYIv5f_HvwtmlI-IdFtVp7ko,1933
-deltafi-2.0rc1719271450675.dist-info/METADATA,sha256=nPoSv2XejK3Lhqco7TFZaE8uOdPO5-ivIT3l-zB_aOo,1446
-deltafi-2.0rc1719271450675.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-deltafi-2.0rc1719271450675.dist-info/RECORD,,
+deltafi-2.0rc1720728217472.dist-info/METADATA,sha256=lhhN66ZvCacThjEosjup0r8FOzyPPvEs-Lt0LWQnFQc,1446
+deltafi-2.0rc1720728217472.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+deltafi-2.0rc1720728217472.dist-info/RECORD,,