deltafi 2.0rc1719271450675__tar.gz → 2.0rc1720817063181__tar.gz

{deltafi-2.0rc1719271450675 → deltafi-2.0rc1720817063181}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltafi
-Version: 2.0rc1719271450675
+Version: 2.0rc1720817063181
 Summary: SDK for DeltaFi plugins and actions
 License: Apache License, Version 2.0
 Keywords: deltafi

{deltafi-2.0rc1719271450675 → deltafi-2.0rc1720817063181}/deltafi/action.py

@@ -38,8 +38,8 @@ class Action(ABC):
     def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
         pass
 
-    def collect(self, action_inputs: List[Any]):
-        raise RuntimeError(f"Collect is not supported for {self.__class__.__name__}")
+    def join(self, action_inputs: List[Any]):
+        raise RuntimeError(f"Join is not supported for {self.__class__.__name__}")
 
     @abstractmethod
     def execute(self, context: Context, action_input: Any, params: BaseModel):
@@ -48,10 +48,10 @@ class Action(ABC):
     def execute_action(self, event):
         if event.delta_file_messages is None or not len(event.delta_file_messages):
             raise RuntimeError(f"Received event with no delta file messages for did {event.context.did}")
-        if event.context.collect is not None:
+        if event.context.join is not None:
             result = self.execute(
                 event.context,
-                self.collect([self.build_input(event.context, delta_file_message)
+                self.join([self.build_input(event.context, delta_file_message)
                               for delta_file_message in event.delta_file_messages]),
                 self.param_class().model_validate(event.params))
         else:
@@ -119,7 +119,7 @@ class TransformAction(Action, ABC):
     def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
         return TransformInput(content=delta_file_message.content_list, metadata=delta_file_message.metadata)
 
-    def collect(self, transform_inputs: List[TransformInput]):
+    def join(self, transform_inputs: List[TransformInput]):
         all_content = []
         all_metadata = {}
         for transform_input in transform_inputs:

{deltafi-2.0rc1719271450675 → deltafi-2.0rc1720817063181}/deltafi/domain.py

@@ -50,8 +50,8 @@ class Context(NamedTuple):
     hostname: str
     system_name: str
     content_service: ContentService
-    collect: dict = None
-    collected_dids: List[str] = None
+    join: dict = None
+    joined_dids: List[str] = None
     memo: str = None
     logger: Logger = None
 
@@ -94,14 +94,14 @@ class Context(NamedTuple):
             system_name = context['systemName']
         else:
             system_name = None
-        if 'collect' in context:
-            collect = context['collect']
+        if 'join' in context:
+            join = context['join']
         else:
-            collect = None
-        if 'collectedDids' in context:
-            collected_dids = context['collectedDids']
+            join = None
+        if 'joinedDids' in context:
+            joined_dids = context['joinedDids']
         else:
-            collected_dids = None
+            joined_dids = None
         if 'memo' in context:
             memo = context['memo']
         else:
@@ -117,8 +117,8 @@ class Context(NamedTuple):
                        action_version=action_version,
                        hostname=hostname,
                        system_name=system_name,
-                       collect=collect,
-                       collected_dids=collected_dids,
+                       join=join,
+                       joined_dids=joined_dids,
                        memo=memo,
                        content_service=content_service,
                        logger=logger)

{deltafi-2.0rc1719271450675 → deltafi-2.0rc1720817063181}/deltafi/test_kit/framework.py

@@ -37,15 +37,20 @@ class IOContent:
     The IOContent class holds the details for loading input or output
      content into the test framework.
     Attributes:
-        file_name (str): The name of file in test/data.
-        content_name (str): The name of the content.
-        content_type (str): The media type of the content
-        offset (int): Offset to use in Segment
-        content_bytes (str): Bypass file read, and uses these bytes for content
+        file_name (str)    : The name of file in test/data.
+        content_name (str) : The name of the content.
+        content_type (str) : The media type of the content
+        offset (int)       : Offset to use in Segment
+        content_bytes (str): Optional.  If set to a String of length greater than zero, indicates to consumers of this
+                             IOContent that they should bypass file read and use these bytes for content.
+        no_content (bool)  : Optional.  If 'True', then consumers should not attempt to interpret content but should
+                             apply other aspects of this IOContent.  When 'True', 'content_bytes' should be ignored and
+                             loaded content, if any, should be interpreted as empty String or otherwise as documented by
+                             the consumer.
     """
 
     def __init__(self, file_name: str, content_name: str = None, content_type: str = None, offset: int = 0,
-                 content_bytes: str = ""):
+                 content_bytes: str = "", no_content: bool = False):
         self.file_name = file_name
         if content_name is None:
             self.content_name = file_name
@@ -56,7 +61,11 @@ class IOContent:
         else:
             self.content_type = content_type
         self.offset = offset
-        self.content_bytes = content_bytes
+        self.no_content = no_content
+        if no_content:
+            self.content_bytes = None
+        else:
+            self.content_bytes = content_bytes
         self.segment_uuid = uuid.uuid4()
 
     @classmethod
@@ -79,7 +88,10 @@ class LoadedContent:
         if data is not None:
             self.data = data
         else:
-            self.data = ioc.content_bytes
+            if ioc.no_content:
+                self.data = ""
+            else:
+                self.data = ioc.content_bytes
         self.segment = Segment.from_dict(
             {"uuid": str(ioc.segment_uuid), "offset": self.offset, "size": len(self.data), "did": did})
 
@@ -127,6 +139,8 @@ class TestCaseBase(ABC):
         - inputs: (optional) List[IOContent]: input content to action
         - parameters: (optional) Dict: map of action input parameters
         - in_meta: (optional) Dict: map of metadata as input to action
+        - join_meta: (optional): List[Dict]: When a List is provided, this enables the JOIN portion of an action.
+        When using JOIN, join_meta must match the size of inputs, though the Dict can be empty
         - did: (optional): str: overrides random DID
         """
         if "action" in data:
@@ -153,6 +167,7 @@ class TestCaseBase(ABC):
         self.err_or_filt_cause = None
         self.err_or_filt_context = None
         self.err_or_filt_annotations = None
+        self.join_meta = data["join_meta"] if "join_meta" in data else None
         self.expected_metrics = []
 
     def add_metric(self, metric: Metric):
@@ -163,7 +178,7 @@ class TestCaseBase(ABC):
         A Sets the expected output of the action to an Error Result
         :param cause: the expected error cause
         :param context: the expected error context
-        :param annotations (Optional): Dict: the expected annotations
+        :param annotations: Dict: (Optional) the expected annotations
         """
         self.expected_result_type = ErrorResult
         self.err_or_filt_cause = cause
@@ -174,8 +189,8 @@ class TestCaseBase(ABC):
         """
         A Sets the expected output of the action to a Filter Result
         :param cause: the expected filter cause (message)
-        :param context (Optional): the expected filter context
-        :param annotations (Optional): Dict: the expected annotations
+        :param context: (Optional) the expected filter context
+        :param annotations: Dict: (Optional) the expected annotations
         """
         self.expected_result_type = FilterResult
         self.err_or_filt_cause = cause
@@ -217,12 +232,12 @@ class ActionTest(ABC):
 
         # Load inputs
         for input_ioc in test_case.inputs:
-            if len(input_ioc.content_bytes) == 0:
+            if not input_ioc.no_content and len(input_ioc.content_bytes) == 0:
                 self.loaded_inputs.append(LoadedContent(self.did, input_ioc, self.load_file(input_ioc)))
             else:
                 self.loaded_inputs.append(LoadedContent(self.did, input_ioc, None))
 
-    def make_content_list(self, test_case: TestCaseBase):
+    def make_content_list(self):
         content_list = []
         for loaded_input in self.loaded_inputs:
             c = Content(name=loaded_input.name, segments=[loaded_input.segment], media_type=loaded_input.content_type,
@@ -232,15 +247,25 @@ class ActionTest(ABC):
 
         return content_list
 
-    def make_df_msg(self, test_case: TestCaseBase):
-        content_list = self.make_content_list(test_case)
+    def make_df_msgs(self, test_case: TestCaseBase):
+        content_list = self.make_content_list()
         self.content_service.load(self.loaded_inputs)
 
-        return DeltaFileMessage(metadata=test_case.in_meta,
-                                content_list=content_list)
+        delta_file_messages = []
+
+        if test_case.join_meta is None:
+            delta_file_messages.append(DeltaFileMessage(metadata=test_case.in_meta, content_list=content_list))
+        else:
+            for index, content in enumerate(content_list):
+                delta_file_messages.append(DeltaFileMessage(
+                    metadata=test_case.join_meta[index],
+                    content_list=[content]))
+
+        return delta_file_messages
 
     def make_context(self, test_case: TestCaseBase):
         action_name = INGRESS_FLOW + "." + test_case.action.__class__.__name__
+        join = {} if test_case.join_meta else None
         return Context(
             did=self.did,
             delta_file_name=test_case.file_name,
@@ -253,10 +278,11 @@ class ActionTest(ABC):
             hostname=HOSTNAME,
             system_name=SYSTEM,
             content_service=self.content_service,
+            join=join,
             logger=get_logger())
 
     def make_event(self, test_case: TestCaseBase):
-        return Event(delta_file_messages=[self.make_df_msg(test_case)], context=self.make_context(test_case),
+        return Event(delta_file_messages=self.make_df_msgs(test_case), context=self.make_context(test_case),
                      params=test_case.parameters, queue_name="", return_address="")
 
     def call_action(self, test_case: TestCaseBase):
@@ -311,7 +337,7 @@ class ActionTest(ABC):
     def compare_content_list(self, comparator: CompareHelper, expected_outputs: List[IOContent], content: List):
         assert_equal_len(expected_outputs, content)
         for index, expected_ioc in enumerate(expected_outputs):
-            if len(expected_ioc.content_bytes) == 0:
+            if not expected_ioc.no_content and len(expected_ioc.content_bytes) == 0:
                 expected = LoadedContent(self.did, expected_ioc, self.load_file(expected_ioc))
             else:
                 expected = LoadedContent(self.did, expected_ioc, None)

{deltafi-2.0rc1719271450675 → deltafi-2.0rc1720817063181}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "deltafi"
-version = "2.0rc1719271450675"
+version = "2.0rc1720817063181"
 description = "SDK for DeltaFi plugins and actions"
 authors = ["DeltaFi <deltafi@systolic.com>"]
 license = "Apache License, Version 2.0"

deltafi-2.0rc1719271450675/deltafi/test_kit/domain.py

@@ -1,59 +0,0 @@
-#
-#    DeltaFi - Data transformation and enrichment platform
-#
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-#
-
-from deltafi.result import DomainResult
-
-from .assertions import *
-from .framework import TestCaseBase, ActionTest
-
-
-class DomainTestCase(TestCaseBase):
-    def __init__(self, fields: Dict):
-        super().__init__(fields)
-        self.annotations = {}
-
-    def expect_domain_result(self, annotations: Dict):
-        self.expected_result_type = DomainResult
-        self.annotations = annotations
-
-
-class DomainActionTest(ActionTest):
-    def __init__(self, package_name: str):
-        """
-        Provides structure for testing DeltaFi Domain action
-        Args:
-            package_name: name of the actions package for finding resources
-        """
-        super().__init__(package_name)
-
-    def domain(self, test_case: DomainTestCase):
-        if test_case.expected_result_type == DomainResult:
-            self.expect_domain_result(test_case)
-        else:
-            super().execute(test_case)
-
-    def expect_domain_result(self, test_case: DomainTestCase):
-        result = super().run_and_check_result_type(test_case, DomainResult)
-        self.assert_domain_result(test_case, result)
-
-    def assert_domain_result(self, test_case: DomainTestCase, result: DomainResult):
-        # Check metrics
-        self.compare_metrics(test_case.expected_metrics, result.metrics)
-
-        # Check annotations
-        assert_keys_and_values(test_case.annotations, result.annotations)

deltafi-2.0rc1719271450675/deltafi/test_kit/enrich.py

@@ -1,70 +0,0 @@
-#
-#    DeltaFi - Data transformation and enrichment platform
-#
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-#
-
-from deltafi.result import EnrichResult
-
-from .assertions import *
-from .framework import TestCaseBase, ActionTest
-
-
-class EnrichTestCase(TestCaseBase):
-    def __init__(self, fields: Dict):
-        super().__init__(fields)
-        self.enrichments = []
-        self.annotations = {}
-
-    def expect_enrich_result(self, annotations: Dict):
-        self.expected_result_type = EnrichResult
-        self.annotations = annotations
-
-    def add_enrichment(self, name: str, value: str, media_type: str):
-        self.enrichments.append({'name': name, 'value': value, 'mediaType': media_type})
-
-
-class EnrichActionTest(ActionTest):
-    def __init__(self, package_name: str):
-        """
-        Provides structure for testing DeltaFi Enrich action
-        Args:
-            package_name: name of the actions package for finding resources
-        """
-        super().__init__(package_name)
-
-    def enrich(self, test_case: EnrichTestCase):
-        if test_case.expected_result_type == EnrichResult:
-            self.expect_enrich_result(test_case)
-        else:
-            super().execute(test_case)
-
-    def expect_enrich_result(self, test_case: EnrichTestCase):
-        result = super().run_and_check_result_type(test_case, EnrichResult)
-        self.assert_enrich_result(test_case, result)
-
-    def assert_enrich_result(self, test_case: EnrichTestCase, result: EnrichResult):
-        # Check metrics
-        self.compare_metrics(test_case.expected_metrics, result.metrics)
-
-        # Check annotations
-        assert_keys_and_values(test_case.annotations, result.annotations)
-
-        # Check enrichments
-        assert_equal_len(test_case.enrichments, result.enrichments)
-        if len(test_case.enrichments) > 0:
-            for index, expected in enumerate(test_case.enrichments):
-                actual = result.enrichments[index]
-                assert_equal(expected, actual)

deltafi-2.0rc1719271450675/deltafi/test_kit/validate.py

@@ -1,54 +0,0 @@
-#
-#    DeltaFi - Data transformation and enrichment platform
-#
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-#
-
-from deltafi.result import ValidateResult
-
-from .assertions import *
-from .framework import TestCaseBase, ActionTest
-
-
-class ValidateTestCase(TestCaseBase):
-    def __init__(self, fields: Dict):
-        super().__init__(fields)
-
-    def expect_validate_result(self):
-        self.expected_result_type = ValidateResult
-
-
-class ValidateActionTest(ActionTest):
-    def __init__(self, package_name: str):
-        """
-        Provides structure for testing DeltaFi Validate action
-        Args:
-            package_name: name of the actions package for finding resources
-        """
-        super().__init__(package_name)
-
-    def validate(self, test_case: ValidateTestCase):
-        if test_case.expected_result_type == ValidateResult:
-            self.expect_validate_result(test_case)
-        else:
-            super().execute(test_case)
-
-    def expect_validate_result(self, test_case: ValidateTestCase):
-        result = super().run_and_check_result_type(test_case, ValidateResult)
-        self.assert_validate_result(test_case, result)
-
-    def assert_validate_result(self, test_case: ValidateTestCase, result: ValidateResult):
-        # Check metrics
-        self.compare_metrics(test_case.expected_metrics, result.metrics)