deltafi 2.0rc1713535554033__tar.gz → 2.0rc1715691282958__tar.gz

{deltafi-2.0rc1713535554033 → deltafi-2.0rc1715691282958}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltafi
-Version: 2.0rc1713535554033
+Version: 2.0rc1715691282958
 Summary: SDK for DeltaFi plugins and actions
 License: Apache License, Version 2.0
 Keywords: deltafi
@@ -20,11 +20,11 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Software Development
 Requires-Dist: deepdiff (>=6.7.1)
 Requires-Dist: json-logging (>=1.3.0)
-Requires-Dist: minio (>=7.2.3)
-Requires-Dist: pydantic (>=2.5.3)
-Requires-Dist: redis (>=5.0.1)
+Requires-Dist: minio (>=7.2.5)
+Requires-Dist: pydantic (>=2.7.1)
+Requires-Dist: redis (>=5.0.4)
 Requires-Dist: requests (>=2.31.0)
-Requires-Dist: urllib3 (>=2.1.0)
+Requires-Dist: urllib3 (>=2.2.1)
 Project-URL: Bug Reports, https://chat.deltafi.org/deltafi/channels/bug-reports
 Project-URL: Documentation, https://docs.deltafi.org/#/
 Project-URL: Source Code, https://gitlab.com/deltafi/deltafi

{deltafi-2.0rc1713535554033 → deltafi-2.0rc1715691282958}/deltafi/action.py

@@ -20,8 +20,8 @@ from abc import ABC, abstractmethod
 from typing import Any, List
 
 from deltafi.actiontype import ActionType
-from deltafi.genericmodel import GenericModel
 from deltafi.domain import Context, DeltaFileMessage
+from deltafi.genericmodel import GenericModel
 from deltafi.input import EgressInput, TransformInput
 from deltafi.result import *
 from pydantic import BaseModel
@@ -48,20 +48,23 @@ class Action(ABC):
     def execute_action(self, event):
         if event.delta_file_messages is None or not len(event.delta_file_messages):
             raise RuntimeError(f"Received event with no delta file messages for did {event.context.did}")
-
         if event.context.collect is not None:
-            result = self.execute(event.context, self.collect([self.build_input(event.context, delta_file_message)
-                                                               for delta_file_message in event.delta_file_messages]),
-                                  self.param_class().model_validate(event.params))
+            result = self.execute(
+                event.context,
+                self.collect([self.build_input(event.context, delta_file_message)
+                              for delta_file_message in event.delta_file_messages]),
+                self.param_class().model_validate(event.params))
         else:
-            result = self.execute(event.context, self.build_input(event.context, event.delta_file_messages[0]),
-                                  self.param_class().model_validate(event.params))
+            result = self.execute(
+                event.context,
+                self.build_input(event.context, event.delta_file_messages[0]),
+                self.param_class().model_validate(event.params))
 
         self.validate_type(result)
         return result
 
     @staticmethod
-    def param_class( ):
+    def param_class():
         """Factory method to create and return an empty GenericModel instance.
 
         Returns
@@ -110,7 +113,8 @@ class TimedIngressAction(Action, ABC):
 
 class TransformAction(Action, ABC):
     def __init__(self, description: str):
-        super().__init__(ActionType.TRANSFORM, description, (TransformResult, TransformResults, ErrorResult, FilterResult))
+        super().__init__(ActionType.TRANSFORM, description,
+                         (TransformResult, TransformResults, ErrorResult, FilterResult))
 
     def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
         return TransformInput(content=delta_file_message.content_list, metadata=delta_file_message.metadata)

{deltafi-2.0rc1713535554033 → deltafi-2.0rc1715691282958}/deltafi/actiontype.py

@@ -20,7 +20,9 @@ from enum import Enum
 
 
 class ActionType(Enum):
+    INGRESS = "INGRESS"
     TIMED_INGRESS = "TIMED_INGRESS"
     TRANSFORM = "TRANSFORM"
     EGRESS = "EGRESS"
+    PUBLISH = "PUBLISH"
     UNKNOWN = "UNKNOWN"

{deltafi-2.0rc1713535554033 → deltafi-2.0rc1715691282958}/deltafi/domain.py

@@ -40,13 +40,15 @@ class ActionExecution(NamedTuple):
 
 class Context(NamedTuple):
     did: str
-    action_flow: str
+    delta_file_name: str
+    data_source: str
+    flow_name: str
+    flow_id: str
     action_name: str
-    source_filename: str
-    ingress_flow: str
-    egress_flow: str
-    system: str
+    action_id: str
+    action_version: str
     hostname: str
+    system_name: str
     content_service: ContentService
     collect: dict = None
     collected_dids: List[str] = None
@@ -56,19 +58,42 @@ class Context(NamedTuple):
     @classmethod
     def create(cls, context: dict, hostname: str, content_service: ContentService, logger: Logger):
         did = context['did']
-        action_name_parts = context['name'].split(".")
-        action_flow = action_name_parts[0]
-        action_name = action_name_parts[1]
-        if 'sourceFilename' in context:
-            source_filename = context['sourceFilename']
+        if 'deltaFileName' in context:
+            delta_file_name = context['deltaFileName']
         else:
-            source_filename = None
-        ingress_flow = context['ingressFlow']
-        if 'egressFlow' in context:
-            egress_flow = context['egressFlow']
+            delta_file_name = None
+        if 'dataSource' in context:
+            data_source = context['dataSource']
         else:
-            egress_flow = None
-        system = context['systemName']
+            data_source = None
+        if 'flowName' in context:
+            flow_name = context['flowName']
+        else:
+            flow_name = None
+        if 'flowId' in context:
+            flow_id = context['flowId']
+        else:
+            flow_id = None
+        if 'actionName' in context:
+            action_name = context['actionName']
+        else:
+            action_name = None
+        if 'actionId' in context:
+            action_id = context['actionId']
+        else:
+            action_id = None
+        if 'actionVersion' in context:
+            action_version = context['actionVersion']
+        else:
+            action_version = None
+        if 'hostname' in context:
+            hostname = context['hostname']
+        else:
+            hostname = None
+        if 'systemName' in context:
+            system_name = context['systemName']
+        else:
+            system_name = None
         if 'collect' in context:
             collect = context['collect']
         else:
@@ -81,18 +106,21 @@ class Context(NamedTuple):
             memo = context['memo']
         else:
             memo = None
+
         return Context(did=did,
-                       action_flow=action_flow,
+                       delta_file_name=delta_file_name,
+                       data_source=data_source,
+                       flow_name=flow_name,
+                       flow_id=flow_id,
                        action_name=action_name,
-                       source_filename=source_filename,
-                       ingress_flow=ingress_flow,
-                       egress_flow=egress_flow,
-                       system=system,
+                       action_id=action_id,
+                       action_version=action_version,
                        hostname=hostname,
-                       content_service=content_service,
+                       system_name=system_name,
                        collect=collect,
                        collected_dids=collected_dids,
                        memo=memo,
+                       content_service=content_service,
                        logger=logger)
 
 
@@ -197,7 +225,6 @@ class Content:
 
         return new_segments
 
-
     def get_size(self):
         """
         Returns the size of the content in bytes.
@@ -318,7 +345,8 @@ class Event(NamedTuple):
 
     @classmethod
     def create(cls, event: dict, hostname: str, content_service: ContentService, logger: Logger):
-        delta_file_messages = [DeltaFileMessage.from_dict(delta_file_message, content_service) for delta_file_message in event['deltaFileMessages']]
+        delta_file_messages = [DeltaFileMessage.from_dict(delta_file_message, content_service) for delta_file_message in
+                               event['deltaFileMessages']]
         context = Context.create(event['actionContext'], hostname, content_service, logger)
         params = event['actionParams']
         queue_name = None

{deltafi-2.0rc1713535554033 → deltafi-2.0rc1715691282958}/deltafi/plugin.py

@@ -46,9 +46,9 @@ def _coordinates():
 
 
 def _setup_queue(max_connections):
-    redis_url = os.getenv('REDIS_URL', 'http://deltafi-redis-master:6379')
-    password = os.getenv('REDIS_PASSWORD')
-    return ActionEventQueue(redis_url, max_connections, password)
+    url = os.getenv('VALKEY_URL', 'http://deltafi-valkey-master:6379')
+    password = os.getenv('VALKEY_PASSWORD')
+    return ActionEventQueue(url, max_connections, password)
 
 
 def _setup_content_service():
@@ -286,7 +286,10 @@ class Plugin(object):
 
                 response = {
                     'did': event.context.did,
-                    'action': event.context.action_flow + "." + event.context.action_name,
+                    'flowName': event.context.flow_name,
+                    'flowId': event.context.flow_id,
+                    'actionName': event.context.action_name,
+                    'actionId': event.context.action_id,
                     'start': start_time,
                     'stop': time.time(),
                     'type': result.result_type,

{deltafi-2.0rc1713535554033 → deltafi-2.0rc1715691282958}/deltafi/result.py

@@ -96,12 +96,12 @@ class FilterResult(Result):
 
 
 class IngressResultItem:
-    def __init__(self, context: Context, filename: str):
+    def __init__(self, context: Context, delta_file_name: str):
         self.context = context
-        self.filename = filename
         self._did = str(uuid.uuid4())
         self.content = []
         self.metadata = {}
+        self.delta_file_name = delta_file_name
 
     @property
     def did(self):
@@ -140,7 +140,7 @@ class IngressResultItem:
     def response(self):
         return {
             'did': self._did,
-            'filename': self.filename,
+            'deltaFileName': self.delta_file_name,
             'metadata': self.metadata,
             'content': [content.json() for content in self.content]
         }
@@ -156,8 +156,8 @@ class IngressResult(Result):
     def __init__(self, context: Context):
         super().__init__('ingress', 'INGRESS', context)
         self.memo = None
-        self.execute_immediate = False
         self.ingress_result_items = []
+        self.execute_immediate = False
         self.status = IngressStatusEnum.HEALTHY
         self.statusMessage = None
 

deltafi-2.0rc1715691282958/deltafi/test_kit/compare_helpers.py

@@ -0,0 +1,293 @@
+#
+#    DeltaFi - Data transformation and enrichment platform
+#
+#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+import json
+import re
+from abc import ABC
+from abc import abstractmethod
+from itertools import repeat
+
+from deepdiff import DeepDiff, DeepSearch
+
+from .assertions import *
+
+
+class CompareHelper(ABC):
+    @abstractmethod
+    def compare(self, expected: str, actual: str, label: str):
+        pass
+
+
+class GenericCompareHelper(CompareHelper):
+    def compare(self, expected: str, actual: str, label: str):
+        assert_equal_with_label(expected, actual, label)
+
+
+class JsonCompareHelper(CompareHelper):
+    """Provides helper functions for comparing JSON/dict objects.
+
+            Are these two JSON/dict objects equivalent?
+                - compare(...)
+
+            Are these two lists equivalent?
+                - compare_lists(...)
+
+            Is this list a subset/superset of that list?
+                - compare_lists_subset(...)
+                - compare_lists_superset(...)
+
+            Select a list of values from an existing list and put them into a new list to facilitate list comparisons:
+                - create_list_from_list_using_filter_regex(...)
+
+            Is this value found (or not) in this JSON/dict object?
+                - is_found(...)
+                - is_not_found(...)
+        """
+
+    def __init__(self, regex_exclusion_list=None, ignore_order=True):
+        """Creates and configures a JsonCompareHelper object.  If the optional 'ignore_order' is true, then the order of
+                data is ignored when checking else order is enforced.  The optional 'regex_exclusion_list' excludes
+                paths within the object from comparison; if empty or not provided, then no excludes are applied."""
+        if regex_exclusion_list is None:
+            regex_exclusion_list = []
+        self.excludes = regex_exclusion_list
+        self.ignore_order = ignore_order
+
+    def __perform_find(self, obj: object, item):
+        """Returns a dict of matches of the 'item' in the object 'obj'.  Both keys and values of dicts are included in
+                the search.  The item may be compiled regex pattern or a string that compiles to a regex pattern.  The
+                returned dict is empty if there are no matches. Excludes path(s) determined by the constructor."""
+        return DeepSearch(obj, item, verbose_level=2, exclude_regex_paths=self.excludes, use_regexp=True)
+
+    def is_not_found(self, obj: object, item):
+        """Returns None if there are no occurrences of 'item' in object 'obj' else raises a ValueError.  Both keys and
+                values of dicts are included in the search.  If 'item' is a list, then all elements of item must not be
+                found in list, else a ValueError is raised.  The argument 'item' may be a compiled regex pattern, a
+                string that compiles to a regex pattern, or a list of either or both.  Excludes path(s) and failure on
+                ordering of elements are determined by the constructor."""
+
+        all_matches = []
+
+        if isinstance(item, list):
+            for value in item:
+                matches = self.__perform_find(obj, value)
+                if len(matches) > 0:
+                    all_matches.append(matches)
+        else:
+            matches = self.__perform_find(obj, item)
+            if len(matches) > 0:
+                all_matches.append(matches)
+
+        if len(all_matches) > 0:
+            raise ValueError("Matches found for items '" + f"{all_matches}" + "'")
+
+        assert len(all_matches) == 0
+
+    def is_found(self, obj: object, item):
+        """Returns None if 'item' occurs in object 'obj' else raises a ValueError.  Both keys and values of dicts are
+                included in the search.  If 'item' is a list, then all elements of item must occur in the object else a
+                ValueError is returned.  The argument 'item' may be a compiled regex pattern, a string that compiles to
+                a regex pattern, or a list of either or both. Excludes path(s) and failure on ordering of elements are
+                determined by the constructor."""
+
+        not_found_items = []
+
+        if isinstance(item, list):
+            for value in item:
+                matches = self.__perform_find(obj, value)
+                if len(matches) == 0:
+                    not_found_items.append(value)
+        else:
+            matches = self.__perform_find(obj, item)
+            if len(matches) == 0:
+                not_found_items.append(item)
+
+        if len(not_found_items) > 0:
+            raise ValueError("No matches found for items '" + f"{not_found_items}" + "'")
+
+        assert len(not_found_items) == 0
+
+    def __perform_diff(self, expected, actual):
+        """Returns a dict with differences between 'expected' and 'actual'.  The returned dict is empty if 'expected'
+                and 'actual' are equivalent.  Both 'expected' and 'actual' must be dicts. Excludes path and failure on
+                ordering of elements are determined by the constructor.  Elements must match number of repetitions."""
+        return DeepDiff(expected, actual, ignore_order=self.ignore_order, report_repetition=True,
+                        exclude_regex_paths=self.excludes)
+
+    def __perform_diff_with_eval(self, expected, actual):
+        """Returns None if 'expected' and 'actual' are equivalent else returns a ValueError. Both 'expected' and
+                'actual' must be dicts.  Excludes path and failure on ordering of elements are determined by the
+                constructor.  Elements must match number of repetitions."""
+
+        diffs = self.__perform_diff(expected, actual)
+
+        if len(diffs) > 0:
+            raise ValueError(f"{diffs}")
+
+        assert len(diffs) == 0
+
+    def compare(self, expected, actual, label: str):
+        """Returns None if 'expected' and 'actual' are equivalent else returns a ValueError. Both 'expected' and
+                'actual' must be either dicts or strings that parse as JSON to dicts. Excludes path and failure on
+                ordering of elements are determined by the constructor.  Elements must match number of repetitions."""
+
+        if isinstance(expected, str):
+            exp = json.loads(expected)
+        else:
+            exp = expected
+
+        if isinstance(actual, str):
+            act = json.loads(actual)
+        else:
+            act = actual
+
+        return self.__perform_diff_with_eval(exp, act)
+
+    def compare_lists(self, expected, actual: list, label: str):
+        """Returns None if 'actual' is equivalent to 'expected' else returns a ValueError.
+
+                The 'actual' argument must be a list.  The 'expected' argument may be a list or a dict.  If a list, then
+                'expected' and 'actual' are compared against each other.  If a dict, then 'actual' is equivalent if it
+                contains elements with the same repetitions as defined in the 'expected' dict with the key equal to the
+                 element in 'actual' and the value equal to the number of repetitions.
+
+                Order of elements is ignored.  Elements must match number of repetitions."""
+
+        expected_list = []
+
+        if isinstance(expected, dict):
+            for key, value in expected.items():
+                expected_list += list(repeat(key, value))
+        else:
+            expected_list = expected
+
+        return self.__perform_diff_with_eval({"json-compare-helper-internal-list": expected_list},
+                                             {"json-compare-helper-internal-list": actual})
+
+    @staticmethod
+    def compare_lists_subset(expected_subset, actual: list):
+        """Returns None if the 'actual' list contains at least 'expected_subset' else returns a ValueError.  The
+                'actual' list may contain 0 or more additional elements than defined in 'expected_subset'.
+
+                The 'actual' argument must be a list.  The argument 'expected_subset' may be a list or a dict.  In the
+                latter case, the key defines the item that must appear in 'actual' and the value defines the number of
+                 repetitions.
+
+                Order of elements is ignored.  Elements must match number of repetitions."""
+
+        expected_subset_map = {}
+
+        if isinstance(expected_subset, list):
+            for item in expected_subset:
+                value = expected_subset_map.get(item)
+                if value is None:
+                    expected_subset_map[item] = 1
+                else:
+                    expected_subset_map[item] = value + 1
+        else:
+            expected_subset_map = expected_subset
+
+        actual_map = {}
+
+        for item in actual:
+            value = actual_map.get(item)
+            if value is None:
+                actual_map[item] = 1
+            else:
+                actual_map[item] = value + 1
+
+        for key, value in expected_subset_map.items():
+            actual_value = actual_map.get(key)
+            if actual_value is None:
+                raise ValueError("Actual list did not contain element '" + str(key) + "'")
+            else:
+                if actual_value != value:
+                    raise ValueError("Actual list had item '" + str(key) + "' with repetition " + str(actual_value)
+                                     + " but required repetition " + str(value))
+
+        return None
+
+    @staticmethod
+    def compare_lists_superset(expected_superset, actual: list):
+        """Returns None if the 'actual' list contains only elements that appear in 'expected_superset' else returns a
+                ValueError.  The 'actual' list cannot contain more elements than are defined in 'expected_superset'; the
+                'expected_superset' may define 0 or more values not contained in 'actual'.
+
+                The 'actual' argument must be a list.  The argument 'expected_superset' may be a list or a dict.  In the
+                latter case, the key defines the item that must appear in 'actual' and the value defines the number of
+                 repetitions.
+
+                Order of elements is ignored.  Elements must match number of repetitions."""
+
+        expected_superset_map = {}
+
+        if isinstance(expected_superset, list):
+            for item in expected_superset:
+                value = expected_superset_map.get(item)
+                if value is None:
+                    expected_superset_map[item] = 1
+                else:
+                    expected_superset_map[item] = value + 1
+        else:
+            expected_superset_map = expected_superset
+
+        actual_map = {}
+
+        for item in actual:
+            value = actual_map.get(item)
+            if value is None:
+                actual_map[item] = 1
+            else:
+                actual_map[item] = value + 1
+
+        for key, value in actual_map.items():
+            expected_value = expected_superset_map.get(key)
+            if expected_value is None:
+                raise ValueError("Actual list contained element '" + str(key)
+                                 + "' that did not appear in the expected superset")
+            else:
+                if expected_value != value:
+                    raise ValueError("Actual list had item '" + str(key) + "' with repetition " + str(value)
+                                     + " but required repetition " + str(expected_value))
+
+        return None
+
+    @staticmethod
+    def create_list_from_list_using_filter_regex(regex, source_list: list):
+        """Creates and returns a list of those items in the 'source_list' matching the regex in the 'regex'.  The
+                argument 'regex' may be a string or a compiled regex object.
+
+                For the 'regex', consider using anchors to explicitly match from the start (or end) of a string.  For
+                example, a string such as "^malware--" will explicitly match from the start of a string."""
+
+        def search_funct(element):
+
+            if isinstance(regex, str):
+                # 'regex' is a string
+                match = re.search(regex, element)
+            else:
+                # assume 'regex' is a compiled regex pattern object
+                match = regex.match(element)
+
+            if match is None:
+                return False
+            else:
+                return True
+
+        return list(filter(search_funct, source_list))
+

{deltafi-2.0rc1713535554033 → deltafi-2.0rc1715691282958}/deltafi/test_kit/framework.py

@@ -188,21 +188,10 @@ class ActionTest(ABC):
 
     def __reset__(self, did: str):
         self.content_service = InternalContentService()
-<<<<<<< HEAD
         if did is None:
             self.did = str(uuid.uuid4())
         else:
             self.did = did
-        self.expected_outputs = []
-||||||| parent of 831733c7 (2.0 Refactor)
-        self.did = str(uuid.uuid4())
-        self.expected_outputs = []
-=======
-        if did is None:
-            self.did = str(uuid.uuid4())
-        else:
-            self.did = did
->>>>>>> 831733c7 (2.0 Refactor)
         self.loaded_inputs = []
         self.res_path = ""
 
@@ -237,36 +226,23 @@ class ActionTest(ABC):
         content_list = self.make_content_list(test_case)
         self.content_service.load(self.loaded_inputs)
 
-<<<<<<< HEAD
-        return DeltaFileMessage(
-            metadata=test_case.in_meta,
-            content_list=content_list,
-            domains=test_case.in_domains,
-            enrichments=test_case.in_enrichments)
-||||||| parent of 831733c7 (2.0 Refactor)
-        return DeltaFileMessage(metadata=test_case.in_meta,
-                                content_list=content_list,
-                                domains=test_case.in_domains,
-                                enrichments=test_case.in_enrichments)
-=======
         return DeltaFileMessage(metadata=test_case.in_meta,
                                 content_list=content_list)
->>>>>>> 831733c7 (2.0 Refactor)
 
     def make_context(self, test_case: TestCaseBase):
         action_name = INGRESS_FLOW + "." + test_case.action.__class__.__name__
         return Context(
             did=self.did,
-            action_flow=INGRESS_FLOW,
+            delta_file_name=test_case.file_name,
+            data_source="DATASRC",
+            flow_name=INGRESS_FLOW,
+            flow_id="FLOWID",
             action_name=action_name,
-            source_filename=test_case.file_name,
-            ingress_flow=INGRESS_FLOW,
-            egress_flow=EGRESS_FLOW,
-            system=SYSTEM,
+            action_id="ACTIONID",
+            action_version="1.0",
             hostname=HOSTNAME,
+            system_name=SYSTEM,
             content_service=self.content_service,
-            collect=None,
-            collected_dids=None,
             logger=get_logger())
 
     def make_event(self, test_case: TestCaseBase):
@@ -312,39 +288,27 @@ class ActionTest(ABC):
         else:
             raise ValueError(f"unknown type: {test_case.expected_result_type}")
 
-    def compare_content_details(self, expected: LoadedContent, actual: Content):
+    @staticmethod
+    def compare_content_details(expected: LoadedContent, actual: Content):
         assert_equal(expected.content_type, actual.media_type)
         assert_equal(expected.name, actual.name)
 
-    def compare_one_content(self, comparitor: CompareHelper, expected: LoadedContent, actual, index):
+    def compare_one_content(self, comparator: CompareHelper, expected: LoadedContent, actual, index):
         self.compare_content_details(expected, actual)
         seg_id = actual.segments[0].uuid
-        comparitor.compare(expected.data, self.content_service.get_output(seg_id), f"Content[{index}]")
+        comparator.compare(expected.data, self.content_service.get_output(seg_id), f"Content[{index}]")
 
-    def compare_content_list(self, comparitor: CompareHelper, expected_outputs: List[IOContent], content: List):
+    def compare_content_list(self, comparator: CompareHelper, expected_outputs: List[IOContent], content: List):
         assert_equal_len(expected_outputs, content)
         for index, expected_ioc in enumerate(expected_outputs):
             if len(expected_ioc.content_bytes) == 0:
-                expected = LoadedContent(self.did, expected_ioc, self.load_file(output_ioc))
+                expected = LoadedContent(self.did, expected_ioc, self.load_file(expected_ioc))
             else:
                 expected = LoadedContent(self.did, expected_ioc, None)
-            self.compare_one_content(comparitor, expected, content[index], index)
-
-    def compare_one_metric(self, expected: Metric, result: Metric):
-        assert expected.name == result.name
-        assert_equal_with_label(expected.value, result.value, expected.name)
-        assert_keys_and_values(expected.tags, result.tags)
-
-            expected_value = expected['value']
-            if type(expected_value) == str:
-                comparitor.compare(expected_value, actual['value'], f"Domain[{index}]")
-            elif type(expected_value) == IOContent:
-                expected_data = self.load_file(expected_value)
-                comparitor.compare(expected_data, actual['value'], f"Domain[{index}]")
-            else:
-                raise ValueError(f"unknown expected_value type: {type(expected_value)}")
+            self.compare_one_content(comparator, expected, content[index], index)
 
-    def compare_one_metric(self, expected: Metric, result: Metric):
+    @staticmethod
+    def compare_one_metric(expected: Metric, result: Metric):
         assert expected.name == result.name
         assert_equal_with_label(expected.value, result.value, expected.name)
         assert_keys_and_values(expected.tags, result.tags)

{deltafi-2.0rc1713535554033 → deltafi-2.0rc1715691282958}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "deltafi"
-version = "2.0rc1713535554033"
+version = "2.0rc1715691282958"
 description = "SDK for DeltaFi plugins and actions"
 authors = ["DeltaFi <deltafi@systolic.com>"]
 license = "Apache License, Version 2.0"
@@ -23,19 +23,19 @@ classifiers = [
 python = "^3.9"
 deepdiff = ">=6.7.1"
 json-logging = ">=1.3.0"
-minio = ">=7.2.3"
-pydantic = ">=2.5.3"
-redis = ">=5.0.1"
+minio = ">=7.2.5"
+pydantic = ">=2.7.1"
+redis = ">=5.0.4"
 requests = ">=2.31.0"
-urllib3 = ">=2.1.0"
+urllib3 = ">=2.2.1"
 
 [tool.poetry.group.test]
 optional = true
 
 [tool.poetry.group.test.dependencies]
-pytest = ">=7.4.4"
-pytest-mock = ">=3.12.0"
-mockito = ">=1.4.0"
+pytest = ">=8.1.1"
+pytest-mock = ">=3.14.0"
+mockito = ">=1.5.0"
 
 [tool.poetry.urls]
 'Source Code' = "https://gitlab.com/deltafi/deltafi"

deltafi-2.0rc1713535554033/deltafi/test_kit/compare_helpers.py

@@ -1,50 +0,0 @@
-#
-#    DeltaFi - Data transformation and enrichment platform
-#
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-#
-
-import json
-from abc import ABC
-from abc import abstractmethod
-from typing import List
-
-from deepdiff import DeepDiff
-
-from .assertions import *
-
-
-class CompareHelper(ABC):
-    @abstractmethod
-    def compare(self, expected: str, actual: str, label: str):
-        pass
-
-
-class GenericCompareHelper(CompareHelper):
-    def compare(self, expected: str, actual: str, label: str):
-        assert_equal_with_label(expected, actual, label)
-
-
-class JsonCompareHelper(CompareHelper):
-    def __init__(self, regex_exclusion_list: List):
-        self.excludes = regex_exclusion_list
-
-    def compare(self, expected: str, actual: str, label: str):
-        exp = json.loads(expected)
-        act = json.loads(actual)
-        diffs = DeepDiff(exp, act, exclude_regex_paths=self.excludes)
-        if len(diffs) > 0:
-            raise ValueError(f"{diffs}")
-        assert len(diffs) == 0