deltafi 2.0rc1714076410617__tar.gz → 2.0rc1716000130240__tar.gz

{deltafi-2.0rc1714076410617 → deltafi-2.0rc1716000130240}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltafi
-Version: 2.0rc1714076410617
+Version: 2.0rc1716000130240
 Summary: SDK for DeltaFi plugins and actions
 License: Apache License, Version 2.0
 Keywords: deltafi
@@ -20,11 +20,11 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Software Development
 Requires-Dist: deepdiff (>=6.7.1)
 Requires-Dist: json-logging (>=1.3.0)
-Requires-Dist: minio (>=7.2.3)
-Requires-Dist: pydantic (>=2.5.3)
-Requires-Dist: redis (>=5.0.1)
+Requires-Dist: minio (>=7.2.5)
+Requires-Dist: pydantic (>=2.7.1)
+Requires-Dist: redis (>=5.0.4)
 Requires-Dist: requests (>=2.31.0)
-Requires-Dist: urllib3 (>=2.1.0)
+Requires-Dist: urllib3 (>=2.2.1)
 Project-URL: Bug Reports, https://chat.deltafi.org/deltafi/channels/bug-reports
 Project-URL: Documentation, https://docs.deltafi.org/#/
 Project-URL: Source Code, https://gitlab.com/deltafi/deltafi

{deltafi-2.0rc1714076410617 → deltafi-2.0rc1716000130240}/deltafi/action.py

@@ -20,8 +20,8 @@ from abc import ABC, abstractmethod
 from typing import Any, List
 
 from deltafi.actiontype import ActionType
-from deltafi.genericmodel import GenericModel
 from deltafi.domain import Context, DeltaFileMessage
+from deltafi.genericmodel import GenericModel
 from deltafi.input import EgressInput, TransformInput
 from deltafi.result import *
 from pydantic import BaseModel
@@ -48,20 +48,23 @@ class Action(ABC):
     def execute_action(self, event):
         if event.delta_file_messages is None or not len(event.delta_file_messages):
             raise RuntimeError(f"Received event with no delta file messages for did {event.context.did}")
-
         if event.context.collect is not None:
-            result = self.execute(event.context, self.collect([self.build_input(event.context, delta_file_message)
-                                                               for delta_file_message in event.delta_file_messages]),
-                                  self.param_class().model_validate(event.params))
+            result = self.execute(
+                event.context,
+                self.collect([self.build_input(event.context, delta_file_message)
+                              for delta_file_message in event.delta_file_messages]),
+                self.param_class().model_validate(event.params))
         else:
-            result = self.execute(event.context, self.build_input(event.context, event.delta_file_messages[0]),
-                                  self.param_class().model_validate(event.params))
+            result = self.execute(
+                event.context,
+                self.build_input(event.context, event.delta_file_messages[0]),
+                self.param_class().model_validate(event.params))
 
         self.validate_type(result)
         return result
 
     @staticmethod
-    def param_class( ):
+    def param_class():
         """Factory method to create and return an empty GenericModel instance.
 
         Returns
@@ -110,7 +113,8 @@ class TimedIngressAction(Action, ABC):
 
 class TransformAction(Action, ABC):
     def __init__(self, description: str):
-        super().__init__(ActionType.TRANSFORM, description, (TransformResult, TransformResults, ErrorResult, FilterResult))
+        super().__init__(ActionType.TRANSFORM, description,
+                         (TransformResult, TransformResults, ErrorResult, FilterResult))
 
     def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
         return TransformInput(content=delta_file_message.content_list, metadata=delta_file_message.metadata)

{deltafi-2.0rc1714076410617 → deltafi-2.0rc1716000130240}/deltafi/actiontype.py

@@ -20,7 +20,9 @@ from enum import Enum
 
 
 class ActionType(Enum):
+    INGRESS = "INGRESS"
     TIMED_INGRESS = "TIMED_INGRESS"
     TRANSFORM = "TRANSFORM"
     EGRESS = "EGRESS"
+    PUBLISH = "PUBLISH"
     UNKNOWN = "UNKNOWN"

{deltafi-2.0rc1714076410617 → deltafi-2.0rc1716000130240}/deltafi/domain.py

@@ -40,13 +40,15 @@ class ActionExecution(NamedTuple):
 
 class Context(NamedTuple):
     did: str
-    action_flow: str
+    delta_file_name: str
+    data_source: str
+    flow_name: str
+    flow_id: str
     action_name: str
-    source_filename: str
-    ingress_flow: str
-    egress_flow: str
-    system: str
+    action_id: str
+    action_version: str
     hostname: str
+    system_name: str
     content_service: ContentService
     collect: dict = None
     collected_dids: List[str] = None
@@ -56,19 +58,42 @@ class Context(NamedTuple):
     @classmethod
     def create(cls, context: dict, hostname: str, content_service: ContentService, logger: Logger):
         did = context['did']
-        action_name_parts = context['name'].split(".")
-        action_flow = action_name_parts[0]
-        action_name = action_name_parts[1]
-        if 'sourceFilename' in context:
-            source_filename = context['sourceFilename']
+        if 'deltaFileName' in context:
+            delta_file_name = context['deltaFileName']
         else:
-            source_filename = None
-        ingress_flow = context['ingressFlow']
-        if 'egressFlow' in context:
-            egress_flow = context['egressFlow']
+            delta_file_name = None
+        if 'dataSource' in context:
+            data_source = context['dataSource']
         else:
-            egress_flow = None
-        system = context['systemName']
+            data_source = None
+        if 'flowName' in context:
+            flow_name = context['flowName']
+        else:
+            flow_name = None
+        if 'flowId' in context:
+            flow_id = context['flowId']
+        else:
+            flow_id = None
+        if 'actionName' in context:
+            action_name = context['actionName']
+        else:
+            action_name = None
+        if 'actionId' in context:
+            action_id = context['actionId']
+        else:
+            action_id = None
+        if 'actionVersion' in context:
+            action_version = context['actionVersion']
+        else:
+            action_version = None
+        if 'hostname' in context:
+            hostname = context['hostname']
+        else:
+            hostname = None
+        if 'systemName' in context:
+            system_name = context['systemName']
+        else:
+            system_name = None
         if 'collect' in context:
             collect = context['collect']
         else:
@@ -81,18 +106,21 @@ class Context(NamedTuple):
             memo = context['memo']
         else:
             memo = None
+
         return Context(did=did,
-                       action_flow=action_flow,
+                       delta_file_name=delta_file_name,
+                       data_source=data_source,
+                       flow_name=flow_name,
+                       flow_id=flow_id,
                        action_name=action_name,
-                       source_filename=source_filename,
-                       ingress_flow=ingress_flow,
-                       egress_flow=egress_flow,
-                       system=system,
+                       action_id=action_id,
+                       action_version=action_version,
                        hostname=hostname,
-                       content_service=content_service,
+                       system_name=system_name,
                        collect=collect,
                        collected_dids=collected_dids,
                        memo=memo,
+                       content_service=content_service,
                        logger=logger)
 
 
@@ -197,7 +225,6 @@ class Content:
 
         return new_segments
 
-
     def get_size(self):
         """
         Returns the size of the content in bytes.
@@ -318,7 +345,8 @@ class Event(NamedTuple):
 
     @classmethod
     def create(cls, event: dict, hostname: str, content_service: ContentService, logger: Logger):
-        delta_file_messages = [DeltaFileMessage.from_dict(delta_file_message, content_service) for delta_file_message in event['deltaFileMessages']]
+        delta_file_messages = [DeltaFileMessage.from_dict(delta_file_message, content_service) for delta_file_message in
+                               event['deltaFileMessages']]
         context = Context.create(event['actionContext'], hostname, content_service, logger)
         params = event['actionParams']
         queue_name = None

{deltafi-2.0rc1714076410617 → deltafi-2.0rc1716000130240}/deltafi/plugin.py

@@ -46,9 +46,9 @@ def _coordinates():
 
 
 def _setup_queue(max_connections):
-    redis_url = os.getenv('REDIS_URL', 'http://deltafi-redis-master:6379')
-    password = os.getenv('REDIS_PASSWORD')
-    return ActionEventQueue(redis_url, max_connections, password)
+    url = os.getenv('VALKEY_URL', 'http://deltafi-valkey-master:6379')
+    password = os.getenv('VALKEY_PASSWORD')
+    return ActionEventQueue(url, max_connections, password)
 
 
 def _setup_content_service():
@@ -286,7 +286,10 @@ class Plugin(object):
 
                 response = {
                     'did': event.context.did,
-                    'action': event.context.action_flow + "." + event.context.action_name,
+                    'flowName': event.context.flow_name,
+                    'flowId': event.context.flow_id,
+                    'actionName': event.context.action_name,
+                    'actionId': event.context.action_id,
                     'start': start_time,
                     'stop': time.time(),
                     'type': result.result_type,

{deltafi-2.0rc1714076410617 → deltafi-2.0rc1716000130240}/deltafi/result.py

@@ -96,12 +96,12 @@ class FilterResult(Result):
 
 
 class IngressResultItem:
-    def __init__(self, context: Context, filename: str):
+    def __init__(self, context: Context, delta_file_name: str):
         self.context = context
-        self.filename = filename
         self._did = str(uuid.uuid4())
         self.content = []
         self.metadata = {}
+        self.delta_file_name = delta_file_name
 
     @property
     def did(self):
@@ -140,7 +140,7 @@ class IngressResultItem:
     def response(self):
         return {
             'did': self._did,
-            'filename': self.filename,
+            'deltaFileName': self.delta_file_name,
             'metadata': self.metadata,
             'content': [content.json() for content in self.content]
         }
@@ -156,10 +156,10 @@ class IngressResult(Result):
     def __init__(self, context: Context):
         super().__init__('ingress', 'INGRESS', context)
         self.memo = None
-        self.execute_immediate = False
         self.ingress_result_items = []
+        self.execute_immediate = False
         self.status = IngressStatusEnum.HEALTHY
-        self.statusMessage = None
+        self.status_message = None
 
     def add_item(self, ingress_result_item: IngressResultItem):
         self.ingress_result_items.append(ingress_result_item)
@@ -171,7 +171,7 @@ class IngressResult(Result):
             'executeImmediate': self.execute_immediate,
             'ingressItems': [ingress_result_item.response() for ingress_result_item in self.ingress_result_items],
             'status': self.status.value,
-            'statusMessage': self.statusMessage
+            'statusMessage': self.status_message
         }
 
 

{deltafi-2.0rc1714076410617 → deltafi-2.0rc1716000130240}/deltafi/test_kit/compare_helpers.py

@@ -69,14 +69,17 @@ class JsonCompareHelper(CompareHelper):
         self.ignore_order = ignore_order
 
     def __perform_find(self, obj: object, item):
-        """Returns a dict of matches of the 'item' in the object 'obj'.  The returned dict is empty if there are no
-                matches. Excludes path determined by the constructor."""
-        return DeepSearch(obj, item, verbose_level=2, exclude_regex_paths=self.excludes)
+        """Returns a dict of matches of the 'item' in the object 'obj'.  Both keys and values of dicts are included in
+                the search.  The item may be compiled regex pattern or a string that compiles to a regex pattern.  The
+                returned dict is empty if there are no matches. Excludes path(s) determined by the constructor."""
+        return DeepSearch(obj, item, verbose_level=2, exclude_regex_paths=self.excludes, use_regexp=True)
 
     def is_not_found(self, obj: object, item):
-        """Returns None if there are no occurrences of 'item' in object 'obj' else returns a ValueError.  The argument
-                'item' may be a scalar or a list.  Excludes path and failure on ordering of elements are determined by
-                the constructor."""
+        """Returns None if there are no occurrences of 'item' in object 'obj' else raises a ValueError.  Both keys and
+                values of dicts are included in the search.  If 'item' is a list, then all elements of item must not be
+                found in list, else a ValueError is raised.  The argument 'item' may be a compiled regex pattern, a
+                string that compiles to a regex pattern, or a list of either or both.  Excludes path(s) and failure on
+                ordering of elements are determined by the constructor."""
 
         all_matches = []
 
@@ -91,31 +94,33 @@ class JsonCompareHelper(CompareHelper):
                 all_matches.append(matches)
 
         if len(all_matches) > 0:
-            raise ValueError(f"{all_matches}")
+            raise ValueError("Matches found for items '" + f"{all_matches}" + "'")
 
         assert len(all_matches) == 0
 
     def is_found(self, obj: object, item):
-        """Returns None if there are no occurrences of 'item' in object 'obj' else returns a ValueError.  The argument
-                'item' may be a scalar or a list.  Excludes path and failure on ordering of elements are determined by
-                the constructor."""
+        """Returns None if 'item' occurs in object 'obj' else raises a ValueError.  Both keys and values of dicts are
+                included in the search.  If 'item' is a list, then all elements of item must occur in the object else a
+                ValueError is returned.  The argument 'item' may be a compiled regex pattern, a string that compiles to
+                a regex pattern, or a list of either or both. Excludes path(s) and failure on ordering of elements are
+                determined by the constructor."""
 
-        all_matches = []
+        not_found_items = []
 
         if isinstance(item, list):
             for value in item:
                 matches = self.__perform_find(obj, value)
-                if len(matches) > 0:
-                    all_matches.append(matches)
+                if len(matches) == 0:
+                    not_found_items.append(value)
         else:
             matches = self.__perform_find(obj, item)
-            if len(matches) > 0:
-                all_matches.append(matches)
+            if len(matches) == 0:
+                not_found_items.append(item)
 
-        if len(all_matches) == 0:
-            raise ValueError("No matches found for '" + str(item) + "'")
+        if len(not_found_items) > 0:
+            raise ValueError("No matches found for items '" + f"{not_found_items}" + "'")
 
-        assert len(all_matches) > 0
+        assert len(not_found_items) == 0
 
     def __perform_diff(self, expected, actual):
         """Returns a dict with differences between 'expected' and 'actual'.  The returned dict is empty if 'expected'

{deltafi-2.0rc1714076410617 → deltafi-2.0rc1716000130240}/deltafi/test_kit/framework.py

@@ -57,6 +57,7 @@ class IOContent:
             self.content_type = content_type
         self.offset = offset
         self.content_bytes = content_bytes
+        self.segment_uuid = uuid.uuid4()
 
     @classmethod
     def file_type(cls, name: str):
@@ -80,7 +81,7 @@ class LoadedContent:
         else:
             self.data = ioc.content_bytes
         self.segment = Segment.from_dict(
-            {"uuid": str(uuid.uuid4()), "offset": self.offset, "size": len(self.data), "did": did})
+            {"uuid": str(ioc.segment_uuid), "offset": self.offset, "size": len(self.data), "did": did})
 
 
 class InternalContentService:
@@ -188,21 +189,10 @@ class ActionTest(ABC):
 
     def __reset__(self, did: str):
         self.content_service = InternalContentService()
-<<<<<<< HEAD
         if did is None:
             self.did = str(uuid.uuid4())
         else:
             self.did = did
-        self.expected_outputs = []
-||||||| parent of 831733c7 (2.0 Refactor)
-        self.did = str(uuid.uuid4())
-        self.expected_outputs = []
-=======
-        if did is None:
-            self.did = str(uuid.uuid4())
-        else:
-            self.did = did
->>>>>>> 831733c7 (2.0 Refactor)
         self.loaded_inputs = []
         self.res_path = ""
 
@@ -237,36 +227,23 @@ class ActionTest(ABC):
         content_list = self.make_content_list(test_case)
         self.content_service.load(self.loaded_inputs)
 
-<<<<<<< HEAD
-        return DeltaFileMessage(
-            metadata=test_case.in_meta,
-            content_list=content_list,
-            domains=test_case.in_domains,
-            enrichments=test_case.in_enrichments)
-||||||| parent of 831733c7 (2.0 Refactor)
-        return DeltaFileMessage(metadata=test_case.in_meta,
-                                content_list=content_list,
-                                domains=test_case.in_domains,
-                                enrichments=test_case.in_enrichments)
-=======
         return DeltaFileMessage(metadata=test_case.in_meta,
                                 content_list=content_list)
->>>>>>> 831733c7 (2.0 Refactor)
 
     def make_context(self, test_case: TestCaseBase):
         action_name = INGRESS_FLOW + "." + test_case.action.__class__.__name__
         return Context(
             did=self.did,
-            action_flow=INGRESS_FLOW,
+            delta_file_name=test_case.file_name,
+            data_source="DATASRC",
+            flow_name=INGRESS_FLOW,
+            flow_id="FLOWID",
             action_name=action_name,
-            source_filename=test_case.file_name,
-            ingress_flow=INGRESS_FLOW,
-            egress_flow=EGRESS_FLOW,
-            system=SYSTEM,
+            action_id="ACTIONID",
+            action_version="1.0",
             hostname=HOSTNAME,
+            system_name=SYSTEM,
             content_service=self.content_service,
-            collect=None,
-            collected_dids=None,
             logger=get_logger())
 
     def make_event(self, test_case: TestCaseBase):
@@ -312,39 +289,27 @@ class ActionTest(ABC):
         else:
             raise ValueError(f"unknown type: {test_case.expected_result_type}")
 
-    def compare_content_details(self, expected: LoadedContent, actual: Content):
+    @staticmethod
+    def compare_content_details(expected: LoadedContent, actual: Content):
         assert_equal(expected.content_type, actual.media_type)
         assert_equal(expected.name, actual.name)
 
-    def compare_one_content(self, comparitor: CompareHelper, expected: LoadedContent, actual, index):
+    def compare_one_content(self, comparator: CompareHelper, expected: LoadedContent, actual, index):
         self.compare_content_details(expected, actual)
         seg_id = actual.segments[0].uuid
-        comparitor.compare(expected.data, self.content_service.get_output(seg_id), f"Content[{index}]")
+        comparator.compare(expected.data, self.content_service.get_output(seg_id), f"Content[{index}]")
 
-    def compare_content_list(self, comparitor: CompareHelper, expected_outputs: List[IOContent], content: List):
+    def compare_content_list(self, comparator: CompareHelper, expected_outputs: List[IOContent], content: List):
         assert_equal_len(expected_outputs, content)
         for index, expected_ioc in enumerate(expected_outputs):
             if len(expected_ioc.content_bytes) == 0:
-                expected = LoadedContent(self.did, expected_ioc, self.load_file(output_ioc))
+                expected = LoadedContent(self.did, expected_ioc, self.load_file(expected_ioc))
             else:
                 expected = LoadedContent(self.did, expected_ioc, None)
-            self.compare_one_content(comparitor, expected, content[index], index)
-
-    def compare_one_metric(self, expected: Metric, result: Metric):
-        assert expected.name == result.name
-        assert_equal_with_label(expected.value, result.value, expected.name)
-        assert_keys_and_values(expected.tags, result.tags)
-
-            expected_value = expected['value']
-            if type(expected_value) == str:
-                comparitor.compare(expected_value, actual['value'], f"Domain[{index}]")
-            elif type(expected_value) == IOContent:
-                expected_data = self.load_file(expected_value)
-                comparitor.compare(expected_data, actual['value'], f"Domain[{index}]")
-            else:
-                raise ValueError(f"unknown expected_value type: {type(expected_value)}")
+            self.compare_one_content(comparator, expected, content[index], index)
 
-    def compare_one_metric(self, expected: Metric, result: Metric):
+    @staticmethod
+    def compare_one_metric(expected: Metric, result: Metric):
         assert expected.name == result.name
         assert_equal_with_label(expected.value, result.value, expected.name)
         assert_keys_and_values(expected.tags, result.tags)

{deltafi-2.0rc1714076410617 → deltafi-2.0rc1716000130240}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "deltafi"
-version = "2.0rc1714076410617"
+version = "2.0rc1716000130240"
 description = "SDK for DeltaFi plugins and actions"
 authors = ["DeltaFi <deltafi@systolic.com>"]
 license = "Apache License, Version 2.0"
@@ -23,19 +23,19 @@ classifiers = [
 python = "^3.9"
 deepdiff = ">=6.7.1"
 json-logging = ">=1.3.0"
-minio = ">=7.2.3"
-pydantic = ">=2.5.3"
-redis = ">=5.0.1"
+minio = ">=7.2.5"
+pydantic = ">=2.7.1"
+redis = ">=5.0.4"
 requests = ">=2.31.0"
-urllib3 = ">=2.1.0"
+urllib3 = ">=2.2.1"
 
 [tool.poetry.group.test]
 optional = true
 
 [tool.poetry.group.test.dependencies]
-pytest = ">=7.4.4"
-pytest-mock = ">=3.12.0"
-mockito = ">=1.4.0"
+pytest = ">=8.1.1"
+pytest-mock = ">=3.14.0"
+mockito = ">=1.5.0"
 
 [tool.poetry.urls]
 'Source Code' = "https://gitlab.com/deltafi/deltafi"