deltafi 2.0rc1720728217472__tar.gz → 2.0rc1722532930396__tar.gz

{deltafi-2.0rc1720728217472 → deltafi-2.0rc1722532930396}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltafi
-Version: 2.0rc1720728217472
+Version: 2.0rc1722532930396
 Summary: SDK for DeltaFi plugins and actions
 License: Apache License, Version 2.0
 Keywords: deltafi
@@ -20,11 +20,11 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Software Development
 Requires-Dist: deepdiff (>=6.7.1)
 Requires-Dist: json-logging (>=1.3.0)
-Requires-Dist: minio (>=7.2.3)
-Requires-Dist: pydantic (>=2.5.3)
-Requires-Dist: redis (>=5.0.1)
-Requires-Dist: requests (>=2.31.0)
-Requires-Dist: urllib3 (>=2.1.0)
+Requires-Dist: minio (>=7.2.7)
+Requires-Dist: pydantic (>=2.8.2)
+Requires-Dist: redis (>=5.0.7)
+Requires-Dist: requests (>=2.32.3)
+Requires-Dist: urllib3 (>=2.2.2)
 Project-URL: Bug Reports, https://chat.deltafi.org/deltafi/channels/bug-reports
 Project-URL: Documentation, https://docs.deltafi.org/#/
 Project-URL: Source Code, https://gitlab.com/deltafi/deltafi

deltafi-2.0rc1722532930396/deltafi/action.py

@@ -0,0 +1,139 @@
+#
+#    DeltaFi - Data transformation and enrichment platform
+#
+#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+from abc import ABC, abstractmethod
+from typing import Any, List
+
+from deltafi.actiontype import ActionType
+from deltafi.domain import Context, DeltaFileMessage
+from deltafi.genericmodel import GenericModel
+from deltafi.input import EgressInput, TransformInput
+from deltafi.result import *
+from pydantic import BaseModel
+
+
+class Action(ABC):
+    def __init__(self, action_type: ActionType, description: str, valid_result_types: tuple):
+        self.action_type = action_type
+        self.description = description
+        self.valid_result_types = valid_result_types
+        self.action_execution = None
+
+    @abstractmethod
+    def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
+        pass
+
+    def join(self, action_inputs: List[Any]):
+        raise RuntimeError(f"Join is not supported for {self.__class__.__name__}")
+
+    @abstractmethod
+    def execute(self, context: Context, action_input: Any, params: BaseModel):
+        pass
+
+    def execute_action(self, event):
+        if event.delta_file_messages is None or not len(event.delta_file_messages):
+            raise RuntimeError(f"Received event with no delta file messages for did {event.context.did}")
+        if event.context.join is not None:
+            result = self.execute(
+                event.context,
+                self.join([self.build_input(event.context, delta_file_message)
+                              for delta_file_message in event.delta_file_messages]),
+                self.param_class().model_validate(event.params))
+        else:
+            result = self.execute(
+                event.context,
+                self.build_input(event.context, event.delta_file_messages[0]),
+                self.param_class().model_validate(event.params))
+
+        self.validate_type(result)
+        return result
+
+    @staticmethod
+    def param_class():
+        """Factory method to create and return an empty GenericModel instance.
+
+        All action parameter classes must inherit pydantic.BaseModel.
+        Use of complex types in custom action parameter classes must specify
+        the internal types when defined. E.g., dict[str, str], or List[str]
+
+        Returns
+        -------
+        GenericModel
+            an empty GenericModel instance
+        """
+        return GenericModel
+
+    def validate_type(self, result):
+        if not isinstance(result, self.valid_result_types):
+            raise ValueError(f"{self.__class__.__name__} must return one of "
+                             f"{[result_type.__name__ for result_type in self.valid_result_types]} "
+                             f"but a {result.__class__.__name__} was returned")
+
+
+class EgressAction(Action, ABC):
+    def __init__(self, description: str):
+        super().__init__(ActionType.EGRESS, description, (EgressResult, ErrorResult, FilterResult))
+
+    def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
+        return EgressInput(content=delta_file_message.content_list[0], metadata=delta_file_message.metadata)
+
+    @abstractmethod
+    def egress(self, context: Context, params: BaseModel, egress_input: EgressInput):
+        pass
+
+    def execute(self, context: Context, egress_input: EgressInput, params: BaseModel):
+        return self.egress(context, params, egress_input)
+
+
+class TimedIngressAction(Action, ABC):
+    def __init__(self, description: str):
+        super().__init__(ActionType.TIMED_INGRESS, description, (IngressResult, ErrorResult))
+
+    def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
+        return None
+
+    @abstractmethod
+    def ingress(self, context: Context, params: BaseModel):
+        pass
+
+    def execute(self, context: Context, input_placeholder: Any, params: BaseModel):
+        return self.ingress(context, params)
+
+
+class TransformAction(Action, ABC):
+    def __init__(self, description: str):
+        super().__init__(ActionType.TRANSFORM, description,
+                         (TransformResult, TransformResults, ErrorResult, FilterResult))
+
+    def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
+        return TransformInput(content=delta_file_message.content_list, metadata=delta_file_message.metadata)
+
+    def join(self, transform_inputs: List[TransformInput]):
+        all_content = []
+        all_metadata = {}
+        for transform_input in transform_inputs:
+            all_content += transform_input.content
+            all_metadata.update(transform_input.metadata)
+        return TransformInput(content=all_content, metadata=all_metadata)
+
+    @abstractmethod
+    def transform(self, context: Context, params: BaseModel, transform_input: TransformInput):
+        pass
+
+    def execute(self, context: Context, transform_input: TransformInput, params: BaseModel):
+        return self.transform(context, params, transform_input)

{deltafi-2.0rc1720728217472 → deltafi-2.0rc1722532930396}/deltafi/actiontype.py

@@ -20,13 +20,9 @@ from enum import Enum
 
 
 class ActionType(Enum):
+    INGRESS = "INGRESS"
     TIMED_INGRESS = "TIMED_INGRESS"
     TRANSFORM = "TRANSFORM"
-    LOAD = "LOAD"
-    DOMAIN = "DOMAIN"
-    ENRICH = "ENRICH"
-    FORMAT = "FORMAT"
-    VALIDATE = "VALIDATE"
     EGRESS = "EGRESS"
-    DELETE = "DELETE"
+    PUBLISH = "PUBLISH"
     UNKNOWN = "UNKNOWN"

{deltafi-2.0rc1720728217472 → deltafi-2.0rc1722532930396}/deltafi/domain.py

@@ -40,59 +40,87 @@ class ActionExecution(NamedTuple):
 
 class Context(NamedTuple):
     did: str
-    action_flow: str
+    delta_file_name: str
+    data_source: str
+    flow_name: str
+    flow_id: str
     action_name: str
-    source_filename: str
-    ingress_flow: str
-    egress_flow: str
-    system: str
+    action_id: str
+    action_version: str
     hostname: str
+    system_name: str
     content_service: ContentService
-    collect: dict = None
-    collected_dids: List[str] = None
+    join: dict = None
+    joined_dids: List[str] = None
     memo: str = None
     logger: Logger = None
 
     @classmethod
-    def create(cls, context: dict, hostname: str, content_service: ContentService, logger: Logger):
+    def create(cls, context: dict, content_service: ContentService, logger: Logger):
         did = context['did']
-        action_name_parts = context['name'].split(".")
-        action_flow = action_name_parts[0]
-        action_name = action_name_parts[1]
-        if 'sourceFilename' in context:
-            source_filename = context['sourceFilename']
+        if 'deltaFileName' in context:
+            delta_file_name = context['deltaFileName']
         else:
-            source_filename = None
-        ingress_flow = context['ingressFlow']
-        if 'egressFlow' in context:
-            egress_flow = context['egressFlow']
+            delta_file_name = None
+        if 'dataSource' in context:
+            data_source = context['dataSource']
         else:
-            egress_flow = None
-        system = context['systemName']
-        if 'collect' in context:
-            collect = context['collect']
+            data_source = None
+        if 'flowName' in context:
+            flow_name = context['flowName']
         else:
-            collect = None
-        if 'collectedDids' in context:
-            collected_dids = context['collectedDids']
+            flow_name = None
+        if 'flowId' in context:
+            flow_id = context['flowId']
         else:
-            collected_dids = None
+            flow_id = None
+        if 'actionName' in context:
+            action_name = context['actionName']
+        else:
+            action_name = None
+        if 'actionId' in context:
+            action_id = context['actionId']
+        else:
+            action_id = None
+        if 'actionVersion' in context:
+            action_version = context['actionVersion']
+        else:
+            action_version = None
+        if 'hostname' in context:
+            hostname = context['hostname']
+        else:
+            hostname = None
+        if 'systemName' in context:
+            system_name = context['systemName']
+        else:
+            system_name = None
+        if 'join' in context:
+            join = context['join']
+        else:
+            join = None
+        if 'joinedDids' in context:
+            joined_dids = context['joinedDids']
+        else:
+            joined_dids = None
         if 'memo' in context:
             memo = context['memo']
         else:
             memo = None
+
         return Context(did=did,
-                       action_flow=action_flow,
+                       delta_file_name=delta_file_name,
+                       data_source=data_source,
+                       flow_name=flow_name,
+                       flow_id=flow_id,
                        action_name=action_name,
-                       source_filename=source_filename,
-                       ingress_flow=ingress_flow,
-                       egress_flow=egress_flow,
-                       system=system,
+                       action_id=action_id,
+                       action_version=action_version,
                        hostname=hostname,
-                       content_service=content_service,
-                       collect=collect,
-                       collected_dids=collected_dids,
+                       system_name=system_name,
+                       join=join,
+                       joined_dids=joined_dids,
                        memo=memo,
+                       content_service=content_service,
                        logger=logger)
 
 
@@ -197,7 +225,6 @@ class Content:
 
         return new_segments
 
-
     def get_size(self):
         """
         Returns the size of the content in bytes.
@@ -296,41 +323,17 @@ class Content:
                        content_service=content_service)
 
 
-class Domain(NamedTuple):
-    name: str
-    value: str
-    media_type: str
-
-    @classmethod
-    def from_dict(cls, domain: dict):
-        name = domain['name']
-        if 'value' in domain:
-            value = domain['value']
-        else:
-            value = None
-        media_type = domain['mediaType']
-        return Domain(name=name,
-                      value=value,
-                      media_type=media_type)
-
-
 class DeltaFileMessage(NamedTuple):
     metadata: Dict[str, str]
     content_list: List[Content]
-    domains: List[Domain]
-    enrichments: List[Domain]
 
     @classmethod
     def from_dict(cls, delta_file_message: dict, content_service: ContentService):
         metadata = delta_file_message['metadata']
         content_list = [Content.from_dict(content, content_service) for content in delta_file_message['contentList']]
-        domains = [Domain.from_dict(domain) for domain in delta_file_message['domains']] if 'domains' in delta_file_message else []
-        enrichments = [Domain.from_dict(domain) for domain in delta_file_message['enrichments']] if 'enrichments' in delta_file_message else []
 
         return DeltaFileMessage(metadata=metadata,
-                                content_list=content_list,
-                                domains=domains,
-                                enrichments=enrichments)
+                                content_list=content_list)
 
 
 class Event(NamedTuple):
@@ -341,9 +344,10 @@ class Event(NamedTuple):
     return_address: str
 
     @classmethod
-    def create(cls, event: dict, hostname: str, content_service: ContentService, logger: Logger):
-        delta_file_messages = [DeltaFileMessage.from_dict(delta_file_message, content_service) for delta_file_message in event['deltaFileMessages']]
-        context = Context.create(event['actionContext'], hostname, content_service, logger)
+    def create(cls, event: dict, content_service: ContentService, logger: Logger):
+        delta_file_messages = [DeltaFileMessage.from_dict(delta_file_message, content_service) for delta_file_message in
+                               event['deltaFileMessages']]
+        context = Context.create(event['actionContext'], content_service, logger)
         params = event['actionParams']
         queue_name = None
         if 'queueName' in event:

{deltafi-2.0rc1720728217472 → deltafi-2.0rc1722532930396}/deltafi/exception.py

@@ -23,16 +23,6 @@ class ExpectedContentException(RuntimeError):
         self.size = size
 
 
-class MissingDomainException(RuntimeError):
-    def __init__(self, name):
-        self.name = name
-
-
-class MissingEnrichmentException(RuntimeError):
-    def __init__(self, name):
-        self.name = name
-
-
 class MissingMetadataException(RuntimeError):
     def __init__(self, key):
         self.key = key

{deltafi-2.0rc1720728217472 → deltafi-2.0rc1722532930396}/deltafi/genericmodel.py

@@ -22,7 +22,9 @@
 
 Provides an empty subclass of pydantic.BaseModel.
 
-Starting Pydantic v2, the BaseModel cannot be directly instantiated.  This class provides for instantiation of GenericModel objects that inherit from BaseModel.
+All action parameter classes must inherit pydantic.BaseModel.
+
+Starting Pydantic v2, the BaseModel cannot be directly instantiated. This class provides for instantiation of GenericModel objects that inherit from BaseModel.
 
 This class does not define fields for validation or any other purpose.
 """

deltafi-2.0rc1722532930396/deltafi/input.py

@@ -0,0 +1,52 @@
+#
+#    DeltaFi - Data transformation and enrichment platform
+#
+#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+from deltafi.domain import *
+from deltafi.exception import MissingMetadataException, ExpectedContentException
+
+class EgressInput(NamedTuple):
+    content: Content
+    metadata: dict
+
+
+class TransformInput(NamedTuple):
+    content: List[Content]
+    metadata: dict
+
+    def has_content(self) -> bool:
+        return len(self.content) > 0
+
+    def content_at(self, index: int) -> Content:
+        if len(self.content) < index + 1:
+            raise ExpectedContentException(index, len(self.content))
+        return self.content[index]
+
+    def first_content(self):
+        return self.content_at(0)
+
+    def get_metadata(self, key: str):
+        if key in self.metadata:
+            return self.metadata[key]
+        else:
+            raise MissingMetadataException(key)
+
+    def get_metadata_or_else(self, key: str, default: str) -> str:
+        if key in self.metadata:
+            return self.metadata[key]
+        else:
+            return default

{deltafi-2.0rc1720728217472 → deltafi-2.0rc1722532930396}/deltafi/logger.py

@@ -18,7 +18,7 @@
 
 import logging
 import sys
-from datetime import datetime
+from datetime import datetime, UTC
 
 import json_logging
 
@@ -42,7 +42,7 @@ def _sanitize_log_msg(record):
 class JSONLogFormatter(json_logging.JSONLogFormatter):
 
     def _format_log_object(self, record, request_util):
-        utcnow = datetime.utcnow()
+        utcnow = datetime.now(UTC)
 
         json_log_object = {
             'timestamp': json_logging.util.iso_time_format(utcnow),

{deltafi-2.0rc1720728217472 → deltafi-2.0rc1722532930396}/deltafi/plugin.py

@@ -34,8 +34,7 @@ from importlib import metadata
 import requests
 from deltafi.actioneventqueue import ActionEventQueue
 from deltafi.domain import Event, ActionExecution
-from deltafi.exception import ExpectedContentException, MissingDomainException, MissingEnrichmentException, \
-    MissingMetadataException
+from deltafi.exception import ExpectedContentException, MissingMetadataException
 from deltafi.logger import get_logger
 from deltafi.result import ErrorResult
 from deltafi.storage import ContentService
@@ -47,9 +46,9 @@ def _coordinates():
 
 
 def _setup_queue(max_connections):
-    redis_url = os.getenv('REDIS_URL', 'http://deltafi-redis-master:6379')
-    password = os.getenv('REDIS_PASSWORD')
-    return ActionEventQueue(redis_url, max_connections, password)
+    url = os.getenv('VALKEY_URL', 'http://deltafi-valkey-master:6379')
+    password = os.getenv('VALKEY_PASSWORD')
+    return ActionEventQueue(url, max_connections, password)
 
 
 def _setup_content_service():
@@ -165,8 +164,6 @@ class Plugin(object):
             'name': self.action_name(action),
             'description': action.description,
             'type': action.action_type.name,
-            'requiresDomains': action.requires_domains,
-            'requiresEnrichments': action.requires_enrichments,
             'schema': action.param_class().model_json_schema()
         }
 
@@ -219,7 +216,8 @@ class Plugin(object):
         for action in self.actions:
             threading.Thread(target=self._do_action, args=(action,)).start()
 
-        threading.Thread(target=self._heartbeat).start()
+        hb_thread = threading.Thread(target=self._heartbeat)
+        hb_thread.start()
 
         self.logger.info("All threads running")
 
@@ -227,6 +225,7 @@ class Plugin(object):
         f.close()
 
         self.logger.info("Application initialization complete")
+        hb_thread.join()
 
     def _heartbeat(self):
         long_running_actions = set()
@@ -263,7 +262,7 @@ class Plugin(object):
         while True:
             try:
                 event_string = self.queue.take(self.action_name(action))
-                event = Event.create(json.loads(event_string), self.hostname, self.content_service, action_logger)
+                event = Event.create(json.loads(event_string), self.content_service, action_logger)
                 start_time = time.time()
                 action_logger.debug(f"Processing event for did {event.context.did}")
 
@@ -277,14 +276,6 @@ class Plugin(object):
                                          f"Action attempted to look up element {e.index + 1} (index {e.index}) from "
                                          f"content list of size {e.size}",
                                          f"{str(e)}\n{traceback.format_exc()}")
-                except MissingDomainException as e:
-                    result = ErrorResult(event.context,
-                                         f"Action attempted to access domain {e.name}, which does not exist",
-                                         f"{str(e)}\n{traceback.format_exc()}")
-                except MissingEnrichmentException as e:
-                    result = ErrorResult(event.context,
-                                         f"Action attempted to access enrichment {e.name}, which does not exist",
-                                         f"{str(e)}\n{traceback.format_exc()}")
                 except MissingMetadataException as e:
                     result = ErrorResult(event.context,
                                          f"Missing metadata with key {e.key}",
@@ -297,7 +288,10 @@ class Plugin(object):
 
                 response = {
                     'did': event.context.did,
-                    'action': event.context.action_flow + "." + event.context.action_name,
+                    'flowName': event.context.flow_name,
+                    'flowId': event.context.flow_id,
+                    'actionName': event.context.action_name,
+                    'actionId': event.context.action_id,
                     'start': start_time,
                     'stop': time.time(),
                     'type': result.result_type,