deltafi 1.2.20__tar.gz → 2.0.0__tar.gz

{deltafi-1.2.20 → deltafi-2.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltafi
-Version: 1.2.20
+Version: 2.0.0
 Summary: SDK for DeltaFi plugins and actions
 License: Apache License, Version 2.0
 Keywords: deltafi
@@ -17,14 +17,16 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Software Development
+Requires-Dist: PyYAML (==6.0.2)
 Requires-Dist: deepdiff (==6.7.1)
 Requires-Dist: json-logging (==1.3.0)
-Requires-Dist: minio (==7.2.5)
-Requires-Dist: pydantic (==2.7.1)
-Requires-Dist: redis (==5.0.4)
-Requires-Dist: requests (==2.31.0)
-Requires-Dist: urllib3 (==2.2.1)
+Requires-Dist: minio (==7.2.10)
+Requires-Dist: pydantic (==2.9.2)
+Requires-Dist: redis (==5.2.0)
+Requires-Dist: requests (==2.32.3)
+Requires-Dist: urllib3 (==2.2.3)
 Project-URL: Bug Reports, https://chat.deltafi.org/deltafi/channels/bug-reports
 Project-URL: Documentation, https://docs.deltafi.org/#/
 Project-URL: Source Code, https://gitlab.com/deltafi/deltafi

{deltafi-1.2.20 → deltafi-2.0.0}/deltafi/__init__.py

@@ -1,7 +1,7 @@
 #
 #    DeltaFi - Data transformation and enrichment platform
 #
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#    Copyright 2021-2024 DeltaFi Contributors <deltafi@deltafi.org>
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.

deltafi-2.0.0/deltafi/action.py

@@ -0,0 +1,147 @@
+#
+#    DeltaFi - Data transformation and enrichment platform
+#
+#    Copyright 2021-2024 DeltaFi Contributors <deltafi@deltafi.org>
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+from abc import ABC, abstractmethod
+from typing import Any, List
+
+from deltafi.actiontype import ActionType
+from deltafi.domain import Context, DeltaFileMessage
+from deltafi.genericmodel import GenericModel
+from deltafi.input import EgressInput, TransformInput
+from deltafi.result import *
+from pydantic import BaseModel
+
+
+class Join(ABC):
+    def join(self, transform_inputs: List[TransformInput]):
+        all_content = []
+        all_metadata = {}
+        for transform_input in transform_inputs:
+            all_content += transform_input.content
+            all_metadata.update(transform_input.metadata)
+        return TransformInput(content=all_content, metadata=all_metadata)
+
+
+class Action(ABC):
+    def __init__(self, action_type: ActionType, description: str, valid_result_types: tuple):
+        self.action_type = action_type
+        self.description = description
+        self.valid_result_types = valid_result_types
+        self.action_execution = None
+
+    @abstractmethod
+    def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
+        pass
+
+    def execute_join_action(self, event):
+        raise RuntimeError(f"Join is not supported for {self.__class__.__name__}")
+
+    @abstractmethod
+    def execute(self, context: Context, action_input: Any, params: BaseModel):
+        pass
+
+    def execute_action(self, event):
+        if event.delta_file_messages is None or not len(event.delta_file_messages):
+            raise RuntimeError(f"Received event with no delta file messages for did {event.context.did}")
+        if event.context.join is not None:
+            result = self.execute_join_action(event)
+        else:
+            result = self.execute(
+                event.context,
+                self.build_input(event.context, event.delta_file_messages[0]),
+                self.param_class().model_validate(event.params))
+
+        self.validate_type(result)
+        return result
+
+    @staticmethod
+    def param_class():
+        """Factory method to create and return an empty GenericModel instance.
+
+        All action parameter classes must inherit pydantic.BaseModel.
+        Use of complex types in custom action parameter classes must specify
+        the internal types when defined. E.g., dict[str, str], or List[str]
+
+        Returns
+        -------
+        GenericModel
+            an empty GenericModel instance
+        """
+        return GenericModel
+
+    def validate_type(self, result):
+        if not isinstance(result, self.valid_result_types):
+            raise ValueError(f"{self.__class__.__name__} must return one of "
+                             f"{[result_type.__name__ for result_type in self.valid_result_types]} "
+                             f"but a {result.__class__.__name__} was returned")
+
+
+class EgressAction(Action, ABC):
+    def __init__(self, description: str):
+        super().__init__(ActionType.EGRESS, description, (EgressResult, ErrorResult, FilterResult))
+
+    def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
+        return EgressInput(content=delta_file_message.content_list[0], metadata=delta_file_message.metadata)
+
+    @abstractmethod
+    def egress(self, context: Context, params: BaseModel, egress_input: EgressInput):
+        pass
+
+    def execute(self, context: Context, egress_input: EgressInput, params: BaseModel):
+        return self.egress(context, params, egress_input)
+
+
+class TimedIngressAction(Action, ABC):
+    def __init__(self, description: str):
+        super().__init__(ActionType.TIMED_INGRESS, description, (IngressResult, ErrorResult))
+
+    def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
+        return None
+
+    @abstractmethod
+    def ingress(self, context: Context, params: BaseModel):
+        pass
+
+    def execute(self, context: Context, input_placeholder: Any, params: BaseModel):
+        return self.ingress(context, params)
+
+
+class TransformAction(Action, ABC):
+    def __init__(self, description: str):
+        super().__init__(ActionType.TRANSFORM, description,
+                         (TransformResult, TransformResults, ErrorResult, FilterResult))
+
+    def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
+        return TransformInput(content=delta_file_message.content_list, metadata=delta_file_message.metadata)
+
+    def execute_join_action(self, event):
+        if isinstance(self, Join):
+            return self.execute(
+                event.context,
+                self.join([self.build_input(event.context, delta_file_message)
+                           for delta_file_message in event.delta_file_messages]),
+                self.param_class().model_validate(event.params))
+        else:
+            super().execute_join_action(event)
+
+    @abstractmethod
+    def transform(self, context: Context, params: BaseModel, transform_input: TransformInput):
+        pass
+
+    def execute(self, context: Context, transform_input: TransformInput, params: BaseModel):
+        return self.transform(context, params, transform_input)

{deltafi-1.2.20 → deltafi-2.0.0}/deltafi/actioneventqueue.py

@@ -1,7 +1,7 @@
 #
 #    DeltaFi - Data transformation and enrichment platform
 #
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#    Copyright 2021-2024 DeltaFi Contributors <deltafi@deltafi.org>
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.

{deltafi-1.2.20 → deltafi-2.0.0}/deltafi/actiontype.py

@@ -1,7 +1,7 @@
 #
 #    DeltaFi - Data transformation and enrichment platform
 #
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#    Copyright 2021-2024 DeltaFi Contributors <deltafi@deltafi.org>
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -20,13 +20,9 @@ from enum import Enum
 
 
 class ActionType(Enum):
+    INGRESS = "INGRESS"
     TIMED_INGRESS = "TIMED_INGRESS"
     TRANSFORM = "TRANSFORM"
-    LOAD = "LOAD"
-    DOMAIN = "DOMAIN"
-    ENRICH = "ENRICH"
-    FORMAT = "FORMAT"
-    VALIDATE = "VALIDATE"
     EGRESS = "EGRESS"
-    DELETE = "DELETE"
+    PUBLISH = "PUBLISH"
     UNKNOWN = "UNKNOWN"

{deltafi-1.2.20 → deltafi-2.0.0}/deltafi/domain.py

@@ -1,7 +1,7 @@
 #
 #    DeltaFi - Data transformation and enrichment platform
 #
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#    Copyright 2021-2024 DeltaFi Contributors <deltafi@deltafi.org>
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -20,6 +20,7 @@ import copy
 from datetime import datetime, timedelta, timezone
 from logging import Logger
 from typing import Dict, List, NamedTuple
+from uuid import uuid4
 
 from deltafi.storage import ContentService, Segment
 
@@ -40,61 +41,88 @@ class ActionExecution(NamedTuple):
 
 class Context(NamedTuple):
     did: str
-    action_flow: str
+    delta_file_name: str
+    data_source: str
+    flow_name: str
+    flow_id: str
     action_name: str
-    source_filename: str
-    ingress_flow: str
-    egress_flow: str
-    system: str
+    action_version: str
     hostname: str
+    system_name: str
     content_service: ContentService
-    collect: dict = None
-    collected_dids: List[str] = None
+    join: dict = None
+    joined_dids: List[str] = None
     memo: str = None
     logger: Logger = None
+    saved_content: List = []
 
     @classmethod
-    def create(cls, context: dict, hostname: str, content_service: ContentService, logger: Logger):
+    def create(cls, context: dict, content_service: ContentService, logger: Logger):
         did = context['did']
-        action_name_parts = context['name'].split(".")
-        action_flow = action_name_parts[0]
-        action_name = action_name_parts[1]
-        if 'sourceFilename' in context:
-            source_filename = context['sourceFilename']
+        if 'deltaFileName' in context:
+            delta_file_name = context['deltaFileName']
         else:
-            source_filename = None
-        ingress_flow = context['ingressFlow']
-        if 'egressFlow' in context:
-            egress_flow = context['egressFlow']
+            delta_file_name = None
+        if 'dataSource' in context:
+            data_source = context['dataSource']
         else:
-            egress_flow = None
-        system = context['systemName']
-        if 'collect' in context:
-            collect = context['collect']
+            data_source = None
+        if 'flowName' in context:
+            flow_name = context['flowName']
         else:
-            collect = None
-        if 'collectedDids' in context:
-            collected_dids = context['collectedDids']
+            flow_name = None
+        if 'flowId' in context:
+            flow_id = context['flowId']
         else:
-            collected_dids = None
+            flow_id = None
+        if 'actionName' in context:
+            action_name = context['actionName']
+        else:
+            action_name = None
+        if 'actionVersion' in context:
+            action_version = context['actionVersion']
+        else:
+            action_version = None
+        if 'hostname' in context:
+            hostname = context['hostname']
+        else:
+            hostname = None
+        if 'systemName' in context:
+            system_name = context['systemName']
+        else:
+            system_name = None
+        if 'join' in context:
+            join = context['join']
+        else:
+            join = None
+        if 'joinedDids' in context:
+            joined_dids = context['joinedDids']
+        else:
+            joined_dids = None
         if 'memo' in context:
             memo = context['memo']
         else:
             memo = None
+
         return Context(did=did,
-                       action_flow=action_flow,
+                       delta_file_name=delta_file_name,
+                       data_source=data_source,
+                       flow_name=flow_name,
+                       flow_id=flow_id,
                        action_name=action_name,
-                       source_filename=source_filename,
-                       ingress_flow=ingress_flow,
-                       egress_flow=egress_flow,
-                       system=system,
+                       action_version=action_version,
                        hostname=hostname,
-                       content_service=content_service,
-                       collect=collect,
-                       collected_dids=collected_dids,
+                       system_name=system_name,
+                       join=join,
+                       joined_dids=joined_dids,
                        memo=memo,
+                       content_service=content_service,
+                       saved_content=[],
                        logger=logger)
 
+    def child_context(self):
+        return self._replace(did=str(uuid4()))
+
 
 class Content:
     """
@@ -197,7 +225,6 @@ class Content:
 
         return new_segments
 
-
     def get_size(self):
         """
         Returns the size of the content in bytes.
@@ -264,6 +291,12 @@ class Content:
         """
         self.segments.extend(other_content.segments)
 
+    def get_segment_names(self):
+        segment_names = {}
+        for seg in self.segments:
+            segment_names[seg.id()] = seg
+        return segment_names
+
     def __eq__(self, other):
         if isinstance(other, Content):
             return (self.name == other.name and
@@ -296,41 +329,17 @@ class Content:
                        content_service=content_service)
 
 
-class Domain(NamedTuple):
-    name: str
-    value: str
-    media_type: str
-
-    @classmethod
-    def from_dict(cls, domain: dict):
-        name = domain['name']
-        if 'value' in domain:
-            value = domain['value']
-        else:
-            value = None
-        media_type = domain['mediaType']
-        return Domain(name=name,
-                      value=value,
-                      media_type=media_type)
-
-
 class DeltaFileMessage(NamedTuple):
     metadata: Dict[str, str]
     content_list: List[Content]
-    domains: List[Domain]
-    enrichments: List[Domain]
 
     @classmethod
     def from_dict(cls, delta_file_message: dict, content_service: ContentService):
         metadata = delta_file_message['metadata']
         content_list = [Content.from_dict(content, content_service) for content in delta_file_message['contentList']]
-        domains = [Domain.from_dict(domain) for domain in delta_file_message['domains']] if 'domains' in delta_file_message else []
-        enrichments = [Domain.from_dict(domain) for domain in delta_file_message['enrichments']] if 'enrichments' in delta_file_message else []
 
         return DeltaFileMessage(metadata=metadata,
-                                content_list=content_list,
-                                domains=domains,
-                                enrichments=enrichments)
+                                content_list=content_list)
 
 
 class Event(NamedTuple):
@@ -341,9 +350,10 @@ class Event(NamedTuple):
     return_address: str
 
     @classmethod
-    def create(cls, event: dict, hostname: str, content_service: ContentService, logger: Logger):
-        delta_file_messages = [DeltaFileMessage.from_dict(delta_file_message, content_service) for delta_file_message in event['deltaFileMessages']]
-        context = Context.create(event['actionContext'], hostname, content_service, logger)
+    def create(cls, event: dict, content_service: ContentService, logger: Logger):
+        delta_file_messages = [DeltaFileMessage.from_dict(delta_file_message, content_service) for delta_file_message in
+                               event['deltaFileMessages']]
+        context = Context.create(event['actionContext'], content_service, logger)
         params = event['actionParams']
         queue_name = None
         if 'queueName' in event:

{deltafi-1.2.20 → deltafi-2.0.0}/deltafi/exception.py

@@ -1,7 +1,7 @@
 #
 #    DeltaFi - Data transformation and enrichment platform
 #
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#    Copyright 2021-2024 DeltaFi Contributors <deltafi@deltafi.org>
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -23,16 +23,6 @@ class ExpectedContentException(RuntimeError):
         self.size = size
 
 
-class MissingDomainException(RuntimeError):
-    def __init__(self, name):
-        self.name = name
-
-
-class MissingEnrichmentException(RuntimeError):
-    def __init__(self, name):
-        self.name = name
-
-
 class MissingMetadataException(RuntimeError):
     def __init__(self, key):
         self.key = key

{deltafi-1.2.20 → deltafi-2.0.0}/deltafi/genericmodel.py

@@ -1,7 +1,7 @@
 #
 #    DeltaFi - Data transformation and enrichment platform
 #
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#    Copyright 2021-2024 DeltaFi Contributors <deltafi@deltafi.org>
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -22,7 +22,9 @@
 
 Provides an empty subclass of pydantic.BaseModel.
 
-Starting Pydantic v2, the BaseModel cannot be directly instantiated.  This class provides for instantiation of GenericModel objects that inherit from BaseModel.
+All action parameter classes must inherit pydantic.BaseModel.
+
+Starting Pydantic v2, the BaseModel cannot be directly instantiated. This class provides for instantiation of GenericModel objects that inherit from BaseModel.
 
 This class does not define fields for validation or any other purpose.
 """

deltafi-2.0.0/deltafi/input.py

@@ -0,0 +1,52 @@
+#
+#    DeltaFi - Data transformation and enrichment platform
+#
+#    Copyright 2021-2024 DeltaFi Contributors <deltafi@deltafi.org>
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+from deltafi.domain import *
+from deltafi.exception import MissingMetadataException, ExpectedContentException
+
+class EgressInput(NamedTuple):
+    content: Content
+    metadata: dict
+
+
+class TransformInput(NamedTuple):
+    content: List[Content]
+    metadata: dict
+
+    def has_content(self) -> bool:
+        return len(self.content) > 0
+
+    def content_at(self, index: int) -> Content:
+        if len(self.content) < index + 1:
+            raise ExpectedContentException(index, len(self.content))
+        return self.content[index]
+
+    def first_content(self):
+        return self.content_at(0)
+
+    def get_metadata(self, key: str):
+        if key in self.metadata:
+            return self.metadata[key]
+        else:
+            raise MissingMetadataException(key)
+
+    def get_metadata_or_else(self, key: str, default: str) -> str:
+        if key in self.metadata:
+            return self.metadata[key]
+        else:
+            return default

{deltafi-1.2.20 → deltafi-2.0.0}/deltafi/logger.py

@@ -1,7 +1,7 @@
 #
 #    DeltaFi - Data transformation and enrichment platform
 #
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#    Copyright 2021-2024 DeltaFi Contributors <deltafi@deltafi.org>
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -18,14 +18,14 @@
 
 import logging
 import sys
-from datetime import datetime
+from datetime import datetime, UTC
 
 import json_logging
 
 
 def get_logger(name: str = None) -> logging.Logger:
     logger = logging.getLogger(name)
-    logger.setLevel(logging.DEBUG)
+    logger.setLevel(logging.INFO)
     logger.addHandler(logging.StreamHandler(sys.stdout))
     logger.propagate = False
 
@@ -42,7 +42,7 @@ def _sanitize_log_msg(record):
 class JSONLogFormatter(json_logging.JSONLogFormatter):
 
     def _format_log_object(self, record, request_util):
-        utcnow = datetime.utcnow()
+        utcnow = datetime.now(UTC)
 
         json_log_object = {
             'timestamp': json_logging.util.iso_time_format(utcnow),

{deltafi-1.2.20 → deltafi-2.0.0}/deltafi/metric.py

@@ -1,7 +1,7 @@
 #
 #    DeltaFi - Data transformation and enrichment platform
 #
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#    Copyright 2021-2024 DeltaFi Contributors <deltafi@deltafi.org>
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -22,7 +22,7 @@ from typing import Dict, NamedTuple
 class Metric(NamedTuple):
     name: str
     value: int
-    tags: Dict[str, str]
+    tags: Dict[str, str] = {}
 
     def json(self):
         return {