deltafi 2.0rc1719271450675__tar.gz → 2.0rc1720728217472__tar.gz

{deltafi-2.0rc1719271450675 → deltafi-2.0rc1720728217472}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltafi
-Version: 2.0rc1719271450675
+Version: 2.0rc1720728217472
 Summary: SDK for DeltaFi plugins and actions
 License: Apache License, Version 2.0
 Keywords: deltafi
@@ -20,11 +20,11 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Software Development
 Requires-Dist: deepdiff (>=6.7.1)
 Requires-Dist: json-logging (>=1.3.0)
-Requires-Dist: minio (>=7.2.5)
-Requires-Dist: pydantic (>=2.7.1)
-Requires-Dist: redis (>=5.0.4)
+Requires-Dist: minio (>=7.2.3)
+Requires-Dist: pydantic (>=2.5.3)
+Requires-Dist: redis (>=5.0.1)
 Requires-Dist: requests (>=2.31.0)
-Requires-Dist: urllib3 (>=2.2.1)
+Requires-Dist: urllib3 (>=2.1.0)
 Project-URL: Bug Reports, https://chat.deltafi.org/deltafi/channels/bug-reports
 Project-URL: Documentation, https://docs.deltafi.org/#/
 Project-URL: Source Code, https://gitlab.com/deltafi/deltafi

deltafi-2.0rc1720728217472/deltafi/action.py

@@ -0,0 +1,237 @@
+#
+#    DeltaFi - Data transformation and enrichment platform
+#
+#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+from abc import ABC, abstractmethod
+from typing import Any, List
+
+from deltafi.actiontype import ActionType
+from deltafi.genericmodel import GenericModel
+from deltafi.domain import Context, DeltaFileMessage
+from deltafi.input import DomainInput, EgressInput, EnrichInput, FormatInput, LoadInput, TransformInput, ValidateInput
+from deltafi.result import *
+from pydantic import BaseModel
+
+
+class Action(ABC):
+    def __init__(self, action_type: ActionType, description: str, requires_domains: List[str],
+                 requires_enrichments: List[str], valid_result_types: tuple):
+        self.action_type = action_type
+        self.description = description
+        self.requires_domains = requires_domains
+        self.requires_enrichments = requires_enrichments
+        self.valid_result_types = valid_result_types
+        self.action_execution = None
+
+    @abstractmethod
+    def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
+        pass
+
+    def collect(self, action_inputs: List[Any]):
+        raise RuntimeError(f"Collect is not supported for {self.__class__.__name__}")
+
+    @abstractmethod
+    def execute(self, context: Context, action_input: Any, params: BaseModel):
+        pass
+
+    def execute_action(self, event):
+        if event.delta_file_messages is None or not len(event.delta_file_messages):
+            raise RuntimeError(f"Received event with no delta file messages for did {event.context.did}")
+
+        if event.context.collect is not None:
+            result = self.execute(event.context, self.collect([self.build_input(event.context, delta_file_message)
+                                                               for delta_file_message in event.delta_file_messages]),
+                                  self.param_class().model_validate(event.params))
+        else:
+            result = self.execute(event.context, self.build_input(event.context, event.delta_file_messages[0]),
+                                  self.param_class().model_validate(event.params))
+
+        self.validate_type(result)
+        return result
+
+    @staticmethod
+    def param_class( ):
+        """Factory method to create and return an empty GenericModel instance.
+
+        Returns
+        -------
+        GenericModel
+            an empty GenericModel instance
+        """
+        return GenericModel
+
+    def validate_type(self, result):
+        if not isinstance(result, self.valid_result_types):
+            raise ValueError(f"{self.__class__.__name__} must return one of "
+                             f"{[result_type.__name__ for result_type in self.valid_result_types]} "
+                             f"but a {result.__class__.__name__} was returned")
+
+
+class DomainAction(Action, ABC):
+    def __init__(self, description: str, requires_domains: List[str]):
+        super().__init__(ActionType.DOMAIN, description, requires_domains, [], (DomainResult, ErrorResult))
+
+    def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
+        return DomainInput(content=delta_file_message.content_list, metadata=delta_file_message.metadata,
+                           domains={domain.name: domain for domain in delta_file_message.domains})
+
+    @abstractmethod
+    def domain(self, context: Context, params: BaseModel, domain_input: DomainInput):
+        pass
+
+    def execute(self, context: Context, domain_input: DomainInput, params: BaseModel):
+        return self.domain(context, params, domain_input)
+
+
+class EgressAction(Action, ABC):
+    def __init__(self, description: str):
+        super().__init__(ActionType.EGRESS, description, [], [], (EgressResult, ErrorResult, FilterResult))
+
+    def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
+        return EgressInput(content=delta_file_message.content_list[0], metadata=delta_file_message.metadata)
+
+    @abstractmethod
+    def egress(self, context: Context, params: BaseModel, egress_input: EgressInput):
+        pass
+
+    def execute(self, context: Context, egress_input: EgressInput, params: BaseModel):
+        return self.egress(context, params, egress_input)
+
+
+class EnrichAction(Action, ABC):
+    def __init__(self, description: str, requires_domains: List[str], requires_enrichments: List[str]):
+        super().__init__(ActionType.ENRICH, description, requires_domains, requires_enrichments,
+                         (EnrichResult, ErrorResult))
+
+    def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
+        return EnrichInput(content=delta_file_message.content_list, metadata=delta_file_message.metadata,
+                           domains={domain.name: domain for domain in delta_file_message.domains},
+                           enrichments={domain.name: domain for domain in delta_file_message.enrichments})
+
+    @abstractmethod
+    def enrich(self, context: Context, params: BaseModel, enrich_input: EnrichInput):
+        pass
+
+    def execute(self, context: Context, enrich_input: EnrichInput, params: BaseModel):
+        return self.enrich(context, params, enrich_input)
+
+
+class FormatAction(Action, ABC):
+    def __init__(self, description: str, requires_domains: List[str], requires_enrichments: List[str]):
+        super().__init__(ActionType.FORMAT, description, requires_domains, requires_enrichments,
+                         (FormatResult, FormatManyResult, ErrorResult, FilterResult))
+
+    def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
+        return FormatInput(content=delta_file_message.content_list, metadata=delta_file_message.metadata,
+                           domains={domain.name: domain for domain in delta_file_message.domains},
+                           enrichments={domain.name: domain for domain in delta_file_message.enrichments})
+
+    def collect(self, format_inputs: List[FormatInput]):
+        all_content = []
+        all_metadata = {}
+        all_domains = {}
+        all_enrichments = {}
+        for format_input in format_inputs:
+            all_content += format_input.content
+            all_metadata.update(format_input.metadata)
+            all_domains.update(format_input.domains)
+            all_enrichments.update(format_input.enrichments)
+        return FormatInput(content=all_content, metadata=all_metadata, domains=all_domains, enrichments=all_enrichments)
+
+    @abstractmethod
+    def format(self, context: Context, params: BaseModel, format_input: FormatInput):
+        pass
+
+    def execute(self, context: Context, format_input: FormatInput, params: BaseModel):
+        return self.format(context, params, format_input)
+
+
+class LoadAction(Action, ABC):
+    def __init__(self, description: str):
+        super().__init__(ActionType.LOAD, description, [], [],
+                         (LoadResult, LoadManyResult, ErrorResult, FilterResult, ReinjectResult))
+
+    def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
+        return LoadInput(content=delta_file_message.content_list, metadata=delta_file_message.metadata)
+
+    def collect(self, load_inputs: List[LoadInput]):
+        all_content = []
+        all_metadata = {}
+        for load_input in load_inputs:
+            all_content += load_input.content
+            all_metadata.update(load_input.metadata)
+        return LoadInput(content=all_content, metadata=all_metadata)
+
+    @abstractmethod
+    def load(self, context: Context, params: BaseModel, load_input: LoadInput):
+        pass
+
+    def execute(self, context: Context, load_input: LoadInput, params: BaseModel):
+        return self.load(context, params, load_input)
+
+
+class TimedIngressAction(Action, ABC):
+    def __init__(self, description: str):
+        super().__init__(ActionType.TIMED_INGRESS, description, [], [], (IngressResult, ErrorResult))
+
+    def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
+        return None
+
+    @abstractmethod
+    def ingress(self, context: Context, params: BaseModel):
+        pass
+
+    def execute(self, context: Context, input_placeholder: Any, params: BaseModel):
+        return self.ingress(context, params)
+
+
+class TransformAction(Action, ABC):
+    def __init__(self, description: str):
+        super().__init__(ActionType.TRANSFORM, description, [], [], (TransformResult, ErrorResult, FilterResult, ReinjectResult))
+
+    def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
+        return TransformInput(content=delta_file_message.content_list, metadata=delta_file_message.metadata)
+
+    def collect(self, transform_inputs: List[TransformInput]):
+        all_content = []
+        all_metadata = {}
+        for transform_input in transform_inputs:
+            all_content += transform_input.content
+            all_metadata.update(transform_input.metadata)
+        return TransformInput(content=all_content, metadata=all_metadata)
+
+    @abstractmethod
+    def transform(self, context: Context, params: BaseModel, transform_input: TransformInput):
+        pass
+
+    def execute(self, context: Context, transform_input: TransformInput, params: BaseModel):
+        return self.transform(context, params, transform_input)
+
+
+class ValidateAction(Action, ABC):
+    def __init__(self, description: str):
+        super().__init__(ActionType.VALIDATE, description, [], [], (ValidateResult, ErrorResult, FilterResult))
+
+    def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
+        return ValidateInput(content=delta_file_message.content_list[0], metadata=delta_file_message.metadata)
+
+    @abstractmethod
+    def validate(self, context: Context, params: BaseModel, validate_input: ValidateInput):
+        pass
+
+    def execute(self, context: Context, validate_input: ValidateInput, params: BaseModel):
+        return self.validate(context, params, validate_input)

{deltafi-2.0rc1719271450675 → deltafi-2.0rc1720728217472}/deltafi/actiontype.py

@@ -20,9 +20,13 @@ from enum import Enum
 
 
 class ActionType(Enum):
-    INGRESS = "INGRESS"
     TIMED_INGRESS = "TIMED_INGRESS"
     TRANSFORM = "TRANSFORM"
+    LOAD = "LOAD"
+    DOMAIN = "DOMAIN"
+    ENRICH = "ENRICH"
+    FORMAT = "FORMAT"
+    VALIDATE = "VALIDATE"
     EGRESS = "EGRESS"
-    PUBLISH = "PUBLISH"
+    DELETE = "DELETE"
     UNKNOWN = "UNKNOWN"

{deltafi-2.0rc1719271450675 → deltafi-2.0rc1720728217472}/deltafi/domain.py

@@ -40,15 +40,13 @@ class ActionExecution(NamedTuple):
 
 class Context(NamedTuple):
     did: str
-    delta_file_name: str
-    data_source: str
-    flow_name: str
-    flow_id: str
+    action_flow: str
     action_name: str
-    action_id: str
-    action_version: str
+    source_filename: str
+    ingress_flow: str
+    egress_flow: str
+    system: str
     hostname: str
-    system_name: str
     content_service: ContentService
     collect: dict = None
     collected_dids: List[str] = None
@@ -56,44 +54,21 @@ class Context(NamedTuple):
     logger: Logger = None
 
     @classmethod
-    def create(cls, context: dict, content_service: ContentService, logger: Logger):
+    def create(cls, context: dict, hostname: str, content_service: ContentService, logger: Logger):
         did = context['did']
-        if 'deltaFileName' in context:
-            delta_file_name = context['deltaFileName']
+        action_name_parts = context['name'].split(".")
+        action_flow = action_name_parts[0]
+        action_name = action_name_parts[1]
+        if 'sourceFilename' in context:
+            source_filename = context['sourceFilename']
         else:
-            delta_file_name = None
-        if 'dataSource' in context:
-            data_source = context['dataSource']
+            source_filename = None
+        ingress_flow = context['ingressFlow']
+        if 'egressFlow' in context:
+            egress_flow = context['egressFlow']
         else:
-            data_source = None
-        if 'flowName' in context:
-            flow_name = context['flowName']
-        else:
-            flow_name = None
-        if 'flowId' in context:
-            flow_id = context['flowId']
-        else:
-            flow_id = None
-        if 'actionName' in context:
-            action_name = context['actionName']
-        else:
-            action_name = None
-        if 'actionId' in context:
-            action_id = context['actionId']
-        else:
-            action_id = None
-        if 'actionVersion' in context:
-            action_version = context['actionVersion']
-        else:
-            action_version = None
-        if 'hostname' in context:
-            hostname = context['hostname']
-        else:
-            hostname = None
-        if 'systemName' in context:
-            system_name = context['systemName']
-        else:
-            system_name = None
+            egress_flow = None
+        system = context['systemName']
         if 'collect' in context:
             collect = context['collect']
         else:
@@ -106,21 +81,18 @@ class Context(NamedTuple):
             memo = context['memo']
         else:
             memo = None
-
         return Context(did=did,
-                       delta_file_name=delta_file_name,
-                       data_source=data_source,
-                       flow_name=flow_name,
-                       flow_id=flow_id,
+                       action_flow=action_flow,
                        action_name=action_name,
-                       action_id=action_id,
-                       action_version=action_version,
+                       source_filename=source_filename,
+                       ingress_flow=ingress_flow,
+                       egress_flow=egress_flow,
+                       system=system,
                        hostname=hostname,
-                       system_name=system_name,
+                       content_service=content_service,
                        collect=collect,
                        collected_dids=collected_dids,
                        memo=memo,
-                       content_service=content_service,
                        logger=logger)
 
 
@@ -225,6 +197,7 @@ class Content:
 
         return new_segments
 
+
     def get_size(self):
         """
         Returns the size of the content in bytes.
@@ -323,17 +296,41 @@ class Content:
                        content_service=content_service)
 
 
+class Domain(NamedTuple):
+    name: str
+    value: str
+    media_type: str
+
+    @classmethod
+    def from_dict(cls, domain: dict):
+        name = domain['name']
+        if 'value' in domain:
+            value = domain['value']
+        else:
+            value = None
+        media_type = domain['mediaType']
+        return Domain(name=name,
+                      value=value,
+                      media_type=media_type)
+
+
 class DeltaFileMessage(NamedTuple):
     metadata: Dict[str, str]
     content_list: List[Content]
+    domains: List[Domain]
+    enrichments: List[Domain]
 
     @classmethod
     def from_dict(cls, delta_file_message: dict, content_service: ContentService):
         metadata = delta_file_message['metadata']
         content_list = [Content.from_dict(content, content_service) for content in delta_file_message['contentList']]
+        domains = [Domain.from_dict(domain) for domain in delta_file_message['domains']] if 'domains' in delta_file_message else []
+        enrichments = [Domain.from_dict(domain) for domain in delta_file_message['enrichments']] if 'enrichments' in delta_file_message else []
 
         return DeltaFileMessage(metadata=metadata,
-                                content_list=content_list)
+                                content_list=content_list,
+                                domains=domains,
+                                enrichments=enrichments)
 
 
 class Event(NamedTuple):
@@ -344,10 +341,9 @@ class Event(NamedTuple):
     return_address: str
 
     @classmethod
-    def create(cls, event: dict, content_service: ContentService, logger: Logger):
-        delta_file_messages = [DeltaFileMessage.from_dict(delta_file_message, content_service) for delta_file_message in
-                               event['deltaFileMessages']]
-        context = Context.create(event['actionContext'], content_service, logger)
+    def create(cls, event: dict, hostname: str, content_service: ContentService, logger: Logger):
+        delta_file_messages = [DeltaFileMessage.from_dict(delta_file_message, content_service) for delta_file_message in event['deltaFileMessages']]
+        context = Context.create(event['actionContext'], hostname, content_service, logger)
         params = event['actionParams']
         queue_name = None
         if 'queueName' in event:

{deltafi-2.0rc1719271450675 → deltafi-2.0rc1720728217472}/deltafi/exception.py

@@ -23,6 +23,16 @@ class ExpectedContentException(RuntimeError):
         self.size = size
 
 
+class MissingDomainException(RuntimeError):
+    def __init__(self, name):
+        self.name = name
+
+
+class MissingEnrichmentException(RuntimeError):
+    def __init__(self, name):
+        self.name = name
+
+
 class MissingMetadataException(RuntimeError):
     def __init__(self, key):
         self.key = key

deltafi-2.0rc1720728217472/deltafi/input.py

@@ -0,0 +1,216 @@
+#
+#    DeltaFi - Data transformation and enrichment platform
+#
+#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+from deltafi.domain import *
+from deltafi.exception import MissingMetadataException, ExpectedContentException, MissingDomainException, \
+    MissingEnrichmentException
+
+
+class DomainInput(NamedTuple):
+    content: List[Content]
+    metadata: Dict[str, str]
+    domains: Dict[str, Domain]
+
+    def has_content(self) -> bool:
+        return len(self.content) > 0
+
+    def content_at(self, index: int) -> Content:
+        if len(self.content) < index + 1:
+            raise ExpectedContentException(index, len(self.content))
+        return self.content[index]
+
+    def first_content(self):
+        return self.content_at(0)
+
+    def get_metadata(self, key: str):
+        if key in self.metadata:
+            return self.metadata[key]
+        else:
+            raise MissingMetadataException(key)
+
+    def get_metadata_or_else(self, key: str, default: str) -> str:
+        if key in self.metadata:
+            return self.metadata[key]
+        else:
+            return default
+
+    def has_domain(self, name: str) -> bool:
+        return name in self.domains
+
+    def domain(self, name: str) -> Domain:
+        if not self.has_domain(name):
+            raise MissingDomainException(name)
+        return self.domains[name]
+
+
+class EgressInput(NamedTuple):
+    content: Content
+    metadata: dict
+
+
+class EnrichInput(NamedTuple):
+    content: List[Content]
+    metadata: dict
+    domains: Dict[str, Domain]
+    enrichments: Dict[str, Domain]
+
+    def has_content(self) -> bool:
+        return len(self.content) > 0
+
+    def content_at(self, index: int) -> Content:
+        if len(self.content) < index + 1:
+            raise ExpectedContentException(index, len(self.content))
+        return self.content[index]
+
+    def first_content(self):
+        return self.content_at(0)
+
+    def get_metadata(self, key: str):
+        if key in self.metadata:
+            return self.metadata[key]
+        else:
+            raise MissingMetadataException(key)
+
+    def get_metadata_or_else(self, key: str, default: str) -> str:
+        if key in self.metadata:
+            return self.metadata[key]
+        else:
+            return default
+
+    def has_domain(self, name: str) -> bool:
+        return name in self.domains
+
+    def domain(self, name: str) -> Domain:
+        if not self.has_domain(name):
+            raise MissingDomainException(name)
+        return self.domains[name]
+
+    def has_enrichment(self, name: str) -> bool:
+        return name in self.enrichments
+
+    def enrichment(self, name: str) -> Domain:
+        if not self.has_enrichment(name):
+            raise MissingEnrichmentException(name)
+        return self.enrichments[name]
+
+
+class FormatInput(NamedTuple):
+    content: List[Content]
+    metadata: dict
+    domains: Dict[str, Domain]
+    enrichments: Dict[str, Domain]
+
+    def has_content(self) -> bool:
+        return len(self.content) > 0
+
+    def content_at(self, index: int) -> Content:
+        if len(self.content) < index + 1:
+            raise ExpectedContentException(index, len(self.content))
+        return self.content[index]
+
+    def first_content(self):
+        return self.content_at(0)
+
+    def get_metadata(self, key: str):
+        if key in self.metadata:
+            return self.metadata[key]
+        else:
+            raise MissingMetadataException(key)
+
+    def get_metadata_or_else(self, key: str, default: str) -> str:
+        if key in self.metadata:
+            return self.metadata[key]
+        else:
+            return default
+
+    def has_domain(self, name: str) -> bool:
+        return name in self.domains
+
+    def domain(self, name: str) -> Domain:
+        if not self.has_domain(name):
+            raise MissingDomainException(name)
+        return self.domains[name]
+
+    def has_enrichment(self, name: str) -> bool:
+        return name in self.enrichments
+
+    def enrichment(self, name: str) -> Domain:
+        if not self.has_enrichment(name):
+            raise MissingEnrichmentException(name)
+        return self.enrichments[name]
+
+
+class LoadInput(NamedTuple):
+    content: List[Content]
+    metadata: dict
+
+    def has_content(self) -> bool:
+        return len(self.content) > 0
+
+    def content_at(self, index: int) -> Content:
+        if len(self.content) < index + 1:
+            raise ExpectedContentException(index, len(self.content))
+        return self.content[index]
+
+    def first_content(self):
+        return self.content_at(0)
+
+    def get_metadata(self, key: str):
+        if key in self.metadata:
+            return self.metadata[key]
+        else:
+            raise MissingMetadataException(key)
+
+    def get_metadata_or_else(self, key: str, default: str) -> str:
+        if key in self.metadata:
+            return self.metadata[key]
+        else:
+            return default
+
+
+class TransformInput(NamedTuple):
+    content: List[Content]
+    metadata: dict
+
+    def has_content(self) -> bool:
+        return len(self.content) > 0
+
+    def content_at(self, index: int) -> Content:
+        if len(self.content) < index + 1:
+            raise ExpectedContentException(index, len(self.content))
+        return self.content[index]
+
+    def first_content(self):
+        return self.content_at(0)
+
+    def get_metadata(self, key: str):
+        if key in self.metadata:
+            return self.metadata[key]
+        else:
+            raise MissingMetadataException(key)
+
+    def get_metadata_or_else(self, key: str, default: str) -> str:
+        if key in self.metadata:
+            return self.metadata[key]
+        else:
+            return default
+
+
+class ValidateInput(NamedTuple):
+    content: Content
+    metadata: dict