deltafi 2.0rc1719271450675__py3-none-any.whl → 2.0rc1720728217472__py3-none-any.whl

deltafi/action.py

@@ -20,17 +20,20 @@ from abc import ABC, abstractmethod
 from typing import Any, List
 
 from deltafi.actiontype import ActionType
-from deltafi.domain import Context, DeltaFileMessage
 from deltafi.genericmodel import GenericModel
-from deltafi.input import EgressInput, TransformInput
+from deltafi.domain import Context, DeltaFileMessage
+from deltafi.input import DomainInput, EgressInput, EnrichInput, FormatInput, LoadInput, TransformInput, ValidateInput
 from deltafi.result import *
 from pydantic import BaseModel
 
 
 class Action(ABC):
-    def __init__(self, action_type: ActionType, description: str, valid_result_types: tuple):
+    def __init__(self, action_type: ActionType, description: str, requires_domains: List[str],
+                 requires_enrichments: List[str], valid_result_types: tuple):
         self.action_type = action_type
         self.description = description
+        self.requires_domains = requires_domains
+        self.requires_enrichments = requires_enrichments
         self.valid_result_types = valid_result_types
         self.action_execution = None
 
@@ -48,23 +51,20 @@ class Action(ABC):
     def execute_action(self, event):
         if event.delta_file_messages is None or not len(event.delta_file_messages):
             raise RuntimeError(f"Received event with no delta file messages for did {event.context.did}")
+
         if event.context.collect is not None:
-            result = self.execute(
-                event.context,
-                self.collect([self.build_input(event.context, delta_file_message)
-                              for delta_file_message in event.delta_file_messages]),
-                self.param_class().model_validate(event.params))
+            result = self.execute(event.context, self.collect([self.build_input(event.context, delta_file_message)
+                                                               for delta_file_message in event.delta_file_messages]),
+                                  self.param_class().model_validate(event.params))
         else:
-            result = self.execute(
-                event.context,
-                self.build_input(event.context, event.delta_file_messages[0]),
-                self.param_class().model_validate(event.params))
+            result = self.execute(event.context, self.build_input(event.context, event.delta_file_messages[0]),
+                                  self.param_class().model_validate(event.params))
 
         self.validate_type(result)
         return result
 
     @staticmethod
-    def param_class():
+    def param_class( ):
         """Factory method to create and return an empty GenericModel instance.
 
         Returns
@@ -81,9 +81,25 @@ class Action(ABC):
                              f"but a {result.__class__.__name__} was returned")
 
 
+class DomainAction(Action, ABC):
+    def __init__(self, description: str, requires_domains: List[str]):
+        super().__init__(ActionType.DOMAIN, description, requires_domains, [], (DomainResult, ErrorResult))
+
+    def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
+        return DomainInput(content=delta_file_message.content_list, metadata=delta_file_message.metadata,
+                           domains={domain.name: domain for domain in delta_file_message.domains})
+
+    @abstractmethod
+    def domain(self, context: Context, params: BaseModel, domain_input: DomainInput):
+        pass
+
+    def execute(self, context: Context, domain_input: DomainInput, params: BaseModel):
+        return self.domain(context, params, domain_input)
+
+
 class EgressAction(Action, ABC):
     def __init__(self, description: str):
-        super().__init__(ActionType.EGRESS, description, (EgressResult, ErrorResult, FilterResult))
+        super().__init__(ActionType.EGRESS, description, [], [], (EgressResult, ErrorResult, FilterResult))
 
     def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
         return EgressInput(content=delta_file_message.content_list[0], metadata=delta_file_message.metadata)
@@ -96,9 +112,81 @@ class EgressAction(Action, ABC):
         return self.egress(context, params, egress_input)
 
 
+class EnrichAction(Action, ABC):
+    def __init__(self, description: str, requires_domains: List[str], requires_enrichments: List[str]):
+        super().__init__(ActionType.ENRICH, description, requires_domains, requires_enrichments,
+                         (EnrichResult, ErrorResult))
+
+    def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
+        return EnrichInput(content=delta_file_message.content_list, metadata=delta_file_message.metadata,
+                           domains={domain.name: domain for domain in delta_file_message.domains},
+                           enrichments={domain.name: domain for domain in delta_file_message.enrichments})
+
+    @abstractmethod
+    def enrich(self, context: Context, params: BaseModel, enrich_input: EnrichInput):
+        pass
+
+    def execute(self, context: Context, enrich_input: EnrichInput, params: BaseModel):
+        return self.enrich(context, params, enrich_input)
+
+
+class FormatAction(Action, ABC):
+    def __init__(self, description: str, requires_domains: List[str], requires_enrichments: List[str]):
+        super().__init__(ActionType.FORMAT, description, requires_domains, requires_enrichments,
+                         (FormatResult, FormatManyResult, ErrorResult, FilterResult))
+
+    def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
+        return FormatInput(content=delta_file_message.content_list, metadata=delta_file_message.metadata,
+                           domains={domain.name: domain for domain in delta_file_message.domains},
+                           enrichments={domain.name: domain for domain in delta_file_message.enrichments})
+
+    def collect(self, format_inputs: List[FormatInput]):
+        all_content = []
+        all_metadata = {}
+        all_domains = {}
+        all_enrichments = {}
+        for format_input in format_inputs:
+            all_content += format_input.content
+            all_metadata.update(format_input.metadata)
+            all_domains.update(format_input.domains)
+            all_enrichments.update(format_input.enrichments)
+        return FormatInput(content=all_content, metadata=all_metadata, domains=all_domains, enrichments=all_enrichments)
+
+    @abstractmethod
+    def format(self, context: Context, params: BaseModel, format_input: FormatInput):
+        pass
+
+    def execute(self, context: Context, format_input: FormatInput, params: BaseModel):
+        return self.format(context, params, format_input)
+
+
+class LoadAction(Action, ABC):
+    def __init__(self, description: str):
+        super().__init__(ActionType.LOAD, description, [], [],
+                         (LoadResult, LoadManyResult, ErrorResult, FilterResult, ReinjectResult))
+
+    def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
+        return LoadInput(content=delta_file_message.content_list, metadata=delta_file_message.metadata)
+
+    def collect(self, load_inputs: List[LoadInput]):
+        all_content = []
+        all_metadata = {}
+        for load_input in load_inputs:
+            all_content += load_input.content
+            all_metadata.update(load_input.metadata)
+        return LoadInput(content=all_content, metadata=all_metadata)
+
+    @abstractmethod
+    def load(self, context: Context, params: BaseModel, load_input: LoadInput):
+        pass
+
+    def execute(self, context: Context, load_input: LoadInput, params: BaseModel):
+        return self.load(context, params, load_input)
+
+
 class TimedIngressAction(Action, ABC):
     def __init__(self, description: str):
-        super().__init__(ActionType.TIMED_INGRESS, description, (IngressResult, ErrorResult))
+        super().__init__(ActionType.TIMED_INGRESS, description, [], [], (IngressResult, ErrorResult))
 
     def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
         return None
@@ -113,8 +201,7 @@ class TimedIngressAction(Action, ABC):
 
 class TransformAction(Action, ABC):
     def __init__(self, description: str):
-        super().__init__(ActionType.TRANSFORM, description,
-                         (TransformResult, TransformResults, ErrorResult, FilterResult))
+        super().__init__(ActionType.TRANSFORM, description, [], [], (TransformResult, ErrorResult, FilterResult, ReinjectResult))
 
     def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
         return TransformInput(content=delta_file_message.content_list, metadata=delta_file_message.metadata)
@@ -133,3 +220,18 @@ class TransformAction(Action, ABC):
 
     def execute(self, context: Context, transform_input: TransformInput, params: BaseModel):
         return self.transform(context, params, transform_input)
+
+
+class ValidateAction(Action, ABC):
+    def __init__(self, description: str):
+        super().__init__(ActionType.VALIDATE, description, [], [], (ValidateResult, ErrorResult, FilterResult))
+
+    def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
+        return ValidateInput(content=delta_file_message.content_list[0], metadata=delta_file_message.metadata)
+
+    @abstractmethod
+    def validate(self, context: Context, params: BaseModel, validate_input: ValidateInput):
+        pass
+
+    def execute(self, context: Context, validate_input: ValidateInput, params: BaseModel):
+        return self.validate(context, params, validate_input)

deltafi/actiontype.py

@@ -20,9 +20,13 @@ from enum import Enum
 
 
 class ActionType(Enum):
-    INGRESS = "INGRESS"
     TIMED_INGRESS = "TIMED_INGRESS"
     TRANSFORM = "TRANSFORM"
+    LOAD = "LOAD"
+    DOMAIN = "DOMAIN"
+    ENRICH = "ENRICH"
+    FORMAT = "FORMAT"
+    VALIDATE = "VALIDATE"
     EGRESS = "EGRESS"
-    PUBLISH = "PUBLISH"
+    DELETE = "DELETE"
     UNKNOWN = "UNKNOWN"

deltafi/domain.py

@@ -40,15 +40,13 @@ class ActionExecution(NamedTuple):
 
 class Context(NamedTuple):
     did: str
-    delta_file_name: str
-    data_source: str
-    flow_name: str
-    flow_id: str
+    action_flow: str
     action_name: str
-    action_id: str
-    action_version: str
+    source_filename: str
+    ingress_flow: str
+    egress_flow: str
+    system: str
     hostname: str
-    system_name: str
     content_service: ContentService
     collect: dict = None
     collected_dids: List[str] = None
@@ -56,44 +54,21 @@ class Context(NamedTuple):
     logger: Logger = None
 
     @classmethod
-    def create(cls, context: dict, content_service: ContentService, logger: Logger):
+    def create(cls, context: dict, hostname: str, content_service: ContentService, logger: Logger):
         did = context['did']
-        if 'deltaFileName' in context:
-            delta_file_name = context['deltaFileName']
+        action_name_parts = context['name'].split(".")
+        action_flow = action_name_parts[0]
+        action_name = action_name_parts[1]
+        if 'sourceFilename' in context:
+            source_filename = context['sourceFilename']
         else:
-            delta_file_name = None
-        if 'dataSource' in context:
-            data_source = context['dataSource']
+            source_filename = None
+        ingress_flow = context['ingressFlow']
+        if 'egressFlow' in context:
+            egress_flow = context['egressFlow']
         else:
-            data_source = None
-        if 'flowName' in context:
-            flow_name = context['flowName']
-        else:
-            flow_name = None
-        if 'flowId' in context:
-            flow_id = context['flowId']
-        else:
-            flow_id = None
-        if 'actionName' in context:
-            action_name = context['actionName']
-        else:
-            action_name = None
-        if 'actionId' in context:
-            action_id = context['actionId']
-        else:
-            action_id = None
-        if 'actionVersion' in context:
-            action_version = context['actionVersion']
-        else:
-            action_version = None
-        if 'hostname' in context:
-            hostname = context['hostname']
-        else:
-            hostname = None
-        if 'systemName' in context:
-            system_name = context['systemName']
-        else:
-            system_name = None
+            egress_flow = None
+        system = context['systemName']
         if 'collect' in context:
             collect = context['collect']
         else:
@@ -106,21 +81,18 @@ class Context(NamedTuple):
             memo = context['memo']
         else:
             memo = None
-
         return Context(did=did,
-                       delta_file_name=delta_file_name,
-                       data_source=data_source,
-                       flow_name=flow_name,
-                       flow_id=flow_id,
+                       action_flow=action_flow,
                        action_name=action_name,
-                       action_id=action_id,
-                       action_version=action_version,
+                       source_filename=source_filename,
+                       ingress_flow=ingress_flow,
+                       egress_flow=egress_flow,
+                       system=system,
                        hostname=hostname,
-                       system_name=system_name,
+                       content_service=content_service,
                        collect=collect,
                        collected_dids=collected_dids,
                        memo=memo,
-                       content_service=content_service,
                        logger=logger)
 
 
@@ -225,6 +197,7 @@ class Content:
 
         return new_segments
 
+
     def get_size(self):
         """
         Returns the size of the content in bytes.
@@ -323,17 +296,41 @@ class Content:
                        content_service=content_service)
 
 
+class Domain(NamedTuple):
+    name: str
+    value: str
+    media_type: str
+
+    @classmethod
+    def from_dict(cls, domain: dict):
+        name = domain['name']
+        if 'value' in domain:
+            value = domain['value']
+        else:
+            value = None
+        media_type = domain['mediaType']
+        return Domain(name=name,
+                      value=value,
+                      media_type=media_type)
+
+
 class DeltaFileMessage(NamedTuple):
     metadata: Dict[str, str]
     content_list: List[Content]
+    domains: List[Domain]
+    enrichments: List[Domain]
 
     @classmethod
     def from_dict(cls, delta_file_message: dict, content_service: ContentService):
         metadata = delta_file_message['metadata']
         content_list = [Content.from_dict(content, content_service) for content in delta_file_message['contentList']]
+        domains = [Domain.from_dict(domain) for domain in delta_file_message['domains']] if 'domains' in delta_file_message else []
+        enrichments = [Domain.from_dict(domain) for domain in delta_file_message['enrichments']] if 'enrichments' in delta_file_message else []
 
         return DeltaFileMessage(metadata=metadata,
-                                content_list=content_list)
+                                content_list=content_list,
+                                domains=domains,
+                                enrichments=enrichments)
 
 
 class Event(NamedTuple):
@@ -344,10 +341,9 @@ class Event(NamedTuple):
     return_address: str
 
     @classmethod
-    def create(cls, event: dict, content_service: ContentService, logger: Logger):
-        delta_file_messages = [DeltaFileMessage.from_dict(delta_file_message, content_service) for delta_file_message in
-                               event['deltaFileMessages']]
-        context = Context.create(event['actionContext'], content_service, logger)
+    def create(cls, event: dict, hostname: str, content_service: ContentService, logger: Logger):
+        delta_file_messages = [DeltaFileMessage.from_dict(delta_file_message, content_service) for delta_file_message in event['deltaFileMessages']]
+        context = Context.create(event['actionContext'], hostname, content_service, logger)
         params = event['actionParams']
         queue_name = None
         if 'queueName' in event:

deltafi/exception.py

@@ -23,6 +23,16 @@ class ExpectedContentException(RuntimeError):
         self.size = size
 
 
+class MissingDomainException(RuntimeError):
+    def __init__(self, name):
+        self.name = name
+
+
+class MissingEnrichmentException(RuntimeError):
+    def __init__(self, name):
+        self.name = name
+
+
 class MissingMetadataException(RuntimeError):
     def __init__(self, key):
         self.key = key

deltafi/input.py

@@ -17,13 +17,172 @@
 #
 
 from deltafi.domain import *
-from deltafi.exception import MissingMetadataException, ExpectedContentException
+from deltafi.exception import MissingMetadataException, ExpectedContentException, MissingDomainException, \
+    MissingEnrichmentException
+
+
+class DomainInput(NamedTuple):
+    content: List[Content]
+    metadata: Dict[str, str]
+    domains: Dict[str, Domain]
+
+    def has_content(self) -> bool:
+        return len(self.content) > 0
+
+    def content_at(self, index: int) -> Content:
+        if len(self.content) < index + 1:
+            raise ExpectedContentException(index, len(self.content))
+        return self.content[index]
+
+    def first_content(self):
+        return self.content_at(0)
+
+    def get_metadata(self, key: str):
+        if key in self.metadata:
+            return self.metadata[key]
+        else:
+            raise MissingMetadataException(key)
+
+    def get_metadata_or_else(self, key: str, default: str) -> str:
+        if key in self.metadata:
+            return self.metadata[key]
+        else:
+            return default
+
+    def has_domain(self, name: str) -> bool:
+        return name in self.domains
+
+    def domain(self, name: str) -> Domain:
+        if not self.has_domain(name):
+            raise MissingDomainException(name)
+        return self.domains[name]
+
 
 class EgressInput(NamedTuple):
     content: Content
     metadata: dict
 
 
+class EnrichInput(NamedTuple):
+    content: List[Content]
+    metadata: dict
+    domains: Dict[str, Domain]
+    enrichments: Dict[str, Domain]
+
+    def has_content(self) -> bool:
+        return len(self.content) > 0
+
+    def content_at(self, index: int) -> Content:
+        if len(self.content) < index + 1:
+            raise ExpectedContentException(index, len(self.content))
+        return self.content[index]
+
+    def first_content(self):
+        return self.content_at(0)
+
+    def get_metadata(self, key: str):
+        if key in self.metadata:
+            return self.metadata[key]
+        else:
+            raise MissingMetadataException(key)
+
+    def get_metadata_or_else(self, key: str, default: str) -> str:
+        if key in self.metadata:
+            return self.metadata[key]
+        else:
+            return default
+
+    def has_domain(self, name: str) -> bool:
+        return name in self.domains
+
+    def domain(self, name: str) -> Domain:
+        if not self.has_domain(name):
+            raise MissingDomainException(name)
+        return self.domains[name]
+
+    def has_enrichment(self, name: str) -> bool:
+        return name in self.enrichments
+
+    def enrichment(self, name: str) -> Domain:
+        if not self.has_enrichment(name):
+            raise MissingEnrichmentException(name)
+        return self.enrichments[name]
+
+
+class FormatInput(NamedTuple):
+    content: List[Content]
+    metadata: dict
+    domains: Dict[str, Domain]
+    enrichments: Dict[str, Domain]
+
+    def has_content(self) -> bool:
+        return len(self.content) > 0
+
+    def content_at(self, index: int) -> Content:
+        if len(self.content) < index + 1:
+            raise ExpectedContentException(index, len(self.content))
+        return self.content[index]
+
+    def first_content(self):
+        return self.content_at(0)
+
+    def get_metadata(self, key: str):
+        if key in self.metadata:
+            return self.metadata[key]
+        else:
+            raise MissingMetadataException(key)
+
+    def get_metadata_or_else(self, key: str, default: str) -> str:
+        if key in self.metadata:
+            return self.metadata[key]
+        else:
+            return default
+
+    def has_domain(self, name: str) -> bool:
+        return name in self.domains
+
+    def domain(self, name: str) -> Domain:
+        if not self.has_domain(name):
+            raise MissingDomainException(name)
+        return self.domains[name]
+
+    def has_enrichment(self, name: str) -> bool:
+        return name in self.enrichments
+
+    def enrichment(self, name: str) -> Domain:
+        if not self.has_enrichment(name):
+            raise MissingEnrichmentException(name)
+        return self.enrichments[name]
+
+
+class LoadInput(NamedTuple):
+    content: List[Content]
+    metadata: dict
+
+    def has_content(self) -> bool:
+        return len(self.content) > 0
+
+    def content_at(self, index: int) -> Content:
+        if len(self.content) < index + 1:
+            raise ExpectedContentException(index, len(self.content))
+        return self.content[index]
+
+    def first_content(self):
+        return self.content_at(0)
+
+    def get_metadata(self, key: str):
+        if key in self.metadata:
+            return self.metadata[key]
+        else:
+            raise MissingMetadataException(key)
+
+    def get_metadata_or_else(self, key: str, default: str) -> str:
+        if key in self.metadata:
+            return self.metadata[key]
+        else:
+            return default
+
+
 class TransformInput(NamedTuple):
     content: List[Content]
     metadata: dict
@@ -50,3 +209,8 @@ class TransformInput(NamedTuple):
             return self.metadata[key]
         else:
             return default
+
+
+class ValidateInput(NamedTuple):
+    content: Content
+    metadata: dict

deltafi/plugin.py

@@ -34,7 +34,8 @@ from importlib import metadata
 import requests
 from deltafi.actioneventqueue import ActionEventQueue
 from deltafi.domain import Event, ActionExecution
-from deltafi.exception import ExpectedContentException, MissingMetadataException
+from deltafi.exception import ExpectedContentException, MissingDomainException, MissingEnrichmentException, \
+    MissingMetadataException
 from deltafi.logger import get_logger
 from deltafi.result import ErrorResult
 from deltafi.storage import ContentService
@@ -46,9 +47,9 @@ def _coordinates():
 
 
 def _setup_queue(max_connections):
-    url = os.getenv('VALKEY_URL', 'http://deltafi-valkey-master:6379')
-    password = os.getenv('VALKEY_PASSWORD')
-    return ActionEventQueue(url, max_connections, password)
+    redis_url = os.getenv('REDIS_URL', 'http://deltafi-redis-master:6379')
+    password = os.getenv('REDIS_PASSWORD')
+    return ActionEventQueue(redis_url, max_connections, password)
 
 
 def _setup_content_service():
@@ -164,6 +165,8 @@ class Plugin(object):
             'name': self.action_name(action),
             'description': action.description,
             'type': action.action_type.name,
+            'requiresDomains': action.requires_domains,
+            'requiresEnrichments': action.requires_enrichments,
             'schema': action.param_class().model_json_schema()
         }
 
@@ -260,7 +263,7 @@ class Plugin(object):
         while True:
             try:
                 event_string = self.queue.take(self.action_name(action))
-                event = Event.create(json.loads(event_string), self.content_service, action_logger)
+                event = Event.create(json.loads(event_string), self.hostname, self.content_service, action_logger)
                 start_time = time.time()
                 action_logger.debug(f"Processing event for did {event.context.did}")
 
@@ -274,6 +277,14 @@ class Plugin(object):
                                          f"Action attempted to look up element {e.index + 1} (index {e.index}) from "
                                          f"content list of size {e.size}",
                                          f"{str(e)}\n{traceback.format_exc()}")
+                except MissingDomainException as e:
+                    result = ErrorResult(event.context,
+                                         f"Action attempted to access domain {e.name}, which does not exist",
+                                         f"{str(e)}\n{traceback.format_exc()}")
+                except MissingEnrichmentException as e:
+                    result = ErrorResult(event.context,
+                                         f"Action attempted to access enrichment {e.name}, which does not exist",
+                                         f"{str(e)}\n{traceback.format_exc()}")
                 except MissingMetadataException as e:
                     result = ErrorResult(event.context,
                                          f"Missing metadata with key {e.key}",
@@ -286,10 +297,7 @@ class Plugin(object):
 
                 response = {
                     'did': event.context.did,
-                    'flowName': event.context.flow_name,
-                    'flowId': event.context.flow_id,
-                    'actionName': event.context.action_name,
-                    'actionId': event.context.action_id,
+                    'action': event.context.action_flow + "." + event.context.action_name,
                     'start': start_time,
                     'stop': time.time(),
                     'type': result.result_type,