deltafi 2.0rc1720728217472__py3-none-any.whl → 2.0rc1720817063181__py3-none-any.whl

deltafi/action.py

@@ -20,20 +20,17 @@ from abc import ABC, abstractmethod
 from typing import Any, List
 
 from deltafi.actiontype import ActionType
-from deltafi.genericmodel import GenericModel
 from deltafi.domain import Context, DeltaFileMessage
-from deltafi.input import DomainInput, EgressInput, EnrichInput, FormatInput, LoadInput, TransformInput, ValidateInput
+from deltafi.genericmodel import GenericModel
+from deltafi.input import EgressInput, TransformInput
 from deltafi.result import *
 from pydantic import BaseModel
 
 
 class Action(ABC):
-    def __init__(self, action_type: ActionType, description: str, requires_domains: List[str],
-                 requires_enrichments: List[str], valid_result_types: tuple):
+    def __init__(self, action_type: ActionType, description: str, valid_result_types: tuple):
         self.action_type = action_type
         self.description = description
-        self.requires_domains = requires_domains
-        self.requires_enrichments = requires_enrichments
         self.valid_result_types = valid_result_types
         self.action_execution = None
 
@@ -41,8 +38,8 @@ class Action(ABC):
     def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
         pass
 
-    def collect(self, action_inputs: List[Any]):
-        raise RuntimeError(f"Collect is not supported for {self.__class__.__name__}")
+    def join(self, action_inputs: List[Any]):
+        raise RuntimeError(f"Join is not supported for {self.__class__.__name__}")
 
     @abstractmethod
     def execute(self, context: Context, action_input: Any, params: BaseModel):
@@ -51,20 +48,23 @@ class Action(ABC):
     def execute_action(self, event):
         if event.delta_file_messages is None or not len(event.delta_file_messages):
             raise RuntimeError(f"Received event with no delta file messages for did {event.context.did}")
-
-        if event.context.collect is not None:
-            result = self.execute(event.context, self.collect([self.build_input(event.context, delta_file_message)
-                                                               for delta_file_message in event.delta_file_messages]),
-                                  self.param_class().model_validate(event.params))
+        if event.context.join is not None:
+            result = self.execute(
+                event.context,
+                self.join([self.build_input(event.context, delta_file_message)
+                              for delta_file_message in event.delta_file_messages]),
+                self.param_class().model_validate(event.params))
         else:
-            result = self.execute(event.context, self.build_input(event.context, event.delta_file_messages[0]),
-                                  self.param_class().model_validate(event.params))
+            result = self.execute(
+                event.context,
+                self.build_input(event.context, event.delta_file_messages[0]),
+                self.param_class().model_validate(event.params))
 
         self.validate_type(result)
         return result
 
     @staticmethod
-    def param_class( ):
+    def param_class():
         """Factory method to create and return an empty GenericModel instance.
 
         Returns
@@ -81,25 +81,9 @@ class Action(ABC):
                              f"but a {result.__class__.__name__} was returned")
 
 
-class DomainAction(Action, ABC):
-    def __init__(self, description: str, requires_domains: List[str]):
-        super().__init__(ActionType.DOMAIN, description, requires_domains, [], (DomainResult, ErrorResult))
-
-    def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
-        return DomainInput(content=delta_file_message.content_list, metadata=delta_file_message.metadata,
-                           domains={domain.name: domain for domain in delta_file_message.domains})
-
-    @abstractmethod
-    def domain(self, context: Context, params: BaseModel, domain_input: DomainInput):
-        pass
-
-    def execute(self, context: Context, domain_input: DomainInput, params: BaseModel):
-        return self.domain(context, params, domain_input)
-
-
 class EgressAction(Action, ABC):
     def __init__(self, description: str):
-        super().__init__(ActionType.EGRESS, description, [], [], (EgressResult, ErrorResult, FilterResult))
+        super().__init__(ActionType.EGRESS, description, (EgressResult, ErrorResult, FilterResult))
 
     def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
         return EgressInput(content=delta_file_message.content_list[0], metadata=delta_file_message.metadata)
@@ -112,81 +96,9 @@ class EgressAction(Action, ABC):
         return self.egress(context, params, egress_input)
 
 
-class EnrichAction(Action, ABC):
-    def __init__(self, description: str, requires_domains: List[str], requires_enrichments: List[str]):
-        super().__init__(ActionType.ENRICH, description, requires_domains, requires_enrichments,
-                         (EnrichResult, ErrorResult))
-
-    def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
-        return EnrichInput(content=delta_file_message.content_list, metadata=delta_file_message.metadata,
-                           domains={domain.name: domain for domain in delta_file_message.domains},
-                           enrichments={domain.name: domain for domain in delta_file_message.enrichments})
-
-    @abstractmethod
-    def enrich(self, context: Context, params: BaseModel, enrich_input: EnrichInput):
-        pass
-
-    def execute(self, context: Context, enrich_input: EnrichInput, params: BaseModel):
-        return self.enrich(context, params, enrich_input)
-
-
-class FormatAction(Action, ABC):
-    def __init__(self, description: str, requires_domains: List[str], requires_enrichments: List[str]):
-        super().__init__(ActionType.FORMAT, description, requires_domains, requires_enrichments,
-                         (FormatResult, FormatManyResult, ErrorResult, FilterResult))
-
-    def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
-        return FormatInput(content=delta_file_message.content_list, metadata=delta_file_message.metadata,
-                           domains={domain.name: domain for domain in delta_file_message.domains},
-                           enrichments={domain.name: domain for domain in delta_file_message.enrichments})
-
-    def collect(self, format_inputs: List[FormatInput]):
-        all_content = []
-        all_metadata = {}
-        all_domains = {}
-        all_enrichments = {}
-        for format_input in format_inputs:
-            all_content += format_input.content
-            all_metadata.update(format_input.metadata)
-            all_domains.update(format_input.domains)
-            all_enrichments.update(format_input.enrichments)
-        return FormatInput(content=all_content, metadata=all_metadata, domains=all_domains, enrichments=all_enrichments)
-
-    @abstractmethod
-    def format(self, context: Context, params: BaseModel, format_input: FormatInput):
-        pass
-
-    def execute(self, context: Context, format_input: FormatInput, params: BaseModel):
-        return self.format(context, params, format_input)
-
-
-class LoadAction(Action, ABC):
-    def __init__(self, description: str):
-        super().__init__(ActionType.LOAD, description, [], [],
-                         (LoadResult, LoadManyResult, ErrorResult, FilterResult, ReinjectResult))
-
-    def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
-        return LoadInput(content=delta_file_message.content_list, metadata=delta_file_message.metadata)
-
-    def collect(self, load_inputs: List[LoadInput]):
-        all_content = []
-        all_metadata = {}
-        for load_input in load_inputs:
-            all_content += load_input.content
-            all_metadata.update(load_input.metadata)
-        return LoadInput(content=all_content, metadata=all_metadata)
-
-    @abstractmethod
-    def load(self, context: Context, params: BaseModel, load_input: LoadInput):
-        pass
-
-    def execute(self, context: Context, load_input: LoadInput, params: BaseModel):
-        return self.load(context, params, load_input)
-
-
 class TimedIngressAction(Action, ABC):
     def __init__(self, description: str):
-        super().__init__(ActionType.TIMED_INGRESS, description, [], [], (IngressResult, ErrorResult))
+        super().__init__(ActionType.TIMED_INGRESS, description, (IngressResult, ErrorResult))
 
     def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
         return None
@@ -201,12 +113,13 @@ class TimedIngressAction(Action, ABC):
 
 class TransformAction(Action, ABC):
     def __init__(self, description: str):
-        super().__init__(ActionType.TRANSFORM, description, [], [], (TransformResult, ErrorResult, FilterResult, ReinjectResult))
+        super().__init__(ActionType.TRANSFORM, description,
+                         (TransformResult, TransformResults, ErrorResult, FilterResult))
 
     def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
         return TransformInput(content=delta_file_message.content_list, metadata=delta_file_message.metadata)
 
-    def collect(self, transform_inputs: List[TransformInput]):
+    def join(self, transform_inputs: List[TransformInput]):
         all_content = []
         all_metadata = {}
         for transform_input in transform_inputs:
@@ -220,18 +133,3 @@ class TransformAction(Action, ABC):
 
     def execute(self, context: Context, transform_input: TransformInput, params: BaseModel):
         return self.transform(context, params, transform_input)
-
-
-class ValidateAction(Action, ABC):
-    def __init__(self, description: str):
-        super().__init__(ActionType.VALIDATE, description, [], [], (ValidateResult, ErrorResult, FilterResult))
-
-    def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
-        return ValidateInput(content=delta_file_message.content_list[0], metadata=delta_file_message.metadata)
-
-    @abstractmethod
-    def validate(self, context: Context, params: BaseModel, validate_input: ValidateInput):
-        pass
-
-    def execute(self, context: Context, validate_input: ValidateInput, params: BaseModel):
-        return self.validate(context, params, validate_input)

deltafi/actiontype.py

@@ -20,13 +20,9 @@ from enum import Enum
 
 
 class ActionType(Enum):
+    INGRESS = "INGRESS"
     TIMED_INGRESS = "TIMED_INGRESS"
     TRANSFORM = "TRANSFORM"
-    LOAD = "LOAD"
-    DOMAIN = "DOMAIN"
-    ENRICH = "ENRICH"
-    FORMAT = "FORMAT"
-    VALIDATE = "VALIDATE"
     EGRESS = "EGRESS"
-    DELETE = "DELETE"
+    PUBLISH = "PUBLISH"
     UNKNOWN = "UNKNOWN"

deltafi/domain.py

@@ -40,59 +40,87 @@ class ActionExecution(NamedTuple):
 
 class Context(NamedTuple):
     did: str
-    action_flow: str
+    delta_file_name: str
+    data_source: str
+    flow_name: str
+    flow_id: str
     action_name: str
-    source_filename: str
-    ingress_flow: str
-    egress_flow: str
-    system: str
+    action_id: str
+    action_version: str
     hostname: str
+    system_name: str
     content_service: ContentService
-    collect: dict = None
-    collected_dids: List[str] = None
+    join: dict = None
+    joined_dids: List[str] = None
     memo: str = None
     logger: Logger = None
 
     @classmethod
-    def create(cls, context: dict, hostname: str, content_service: ContentService, logger: Logger):
+    def create(cls, context: dict, content_service: ContentService, logger: Logger):
         did = context['did']
-        action_name_parts = context['name'].split(".")
-        action_flow = action_name_parts[0]
-        action_name = action_name_parts[1]
-        if 'sourceFilename' in context:
-            source_filename = context['sourceFilename']
+        if 'deltaFileName' in context:
+            delta_file_name = context['deltaFileName']
         else:
-            source_filename = None
-        ingress_flow = context['ingressFlow']
-        if 'egressFlow' in context:
-            egress_flow = context['egressFlow']
+            delta_file_name = None
+        if 'dataSource' in context:
+            data_source = context['dataSource']
         else:
-            egress_flow = None
-        system = context['systemName']
-        if 'collect' in context:
-            collect = context['collect']
+            data_source = None
+        if 'flowName' in context:
+            flow_name = context['flowName']
         else:
-            collect = None
-        if 'collectedDids' in context:
-            collected_dids = context['collectedDids']
+            flow_name = None
+        if 'flowId' in context:
+            flow_id = context['flowId']
         else:
-            collected_dids = None
+            flow_id = None
+        if 'actionName' in context:
+            action_name = context['actionName']
+        else:
+            action_name = None
+        if 'actionId' in context:
+            action_id = context['actionId']
+        else:
+            action_id = None
+        if 'actionVersion' in context:
+            action_version = context['actionVersion']
+        else:
+            action_version = None
+        if 'hostname' in context:
+            hostname = context['hostname']
+        else:
+            hostname = None
+        if 'systemName' in context:
+            system_name = context['systemName']
+        else:
+            system_name = None
+        if 'join' in context:
+            join = context['join']
+        else:
+            join = None
+        if 'joinedDids' in context:
+            joined_dids = context['joinedDids']
+        else:
+            joined_dids = None
         if 'memo' in context:
             memo = context['memo']
         else:
             memo = None
+
         return Context(did=did,
-                       action_flow=action_flow,
+                       delta_file_name=delta_file_name,
+                       data_source=data_source,
+                       flow_name=flow_name,
+                       flow_id=flow_id,
                        action_name=action_name,
-                       source_filename=source_filename,
-                       ingress_flow=ingress_flow,
-                       egress_flow=egress_flow,
-                       system=system,
+                       action_id=action_id,
+                       action_version=action_version,
                        hostname=hostname,
-                       content_service=content_service,
-                       collect=collect,
-                       collected_dids=collected_dids,
+                       system_name=system_name,
+                       join=join,
+                       joined_dids=joined_dids,
                        memo=memo,
+                       content_service=content_service,
                        logger=logger)
 
 
@@ -197,7 +225,6 @@ class Content:
 
         return new_segments
 
-
     def get_size(self):
         """
         Returns the size of the content in bytes.
@@ -296,41 +323,17 @@ class Content:
                        content_service=content_service)
 
 
-class Domain(NamedTuple):
-    name: str
-    value: str
-    media_type: str
-
-    @classmethod
-    def from_dict(cls, domain: dict):
-        name = domain['name']
-        if 'value' in domain:
-            value = domain['value']
-        else:
-            value = None
-        media_type = domain['mediaType']
-        return Domain(name=name,
-                      value=value,
-                      media_type=media_type)
-
-
 class DeltaFileMessage(NamedTuple):
     metadata: Dict[str, str]
     content_list: List[Content]
-    domains: List[Domain]
-    enrichments: List[Domain]
 
     @classmethod
     def from_dict(cls, delta_file_message: dict, content_service: ContentService):
         metadata = delta_file_message['metadata']
         content_list = [Content.from_dict(content, content_service) for content in delta_file_message['contentList']]
-        domains = [Domain.from_dict(domain) for domain in delta_file_message['domains']] if 'domains' in delta_file_message else []
-        enrichments = [Domain.from_dict(domain) for domain in delta_file_message['enrichments']] if 'enrichments' in delta_file_message else []
 
         return DeltaFileMessage(metadata=metadata,
-                                content_list=content_list,
-                                domains=domains,
-                                enrichments=enrichments)
+                                content_list=content_list)
 
 
 class Event(NamedTuple):
@@ -341,9 +344,10 @@ class Event(NamedTuple):
     return_address: str
 
     @classmethod
-    def create(cls, event: dict, hostname: str, content_service: ContentService, logger: Logger):
-        delta_file_messages = [DeltaFileMessage.from_dict(delta_file_message, content_service) for delta_file_message in event['deltaFileMessages']]
-        context = Context.create(event['actionContext'], hostname, content_service, logger)
+    def create(cls, event: dict, content_service: ContentService, logger: Logger):
+        delta_file_messages = [DeltaFileMessage.from_dict(delta_file_message, content_service) for delta_file_message in
+                               event['deltaFileMessages']]
+        context = Context.create(event['actionContext'], content_service, logger)
         params = event['actionParams']
         queue_name = None
         if 'queueName' in event:

deltafi/exception.py

@@ -23,16 +23,6 @@ class ExpectedContentException(RuntimeError):
         self.size = size
 
 
-class MissingDomainException(RuntimeError):
-    def __init__(self, name):
-        self.name = name
-
-
-class MissingEnrichmentException(RuntimeError):
-    def __init__(self, name):
-        self.name = name
-
-
 class MissingMetadataException(RuntimeError):
     def __init__(self, key):
         self.key = key

deltafi/input.py

@@ -17,172 +17,13 @@
 #
 
 from deltafi.domain import *
-from deltafi.exception import MissingMetadataException, ExpectedContentException, MissingDomainException, \
-    MissingEnrichmentException
-
-
-class DomainInput(NamedTuple):
-    content: List[Content]
-    metadata: Dict[str, str]
-    domains: Dict[str, Domain]
-
-    def has_content(self) -> bool:
-        return len(self.content) > 0
-
-    def content_at(self, index: int) -> Content:
-        if len(self.content) < index + 1:
-            raise ExpectedContentException(index, len(self.content))
-        return self.content[index]
-
-    def first_content(self):
-        return self.content_at(0)
-
-    def get_metadata(self, key: str):
-        if key in self.metadata:
-            return self.metadata[key]
-        else:
-            raise MissingMetadataException(key)
-
-    def get_metadata_or_else(self, key: str, default: str) -> str:
-        if key in self.metadata:
-            return self.metadata[key]
-        else:
-            return default
-
-    def has_domain(self, name: str) -> bool:
-        return name in self.domains
-
-    def domain(self, name: str) -> Domain:
-        if not self.has_domain(name):
-            raise MissingDomainException(name)
-        return self.domains[name]
-
+from deltafi.exception import MissingMetadataException, ExpectedContentException
 
 class EgressInput(NamedTuple):
     content: Content
     metadata: dict
 
 
-class EnrichInput(NamedTuple):
-    content: List[Content]
-    metadata: dict
-    domains: Dict[str, Domain]
-    enrichments: Dict[str, Domain]
-
-    def has_content(self) -> bool:
-        return len(self.content) > 0
-
-    def content_at(self, index: int) -> Content:
-        if len(self.content) < index + 1:
-            raise ExpectedContentException(index, len(self.content))
-        return self.content[index]
-
-    def first_content(self):
-        return self.content_at(0)
-
-    def get_metadata(self, key: str):
-        if key in self.metadata:
-            return self.metadata[key]
-        else:
-            raise MissingMetadataException(key)
-
-    def get_metadata_or_else(self, key: str, default: str) -> str:
-        if key in self.metadata:
-            return self.metadata[key]
-        else:
-            return default
-
-    def has_domain(self, name: str) -> bool:
-        return name in self.domains
-
-    def domain(self, name: str) -> Domain:
-        if not self.has_domain(name):
-            raise MissingDomainException(name)
-        return self.domains[name]
-
-    def has_enrichment(self, name: str) -> bool:
-        return name in self.enrichments
-
-    def enrichment(self, name: str) -> Domain:
-        if not self.has_enrichment(name):
-            raise MissingEnrichmentException(name)
-        return self.enrichments[name]
-
-
-class FormatInput(NamedTuple):
-    content: List[Content]
-    metadata: dict
-    domains: Dict[str, Domain]
-    enrichments: Dict[str, Domain]
-
-    def has_content(self) -> bool:
-        return len(self.content) > 0
-
-    def content_at(self, index: int) -> Content:
-        if len(self.content) < index + 1:
-            raise ExpectedContentException(index, len(self.content))
-        return self.content[index]
-
-    def first_content(self):
-        return self.content_at(0)
-
-    def get_metadata(self, key: str):
-        if key in self.metadata:
-            return self.metadata[key]
-        else:
-            raise MissingMetadataException(key)
-
-    def get_metadata_or_else(self, key: str, default: str) -> str:
-        if key in self.metadata:
-            return self.metadata[key]
-        else:
-            return default
-
-    def has_domain(self, name: str) -> bool:
-        return name in self.domains
-
-    def domain(self, name: str) -> Domain:
-        if not self.has_domain(name):
-            raise MissingDomainException(name)
-        return self.domains[name]
-
-    def has_enrichment(self, name: str) -> bool:
-        return name in self.enrichments
-
-    def enrichment(self, name: str) -> Domain:
-        if not self.has_enrichment(name):
-            raise MissingEnrichmentException(name)
-        return self.enrichments[name]
-
-
-class LoadInput(NamedTuple):
-    content: List[Content]
-    metadata: dict
-
-    def has_content(self) -> bool:
-        return len(self.content) > 0
-
-    def content_at(self, index: int) -> Content:
-        if len(self.content) < index + 1:
-            raise ExpectedContentException(index, len(self.content))
-        return self.content[index]
-
-    def first_content(self):
-        return self.content_at(0)
-
-    def get_metadata(self, key: str):
-        if key in self.metadata:
-            return self.metadata[key]
-        else:
-            raise MissingMetadataException(key)
-
-    def get_metadata_or_else(self, key: str, default: str) -> str:
-        if key in self.metadata:
-            return self.metadata[key]
-        else:
-            return default
-
-
 class TransformInput(NamedTuple):
     content: List[Content]
     metadata: dict
@@ -209,8 +50,3 @@ class TransformInput(NamedTuple):
             return self.metadata[key]
         else:
             return default
-
-
-class ValidateInput(NamedTuple):
-    content: Content
-    metadata: dict

deltafi/plugin.py

@@ -34,8 +34,7 @@ from importlib import metadata
 import requests
 from deltafi.actioneventqueue import ActionEventQueue
 from deltafi.domain import Event, ActionExecution
-from deltafi.exception import ExpectedContentException, MissingDomainException, MissingEnrichmentException, \
-    MissingMetadataException
+from deltafi.exception import ExpectedContentException, MissingMetadataException
 from deltafi.logger import get_logger
 from deltafi.result import ErrorResult
 from deltafi.storage import ContentService
@@ -47,9 +46,9 @@ def _coordinates():
 
 
 def _setup_queue(max_connections):
-    redis_url = os.getenv('REDIS_URL', 'http://deltafi-redis-master:6379')
-    password = os.getenv('REDIS_PASSWORD')
-    return ActionEventQueue(redis_url, max_connections, password)
+    url = os.getenv('VALKEY_URL', 'http://deltafi-valkey-master:6379')
+    password = os.getenv('VALKEY_PASSWORD')
+    return ActionEventQueue(url, max_connections, password)
 
 
 def _setup_content_service():
@@ -165,8 +164,6 @@ class Plugin(object):
             'name': self.action_name(action),
             'description': action.description,
             'type': action.action_type.name,
-            'requiresDomains': action.requires_domains,
-            'requiresEnrichments': action.requires_enrichments,
             'schema': action.param_class().model_json_schema()
         }
 
@@ -263,7 +260,7 @@ class Plugin(object):
         while True:
             try:
                 event_string = self.queue.take(self.action_name(action))
-                event = Event.create(json.loads(event_string), self.hostname, self.content_service, action_logger)
+                event = Event.create(json.loads(event_string), self.content_service, action_logger)
                 start_time = time.time()
                 action_logger.debug(f"Processing event for did {event.context.did}")
 
@@ -277,14 +274,6 @@ class Plugin(object):
                                          f"Action attempted to look up element {e.index + 1} (index {e.index}) from "
                                          f"content list of size {e.size}",
                                          f"{str(e)}\n{traceback.format_exc()}")
-                except MissingDomainException as e:
-                    result = ErrorResult(event.context,
-                                         f"Action attempted to access domain {e.name}, which does not exist",
-                                         f"{str(e)}\n{traceback.format_exc()}")
-                except MissingEnrichmentException as e:
-                    result = ErrorResult(event.context,
-                                         f"Action attempted to access enrichment {e.name}, which does not exist",
-                                         f"{str(e)}\n{traceback.format_exc()}")
                 except MissingMetadataException as e:
                     result = ErrorResult(event.context,
                                          f"Missing metadata with key {e.key}",
@@ -297,7 +286,10 @@ class Plugin(object):
 
                 response = {
                     'did': event.context.did,
-                    'action': event.context.action_flow + "." + event.context.action_name,
+                    'flowName': event.context.flow_name,
+                    'flowId': event.context.flow_id,
+                    'actionName': event.context.action_name,
+                    'actionId': event.context.action_id,
                     'start': start_time,
                     'stop': time.time(),
                     'type': result.result_type,