deltafi 2.0rc1720728217472__py3-none-any.whl → 2.0rc1722532930396__py3-none-any.whl

deltafi/action.py

@@ -20,20 +20,17 @@ from abc import ABC, abstractmethod
 from typing import Any, List
 
 from deltafi.actiontype import ActionType
-from deltafi.genericmodel import GenericModel
 from deltafi.domain import Context, DeltaFileMessage
-from deltafi.input import DomainInput, EgressInput, EnrichInput, FormatInput, LoadInput, TransformInput, ValidateInput
+from deltafi.genericmodel import GenericModel
+from deltafi.input import EgressInput, TransformInput
 from deltafi.result import *
 from pydantic import BaseModel
 
 
 class Action(ABC):
-    def __init__(self, action_type: ActionType, description: str, requires_domains: List[str],
-                 requires_enrichments: List[str], valid_result_types: tuple):
+    def __init__(self, action_type: ActionType, description: str, valid_result_types: tuple):
         self.action_type = action_type
         self.description = description
-        self.requires_domains = requires_domains
-        self.requires_enrichments = requires_enrichments
         self.valid_result_types = valid_result_types
         self.action_execution = None
 
@@ -41,8 +38,8 @@ class Action(ABC):
     def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
         pass
 
-    def collect(self, action_inputs: List[Any]):
-        raise RuntimeError(f"Collect is not supported for {self.__class__.__name__}")
+    def join(self, action_inputs: List[Any]):
+        raise RuntimeError(f"Join is not supported for {self.__class__.__name__}")
 
     @abstractmethod
     def execute(self, context: Context, action_input: Any, params: BaseModel):
@@ -51,22 +48,29 @@ class Action(ABC):
     def execute_action(self, event):
         if event.delta_file_messages is None or not len(event.delta_file_messages):
             raise RuntimeError(f"Received event with no delta file messages for did {event.context.did}")
-
-        if event.context.collect is not None:
-            result = self.execute(event.context, self.collect([self.build_input(event.context, delta_file_message)
-                                                               for delta_file_message in event.delta_file_messages]),
-                                  self.param_class().model_validate(event.params))
+        if event.context.join is not None:
+            result = self.execute(
+                event.context,
+                self.join([self.build_input(event.context, delta_file_message)
+                              for delta_file_message in event.delta_file_messages]),
+                self.param_class().model_validate(event.params))
         else:
-            result = self.execute(event.context, self.build_input(event.context, event.delta_file_messages[0]),
-                                  self.param_class().model_validate(event.params))
+            result = self.execute(
+                event.context,
+                self.build_input(event.context, event.delta_file_messages[0]),
+                self.param_class().model_validate(event.params))
 
         self.validate_type(result)
         return result
 
     @staticmethod
-    def param_class( ):
+    def param_class():
         """Factory method to create and return an empty GenericModel instance.
 
+        All action parameter classes must inherit pydantic.BaseModel.
+        Use of complex types in custom action parameter classes must specify
+        the internal types when defined. E.g., dict[str, str], or List[str]
+
         Returns
         -------
         GenericModel
@@ -81,25 +85,9 @@ class Action(ABC):
                              f"but a {result.__class__.__name__} was returned")
 
 
-class DomainAction(Action, ABC):
-    def __init__(self, description: str, requires_domains: List[str]):
-        super().__init__(ActionType.DOMAIN, description, requires_domains, [], (DomainResult, ErrorResult))
-
-    def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
-        return DomainInput(content=delta_file_message.content_list, metadata=delta_file_message.metadata,
-                           domains={domain.name: domain for domain in delta_file_message.domains})
-
-    @abstractmethod
-    def domain(self, context: Context, params: BaseModel, domain_input: DomainInput):
-        pass
-
-    def execute(self, context: Context, domain_input: DomainInput, params: BaseModel):
-        return self.domain(context, params, domain_input)
-
-
 class EgressAction(Action, ABC):
     def __init__(self, description: str):
-        super().__init__(ActionType.EGRESS, description, [], [], (EgressResult, ErrorResult, FilterResult))
+        super().__init__(ActionType.EGRESS, description, (EgressResult, ErrorResult, FilterResult))
 
     def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
         return EgressInput(content=delta_file_message.content_list[0], metadata=delta_file_message.metadata)
@@ -112,81 +100,9 @@ class EgressAction(Action, ABC):
         return self.egress(context, params, egress_input)
 
 
-class EnrichAction(Action, ABC):
-    def __init__(self, description: str, requires_domains: List[str], requires_enrichments: List[str]):
-        super().__init__(ActionType.ENRICH, description, requires_domains, requires_enrichments,
-                         (EnrichResult, ErrorResult))
-
-    def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
-        return EnrichInput(content=delta_file_message.content_list, metadata=delta_file_message.metadata,
-                           domains={domain.name: domain for domain in delta_file_message.domains},
-                           enrichments={domain.name: domain for domain in delta_file_message.enrichments})
-
-    @abstractmethod
-    def enrich(self, context: Context, params: BaseModel, enrich_input: EnrichInput):
-        pass
-
-    def execute(self, context: Context, enrich_input: EnrichInput, params: BaseModel):
-        return self.enrich(context, params, enrich_input)
-
-
-class FormatAction(Action, ABC):
-    def __init__(self, description: str, requires_domains: List[str], requires_enrichments: List[str]):
-        super().__init__(ActionType.FORMAT, description, requires_domains, requires_enrichments,
-                         (FormatResult, FormatManyResult, ErrorResult, FilterResult))
-
-    def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
-        return FormatInput(content=delta_file_message.content_list, metadata=delta_file_message.metadata,
-                           domains={domain.name: domain for domain in delta_file_message.domains},
-                           enrichments={domain.name: domain for domain in delta_file_message.enrichments})
-
-    def collect(self, format_inputs: List[FormatInput]):
-        all_content = []
-        all_metadata = {}
-        all_domains = {}
-        all_enrichments = {}
-        for format_input in format_inputs:
-            all_content += format_input.content
-            all_metadata.update(format_input.metadata)
-            all_domains.update(format_input.domains)
-            all_enrichments.update(format_input.enrichments)
-        return FormatInput(content=all_content, metadata=all_metadata, domains=all_domains, enrichments=all_enrichments)
-
-    @abstractmethod
-    def format(self, context: Context, params: BaseModel, format_input: FormatInput):
-        pass
-
-    def execute(self, context: Context, format_input: FormatInput, params: BaseModel):
-        return self.format(context, params, format_input)
-
-
-class LoadAction(Action, ABC):
-    def __init__(self, description: str):
-        super().__init__(ActionType.LOAD, description, [], [],
-                         (LoadResult, LoadManyResult, ErrorResult, FilterResult, ReinjectResult))
-
-    def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
-        return LoadInput(content=delta_file_message.content_list, metadata=delta_file_message.metadata)
-
-    def collect(self, load_inputs: List[LoadInput]):
-        all_content = []
-        all_metadata = {}
-        for load_input in load_inputs:
-            all_content += load_input.content
-            all_metadata.update(load_input.metadata)
-        return LoadInput(content=all_content, metadata=all_metadata)
-
-    @abstractmethod
-    def load(self, context: Context, params: BaseModel, load_input: LoadInput):
-        pass
-
-    def execute(self, context: Context, load_input: LoadInput, params: BaseModel):
-        return self.load(context, params, load_input)
-
-
 class TimedIngressAction(Action, ABC):
     def __init__(self, description: str):
-        super().__init__(ActionType.TIMED_INGRESS, description, [], [], (IngressResult, ErrorResult))
+        super().__init__(ActionType.TIMED_INGRESS, description, (IngressResult, ErrorResult))
 
     def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
         return None
@@ -201,12 +117,13 @@ class TimedIngressAction(Action, ABC):
 
 class TransformAction(Action, ABC):
     def __init__(self, description: str):
-        super().__init__(ActionType.TRANSFORM, description, [], [], (TransformResult, ErrorResult, FilterResult, ReinjectResult))
+        super().__init__(ActionType.TRANSFORM, description,
+                         (TransformResult, TransformResults, ErrorResult, FilterResult))
 
     def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
         return TransformInput(content=delta_file_message.content_list, metadata=delta_file_message.metadata)
 
-    def collect(self, transform_inputs: List[TransformInput]):
+    def join(self, transform_inputs: List[TransformInput]):
         all_content = []
         all_metadata = {}
         for transform_input in transform_inputs:
@@ -220,18 +137,3 @@ class TransformAction(Action, ABC):
 
     def execute(self, context: Context, transform_input: TransformInput, params: BaseModel):
         return self.transform(context, params, transform_input)
-
-
-class ValidateAction(Action, ABC):
-    def __init__(self, description: str):
-        super().__init__(ActionType.VALIDATE, description, [], [], (ValidateResult, ErrorResult, FilterResult))
-
-    def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
-        return ValidateInput(content=delta_file_message.content_list[0], metadata=delta_file_message.metadata)
-
-    @abstractmethod
-    def validate(self, context: Context, params: BaseModel, validate_input: ValidateInput):
-        pass
-
-    def execute(self, context: Context, validate_input: ValidateInput, params: BaseModel):
-        return self.validate(context, params, validate_input)

deltafi/actiontype.py

@@ -20,13 +20,9 @@ from enum import Enum
 
 
 class ActionType(Enum):
+    INGRESS = "INGRESS"
     TIMED_INGRESS = "TIMED_INGRESS"
     TRANSFORM = "TRANSFORM"
-    LOAD = "LOAD"
-    DOMAIN = "DOMAIN"
-    ENRICH = "ENRICH"
-    FORMAT = "FORMAT"
-    VALIDATE = "VALIDATE"
     EGRESS = "EGRESS"
-    DELETE = "DELETE"
+    PUBLISH = "PUBLISH"
     UNKNOWN = "UNKNOWN"

deltafi/domain.py

@@ -40,59 +40,87 @@ class ActionExecution(NamedTuple):
 
 class Context(NamedTuple):
     did: str
-    action_flow: str
+    delta_file_name: str
+    data_source: str
+    flow_name: str
+    flow_id: str
     action_name: str
-    source_filename: str
-    ingress_flow: str
-    egress_flow: str
-    system: str
+    action_id: str
+    action_version: str
     hostname: str
+    system_name: str
     content_service: ContentService
-    collect: dict = None
-    collected_dids: List[str] = None
+    join: dict = None
+    joined_dids: List[str] = None
     memo: str = None
     logger: Logger = None
 
     @classmethod
-    def create(cls, context: dict, hostname: str, content_service: ContentService, logger: Logger):
+    def create(cls, context: dict, content_service: ContentService, logger: Logger):
         did = context['did']
-        action_name_parts = context['name'].split(".")
-        action_flow = action_name_parts[0]
-        action_name = action_name_parts[1]
-        if 'sourceFilename' in context:
-            source_filename = context['sourceFilename']
+        if 'deltaFileName' in context:
+            delta_file_name = context['deltaFileName']
         else:
-            source_filename = None
-        ingress_flow = context['ingressFlow']
-        if 'egressFlow' in context:
-            egress_flow = context['egressFlow']
+            delta_file_name = None
+        if 'dataSource' in context:
+            data_source = context['dataSource']
         else:
-            egress_flow = None
-        system = context['systemName']
-        if 'collect' in context:
-            collect = context['collect']
+            data_source = None
+        if 'flowName' in context:
+            flow_name = context['flowName']
         else:
-            collect = None
-        if 'collectedDids' in context:
-            collected_dids = context['collectedDids']
+            flow_name = None
+        if 'flowId' in context:
+            flow_id = context['flowId']
         else:
-            collected_dids = None
+            flow_id = None
+        if 'actionName' in context:
+            action_name = context['actionName']
+        else:
+            action_name = None
+        if 'actionId' in context:
+            action_id = context['actionId']
+        else:
+            action_id = None
+        if 'actionVersion' in context:
+            action_version = context['actionVersion']
+        else:
+            action_version = None
+        if 'hostname' in context:
+            hostname = context['hostname']
+        else:
+            hostname = None
+        if 'systemName' in context:
+            system_name = context['systemName']
+        else:
+            system_name = None
+        if 'join' in context:
+            join = context['join']
+        else:
+            join = None
+        if 'joinedDids' in context:
+            joined_dids = context['joinedDids']
+        else:
+            joined_dids = None
         if 'memo' in context:
             memo = context['memo']
         else:
             memo = None
+
         return Context(did=did,
-                       action_flow=action_flow,
+                       delta_file_name=delta_file_name,
+                       data_source=data_source,
+                       flow_name=flow_name,
+                       flow_id=flow_id,
                        action_name=action_name,
-                       source_filename=source_filename,
-                       ingress_flow=ingress_flow,
-                       egress_flow=egress_flow,
-                       system=system,
+                       action_id=action_id,
+                       action_version=action_version,
                        hostname=hostname,
-                       content_service=content_service,
-                       collect=collect,
-                       collected_dids=collected_dids,
+                       system_name=system_name,
+                       join=join,
+                       joined_dids=joined_dids,
                        memo=memo,
+                       content_service=content_service,
                        logger=logger)
 
 
@@ -197,7 +225,6 @@ class Content:
 
         return new_segments
 
-
     def get_size(self):
         """
         Returns the size of the content in bytes.
@@ -296,41 +323,17 @@ class Content:
                        content_service=content_service)
 
 
-class Domain(NamedTuple):
-    name: str
-    value: str
-    media_type: str
-
-    @classmethod
-    def from_dict(cls, domain: dict):
-        name = domain['name']
-        if 'value' in domain:
-            value = domain['value']
-        else:
-            value = None
-        media_type = domain['mediaType']
-        return Domain(name=name,
-                      value=value,
-                      media_type=media_type)
-
-
 class DeltaFileMessage(NamedTuple):
     metadata: Dict[str, str]
     content_list: List[Content]
-    domains: List[Domain]
-    enrichments: List[Domain]
 
     @classmethod
     def from_dict(cls, delta_file_message: dict, content_service: ContentService):
         metadata = delta_file_message['metadata']
         content_list = [Content.from_dict(content, content_service) for content in delta_file_message['contentList']]
-        domains = [Domain.from_dict(domain) for domain in delta_file_message['domains']] if 'domains' in delta_file_message else []
-        enrichments = [Domain.from_dict(domain) for domain in delta_file_message['enrichments']] if 'enrichments' in delta_file_message else []
 
         return DeltaFileMessage(metadata=metadata,
-                                content_list=content_list,
-                                domains=domains,
-                                enrichments=enrichments)
+                                content_list=content_list)
 
 
 class Event(NamedTuple):
@@ -341,9 +344,10 @@ class Event(NamedTuple):
     return_address: str
 
     @classmethod
-    def create(cls, event: dict, hostname: str, content_service: ContentService, logger: Logger):
-        delta_file_messages = [DeltaFileMessage.from_dict(delta_file_message, content_service) for delta_file_message in event['deltaFileMessages']]
-        context = Context.create(event['actionContext'], hostname, content_service, logger)
+    def create(cls, event: dict, content_service: ContentService, logger: Logger):
+        delta_file_messages = [DeltaFileMessage.from_dict(delta_file_message, content_service) for delta_file_message in
+                               event['deltaFileMessages']]
+        context = Context.create(event['actionContext'], content_service, logger)
         params = event['actionParams']
         queue_name = None
         if 'queueName' in event:

deltafi/exception.py

@@ -23,16 +23,6 @@ class ExpectedContentException(RuntimeError):
         self.size = size
 
 
-class MissingDomainException(RuntimeError):
-    def __init__(self, name):
-        self.name = name
-
-
-class MissingEnrichmentException(RuntimeError):
-    def __init__(self, name):
-        self.name = name
-
-
 class MissingMetadataException(RuntimeError):
     def __init__(self, key):
         self.key = key

deltafi/genericmodel.py

@@ -22,7 +22,9 @@
 
 Provides an empty subclass of pydantic.BaseModel.
 
-Starting Pydantic v2, the BaseModel cannot be directly instantiated.  This class provides for instantiation of GenericModel objects that inherit from BaseModel.
+All action parameter classes must inherit pydantic.BaseModel.
+
+Starting Pydantic v2, the BaseModel cannot be directly instantiated. This class provides for instantiation of GenericModel objects that inherit from BaseModel.
 
 This class does not define fields for validation or any other purpose.
 """

deltafi/input.py

@@ -17,172 +17,13 @@
 #
 
 from deltafi.domain import *
-from deltafi.exception import MissingMetadataException, ExpectedContentException, MissingDomainException, \
-    MissingEnrichmentException
-
-
-class DomainInput(NamedTuple):
-    content: List[Content]
-    metadata: Dict[str, str]
-    domains: Dict[str, Domain]
-
-    def has_content(self) -> bool:
-        return len(self.content) > 0
-
-    def content_at(self, index: int) -> Content:
-        if len(self.content) < index + 1:
-            raise ExpectedContentException(index, len(self.content))
-        return self.content[index]
-
-    def first_content(self):
-        return self.content_at(0)
-
-    def get_metadata(self, key: str):
-        if key in self.metadata:
-            return self.metadata[key]
-        else:
-            raise MissingMetadataException(key)
-
-    def get_metadata_or_else(self, key: str, default: str) -> str:
-        if key in self.metadata:
-            return self.metadata[key]
-        else:
-            return default
-
-    def has_domain(self, name: str) -> bool:
-        return name in self.domains
-
-    def domain(self, name: str) -> Domain:
-        if not self.has_domain(name):
-            raise MissingDomainException(name)
-        return self.domains[name]
-
+from deltafi.exception import MissingMetadataException, ExpectedContentException
 
 class EgressInput(NamedTuple):
     content: Content
     metadata: dict
 
 
-class EnrichInput(NamedTuple):
-    content: List[Content]
-    metadata: dict
-    domains: Dict[str, Domain]
-    enrichments: Dict[str, Domain]
-
-    def has_content(self) -> bool:
-        return len(self.content) > 0
-
-    def content_at(self, index: int) -> Content:
-        if len(self.content) < index + 1:
-            raise ExpectedContentException(index, len(self.content))
-        return self.content[index]
-
-    def first_content(self):
-        return self.content_at(0)
-
-    def get_metadata(self, key: str):
-        if key in self.metadata:
-            return self.metadata[key]
-        else:
-            raise MissingMetadataException(key)
-
-    def get_metadata_or_else(self, key: str, default: str) -> str:
-        if key in self.metadata:
-            return self.metadata[key]
-        else:
-            return default
-
-    def has_domain(self, name: str) -> bool:
-        return name in self.domains
-
-    def domain(self, name: str) -> Domain:
-        if not self.has_domain(name):
-            raise MissingDomainException(name)
-        return self.domains[name]
-
-    def has_enrichment(self, name: str) -> bool:
-        return name in self.enrichments
-
-    def enrichment(self, name: str) -> Domain:
-        if not self.has_enrichment(name):
-            raise MissingEnrichmentException(name)
-        return self.enrichments[name]
-
-
-class FormatInput(NamedTuple):
-    content: List[Content]
-    metadata: dict
-    domains: Dict[str, Domain]
-    enrichments: Dict[str, Domain]
-
-    def has_content(self) -> bool:
-        return len(self.content) > 0
-
-    def content_at(self, index: int) -> Content:
-        if len(self.content) < index + 1:
-            raise ExpectedContentException(index, len(self.content))
-        return self.content[index]
-
-    def first_content(self):
-        return self.content_at(0)
-
-    def get_metadata(self, key: str):
-        if key in self.metadata:
-            return self.metadata[key]
-        else:
-            raise MissingMetadataException(key)
-
-    def get_metadata_or_else(self, key: str, default: str) -> str:
-        if key in self.metadata:
-            return self.metadata[key]
-        else:
-            return default
-
-    def has_domain(self, name: str) -> bool:
-        return name in self.domains
-
-    def domain(self, name: str) -> Domain:
-        if not self.has_domain(name):
-            raise MissingDomainException(name)
-        return self.domains[name]
-
-    def has_enrichment(self, name: str) -> bool:
-        return name in self.enrichments
-
-    def enrichment(self, name: str) -> Domain:
-        if not self.has_enrichment(name):
-            raise MissingEnrichmentException(name)
-        return self.enrichments[name]
-
-
-class LoadInput(NamedTuple):
-    content: List[Content]
-    metadata: dict
-
-    def has_content(self) -> bool:
-        return len(self.content) > 0
-
-    def content_at(self, index: int) -> Content:
-        if len(self.content) < index + 1:
-            raise ExpectedContentException(index, len(self.content))
-        return self.content[index]
-
-    def first_content(self):
-        return self.content_at(0)
-
-    def get_metadata(self, key: str):
-        if key in self.metadata:
-            return self.metadata[key]
-        else:
-            raise MissingMetadataException(key)
-
-    def get_metadata_or_else(self, key: str, default: str) -> str:
-        if key in self.metadata:
-            return self.metadata[key]
-        else:
-            return default
-
-
 class TransformInput(NamedTuple):
     content: List[Content]
     metadata: dict
@@ -209,8 +50,3 @@ class TransformInput(NamedTuple):
             return self.metadata[key]
         else:
             return default
-
-
-class ValidateInput(NamedTuple):
-    content: Content
-    metadata: dict

deltafi/logger.py

@@ -18,7 +18,7 @@
 
 import logging
 import sys
-from datetime import datetime
+from datetime import datetime, UTC
 
 import json_logging
 
@@ -42,7 +42,7 @@ def _sanitize_log_msg(record):
 class JSONLogFormatter(json_logging.JSONLogFormatter):
 
     def _format_log_object(self, record, request_util):
-        utcnow = datetime.utcnow()
+        utcnow = datetime.now(UTC)
 
         json_log_object = {
             'timestamp': json_logging.util.iso_time_format(utcnow),