deltafi 0.109.0__py3-none-any.whl → 2.40.0__py3-none-any.whl

deltafi/__init__.py

@@ -1,7 +1,7 @@
 #
 #    DeltaFi - Data transformation and enrichment platform
 #
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#    Copyright 2021-2025 DeltaFi Contributors <deltafi@deltafi.org>
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -15,3 +15,5 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
+
+#

deltafi/action.py

@@ -1,7 +1,7 @@
 #
 #    DeltaFi - Data transformation and enrichment platform
 #
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#    Copyright 2021-2025 DeltaFi Contributors <deltafi@deltafi.org>
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -17,148 +17,308 @@
 #
 
 from abc import ABC, abstractmethod
+from typing import Any, List
+
+from pydantic import BaseModel
 
 from deltafi.actiontype import ActionType
-from deltafi.domain import Context, DeltaFileMessage
-from deltafi.input import DomainInput, EgressInput, EnrichInput, FormatInput, LoadInput, TransformInput, ValidateInput
+from deltafi.domain import DeltaFileMessage
+from deltafi.genericmodel import GenericModel
+from deltafi.input import EgressInput, TransformInput
 from deltafi.result import *
-from pydantic import BaseModel
 
 
-class Action(ABC):
-    def __init__(self, action_type: ActionType, description: str, requires_domains: List[str],
-                 requires_enrichments: List[str]):
-        self.action_type = action_type
-        self.description = description
-        self.requires_domains = requires_domains
-        self.requires_enrichments = requires_enrichments
+class Join(ABC):
+    def join(self, transform_inputs: List[TransformInput]):
+        all_content = []
+        all_metadata = {}
+        for transform_input in transform_inputs:
+            all_content += transform_input.content
+            all_metadata.update(transform_input.metadata)
+        return TransformInput(content=all_content, metadata=all_metadata)
 
-    @abstractmethod
-    def execute(self, event):
-        pass
 
-    def param_class(self):
-        return BaseModel
+class ContentSpec:
+    name: str
+    media_type: str
+    description: str
 
-    def validate_type(self, result, types: tuple):
-        if not isinstance(result, types):
-            raise ValueError(f"{self.__class__.__name__} must return one of "
-                             f"{[result_type.__name__ for result_type in types]} "
-                             f"but a {result.__class__.__name__} was returned")
+    def __init__(self, name: str = None, media_type: str = None, description: str = None):
+        self.name = name
+        self.media_type = media_type
+        self.description = description
 
+    def json(self):
+        json_dictionary = {}
+        if self.name is not None:
+            json_dictionary['name'] = self.name
+        if self.media_type is not None:
+            json_dictionary['mediaType'] = self.media_type
+        if self.description is not None:
+            json_dictionary['description'] = self.description
+        return json_dictionary
 
-class DomainAction(Action):
-    def __init__(self, description: str, requires_domains: List[str]):
-        super().__init__(ActionType.DOMAIN, description, requires_domains, [])
 
-    def execute(self, event):
-        domain_input = DomainInput(content=event.delta_file_messages[0].content_list,
-                                   metadata=event.delta_file_messages[0].metadata,
-                                   domains={domain.name: domain for domain in event.delta_file_messages[0].domains})
-        result = self.domain(event.context, self.param_class().parse_obj(event.params), domain_input)
-        self.validate_type(result, (DomainResult, ErrorResult))
-        return result
+class KeyedDescription:
+    key: str
+    description: str
 
-    @abstractmethod
-    def domain(self, context: Context, params: BaseModel, domain_input: DomainInput):
-        pass
+    def __init__(self, key: str, description: str):
+        self.key = key
+        self.description = description
 
+    def json(self):
+        json_dictionary = {}
+        if self.key is not None:
+            json_dictionary['key'] = self.key
+            json_dictionary['description'] = self.description
+        return json_dictionary
+
+
+class InputSpec:
+    content_summary: str
+    content_specs: List[ContentSpec]
+    metadata_summary: str
+    metadata_descriptions: List[KeyedDescription]
+
+    def __init__(self, content_summary: str = None, content_specs: List[ContentSpec] = None,
+                 metadata_summary: str = None, metadata_descriptions: List[KeyedDescription] = None):
+        self.content_summary = content_summary
+        self.content_specs = content_specs
+        self.metadata_summary = metadata_summary
+        self.metadata_descriptions = metadata_descriptions
+
+    def json(self):
+        json_dictionary = {}
+        if self.content_summary is not None:
+            json_dictionary['contentSummary'] = self.content_summary
+        if self.content_specs is not None:
+            json_dictionary['contentSpecs'] = [cs.json() for cs in self.content_specs]
+        if self.metadata_summary is not None:
+            json_dictionary['metadataSummary'] = self.metadata_summary
+        if self.metadata_descriptions is not None:
+            json_dictionary['metadataDescriptions'] = [md.json() for md in self.metadata_descriptions]
+        return json_dictionary
+
+
+class OutputSpec:
+    content_summary: str
+    content_specs: List[ContentSpec]
+    metadata_summary: str
+    metadata_descriptions: List[KeyedDescription]
+    passthrough: bool
+    annotations_summary: str
+    annotation_descriptions: List[KeyedDescription]
+
+    def __init__(self, content_summary: str = None, content_specs: List[ContentSpec] = None,
+                 metadata_summary: str = None, metadata_descriptions: List[KeyedDescription] = None,
+                 passthrough: bool = False, annotations_summary: str = None,
+                 annotation_descriptions: List[KeyedDescription] = None):
+        self.content_summary = content_summary
+        self.content_specs = content_specs
+        self.metadata_summary = metadata_summary
+        self.metadata_descriptions = metadata_descriptions
+        self.passthrough = passthrough
+        self.annotations_summary = annotations_summary
+        self.annotation_descriptions = annotation_descriptions
+
+    def json(self):
+        json_dictionary = {}
+        if self.content_summary is not None:
+            json_dictionary['contentSummary'] = self.content_summary
+        if self.content_specs is not None:
+            json_dictionary['contentSpecs'] = [cs.json() for cs in self.content_specs]
+        if self.metadata_summary is not None:
+            json_dictionary['metadataSummary'] = self.metadata_summary
+        if self.metadata_descriptions is not None:
+            json_dictionary['metadataDescriptions'] = [md.json() for md in self.metadata_descriptions]
+        if self.passthrough is not None:
+            json_dictionary['passthrough'] = self.passthrough
+        if self.annotations_summary is not None:
+            json_dictionary['annotationsSummary'] = self.annotations_summary
+        if self.annotation_descriptions is not None:
+            json_dictionary['annotationDescriptions'] = [ad.json() for ad in self.annotation_descriptions]
+        return json_dictionary
+
+
+class DescriptionWithConditions:
+    description: str
+    conditions: List[str]
+
+    def __init__(self, description: str = None, conditions: List[str] = None):
+        self.description = description
+        self.conditions = conditions
+
+    def json(self):
+        json_dictionary = {}
+        if self.description is not None:
+            json_dictionary['description'] = self.description
+        if self.conditions is not None:
+            json_dictionary['conditions'] = [c for c in self.conditions]
+        return json_dictionary
+
+
+class ActionOptions:
+    description: str
+    input_spec: InputSpec
+    output_spec: OutputSpec
+    filters: List[DescriptionWithConditions] = None
+    errors: List[DescriptionWithConditions] = None
+    notes: List[str]
+    details: str
+
+    def __init__(self, description: str = None, input_spec: InputSpec = None, output_spec: OutputSpec = None,
+                 filters: List = None, errors: List = None, notes: List[str] = None, details: str = None):
+        self.description = description
+        self.input_spec = input_spec
+        self.output_spec = output_spec
+        if filters is not None:
+            self.filters = []
+            for f in filters:
+                if isinstance(f, DescriptionWithConditions):
+                    self.filters.append(f)
+                else:
+                    self.filters.append(DescriptionWithConditions(description=f))
+        if errors is not None:
+            self.errors = []
+            for e in errors:
+                if isinstance(e, DescriptionWithConditions):
+                    self.errors.append(e)
+                else:
+                    self.errors.append(DescriptionWithConditions(description=e))
+        self.notes = notes
+        self.details = details
+
+    def json(self):
+        json_dictionary = {}
+        if self.description is not None:
+            json_dictionary['description'] = self.description
+        if self.input_spec is not None:
+            json_dictionary['inputSpec'] = self.input_spec.json()
+        if self.output_spec is not None:
+            json_dictionary['outputSpec'] = self.output_spec.json()
+        if self.filters is not None:
+            json_dictionary['filters'] = [f.json() for f in self.filters]
+        if self.errors is not None:
+            json_dictionary['errors'] = [e.json() for e in self.errors]
+        if self.notes is not None:
+            json_dictionary['notes'] = [n for n in self.notes]
+        if self.details is not None:
+            json_dictionary['details'] = self.details
+        return json_dictionary
 
-class EgressAction(Action):
-    def __init__(self, description: str):
-        super().__init__(ActionType.EGRESS, description, [], [])
 
-    def execute(self, event):
-        egress_input = EgressInput(content=event.delta_file_messages[0].content_list[0],
-                                   metadata=event.delta_file_messages[0].metadata)
-        result = self.egress(event.context, self.param_class().parse_obj(event.params), egress_input)
-        self.validate_type(result, (EgressResult, ErrorResult, FilterResult))
-        return result
+class Action(ABC):
+    def __init__(self, action_type: ActionType, description: str, valid_result_types: tuple,
+                 action_options: ActionOptions = None):
+        self.action_type = action_type
+        if action_options is None:
+            self.action_options = ActionOptions(description=description)
+        else:
+            self.action_options = action_options
+        self.valid_result_types = valid_result_types
 
     @abstractmethod
-    def egress(self, context: Context, params: BaseModel, egress_input: EgressInput):
+    def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
         pass
 
-
-class EnrichAction(Action):
-    def __init__(self, description: str, requires_domains: List[str], requires_enrichments: List[str]):
-        super().__init__(ActionType.ENRICH, description, requires_domains, requires_enrichments)
-
-    def execute(self, event):
-        enrich_input = EnrichInput(content=event.delta_file_messages[0].content_list,
-                                   metadata=event.delta_file_messages[0].metadata,
-                                   domains={domain.name: domain for domain in event.delta_file_messages[0].domains},
-                                   enrichment={domain.name: domain for domain in event.delta_file_messages[0].enrichment})
-        result = self.enrich(event.context, self.param_class().parse_obj(event.params), enrich_input)
-        self.validate_type(result, (EnrichResult, ErrorResult))
-        return result
+    def execute_join_action(self, event):
+        raise RuntimeError(f"Join is not supported for {self.__class__.__name__}")
 
     @abstractmethod
-    def enrich(self, context: Context, params: BaseModel, enrich_input: EnrichInput):
+    def execute(self, context: Context, action_input: Any, params: BaseModel):
         pass
 
+    def execute_action(self, event):
+        if event.delta_file_messages is None or not len(event.delta_file_messages):
+            raise RuntimeError(f"Received event with no delta file messages for did {event.context.did}")
+        if event.context.join is not None:
+            result = self.execute_join_action(event)
+        else:
+            result = self.execute(
+                event.context,
+                self.build_input(event.context, event.delta_file_messages[0]),
+                self.param_class().model_validate(event.params))
+
+        self.validate_type(result)
+        return result
 
-class FormatAction(Action):
-    def __init__(self, description: str, requires_domains: List[str], requires_enrichments: List[str]):
-        super().__init__(ActionType.FORMAT, description, requires_domains, requires_enrichments)
+    @staticmethod
+    def param_class():
+        """Factory method to create and return an empty GenericModel instance.
 
-    def execute(self, event):
-        format_input = FormatInput(content=event.delta_file_messages[0].content_list,
-                                   metadata=event.delta_file_messages[0].metadata,
-                                   domains={domain.name: domain for domain in event.delta_file_messages[0].domains},
-                                   enrichment={domain.name: domain for domain in event.delta_file_messages[0].enrichment})
-        result = self.format(event.context, self.param_class().parse_obj(event.params), format_input)
-        self.validate_type(result, (FormatResult, FormatManyResult, ErrorResult, FilterResult))
-        return result
+        All action parameter classes must inherit pydantic.BaseModel.
+        Use of complex types in custom action parameter classes must specify
+        the internal types when defined. E.g., dict[str, str], or List[str]
 
-    @abstractmethod
-    def format(self, context: Context, params: BaseModel, format_input: FormatInput):
-        pass
+        Returns
+        -------
+        GenericModel
+            an empty GenericModel instance
+        """
+        return GenericModel
 
+    def validate_type(self, result):
+        if not isinstance(result, self.valid_result_types):
+            raise ValueError(f"{self.__class__.__name__} must return one of "
+                             f"{[result_type.__name__ for result_type in self.valid_result_types]} "
+                             f"but a {result.__class__.__name__} was returned")
 
-class LoadAction(Action):
-    def __init__(self, description: str):
-        super().__init__(ActionType.LOAD, description, [], [])
 
-    def execute(self, event):
-        load_input = LoadInput(content=event.delta_file_messages[0].content_list,
-                               metadata=event.delta_file_messages[0].metadata)
-        result = self.load(event.context, self.param_class().parse_obj(event.params), load_input)
-        self.validate_type(result, (LoadResult, LoadManyResult, ErrorResult, FilterResult, ReinjectResult))
-        return result
+class EgressAction(Action, ABC):
+    def __init__(self, description: str, action_options: ActionOptions = None):
+        super().__init__(ActionType.EGRESS, description, (EgressResult, ErrorResult, FilterResult), action_options)
+
+    def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
+        content = None
+        if delta_file_message.content_list is not None and len(delta_file_message.content_list) > 0:
+            content = delta_file_message.content_list[0]
+        return EgressInput(content=content, metadata=delta_file_message.metadata)
 
     @abstractmethod
-    def load(self, context: Context, params: BaseModel, load_input: LoadInput):
+    def egress(self, context: Context, params: BaseModel, egress_input: EgressInput):
         pass
 
+    def execute(self, context: Context, egress_input: EgressInput, params: BaseModel):
+        return self.egress(context, params, egress_input)
 
-class TransformAction(Action):
-    def __init__(self, description: str):
-        super().__init__(ActionType.TRANSFORM, description, [], [])
 
-    def execute(self, event):
-        transform_input = TransformInput(content=event.delta_file_messages[0].content_list,
-                                         metadata=event.delta_file_messages[0].metadata)
-        result = self.transform(event.context, self.param_class().parse_obj(event.params), transform_input)
-        self.validate_type(result, (TransformResult, ErrorResult, FilterResult))
-        return result
+class TimedIngressAction(Action, ABC):
+    def __init__(self, description: str, action_options: ActionOptions = None):
+        super().__init__(ActionType.TIMED_INGRESS, description, (IngressResult, ErrorResult), action_options)
+
+    def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
+        return None
 
     @abstractmethod
-    def transform(self, context: Context, params: BaseModel, transform_input: TransformInput):
+    def ingress(self, context: Context, params: BaseModel):
         pass
 
+    def execute(self, context: Context, input_placeholder: Any, params: BaseModel):
+        return self.ingress(context, params)
 
-class ValidateAction(Action):
-    def __init__(self, description: str):
-        super().__init__(ActionType.VALIDATE, description, [], [])
 
-    def execute(self, event):
-        validate_input = ValidateInput(content=event.delta_file_messages[0].content_list[0],
-                                       metadata=event.delta_file_messages[0].metadata)
-        result = self.validate(event.context, self.param_class().parse_obj(event.params), validate_input)
-        self.validate_type(result, (ValidateResult, ErrorResult, FilterResult))
-        return result
+class TransformAction(Action, ABC):
+    def __init__(self, description: str, action_options: ActionOptions = None):
+        super().__init__(ActionType.TRANSFORM, description,
+                         (TransformResult, TransformResults, ErrorResult, FilterResult), action_options)
+
+    def build_input(self, context: Context, delta_file_message: DeltaFileMessage):
+        return TransformInput(content=delta_file_message.content_list, metadata=delta_file_message.metadata)
+
+    def execute_join_action(self, event):
+        if isinstance(self, Join):
+            return self.execute(
+                event.context,
+                self.join([self.build_input(event.context, delta_file_message)
+                           for delta_file_message in event.delta_file_messages]),
+                self.param_class().model_validate(event.params))
+        else:
+            super().execute_join_action(event)
 
     @abstractmethod
-    def validate(self, context: Context, params: BaseModel, validate_input: ValidateInput):
+    def transform(self, context: Context, params: BaseModel, transform_input: TransformInput):
         pass
+
+    def execute(self, context: Context, transform_input: TransformInput, params: BaseModel):
+        return self.transform(context, params, transform_input)

deltafi/actioneventqueue.py

@@ -1,7 +1,7 @@
 #
 #    DeltaFi - Data transformation and enrichment platform
 #
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#    Copyright 2021-2025 DeltaFi Contributors <deltafi@deltafi.org>
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -17,14 +17,19 @@
 #
 
 from datetime import datetime, timezone
+from typing import List
 from urllib.parse import urlparse
 
+import json
 import redis
 import time
 
+HEARTBEAT_HASH = "org.deltafi.action-queue.heartbeat"
+LONG_RUNNING_TASKS_HASH = "org.deltafi.action-queue.long-running-tasks"
+
 
 class ActionEventQueue:
-    def __init__(self, url, max_connections, password):
+    def __init__(self, url, max_connections, password, app_name):
         parsed = urlparse(url)
         self.pool = redis.ConnectionPool(
             max_connections=max_connections,
@@ -32,6 +37,7 @@ class ActionEventQueue:
             port=parsed.port,
             password=password)
         self.connection = None
+        self.app_name = app_name
 
     def get_connection(self):
         if self.connection is None:
@@ -49,7 +55,7 @@ class ActionEventQueue:
         added = conn.zadd(name, {item: now}, nx=True)
         return added
 
-    def take(self, name: str) -> str:
+    def take(self, name: List[str]) -> str:
         conn = self.get_connection()
         setkey, item, score = conn.bzpopmin(name, 0)
         return item
@@ -57,4 +63,23 @@ class ActionEventQueue:
     def heartbeat(self, name: str):
         conn = self.get_connection()
         utcnow = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
-        conn.hset("org.deltafi.action-queue.heartbeat", name, utcnow)
+        conn.hset(HEARTBEAT_HASH, name, utcnow)
+
+    def record_long_running_task(self, action_execution):
+        try:
+            key = action_execution.key
+            start_time = action_execution.start_time.isoformat().replace("+00:00", "Z")
+            heartbeat_time = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
+            values = [start_time, heartbeat_time]
+            if self.app_name is not None:
+                values.append(self.app_name)
+            value = json.dumps(values)
+            conn = self.get_connection()
+            conn.hset(LONG_RUNNING_TASKS_HASH, key, value)
+        except Exception as e:
+            print(f"Unable to convert long running task information to JSON: {str(e)}")
+
+    def remove_long_running_task(self, action_execution):
+        key = action_execution.key
+        conn = self.get_connection()
+        conn.hdel(LONG_RUNNING_TASKS_HASH, key)

deltafi/actiontype.py

@@ -1,7 +1,7 @@
 #
 #    DeltaFi - Data transformation and enrichment platform
 #
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#    Copyright 2021-2025 DeltaFi Contributors <deltafi@deltafi.org>
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -20,13 +20,9 @@ from enum import Enum
 
 
 class ActionType(Enum):
-    INGRESS = "ingress"
-    TRANSFORM = "transform"
-    LOAD = "load"
-    DOMAIN = "domain"
-    ENRICH = "enrich"
-    FORMAT = "format"
-    VALIDATE = "validate"
-    EGRESS = "egress"
-    DELETE = "delete"
-    UNKNOWN = "unknown"
+    INGRESS = "INGRESS"
+    TIMED_INGRESS = "TIMED_INGRESS"
+    TRANSFORM = "TRANSFORM"
+    EGRESS = "EGRESS"
+    PUBLISH = "PUBLISH"
+    UNKNOWN = "UNKNOWN"