deltafi 2.0rc1720728217472__py3-none-any.whl → 2.0rc1722532930396__py3-none-any.whl

deltafi/plugin.py

@@ -34,8 +34,7 @@ from importlib import metadata
 import requests
 from deltafi.actioneventqueue import ActionEventQueue
 from deltafi.domain import Event, ActionExecution
-from deltafi.exception import ExpectedContentException, MissingDomainException, MissingEnrichmentException, \
-    MissingMetadataException
+from deltafi.exception import ExpectedContentException, MissingMetadataException
 from deltafi.logger import get_logger
 from deltafi.result import ErrorResult
 from deltafi.storage import ContentService
@@ -47,9 +46,9 @@ def _coordinates():
 
 
 def _setup_queue(max_connections):
-    redis_url = os.getenv('REDIS_URL', 'http://deltafi-redis-master:6379')
-    password = os.getenv('REDIS_PASSWORD')
-    return ActionEventQueue(redis_url, max_connections, password)
+    url = os.getenv('VALKEY_URL', 'http://deltafi-valkey-master:6379')
+    password = os.getenv('VALKEY_PASSWORD')
+    return ActionEventQueue(url, max_connections, password)
 
 
 def _setup_content_service():
@@ -165,8 +164,6 @@ class Plugin(object):
             'name': self.action_name(action),
             'description': action.description,
             'type': action.action_type.name,
-            'requiresDomains': action.requires_domains,
-            'requiresEnrichments': action.requires_enrichments,
             'schema': action.param_class().model_json_schema()
         }
 
@@ -219,7 +216,8 @@ class Plugin(object):
         for action in self.actions:
             threading.Thread(target=self._do_action, args=(action,)).start()
 
-        threading.Thread(target=self._heartbeat).start()
+        hb_thread = threading.Thread(target=self._heartbeat)
+        hb_thread.start()
 
         self.logger.info("All threads running")
 
@@ -227,6 +225,7 @@ class Plugin(object):
         f.close()
 
         self.logger.info("Application initialization complete")
+        hb_thread.join()
 
     def _heartbeat(self):
         long_running_actions = set()
@@ -263,7 +262,7 @@ class Plugin(object):
         while True:
             try:
                 event_string = self.queue.take(self.action_name(action))
-                event = Event.create(json.loads(event_string), self.hostname, self.content_service, action_logger)
+                event = Event.create(json.loads(event_string), self.content_service, action_logger)
                 start_time = time.time()
                 action_logger.debug(f"Processing event for did {event.context.did}")
 
@@ -277,14 +276,6 @@ class Plugin(object):
                                          f"Action attempted to look up element {e.index + 1} (index {e.index}) from "
                                          f"content list of size {e.size}",
                                          f"{str(e)}\n{traceback.format_exc()}")
-                except MissingDomainException as e:
-                    result = ErrorResult(event.context,
-                                         f"Action attempted to access domain {e.name}, which does not exist",
-                                         f"{str(e)}\n{traceback.format_exc()}")
-                except MissingEnrichmentException as e:
-                    result = ErrorResult(event.context,
-                                         f"Action attempted to access enrichment {e.name}, which does not exist",
-                                         f"{str(e)}\n{traceback.format_exc()}")
                 except MissingMetadataException as e:
                     result = ErrorResult(event.context,
                                          f"Missing metadata with key {e.key}",
@@ -297,7 +288,10 @@ class Plugin(object):
 
                 response = {
                     'did': event.context.did,
-                    'action': event.context.action_flow + "." + event.context.action_name,
+                    'flowName': event.context.flow_name,
+                    'flowId': event.context.flow_id,
+                    'actionName': event.context.action_name,
+                    'actionId': event.context.action_id,
                     'start': start_time,
                     'stop': time.time(),
                     'type': result.result_type,

deltafi/result.py

@@ -17,9 +17,9 @@
 #
 
 import abc
-from enum import Enum
 import uuid
-from typing import Dict, List
+from enum import Enum
+from typing import NamedTuple
 
 from deltafi.domain import Content, Context
 from deltafi.metric import Metric
@@ -44,22 +44,8 @@ class Result:
 
     def add_metric(self, metric: Metric):
         self.metrics.append(metric)
-
-
-class DomainResult(Result):
-    def __init__(self, context: Context):
-        super().__init__('domain', 'DOMAIN', context)
-        self.annotations = {}
-
-    def annotate(self, key: str, value: str):
-        self.annotations[key] = value
         return self
 
-    def response(self):
-        return {
-            'annotations': self.annotations
-        }
-
 
 class EgressResult(Result):
     def __init__(self, context: Context, destination: str, bytes_egressed: int):
@@ -71,31 +57,6 @@ class EgressResult(Result):
         return None
 
 
-class EnrichResult(Result):
-    def __init__(self, context: Context):
-        super().__init__('enrich', 'ENRICH', context)
-        self.enrichments = []
-        self.annotations = {}
-
-    def enrich(self, name: str, value: str, media_type: str):
-        self.enrichments.append({
-            'name': name,
-            'value': value,
-            'mediaType': media_type
-        })
-        return self
-
-    def annotate(self, key: str, value: str):
-        self.annotations[key] = value
-        return self
-
-    def response(self):
-        return {
-            'enrichments': self.enrichments,
-            'annotations': self.annotations
-        }
-
-
 class ErrorResult(Result):
     def __init__(self, context: Context, error_cause: str, error_context: str):
         super().__init__('error', 'ERROR', context)
@@ -116,7 +77,7 @@ class ErrorResult(Result):
 
 
 class FilterResult(Result):
-    def __init__(self, context: Context, filtered_cause: str, filtered_context: str=None):
+    def __init__(self, context: Context, filtered_cause: str, filtered_context: str = None):
         super().__init__('filter', 'FILTER', context)
         self.filtered_cause = filtered_cause
         self.filtered_context = filtered_context
@@ -134,87 +95,13 @@ class FilterResult(Result):
         }
 
 
-class FormatResult(Result):
-    def __init__(self, context: Context):
-        super().__init__('format', 'FORMAT', context)
-        self.content = None
-        self.delete_metadata_keys = []
-        self.metadata = {}
-
-    def set_metadata(self, metadata: dict):
-        self.metadata = metadata
-        return self
-
-    def add_metadata(self, key: str, value: str):
-        self.metadata[key] = value
-        return self
-
-    def delete_metadata_key(self, key: str):
-        self.delete_metadata_keys.append(key)
-        return self
-
-    def set_content(self, content: Content):
-        self.content = content
-        return self
-
-    def save_string_content(self, string_data: str, name: str, media_type: str):
-        segment = self.context.content_service.put_str(self.context.did, string_data)
-        self.content = Content(name=name, segments=[segment], media_type=media_type,
-                               content_service=self.context.content_service)
-        return self
-
-    def save_byte_content(self, byte_data: bytes, name: str, media_type: str):
-        segment = self.context.content_service.put_bytes(self.context.did, byte_data)
-        self.content = Content(name=name, segments=[segment], media_type=media_type,
-                               content_service=self.context.content_service)
-        return self
-
-    def response(self):
-        return {
-            'content': self.content.json(),
-            'metadata': self.metadata,
-            'deleteMetadataKeys': self.delete_metadata_keys
-        }
-
-
-class ChildFormatResult:
-    def __init__(self, format_result: FormatResult = None):
-        self._did = str(uuid.uuid4())
-        self.format_result = format_result
-
-    @property
-    def did(self):
-        return self._did
-
-    def response(self):
-        res = self.format_result.response()
-        res["did"] = self._did
-        return res
-
-
-class FormatManyResult(Result):
-    def __init__(self, context: Context):
-        super().__init__('formatMany', 'FORMAT_MANY', context)
-        self.format_results = []
-
-    def add_format_result(self, format_result):
-        if isinstance(format_result, ChildFormatResult):
-            self.format_results.append(format_result)
-        else:
-            self.format_results.append(ChildFormatResult(format_result))
-        return self
-
-    def response(self):
-        return [format_result.response() for format_result in self.format_results]
-
-
 class IngressResultItem:
-    def __init__(self, context: Context, filename: str):
+    def __init__(self, context: Context, delta_file_name: str):
         self.context = context
-        self.filename = filename
         self._did = str(uuid.uuid4())
         self.content = []
         self.metadata = {}
+        self.delta_file_name = delta_file_name
 
     @property
     def did(self):
@@ -253,7 +140,7 @@ class IngressResultItem:
     def response(self):
         return {
             'did': self._did,
-            'filename': self.filename,
+            'deltaFileName': self.delta_file_name,
             'metadata': self.metadata,
             'content': [content.json() for content in self.content]
         }
@@ -269,10 +156,10 @@ class IngressResult(Result):
     def __init__(self, context: Context):
         super().__init__('ingress', 'INGRESS', context)
         self.memo = None
-        self.execute_immediate = False
         self.ingress_result_items = []
+        self.execute_immediate = False
         self.status = IngressStatusEnum.HEALTHY
-        self.statusMessage = None
+        self.status_message = None
 
     def add_item(self, ingress_result_item: IngressResultItem):
         self.ingress_result_items.append(ingress_result_item)
@@ -284,17 +171,16 @@ class IngressResult(Result):
             'executeImmediate': self.execute_immediate,
             'ingressItems': [ingress_result_item.response() for ingress_result_item in self.ingress_result_items],
             'status': self.status.value,
-            'statusMessage': self.statusMessage
+            'statusMessage': self.status_message
         }
 
 
-class LoadResult(Result):
+class TransformResult(Result):
     def __init__(self, context: Context):
-        super().__init__('load', 'LOAD', context)
+        super().__init__('transform', 'TRANSFORM', context)
         self.content = []
-        self.metadata = {}
-        self.domains = []
         self.annotations = {}
+        self.metadata = {}
         self.delete_metadata_keys = []
 
     # content can be a single Content or a List[Content]
@@ -327,13 +213,6 @@ class LoadResult(Result):
         self.metadata[key] = value
         return self
 
-    def add_domain(self, name: str, value: str, media_type: str):
-        self.domains.append({
-            'name': name,
-            'value': value,
-            'mediaType': media_type})
-        return self
-
     def annotate(self, key: str, value: str):
         self.annotations[key] = value
         return self
@@ -342,133 +221,41 @@ class LoadResult(Result):
         self.delete_metadata_keys.append(key)
         return self
 
-    def response(self):
+    def json(self):
         return {
-            'domains': self.domains,
             'content': [content.json() for content in self.content],
-            'metadata': self.metadata,
             'annotations': self.annotations,
+            'metadata': self.metadata,
             'deleteMetadataKeys': self.delete_metadata_keys
         }
 
-
-class ChildLoadResult:
-    def __init__(self, load_result: LoadResult = None):
-        self._did = str(uuid.uuid4())
-        self.load_result = load_result
-
-    @property
-    def did(self):
-        return self._did
-
-    def response(self):
-        res = self.load_result.response()
-        res["did"] = self._did
-        return res
-
-
-class LoadManyResult(Result):
-    def __init__(self, context: Context):
-        super().__init__('loadMany', 'LOAD_MANY', context)
-        self.load_results = []
-
-    def add_load_result(self, load_result):
-        if isinstance(load_result, ChildLoadResult):
-            self.load_results.append(load_result)
-        else:
-            self.load_results.append(ChildLoadResult(load_result))
-        return self
-
     def response(self):
-        return [load_result.response() for load_result in self.load_results]
-
-
-class ReinjectResult(Result):
-    class ReinjectChild:
-        def __init__(self, filename: str, flow: str, content: List[Content], metadata: Dict[str, str]):
-            self.filename = filename
-            self.flow = flow
-            self.content = content
-            self.metadata = metadata
+        return [self.json()]
 
-        def json(self):
-            return {
-                'filename': self.filename,
-                'flow': self.flow,
-                'metadata': self.metadata,
-                'content': [content.json() for content in self.content]
-            }
-
-    def __init__(self, context: Context):
-        super().__init__('reinject', 'REINJECT', context)
-        self.children = []
 
-    def add_child(self, filename: str, flow: str, content: List[Content], metadata: Dict[str, str]):
-        child = ReinjectResult.ReinjectChild(filename, flow, content, metadata)
-        self.children.append(child)
+class NamedTransformResult(NamedTuple):
+    result: TransformResult
+    name: str
 
-    def response(self):
-        return [child.json() for child in self.children]
+    def json(self):
+        j = self.result.json()
+        if self.name is not None:
+            j['name'] = self.name
+        return j
 
 
-class TransformResult(Result):
+class TransformResults(Result):
     def __init__(self, context: Context):
         super().__init__('transform', 'TRANSFORM', context)
-        self.content = []
-        self.metadata = {}
-        self.annotations = {}
-        self.delete_metadata_keys = []
-
-    # content can be a single Content or a List[Content]
-    def add_content(self, content):
-        if content:
-            if type(content) == list:
-                self.content.extend(content)
-            else:
-                self.content.append(content)
+        self.named_results = []
 
-        return self
-
-    def save_string_content(self, string_data: str, name: str, media_type: str):
-        segment = self.context.content_service.put_str(self.context.did, string_data)
-        self.content.append(
-            Content(name=name, segments=[segment], media_type=media_type, content_service=self.context.content_service))
-        return self
-
-    def save_byte_content(self, byte_data: bytes, name: str, media_type: str):
-        segment = self.context.content_service.put_bytes(self.context.did, byte_data)
-        self.content.append(
-            Content(name=name, segments=[segment], media_type=media_type, content_service=self.context.content_service))
-        return self
-
-    def set_metadata(self, metadata: dict):
-        self.metadata = metadata
-        return self
-
-    def add_metadata(self, key: str, value: str):
-        self.metadata[key] = value
-        return self
-
-    def annotate(self, key: str, value: str):
-        self.annotations[key] = value
-        return self
-
-    def delete_metadata_key(self, key: str):
-        self.delete_metadata_keys.append(key)
+    def add_result(self, result: TransformResult, name: str = None):
+        self.named_results.append(NamedTransformResult(result, name))
         return self
 
     def response(self):
-        return {
-            'content': [content.json() for content in self.content],
-            'metadata': self.metadata,
-            'annotations': self.annotations,
-            'deleteMetadataKeys': self.delete_metadata_keys
-        }
-
-
-class ValidateResult(Result):
-    def __init__(self, context: Context):
-        super().__init__(None, 'VALIDATE', context)
-
-    def response(self):
-        return None
+        transform_events = []
+        for named_result in self.named_results:
+            json_dict = named_result.json()
+            transform_events.append(json_dict)
+        return transform_events