deltafi 1.2.20__py3-none-any.whl → 2.0.0__py3-none-any.whl

deltafi/input.py

@@ -1,7 +1,7 @@
 #
 #    DeltaFi - Data transformation and enrichment platform
 #
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#    Copyright 2021-2024 DeltaFi Contributors <deltafi@deltafi.org>
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -17,172 +17,13 @@
 #
 
 from deltafi.domain import *
-from deltafi.exception import MissingMetadataException, ExpectedContentException, MissingDomainException, \
-    MissingEnrichmentException
-
-
-class DomainInput(NamedTuple):
-    content: List[Content]
-    metadata: Dict[str, str]
-    domains: Dict[str, Domain]
-
-    def has_content(self) -> bool:
-        return len(self.content) > 0
-
-    def content_at(self, index: int) -> Content:
-        if len(self.content) < index + 1:
-            raise ExpectedContentException(index, len(self.content))
-        return self.content[index]
-
-    def first_content(self):
-        return self.content_at(0)
-
-    def get_metadata(self, key: str):
-        if key in self.metadata:
-            return self.metadata[key]
-        else:
-            raise MissingMetadataException(key)
-
-    def get_metadata_or_else(self, key: str, default: str) -> str:
-        if key in self.metadata:
-            return self.metadata[key]
-        else:
-            return default
-
-    def has_domain(self, name: str) -> bool:
-        return name in self.domains
-
-    def domain(self, name: str) -> Domain:
-        if not self.has_domain(name):
-            raise MissingDomainException(name)
-        return self.domains[name]
-
+from deltafi.exception import MissingMetadataException, ExpectedContentException
 
 class EgressInput(NamedTuple):
     content: Content
     metadata: dict
 
 
-class EnrichInput(NamedTuple):
-    content: List[Content]
-    metadata: dict
-    domains: Dict[str, Domain]
-    enrichments: Dict[str, Domain]
-
-    def has_content(self) -> bool:
-        return len(self.content) > 0
-
-    def content_at(self, index: int) -> Content:
-        if len(self.content) < index + 1:
-            raise ExpectedContentException(index, len(self.content))
-        return self.content[index]
-
-    def first_content(self):
-        return self.content_at(0)
-
-    def get_metadata(self, key: str):
-        if key in self.metadata:
-            return self.metadata[key]
-        else:
-            raise MissingMetadataException(key)
-
-    def get_metadata_or_else(self, key: str, default: str) -> str:
-        if key in self.metadata:
-            return self.metadata[key]
-        else:
-            return default
-
-    def has_domain(self, name: str) -> bool:
-        return name in self.domains
-
-    def domain(self, name: str) -> Domain:
-        if not self.has_domain(name):
-            raise MissingDomainException(name)
-        return self.domains[name]
-
-    def has_enrichment(self, name: str) -> bool:
-        return name in self.enrichments
-
-    def enrichment(self, name: str) -> Domain:
-        if not self.has_enrichment(name):
-            raise MissingEnrichmentException(name)
-        return self.enrichments[name]
-
-
-class FormatInput(NamedTuple):
-    content: List[Content]
-    metadata: dict
-    domains: Dict[str, Domain]
-    enrichments: Dict[str, Domain]
-
-    def has_content(self) -> bool:
-        return len(self.content) > 0
-
-    def content_at(self, index: int) -> Content:
-        if len(self.content) < index + 1:
-            raise ExpectedContentException(index, len(self.content))
-        return self.content[index]
-
-    def first_content(self):
-        return self.content_at(0)
-
-    def get_metadata(self, key: str):
-        if key in self.metadata:
-            return self.metadata[key]
-        else:
-            raise MissingMetadataException(key)
-
-    def get_metadata_or_else(self, key: str, default: str) -> str:
-        if key in self.metadata:
-            return self.metadata[key]
-        else:
-            return default
-
-    def has_domain(self, name: str) -> bool:
-        return name in self.domains
-
-    def domain(self, name: str) -> Domain:
-        if not self.has_domain(name):
-            raise MissingDomainException(name)
-        return self.domains[name]
-
-    def has_enrichment(self, name: str) -> bool:
-        return name in self.enrichments
-
-    def enrichment(self, name: str) -> Domain:
-        if not self.has_enrichment(name):
-            raise MissingEnrichmentException(name)
-        return self.enrichments[name]
-
-
-class LoadInput(NamedTuple):
-    content: List[Content]
-    metadata: dict
-
-    def has_content(self) -> bool:
-        return len(self.content) > 0
-
-    def content_at(self, index: int) -> Content:
-        if len(self.content) < index + 1:
-            raise ExpectedContentException(index, len(self.content))
-        return self.content[index]
-
-    def first_content(self):
-        return self.content_at(0)
-
-    def get_metadata(self, key: str):
-        if key in self.metadata:
-            return self.metadata[key]
-        else:
-            raise MissingMetadataException(key)
-
-    def get_metadata_or_else(self, key: str, default: str) -> str:
-        if key in self.metadata:
-            return self.metadata[key]
-        else:
-            return default
-
-
 class TransformInput(NamedTuple):
     content: List[Content]
     metadata: dict
@@ -209,8 +50,3 @@ class TransformInput(NamedTuple):
             return self.metadata[key]
         else:
             return default
-
-
-class ValidateInput(NamedTuple):
-    content: Content
-    metadata: dict

deltafi/logger.py

@@ -1,7 +1,7 @@
 #
 #    DeltaFi - Data transformation and enrichment platform
 #
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#    Copyright 2021-2024 DeltaFi Contributors <deltafi@deltafi.org>
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -18,14 +18,14 @@
 
 import logging
 import sys
-from datetime import datetime
+from datetime import datetime, UTC
 
 import json_logging
 
 
 def get_logger(name: str = None) -> logging.Logger:
     logger = logging.getLogger(name)
-    logger.setLevel(logging.DEBUG)
+    logger.setLevel(logging.INFO)
     logger.addHandler(logging.StreamHandler(sys.stdout))
     logger.propagate = False
 
@@ -42,7 +42,7 @@ def _sanitize_log_msg(record):
 class JSONLogFormatter(json_logging.JSONLogFormatter):
 
     def _format_log_object(self, record, request_util):
-        utcnow = datetime.utcnow()
+        utcnow = datetime.now(UTC)
 
         json_log_object = {
             'timestamp': json_logging.util.iso_time_format(utcnow),

deltafi/metric.py

@@ -1,7 +1,7 @@
 #
 #    DeltaFi - Data transformation and enrichment platform
 #
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#    Copyright 2021-2024 DeltaFi Contributors <deltafi@deltafi.org>
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -22,7 +22,7 @@ from typing import Dict, NamedTuple
 class Metric(NamedTuple):
     name: str
     value: int
-    tags: Dict[str, str]
+    tags: Dict[str, str] = {}
 
     def json(self):
         return {

deltafi/plugin.py

@@ -1,7 +1,7 @@
 #
 #    DeltaFi - Data transformation and enrichment platform
 #
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#    Copyright 2021-2024 DeltaFi Contributors <deltafi@deltafi.org>
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -16,30 +16,30 @@
 #    limitations under the License.
 #
 
+import importlib
+import inspect
 import json
 import os
+import pkgutil
 import sys
 import threading
 import time
 import traceback
+import yaml
 from datetime import datetime, timezone, timedelta
+from importlib import metadata
 from os.path import isdir, isfile, join
 from pathlib import Path
 from typing import List
-import importlib
-import inspect
-import pkgutil
 
-from importlib import metadata
 import requests
+from deltafi.action import Action, Join
 from deltafi.actioneventqueue import ActionEventQueue
 from deltafi.domain import Event, ActionExecution
-from deltafi.exception import ExpectedContentException, MissingDomainException, MissingEnrichmentException, \
-    MissingMetadataException
+from deltafi.exception import ExpectedContentException, MissingMetadataException
 from deltafi.logger import get_logger
-from deltafi.result import ErrorResult
+from deltafi.result import ErrorResult, IngressResult, TransformResult, TransformResults
 from deltafi.storage import ContentService
-from deltafi.action import Action
 
 
 def _coordinates():
@@ -47,9 +47,9 @@ def _coordinates():
 
 
 def _setup_queue(max_connections):
-    redis_url = os.getenv('REDIS_URL', 'http://deltafi-redis-master:6379')
-    password = os.getenv('REDIS_PASSWORD')
-    return ActionEventQueue(redis_url, max_connections, password)
+    url = os.getenv('VALKEY_URL', 'http://deltafi-valkey-master:6379')
+    password = os.getenv('VALKEY_PASSWORD')
+    return ActionEventQueue(url, max_connections, password)
 
 
 def _setup_content_service():
@@ -94,6 +94,8 @@ class Plugin(object):
         self.queue = None
         self.actions = []
         self.core_url = os.getenv('CORE_URL')
+        self.image = os.getenv('IMAGE')
+        self.image_pull_secret = os.getenv('IMAGE_PULL_SECRET')
         action_classes = []
         if actions is not None and len(actions):
             action_classes.extend(actions)
@@ -160,16 +162,39 @@ class Plugin(object):
     def action_name(self, action):
         return f"{self.coordinates.group_id}.{action.__class__.__name__}"
 
+    def _load_action_docs(self, action):
+        docs_path = str(Path(os.path.dirname(os.path.abspath(sys.argv[0]))) / 'docs')
+        if not isdir(docs_path):
+            return None
+
+        action_docs_file = join(docs_path, action.__class__.__name__ + '.md')
+        if not isfile(action_docs_file):
+            return None
+
+        return open(action_docs_file).read()
+
     def _action_json(self, action):
         return {
             'name': self.action_name(action),
             'description': action.description,
             'type': action.action_type.name,
-            'requiresDomains': action.requires_domains,
-            'requiresEnrichments': action.requires_enrichments,
-            'schema': action.param_class().model_json_schema()
+            'supportsJoin': isinstance(action, Join),
+            'schema': action.param_class().model_json_schema(),
+            'docsMarkdown': self._load_action_docs(action)
         }
 
+    def _integration_tests(self):
+        tests_path = str(Path(os.path.dirname(os.path.abspath(sys.argv[0]))) / 'integration')
+
+        test_files = []
+        if isdir(tests_path):
+            test_files = [f for f in os.listdir(tests_path) if isfile(join(tests_path, f))]
+        else:
+            self.logger.warning(f"tests directory ({tests_path}) does not exist. No tests will be installed.")
+
+        tests = [json.load(open(join(tests_path, f))) for f in test_files]
+        return tests
+
     def registration_json(self):
         flows_path = str(Path(os.path.dirname(os.path.abspath(sys.argv[0]))) / 'flows')
 
@@ -187,15 +212,18 @@ class Plugin(object):
         actions = [self._action_json(action) for action in self.actions]
 
         return {
-                'pluginCoordinates': self.coordinates.__json__(),
-                'displayName': self.display_name,
-                'description': self.description,
-                'actionKitVersion': metadata.version('deltafi'),
-                'dependencies': [],
-                'actions': actions,
-                'variables': variables,
-                'flowPlans': flows
-            }
+            'pluginCoordinates': self.coordinates.__json__(),
+            'displayName': self.display_name,
+            'description': self.description,
+            'actionKitVersion': metadata.version('deltafi'),
+            'image': self.image,
+            'imagePullSecret': self.image_pull_secret,
+            'dependencies': [],
+            'actions': actions,
+            'variables': variables,
+            'flowPlans': flows,
+            'integrationTests': self._integration_tests()
+        }
 
     def _register(self):
         url = f"{self.core_url}/plugins"
@@ -258,6 +286,22 @@ class Plugin(object):
             finally:
                 time.sleep(10)
 
+    @staticmethod
+    def to_response(event, start_time, stop_time, result):
+        response = {
+            'did': event.context.did,
+            'flowName': event.context.flow_name,
+            'flowId': event.context.flow_id,
+            'actionName': event.context.action_name,
+            'start': start_time,
+            'stop': stop_time,
+            'type': result.result_type,
+            'metrics': [metric.json() for metric in result.metrics]
+        }
+        if result.result_key is not None:
+            response[result.result_key] = result.response()
+        return response
+
     def _do_action(self, action):
         action_logger = get_logger(self.action_name(action))
 
@@ -265,7 +309,7 @@ class Plugin(object):
         while True:
             try:
                 event_string = self.queue.take(self.action_name(action))
-                event = Event.create(json.loads(event_string), self.hostname, self.content_service, action_logger)
+                event = Event.create(json.loads(event_string), self.content_service, action_logger)
                 start_time = time.time()
                 action_logger.debug(f"Processing event for did {event.context.did}")
 
@@ -279,14 +323,6 @@ class Plugin(object):
                                          f"Action attempted to look up element {e.index + 1} (index {e.index}) from "
                                          f"content list of size {e.size}",
                                          f"{str(e)}\n{traceback.format_exc()}")
-                except MissingDomainException as e:
-                    result = ErrorResult(event.context,
-                                         f"Action attempted to access domain {e.name}, which does not exist",
-                                         f"{str(e)}\n{traceback.format_exc()}")
-                except MissingEnrichmentException as e:
-                    result = ErrorResult(event.context,
-                                         f"Action attempted to access enrichment {e.name}, which does not exist",
-                                         f"{str(e)}\n{traceback.format_exc()}")
                 except MissingMetadataException as e:
                     result = ErrorResult(event.context,
                                          f"Missing metadata with key {e.key}",
@@ -297,16 +333,10 @@ class Plugin(object):
 
                 action.action_execution = None
 
-                response = {
-                    'did': event.context.did,
-                    'action': event.context.action_flow + "." + event.context.action_name,
-                    'start': start_time,
-                    'stop': time.time(),
-                    'type': result.result_type,
-                    'metrics': [metric.json() for metric in result.metrics]
-                }
-                if result.result_key is not None:
-                    response[result.result_key] = result.response()
+                response = Plugin.to_response(
+                    event, start_time, time.time(), result)
+
+                Plugin.orphaned_content_check(action_logger, event.context, result, response)
 
                 topic = 'dgs'
                 if event.return_address:
@@ -315,3 +345,42 @@ class Plugin(object):
             except BaseException as e:
                 action_logger.error(f"Unexpected {type(e)} error: {str(e)}\n{traceback.format_exc()}")
                 time.sleep(1)
+
+    @staticmethod
+    def orphaned_content_check(logger, context, result, response):
+        if len(context.saved_content) > 0:
+            to_delete = Plugin.find_unused_content(context.saved_content, result)
+            if len(to_delete) > 0:
+                errors = context.content_service.delete_all(to_delete)
+                for e in errors:
+                    logger.error(f"Unable to delete object(s), {e}")
+                logger.warning(
+                    f"Deleted {len(to_delete)} unused content entries for did {context.did} due to a {response['type']} event by {response['actionName']}")
+
+    @staticmethod
+    def find_unused_content(saved_content, result):
+        segments_in_use = Plugin.used_segment_names(result)
+        saved_segments = Plugin.get_segment_names(saved_content)
+        to_delete = []
+        for key, value in saved_segments.items():
+            if key not in segments_in_use:
+                to_delete.append(value)
+        return to_delete
+
+    @staticmethod
+    def used_segment_names(result):
+        segment_names = {}
+        if isinstance(result, TransformResult):
+            segment_names.update(result.get_segment_names())
+        elif isinstance(result, TransformResults):
+            segment_names.update(result.get_segment_names())
+        elif isinstance(result, IngressResult):
+            segment_names.update(result.get_segment_names())
+        return segment_names
+
+    @staticmethod
+    def get_segment_names(content_list):
+        segment_names = {}
+        for content in content_list:
+            segment_names.update(content.get_segment_names())
+        return segment_names