deltafi 2.0rc1705024454242__py3-none-any.whl → 2.4.0__py3-none-any.whl

deltafi/plugin.py

@@ -1,7 +1,7 @@
 #
 #    DeltaFi - Data transformation and enrichment platform
 #
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#    Copyright 2021-2025 DeltaFi Contributors <deltafi@deltafi.org>
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -16,40 +16,109 @@
 #    limitations under the License.
 #
 
+import importlib
+import inspect
 import json
 import os
+import pkgutil
 import sys
 import threading
 import time
 import traceback
 from datetime import datetime, timezone, timedelta
+from importlib import metadata
 from os.path import isdir, isfile, join
 from pathlib import Path
 from typing import List
-import importlib
-import inspect
-import pkgutil
 
-from importlib import metadata
 import requests
+import yaml
+from deltafi.action import Action, Join
 from deltafi.actioneventqueue import ActionEventQueue
 from deltafi.domain import Event, ActionExecution
-from deltafi.exception import ExpectedContentException, MissingDomainException, MissingEnrichmentException, \
-    MissingMetadataException
+from deltafi.exception import ExpectedContentException, MissingMetadataException
 from deltafi.logger import get_logger
-from deltafi.result import ErrorResult
+from deltafi.result import ErrorResult, IngressResult, TransformResult, TransformResults
 from deltafi.storage import ContentService
-from deltafi.action import Action
 
 
 def _coordinates():
     return PluginCoordinates(os.getenv('PROJECT_GROUP'), os.getenv('PROJECT_NAME'), os.getenv('PROJECT_VERSION'))
 
 
+def _valid_file(filename: str):
+    return isfile(filename) and \
+        (filename.endswith(".json")
+         or filename.endswith(".yaml")
+         or filename.endswith(".yml"))
+
+
+def _read_valid_files(path: str):
+    """
+    Read the contents of a directory, and returns a filtered list of files
+    that can be read/parsed for plugin usage, and ignores everything else.
+    :param path: name of the directory to scan
+    :return: list of filtered, parsable files
+    """
+    files = []
+    if isdir(path):
+        files = [f for f in os.listdir(path) if _valid_file(join(path, f))]
+    return files
+
+
+def _load_resource(path: str, filename: str):
+    """
+    Read the content of a JSON or YAML file, and return a Python
+    object of its contents, typically as a dict or list.
+    To avoid exceptions, use only files returned by _read_valid_files().
+    :param path: directory which contains the file to load
+    :param filename: name of the file to load
+    :return: dict or list of file contents
+    """
+    with open(join(path, filename)) as file_in:
+        if filename.endswith(".json"):
+            return json.load(file_in)
+        elif filename.endswith(".yaml") or filename.endswith(".yml"):
+            results = []
+            yaml_docs = yaml.safe_load_all(file_in)
+            for doc_iter in yaml_docs:
+                # yaml_docs must be iterated
+                results.append(doc_iter)
+            if len(results) == 1:
+                # Single document YAML file
+                return results[0]
+            else:
+                # Multi-document YAML file
+                return results
+    raise RuntimeError(f"File type not supported: {filename}")
+
+
+def _load__all_resource(path: str, file_list: List[str]):
+    resources = []
+    for f in file_list:
+        r = _load_resource(path, f)
+        if isinstance(r, list):
+            resources.extend(r)
+        else:
+            resources.append(r)
+    return resources
+
+
+def _find_variables_filename(names: List[str]):
+    if 'variables.json' in names:
+        return 'variables.json'
+    elif 'variables.yaml' in names:
+        return 'variables.yaml'
+    elif 'variables.yml' in names:
+        return 'variables.yml'
+    else:
+        return None
+
+
 def _setup_queue(max_connections):
-    redis_url = os.getenv('REDIS_URL', 'http://deltafi-redis-master:6379')
-    password = os.getenv('REDIS_PASSWORD')
-    return ActionEventQueue(redis_url, max_connections, password)
+    url = os.getenv('VALKEY_URL', 'http://deltafi-valkey-master:6379')
+    password = os.getenv('VALKEY_PASSWORD')
+    return ActionEventQueue(url, max_connections, password)
 
 
 def _setup_content_service():
@@ -94,6 +163,8 @@ class Plugin(object):
         self.queue = None
         self.actions = []
         self.core_url = os.getenv('CORE_URL')
+        self.image = os.getenv('IMAGE')
+        self.image_pull_secret = os.getenv('IMAGE_PULL_SECRET')
         action_classes = []
         if actions is not None and len(actions):
             action_classes.extend(actions)
@@ -160,40 +231,74 @@ class Plugin(object):
     def action_name(self, action):
         return f"{self.coordinates.group_id}.{action.__class__.__name__}"
 
+    def _load_action_docs(self, action):
+        docs_path = str(Path(os.path.dirname(os.path.abspath(sys.argv[0]))) / 'docs')
+        if not isdir(docs_path):
+            return None
+
+        action_docs_file = join(docs_path, action.__class__.__name__ + '.md')
+        if not isfile(action_docs_file):
+            return None
+
+        return open(action_docs_file).read()
+
     def _action_json(self, action):
         return {
             'name': self.action_name(action),
             'description': action.description,
             'type': action.action_type.name,
-            'schema': action.param_class().model_json_schema()
+            'supportsJoin': isinstance(action, Join),
+            'schema': action.param_class().model_json_schema(),
+            'docsMarkdown': self._load_action_docs(action)
         }
 
+    @staticmethod
+    def load_integration_tests(tests_path: str):
+        test_files = _read_valid_files(tests_path)
+        return _load__all_resource(tests_path, test_files)
+
+    @staticmethod
+    def load_variables(flows_path: str, flow_files: List[str]):
+        variables = []
+        variables_filename = _find_variables_filename(flow_files)
+        if variables_filename is not None:
+            flow_files.remove(variables_filename)
+            variables = _load__all_resource(flows_path, [variables_filename])
+        return variables
+
     def registration_json(self):
         flows_path = str(Path(os.path.dirname(os.path.abspath(sys.argv[0]))) / 'flows')
+        tests_path = str(Path(os.path.dirname(os.path.abspath(sys.argv[0]))) / 'integration')
 
-        flow_files = []
         variables = []
-        if isdir(flows_path):
-            flow_files = [f for f in os.listdir(flows_path) if isfile(join(flows_path, f))]
-            if 'variables.json' in flow_files:
-                flow_files.remove('variables.json')
-                variables = json.load(open(join(flows_path, 'variables.json')))
+        flow_files = _read_valid_files(flows_path)
+        if len(flow_files) == 0:
+            self.logger.warning(
+                f"Flows directory ({flows_path}) does not exist or contains no valid files. No flows will be installed.")
         else:
-            self.logger.warning(f"Flows directory ({flows_path}) does not exist. No flows will be installed.")
+            variables = self.load_variables(flows_path, flow_files)
 
-        flows = [json.load(open(join(flows_path, f))) for f in flow_files]
+        flows = _load__all_resource(flows_path, flow_files)
         actions = [self._action_json(action) for action in self.actions]
 
+        test_files = self.load_integration_tests(tests_path)
+        if len(test_files) == 0:
+            self.logger.warning(
+                f"tests directory ({tests_path}) does not exist or contains no valid files. No tests will be installed.")
+
         return {
-                'pluginCoordinates': self.coordinates.__json__(),
-                'displayName': self.display_name,
-                'description': self.description,
-                'actionKitVersion': metadata.version('deltafi'),
-                'dependencies': [],
-                'actions': actions,
-                'variables': variables,
-                'flowPlans': flows
-            }
+            'pluginCoordinates': self.coordinates.__json__(),
+            'displayName': self.display_name,
+            'description': self.description,
+            'actionKitVersion': metadata.version('deltafi'),
+            'image': self.image,
+            'imagePullSecret': self.image_pull_secret,
+            'dependencies': [],
+            'actions': actions,
+            'variables': variables,
+            'flowPlans': flows,
+            'integrationTests': test_files
+        }
 
     def _register(self):
         url = f"{self.core_url}/plugins"
@@ -217,7 +322,8 @@ class Plugin(object):
         for action in self.actions:
             threading.Thread(target=self._do_action, args=(action,)).start()
 
-        threading.Thread(target=self._heartbeat).start()
+        hb_thread = threading.Thread(target=self._heartbeat)
+        hb_thread.start()
 
         self.logger.info("All threads running")
 
@@ -225,6 +331,7 @@ class Plugin(object):
         f.close()
 
         self.logger.info("Application initialization complete")
+        hb_thread.join()
 
     def _heartbeat(self):
         long_running_actions = set()
@@ -254,6 +361,22 @@ class Plugin(object):
             finally:
                 time.sleep(10)
 
+    @staticmethod
+    def to_response(event, start_time, stop_time, result):
+        response = {
+            'did': event.context.did,
+            'flowName': event.context.flow_name,
+            'flowId': event.context.flow_id,
+            'actionName': event.context.action_name,
+            'start': start_time,
+            'stop': stop_time,
+            'type': result.result_type,
+            'metrics': [metric.json() for metric in result.metrics]
+        }
+        if result.result_key is not None:
+            response[result.result_key] = result.response()
+        return response
+
     def _do_action(self, action):
         action_logger = get_logger(self.action_name(action))
 
@@ -261,7 +384,7 @@ class Plugin(object):
         while True:
             try:
                 event_string = self.queue.take(self.action_name(action))
-                event = Event.create(json.loads(event_string), self.hostname, self.content_service, action_logger)
+                event = Event.create(json.loads(event_string), self.content_service, action_logger)
                 start_time = time.time()
                 action_logger.debug(f"Processing event for did {event.context.did}")
 
@@ -275,14 +398,6 @@ class Plugin(object):
                                          f"Action attempted to look up element {e.index + 1} (index {e.index}) from "
                                          f"content list of size {e.size}",
                                          f"{str(e)}\n{traceback.format_exc()}")
-                except MissingDomainException as e:
-                    result = ErrorResult(event.context,
-                                         f"Action attempted to access domain {e.name}, which does not exist",
-                                         f"{str(e)}\n{traceback.format_exc()}")
-                except MissingEnrichmentException as e:
-                    result = ErrorResult(event.context,
-                                         f"Action attempted to access enrichment {e.name}, which does not exist",
-                                         f"{str(e)}\n{traceback.format_exc()}")
                 except MissingMetadataException as e:
                     result = ErrorResult(event.context,
                                          f"Missing metadata with key {e.key}",
@@ -293,16 +408,10 @@ class Plugin(object):
 
                 action.action_execution = None
 
-                response = {
-                    'did': event.context.did,
-                    'action': event.context.action_flow + "." + event.context.action_name,
-                    'start': start_time,
-                    'stop': time.time(),
-                    'type': result.result_type,
-                    'metrics': [metric.json() for metric in result.metrics]
-                }
-                if result.result_key is not None:
-                    response[result.result_key] = result.response()
+                response = Plugin.to_response(
+                    event, start_time, time.time(), result)
+
+                Plugin.orphaned_content_check(action_logger, event.context, result, response)
 
                 topic = 'dgs'
                 if event.return_address:
@@ -311,3 +420,42 @@ class Plugin(object):
             except BaseException as e:
                 action_logger.error(f"Unexpected {type(e)} error: {str(e)}\n{traceback.format_exc()}")
                 time.sleep(1)
+
+    @staticmethod
+    def orphaned_content_check(logger, context, result, response):
+        if len(context.saved_content) > 0:
+            to_delete = Plugin.find_unused_content(context.saved_content, result)
+            if len(to_delete) > 0:
+                errors = context.content_service.delete_all(to_delete)
+                for e in errors:
+                    logger.error(f"Unable to delete object(s), {e}")
+                logger.warning(
+                    f"Deleted {len(to_delete)} unused content entries for did {context.did} due to a {response['type']} event by {response['actionName']}")
+
+    @staticmethod
+    def find_unused_content(saved_content, result):
+        segments_in_use = Plugin.used_segment_names(result)
+        saved_segments = Plugin.get_segment_names(saved_content)
+        to_delete = []
+        for key, value in saved_segments.items():
+            if key not in segments_in_use:
+                to_delete.append(value)
+        return to_delete
+
+    @staticmethod
+    def used_segment_names(result):
+        segment_names = {}
+        if isinstance(result, TransformResult):
+            segment_names.update(result.get_segment_names())
+        elif isinstance(result, TransformResults):
+            segment_names.update(result.get_segment_names())
+        elif isinstance(result, IngressResult):
+            segment_names.update(result.get_segment_names())
+        return segment_names
+
+    @staticmethod
+    def get_segment_names(content_list):
+        segment_names = {}
+        for content in content_list:
+            segment_names.update(content.get_segment_names())
+        return segment_names

deltafi/result.py

@@ -1,7 +1,7 @@
 #
 #    DeltaFi - Data transformation and enrichment platform
 #
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#    Copyright 2021-2025 DeltaFi Contributors <deltafi@deltafi.org>
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -17,17 +17,13 @@
 #
 
 import abc
-from enum import Enum
 import uuid
-from typing import Dict, List
+from enum import Enum
+from typing import NamedTuple
 
 from deltafi.domain import Content, Context
 from deltafi.metric import Metric
 
-ENDPOINT_TAG = "endpoint"
-FILES_OUT = "files_out"
-BYTES_OUT = "bytes_out"
-
 
 class Result:
     __metaclass__ = abc.ABCMeta
@@ -44,13 +40,12 @@ class Result:
 
     def add_metric(self, metric: Metric):
         self.metrics.append(metric)
+        return self
 
 
 class EgressResult(Result):
-    def __init__(self, context: Context, destination: str, bytes_egressed: int):
+    def __init__(self, context: Context):
         super().__init__(None, 'EGRESS', context)
-        self.add_metric(Metric(FILES_OUT, 1, {ENDPOINT_TAG: destination}))
-        self.add_metric(Metric(BYTES_OUT, bytes_egressed, {ENDPOINT_TAG: destination}))
 
     def response(self):
         return None
@@ -76,7 +71,7 @@ class ErrorResult(Result):
 
 
 class FilterResult(Result):
-    def __init__(self, context: Context, filtered_cause: str, filtered_context: str=None):
+    def __init__(self, context: Context, filtered_cause: str, filtered_context: str = None):
         super().__init__('filter', 'FILTER', context)
         self.filtered_cause = filtered_cause
         self.filtered_context = filtered_context
@@ -95,12 +90,13 @@ class FilterResult(Result):
 
 
 class IngressResultItem:
-    def __init__(self, context: Context, filename: str):
+    def __init__(self, context: Context, delta_file_name: str):
         self.context = context
-        self.filename = filename
         self._did = str(uuid.uuid4())
         self.content = []
         self.metadata = {}
+        self.annotations = {}
+        self.delta_file_name = delta_file_name
 
     @property
     def did(self):
@@ -117,15 +113,17 @@ class IngressResultItem:
         return self
 
     def save_string_content(self, string_data: str, name: str, media_type: str):
-        segment = self.context.content_service.put_str(self.context.did, string_data)
-        self.content.append(
-            Content(name=name, segments=[segment], media_type=media_type, content_service=self.context.content_service))
+        segment = self.context.content_service.put_str(self._did, string_data)
+        c = Content(name=name, segments=[segment], media_type=media_type, content_service=self.context.content_service)
+        self.content.append(c)
+        self.context.saved_content.append(c)
         return self
 
     def save_byte_content(self, byte_data: bytes, name: str, media_type: str):
-        segment = self.context.content_service.put_bytes(self.context.did, byte_data)
-        self.content.append(
-            Content(name=name, segments=[segment], media_type=media_type, content_service=self.context.content_service))
+        segment = self.context.content_service.put_bytes(self._did, byte_data)
+        c = Content(name=name, segments=[segment], media_type=media_type, content_service=self.context.content_service)
+        self.content.append(c)
+        self.context.saved_content.append(c)
         return self
 
     def set_metadata(self, metadata: dict):
@@ -136,12 +134,23 @@ class IngressResultItem:
         self.metadata[key] = value
         return self
 
+    def get_segment_names(self):
+        segment_names = {}
+        for c in self.content:
+            segment_names.update(c.get_segment_names())
+        return segment_names
+
+    def annotate(self, key: str, value: str):
+                    self.annotations[key] = value
+                    return self
+
     def response(self):
         return {
             'did': self._did,
-            'filename': self.filename,
+            'deltaFileName': self.delta_file_name,
             'metadata': self.metadata,
-            'content': [content.json() for content in self.content]
+            'content': [content.json() for content in self.content],
+            'annotations': self.annotations
         }
 
 
@@ -155,22 +164,28 @@ class IngressResult(Result):
     def __init__(self, context: Context):
         super().__init__('ingress', 'INGRESS', context)
         self.memo = None
-        self.execute_immediate = False
         self.ingress_result_items = []
+        self.execute_immediate = False
         self.status = IngressStatusEnum.HEALTHY
-        self.statusMessage = None
+        self.status_message = None
 
     def add_item(self, ingress_result_item: IngressResultItem):
         self.ingress_result_items.append(ingress_result_item)
         return self
 
+    def get_segment_names(self):
+        segment_names = {}
+        for ingress_item in self.ingress_result_items:
+            segment_names.update(ingress_item.get_segment_names())
+        return segment_names
+
     def response(self):
         return {
             'memo': self.memo,
             'executeImmediate': self.execute_immediate,
             'ingressItems': [ingress_result_item.response() for ingress_result_item in self.ingress_result_items],
             'status': self.status.value,
-            'statusMessage': self.statusMessage
+            'statusMessage': self.status_message
         }
 
 
@@ -178,8 +193,8 @@ class TransformResult(Result):
     def __init__(self, context: Context):
         super().__init__('transform', 'TRANSFORM', context)
         self.content = []
-        self.metadata = {}
         self.annotations = {}
+        self.metadata = {}
         self.delete_metadata_keys = []
 
     # content can be a single Content or a List[Content]
@@ -194,14 +209,16 @@ class TransformResult(Result):
 
     def save_string_content(self, string_data: str, name: str, media_type: str):
         segment = self.context.content_service.put_str(self.context.did, string_data)
-        self.content.append(
-            Content(name=name, segments=[segment], media_type=media_type, content_service=self.context.content_service))
+        c = Content(name=name, segments=[segment], media_type=media_type, content_service=self.context.content_service)
+        self.content.append(c)
+        self.context.saved_content.append(c)
         return self
 
     def save_byte_content(self, byte_data: bytes, name: str, media_type: str):
         segment = self.context.content_service.put_bytes(self.context.did, byte_data)
-        self.content.append(
-            Content(name=name, segments=[segment], media_type=media_type, content_service=self.context.content_service))
+        c = Content(name=name, segments=[segment], media_type=media_type, content_service=self.context.content_service)
+        self.content.append(c)
+        self.context.saved_content.append(c)
         return self
 
     def set_metadata(self, metadata: dict):
@@ -220,10 +237,57 @@ class TransformResult(Result):
         self.delete_metadata_keys.append(key)
         return self
 
-    def response(self):
+    def get_segment_names(self):
+        segment_names = {}
+        for c in self.content:
+            segment_names.update(c.get_segment_names())
+        return segment_names
+
+    def json(self):
         return {
+            'did': self.context.did,
             'content': [content.json() for content in self.content],
-            'metadata': self.metadata,
             'annotations': self.annotations,
+            'metadata': self.metadata,
             'deleteMetadataKeys': self.delete_metadata_keys
         }
+
+    def response(self):
+        return [self.json()]
+
+
+class ChildTransformResult(TransformResult):
+    delta_file_name: str
+
+    def __init__(self, context: Context, delta_file_name: str = None):
+        super().__init__(context.child_context())
+        self.delta_file_name = delta_file_name
+
+    def json(self):
+        j = super().json()
+        if self.delta_file_name is not None:
+            j['name'] = self.delta_file_name
+        return j
+
+
+class TransformResults(Result):
+    def __init__(self, context: Context):
+        super().__init__('transform', 'TRANSFORM', context)
+        self.child_results = []
+
+    def add_result(self, result: ChildTransformResult):
+        self.child_results.append(result)
+        return self
+
+    def get_segment_names(self):
+        segment_names = {}
+        for child_result in self.child_results:
+            segment_names.update(child_result.get_segment_names())
+        return segment_names
+
+    def response(self):
+        transform_events = []
+        for child_result in self.child_results:
+            json_dict = child_result.json()
+            transform_events.append(json_dict)
+        return transform_events

deltafi/storage.py

@@ -1,7 +1,7 @@
 #
 #    DeltaFi - Data transformation and enrichment platform
 #
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#    Copyright 2021-2025 DeltaFi Contributors <deltafi@deltafi.org>
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -22,6 +22,7 @@ from typing import List, NamedTuple
 from urllib.parse import urlparse
 
 import minio
+from minio.deleteobjects import DeleteObject
 
 BUCKET = 'storage'
 
@@ -86,3 +87,7 @@ class ContentService:
 
     def put_str(self, did, string_data):
         return self.put_bytes(did, string_data.encode('utf-8'))
+
+    def delete_all(self, segments: List[Segment]):
+        delete_objects = [DeleteObject(seg.id()) for seg in segments]
+        return self.minio_client.remove_objects(BUCKET, delete_objects)

deltafi/test_kit/__init__.py

@@ -1,7 +1,7 @@
 #
 #    DeltaFi - Data transformation and enrichment platform
 #
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#    Copyright 2021-2025 DeltaFi Contributors <deltafi@deltafi.org>
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.

deltafi/test_kit/assertions.py

@@ -1,7 +1,7 @@
 #
 #    DeltaFi - Data transformation and enrichment platform
 #
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#    Copyright 2021-2025 DeltaFi Contributors <deltafi@deltafi.org>
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -29,10 +29,18 @@ def assert_equal_with_label(e, a, l):
     assert e == a, f"{l}. Expected:\n<<{e}>>\nBut was:\n<<{a}>>"
 
 
+def assert_equal_short(e, a, l):
+    assert e == a, f"{l}. E:{e}, A::{a}"
+
+
 def assert_equal_len(e, a):
     assert len(e) == len(a), f"{len(e)} != {len(a)}"
 
 
+def assert_equal_len_with_label(e, a, l):
+    assert len(e) == len(a), f"{l}. {len(e)} != {len(a)}"
+
+
 def assert_key_in(k, m):
     assert k in m, f"{k} not found"
 
@@ -45,4 +53,4 @@ def assert_keys_and_values(expected: Dict, actual: Dict):
     for key in expected:
         assert_key_in(key, actual)
         if expected[key] != IGNORE_VALUE:
-            assert_equal(expected[key], actual[key])
+            assert_equal_short(expected[key], actual[key], f"invalid value for key {key}")