deltafi 2.0rc1730772574854__py3-none-any.whl → 2.1.0__py3-none-any.whl

deltafi/domain.py

@@ -20,6 +20,7 @@ import copy
 from datetime import datetime, timedelta, timezone
 from logging import Logger
 from typing import Dict, List, NamedTuple
+from uuid import uuid4
 
 from deltafi.storage import ContentService, Segment
 
@@ -119,6 +120,9 @@ class Context(NamedTuple):
                        saved_content=[],
                        logger=logger)
 
+    def child_context(self):
+        return self._replace(did=str(uuid4()))
+
 
 class Content:
     """

deltafi/plugin.py

@@ -32,6 +32,7 @@ from pathlib import Path
 from typing import List
 
 import requests
+import yaml
 from deltafi.action import Action, Join
 from deltafi.actioneventqueue import ActionEventQueue
 from deltafi.domain import Event, ActionExecution
@@ -45,6 +46,75 @@ def _coordinates():
     return PluginCoordinates(os.getenv('PROJECT_GROUP'), os.getenv('PROJECT_NAME'), os.getenv('PROJECT_VERSION'))
 
 
+def _valid_file(filename: str):
+    return isfile(filename) and \
+        (filename.endswith(".json")
+         or filename.endswith(".yaml")
+         or filename.endswith(".yml"))
+
+
+def _read_valid_files(path: str):
+    """
+    Read the contents of a directory, and returns a filtered list of files
+    that can be read/parsed for plugin usage, and ignores everything else.
+    :param path: name of the directory to scan
+    :return: list of filtered, parsable files
+    """
+    files = []
+    if isdir(path):
+        files = [f for f in os.listdir(path) if _valid_file(join(path, f))]
+    return files
+
+
+def _load_resource(path: str, filename: str):
+    """
+    Read the content of a JSON or YAML file, and return a Python
+    object of its contents, typically as a dict or list.
+    To avoid exceptions, use only files returned by _read_valid_files().
+    :param path: directory which contains the file to load
+    :param filename: name of the file to load
+    :return: dict or list of file contents
+    """
+    with open(join(path, filename)) as file_in:
+        if filename.endswith(".json"):
+            return json.load(file_in)
+        elif filename.endswith(".yaml") or filename.endswith(".yml"):
+            results = []
+            yaml_docs = yaml.safe_load_all(file_in)
+            for doc_iter in yaml_docs:
+                # yaml_docs must be iterated
+                results.append(doc_iter)
+            if len(results) == 1:
+                # Single document YAML file
+                return results[0]
+            else:
+                # Multi-document YAML file
+                return results
+    raise RuntimeError(f"File type not supported: {filename}")
+
+
+def _load__all_resource(path: str, file_list: List[str]):
+    resources = []
+    for f in file_list:
+        r = _load_resource(path, f)
+        if isinstance(r, list):
+            resources.extend(r)
+        else:
+            resources.append(r)
+    return resources
+
+
+def _find_variables_filename(names: List[str]):
+    if 'variables.json' in names:
+        return 'variables.json'
+    elif 'variables.yaml' in names:
+        return 'variables.yaml'
+    elif 'variables.yml' in names:
+        return 'variables.yml'
+    else:
+        return None
+
+
 def _setup_queue(max_connections):
     url = os.getenv('VALKEY_URL', 'http://deltafi-valkey-master:6379')
     password = os.getenv('VALKEY_PASSWORD')
@@ -182,22 +252,40 @@ class Plugin(object):
             'docsMarkdown': self._load_action_docs(action)
         }
 
+    @staticmethod
+    def load_integration_tests(tests_path: str):
+        test_files = _read_valid_files(tests_path)
+        return _load__all_resource(tests_path, test_files)
+
+    @staticmethod
+    def load_variables(flows_path: str, flow_files: List[str]):
+        variables = []
+        variables_filename = _find_variables_filename(flow_files)
+        if variables_filename is not None:
+            flow_files.remove(variables_filename)
+            variables = _load__all_resource(flows_path, [variables_filename])
+        return variables
+
     def registration_json(self):
         flows_path = str(Path(os.path.dirname(os.path.abspath(sys.argv[0]))) / 'flows')
+        tests_path = str(Path(os.path.dirname(os.path.abspath(sys.argv[0]))) / 'integration')
 
-        flow_files = []
         variables = []
-        if isdir(flows_path):
-            flow_files = [f for f in os.listdir(flows_path) if isfile(join(flows_path, f))]
-            if 'variables.json' in flow_files:
-                flow_files.remove('variables.json')
-                variables = json.load(open(join(flows_path, 'variables.json')))
+        flow_files = _read_valid_files(flows_path)
+        if len(flow_files) == 0:
+            self.logger.warning(
+                f"Flows directory ({flows_path}) does not exist or contains no valid files. No flows will be installed.")
         else:
-            self.logger.warning(f"Flows directory ({flows_path}) does not exist. No flows will be installed.")
+            variables = self.load_variables(flows_path, flow_files)
 
-        flows = [json.load(open(join(flows_path, f))) for f in flow_files]
+        flows = _load__all_resource(flows_path, flow_files)
         actions = [self._action_json(action) for action in self.actions]
 
+        test_files = self.load_integration_tests(tests_path)
+        if len(test_files) == 0:
+            self.logger.warning(
+                f"tests directory ({tests_path}) does not exist or contains no valid files. No tests will be installed.")
+
         return {
             'pluginCoordinates': self.coordinates.__json__(),
             'displayName': self.display_name,
@@ -208,7 +296,8 @@ class Plugin(object):
             'dependencies': [],
             'actions': actions,
             'variables': variables,
-            'flowPlans': flows
+            'flowPlans': flows,
+            'integrationTests': test_files
         }
 
     def _register(self):

deltafi/result.py

@@ -239,6 +239,7 @@ class TransformResult(Result):
 
     def json(self):
         return {
+            'did': self.context.did,
             'content': [content.json() for content in self.content],
             'annotations': self.annotations,
             'metadata': self.metadata,
@@ -249,38 +250,38 @@ class TransformResult(Result):
         return [self.json()]
 
 
-class NamedTransformResult(NamedTuple):
-    result: TransformResult
-    name: str
+class ChildTransformResult(TransformResult):
+    delta_file_name: str
 
-    def get_segment_names(self):
-        return self.result.get_segment_names()
+    def __init__(self, context: Context, delta_file_name: str = None):
+        super().__init__(context.child_context())
+        self.delta_file_name = delta_file_name
 
     def json(self):
-        j = self.result.json()
-        if self.name is not None:
-            j['name'] = self.name
+        j = super().json()
+        if self.delta_file_name is not None:
+            j['name'] = self.delta_file_name
         return j
 
 
 class TransformResults(Result):
     def __init__(self, context: Context):
         super().__init__('transform', 'TRANSFORM', context)
-        self.named_results = []
+        self.child_results = []
 
-    def add_result(self, result: TransformResult, name: str = None):
-        self.named_results.append(NamedTransformResult(result, name))
+    def add_result(self, result: ChildTransformResult):
+        self.child_results.append(result)
         return self
 
     def get_segment_names(self):
         segment_names = {}
-        for named_result in self.named_results:
-            segment_names.update(named_result.get_segment_names())
+        for child_result in self.child_results:
+            segment_names.update(child_result.get_segment_names())
         return segment_names
 
     def response(self):
         transform_events = []
-        for named_result in self.named_results:
-            json_dict = named_result.json()
+        for child_result in self.child_results:
+            json_dict = child_result.json()
             transform_events.append(json_dict)
         return transform_events

deltafi/test_kit/framework.py

@@ -137,6 +137,7 @@ class TestCaseBase(ABC):
         - compare_tool: (optional) CompareHelper instanced for comparing output content
         - inputs: (optional) List[IOContent]: input content to action
         - parameters: (optional) Dict: map of action input parameters
+        - in_memo: (optional) str: Input 'memo' value for a TimedIngress context
         - in_meta: (optional) Dict: map of metadata as input to action
         - join_meta: (optional): List[Dict]: When a List is provided, this enables the JOIN portion of an action.
         When using JOIN, join_meta must match the size of inputs, though the Dict can be empty
@@ -161,6 +162,7 @@ class TestCaseBase(ABC):
         self.file_name = data["file_name"] if "file_name" in data else "filename"
         self.parameters = data["parameters"] if "parameters" in data else {}
         self.in_meta = data["in_meta"] if "in_meta" in data else {}
+        self.in_memo = data["in_memo"] if "in_memo" in data else None
         self.use_did = data["did"] if "did" in data else None
         self.expected_result_type = None
         self.err_or_filt_cause = None
@@ -280,6 +282,7 @@ class ActionTest(ABC):
             content_service=self.content_service,
             saved_content=[],
             join=join,
+            memo=test_case.in_memo,
             logger=get_logger())
         return self.context
 

deltafi/test_kit/timed_ingress.py

@@ -0,0 +1,96 @@
+#
+#    DeltaFi - Data transformation and enrichment platform
+#
+#    Copyright 2021-2024 DeltaFi Contributors <deltafi@deltafi.org>
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+from typing import List
+
+from deltafi.result import IngressResult, IngressResultItem, IngressStatusEnum
+
+from .assertions import *
+from .framework import TestCaseBase, ActionTest, IOContent
+
+
+class TimedIngressTestCase(TestCaseBase):
+    def __init__(self, fields: Dict):
+        super().__init__(fields)
+        self.memo = None
+        self.results = []
+        self.execute_immediate = False
+        self.status = IngressStatusEnum.HEALTHY
+        self.status_message = None
+
+    def expect_ingress_result(self,
+                              memo: str = None,
+                              exec_immed: bool = False,
+                              status: IngressStatusEnum = IngressStatusEnum.HEALTHY,
+                              status_message: str = None):
+        self.expected_result_type = IngressResult
+        self.memo = memo
+        self.execute_immediate = exec_immed
+        self.status = status
+        self.status_message = status_message
+
+    def add_ingress_result_item(self, content: List[IOContent], metadata: Dict, name: str = None):
+        self.results.append(
+            {
+                'content': content,
+                'metadata': metadata,
+                'name': name
+            }
+        )
+
+
+class TimedIngressActionTest(ActionTest):
+    def __init__(self, package_name: str):
+        """
+        Provides structure for testing DeltaFi TimedIngress action
+        Args:
+            package_name: name of the actions package for finding resources
+        """
+        super().__init__(package_name)
+
+    def ingress(self, test_case: TimedIngressTestCase):
+        if test_case.expected_result_type == IngressResult:
+            self.expect_ingress_result(test_case)
+        else:
+            super().execute(test_case)
+
+    def expect_ingress_result(self, test_case: TimedIngressTestCase):
+        result = super().run_and_check_result_type(test_case, IngressResult)
+        self.assert_ingress_result(test_case, result)
+
+    def assert_ingress_result(self, test_case: TimedIngressTestCase, result: IngressResult):
+        assert_equal(test_case.memo, result.memo)
+        assert_equal(test_case.execute_immediate, result.execute_immediate)
+        assert_equal(test_case.status, result.status)
+        assert_equal(test_case.status_message, result.status_message)
+
+        assert_equal_len(test_case.results, result.ingress_result_items)
+        for index, ingress_item in enumerate(result.ingress_result_items):
+            self.compare_one_ingress_item(test_case, ingress_item, index)
+            expected = test_case.results[index]
+            if 'name' in expected:
+                assert_equal_with_label(expected["name"], ingress_item.delta_file_name, f"name[{index}]")
+
+    def compare_one_ingress_item(self, test_case: TimedIngressTestCase, result: IngressResultItem, index: int):
+        expected = test_case.results[index]
+
+        # Check output
+        self.compare_content_list(test_case.compare_tool, expected['content'], result.content)
+
+        # Check metadata
+        assert_keys_and_values(expected['metadata'], result.metadata)

deltafi/test_kit/transform.py

@@ -74,12 +74,12 @@ class TransformActionTest(ActionTest):
         self.assert_transform_results(test_case, result)
 
     def assert_transform_results(self, test_case: TransformTestCase, result: TransformResults):
-        assert_equal_len(test_case.results, result.named_results)
-        for index, named_result in enumerate(result.named_results):
-            self.compare_one_transform_result(test_case, named_result.result, index)
+        assert_equal_len(test_case.results, result.child_results)
+        for index, child_result in enumerate(result.child_results):
+            self.compare_one_transform_result(test_case, child_result, index)
             expected = test_case.results[index]
             if 'name' in expected:
-                assert_equal_with_label(expected["name"], named_result.name, f"name[{index}]")
+                assert_equal_with_label(expected["name"], child_result.delta_file_name, f"name[{index}]")
 
     def assert_transform_result(self, test_case: TransformTestCase, result: TransformResult):
         # Check metrics

{deltafi-2.0rc1730772574854.dist-info → deltafi-2.1.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltafi
-Version: 2.0rc1730772574854
+Version: 2.1.0
 Summary: SDK for DeltaFi plugins and actions
 License: Apache License, Version 2.0
 Keywords: deltafi
@@ -19,13 +19,14 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Software Development
+Requires-Dist: PyYAML (==6.0.2)
 Requires-Dist: deepdiff (==6.7.1)
 Requires-Dist: json-logging (==1.3.0)
-Requires-Dist: minio (==7.2.7)
-Requires-Dist: pydantic (==2.8.2)
-Requires-Dist: redis (==5.0.7)
+Requires-Dist: minio (==7.2.10)
+Requires-Dist: pydantic (==2.9.2)
+Requires-Dist: redis (==5.2.0)
 Requires-Dist: requests (==2.32.3)
-Requires-Dist: urllib3 (==2.2.2)
+Requires-Dist: urllib3 (==2.2.3)
 Project-URL: Bug Reports, https://chat.deltafi.org/deltafi/channels/bug-reports
 Project-URL: Documentation, https://docs.deltafi.org/#/
 Project-URL: Source Code, https://gitlab.com/deltafi/deltafi

{deltafi-2.0rc1730772574854.dist-info → deltafi-2.1.0.dist-info}/RECORD

@@ -2,22 +2,23 @@ deltafi/__init__.py,sha256=sSGWjpvzcCzgLywoEg55z_0satt0g_LPTwNIWtylgF4,709
 deltafi/action.py,sha256=kuDY7-qcJsiRTB3ZigiYg-XOcNeahnDU3NGSZBy10vM,5753
 deltafi/actioneventqueue.py,sha256=_05aDeqVpUbGWG3jFf0AalOsVfAvOTn0oBxjpQIQxIs,2847
 deltafi/actiontype.py,sha256=CpVI0wk9C-eu44e0jYvqBzE6yIX_zfRrW5RCf4ohu4Y,913
-deltafi/domain.py,sha256=YIfkxnjIuDXRlEm0MvrKh0CY5KczTZqZiTmnhc7wkOo,12082
+deltafi/domain.py,sha256=wCWqCjWbQ02thAMHYXTuUMHY58QZyyTRkYNZFd3kPPo,12182
 deltafi/exception.py,sha256=LNNFqc_lA89I-RvnVsbTQ2vnWQ9WZXdQqLwiXwNgc_w,943
 deltafi/genericmodel.py,sha256=WU8zfqEO_n84CZ0KpH9FhgTsL9jyU0EXSuhL0IdLWFw,1152
 deltafi/input.py,sha256=ydAhuw68N9qMeacc9YE4U79zRqxWiinZNkm1AxoFBEk,1656
 deltafi/logger.py,sha256=mKfJTnuupf3sto6hV-SIXajtcP-xTSSdJ2Ufd-lJPTo,2140
 deltafi/metric.py,sha256=79Gb2C1qYXeQYshgFPu2BcMT5oEb0LhmNfwyulSqsIc,972
-deltafi/plugin.py,sha256=FzzUEahdpWzGCULf7qhkfBsxZg4YgHGthq2wg0d2Ajc,14952
-deltafi/result.py,sha256=Xv156aM-PgYNB1Y6L-0Wl22YMj46srt88aAXtgc0U74,8957
+deltafi/plugin.py,sha256=QQ197MWyi3dCkax6rjA-kylbfMrMWHEaG8jIVsOGRW4,17980
+deltafi/result.py,sha256=2mefttQAdvYhsw6IjvouY4L5W1n3s6tP7oJUhLtyyVk,9047
 deltafi/storage.py,sha256=vCE29Yzk5s7ub4O1PdaxXBKPBQy4OHs7j46z3O5MezE,2979
 deltafi/test_kit/__init__.py,sha256=sSGWjpvzcCzgLywoEg55z_0satt0g_LPTwNIWtylgF4,709
 deltafi/test_kit/assertions.py,sha256=MdUXENLn0aDvknMtsnSAb-DwvpzXlMkaYQ6-RRkWG8s,1378
 deltafi/test_kit/compare_helpers.py,sha256=qRINvCQqBUaalT1DG2oz2egSkUjiSOfqKF5U7WyeT_g,12957
 deltafi/test_kit/constants.py,sha256=Suygx9CEob2Skw4UyzzMCibQ8hRhGHC_d_Xab8AJMFE,833
 deltafi/test_kit/egress.py,sha256=53SBeJJmphDl-jZdqkC1dKYWZ4ATWlzY02TUt0Sj1zs,1899
-deltafi/test_kit/framework.py,sha256=g7NRl_KxUFio8aZ1xo1gEBKqL9qquYx9qAyFpPLHsmM,15150
-deltafi/test_kit/transform.py,sha256=bnV6kcOkuShi_9uCtzkWcgz-wMejnI9Q2_MbyVm_A7U,4085
-deltafi-2.0rc1730772574854.dist-info/METADATA,sha256=2INpnVNi_YzmOEk2MWkTiRE9jYfi2tbHEezzTLIGGPs,1497
-deltafi-2.0rc1730772574854.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-deltafi-2.0rc1730772574854.dist-info/RECORD,,
+deltafi/test_kit/framework.py,sha256=6txr5vnLSFIDaXgx-cXjDuYJSPcSHR7ez1x4N8TsNNw,15337
+deltafi/test_kit/timed_ingress.py,sha256=ztE57hCuoZTrtFn_HlyHpHMFJN5Oyo3pPhVfbAI_weM,3790
+deltafi/test_kit/transform.py,sha256=Qj9LIyZ8RYj0tCeq7agrDlEkkgtBR86TNinV9oncTnM,4089
+deltafi-2.1.0.dist-info/METADATA,sha256=3NnbYSKHSIZx_m3Gh9wRBQyy3v0wzNV_QHnE5tQlCMs,1517
+deltafi-2.1.0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+deltafi-2.1.0.dist-info/RECORD,,