deltafi 2.0rc1731040010625__py3-none-any.whl → 2.1.1__py3-none-any.whl

deltafi/plugin.py

@@ -25,7 +25,6 @@ import sys
 import threading
 import time
 import traceback
-import yaml
 from datetime import datetime, timezone, timedelta
 from importlib import metadata
 from os.path import isdir, isfile, join
@@ -33,6 +32,7 @@ from pathlib import Path
 from typing import List
 
 import requests
+import yaml
 from deltafi.action import Action, Join
 from deltafi.actioneventqueue import ActionEventQueue
 from deltafi.domain import Event, ActionExecution
@@ -46,6 +46,75 @@ def _coordinates():
     return PluginCoordinates(os.getenv('PROJECT_GROUP'), os.getenv('PROJECT_NAME'), os.getenv('PROJECT_VERSION'))
 
 
+def _valid_file(filename: str):
+    return isfile(filename) and \
+        (filename.endswith(".json")
+         or filename.endswith(".yaml")
+         or filename.endswith(".yml"))
+
+
+def _read_valid_files(path: str):
+    """
+    Read the contents of a directory, and returns a filtered list of files
+    that can be read/parsed for plugin usage, and ignores everything else.
+    :param path: name of the directory to scan
+    :return: list of filtered, parsable files
+    """
+    files = []
+    if isdir(path):
+        files = [f for f in os.listdir(path) if _valid_file(join(path, f))]
+    return files
+
+
+def _load_resource(path: str, filename: str):
+    """
+    Read the content of a JSON or YAML file, and return a Python
+    object of its contents, typically as a dict or list.
+    To avoid exceptions, use only files returned by _read_valid_files().
+    :param path: directory which contains the file to load
+    :param filename: name of the file to load
+    :return: dict or list of file contents
+    """
+    with open(join(path, filename)) as file_in:
+        if filename.endswith(".json"):
+            return json.load(file_in)
+        elif filename.endswith(".yaml") or filename.endswith(".yml"):
+            results = []
+            yaml_docs = yaml.safe_load_all(file_in)
+            for doc_iter in yaml_docs:
+                # yaml_docs must be iterated
+                results.append(doc_iter)
+            if len(results) == 1:
+                # Single document YAML file
+                return results[0]
+            else:
+                # Multi-document YAML file
+                return results
+    raise RuntimeError(f"File type not supported: {filename}")
+
+
+def _load__all_resource(path: str, file_list: List[str]):
+    resources = []
+    for f in file_list:
+        r = _load_resource(path, f)
+        if isinstance(r, list):
+            resources.extend(r)
+        else:
+            resources.append(r)
+    return resources
+
+
+def _find_variables_filename(names: List[str]):
+    if 'variables.json' in names:
+        return 'variables.json'
+    elif 'variables.yaml' in names:
+        return 'variables.yaml'
+    elif 'variables.yml' in names:
+        return 'variables.yml'
+    else:
+        return None
+
+
 def _setup_queue(max_connections):
     url = os.getenv('VALKEY_URL', 'http://deltafi-valkey-master:6379')
     password = os.getenv('VALKEY_PASSWORD')
@@ -183,34 +252,40 @@ class Plugin(object):
             'docsMarkdown': self._load_action_docs(action)
         }
 
-    def _integration_tests(self):
-        tests_path = str(Path(os.path.dirname(os.path.abspath(sys.argv[0]))) / 'integration')
-
-        test_files = []
-        if isdir(tests_path):
-            test_files = [f for f in os.listdir(tests_path) if isfile(join(tests_path, f))]
-        else:
-            self.logger.warning(f"tests directory ({tests_path}) does not exist. No tests will be installed.")
+    @staticmethod
+    def load_integration_tests(tests_path: str):
+        test_files = _read_valid_files(tests_path)
+        return _load__all_resource(tests_path, test_files)
 
-        tests = [json.load(open(join(tests_path, f))) for f in test_files]
-        return tests
+    @staticmethod
+    def load_variables(flows_path: str, flow_files: List[str]):
+        variables = []
+        variables_filename = _find_variables_filename(flow_files)
+        if variables_filename is not None:
+            flow_files.remove(variables_filename)
+            variables = _load__all_resource(flows_path, [variables_filename])
+        return variables
 
     def registration_json(self):
         flows_path = str(Path(os.path.dirname(os.path.abspath(sys.argv[0]))) / 'flows')
+        tests_path = str(Path(os.path.dirname(os.path.abspath(sys.argv[0]))) / 'integration')
 
-        flow_files = []
         variables = []
-        if isdir(flows_path):
-            flow_files = [f for f in os.listdir(flows_path) if isfile(join(flows_path, f))]
-            if 'variables.json' in flow_files:
-                flow_files.remove('variables.json')
-                variables = json.load(open(join(flows_path, 'variables.json')))
+        flow_files = _read_valid_files(flows_path)
+        if len(flow_files) == 0:
+            self.logger.warning(
+                f"Flows directory ({flows_path}) does not exist or contains no valid files. No flows will be installed.")
         else:
-            self.logger.warning(f"Flows directory ({flows_path}) does not exist. No flows will be installed.")
+            variables = self.load_variables(flows_path, flow_files)
 
-        flows = [json.load(open(join(flows_path, f))) for f in flow_files]
+        flows = _load__all_resource(flows_path, flow_files)
         actions = [self._action_json(action) for action in self.actions]
 
+        test_files = self.load_integration_tests(tests_path)
+        if len(test_files) == 0:
+            self.logger.warning(
+                f"tests directory ({tests_path}) does not exist or contains no valid files. No tests will be installed.")
+
         return {
             'pluginCoordinates': self.coordinates.__json__(),
             'displayName': self.display_name,
@@ -222,7 +297,7 @@ class Plugin(object):
             'actions': actions,
             'variables': variables,
             'flowPlans': flows,
-            'integrationTests': self._integration_tests()
+            'integrationTests': test_files
         }
 
     def _register(self):

deltafi/test_kit/framework.py

@@ -137,6 +137,7 @@ class TestCaseBase(ABC):
         - compare_tool: (optional) CompareHelper instanced for comparing output content
         - inputs: (optional) List[IOContent]: input content to action
         - parameters: (optional) Dict: map of action input parameters
+        - in_memo: (optional) str: Input 'memo' value for a TimedIngress context
         - in_meta: (optional) Dict: map of metadata as input to action
         - join_meta: (optional): List[Dict]: When a List is provided, this enables the JOIN portion of an action.
         When using JOIN, join_meta must match the size of inputs, though the Dict can be empty
@@ -161,6 +162,7 @@ class TestCaseBase(ABC):
         self.file_name = data["file_name"] if "file_name" in data else "filename"
         self.parameters = data["parameters"] if "parameters" in data else {}
         self.in_meta = data["in_meta"] if "in_meta" in data else {}
+        self.in_memo = data["in_memo"] if "in_memo" in data else None
         self.use_did = data["did"] if "did" in data else None
         self.expected_result_type = None
         self.err_or_filt_cause = None
@@ -280,6 +282,7 @@ class ActionTest(ABC):
             content_service=self.content_service,
             saved_content=[],
             join=join,
+            memo=test_case.in_memo,
             logger=get_logger())
         return self.context
 

deltafi/test_kit/timed_ingress.py

@@ -0,0 +1,96 @@
+#
+#    DeltaFi - Data transformation and enrichment platform
+#
+#    Copyright 2021-2024 DeltaFi Contributors <deltafi@deltafi.org>
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+from typing import List
+
+from deltafi.result import IngressResult, IngressResultItem, IngressStatusEnum
+
+from .assertions import *
+from .framework import TestCaseBase, ActionTest, IOContent
+
+
+class TimedIngressTestCase(TestCaseBase):
+    def __init__(self, fields: Dict):
+        super().__init__(fields)
+        self.memo = None
+        self.results = []
+        self.execute_immediate = False
+        self.status = IngressStatusEnum.HEALTHY
+        self.status_message = None
+
+    def expect_ingress_result(self,
+                              memo: str = None,
+                              exec_immed: bool = False,
+                              status: IngressStatusEnum = IngressStatusEnum.HEALTHY,
+                              status_message: str = None):
+        self.expected_result_type = IngressResult
+        self.memo = memo
+        self.execute_immediate = exec_immed
+        self.status = status
+        self.status_message = status_message
+
+    def add_ingress_result_item(self, content: List[IOContent], metadata: Dict, name: str = None):
+        self.results.append(
+            {
+                'content': content,
+                'metadata': metadata,
+                'name': name
+            }
+        )
+
+
+class TimedIngressActionTest(ActionTest):
+    def __init__(self, package_name: str):
+        """
+        Provides structure for testing DeltaFi TimedIngress action
+        Args:
+            package_name: name of the actions package for finding resources
+        """
+        super().__init__(package_name)
+
+    def ingress(self, test_case: TimedIngressTestCase):
+        if test_case.expected_result_type == IngressResult:
+            self.expect_ingress_result(test_case)
+        else:
+            super().execute(test_case)
+
+    def expect_ingress_result(self, test_case: TimedIngressTestCase):
+        result = super().run_and_check_result_type(test_case, IngressResult)
+        self.assert_ingress_result(test_case, result)
+
+    def assert_ingress_result(self, test_case: TimedIngressTestCase, result: IngressResult):
+        assert_equal(test_case.memo, result.memo)
+        assert_equal(test_case.execute_immediate, result.execute_immediate)
+        assert_equal(test_case.status, result.status)
+        assert_equal(test_case.status_message, result.status_message)
+
+        assert_equal_len(test_case.results, result.ingress_result_items)
+        for index, ingress_item in enumerate(result.ingress_result_items):
+            self.compare_one_ingress_item(test_case, ingress_item, index)
+            expected = test_case.results[index]
+            if 'name' in expected:
+                assert_equal_with_label(expected["name"], ingress_item.delta_file_name, f"name[{index}]")
+
+    def compare_one_ingress_item(self, test_case: TimedIngressTestCase, result: IngressResultItem, index: int):
+        expected = test_case.results[index]
+
+        # Check output
+        self.compare_content_list(test_case.compare_tool, expected['content'], result.content)
+
+        # Check metadata
+        assert_keys_and_values(expected['metadata'], result.metadata)

{deltafi-2.0rc1731040010625.dist-info → deltafi-2.1.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltafi
-Version: 2.0rc1731040010625
+Version: 2.1.1
 Summary: SDK for DeltaFi plugins and actions
 License: Apache License, Version 2.0
 Keywords: deltafi

{deltafi-2.0rc1731040010625.dist-info → deltafi-2.1.1.dist-info}/RECORD

@@ -8,7 +8,7 @@ deltafi/genericmodel.py,sha256=WU8zfqEO_n84CZ0KpH9FhgTsL9jyU0EXSuhL0IdLWFw,1152
 deltafi/input.py,sha256=ydAhuw68N9qMeacc9YE4U79zRqxWiinZNkm1AxoFBEk,1656
 deltafi/logger.py,sha256=mKfJTnuupf3sto6hV-SIXajtcP-xTSSdJ2Ufd-lJPTo,2140
 deltafi/metric.py,sha256=79Gb2C1qYXeQYshgFPu2BcMT5oEb0LhmNfwyulSqsIc,972
-deltafi/plugin.py,sha256=7e4wblHAG41UCOZ1RcWjkw-0RJF4_dAksx5b4XCNIU4,15521
+deltafi/plugin.py,sha256=QQ197MWyi3dCkax6rjA-kylbfMrMWHEaG8jIVsOGRW4,17980
 deltafi/result.py,sha256=2mefttQAdvYhsw6IjvouY4L5W1n3s6tP7oJUhLtyyVk,9047
 deltafi/storage.py,sha256=vCE29Yzk5s7ub4O1PdaxXBKPBQy4OHs7j46z3O5MezE,2979
 deltafi/test_kit/__init__.py,sha256=sSGWjpvzcCzgLywoEg55z_0satt0g_LPTwNIWtylgF4,709
@@ -16,8 +16,9 @@ deltafi/test_kit/assertions.py,sha256=MdUXENLn0aDvknMtsnSAb-DwvpzXlMkaYQ6-RRkWG8
 deltafi/test_kit/compare_helpers.py,sha256=qRINvCQqBUaalT1DG2oz2egSkUjiSOfqKF5U7WyeT_g,12957
 deltafi/test_kit/constants.py,sha256=Suygx9CEob2Skw4UyzzMCibQ8hRhGHC_d_Xab8AJMFE,833
 deltafi/test_kit/egress.py,sha256=53SBeJJmphDl-jZdqkC1dKYWZ4ATWlzY02TUt0Sj1zs,1899
-deltafi/test_kit/framework.py,sha256=g7NRl_KxUFio8aZ1xo1gEBKqL9qquYx9qAyFpPLHsmM,15150
+deltafi/test_kit/framework.py,sha256=6txr5vnLSFIDaXgx-cXjDuYJSPcSHR7ez1x4N8TsNNw,15337
+deltafi/test_kit/timed_ingress.py,sha256=ztE57hCuoZTrtFn_HlyHpHMFJN5Oyo3pPhVfbAI_weM,3790
 deltafi/test_kit/transform.py,sha256=Qj9LIyZ8RYj0tCeq7agrDlEkkgtBR86TNinV9oncTnM,4089
-deltafi-2.0rc1731040010625.dist-info/METADATA,sha256=515J328ll7iZ6v_RfUpNTeOCPkK242WEYZjzekh63FE,1530
-deltafi-2.0rc1731040010625.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-deltafi-2.0rc1731040010625.dist-info/RECORD,,
+deltafi-2.1.1.dist-info/METADATA,sha256=TMTU52HQuV2krmuKdx-wf6_3V944bfJ6tlRhdW6jAfw,1517
+deltafi-2.1.1.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+deltafi-2.1.1.dist-info/RECORD,,