deltafi 2.0rc1719271450675__py3-none-any.whl → 2.0rc1720728217472__py3-none-any.whl

deltafi/result.py

@@ -17,9 +17,9 @@
 #
 
 import abc
-import uuid
 from enum import Enum
-from typing import NamedTuple
+import uuid
+from typing import Dict, List
 
 from deltafi.domain import Content, Context
 from deltafi.metric import Metric
@@ -44,8 +44,22 @@ class Result:
 
     def add_metric(self, metric: Metric):
         self.metrics.append(metric)
+
+
+class DomainResult(Result):
+    def __init__(self, context: Context):
+        super().__init__('domain', 'DOMAIN', context)
+        self.annotations = {}
+
+    def annotate(self, key: str, value: str):
+        self.annotations[key] = value
         return self
 
+    def response(self):
+        return {
+            'annotations': self.annotations
+        }
+
 
 class EgressResult(Result):
     def __init__(self, context: Context, destination: str, bytes_egressed: int):
@@ -57,6 +71,31 @@ class EgressResult(Result):
         return None
 
 
+class EnrichResult(Result):
+    def __init__(self, context: Context):
+        super().__init__('enrich', 'ENRICH', context)
+        self.enrichments = []
+        self.annotations = {}
+
+    def enrich(self, name: str, value: str, media_type: str):
+        self.enrichments.append({
+            'name': name,
+            'value': value,
+            'mediaType': media_type
+        })
+        return self
+
+    def annotate(self, key: str, value: str):
+        self.annotations[key] = value
+        return self
+
+    def response(self):
+        return {
+            'enrichments': self.enrichments,
+            'annotations': self.annotations
+        }
+
+
 class ErrorResult(Result):
     def __init__(self, context: Context, error_cause: str, error_context: str):
         super().__init__('error', 'ERROR', context)
@@ -77,7 +116,7 @@ class ErrorResult(Result):
 
 
 class FilterResult(Result):
-    def __init__(self, context: Context, filtered_cause: str, filtered_context: str = None):
+    def __init__(self, context: Context, filtered_cause: str, filtered_context: str=None):
         super().__init__('filter', 'FILTER', context)
         self.filtered_cause = filtered_cause
         self.filtered_context = filtered_context
@@ -95,13 +134,87 @@ class FilterResult(Result):
         }
 
 
+class FormatResult(Result):
+    def __init__(self, context: Context):
+        super().__init__('format', 'FORMAT', context)
+        self.content = None
+        self.delete_metadata_keys = []
+        self.metadata = {}
+
+    def set_metadata(self, metadata: dict):
+        self.metadata = metadata
+        return self
+
+    def add_metadata(self, key: str, value: str):
+        self.metadata[key] = value
+        return self
+
+    def delete_metadata_key(self, key: str):
+        self.delete_metadata_keys.append(key)
+        return self
+
+    def set_content(self, content: Content):
+        self.content = content
+        return self
+
+    def save_string_content(self, string_data: str, name: str, media_type: str):
+        segment = self.context.content_service.put_str(self.context.did, string_data)
+        self.content = Content(name=name, segments=[segment], media_type=media_type,
+                               content_service=self.context.content_service)
+        return self
+
+    def save_byte_content(self, byte_data: bytes, name: str, media_type: str):
+        segment = self.context.content_service.put_bytes(self.context.did, byte_data)
+        self.content = Content(name=name, segments=[segment], media_type=media_type,
+                               content_service=self.context.content_service)
+        return self
+
+    def response(self):
+        return {
+            'content': self.content.json(),
+            'metadata': self.metadata,
+            'deleteMetadataKeys': self.delete_metadata_keys
+        }
+
+
+class ChildFormatResult:
+    def __init__(self, format_result: FormatResult = None):
+        self._did = str(uuid.uuid4())
+        self.format_result = format_result
+
+    @property
+    def did(self):
+        return self._did
+
+    def response(self):
+        res = self.format_result.response()
+        res["did"] = self._did
+        return res
+
+
+class FormatManyResult(Result):
+    def __init__(self, context: Context):
+        super().__init__('formatMany', 'FORMAT_MANY', context)
+        self.format_results = []
+
+    def add_format_result(self, format_result):
+        if isinstance(format_result, ChildFormatResult):
+            self.format_results.append(format_result)
+        else:
+            self.format_results.append(ChildFormatResult(format_result))
+        return self
+
+    def response(self):
+        return [format_result.response() for format_result in self.format_results]
+
+
 class IngressResultItem:
-    def __init__(self, context: Context, delta_file_name: str):
+    def __init__(self, context: Context, filename: str):
         self.context = context
+        self.filename = filename
         self._did = str(uuid.uuid4())
         self.content = []
         self.metadata = {}
-        self.delta_file_name = delta_file_name
 
     @property
     def did(self):
@@ -140,7 +253,7 @@ class IngressResultItem:
     def response(self):
         return {
             'did': self._did,
-            'deltaFileName': self.delta_file_name,
+            'filename': self.filename,
             'metadata': self.metadata,
             'content': [content.json() for content in self.content]
         }
@@ -156,10 +269,10 @@ class IngressResult(Result):
     def __init__(self, context: Context):
         super().__init__('ingress', 'INGRESS', context)
         self.memo = None
-        self.ingress_result_items = []
         self.execute_immediate = False
+        self.ingress_result_items = []
         self.status = IngressStatusEnum.HEALTHY
-        self.status_message = None
+        self.statusMessage = None
 
     def add_item(self, ingress_result_item: IngressResultItem):
         self.ingress_result_items.append(ingress_result_item)
@@ -171,16 +284,17 @@ class IngressResult(Result):
             'executeImmediate': self.execute_immediate,
             'ingressItems': [ingress_result_item.response() for ingress_result_item in self.ingress_result_items],
             'status': self.status.value,
-            'statusMessage': self.status_message
+            'statusMessage': self.statusMessage
         }
 
 
-class TransformResult(Result):
+class LoadResult(Result):
     def __init__(self, context: Context):
-        super().__init__('transform', 'TRANSFORM', context)
+        super().__init__('load', 'LOAD', context)
         self.content = []
-        self.annotations = {}
         self.metadata = {}
+        self.domains = []
+        self.annotations = {}
         self.delete_metadata_keys = []
 
     # content can be a single Content or a List[Content]
@@ -213,6 +327,13 @@ class TransformResult(Result):
         self.metadata[key] = value
         return self
 
+    def add_domain(self, name: str, value: str, media_type: str):
+        self.domains.append({
+            'name': name,
+            'value': value,
+            'mediaType': media_type})
+        return self
+
     def annotate(self, key: str, value: str):
         self.annotations[key] = value
         return self
@@ -221,41 +342,133 @@ class TransformResult(Result):
         self.delete_metadata_keys.append(key)
         return self
 
-    def json(self):
+    def response(self):
         return {
+            'domains': self.domains,
             'content': [content.json() for content in self.content],
-            'annotations': self.annotations,
             'metadata': self.metadata,
+            'annotations': self.annotations,
             'deleteMetadataKeys': self.delete_metadata_keys
         }
 
+
+class ChildLoadResult:
+    def __init__(self, load_result: LoadResult = None):
+        self._did = str(uuid.uuid4())
+        self.load_result = load_result
+
+    @property
+    def did(self):
+        return self._did
+
+    def response(self):
+        res = self.load_result.response()
+        res["did"] = self._did
+        return res
+
+
+class LoadManyResult(Result):
+    def __init__(self, context: Context):
+        super().__init__('loadMany', 'LOAD_MANY', context)
+        self.load_results = []
+
+    def add_load_result(self, load_result):
+        if isinstance(load_result, ChildLoadResult):
+            self.load_results.append(load_result)
+        else:
+            self.load_results.append(ChildLoadResult(load_result))
+        return self
+
     def response(self):
-        return [self.json()]
+        return [load_result.response() for load_result in self.load_results]
+
+
+class ReinjectResult(Result):
+    class ReinjectChild:
+        def __init__(self, filename: str, flow: str, content: List[Content], metadata: Dict[str, str]):
+            self.filename = filename
+            self.flow = flow
+            self.content = content
+            self.metadata = metadata
 
+        def json(self):
+            return {
+                'filename': self.filename,
+                'flow': self.flow,
+                'metadata': self.metadata,
+                'content': [content.json() for content in self.content]
+            }
+
+    def __init__(self, context: Context):
+        super().__init__('reinject', 'REINJECT', context)
+        self.children = []
 
-class NamedTransformResult(NamedTuple):
-    result: TransformResult
-    name: str
+    def add_child(self, filename: str, flow: str, content: List[Content], metadata: Dict[str, str]):
+        child = ReinjectResult.ReinjectChild(filename, flow, content, metadata)
+        self.children.append(child)
 
-    def json(self):
-        j = self.result.json()
-        if self.name is not None:
-            j['name'] = self.name
-        return j
+    def response(self):
+        return [child.json() for child in self.children]
 
 
-class TransformResults(Result):
+class TransformResult(Result):
     def __init__(self, context: Context):
         super().__init__('transform', 'TRANSFORM', context)
-        self.named_results = []
+        self.content = []
+        self.metadata = {}
+        self.annotations = {}
+        self.delete_metadata_keys = []
+
+    # content can be a single Content or a List[Content]
+    def add_content(self, content):
+        if content:
+            if type(content) == list:
+                self.content.extend(content)
+            else:
+                self.content.append(content)
 
-    def add_result(self, result: TransformResult, name: str = None):
-        self.named_results.append(NamedTransformResult(result, name))
+        return self
+
+    def save_string_content(self, string_data: str, name: str, media_type: str):
+        segment = self.context.content_service.put_str(self.context.did, string_data)
+        self.content.append(
+            Content(name=name, segments=[segment], media_type=media_type, content_service=self.context.content_service))
+        return self
+
+    def save_byte_content(self, byte_data: bytes, name: str, media_type: str):
+        segment = self.context.content_service.put_bytes(self.context.did, byte_data)
+        self.content.append(
+            Content(name=name, segments=[segment], media_type=media_type, content_service=self.context.content_service))
+        return self
+
+    def set_metadata(self, metadata: dict):
+        self.metadata = metadata
+        return self
+
+    def add_metadata(self, key: str, value: str):
+        self.metadata[key] = value
+        return self
+
+    def annotate(self, key: str, value: str):
+        self.annotations[key] = value
+        return self
+
+    def delete_metadata_key(self, key: str):
+        self.delete_metadata_keys.append(key)
         return self
 
     def response(self):
-        transform_events = []
-        for named_result in self.named_results:
-            json_dict = named_result.json()
-            transform_events.append(json_dict)
-        return transform_events
+        return {
+            'content': [content.json() for content in self.content],
+            'metadata': self.metadata,
+            'annotations': self.annotations,
+            'deleteMetadataKeys': self.delete_metadata_keys
+        }
+
+
+class ValidateResult(Result):
+    def __init__(self, context: Context):
+        super().__init__(None, 'VALIDATE', context)
+
+    def response(self):
+        return None

deltafi/test_kit/format.py

@@ -0,0 +1,105 @@
+#
+#    DeltaFi - Data transformation and enrichment platform
+#
+#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+from typing import List
+
+from deltafi.result import FormatResult, FormatManyResult
+
+from .assertions import *
+from .framework import TestCaseBase, ActionTest
+
+
+class FormatTestCase(TestCaseBase):
+    def __init__(self, fields: Dict):
+        super().__init__(fields)
+        self.metadata = {}
+        self.delete_metadata_keys = []
+        self.expected_format_many_result = []
+
+    def expect_format_result(self, metadata: Dict, delete_metadata_keys: List[str]):
+        self.expected_result_type = FormatResult
+        self.metadata = metadata
+        self.delete_metadata_keys = delete_metadata_keys
+
+    def add_format_many_result(self, metadata: Dict, delete_metadata_keys: List):
+        self.expected_result_type = FormatManyResult
+        self.expected_format_many_result.append(
+            {
+                "metadata": metadata,
+                "delete_metadata_keys": delete_metadata_keys
+            }
+        )
+
+
+class FormatActionTest(ActionTest):
+    def __init__(self, package_name: str):
+        """
+        Provides structure for testing DeltaFi Format action
+        Args:
+            package_name: name of the actions package for finding resources
+        """
+        super().__init__(package_name)
+
+    def format(self, test_case: FormatTestCase):
+        if test_case.expected_result_type == FormatManyResult:
+            self.expect_format_many_result(test_case)
+        elif test_case.expected_result_type == FormatResult:
+            self.expect_format_result(test_case)
+        else:
+            super().execute(test_case)
+
+    def expect_format_result(self, test_case: FormatTestCase):
+        result = super().run_and_check_result_type(test_case, FormatResult)
+        self.assert_format_result(test_case, result)
+
+    def expect_format_many_result(self, test_case: FormatTestCase):
+        result = super().run_and_check_result_type(test_case, FormatManyResult)
+        self.assert_format_many_result(test_case, result)
+
+    def assert_format_result(self, test_case: FormatTestCase, result: FormatResult):
+        # Check metrics
+        self.compare_metrics(test_case.expected_metrics, result.metrics)
+
+        # Check output
+        if result.content is None:
+            self.compare_all_output(test_case.compare_tool, [])
+        else:
+            self.compare_all_output(test_case.compare_tool, [result.content])
+
+        # Check metadata
+        assert_keys_and_values(test_case.metadata, result.metadata)
+
+        # Check deleted metadata
+        for key in test_case.delete_metadata_keys:
+            assert_key_in(key, result.delete_metadata_keys)
+
+    def assert_format_many_result(self, test_case: FormatTestCase, actual: FormatManyResult):
+        # Check metrics
+        self.compare_metrics(test_case.expected_metrics, actual.metrics)
+
+        assert_equal_len(test_case.expected_format_many_result, actual.format_results)
+        for index, expected_child_result in enumerate(test_case.expected_format_many_result):
+            actual_child = actual.format_results[index]
+            self.compare_one_content(
+                test_case.compare_tool,
+                self.expected_outputs[index],
+                actual_child.format_result.content, index)
+
+            assert_keys_and_values(expected_child_result['metadata'], actual_child.format_result.metadata)
+            for key in expected_child_result['delete_metadata_keys']:
+                assert_key_in(key, actual_child.format_result.delete_metadata_keys)

deltafi/test_kit/framework.py

@@ -103,10 +103,6 @@ class InternalContentService:
         seg_id = segments[0].uuid
         return self.loaded_content[seg_id].data
 
-    def get_bytes(self, segments: List[Segment]):
-        seg_id = segments[0].uuid
-        return self.loaded_content[seg_id].data.encode('utf-8')
-
     def get_output(self, seg_id: str):
         if seg_id in self.outputs:
             return self.outputs[seg_id]
@@ -127,6 +123,8 @@ class TestCaseBase(ABC):
         - inputs: (optional) List[IOContent]: input content to action
         - parameters: (optional) Dict: map of action input parameters
         - in_meta: (optional) Dict: map of metadata as input to action
+        - in_domains: (optional) List[Domain]: list of domains as input to action
+        - in_enrichments: (optional) List[Domain]: list of enrichments as input to action
         - did: (optional): str: overrides random DID
         """
         if "action" in data:
@@ -146,8 +144,11 @@ class TestCaseBase(ABC):
 
         self.inputs = data["inputs"] if "inputs" in data else []
         self.file_name = data["file_name"] if "file_name" in data else "filename"
+        self.outputs = data["outputs"] if "outputs" in data else []
         self.parameters = data["parameters"] if "parameters" in data else {}
         self.in_meta = data["in_meta"] if "in_meta" in data else {}
+        self.in_domains = data["in_domains"] if "in_domains" in data else []
+        self.in_enrichments = data["in_enrichments"] if "in_enrichments" in data else []
         self.use_did = data["did"] if "did" in data else None
         self.expected_result_type = None
         self.err_or_filt_cause = None
@@ -192,6 +193,7 @@ class ActionTest(ABC):
         """
         self.content_service = InternalContentService()
         self.did = ""
+        self.expected_outputs = []
         self.loaded_inputs = []
         self.package_name = package_name
         self.res_path = ""
@@ -202,6 +204,7 @@ class ActionTest(ABC):
             self.did = str(uuid.uuid4())
         else:
             self.did = did
+        self.expected_outputs = []
         self.loaded_inputs = []
         self.res_path = ""
 
@@ -222,6 +225,13 @@ class ActionTest(ABC):
             else:
                 self.loaded_inputs.append(LoadedContent(self.did, input_ioc, None))
 
+        # Load expected outputs
+        for output_ioc in test_case.outputs:
+            if len(output_ioc.content_bytes) == 0:
+                self.expected_outputs.append(LoadedContent(self.did, output_ioc, self.load_file(output_ioc)))
+            else:
+                self.expected_outputs.append(LoadedContent(self.did, output_ioc, None))
+
     def make_content_list(self, test_case: TestCaseBase):
         content_list = []
         for loaded_input in self.loaded_inputs:
@@ -236,23 +246,26 @@ class ActionTest(ABC):
         content_list = self.make_content_list(test_case)
         self.content_service.load(self.loaded_inputs)
 
-        return DeltaFileMessage(metadata=test_case.in_meta,
-                                content_list=content_list)
+        return DeltaFileMessage(
+            metadata=test_case.in_meta,
+            content_list=content_list,
+            domains=test_case.in_domains,
+            enrichments=test_case.in_enrichments)
 
     def make_context(self, test_case: TestCaseBase):
         action_name = INGRESS_FLOW + "." + test_case.action.__class__.__name__
         return Context(
             did=self.did,
-            delta_file_name=test_case.file_name,
-            data_source="DATASRC",
-            flow_name=INGRESS_FLOW,
-            flow_id="FLOWID",
+            action_flow=INGRESS_FLOW,
             action_name=action_name,
-            action_id="ACTIONID",
-            action_version="1.0",
+            source_filename=test_case.file_name,
+            ingress_flow=INGRESS_FLOW,
+            egress_flow=EGRESS_FLOW,
+            system=SYSTEM,
             hostname=HOSTNAME,
-            system_name=SYSTEM,
             content_service=self.content_service,
+            collect=None,
+            collected_dids=None,
             logger=get_logger())
 
     def make_event(self, test_case: TestCaseBase):
@@ -298,27 +311,37 @@ class ActionTest(ABC):
         else:
             raise ValueError(f"unknown type: {test_case.expected_result_type}")
 
-    @staticmethod
-    def compare_content_details(expected: LoadedContent, actual: Content):
+    def compare_content_details(self, expected: LoadedContent, actual: Content):
         assert_equal(expected.content_type, actual.media_type)
         assert_equal(expected.name, actual.name)
 
-    def compare_one_content(self, comparator: CompareHelper, expected: LoadedContent, actual, index):
+    def compare_one_content(self, comparitor: CompareHelper, expected: LoadedContent, actual, index):
         self.compare_content_details(expected, actual)
         seg_id = actual.segments[0].uuid
-        comparator.compare(expected.data, self.content_service.get_output(seg_id), f"Content[{index}]")
-
-    def compare_content_list(self, comparator: CompareHelper, expected_outputs: List[IOContent], content: List):
-        assert_equal_len(expected_outputs, content)
-        for index, expected_ioc in enumerate(expected_outputs):
-            if len(expected_ioc.content_bytes) == 0:
-                expected = LoadedContent(self.did, expected_ioc, self.load_file(expected_ioc))
+        comparitor.compare(expected.data, self.content_service.get_output(seg_id), f"Content[{index}]")
+
+    def compare_all_output(self, comparitor: CompareHelper, content: List):
+        assert_equal_len(self.expected_outputs, content)
+        for index, expected in enumerate(self.expected_outputs):
+            self.compare_one_content(comparitor, expected, content[index], index)
+
+    def compare_domains(self, comparitor: CompareHelper, expected_items: List[Dict], results: List[Dict]):
+        assert_equal_len(expected_items, results)
+        for index, expected in enumerate(expected_items):
+            actual = results[index]
+            assert_equal(expected['name'], actual['name'])
+            assert_equal(expected['mediaType'], actual['mediaType'])
+
+            expected_value = expected['value']
+            if type(expected_value) == str:
+                comparitor.compare(expected_value, actual['value'], f"Domain[{index}]")
+            elif type(expected_value) == IOContent:
+                expected_data = self.load_file(expected_value)
+                comparitor.compare(expected_data, actual['value'], f"Domain[{index}]")
             else:
-                expected = LoadedContent(self.did, expected_ioc, None)
-            self.compare_one_content(comparator, expected, content[index], index)
+                raise ValueError(f"unknown expected_value type: {type(expected_value)}")
 
-    @staticmethod
-    def compare_one_metric(expected: Metric, result: Metric):
+    def compare_one_metric(self, expected: Metric, result: Metric):
         assert expected.name == result.name
         assert_equal_with_label(expected.value, result.value, expected.name)
         assert_keys_and_values(expected.tags, result.tags)