deltafi 2.0rc1720728217472__py3-none-any.whl → 2.0rc1720817063181__py3-none-any.whl

deltafi/result.py

@@ -17,9 +17,9 @@
 #
 
 import abc
-from enum import Enum
 import uuid
-from typing import Dict, List
+from enum import Enum
+from typing import NamedTuple
 
 from deltafi.domain import Content, Context
 from deltafi.metric import Metric
@@ -44,22 +44,8 @@ class Result:
 
     def add_metric(self, metric: Metric):
         self.metrics.append(metric)
-
-
-class DomainResult(Result):
-    def __init__(self, context: Context):
-        super().__init__('domain', 'DOMAIN', context)
-        self.annotations = {}
-
-    def annotate(self, key: str, value: str):
-        self.annotations[key] = value
         return self
 
-    def response(self):
-        return {
-            'annotations': self.annotations
-        }
-
 
 class EgressResult(Result):
     def __init__(self, context: Context, destination: str, bytes_egressed: int):
@@ -71,31 +57,6 @@ class EgressResult(Result):
         return None
 
 
-class EnrichResult(Result):
-    def __init__(self, context: Context):
-        super().__init__('enrich', 'ENRICH', context)
-        self.enrichments = []
-        self.annotations = {}
-
-    def enrich(self, name: str, value: str, media_type: str):
-        self.enrichments.append({
-            'name': name,
-            'value': value,
-            'mediaType': media_type
-        })
-        return self
-
-    def annotate(self, key: str, value: str):
-        self.annotations[key] = value
-        return self
-
-    def response(self):
-        return {
-            'enrichments': self.enrichments,
-            'annotations': self.annotations
-        }
-
-
 class ErrorResult(Result):
     def __init__(self, context: Context, error_cause: str, error_context: str):
         super().__init__('error', 'ERROR', context)
@@ -116,7 +77,7 @@ class ErrorResult(Result):
 
 
 class FilterResult(Result):
-    def __init__(self, context: Context, filtered_cause: str, filtered_context: str=None):
+    def __init__(self, context: Context, filtered_cause: str, filtered_context: str = None):
         super().__init__('filter', 'FILTER', context)
         self.filtered_cause = filtered_cause
         self.filtered_context = filtered_context
@@ -134,87 +95,13 @@ class FilterResult(Result):
         }
 
 
-class FormatResult(Result):
-    def __init__(self, context: Context):
-        super().__init__('format', 'FORMAT', context)
-        self.content = None
-        self.delete_metadata_keys = []
-        self.metadata = {}
-
-    def set_metadata(self, metadata: dict):
-        self.metadata = metadata
-        return self
-
-    def add_metadata(self, key: str, value: str):
-        self.metadata[key] = value
-        return self
-
-    def delete_metadata_key(self, key: str):
-        self.delete_metadata_keys.append(key)
-        return self
-
-    def set_content(self, content: Content):
-        self.content = content
-        return self
-
-    def save_string_content(self, string_data: str, name: str, media_type: str):
-        segment = self.context.content_service.put_str(self.context.did, string_data)
-        self.content = Content(name=name, segments=[segment], media_type=media_type,
-                               content_service=self.context.content_service)
-        return self
-
-    def save_byte_content(self, byte_data: bytes, name: str, media_type: str):
-        segment = self.context.content_service.put_bytes(self.context.did, byte_data)
-        self.content = Content(name=name, segments=[segment], media_type=media_type,
-                               content_service=self.context.content_service)
-        return self
-
-    def response(self):
-        return {
-            'content': self.content.json(),
-            'metadata': self.metadata,
-            'deleteMetadataKeys': self.delete_metadata_keys
-        }
-
-
-class ChildFormatResult:
-    def __init__(self, format_result: FormatResult = None):
-        self._did = str(uuid.uuid4())
-        self.format_result = format_result
-
-    @property
-    def did(self):
-        return self._did
-
-    def response(self):
-        res = self.format_result.response()
-        res["did"] = self._did
-        return res
-
-
-class FormatManyResult(Result):
-    def __init__(self, context: Context):
-        super().__init__('formatMany', 'FORMAT_MANY', context)
-        self.format_results = []
-
-    def add_format_result(self, format_result):
-        if isinstance(format_result, ChildFormatResult):
-            self.format_results.append(format_result)
-        else:
-            self.format_results.append(ChildFormatResult(format_result))
-        return self
-
-    def response(self):
-        return [format_result.response() for format_result in self.format_results]
-
-
 class IngressResultItem:
-    def __init__(self, context: Context, filename: str):
+    def __init__(self, context: Context, delta_file_name: str):
         self.context = context
-        self.filename = filename
         self._did = str(uuid.uuid4())
         self.content = []
         self.metadata = {}
+        self.delta_file_name = delta_file_name
 
     @property
     def did(self):
@@ -253,7 +140,7 @@ class IngressResultItem:
     def response(self):
         return {
             'did': self._did,
-            'filename': self.filename,
+            'deltaFileName': self.delta_file_name,
             'metadata': self.metadata,
             'content': [content.json() for content in self.content]
         }
@@ -269,10 +156,10 @@ class IngressResult(Result):
     def __init__(self, context: Context):
         super().__init__('ingress', 'INGRESS', context)
         self.memo = None
-        self.execute_immediate = False
         self.ingress_result_items = []
+        self.execute_immediate = False
         self.status = IngressStatusEnum.HEALTHY
-        self.statusMessage = None
+        self.status_message = None
 
     def add_item(self, ingress_result_item: IngressResultItem):
         self.ingress_result_items.append(ingress_result_item)
@@ -284,17 +171,16 @@ class IngressResult(Result):
             'executeImmediate': self.execute_immediate,
             'ingressItems': [ingress_result_item.response() for ingress_result_item in self.ingress_result_items],
             'status': self.status.value,
-            'statusMessage': self.statusMessage
+            'statusMessage': self.status_message
         }
 
 
-class LoadResult(Result):
+class TransformResult(Result):
     def __init__(self, context: Context):
-        super().__init__('load', 'LOAD', context)
+        super().__init__('transform', 'TRANSFORM', context)
         self.content = []
-        self.metadata = {}
-        self.domains = []
         self.annotations = {}
+        self.metadata = {}
         self.delete_metadata_keys = []
 
     # content can be a single Content or a List[Content]
@@ -327,13 +213,6 @@ class LoadResult(Result):
         self.metadata[key] = value
         return self
 
-    def add_domain(self, name: str, value: str, media_type: str):
-        self.domains.append({
-            'name': name,
-            'value': value,
-            'mediaType': media_type})
-        return self
-
     def annotate(self, key: str, value: str):
         self.annotations[key] = value
         return self
@@ -342,133 +221,41 @@ class LoadResult(Result):
         self.delete_metadata_keys.append(key)
         return self
 
-    def response(self):
+    def json(self):
         return {
-            'domains': self.domains,
             'content': [content.json() for content in self.content],
-            'metadata': self.metadata,
             'annotations': self.annotations,
+            'metadata': self.metadata,
             'deleteMetadataKeys': self.delete_metadata_keys
         }
 
-
-class ChildLoadResult:
-    def __init__(self, load_result: LoadResult = None):
-        self._did = str(uuid.uuid4())
-        self.load_result = load_result
-
-    @property
-    def did(self):
-        return self._did
-
-    def response(self):
-        res = self.load_result.response()
-        res["did"] = self._did
-        return res
-
-
-class LoadManyResult(Result):
-    def __init__(self, context: Context):
-        super().__init__('loadMany', 'LOAD_MANY', context)
-        self.load_results = []
-
-    def add_load_result(self, load_result):
-        if isinstance(load_result, ChildLoadResult):
-            self.load_results.append(load_result)
-        else:
-            self.load_results.append(ChildLoadResult(load_result))
-        return self
-
     def response(self):
-        return [load_result.response() for load_result in self.load_results]
-
-
-class ReinjectResult(Result):
-    class ReinjectChild:
-        def __init__(self, filename: str, flow: str, content: List[Content], metadata: Dict[str, str]):
-            self.filename = filename
-            self.flow = flow
-            self.content = content
-            self.metadata = metadata
+        return [self.json()]
 
-        def json(self):
-            return {
-                'filename': self.filename,
-                'flow': self.flow,
-                'metadata': self.metadata,
-                'content': [content.json() for content in self.content]
-            }
-
-    def __init__(self, context: Context):
-        super().__init__('reinject', 'REINJECT', context)
-        self.children = []
 
-    def add_child(self, filename: str, flow: str, content: List[Content], metadata: Dict[str, str]):
-        child = ReinjectResult.ReinjectChild(filename, flow, content, metadata)
-        self.children.append(child)
+class NamedTransformResult(NamedTuple):
+    result: TransformResult
+    name: str
 
-    def response(self):
-        return [child.json() for child in self.children]
+    def json(self):
+        j = self.result.json()
+        if self.name is not None:
+            j['name'] = self.name
+        return j
 
 
-class TransformResult(Result):
+class TransformResults(Result):
     def __init__(self, context: Context):
         super().__init__('transform', 'TRANSFORM', context)
-        self.content = []
-        self.metadata = {}
-        self.annotations = {}
-        self.delete_metadata_keys = []
-
-    # content can be a single Content or a List[Content]
-    def add_content(self, content):
-        if content:
-            if type(content) == list:
-                self.content.extend(content)
-            else:
-                self.content.append(content)
+        self.named_results = []
 
-        return self
-
-    def save_string_content(self, string_data: str, name: str, media_type: str):
-        segment = self.context.content_service.put_str(self.context.did, string_data)
-        self.content.append(
-            Content(name=name, segments=[segment], media_type=media_type, content_service=self.context.content_service))
-        return self
-
-    def save_byte_content(self, byte_data: bytes, name: str, media_type: str):
-        segment = self.context.content_service.put_bytes(self.context.did, byte_data)
-        self.content.append(
-            Content(name=name, segments=[segment], media_type=media_type, content_service=self.context.content_service))
-        return self
-
-    def set_metadata(self, metadata: dict):
-        self.metadata = metadata
-        return self
-
-    def add_metadata(self, key: str, value: str):
-        self.metadata[key] = value
-        return self
-
-    def annotate(self, key: str, value: str):
-        self.annotations[key] = value
-        return self
-
-    def delete_metadata_key(self, key: str):
-        self.delete_metadata_keys.append(key)
+    def add_result(self, result: TransformResult, name: str = None):
+        self.named_results.append(NamedTransformResult(result, name))
         return self
 
     def response(self):
-        return {
-            'content': [content.json() for content in self.content],
-            'metadata': self.metadata,
-            'annotations': self.annotations,
-            'deleteMetadataKeys': self.delete_metadata_keys
-        }
-
-
-class ValidateResult(Result):
-    def __init__(self, context: Context):
-        super().__init__(None, 'VALIDATE', context)
-
-    def response(self):
-        return None
+        transform_events = []
+        for named_result in self.named_results:
+            json_dict = named_result.json()
+            transform_events.append(json_dict)
+        return transform_events

deltafi/test_kit/framework.py

@@ -37,15 +37,20 @@ class IOContent:
     The IOContent class holds the details for loading input or output
      content into the test framework.
     Attributes:
-        file_name (str): The name of file in test/data.
-        content_name (str): The name of the content.
-        content_type (str): The media type of the content
-        offset (int): Offset to use in Segment
-        content_bytes (str): Bypass file read, and uses these bytes for content
+        file_name (str)    : The name of file in test/data.
+        content_name (str) : The name of the content.
+        content_type (str) : The media type of the content
+        offset (int)       : Offset to use in Segment
+        content_bytes (str): Optional.  If set to a String of length greater than zero, indicates to consumers of this
+                             IOContent that they should bypass file read and use these bytes for content.
+        no_content (bool)  : Optional.  If 'True', then consumers should not attempt to interpret content but should
+                             apply other aspects of this IOContent.  When 'True', 'content_bytes' should be ignored and
+                             loaded content, if any, should be interpreted as empty String or otherwise as documented by
+                             the consumer.
     """
 
     def __init__(self, file_name: str, content_name: str = None, content_type: str = None, offset: int = 0,
-                 content_bytes: str = ""):
+                 content_bytes: str = "", no_content: bool = False):
         self.file_name = file_name
         if content_name is None:
             self.content_name = file_name
@@ -56,7 +61,11 @@ class IOContent:
         else:
             self.content_type = content_type
         self.offset = offset
-        self.content_bytes = content_bytes
+        self.no_content = no_content
+        if no_content:
+            self.content_bytes = None
+        else:
+            self.content_bytes = content_bytes
         self.segment_uuid = uuid.uuid4()
 
     @classmethod
@@ -79,7 +88,10 @@ class LoadedContent:
         if data is not None:
             self.data = data
         else:
-            self.data = ioc.content_bytes
+            if ioc.no_content:
+                self.data = ""
+            else:
+                self.data = ioc.content_bytes
         self.segment = Segment.from_dict(
             {"uuid": str(ioc.segment_uuid), "offset": self.offset, "size": len(self.data), "did": did})
 
@@ -103,6 +115,10 @@ class InternalContentService:
         seg_id = segments[0].uuid
         return self.loaded_content[seg_id].data
 
+    def get_bytes(self, segments: List[Segment]):
+        seg_id = segments[0].uuid
+        return self.loaded_content[seg_id].data.encode('utf-8')
+
     def get_output(self, seg_id: str):
         if seg_id in self.outputs:
             return self.outputs[seg_id]
@@ -123,8 +139,8 @@ class TestCaseBase(ABC):
         - inputs: (optional) List[IOContent]: input content to action
         - parameters: (optional) Dict: map of action input parameters
         - in_meta: (optional) Dict: map of metadata as input to action
-        - in_domains: (optional) List[Domain]: list of domains as input to action
-        - in_enrichments: (optional) List[Domain]: list of enrichments as input to action
+        - join_meta: (optional): List[Dict]: When a List is provided, this enables the JOIN portion of an action.
+        When using JOIN, join_meta must match the size of inputs, though the Dict can be empty
         - did: (optional): str: overrides random DID
         """
         if "action" in data:
@@ -144,16 +160,14 @@ class TestCaseBase(ABC):
 
         self.inputs = data["inputs"] if "inputs" in data else []
         self.file_name = data["file_name"] if "file_name" in data else "filename"
-        self.outputs = data["outputs"] if "outputs" in data else []
         self.parameters = data["parameters"] if "parameters" in data else {}
         self.in_meta = data["in_meta"] if "in_meta" in data else {}
-        self.in_domains = data["in_domains"] if "in_domains" in data else []
-        self.in_enrichments = data["in_enrichments"] if "in_enrichments" in data else []
         self.use_did = data["did"] if "did" in data else None
         self.expected_result_type = None
         self.err_or_filt_cause = None
         self.err_or_filt_context = None
         self.err_or_filt_annotations = None
+        self.join_meta = data["join_meta"] if "join_meta" in data else None
         self.expected_metrics = []
 
     def add_metric(self, metric: Metric):
@@ -164,7 +178,7 @@ class TestCaseBase(ABC):
         A Sets the expected output of the action to an Error Result
         :param cause: the expected error cause
         :param context: the expected error context
-        :param annotations (Optional): Dict: the expected annotations
+        :param annotations: Dict: (Optional) the expected annotations
         """
         self.expected_result_type = ErrorResult
         self.err_or_filt_cause = cause
@@ -175,8 +189,8 @@ class TestCaseBase(ABC):
         """
         A Sets the expected output of the action to a Filter Result
         :param cause: the expected filter cause (message)
-        :param context (Optional): the expected filter context
-        :param annotations (Optional): Dict: the expected annotations
+        :param context: (Optional) the expected filter context
+        :param annotations: Dict: (Optional) the expected annotations
         """
         self.expected_result_type = FilterResult
         self.err_or_filt_cause = cause
@@ -193,7 +207,6 @@ class ActionTest(ABC):
         """
         self.content_service = InternalContentService()
         self.did = ""
-        self.expected_outputs = []
         self.loaded_inputs = []
         self.package_name = package_name
         self.res_path = ""
@@ -204,7 +217,6 @@ class ActionTest(ABC):
             self.did = str(uuid.uuid4())
         else:
             self.did = did
-        self.expected_outputs = []
         self.loaded_inputs = []
         self.res_path = ""
 
@@ -220,19 +232,12 @@ class ActionTest(ABC):
 
         # Load inputs
         for input_ioc in test_case.inputs:
-            if len(input_ioc.content_bytes) == 0:
+            if not input_ioc.no_content and len(input_ioc.content_bytes) == 0:
                 self.loaded_inputs.append(LoadedContent(self.did, input_ioc, self.load_file(input_ioc)))
             else:
                 self.loaded_inputs.append(LoadedContent(self.did, input_ioc, None))
 
-        # Load expected outputs
-        for output_ioc in test_case.outputs:
-            if len(output_ioc.content_bytes) == 0:
-                self.expected_outputs.append(LoadedContent(self.did, output_ioc, self.load_file(output_ioc)))
-            else:
-                self.expected_outputs.append(LoadedContent(self.did, output_ioc, None))
-
-    def make_content_list(self, test_case: TestCaseBase):
+    def make_content_list(self):
         content_list = []
         for loaded_input in self.loaded_inputs:
             c = Content(name=loaded_input.name, segments=[loaded_input.segment], media_type=loaded_input.content_type,
@@ -242,34 +247,42 @@ class ActionTest(ABC):
 
         return content_list
 
-    def make_df_msg(self, test_case: TestCaseBase):
-        content_list = self.make_content_list(test_case)
+    def make_df_msgs(self, test_case: TestCaseBase):
+        content_list = self.make_content_list()
         self.content_service.load(self.loaded_inputs)
 
-        return DeltaFileMessage(
-            metadata=test_case.in_meta,
-            content_list=content_list,
-            domains=test_case.in_domains,
-            enrichments=test_case.in_enrichments)
+        delta_file_messages = []
+
+        if test_case.join_meta is None:
+            delta_file_messages.append(DeltaFileMessage(metadata=test_case.in_meta, content_list=content_list))
+        else:
+            for index, content in enumerate(content_list):
+                delta_file_messages.append(DeltaFileMessage(
+                    metadata=test_case.join_meta[index],
+                    content_list=[content]))
+
+        return delta_file_messages
 
     def make_context(self, test_case: TestCaseBase):
         action_name = INGRESS_FLOW + "." + test_case.action.__class__.__name__
+        join = {} if test_case.join_meta else None
         return Context(
             did=self.did,
-            action_flow=INGRESS_FLOW,
+            delta_file_name=test_case.file_name,
+            data_source="DATASRC",
+            flow_name=INGRESS_FLOW,
+            flow_id="FLOWID",
             action_name=action_name,
-            source_filename=test_case.file_name,
-            ingress_flow=INGRESS_FLOW,
-            egress_flow=EGRESS_FLOW,
-            system=SYSTEM,
+            action_id="ACTIONID",
+            action_version="1.0",
             hostname=HOSTNAME,
+            system_name=SYSTEM,
             content_service=self.content_service,
-            collect=None,
-            collected_dids=None,
+            join=join,
             logger=get_logger())
 
     def make_event(self, test_case: TestCaseBase):
-        return Event(delta_file_messages=[self.make_df_msg(test_case)], context=self.make_context(test_case),
+        return Event(delta_file_messages=self.make_df_msgs(test_case), context=self.make_context(test_case),
                      params=test_case.parameters, queue_name="", return_address="")
 
     def call_action(self, test_case: TestCaseBase):
@@ -311,37 +324,27 @@ class ActionTest(ABC):
         else:
             raise ValueError(f"unknown type: {test_case.expected_result_type}")
 
-    def compare_content_details(self, expected: LoadedContent, actual: Content):
+    @staticmethod
+    def compare_content_details(expected: LoadedContent, actual: Content):
         assert_equal(expected.content_type, actual.media_type)
         assert_equal(expected.name, actual.name)
 
-    def compare_one_content(self, comparitor: CompareHelper, expected: LoadedContent, actual, index):
+    def compare_one_content(self, comparator: CompareHelper, expected: LoadedContent, actual, index):
         self.compare_content_details(expected, actual)
         seg_id = actual.segments[0].uuid
-        comparitor.compare(expected.data, self.content_service.get_output(seg_id), f"Content[{index}]")
-
-    def compare_all_output(self, comparitor: CompareHelper, content: List):
-        assert_equal_len(self.expected_outputs, content)
-        for index, expected in enumerate(self.expected_outputs):
-            self.compare_one_content(comparitor, expected, content[index], index)
-
-    def compare_domains(self, comparitor: CompareHelper, expected_items: List[Dict], results: List[Dict]):
-        assert_equal_len(expected_items, results)
-        for index, expected in enumerate(expected_items):
-            actual = results[index]
-            assert_equal(expected['name'], actual['name'])
-            assert_equal(expected['mediaType'], actual['mediaType'])
-
-            expected_value = expected['value']
-            if type(expected_value) == str:
-                comparitor.compare(expected_value, actual['value'], f"Domain[{index}]")
-            elif type(expected_value) == IOContent:
-                expected_data = self.load_file(expected_value)
-                comparitor.compare(expected_data, actual['value'], f"Domain[{index}]")
+        comparator.compare(expected.data, self.content_service.get_output(seg_id), f"Content[{index}]")
+
+    def compare_content_list(self, comparator: CompareHelper, expected_outputs: List[IOContent], content: List):
+        assert_equal_len(expected_outputs, content)
+        for index, expected_ioc in enumerate(expected_outputs):
+            if not expected_ioc.no_content and len(expected_ioc.content_bytes) == 0:
+                expected = LoadedContent(self.did, expected_ioc, self.load_file(expected_ioc))
             else:
-                raise ValueError(f"unknown expected_value type: {type(expected_value)}")
+                expected = LoadedContent(self.did, expected_ioc, None)
+            self.compare_one_content(comparator, expected, content[index], index)
 
-    def compare_one_metric(self, expected: Metric, result: Metric):
+    @staticmethod
+    def compare_one_metric(expected: Metric, result: Metric):
         assert expected.name == result.name
         assert_equal_with_label(expected.value, result.value, expected.name)
         assert_keys_and_values(expected.tags, result.tags)