deltafi 1.2.20__py3-none-any.whl → 2.0.0__py3-none-any.whl

deltafi/result.py

@@ -1,7 +1,7 @@
 #
 #    DeltaFi - Data transformation and enrichment platform
 #
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#    Copyright 2021-2024 DeltaFi Contributors <deltafi@deltafi.org>
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -17,17 +17,13 @@
 #
 
 import abc
-from enum import Enum
 import uuid
-from typing import Dict, List
+from enum import Enum
+from typing import NamedTuple
 
 from deltafi.domain import Content, Context
 from deltafi.metric import Metric
 
-ENDPOINT_TAG = "endpoint"
-FILES_OUT = "files_out"
-BYTES_OUT = "bytes_out"
-
 
 class Result:
     __metaclass__ = abc.ABCMeta
@@ -44,58 +40,17 @@ class Result:
 
     def add_metric(self, metric: Metric):
         self.metrics.append(metric)
-
-
-class DomainResult(Result):
-    def __init__(self, context: Context):
-        super().__init__('domain', 'DOMAIN', context)
-        self.annotations = {}
-
-    def annotate(self, key: str, value: str):
-        self.annotations[key] = value
         return self
 
-    def response(self):
-        return {
-            'annotations': self.annotations
-        }
-
 
 class EgressResult(Result):
-    def __init__(self, context: Context, destination: str, bytes_egressed: int):
+    def __init__(self, context: Context):
         super().__init__(None, 'EGRESS', context)
-        self.add_metric(Metric(FILES_OUT, 1, {ENDPOINT_TAG: destination}))
-        self.add_metric(Metric(BYTES_OUT, bytes_egressed, {ENDPOINT_TAG: destination}))
 
     def response(self):
         return None
 
 
-class EnrichResult(Result):
-    def __init__(self, context: Context):
-        super().__init__('enrich', 'ENRICH', context)
-        self.enrichments = []
-        self.annotations = {}
-
-    def enrich(self, name: str, value: str, media_type: str):
-        self.enrichments.append({
-            'name': name,
-            'value': value,
-            'mediaType': media_type
-        })
-        return self
-
-    def annotate(self, key: str, value: str):
-        self.annotations[key] = value
-        return self
-
-    def response(self):
-        return {
-            'enrichments': self.enrichments,
-            'annotations': self.annotations
-        }
-
-
 class ErrorResult(Result):
     def __init__(self, context: Context, error_cause: str, error_context: str):
         super().__init__('error', 'ERROR', context)
@@ -116,7 +71,7 @@ class ErrorResult(Result):
 
 
 class FilterResult(Result):
-    def __init__(self, context: Context, filtered_cause: str, filtered_context: str=None):
+    def __init__(self, context: Context, filtered_cause: str, filtered_context: str = None):
         super().__init__('filter', 'FILTER', context)
         self.filtered_cause = filtered_cause
         self.filtered_context = filtered_context
@@ -134,87 +89,13 @@ class FilterResult(Result):
         }
 
 
-class FormatResult(Result):
-    def __init__(self, context: Context):
-        super().__init__('format', 'FORMAT', context)
-        self.content = None
-        self.delete_metadata_keys = []
-        self.metadata = {}
-
-    def set_metadata(self, metadata: dict):
-        self.metadata = metadata
-        return self
-
-    def add_metadata(self, key: str, value: str):
-        self.metadata[key] = value
-        return self
-
-    def delete_metadata_key(self, key: str):
-        self.delete_metadata_keys.append(key)
-        return self
-
-    def set_content(self, content: Content):
-        self.content = content
-        return self
-
-    def save_string_content(self, string_data: str, name: str, media_type: str):
-        segment = self.context.content_service.put_str(self.context.did, string_data)
-        self.content = Content(name=name, segments=[segment], media_type=media_type,
-                               content_service=self.context.content_service)
-        return self
-
-    def save_byte_content(self, byte_data: bytes, name: str, media_type: str):
-        segment = self.context.content_service.put_bytes(self.context.did, byte_data)
-        self.content = Content(name=name, segments=[segment], media_type=media_type,
-                               content_service=self.context.content_service)
-        return self
-
-    def response(self):
-        return {
-            'content': self.content.json(),
-            'metadata': self.metadata,
-            'deleteMetadataKeys': self.delete_metadata_keys
-        }
-
-
-class ChildFormatResult:
-    def __init__(self, format_result: FormatResult = None):
-        self._did = str(uuid.uuid4())
-        self.format_result = format_result
-
-    @property
-    def did(self):
-        return self._did
-
-    def response(self):
-        res = self.format_result.response()
-        res["did"] = self._did
-        return res
-
-
-class FormatManyResult(Result):
-    def __init__(self, context: Context):
-        super().__init__('formatMany', 'FORMAT_MANY', context)
-        self.format_results = []
-
-    def add_format_result(self, format_result):
-        if isinstance(format_result, ChildFormatResult):
-            self.format_results.append(format_result)
-        else:
-            self.format_results.append(ChildFormatResult(format_result))
-        return self
-
-    def response(self):
-        return [format_result.response() for format_result in self.format_results]
-
-
 class IngressResultItem:
-    def __init__(self, context: Context, filename: str):
+    def __init__(self, context: Context, delta_file_name: str):
         self.context = context
-        self.filename = filename
         self._did = str(uuid.uuid4())
         self.content = []
         self.metadata = {}
+        self.delta_file_name = delta_file_name
 
     @property
     def did(self):
@@ -231,15 +112,17 @@ class IngressResultItem:
         return self
 
     def save_string_content(self, string_data: str, name: str, media_type: str):
-        segment = self.context.content_service.put_str(self.context.did, string_data)
-        self.content.append(
-            Content(name=name, segments=[segment], media_type=media_type, content_service=self.context.content_service))
+        segment = self.context.content_service.put_str(self._did, string_data)
+        c = Content(name=name, segments=[segment], media_type=media_type, content_service=self.context.content_service)
+        self.content.append(c)
+        self.context.saved_content.append(c)
         return self
 
     def save_byte_content(self, byte_data: bytes, name: str, media_type: str):
-        segment = self.context.content_service.put_bytes(self.context.did, byte_data)
-        self.content.append(
-            Content(name=name, segments=[segment], media_type=media_type, content_service=self.context.content_service))
+        segment = self.context.content_service.put_bytes(self._did, byte_data)
+        c = Content(name=name, segments=[segment], media_type=media_type, content_service=self.context.content_service)
+        self.content.append(c)
+        self.context.saved_content.append(c)
         return self
 
     def set_metadata(self, metadata: dict):
@@ -250,10 +133,16 @@ class IngressResultItem:
         self.metadata[key] = value
         return self
 
+    def get_segment_names(self):
+        segment_names = {}
+        for c in self.content:
+            segment_names.update(c.get_segment_names())
+        return segment_names
+
     def response(self):
         return {
             'did': self._did,
-            'filename': self.filename,
+            'deltaFileName': self.delta_file_name,
             'metadata': self.metadata,
             'content': [content.json() for content in self.content]
         }
@@ -269,32 +158,37 @@ class IngressResult(Result):
     def __init__(self, context: Context):
         super().__init__('ingress', 'INGRESS', context)
         self.memo = None
-        self.execute_immediate = False
         self.ingress_result_items = []
+        self.execute_immediate = False
         self.status = IngressStatusEnum.HEALTHY
-        self.statusMessage = None
+        self.status_message = None
 
     def add_item(self, ingress_result_item: IngressResultItem):
         self.ingress_result_items.append(ingress_result_item)
         return self
 
+    def get_segment_names(self):
+        segment_names = {}
+        for ingress_item in self.ingress_result_items:
+            segment_names.update(ingress_item.get_segment_names())
+        return segment_names
+
     def response(self):
         return {
             'memo': self.memo,
             'executeImmediate': self.execute_immediate,
             'ingressItems': [ingress_result_item.response() for ingress_result_item in self.ingress_result_items],
             'status': self.status.value,
-            'statusMessage': self.statusMessage
+            'statusMessage': self.status_message
         }
 
 
-class LoadResult(Result):
+class TransformResult(Result):
     def __init__(self, context: Context):
-        super().__init__('load', 'LOAD', context)
+        super().__init__('transform', 'TRANSFORM', context)
         self.content = []
-        self.metadata = {}
-        self.domains = []
         self.annotations = {}
+        self.metadata = {}
         self.delete_metadata_keys = []
 
     # content can be a single Content or a List[Content]
@@ -309,14 +203,16 @@ class LoadResult(Result):
 
     def save_string_content(self, string_data: str, name: str, media_type: str):
         segment = self.context.content_service.put_str(self.context.did, string_data)
-        self.content.append(
-            Content(name=name, segments=[segment], media_type=media_type, content_service=self.context.content_service))
+        c = Content(name=name, segments=[segment], media_type=media_type, content_service=self.context.content_service)
+        self.content.append(c)
+        self.context.saved_content.append(c)
         return self
 
     def save_byte_content(self, byte_data: bytes, name: str, media_type: str):
         segment = self.context.content_service.put_bytes(self.context.did, byte_data)
-        self.content.append(
-            Content(name=name, segments=[segment], media_type=media_type, content_service=self.context.content_service))
+        c = Content(name=name, segments=[segment], media_type=media_type, content_service=self.context.content_service)
+        self.content.append(c)
+        self.context.saved_content.append(c)
         return self
 
     def set_metadata(self, metadata: dict):
@@ -327,13 +223,6 @@ class LoadResult(Result):
         self.metadata[key] = value
         return self
 
-    def add_domain(self, name: str, value: str, media_type: str):
-        self.domains.append({
-            'name': name,
-            'value': value,
-            'mediaType': media_type})
-        return self
-
     def annotate(self, key: str, value: str):
         self.annotations[key] = value
         return self
@@ -342,133 +231,57 @@ class LoadResult(Result):
         self.delete_metadata_keys.append(key)
         return self
 
-    def response(self):
+    def get_segment_names(self):
+        segment_names = {}
+        for c in self.content:
+            segment_names.update(c.get_segment_names())
+        return segment_names
+
+    def json(self):
         return {
-            'domains': self.domains,
+            'did': self.context.did,
             'content': [content.json() for content in self.content],
-            'metadata': self.metadata,
             'annotations': self.annotations,
+            'metadata': self.metadata,
             'deleteMetadataKeys': self.delete_metadata_keys
         }
 
-
-class ChildLoadResult:
-    def __init__(self, load_result: LoadResult = None):
-        self._did = str(uuid.uuid4())
-        self.load_result = load_result
-
-    @property
-    def did(self):
-        return self._did
-
-    def response(self):
-        res = self.load_result.response()
-        res["did"] = self._did
-        return res
-
-
-class LoadManyResult(Result):
-    def __init__(self, context: Context):
-        super().__init__('loadMany', 'LOAD_MANY', context)
-        self.load_results = []
-
-    def add_load_result(self, load_result):
-        if isinstance(load_result, ChildLoadResult):
-            self.load_results.append(load_result)
-        else:
-            self.load_results.append(ChildLoadResult(load_result))
-        return self
-
     def response(self):
-        return [load_result.response() for load_result in self.load_results]
-
-
-class ReinjectResult(Result):
-    class ReinjectChild:
-        def __init__(self, filename: str, flow: str, content: List[Content], metadata: Dict[str, str]):
-            self.filename = filename
-            self.flow = flow
-            self.content = content
-            self.metadata = metadata
+        return [self.json()]
 
-        def json(self):
-            return {
-                'filename': self.filename,
-                'flow': self.flow,
-                'metadata': self.metadata,
-                'content': [content.json() for content in self.content]
-            }
 
-    def __init__(self, context: Context):
-        super().__init__('reinject', 'REINJECT', context)
-        self.children = []
+class ChildTransformResult(TransformResult):
+    delta_file_name: str
 
-    def add_child(self, filename: str, flow: str, content: List[Content], metadata: Dict[str, str]):
-        child = ReinjectResult.ReinjectChild(filename, flow, content, metadata)
-        self.children.append(child)
+    def __init__(self, context: Context, delta_file_name: str = None):
+        super().__init__(context.child_context())
+        self.delta_file_name = delta_file_name
 
-    def response(self):
-        return [child.json() for child in self.children]
+    def json(self):
+        j = super().json()
+        if self.delta_file_name is not None:
+            j['name'] = self.delta_file_name
+        return j
 
 
-class TransformResult(Result):
+class TransformResults(Result):
     def __init__(self, context: Context):
         super().__init__('transform', 'TRANSFORM', context)
-        self.content = []
-        self.metadata = {}
-        self.annotations = {}
-        self.delete_metadata_keys = []
-
-    # content can be a single Content or a List[Content]
-    def add_content(self, content):
-        if content:
-            if type(content) == list:
-                self.content.extend(content)
-            else:
-                self.content.append(content)
-
-        return self
-
-    def save_string_content(self, string_data: str, name: str, media_type: str):
-        segment = self.context.content_service.put_str(self.context.did, string_data)
-        self.content.append(
-            Content(name=name, segments=[segment], media_type=media_type, content_service=self.context.content_service))
-        return self
-
-    def save_byte_content(self, byte_data: bytes, name: str, media_type: str):
-        segment = self.context.content_service.put_bytes(self.context.did, byte_data)
-        self.content.append(
-            Content(name=name, segments=[segment], media_type=media_type, content_service=self.context.content_service))
-        return self
-
-    def set_metadata(self, metadata: dict):
-        self.metadata = metadata
-        return self
-
-    def add_metadata(self, key: str, value: str):
-        self.metadata[key] = value
-        return self
-
-    def annotate(self, key: str, value: str):
-        self.annotations[key] = value
-        return self
+        self.child_results = []
 
-    def delete_metadata_key(self, key: str):
-        self.delete_metadata_keys.append(key)
+    def add_result(self, result: ChildTransformResult):
+        self.child_results.append(result)
         return self
 
-    def response(self):
-        return {
-            'content': [content.json() for content in self.content],
-            'metadata': self.metadata,
-            'annotations': self.annotations,
-            'deleteMetadataKeys': self.delete_metadata_keys
-        }
-
-
-class ValidateResult(Result):
-    def __init__(self, context: Context):
-        super().__init__(None, 'VALIDATE', context)
+    def get_segment_names(self):
+        segment_names = {}
+        for child_result in self.child_results:
+            segment_names.update(child_result.get_segment_names())
+        return segment_names
 
     def response(self):
-        return None
+        transform_events = []
+        for child_result in self.child_results:
+            json_dict = child_result.json()
+            transform_events.append(json_dict)
+        return transform_events

deltafi/storage.py

@@ -1,7 +1,7 @@
 #
 #    DeltaFi - Data transformation and enrichment platform
 #
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#    Copyright 2021-2024 DeltaFi Contributors <deltafi@deltafi.org>
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -22,6 +22,7 @@ from typing import List, NamedTuple
 from urllib.parse import urlparse
 
 import minio
+from minio.deleteobjects import DeleteObject
 
 BUCKET = 'storage'
 
@@ -86,3 +87,7 @@ class ContentService:
 
     def put_str(self, did, string_data):
         return self.put_bytes(did, string_data.encode('utf-8'))
+
+    def delete_all(self, segments: List[Segment]):
+        delete_objects = [DeleteObject(seg.id()) for seg in segments]
+        return self.minio_client.remove_objects(BUCKET, delete_objects)

deltafi/test_kit/__init__.py

@@ -1,7 +1,7 @@
 #
 #    DeltaFi - Data transformation and enrichment platform
 #
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#    Copyright 2021-2024 DeltaFi Contributors <deltafi@deltafi.org>
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.

deltafi/test_kit/assertions.py

@@ -1,7 +1,7 @@
 #
 #    DeltaFi - Data transformation and enrichment platform
 #
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#    Copyright 2021-2024 DeltaFi Contributors <deltafi@deltafi.org>
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.

deltafi/test_kit/compare_helpers.py

@@ -1,7 +1,7 @@
 #
 #    DeltaFi - Data transformation and enrichment platform
 #
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#    Copyright 2021-2024 DeltaFi Contributors <deltafi@deltafi.org>
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.

deltafi/test_kit/constants.py

@@ -1,7 +1,7 @@
 #
 #    DeltaFi - Data transformation and enrichment platform
 #
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#    Copyright 2021-2024 DeltaFi Contributors <deltafi@deltafi.org>
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.

deltafi/test_kit/egress.py

@@ -1,7 +1,7 @@
 #
 #    DeltaFi - Data transformation and enrichment platform
 #
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#    Copyright 2021-2024 DeltaFi Contributors <deltafi@deltafi.org>
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.