deltafi 2.0rc1725920935509__py3-none-any.whl → 2.0rc1727460733389__py3-none-any.whl

deltafi/__init__.py

@@ -1,7 +1,7 @@
 #
 #    DeltaFi - Data transformation and enrichment platform
 #
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#    Copyright 2021-2024 DeltaFi Contributors <deltafi@deltafi.org>
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.

deltafi/action.py

@@ -1,7 +1,7 @@
 #
 #    DeltaFi - Data transformation and enrichment platform
 #
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#    Copyright 2021-2024 DeltaFi Contributors <deltafi@deltafi.org>
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.

deltafi/actioneventqueue.py

@@ -1,7 +1,7 @@
 #
 #    DeltaFi - Data transformation and enrichment platform
 #
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#    Copyright 2021-2024 DeltaFi Contributors <deltafi@deltafi.org>
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.

deltafi/actiontype.py

@@ -1,7 +1,7 @@
 #
 #    DeltaFi - Data transformation and enrichment platform
 #
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#    Copyright 2021-2024 DeltaFi Contributors <deltafi@deltafi.org>
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.

deltafi/domain.py

@@ -1,7 +1,7 @@
 #
 #    DeltaFi - Data transformation and enrichment platform
 #
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#    Copyright 2021-2024 DeltaFi Contributors <deltafi@deltafi.org>
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -293,6 +293,12 @@ class Content:
         """
         self.segments.extend(other_content.segments)
 
+    def get_segment_names(self):
+        segment_names = {}
+        for seg in self.segments:
+            segment_names[seg.id()] = seg
+        return segment_names
+
     def __eq__(self, other):
         if isinstance(other, Content):
             return (self.name == other.name and

deltafi/exception.py

@@ -1,7 +1,7 @@
 #
 #    DeltaFi - Data transformation and enrichment platform
 #
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#    Copyright 2021-2024 DeltaFi Contributors <deltafi@deltafi.org>
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.

deltafi/genericmodel.py

@@ -1,7 +1,7 @@
 #
 #    DeltaFi - Data transformation and enrichment platform
 #
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#    Copyright 2021-2024 DeltaFi Contributors <deltafi@deltafi.org>
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.

deltafi/input.py

@@ -1,7 +1,7 @@
 #
 #    DeltaFi - Data transformation and enrichment platform
 #
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#    Copyright 2021-2024 DeltaFi Contributors <deltafi@deltafi.org>
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.

deltafi/logger.py

@@ -1,7 +1,7 @@
 #
 #    DeltaFi - Data transformation and enrichment platform
 #
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#    Copyright 2021-2024 DeltaFi Contributors <deltafi@deltafi.org>
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.

deltafi/metric.py

@@ -1,7 +1,7 @@
 #
 #    DeltaFi - Data transformation and enrichment platform
 #
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#    Copyright 2021-2024 DeltaFi Contributors <deltafi@deltafi.org>
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.

deltafi/plugin.py

@@ -1,7 +1,7 @@
 #
 #    DeltaFi - Data transformation and enrichment platform
 #
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#    Copyright 2021-2024 DeltaFi Contributors <deltafi@deltafi.org>
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -32,13 +32,12 @@ from pathlib import Path
 from typing import List
 
 import requests
-
 from deltafi.action import Action, Join
 from deltafi.actioneventqueue import ActionEventQueue
 from deltafi.domain import Event, ActionExecution
 from deltafi.exception import ExpectedContentException, MissingMetadataException
 from deltafi.logger import get_logger
-from deltafi.result import ErrorResult
+from deltafi.result import ErrorResult, IngressResult, TransformResult, TransformResults
 from deltafi.storage import ContentService
 
 
@@ -268,8 +267,7 @@ class Plugin(object):
             'start': start_time,
             'stop': stop_time,
             'type': result.result_type,
-            'metrics': [metric.json() for metric in result.metrics],
-            'savedContent': [content.json() for content in event.context.saved_content]
+            'metrics': [metric.json() for metric in result.metrics]
         }
         if result.result_key is not None:
             response[result.result_key] = result.response()
@@ -309,6 +307,8 @@ class Plugin(object):
                 response = Plugin.to_response(
                     event, start_time, time.time(), result)
 
+                Plugin.orphaned_content_check(action_logger, event.context, result, response)
+
                 topic = 'dgs'
                 if event.return_address:
                     topic += f"-{event.return_address}"
@@ -316,3 +316,42 @@ class Plugin(object):
             except BaseException as e:
                 action_logger.error(f"Unexpected {type(e)} error: {str(e)}\n{traceback.format_exc()}")
                 time.sleep(1)
+
+    @staticmethod
+    def orphaned_content_check(logger, context, result, response):
+        if len(context.saved_content) > 0:
+            to_delete = Plugin.find_unused_content(context.saved_content, result)
+            if len(to_delete) > 0:
+                errors = context.content_service.delete_all(to_delete)
+                for e in errors:
+                    logger.error(f"Unable to delete object(s), {e}")
+                logger.warning(
+                    f"Deleted {len(to_delete)} unused content entries for did {context.did} due to a {response['type']} event by {response['actionName']}")
+
+    @staticmethod
+    def find_unused_content(saved_content, result):
+        segments_in_use = Plugin.used_segment_names(result)
+        saved_segments = Plugin.get_segment_names(saved_content)
+        to_delete = []
+        for key, value in saved_segments.items():
+            if key not in segments_in_use:
+                to_delete.append(value)
+        return to_delete
+
+    @staticmethod
+    def used_segment_names(result):
+        segment_names = {}
+        if isinstance(result, TransformResult):
+            segment_names.update(result.get_segment_names())
+        elif isinstance(result, TransformResults):
+            segment_names.update(result.get_segment_names())
+        elif isinstance(result, IngressResult):
+            segment_names.update(result.get_segment_names())
+        return segment_names
+
+    @staticmethod
+    def get_segment_names(content_list):
+        segment_names = {}
+        for content in content_list:
+            segment_names.update(content.get_segment_names())
+        return segment_names

deltafi/result.py

@@ -1,7 +1,7 @@
 #
 #    DeltaFi - Data transformation and enrichment platform
 #
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#    Copyright 2021-2024 DeltaFi Contributors <deltafi@deltafi.org>
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -139,6 +139,12 @@ class IngressResultItem:
         self.metadata[key] = value
         return self
 
+    def get_segment_names(self):
+        segment_names = {}
+        for c in self.content:
+            segment_names.update(c.get_segment_names())
+        return segment_names
+
     def response(self):
         return {
             'did': self._did,
@@ -167,6 +173,12 @@ class IngressResult(Result):
         self.ingress_result_items.append(ingress_result_item)
         return self
 
+    def get_segment_names(self):
+        segment_names = {}
+        for ingress_item in self.ingress_result_items:
+            segment_names.update(ingress_item.get_segment_names())
+        return segment_names
+
     def response(self):
         return {
             'memo': self.memo,
@@ -225,6 +237,12 @@ class TransformResult(Result):
         self.delete_metadata_keys.append(key)
         return self
 
+    def get_segment_names(self):
+        segment_names = {}
+        for c in self.content:
+            segment_names.update(c.get_segment_names())
+        return segment_names
+
     def json(self):
         return {
             'content': [content.json() for content in self.content],
@@ -241,6 +259,9 @@ class NamedTransformResult(NamedTuple):
     result: TransformResult
     name: str
 
+    def get_segment_names(self):
+        return self.result.get_segment_names()
+
     def json(self):
         j = self.result.json()
         if self.name is not None:
@@ -257,6 +278,12 @@ class TransformResults(Result):
         self.named_results.append(NamedTransformResult(result, name))
         return self
 
+    def get_segment_names(self):
+        segment_names = {}
+        for named_result in self.named_results:
+            segment_names.update(named_result.get_segment_names())
+        return segment_names
+
     def response(self):
         transform_events = []
         for named_result in self.named_results:

deltafi/storage.py

@@ -1,7 +1,7 @@
 #
 #    DeltaFi - Data transformation and enrichment platform
 #
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#    Copyright 2021-2024 DeltaFi Contributors <deltafi@deltafi.org>
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -22,6 +22,7 @@ from typing import List, NamedTuple
 from urllib.parse import urlparse
 
 import minio
+from minio.deleteobjects import DeleteObject
 
 BUCKET = 'storage'
 
@@ -86,3 +87,7 @@ class ContentService:
 
     def put_str(self, did, string_data):
         return self.put_bytes(did, string_data.encode('utf-8'))
+
+    def delete_all(self, segments: List[Segment]):
+        delete_objects = [DeleteObject(seg.id()) for seg in segments]
+        return self.minio_client.remove_objects(BUCKET, delete_objects)

deltafi/test_kit/__init__.py

@@ -1,7 +1,7 @@
 #
 #    DeltaFi - Data transformation and enrichment platform
 #
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#    Copyright 2021-2024 DeltaFi Contributors <deltafi@deltafi.org>
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.

deltafi/test_kit/assertions.py

@@ -1,7 +1,7 @@
 #
 #    DeltaFi - Data transformation and enrichment platform
 #
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#    Copyright 2021-2024 DeltaFi Contributors <deltafi@deltafi.org>
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.

deltafi/test_kit/compare_helpers.py

@@ -1,7 +1,7 @@
 #
 #    DeltaFi - Data transformation and enrichment platform
 #
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#    Copyright 2021-2024 DeltaFi Contributors <deltafi@deltafi.org>
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.

deltafi/test_kit/constants.py

@@ -1,7 +1,7 @@
 #
 #    DeltaFi - Data transformation and enrichment platform
 #
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#    Copyright 2021-2024 DeltaFi Contributors <deltafi@deltafi.org>
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.

deltafi/test_kit/egress.py

@@ -1,7 +1,7 @@
 #
 #    DeltaFi - Data transformation and enrichment platform
 #
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#    Copyright 2021-2024 DeltaFi Contributors <deltafi@deltafi.org>
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.

deltafi/test_kit/framework.py

@@ -1,7 +1,7 @@
 #
 #    DeltaFi - Data transformation and enrichment platform
 #
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#    Copyright 2021-2024 DeltaFi Contributors <deltafi@deltafi.org>
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.

deltafi/test_kit/transform.py

@@ -1,7 +1,7 @@
 #
 #    DeltaFi - Data transformation and enrichment platform
 #
-#    Copyright 2021-2023 DeltaFi Contributors <deltafi@deltafi.org>
+#    Copyright 2021-2024 DeltaFi Contributors <deltafi@deltafi.org>
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.

{deltafi-2.0rc1725920935509.dist-info → deltafi-2.0rc1727460733389.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltafi
-Version: 2.0rc1725920935509
+Version: 2.0rc1727460733389
 Summary: SDK for DeltaFi plugins and actions
 License: Apache License, Version 2.0
 Keywords: deltafi

deltafi-2.0rc1727460733389.dist-info/RECORD

@@ -0,0 +1,23 @@
+deltafi/__init__.py,sha256=sSGWjpvzcCzgLywoEg55z_0satt0g_LPTwNIWtylgF4,709
+deltafi/action.py,sha256=Myc4RTYHLag6hXYd1pHZl3aguDPTrJ0MWN8x9nWG4Sw,5755
+deltafi/actioneventqueue.py,sha256=_05aDeqVpUbGWG3jFf0AalOsVfAvOTn0oBxjpQIQxIs,2847
+deltafi/actiontype.py,sha256=CpVI0wk9C-eu44e0jYvqBzE6yIX_zfRrW5RCf4ohu4Y,913
+deltafi/domain.py,sha256=1nUjpOF48UlytGKHep-lAy06U9RIe0H_dSeDJMsajng,12266
+deltafi/exception.py,sha256=LNNFqc_lA89I-RvnVsbTQ2vnWQ9WZXdQqLwiXwNgc_w,943
+deltafi/genericmodel.py,sha256=WU8zfqEO_n84CZ0KpH9FhgTsL9jyU0EXSuhL0IdLWFw,1152
+deltafi/input.py,sha256=ydAhuw68N9qMeacc9YE4U79zRqxWiinZNkm1AxoFBEk,1656
+deltafi/logger.py,sha256=4ahVx6E3TIZBTcvteaBeYXeAjk1oRR7vHkOienmtx4U,2141
+deltafi/metric.py,sha256=s68b1xNG9UPcpOB-nlTT-SntskJEUCpoKz1gTMRFgHc,967
+deltafi/plugin.py,sha256=20uXIB3XhaQAefBwKV-Mh7s4-sx9iWm9A_xLXbermeU,14374
+deltafi/result.py,sha256=s5jIoCa_deYHyl0T3lXLOcEVTROE2_U_7AQ2n4USgdI,9234
+deltafi/storage.py,sha256=vCE29Yzk5s7ub4O1PdaxXBKPBQy4OHs7j46z3O5MezE,2979
+deltafi/test_kit/__init__.py,sha256=sSGWjpvzcCzgLywoEg55z_0satt0g_LPTwNIWtylgF4,709
+deltafi/test_kit/assertions.py,sha256=MdUXENLn0aDvknMtsnSAb-DwvpzXlMkaYQ6-RRkWG8s,1378
+deltafi/test_kit/compare_helpers.py,sha256=qRINvCQqBUaalT1DG2oz2egSkUjiSOfqKF5U7WyeT_g,12957
+deltafi/test_kit/constants.py,sha256=Suygx9CEob2Skw4UyzzMCibQ8hRhGHC_d_Xab8AJMFE,833
+deltafi/test_kit/egress.py,sha256=53SBeJJmphDl-jZdqkC1dKYWZ4ATWlzY02TUt0Sj1zs,1899
+deltafi/test_kit/framework.py,sha256=w0vdG3sTGYFrd1cQN6r4H5ioXhjRfABJBGs6ZbNBEo0,15184
+deltafi/test_kit/transform.py,sha256=bnV6kcOkuShi_9uCtzkWcgz-wMejnI9Q2_MbyVm_A7U,4085
+deltafi-2.0rc1727460733389.dist-info/METADATA,sha256=9_YeyYlsaZ2DQqd8LV-Uv-t3dBLs9x-nHgzua9pVD14,1446
+deltafi-2.0rc1727460733389.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+deltafi-2.0rc1727460733389.dist-info/RECORD,,

deltafi-2.0rc1725920935509.dist-info/RECORD

@@ -1,23 +0,0 @@
-deltafi/__init__.py,sha256=qv6y7PpBG0ekTN9EuD9Lj8JYOLVqZA6tvHwgjplncAM,709
-deltafi/action.py,sha256=ejORyFe1cFCC04fdNt1VPWHqhNqZGjpLeD9tTQAYs1w,5755
-deltafi/actioneventqueue.py,sha256=mCRE1PFXy_KvaeezTeMze5N400CO1V06zEF8FFD6xZk,2847
-deltafi/actiontype.py,sha256=V7JbTgHQIaWQ2N36pG-XV7kAXhq8qvj954z71iHw_wI,913
-deltafi/domain.py,sha256=kuw0zwC2eMlJcksR5IU5_zJecU23RuUWwY26ks8hQp4,12100
-deltafi/exception.py,sha256=Z8LwxxTiTyuG3v7FAsjfz6cAqRAJFKK2q_2TbEDrIJE,943
-deltafi/genericmodel.py,sha256=LBIhU1rLU4HjoD8ooMfpbEhcRkrTIeGXCP1-0palKic,1152
-deltafi/input.py,sha256=2qQcOWqz7lOXZIZipTz5XtcGESpjdnUAsZ7b9YqHw7M,1656
-deltafi/logger.py,sha256=b6vTKSJjfXfzJIKB7bLrefIvlaIQAxWkeDJYzT1EXes,2141
-deltafi/metric.py,sha256=eIDjZQVO53KuAFoEtjLNFwqMrp_7BC0Td9GLD1tb6sE,967
-deltafi/plugin.py,sha256=oPopncj41OZ5EGzB2DfTaZEZwrIYOM3W8uSfCTq27RA,12679
-deltafi/result.py,sha256=353nPN4tSLskhf9ClJiGM2cy8vVMrjRaxsbzjmu8ONA,8382
-deltafi/storage.py,sha256=toq58EPZgzj2TfDF-YpFUmRnsWSjACA0KQAZzkM04xU,2740
-deltafi/test_kit/__init__.py,sha256=qv6y7PpBG0ekTN9EuD9Lj8JYOLVqZA6tvHwgjplncAM,709
-deltafi/test_kit/assertions.py,sha256=2eahqmbedhnHXMguiJHdaV-6sx4_jvCbxH4HSzx6PV8,1378
-deltafi/test_kit/compare_helpers.py,sha256=Q0e14q4XQ4rlMUbPt3aMHThAXHFRXlue3-M4oAayyJE,12957
-deltafi/test_kit/constants.py,sha256=epz0OS-qILcc8t7iIs5B3m2-wvZx8FpYpyb19ZsImbI,833
-deltafi/test_kit/egress.py,sha256=nF7YYak-_X35m7h2rSIcju86IJBf8cVaFHs7xw7-_b8,1899
-deltafi/test_kit/framework.py,sha256=hHJHyfx5yfBtixLdZGjqbjLdgvZ2idItunOOwQk_35c,15184
-deltafi/test_kit/transform.py,sha256=q9zQdZswTKwU5Om8E9Lwg0G5JI8LEXuRZjT9oam-Zoo,4085
-deltafi-2.0rc1725920935509.dist-info/METADATA,sha256=mEOhAUWSGCUgMx7UfwO0z6K7AT7vFelnlhF-ZA3Djs4,1446
-deltafi-2.0rc1725920935509.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-deltafi-2.0rc1725920935509.dist-info/RECORD,,