datapipelab 0.2.9__py3-none-any.whl → 0.3.0__py3-none-any.whl

datapipelab/app/node/processor/gcp_bucket_node.py

@@ -0,0 +1,42 @@
+from datapipelab.app.node.tnode import TNode
+from datapipelab.logger import logger
+
+class GCPBucketAPINode(TNode):
+    def __init__(self, spark, tnode_config):
+        super().__init__(spark=spark)
+        self.node_name = tnode_config['name']
+        self.credentials_path = tnode_config['options']['credentials_path']
+        self.project_name = tnode_config['options']['project_name']
+        self.bucket_name = tnode_config['options']['bucket_name']
+        self.prefix = tnode_config['options'].get('subdirectory', None)  # Optional subdirectory (prefix) to delete
+
+    def __delete_gcs_folder(self, bucket_name, prefix=None):
+        from google.cloud import storage
+        from google.oauth2 import service_account
+        credentials = service_account.Credentials.from_service_account_file(self.credentials_path)
+        client = storage.Client(credentials=credentials, project=self.project_name)
+
+        bucket = client.bucket(bucket_name)
+
+        if prefix:
+            # Delete only objects under the prefix (subfolder)
+            blobs = bucket.list_blobs(prefix=prefix)
+            deleted = False
+            for blob in blobs:
+                blob.delete()
+                deleted = True
+            if deleted:
+                logger.info(f"Deleted all objects under prefix '{prefix}' in bucket '{bucket_name}'.")
+            else:
+                logger.info(f"No objects found under prefix '{prefix}' in bucket '{bucket_name}'.")
+        else:
+            # Delete the entire bucket (must be empty)
+            try:
+                bucket.delete(force=True)  # force=True to delete non-empty bucket
+                logger.info(f"Bucket '{bucket_name}' deleted.")
+            except Exception as e:
+                logger.info(f"Error deleting bucket '{bucket_name}': {e}")
+
+    def _process(self):
+        self.__delete_gcs_folder(self.bucket_name, self.prefix)
+        return None

datapipelab/pipeline_handler.py

@@ -10,6 +10,7 @@ from datapipelab.app.node.sink.pandas_csv_node import PandasCSVSinkNode
 from datapipelab.app.node.sink.teams_notification_node import TeamsNotificationSinkNode
 from datapipelab.app.node.processor.bigquery_spark_node import BigQuerySparkProcessorNode
 from datapipelab.app.node.processor.bigquery_api_node import BigQueryAPIProcessorNode
+from datapipelab.app.node.processor.gcp_bucket_node import GCPBucketAPINode
 
 
 class PipelineHandler:
@@ -59,6 +60,8 @@ class PipelineHandler:
             processor_df = BigQuerySparkProcessorNode(self.spark, tnode_config).run()
         if tnode_format == 'shell':
             processor_df = ShellProcessorNode(self.spark, tnode_config).run()
+        if tnode_format == 'gcp_bucket_api':
+            processor_df = GCPBucketAPINode(self.spark, tnode_config).run()
         return processor_df
 
     def write_sink_node(self, tnode_config, t_df):

{datapipelab-0.2.9.dist-info → datapipelab-0.3.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datapipelab
-Version: 0.2.9
+Version: 0.3.0
 Summary: A data pipeline library with connectors, sources, processors, and sinks.
 Requires-Dist: json5
 Requires-Dist: loguru

{datapipelab-0.2.9.dist-info → datapipelab-0.3.0.dist-info}/RECORD

@@ -3,7 +3,7 @@ datapipelab/engine.py,sha256=3QRsedRYNov6xIDOZ1tukinFE-SKv39Fn3sNCnD3L6g,442
 datapipelab/logger.py,sha256=Ugv0A4TfD3JWCWXNWu0lURcnfAEyuVrK3IrvVVgcHBo,864
 datapipelab/pipeline.py,sha256=dw9D9KM_hztt9g_YzqoNgQBRyCYR92cRZwrU5duP_Pg,1464
 datapipelab/pipeline_config.py,sha256=2bFAJepViE7rT7CaRANZU07aeQpOYcZ954ISujm9pXA,3816
-datapipelab/pipeline_handler.py,sha256=xlDNHUr4bhlvpj6rpI6Mbfe8jsHG0-rQF4EasRUPU8Y,4783
+datapipelab/pipeline_handler.py,sha256=Q1AzuPgOb9bElclfX-E8PiTGrdOKzFshQUjbNNXp5m0,4980
 datapipelab/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datapipelab/app/connector_node/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datapipelab/app/node/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -13,6 +13,7 @@ datapipelab/app/node/processor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5
 datapipelab/app/node/processor/bigquery_api_node.py,sha256=IclDkGxo9ltGJVkBaHKFPFCSlEEyzefgalaAOLA17bE,1752
 datapipelab/app/node/processor/bigquery_spark_node.py,sha256=S9kIYW0RE5b0RjniKFFBTzA3Tx4_plFdkFQXzhl1xTY,1039
 datapipelab/app/node/processor/custom_node.py,sha256=1nqbJEhNiMP1rmN9ufpUuKO1IkuI2BEM5auW4JceGMA,933
+datapipelab/app/node/processor/gcp_bucket_node.py,sha256=bzV2c89-g5S0OH5bcKKQ-9yKOGwlmOR7h7_5uO6Gnq0,1904
 datapipelab/app/node/processor/shell_node.py,sha256=s3dKgfEqbpUIEiwORERgvp7FNDE5JkFHBo7EnJYBPnA,669
 datapipelab/app/node/processor/spark_node.py,sha256=jzqdffIHUCgOfMFcoqjXdl8wFag-3gafxfNCdssKnwc,483
 datapipelab/app/node/sink/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -26,7 +27,9 @@ datapipelab/app/node/source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMp
 datapipelab/app/node/source/delta_node.py,sha256=gg7SfuKBAAfjk6OX2jNrot9XX61HoBe3us3D8O-dscE,529
 datapipelab/app/node/source/hive_node.py,sha256=h_AMCnnmon7uLRIGsaHAPWEReD3VaWZXnz9r0TpLGNM,478
 datapipelab/app/node/source/spark_node.py,sha256=S_x2atRFPDnXmhCUtcmaLc4BDFd2H4uQq6wnEJb7Uug,480
-datapipelab-0.2.9.dist-info/METADATA,sha256=nIXYk_x-oMW4E1mZ0yDS1YniCbafMy9iGNNwXxcWbZI,220
-datapipelab-0.2.9.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-datapipelab-0.2.9.dist-info/top_level.txt,sha256=HgeBjHvXorKzvNqU5BNPutoI771HtiqVit9_-0Zyrb4,12
-datapipelab-0.2.9.dist-info/RECORD,,
+datapipelab/app/wrapper/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datapipelab/app/wrapper/source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datapipelab-0.3.0.dist-info/METADATA,sha256=uXwxvNNpO_FSDUi8rC5bxPNImGtLuZuSMUV9QDNptT0,220
+datapipelab-0.3.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+datapipelab-0.3.0.dist-info/top_level.txt,sha256=HgeBjHvXorKzvNqU5BNPutoI771HtiqVit9_-0Zyrb4,12
+datapipelab-0.3.0.dist-info/RECORD,,