datapipelab 0.2.9__tar.gz → 0.3.0__tar.gz

{datapipelab-0.2.9 → datapipelab-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datapipelab
-Version: 0.2.9
+Version: 0.3.0
 Summary: A data pipeline library with connectors, sources, processors, and sinks.
 Requires-Dist: json5
 Requires-Dist: loguru

datapipelab-0.3.0/datapipelab/app/node/processor/gcp_bucket_node.py

@@ -0,0 +1,42 @@
+from datapipelab.app.node.tnode import TNode
+from datapipelab.logger import logger
+
+class GCPBucketAPINode(TNode):
+    def __init__(self, spark, tnode_config):
+        super().__init__(spark=spark)
+        self.node_name = tnode_config['name']
+        self.credentials_path = tnode_config['options']['credentials_path']
+        self.project_name = tnode_config['options']['project_name']
+        self.bucket_name = tnode_config['options']['bucket_name']
+        self.prefix = tnode_config['options'].get('subdirectory', None)  # Optional subdirectory (prefix) to delete
+
+    def __delete_gcs_folder(self, bucket_name, prefix=None):
+        from google.cloud import storage
+        from google.oauth2 import service_account
+        credentials = service_account.Credentials.from_service_account_file(self.credentials_path)
+        client = storage.Client(credentials=credentials, project=self.project_name)
+
+        bucket = client.bucket(bucket_name)
+
+        if prefix:
+            # Delete only objects under the prefix (subfolder)
+            blobs = bucket.list_blobs(prefix=prefix)
+            deleted = False
+            for blob in blobs:
+                blob.delete()
+                deleted = True
+            if deleted:
+                logger.info(f"Deleted all objects under prefix '{prefix}' in bucket '{bucket_name}'.")
+            else:
+                logger.info(f"No objects found under prefix '{prefix}' in bucket '{bucket_name}'.")
+        else:
+            # Delete the entire bucket (must be empty)
+            try:
+                bucket.delete(force=True)  # force=True to delete non-empty bucket
+                logger.info(f"Bucket '{bucket_name}' deleted.")
+            except Exception as e:
+                logger.info(f"Error deleting bucket '{bucket_name}': {e}")
+
+    def _process(self):
+        self.__delete_gcs_folder(self.bucket_name, self.prefix)
+        return None

{datapipelab-0.2.9 → datapipelab-0.3.0}/datapipelab/pipeline_handler.py

@@ -10,6 +10,7 @@ from datapipelab.app.node.sink.pandas_csv_node import PandasCSVSinkNode
 from datapipelab.app.node.sink.teams_notification_node import TeamsNotificationSinkNode
 from datapipelab.app.node.processor.bigquery_spark_node import BigQuerySparkProcessorNode
 from datapipelab.app.node.processor.bigquery_api_node import BigQueryAPIProcessorNode
+from datapipelab.app.node.processor.gcp_bucket_node import GCPBucketAPINode
 
 
 class PipelineHandler:
@@ -59,6 +60,8 @@ class PipelineHandler:
             processor_df = BigQuerySparkProcessorNode(self.spark, tnode_config).run()
         if tnode_format == 'shell':
             processor_df = ShellProcessorNode(self.spark, tnode_config).run()
+        if tnode_format == 'gcp_bucket_api':
+            processor_df = GCPBucketAPINode(self.spark, tnode_config).run()
         return processor_df
 
     def write_sink_node(self, tnode_config, t_df):

{datapipelab-0.2.9 → datapipelab-0.3.0}/datapipelab.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datapipelab
-Version: 0.2.9
+Version: 0.3.0
 Summary: A data pipeline library with connectors, sources, processors, and sinks.
 Requires-Dist: json5
 Requires-Dist: loguru

{datapipelab-0.2.9 → datapipelab-0.3.0}/datapipelab.egg-info/SOURCES.txt

@@ -21,6 +21,7 @@ datapipelab/app/node/processor/__init__.py
 datapipelab/app/node/processor/bigquery_api_node.py
 datapipelab/app/node/processor/bigquery_spark_node.py
 datapipelab/app/node/processor/custom_node.py
+datapipelab/app/node/processor/gcp_bucket_node.py
 datapipelab/app/node/processor/shell_node.py
 datapipelab/app/node/processor/spark_node.py
 datapipelab/app/node/sink/__init__.py
@@ -33,4 +34,6 @@ datapipelab/app/node/sink/teams_notification_node.py
 datapipelab/app/node/source/__init__.py
 datapipelab/app/node/source/delta_node.py
 datapipelab/app/node/source/hive_node.py
-datapipelab/app/node/source/spark_node.py
+datapipelab/app/node/source/spark_node.py
+datapipelab/app/wrapper/__init__.py
+datapipelab/app/wrapper/source/__init__.py

{datapipelab-0.2.9 → datapipelab-0.3.0}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='datapipelab',
-    version='0.2.9',
+    version='0.3.0',
     description='A data pipeline library with connectors, sources, processors, and sinks.',
     packages=find_packages(),
     include_package_data=True,