datapipelab 0.1__tar.gz

datapipelab-0.1/MANIFEST.in

@@ -0,0 +1 @@
+include README.md

datapipelab-0.1/PKG-INFO

@@ -0,0 +1,11 @@
+Metadata-Version: 2.4
+Name: datapipelab
+Version: 0.1
+Summary: A data pipeline library with connectors, sources, processors, and sinks.
+Requires-Dist: json5
+Requires-Dist: loguru
+Requires-Dist: azure-storage-blob
+Requires-Dist: google-cloud-storage
+Requires-Dist: pandas
+Dynamic: requires-dist
+Dynamic: summary

datapipelab-0.1/README.md

@@ -0,0 +1 @@
+# DataPipeLab

datapipelab-0.1/app/node/processor/custom_node.py

@@ -0,0 +1,21 @@
+from datapipelab.app.node.tnode import TNode
+from datapipelab.logger import logger
+
+class CustomNode(TNode):
+    def __init__(self, spark, tnode_config):
+        super().__init__(spark=spark)
+        self.tnode_config = tnode_config
+        self.spark = spark
+        module_name = tnode_config['options']['module_name']
+        module_path = tnode_config['options']['module_path']
+        class_name = tnode_config['options']['class_name']
+        self.custom_processor = self.import_module(module_name, module_path, class_name)
+
+    def import_module(self, module_name, module_path, class_name):
+        custom_module = __import__(module_path, fromlist=[module_name])
+        custom_class = getattr(custom_module, class_name)
+        return custom_class(self.spark, self.tnode_config)  # .create_instance(self.t_df)
+
+    def _process(self):
+        logger.info("Custom node process")
+        return self.custom_processor.process()

datapipelab-0.1/app/node/processor/spark_node.py

@@ -0,0 +1,16 @@
+from datapipelab.app.node.tnode import TNode
+
+
+class SparkProcessorNode(TNode):
+    def __init__(self, spark, tnode_config):
+        super().__init__(spark=spark)
+        self.sql_query = tnode_config['options']['query']
+        self.node_name = tnode_config['name']
+
+    def __sql_query(self, sql_query):
+        self.node = self.spark.sql(sql_query)
+
+    def _process(self):
+        self.__sql_query(self.sql_query)
+        self._createOrReplaceTempView()
+        return self.node

datapipelab-0.1/app/node/sink/csv_node.py

@@ -0,0 +1,33 @@
+from pyspark.sql import DataFrame
+from datapipelab.app.node.tnode import TNode
+
+
+class CSVSinkNode(TNode):
+    def __init__(self, spark, tnode_config, t_df):
+        super().__init__(spark=spark)
+        self.output_path = tnode_config['options']['path']
+        self.partition_by = tnode_config['options'].get('partition_by')
+        self.partition_count = tnode_config['options'].get('partition_count', 1)
+        self.overwrite = tnode_config['options'].get('overwrite', False)
+        self.header = tnode_config['options'].get('header', True)
+        self.df = t_df[tnode_config['options']['parents'][0]]
+        self.quote_all = tnode_config['options'].get('quote_all', False)
+        self.ignore_leading_white_space = tnode_config['options'].get('ignore_leading_white_space', True)
+        self.ignore_trailing_white_space = tnode_config['options'].get('ignore_trailing_white_space', True)
+
+    def __write_csv(self):
+        if self.partition_count:
+            if self.partition_by:
+                self.df = self.df.repartition(int(self.partition_count), *self.partition_by)
+            else:
+                self.df = self.df.repartition(int(self.partition_count))
+
+        write_mode = "overwrite" if self.overwrite else "errorifexists"
+
+        (self.df.write.mode(write_mode).option("quoteAll", self.quote_all).option("ignoreLeadingWhiteSpace",
+                                                                                  self.ignore_leading_white_space).option(
+            "ignoreTrailingWhiteSpace", self.ignore_trailing_white_space).option("header", self.header).csv(
+            self.output_path))
+
+    def _process(self):
+        self.__write_csv()

datapipelab-0.1/app/node/sink/delta_node.py

@@ -0,0 +1,46 @@
+from pyspark.sql import DataFrame
+from datapipelab.app.node.tnode import TNode
+from delta.tables import DeltaTable
+
+
+class DeltaSinkNode(TNode):
+    def __init__(self, spark, tnode_config, t_df):
+        super().__init__(spark=spark)
+        self.mode = tnode_config['options']['mode']  # Can be 'append', 'overwrite', or 'upsert'
+        self.partition_by = tnode_config['options'].get('partition_by')
+        self.partition_count = tnode_config['options'].get('partition_count')
+        self.df = t_df[tnode_config['options']['parents'][0]]
+        self.delta_table_path = tnode_config['options']['path']  # Path to the Delta table
+        self.primary_key = tnode_config['options'].get('primary_key', None)
+
+    def __write_append(self):
+        if self.partition_count:
+            self.df = self.df.repartition(int(self.partition_count), self.partition_by)
+        self.df.write.format("delta").mode("append").save(self.delta_table_path)
+
+    def __write_overwrite(self):
+        if self.partition_count:
+            self.df = self.df.repartition(int(self.partition_count), self.partition_by)
+        self.df.write.format("delta").mode("overwrite").option("overwriteSchema", "true").save(self.delta_table_path)
+
+    def __write_upsert(self):
+        delta_table = DeltaTable.forPath(self.spark, self.delta_table_path)
+        primary_key = self.primary_key
+        if primary_key is None:
+            raise ValueError("Primary key must be provided for upsert mode")
+        delta_table.alias("target").merge(
+            self.df.alias("source"),
+            " AND ".join([f"target.{key} = source.{key}" for key in primary_key])
+        ).whenMatchedUpdateAll(
+        ).whenNotMatchedInsertAll(
+        ).execute()
+
+    def _process(self):
+        if self.mode == 'append':
+            self.__write_append()
+        elif self.mode == 'overwrite':
+            self.__write_overwrite()
+        elif self.mode == 'upsert':
+            self.__write_upsert()
+        else:
+            raise ValueError(f"Unsupported mode: {self.mode}")

datapipelab-0.1/app/node/sink/hive_node.py

@@ -0,0 +1,26 @@
+from pyspark.sql import DataFrame
+from datapipelab.app.node.tnode import TNode
+
+
+class HiveSinkNode(TNode):
+    def __init__(self, spark, tnode_config, df):
+        super().__init__(spark=spark)
+        self.mode = tnode_config['mode']
+        self.stream = tnode_config['stream']
+        self.database_name = tnode_config['options']['database']
+        self.table_name = tnode_config['options']['table']
+        self.partition_by = tnode_config['options'].get('partition_by')
+        self.partition_count = tnode_config['options'].get('partition_count')
+        self.overwrite = tnode_config['options']['overwrite']
+        self.df = df
+
+    def __write_dynamic_partition(self):
+        if self.partition_count:
+            if self.partition_by:
+                self.df = self.df.repartition(int(self.partition_count))
+            else:
+                self.df = self.df.repartition(int(self.partition_count), self.partition_by)
+        self.df.write.insertInto(f'{self.database_name}.{self.table_name}', overwrite=self.overwrite)
+
+    def _process(self):
+        self.__write_dynamic_partition()

datapipelab-0.1/app/node/sink/pandas_csv_node.py

@@ -0,0 +1,23 @@
+from pyspark.sql import DataFrame
+from datapipelab.app.node.tnode import TNode
+
+
+class PandasCSVSinkNode(TNode):
+    def __init__(self, spark, tnode_config, t_df):
+        super().__init__(spark=spark)
+        self.mode = tnode_config['options'].get('mode', 'w')
+        # self.stream = tnode_config['stream']
+        self.output_path = tnode_config['options']['path']
+        self.overwrite = tnode_config['options'].get('overwrite', False)
+        self.header = tnode_config['options'].get('header', True)
+        self.df = t_df[tnode_config['options']['parents'][0]]
+
+    def __write_csv(self):
+        import pandas as pd
+        pandas_df = self.df.toPandas()
+        write_mode = "w" if self.overwrite else "x"
+
+        pandas_df.to_csv(self.output_path, mode=write_mode, header=self.header, index=False)
+
+    def _process(self):
+        self.__write_csv()

datapipelab-0.1/app/node/sink/teams_notification_node.py

@@ -0,0 +1,83 @@
+from pyspark.sql import DataFrame
+from datapipelab.app.node.tnode import TNode
+import json
+
+
+class TeamsNotificationSinkNode(TNode):
+    def __init__(self, spark, tnode_config, df=None):
+        super().__init__(spark=spark)
+        self.teams_msg_body = tnode_config['options']['teams_msg_body']
+        self.teams_msg_title = tnode_config['options'].get('teams_msg_title', 'Notification')
+        self.teams_users = tnode_config['options'].get('teams_users', None)
+        self.teams_channel_webhook_url = tnode_config['options']['teams_channel_webhook_url']
+        self.df = df
+
+    def __prepare_teams_notification_payload(self, teams_msg_body: list, teams_msg_title: str = "Notification",
+                                           teams_users: list = None):
+        if teams_users is not None:
+            teams_msg_body.extend([f'<at>{user}</at>' for user in teams_users])
+            print(teams_msg_body)
+            final_msg = '   \n'.join(teams_msg_body)
+            entities = []
+            for user_id in teams_users:
+                mention = {
+                    "type": "mention",
+                    "text": f"<at>{user_id}</at>",
+                    "mentioned": {
+                        "id": f"{user_id}@cantire.com",
+                        "name": f"{user_id}"
+                    }
+                }
+                entities.append(mention)
+            payload = {
+                "type": "message",
+                "attachments": [
+                    {
+                        "contentType": "application/vnd.microsoft.card.adaptive",
+                        "content": {
+                            "type": "AdaptiveCard",
+                            "body": [
+                                {
+                                    "type": "TextBlock",
+                                    "size": "Medium",
+                                    "weight": "Bolder",
+                                    "text": f"{teams_msg_title}"
+                                },
+                                {
+                                    "type": "TextBlock",
+                                    "text": f"{final_msg}",
+                                    "wrap": "true",
+                                    "maxLines": 0
+                                }
+                            ],
+                            "$schema": "http://adaptivecards.io/schemas/adaptive-card.json",
+                            "version": "1.0",
+                            "msteams": {
+                                "entities": entities
+                            }
+                        }
+                    }]
+            }
+        else:
+            final_msg = '   \n'.join(teams_msg_body)
+            payload = {
+                "text": f"{final_msg}",
+                "title": f"{teams_msg_title}"
+            }
+        return payload
+    def __send_teams_notification(self):
+        import requests
+        payload = self.__prepare_teams_notification_payload(self.teams_msg_body, self.teams_msg_title, self.teams_users)
+        headers = {"Content-Type": "application/json"}
+        try:
+            response = requests.post(self.teams_channel_webhook_url, headers=headers, data=json.dumps(payload))
+            if response.status_code == 200:
+                print("Message sent successfully!")
+            else:
+                print(f"Failed to send message: {response.status_code}, {response.text}")
+        except Exception as e:
+            print(f"An error occurred: {e}")
+
+
+    def _process(self):
+        self.__send_teams_notification()

datapipelab-0.1/app/node/source/delta_node.py

@@ -0,0 +1,16 @@
+from datapipelab.app.node.tnode import TNode
+
+
+class DeltaSourceNode(TNode):
+    def __init__(self, spark, tnode_config):
+        super().__init__(spark=spark)
+        self.delta_table_path = tnode_config['options']['path']
+        self.node_name = tnode_config['name']
+
+    def __sql_query(self, delta_table_path):
+        self.node = self.spark.read.format("delta").load(delta_table_path)
+
+    def _process(self):
+        self.__sql_query(self.delta_table_path)
+        self._createOrReplaceTempView()
+        return self.node

datapipelab-0.1/app/node/source/hive_node.py

@@ -0,0 +1,16 @@
+from datapipelab.app.node.tnode import TNode
+
+
+class HiveSourceNode(TNode):
+    def __init__(self, spark, tnode_config):
+        super().__init__(spark=spark)
+        self.sql_query = tnode_config['options']['query']
+        self.node_name = tnode_config['name']
+
+    def __sql_query(self, sql_query):
+        self.node = self.spark.sql(sql_query)
+
+    def _process(self):
+        self.__sql_query(self.sql_query)
+        self._createOrReplaceTempView()
+        return self.node

datapipelab-0.1/app/node/source/spark_node.py

@@ -0,0 +1,16 @@
+from datapipelab.app.node.tnode import TNode
+
+
+class SparkSourceNode(TNode):
+    def __init__(self, spark, tnode_config):
+        super().__init__(spark=spark)
+        self.sql_query = tnode_config['options']['query']
+        self.node_name = tnode_config['name']
+
+    def __sql_query(self, sql_query):
+        self.node = self.spark.sql(sql_query)
+
+    def _process(self):
+        self.__sql_query(self.sql_query)
+        self._createOrReplaceTempView()
+        return self.node

datapipelab-0.1/app/node/tnode.py

@@ -0,0 +1,15 @@
+class TNode:
+    def __init__(self, spark, node_type='SparkDataFrame'):
+        self.node_type = node_type
+        self.node = None
+        self.spark = spark
+
+    def _process(self):
+        raise NotImplementedError("Subclasses must implement _process method")
+
+    # Source and Processor nodes
+    def _createOrReplaceTempView(self):
+        self.node.createOrReplaceTempView(self.node_name)
+
+    def run(self):
+        return self._process()

datapipelab-0.1/datapipelab.egg-info/PKG-INFO

@@ -0,0 +1,11 @@
+Metadata-Version: 2.4
+Name: datapipelab
+Version: 0.1
+Summary: A data pipeline library with connectors, sources, processors, and sinks.
+Requires-Dist: json5
+Requires-Dist: loguru
+Requires-Dist: azure-storage-blob
+Requires-Dist: google-cloud-storage
+Requires-Dist: pandas
+Dynamic: requires-dist
+Dynamic: summary

datapipelab-0.1/datapipelab.egg-info/SOURCES.txt

@@ -0,0 +1,25 @@
+MANIFEST.in
+README.md
+setup.py
+app/__init__.py
+app/connector_node/__init__.py
+app/node/__init__.py
+app/node/tnode.py
+app/node/processor/__init__.py
+app/node/processor/custom_node.py
+app/node/processor/spark_node.py
+app/node/sink/__init__.py
+app/node/sink/csv_node.py
+app/node/sink/delta_node.py
+app/node/sink/hive_node.py
+app/node/sink/pandas_csv_node.py
+app/node/sink/teams_notification_node.py
+app/node/source/__init__.py
+app/node/source/delta_node.py
+app/node/source/hive_node.py
+app/node/source/spark_node.py
+datapipelab.egg-info/PKG-INFO
+datapipelab.egg-info/SOURCES.txt
+datapipelab.egg-info/dependency_links.txt
+datapipelab.egg-info/requires.txt
+datapipelab.egg-info/top_level.txt

datapipelab-0.1/datapipelab.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

datapipelab-0.1/datapipelab.egg-info/requires.txt

@@ -0,0 +1,5 @@
+json5
+loguru
+azure-storage-blob
+google-cloud-storage
+pandas

datapipelab-0.1/datapipelab.egg-info/top_level.txt

@@ -0,0 +1 @@
+app

datapipelab-0.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

datapipelab-0.1/setup.py

@@ -0,0 +1,16 @@
+from setuptools import setup, find_packages
+
+setup(
+    name='datapipelab',
+    version='0.1',
+    description='A data pipeline library with connectors, sources, processors, and sinks.',
+    packages=find_packages(),
+    include_package_data=True,
+    install_requires=[
+        'json5',
+        'loguru',
+        'azure-storage-blob',
+        'google-cloud-storage',
+        'pandas'
+    ],
+)