datapipelab 0.1.4__tar.gz → 0.1.6__tar.gz

{datapipelab-0.1.4 → datapipelab-0.1.6}/PKG-INFO

@@ -1,11 +1,8 @@
 Metadata-Version: 2.4
 Name: datapipelab
-Version: 0.1.4
+Version: 0.1.6
 Summary: A data pipeline library with connectors, sources, processors, and sinks.
 Requires-Dist: json5
 Requires-Dist: loguru
-Requires-Dist: azure-storage-blob
-Requires-Dist: google-cloud-storage
-Requires-Dist: pandas
 Dynamic: requires-dist
 Dynamic: summary

datapipelab-0.1.6/datapipelab/app/node/custom_node.py

@@ -0,0 +1,22 @@
+from datapipelab.app.node.tnode import TNode
+from datapipelab.logger import logger
+
+class CustomNode(TNode):
+    def __init__(self, spark, tnode_config, t_df):
+        super().__init__(spark=spark)
+        self.tnode_config = tnode_config
+        self.spark = spark
+        self.t_df = t_df
+        module_name = tnode_config['options']['module_name']
+        module_path = tnode_config['options']['module_path']
+        class_name = tnode_config['options']['class_name']
+        self.custom_processor = self.import_module(module_name, module_path, class_name)
+
+    def import_module(self, module_name, module_path, class_name):
+        custom_module = __import__(module_path, fromlist=[module_name])
+        custom_class = getattr(custom_module, class_name)
+        return custom_class(self.spark, self.tnode_config)  # .create_instance(self.t_df)
+
+    def _process(self):
+        logger.info(f"Custom node type is {self.tnode_config.get('custom_node_type', 'N/A!')}")
+        return self.custom_processor.process()

datapipelab-0.1.6/datapipelab/app/node/processor/bigquery_api_node.py

@@ -0,0 +1,33 @@
+from datapipelab.app.node.tnode import TNode
+from datapipelab.logger import logger
+
+class BigQueryAPIProcessorNode(TNode):
+    def __init__(self, spark, tnode_config):
+        from google.cloud import bigquery
+        from google.oauth2 import service_account
+        super().__init__(spark=spark)
+        self.sql_query = tnode_config['options']['query']
+        self.node_name = tnode_config['name']
+        self.credentials_path = tnode_config['options']['credentials_path']
+        self.return_as_spark_df = tnode_config['options']['return_as_spark_df']
+        self.project_name = tnode_config['options']['project_name']
+
+    def __sql_query(self, sql_query):
+        credentials = service_account.Credentials.from_service_account_file(self.credentials_path)
+        client = bigquery.Client(credentials=credentials, project=self.project_name)
+
+        # run the job
+        query_job = client.query(sql_query)
+
+        results = query_job.result()
+        rows = [dict(row) for row in results]
+        if self.return_as_spark_df:
+            self.node = self.spark.createDataFrame(rows)
+        else:
+            self.node = None
+        logger.info(rows)
+
+    def _process(self):
+        self.__sql_query(self.sql_query)
+        self._createOrReplaceTempView()
+        return self.node

datapipelab-0.1.6/datapipelab/app/node/processor/bigquery_spark_node.py

@@ -0,0 +1,30 @@
+from datapipelab.app.node.tnode import TNode
+from datapipelab.logger import logger
+
+class BigQuerySparkProcessorNode(TNode):
+    def __init__(self, spark, tnode_config):
+        super().__init__(spark=spark)
+        self.sql_query = tnode_config['options']['query']
+        self.node_name = tnode_config['name']
+        self.credentials_path = tnode_config['options']['materialization_dataset'] # materializationDataset
+        self.return_as_spark_df = tnode_config['options']['parent_project'] # parentProject
+
+    def __sql_query(self, sql_query):
+        credentials = service_account.Credentials.from_service_account_file(self.credentials_path)
+        client = bigquery.Client(credentials=credentials, project=self.project_name)
+
+        # run the job
+        query_job = client.query(sql_query)
+
+        results = query_job.result()
+        rows = [dict(row) for row in results]
+        if self.return_as_spark_df:
+            self.node = self.spark.createDataFrame(rows)
+        else:
+            self.node = None
+        logger.info(rows)
+
+    def _process(self):
+        self.__sql_query(self.sql_query)
+        self._createOrReplaceTempView()
+        return self.node

datapipelab-0.1.6/datapipelab/app/node/processor/shell_node.py

@@ -0,0 +1,23 @@
+from datapipelab.app.node.tnode import TNode
+from datapipelab.logger import logger
+
+class ShellProcessorNode(TNode):
+    def __init__(self, spark, tnode_config):
+
+        super().__init__(spark=spark)
+        self.shell_query = tnode_config['options']['query']
+        self.node_name = tnode_config['name']
+
+    def __shell_query(self):
+        import subprocess
+        # run the job
+        result = subprocess.run(
+            f"{self.shell_query}",
+            shell=True, check=True, executable='/bin/bash'
+        )
+        logger.info(result)
+
+    def _process(self):
+        self.__shell_query()
+        self._createOrReplaceTempView()
+        return self.node

{datapipelab-0.1.4 → datapipelab-0.1.6}/datapipelab/app/node/sink/csv_node.py

@@ -1,9 +1,9 @@
-from pyspark.sql import DataFrame
 from datapipelab.app.node.tnode import TNode
 
 
 class CSVSinkNode(TNode):
     def __init__(self, spark, tnode_config, t_df):
+        from pyspark.sql import DataFrame
         super().__init__(spark=spark)
         self.output_path = tnode_config['options']['path']
         self.partition_by = tnode_config['options'].get('partition_by')

{datapipelab-0.1.4 → datapipelab-0.1.6}/datapipelab/app/node/sink/delta_node.py

@@ -1,10 +1,10 @@
 from pyspark.sql import DataFrame
 from datapipelab.app.node.tnode import TNode
-from delta.tables import DeltaTable
 
 
 class DeltaSinkNode(TNode):
     def __init__(self, spark, tnode_config, t_df):
+        from delta.tables import DeltaTable
         super().__init__(spark=spark)
         self.mode = tnode_config['options']['mode']  # Can be 'append', 'overwrite', or 'upsert'
         self.partition_by = tnode_config['options'].get('partition_by')

{datapipelab-0.1.4 → datapipelab-0.1.6}/datapipelab/app/node/sink/hive_node.py

@@ -1,9 +1,9 @@
-from pyspark.sql import DataFrame
 from datapipelab.app.node.tnode import TNode
 
 
 class HiveSinkNode(TNode):
     def __init__(self, spark, tnode_config, df):
+        from pyspark.sql import DataFrame
         super().__init__(spark=spark)
         self.mode = tnode_config['mode']
         self.stream = tnode_config['stream']

{datapipelab-0.1.4 → datapipelab-0.1.6}/datapipelab/app/node/sink/pandas_csv_node.py

@@ -1,9 +1,9 @@
-from pyspark.sql import DataFrame
 from datapipelab.app.node.tnode import TNode
 
 
 class PandasCSVSinkNode(TNode):
     def __init__(self, spark, tnode_config, t_df):
+        from pyspark.sql import DataFrame
         super().__init__(spark=spark)
         self.mode = tnode_config['options'].get('mode', 'w')
         # self.stream = tnode_config['stream']

{datapipelab-0.1.4 → datapipelab-0.1.6}/datapipelab/app/node/sink/teams_notification_node.py

@@ -1,10 +1,11 @@
-from pyspark.sql import DataFrame
 from datapipelab.app.node.tnode import TNode
-import json
+
 
 
 class TeamsNotificationSinkNode(TNode):
     def __init__(self, spark, tnode_config, df=None):
+        from pyspark.sql import DataFrame
+        import json
         super().__init__(spark=spark)
         self.teams_msg_body = tnode_config['options']['teams_msg_body']
         self.teams_msg_title = tnode_config['options'].get('teams_msg_title', 'Notification')

{datapipelab-0.1.4 → datapipelab-0.1.6}/datapipelab/engine.py

@@ -3,7 +3,7 @@ from datapipelab.logger import logger
 
 
 class Engine:
-    def __init__(self, engine_config_path, spark, params):
+    def __init__(self, engine_config_path, spark, params=None):
         self.engine_config_path = engine_config_path
         self.params = params
         self.pipeline = None

{datapipelab-0.1.4 → datapipelab-0.1.6}/datapipelab/pipeline_config.py

@@ -6,14 +6,14 @@ class PipelineConfig:
         print(config_file)
         self.params = params
         self.config_file = config_file
-        # If config file is instance of string, it is a path to the config file
+        # If the config file is an instance of string, it is a path to the config file
         self.config_file = config_file
         if isinstance(self.config_file, str):
             self.load_json_config_file()
-        elif isinstance(self.config_file, dict):
+        elif isinstance(self.config_file, list):
             self.pipeline_settings = config_file
         else:
-            raise ValueError("Invalid config file type. Must be a string or a dictionary.")
+            raise ValueError("Invalid config file type. Must be a string or a list.")
         self.sources = {}
         self.processors = {}
         self.sinks = {}
@@ -27,6 +27,14 @@ class PipelineConfig:
                 json_config_file = json_config_file.replace(f"{{{key}}}", value)
             # Convert to JSON file
             self.pipeline_settings = json5.loads(json_config_file)
+        if len(self.pipeline_settings) > 0 and self.pipeline_settings[0]['type'] == 'import':
+            self.import_json_config_file()
+
+    def import_json_config_file(self):
+        import_pipeline_settings = []
+        for import_component in self.pipeline_settings:
+            if import_component['type'] == '': # Maybe someone wants to use import in the middle of his config?
+                pass
 
     def create_pipeline_nodes(self):
         for component in self.pipeline_settings:

{datapipelab-0.1.4 → datapipelab-0.1.6}/datapipelab/pipeline_handler.py

@@ -6,6 +6,7 @@ from datapipelab.app.node.sink.delta_node import DeltaSinkNode
 from datapipelab.app.node.sink.csv_node import CSVSinkNode
 from datapipelab.app.node.sink.pandas_csv_node import PandasCSVSinkNode
 from datapipelab.app.node.sink.teams_notification_node import TeamsNotificationSinkNode
+from datapipelab.app.node.processor.bigquery_spark_node import BigQuerySparkProcessorNode
 
 
 class PipelineHandler:
@@ -42,10 +43,12 @@ class PipelineHandler:
         processor_df = None
         if tnode_format == 'custom':
             processor_df = CustomNode(self.spark, tnode_config).run()
-
         if tnode_format == 'query':
             processor_df = SparkProcessorNode(self.spark, tnode_config).run()
-
+        if tnode_format == 'bigquery_api':
+            processor_df = None
+        if tnode_format == 'bigquery_spark':
+            processor_df = datapipelab.app.node.processor.bigquery_spark.BigQuerySparkProcessorNode(self.spark, tnode_config).run()
         return processor_df
 
     def write_sink_node(self, tnode_config, t_df):
@@ -70,5 +73,8 @@ class PipelineHandler:
         if tnode_type == "local":
             if tnode_format == "csv":
                 PandasCSVSinkNode(self.spark, tnode_config, t_df).run()
+        if tnode_type == 'custom':
+            from datapipelab.app.node import custom_node
+            processor_df = custom_node.CustomNode(self.spark, tnode_config).run()
 
 

{datapipelab-0.1.4 → datapipelab-0.1.6}/datapipelab.egg-info/PKG-INFO

@@ -1,11 +1,8 @@
 Metadata-Version: 2.4
 Name: datapipelab
-Version: 0.1.4
+Version: 0.1.6
 Summary: A data pipeline library with connectors, sources, processors, and sinks.
 Requires-Dist: json5
 Requires-Dist: loguru
-Requires-Dist: azure-storage-blob
-Requires-Dist: google-cloud-storage
-Requires-Dist: pandas
 Dynamic: requires-dist
 Dynamic: summary

{datapipelab-0.1.4 → datapipelab-0.1.6}/datapipelab.egg-info/SOURCES.txt

@@ -15,9 +15,13 @@ datapipelab.egg-info/top_level.txt
 datapipelab/app/__init__.py
 datapipelab/app/connector_node/__init__.py
 datapipelab/app/node/__init__.py
+datapipelab/app/node/custom_node.py
 datapipelab/app/node/tnode.py
 datapipelab/app/node/processor/__init__.py
+datapipelab/app/node/processor/bigquery_api_node.py
+datapipelab/app/node/processor/bigquery_spark_node.py
 datapipelab/app/node/processor/custom_node.py
+datapipelab/app/node/processor/shell_node.py
 datapipelab/app/node/processor/spark_node.py
 datapipelab/app/node/sink/__init__.py
 datapipelab/app/node/sink/csv_node.py

datapipelab-0.1.6/datapipelab.egg-info/requires.txt

@@ -0,0 +1,2 @@
+json5
+loguru

{datapipelab-0.1.4 → datapipelab-0.1.6}/setup.py

@@ -2,15 +2,15 @@ from setuptools import setup, find_packages
 
 setup(
     name='datapipelab',
-    version='0.1.4',
+    version='0.1.6',
     description='A data pipeline library with connectors, sources, processors, and sinks.',
     packages=find_packages(),
     include_package_data=True,
     install_requires=[
         'json5',
         'loguru',
-        'azure-storage-blob',
-        'google-cloud-storage',
-        'pandas'
+        # 'azure-storage-blob',
+        # 'google-cloud-storage',
+        # 'pandas'
     ],
 )

datapipelab-0.1.4/datapipelab.egg-info/requires.txt

@@ -1,5 +0,0 @@
-json5
-loguru
-azure-storage-blob
-google-cloud-storage
-pandas