datapipelab 0.1.4__py3-none-any.whl → 0.1.6__py3-none-any.whl

datapipelab/app/node/custom_node.py

@@ -0,0 +1,22 @@
+from datapipelab.app.node.tnode import TNode
+from datapipelab.logger import logger
+
+class CustomNode(TNode):
+    def __init__(self, spark, tnode_config, t_df):
+        super().__init__(spark=spark)
+        self.tnode_config = tnode_config
+        self.spark = spark
+        self.t_df = t_df
+        module_name = tnode_config['options']['module_name']
+        module_path = tnode_config['options']['module_path']
+        class_name = tnode_config['options']['class_name']
+        self.custom_processor = self.import_module(module_name, module_path, class_name)
+
+    def import_module(self, module_name, module_path, class_name):
+        custom_module = __import__(module_path, fromlist=[module_name])
+        custom_class = getattr(custom_module, class_name)
+        return custom_class(self.spark, self.tnode_config)  # .create_instance(self.t_df)
+
+    def _process(self):
+        logger.info(f"Custom node type is {self.tnode_config.get('custom_node_type', 'N/A!')}")
+        return self.custom_processor.process()

datapipelab/app/node/processor/bigquery_api_node.py

@@ -0,0 +1,33 @@
+from datapipelab.app.node.tnode import TNode
+from datapipelab.logger import logger
+
+class BigQueryAPIProcessorNode(TNode):
+    def __init__(self, spark, tnode_config):
+        from google.cloud import bigquery
+        from google.oauth2 import service_account
+        super().__init__(spark=spark)
+        self.sql_query = tnode_config['options']['query']
+        self.node_name = tnode_config['name']
+        self.credentials_path = tnode_config['options']['credentials_path']
+        self.return_as_spark_df = tnode_config['options']['return_as_spark_df']
+        self.project_name = tnode_config['options']['project_name']
+
+    def __sql_query(self, sql_query):
+        credentials = service_account.Credentials.from_service_account_file(self.credentials_path)
+        client = bigquery.Client(credentials=credentials, project=self.project_name)
+
+        # run the job
+        query_job = client.query(sql_query)
+
+        results = query_job.result()
+        rows = [dict(row) for row in results]
+        if self.return_as_spark_df:
+            self.node = self.spark.createDataFrame(rows)
+        else:
+            self.node = None
+        logger.info(rows)
+
+    def _process(self):
+        self.__sql_query(self.sql_query)
+        self._createOrReplaceTempView()
+        return self.node

datapipelab/app/node/processor/bigquery_spark_node.py

@@ -0,0 +1,30 @@
+from datapipelab.app.node.tnode import TNode
+from datapipelab.logger import logger
+
+class BigQuerySparkProcessorNode(TNode):
+    def __init__(self, spark, tnode_config):
+        super().__init__(spark=spark)
+        self.sql_query = tnode_config['options']['query']
+        self.node_name = tnode_config['name']
+        self.credentials_path = tnode_config['options']['materialization_dataset'] # materializationDataset
+        self.return_as_spark_df = tnode_config['options']['parent_project'] # parentProject
+
+    def __sql_query(self, sql_query):
+        credentials = service_account.Credentials.from_service_account_file(self.credentials_path)
+        client = bigquery.Client(credentials=credentials, project=self.project_name)
+
+        # run the job
+        query_job = client.query(sql_query)
+
+        results = query_job.result()
+        rows = [dict(row) for row in results]
+        if self.return_as_spark_df:
+            self.node = self.spark.createDataFrame(rows)
+        else:
+            self.node = None
+        logger.info(rows)
+
+    def _process(self):
+        self.__sql_query(self.sql_query)
+        self._createOrReplaceTempView()
+        return self.node

datapipelab/app/node/processor/shell_node.py

@@ -0,0 +1,23 @@
+from datapipelab.app.node.tnode import TNode
+from datapipelab.logger import logger
+
+class ShellProcessorNode(TNode):
+    def __init__(self, spark, tnode_config):
+
+        super().__init__(spark=spark)
+        self.shell_query = tnode_config['options']['query']
+        self.node_name = tnode_config['name']
+
+    def __shell_query(self):
+        import subprocess
+        # run the job
+        result = subprocess.run(
+            f"{self.shell_query}",
+            shell=True, check=True, executable='/bin/bash'
+        )
+        logger.info(result)
+
+    def _process(self):
+        self.__shell_query()
+        self._createOrReplaceTempView()
+        return self.node

datapipelab/app/node/sink/csv_node.py

@@ -1,9 +1,9 @@
-from pyspark.sql import DataFrame
 from datapipelab.app.node.tnode import TNode
 
 
 class CSVSinkNode(TNode):
     def __init__(self, spark, tnode_config, t_df):
+        from pyspark.sql import DataFrame
         super().__init__(spark=spark)
         self.output_path = tnode_config['options']['path']
         self.partition_by = tnode_config['options'].get('partition_by')

datapipelab/app/node/sink/delta_node.py

@@ -1,10 +1,10 @@
 from pyspark.sql import DataFrame
 from datapipelab.app.node.tnode import TNode
-from delta.tables import DeltaTable
 
 
 class DeltaSinkNode(TNode):
     def __init__(self, spark, tnode_config, t_df):
+        from delta.tables import DeltaTable
         super().__init__(spark=spark)
         self.mode = tnode_config['options']['mode']  # Can be 'append', 'overwrite', or 'upsert'
         self.partition_by = tnode_config['options'].get('partition_by')

datapipelab/app/node/sink/hive_node.py

@@ -1,9 +1,9 @@
-from pyspark.sql import DataFrame
 from datapipelab.app.node.tnode import TNode
 
 
 class HiveSinkNode(TNode):
     def __init__(self, spark, tnode_config, df):
+        from pyspark.sql import DataFrame
         super().__init__(spark=spark)
         self.mode = tnode_config['mode']
         self.stream = tnode_config['stream']

datapipelab/app/node/sink/pandas_csv_node.py

@@ -1,9 +1,9 @@
-from pyspark.sql import DataFrame
 from datapipelab.app.node.tnode import TNode
 
 
 class PandasCSVSinkNode(TNode):
     def __init__(self, spark, tnode_config, t_df):
+        from pyspark.sql import DataFrame
         super().__init__(spark=spark)
         self.mode = tnode_config['options'].get('mode', 'w')
         # self.stream = tnode_config['stream']

datapipelab/app/node/sink/teams_notification_node.py

@@ -1,10 +1,11 @@
-from pyspark.sql import DataFrame
 from datapipelab.app.node.tnode import TNode
-import json
+
 
 
 class TeamsNotificationSinkNode(TNode):
     def __init__(self, spark, tnode_config, df=None):
+        from pyspark.sql import DataFrame
+        import json
         super().__init__(spark=spark)
         self.teams_msg_body = tnode_config['options']['teams_msg_body']
         self.teams_msg_title = tnode_config['options'].get('teams_msg_title', 'Notification')

datapipelab/engine.py

@@ -3,7 +3,7 @@ from datapipelab.logger import logger
 
 
 class Engine:
-    def __init__(self, engine_config_path, spark, params):
+    def __init__(self, engine_config_path, spark, params=None):
         self.engine_config_path = engine_config_path
         self.params = params
         self.pipeline = None

datapipelab/pipeline_config.py

@@ -6,14 +6,14 @@ class PipelineConfig:
         print(config_file)
         self.params = params
         self.config_file = config_file
-        # If config file is instance of string, it is a path to the config file
+        # If the config file is an instance of string, it is a path to the config file
         self.config_file = config_file
         if isinstance(self.config_file, str):
             self.load_json_config_file()
-        elif isinstance(self.config_file, dict):
+        elif isinstance(self.config_file, list):
             self.pipeline_settings = config_file
         else:
-            raise ValueError("Invalid config file type. Must be a string or a dictionary.")
+            raise ValueError("Invalid config file type. Must be a string or a list.")
         self.sources = {}
         self.processors = {}
         self.sinks = {}
@@ -27,6 +27,14 @@ class PipelineConfig:
                 json_config_file = json_config_file.replace(f"{{{key}}}", value)
             # Convert to JSON file
             self.pipeline_settings = json5.loads(json_config_file)
+        if len(self.pipeline_settings) > 0 and self.pipeline_settings[0]['type'] == 'import':
+            self.import_json_config_file()
+
+    def import_json_config_file(self):
+        import_pipeline_settings = []
+        for import_component in self.pipeline_settings:
+            if import_component['type'] == '': # Maybe someone wants to use import in the middle of his config?
+                pass
 
     def create_pipeline_nodes(self):
         for component in self.pipeline_settings:

datapipelab/pipeline_handler.py

@@ -6,6 +6,7 @@ from datapipelab.app.node.sink.delta_node import DeltaSinkNode
 from datapipelab.app.node.sink.csv_node import CSVSinkNode
 from datapipelab.app.node.sink.pandas_csv_node import PandasCSVSinkNode
 from datapipelab.app.node.sink.teams_notification_node import TeamsNotificationSinkNode
+from datapipelab.app.node.processor.bigquery_spark_node import BigQuerySparkProcessorNode
 
 
 class PipelineHandler:
@@ -42,10 +43,12 @@ class PipelineHandler:
         processor_df = None
         if tnode_format == 'custom':
             processor_df = CustomNode(self.spark, tnode_config).run()
-
         if tnode_format == 'query':
             processor_df = SparkProcessorNode(self.spark, tnode_config).run()
-
+        if tnode_format == 'bigquery_api':
+            processor_df = None
+        if tnode_format == 'bigquery_spark':
+            processor_df = datapipelab.app.node.processor.bigquery_spark.BigQuerySparkProcessorNode(self.spark, tnode_config).run()
         return processor_df
 
     def write_sink_node(self, tnode_config, t_df):
@@ -70,5 +73,8 @@ class PipelineHandler:
         if tnode_type == "local":
             if tnode_format == "csv":
                 PandasCSVSinkNode(self.spark, tnode_config, t_df).run()
+        if tnode_type == 'custom':
+            from datapipelab.app.node import custom_node
+            processor_df = custom_node.CustomNode(self.spark, tnode_config).run()
 
 

{datapipelab-0.1.4.dist-info → datapipelab-0.1.6.dist-info}/METADATA

@@ -1,11 +1,8 @@
 Metadata-Version: 2.4
 Name: datapipelab
-Version: 0.1.4
+Version: 0.1.6
 Summary: A data pipeline library with connectors, sources, processors, and sinks.
 Requires-Dist: json5
 Requires-Dist: loguru
-Requires-Dist: azure-storage-blob
-Requires-Dist: google-cloud-storage
-Requires-Dist: pandas
 Dynamic: requires-dist
 Dynamic: summary

datapipelab-0.1.6.dist-info/RECORD

@@ -0,0 +1,31 @@
+datapipelab/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datapipelab/engine.py,sha256=3QRsedRYNov6xIDOZ1tukinFE-SKv39Fn3sNCnD3L6g,442
+datapipelab/logger.py,sha256=Ugv0A4TfD3JWCWXNWu0lURcnfAEyuVrK3IrvVVgcHBo,864
+datapipelab/pipeline.py,sha256=dw9D9KM_hztt9g_YzqoNgQBRyCYR92cRZwrU5duP_Pg,1464
+datapipelab/pipeline_config.py,sha256=2bFAJepViE7rT7CaRANZU07aeQpOYcZ954ISujm9pXA,3816
+datapipelab/pipeline_handler.py,sha256=HCyvyW3Yx2QYWQ-D-ArMDZdGtMRsJ348ZHSrqbRYwd0,3779
+datapipelab/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datapipelab/app/connector_node/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datapipelab/app/node/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datapipelab/app/node/custom_node.py,sha256=VvjwkECTobRhO_fYKUrJCd117B5MoR9P6UKYZfRLhV4,1017
+datapipelab/app/node/tnode.py,sha256=npHG4fFZty5JZ3F_okO9xml-BRhu4DkrZuNE6oaLbvw,446
+datapipelab/app/node/processor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datapipelab/app/node/processor/bigquery_api_node.py,sha256=2kF6hgYOUi7te-aXXr-cTZfkvitBDEPJAHT8KG-i8fE,1293
+datapipelab/app/node/processor/bigquery_spark_node.py,sha256=t8JJqMWTZwBuJUKV7-l72ZLdLVoHHSFJHFNovAY-2nc,1179
+datapipelab/app/node/processor/custom_node.py,sha256=1nqbJEhNiMP1rmN9ufpUuKO1IkuI2BEM5auW4JceGMA,933
+datapipelab/app/node/processor/shell_node.py,sha256=s3dKgfEqbpUIEiwORERgvp7FNDE5JkFHBo7EnJYBPnA,669
+datapipelab/app/node/processor/spark_node.py,sha256=jzqdffIHUCgOfMFcoqjXdl8wFag-3gafxfNCdssKnwc,483
+datapipelab/app/node/sink/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datapipelab/app/node/sink/csv_node.py,sha256=d2hyufP5_Nmql0pfD0KeC4rFu1wXTnBxVsoGl7sWbhM,1681
+datapipelab/app/node/sink/delta_node.py,sha256=iKEdiTjJ7SHJZMrbm0jR5tms5JZ5iCFfQklZbI-Yr2o,2044
+datapipelab/app/node/sink/hive_node.py,sha256=BKSSYb1AexQD1Jl6zP5ak_ibvd0wYRNzC_zhTtNTDRg,1106
+datapipelab/app/node/sink/pandas_csv_node.py,sha256=JsJFt2XRpwxGeJyt_PDUgqZafiQROf1Sk5TUhQPxh4c,870
+datapipelab/app/node/sink/teams_notification_node.py,sha256=6ZufdbhVvRXi3QTQafLo5uKl9kLyDnkYIE_VZFT0QNw,3581
+datapipelab/app/node/source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datapipelab/app/node/source/delta_node.py,sha256=gg7SfuKBAAfjk6OX2jNrot9XX61HoBe3us3D8O-dscE,529
+datapipelab/app/node/source/hive_node.py,sha256=h_AMCnnmon7uLRIGsaHAPWEReD3VaWZXnz9r0TpLGNM,478
+datapipelab/app/node/source/spark_node.py,sha256=S_x2atRFPDnXmhCUtcmaLc4BDFd2H4uQq6wnEJb7Uug,480
+datapipelab-0.1.6.dist-info/METADATA,sha256=yuXR8PtBINVXBFVsFncHbu9cnvJ52Jp8ZCDi-Bsmrnw,220
+datapipelab-0.1.6.dist-info/WHEEL,sha256=Nw36Djuh_5VDukK0H78QzOX-_FQEo6V37m3nkm96gtU,91
+datapipelab-0.1.6.dist-info/top_level.txt,sha256=HgeBjHvXorKzvNqU5BNPutoI771HtiqVit9_-0Zyrb4,12
+datapipelab-0.1.6.dist-info/RECORD,,

{datapipelab-0.1.4.dist-info → datapipelab-0.1.6.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (78.1.0)
+Generator: setuptools (80.7.1)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

datapipelab-0.1.4.dist-info/RECORD

@@ -1,27 +0,0 @@
-datapipelab/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datapipelab/engine.py,sha256=dYm39Yb0Eqt76pwdc-ilzZNxehMKdiuidE557YexHaU,437
-datapipelab/logger.py,sha256=Ugv0A4TfD3JWCWXNWu0lURcnfAEyuVrK3IrvVVgcHBo,864
-datapipelab/pipeline.py,sha256=dw9D9KM_hztt9g_YzqoNgQBRyCYR92cRZwrU5duP_Pg,1464
-datapipelab/pipeline_config.py,sha256=dKVWz_FH5448a1ZE9eIu8ANagEceI_l4QNICbXrRudk,3411
-datapipelab/pipeline_handler.py,sha256=1t5wwsaVUMXXmsEa-Qt-6jtMIyAZmX1hgo2I_UgbtiM,3265
-datapipelab/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datapipelab/app/connector_node/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datapipelab/app/node/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datapipelab/app/node/tnode.py,sha256=npHG4fFZty5JZ3F_okO9xml-BRhu4DkrZuNE6oaLbvw,446
-datapipelab/app/node/processor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datapipelab/app/node/processor/custom_node.py,sha256=1nqbJEhNiMP1rmN9ufpUuKO1IkuI2BEM5auW4JceGMA,933
-datapipelab/app/node/processor/spark_node.py,sha256=jzqdffIHUCgOfMFcoqjXdl8wFag-3gafxfNCdssKnwc,483
-datapipelab/app/node/sink/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datapipelab/app/node/sink/csv_node.py,sha256=YaDporq4L358dJCkheCjpGmDBsYtvai6dnGiS-uf_Mc,1673
-datapipelab/app/node/sink/delta_node.py,sha256=h3gnBsuYoXUlnhCouav-iwdQ4UVZPcvG4_5r9gxY8JM,2036
-datapipelab/app/node/sink/hive_node.py,sha256=E3pP_U7LzCFdZ0LVkqR0LDlU8HZtfbKgOo72yJsLLT0,1098
-datapipelab/app/node/sink/pandas_csv_node.py,sha256=bNF-Sb5pKMV1qAePzyDhwYLB075Rg7tjxfQ6BHcq1Wk,862
-datapipelab/app/node/sink/teams_notification_node.py,sha256=ZDE-F4nfmrK7UnXoSI6mHuhUHWlgE8rUCdPFW5ZXr7E,3564
-datapipelab/app/node/source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datapipelab/app/node/source/delta_node.py,sha256=gg7SfuKBAAfjk6OX2jNrot9XX61HoBe3us3D8O-dscE,529
-datapipelab/app/node/source/hive_node.py,sha256=h_AMCnnmon7uLRIGsaHAPWEReD3VaWZXnz9r0TpLGNM,478
-datapipelab/app/node/source/spark_node.py,sha256=S_x2atRFPDnXmhCUtcmaLc4BDFd2H4uQq6wnEJb7Uug,480
-datapipelab-0.1.4.dist-info/METADATA,sha256=O_P8alSEnX3hM-gLN6OPFfeYykpOO_-ymns8OKUtJU4,312
-datapipelab-0.1.4.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-datapipelab-0.1.4.dist-info/top_level.txt,sha256=HgeBjHvXorKzvNqU5BNPutoI771HtiqVit9_-0Zyrb4,12
-datapipelab-0.1.4.dist-info/RECORD,,