datapipelab 0.1.9__py3-none-any.whl → 0.2.1__py3-none-any.whl

datapipelab/app/node/processor/bigquery_api_node.py

@@ -9,6 +9,8 @@ class BigQueryAPIProcessorNode(TNode):
         self.credentials_path = tnode_config['options']['credentials_path']
         self.return_as_spark_df = tnode_config['options']['return_as_spark_df']
         self.project_name = tnode_config['options']['project_name']
+        self.return_as_python_list = tnode_config['options'].get('return_as_python_list', False)
+        self.return_as_is = tnode_config['options'].get('return_as_is', False)
 
     def __sql_biqquery(self, sql_query):
         from google.cloud import bigquery
@@ -20,12 +22,17 @@ class BigQueryAPIProcessorNode(TNode):
         query_job = client.query(sql_query)
 
         results = query_job.result()
-        rows = [dict(row) for row in results]
         if self.return_as_spark_df:
+            rows = [dict(row) for row in results]
             self.node = self.spark.createDataFrame(rows)
+        elif self.return_as_python_list:
+            rows = [dict(row) for row in results]
+            self.node = rows
+        elif self.return_as_is:
+            self.node = results
         else:
             self.node = None
-        logger.info(rows)
+        logger.info([dict(row) for row in results])  # TODO: Remove this
 
     def _process(self):
         self.__sql_biqquery(self.sql_query)

datapipelab/app/node/sink/hive_node.py

@@ -5,12 +5,12 @@ class HiveSinkNode(TNode):
     def __init__(self, spark, tnode_config, df):
         from pyspark.sql import DataFrame
         super().__init__(spark=spark)
-        self.mode = tnode_config['mode']
-        self.stream = tnode_config['stream']
+        self.mode = tnode_config.get('mode', None)
+        self.stream = tnode_config.get('stream', None)
         self.database_name = tnode_config['options']['database']
         self.table_name = tnode_config['options']['table']
-        self.partition_by = tnode_config['options'].get('partition_by')
-        self.partition_count = tnode_config['options'].get('partition_count')
+        self.partition_by = tnode_config['options'].get('partition_by', None)
+        self.partition_count = tnode_config['options'].get('partition_count', None)
         self.overwrite = tnode_config['options']['overwrite']
         self.df = df
 

datapipelab/app/node/tnode.py

@@ -9,7 +9,8 @@ class TNode:
 
     # Source and Processor nodes
     def _createOrReplaceTempView(self):
-        self.node.createOrReplaceTempView(self.node_name)
+        if self.node is not None:
+            self.node.createOrReplaceTempView(self.node_name)
 
     def run(self):
         return self._process()

datapipelab/pipeline_handler.py

@@ -1,4 +1,5 @@
 from datapipelab.app.node.processor.custom_node import CustomNode
+from datapipelab.app.node.processor.shell_node import ShellProcessorNode
 from datapipelab.app.node.source.spark_node import SparkSourceNode
 from datapipelab.app.node.source.delta_node import DeltaSourceNode
 from datapipelab.app.node.processor.spark_node import SparkProcessorNode
@@ -50,6 +51,8 @@ class PipelineHandler:
             processor_df = BigQueryAPIProcessorNode(self.spark, tnode_config).run()
         if tnode_format == 'bigquery_spark':
             processor_df = BigQuerySparkProcessorNode(self.spark, tnode_config).run()
+        if tnode_format == 'shell':
+            processor_df = ShellProcessorNode(self.spark, tnode_config).run()
         return processor_df
 
     def write_sink_node(self, tnode_config, t_df):
@@ -77,5 +80,10 @@ class PipelineHandler:
         if tnode_type == 'custom':
             from datapipelab.app.node import custom_node
             processor_df = custom_node.CustomNode(self.spark, tnode_config).run()
+        if tnode_type == 'spark':
+            if tnode_format == 'hive':
+                from datapipelab.app.node.sink import hive_node
+                processor_df = hive_node.HiveSinkNode(self.spark, tnode_config, t_df[tnode_name_df]).run()
+
 
 

{datapipelab-0.1.9.dist-info → datapipelab-0.2.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datapipelab
-Version: 0.1.9
+Version: 0.2.1
 Summary: A data pipeline library with connectors, sources, processors, and sinks.
 Requires-Dist: json5
 Requires-Dist: loguru

{datapipelab-0.1.9.dist-info → datapipelab-0.2.1.dist-info}/RECORD

@@ -3,14 +3,14 @@ datapipelab/engine.py,sha256=3QRsedRYNov6xIDOZ1tukinFE-SKv39Fn3sNCnD3L6g,442
 datapipelab/logger.py,sha256=Ugv0A4TfD3JWCWXNWu0lURcnfAEyuVrK3IrvVVgcHBo,864
 datapipelab/pipeline.py,sha256=dw9D9KM_hztt9g_YzqoNgQBRyCYR92cRZwrU5duP_Pg,1464
 datapipelab/pipeline_config.py,sha256=2bFAJepViE7rT7CaRANZU07aeQpOYcZ954ISujm9pXA,3816
-datapipelab/pipeline_handler.py,sha256=lQv6HwwdgZDQvICgABtWiuvZQ9jG9cJjy8s_7qLZr9s,3871
+datapipelab/pipeline_handler.py,sha256=mkSvEo93IqmqJKbMc9R500C8ao7JFQCyPJ813B3COOc,4303
 datapipelab/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datapipelab/app/connector_node/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datapipelab/app/node/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datapipelab/app/node/custom_node.py,sha256=VvjwkECTobRhO_fYKUrJCd117B5MoR9P6UKYZfRLhV4,1017
-datapipelab/app/node/tnode.py,sha256=npHG4fFZty5JZ3F_okO9xml-BRhu4DkrZuNE6oaLbvw,446
+datapipelab/app/node/tnode.py,sha256=-2hnQkIuLwEy7xVTig54TByO7L2l7UujolXMQL0CQJA,484
 datapipelab/app/node/processor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datapipelab/app/node/processor/bigquery_api_node.py,sha256=uBq9Ruk791Sa4qUK7-dnGPklXH68xqWy0yvQsASwHP8,1299
+datapipelab/app/node/processor/bigquery_api_node.py,sha256=fLs8g1R8iRIZIJL66wbXONmStbhR4Q8Mw3QsZgp7MpI,1710
 datapipelab/app/node/processor/bigquery_spark_node.py,sha256=S9kIYW0RE5b0RjniKFFBTzA3Tx4_plFdkFQXzhl1xTY,1039
 datapipelab/app/node/processor/custom_node.py,sha256=1nqbJEhNiMP1rmN9ufpUuKO1IkuI2BEM5auW4JceGMA,933
 datapipelab/app/node/processor/shell_node.py,sha256=s3dKgfEqbpUIEiwORERgvp7FNDE5JkFHBo7EnJYBPnA,669
@@ -18,14 +18,14 @@ datapipelab/app/node/processor/spark_node.py,sha256=jzqdffIHUCgOfMFcoqjXdl8wFag-
 datapipelab/app/node/sink/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datapipelab/app/node/sink/csv_node.py,sha256=d2hyufP5_Nmql0pfD0KeC4rFu1wXTnBxVsoGl7sWbhM,1681
 datapipelab/app/node/sink/delta_node.py,sha256=iKEdiTjJ7SHJZMrbm0jR5tms5JZ5iCFfQklZbI-Yr2o,2044
-datapipelab/app/node/sink/hive_node.py,sha256=BKSSYb1AexQD1Jl6zP5ak_ibvd0wYRNzC_zhTtNTDRg,1106
+datapipelab/app/node/sink/hive_node.py,sha256=hodxZEUDuqnJgw4hraguuNe9bJwv2UFzTbffwCsGGJg,1138
 datapipelab/app/node/sink/pandas_csv_node.py,sha256=JsJFt2XRpwxGeJyt_PDUgqZafiQROf1Sk5TUhQPxh4c,870
 datapipelab/app/node/sink/teams_notification_node.py,sha256=6ZufdbhVvRXi3QTQafLo5uKl9kLyDnkYIE_VZFT0QNw,3581
 datapipelab/app/node/source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datapipelab/app/node/source/delta_node.py,sha256=gg7SfuKBAAfjk6OX2jNrot9XX61HoBe3us3D8O-dscE,529
 datapipelab/app/node/source/hive_node.py,sha256=h_AMCnnmon7uLRIGsaHAPWEReD3VaWZXnz9r0TpLGNM,478
 datapipelab/app/node/source/spark_node.py,sha256=S_x2atRFPDnXmhCUtcmaLc4BDFd2H4uQq6wnEJb7Uug,480
-datapipelab-0.1.9.dist-info/METADATA,sha256=GPQ-3fWDVD8G5z4rv-fjaRZ6L61U_a_pmuagIJw66Us,220
-datapipelab-0.1.9.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
-datapipelab-0.1.9.dist-info/top_level.txt,sha256=HgeBjHvXorKzvNqU5BNPutoI771HtiqVit9_-0Zyrb4,12
-datapipelab-0.1.9.dist-info/RECORD,,
+datapipelab-0.2.1.dist-info/METADATA,sha256=fz0lesIoc1arto7rA5e9IKFw8Bgiofc3QmCcwyb0lN8,220
+datapipelab-0.2.1.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
+datapipelab-0.2.1.dist-info/top_level.txt,sha256=HgeBjHvXorKzvNqU5BNPutoI771HtiqVit9_-0Zyrb4,12
+datapipelab-0.2.1.dist-info/RECORD,,