datapipelab 0.1.8__tar.gz → 0.2.0__tar.gz

{datapipelab-0.1.8 → datapipelab-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datapipelab
-Version: 0.1.8
+Version: 0.2.0
 Summary: A data pipeline library with connectors, sources, processors, and sinks.
 Requires-Dist: json5
 Requires-Dist: loguru

{datapipelab-0.1.8 → datapipelab-0.2.0}/datapipelab/app/node/processor/bigquery_api_node.py

@@ -3,16 +3,18 @@ from datapipelab.logger import logger
 
 class BigQueryAPIProcessorNode(TNode):
     def __init__(self, spark, tnode_config):
-        from google.cloud import bigquery
-        from google.oauth2 import service_account
         super().__init__(spark=spark)
         self.sql_query = tnode_config['options']['query']
         self.node_name = tnode_config['name']
         self.credentials_path = tnode_config['options']['credentials_path']
         self.return_as_spark_df = tnode_config['options']['return_as_spark_df']
         self.project_name = tnode_config['options']['project_name']
+        self.return_as_python_list = tnode_config['options'].get('return_as_python_list', False)
+        self.return_as_is = tnode_config['options'].get('return_as_is', False)
 
     def __sql_biqquery(self, sql_query):
+        from google.cloud import bigquery
+        from google.oauth2 import service_account
         credentials = service_account.Credentials.from_service_account_file(self.credentials_path)
         client = bigquery.Client(credentials=credentials, project=self.project_name)
 
@@ -20,12 +22,17 @@ class BigQueryAPIProcessorNode(TNode):
         query_job = client.query(sql_query)
 
         results = query_job.result()
-        rows = [dict(row) for row in results]
         if self.return_as_spark_df:
+            rows = [dict(row) for row in results]
             self.node = self.spark.createDataFrame(rows)
+        elif self.return_as_python_list:
+            rows = [dict(row) for row in results]
+            self.node = rows
+        elif self.return_as_is:
+            self.node = results
         else:
             self.node = None
-        logger.info(rows)
+        logger.info([dict(row) for row in results])  # TODO: Remove this
 
     def _process(self):
         self.__sql_biqquery(self.sql_query)

{datapipelab-0.1.8 → datapipelab-0.2.0}/datapipelab/app/node/tnode.py

@@ -9,7 +9,8 @@ class TNode:
 
     # Source and Processor nodes
     def _createOrReplaceTempView(self):
-        self.node.createOrReplaceTempView(self.node_name)
+        if self.node is not None:
+            self.node.createOrReplaceTempView(self.node_name)
 
     def run(self):
         return self._process()

{datapipelab-0.1.8 → datapipelab-0.2.0}/datapipelab/pipeline_handler.py

@@ -1,4 +1,5 @@
 from datapipelab.app.node.processor.custom_node import CustomNode
+from datapipelab.app.node.processor.shell_node import ShellProcessorNode
 from datapipelab.app.node.source.spark_node import SparkSourceNode
 from datapipelab.app.node.source.delta_node import DeltaSourceNode
 from datapipelab.app.node.processor.spark_node import SparkProcessorNode
@@ -50,6 +51,8 @@ class PipelineHandler:
             processor_df = BigQueryAPIProcessorNode(self.spark, tnode_config).run()
         if tnode_format == 'bigquery_spark':
             processor_df = BigQuerySparkProcessorNode(self.spark, tnode_config).run()
+        if tnode_format == 'shell':
+            processor_df = ShellProcessorNode(self.spark, tnode_config).run()
         return processor_df
 
     def write_sink_node(self, tnode_config, t_df):

{datapipelab-0.1.8 → datapipelab-0.2.0}/datapipelab.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datapipelab
-Version: 0.1.8
+Version: 0.2.0
 Summary: A data pipeline library with connectors, sources, processors, and sinks.
 Requires-Dist: json5
 Requires-Dist: loguru

{datapipelab-0.1.8 → datapipelab-0.2.0}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='datapipelab',
-    version='0.1.8',
+    version='0.2.0',
     description='A data pipeline library with connectors, sources, processors, and sinks.',
     packages=find_packages(),
     include_package_data=True,