datapipelab 0.1.7__tar.gz → 0.1.9__tar.gz

{datapipelab-0.1.7 → datapipelab-0.1.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datapipelab
-Version: 0.1.7
+Version: 0.1.9
 Summary: A data pipeline library with connectors, sources, processors, and sinks.
 Requires-Dist: json5
 Requires-Dist: loguru

{datapipelab-0.1.7 → datapipelab-0.1.9}/datapipelab/app/node/processor/bigquery_api_node.py

@@ -3,8 +3,6 @@ from datapipelab.logger import logger
 
 class BigQueryAPIProcessorNode(TNode):
     def __init__(self, spark, tnode_config):
-        from google.cloud import bigquery
-        from google.oauth2 import service_account
         super().__init__(spark=spark)
         self.sql_query = tnode_config['options']['query']
         self.node_name = tnode_config['name']
@@ -12,7 +10,9 @@ class BigQueryAPIProcessorNode(TNode):
         self.return_as_spark_df = tnode_config['options']['return_as_spark_df']
         self.project_name = tnode_config['options']['project_name']
 
-    def __sql_query(self, sql_query):
+    def __sql_biqquery(self, sql_query):
+        from google.cloud import bigquery
+        from google.oauth2 import service_account
         credentials = service_account.Credentials.from_service_account_file(self.credentials_path)
         client = bigquery.Client(credentials=credentials, project=self.project_name)
 
@@ -28,6 +28,6 @@ class BigQueryAPIProcessorNode(TNode):
         logger.info(rows)
 
     def _process(self):
-        self.__sql_query(self.sql_query)
+        self.__sql_biqquery(self.sql_query)
         self._createOrReplaceTempView()
         return self.node

datapipelab-0.1.9/datapipelab/app/node/processor/bigquery_spark_node.py

@@ -0,0 +1,22 @@
+from datapipelab.app.node.tnode import TNode
+from datapipelab.logger import logger
+
+
+class BigQuerySparkProcessorNode(TNode):
+    def __init__(self, spark, tnode_config):
+        super().__init__(spark=spark)
+        self.sql_query = tnode_config['options']['query']
+        self.node_name = tnode_config['name']
+        self.materialization_dataset = tnode_config['options']['materialization_dataset']  # materializationDataset
+        self.parent_project = tnode_config['options']['parent_project']  # parentProject
+
+    def __sql_query(self):
+        self.node = self.spark.read.format("bigquery").option("materializationDataset",
+                                                              self.materialization_dataset).option("query",
+                                                                                                   self.sql_query).option(
+            "parentProject", self.parent_project).load()
+
+    def _process(self):
+        self.__sql_query()
+        self._createOrReplaceTempView()
+        return self.node

{datapipelab-0.1.7 → datapipelab-0.1.9}/datapipelab.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datapipelab
-Version: 0.1.7
+Version: 0.1.9
 Summary: A data pipeline library with connectors, sources, processors, and sinks.
 Requires-Dist: json5
 Requires-Dist: loguru

{datapipelab-0.1.7 → datapipelab-0.1.9}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='datapipelab',
-    version='0.1.7',
+    version='0.1.9',
     description='A data pipeline library with connectors, sources, processors, and sinks.',
     packages=find_packages(),
     include_package_data=True,

datapipelab-0.1.7/datapipelab/app/node/processor/bigquery_spark_node.py

@@ -1,30 +0,0 @@
-from datapipelab.app.node.tnode import TNode
-from datapipelab.logger import logger
-
-class BigQuerySparkProcessorNode(TNode):
-    def __init__(self, spark, tnode_config):
-        super().__init__(spark=spark)
-        self.sql_query = tnode_config['options']['query']
-        self.node_name = tnode_config['name']
-        self.credentials_path = tnode_config['options']['materialization_dataset'] # materializationDataset
-        self.return_as_spark_df = tnode_config['options']['parent_project'] # parentProject
-
-    def __sql_query(self, sql_query):
-        credentials = service_account.Credentials.from_service_account_file(self.credentials_path)
-        client = bigquery.Client(credentials=credentials, project=self.project_name)
-
-        # run the job
-        query_job = client.query(sql_query)
-
-        results = query_job.result()
-        rows = [dict(row) for row in results]
-        if self.return_as_spark_df:
-            self.node = self.spark.createDataFrame(rows)
-        else:
-            self.node = None
-        logger.info(rows)
-
-    def _process(self):
-        self.__sql_query(self.sql_query)
-        self._createOrReplaceTempView()
-        return self.node