datapipelab 0.1.7__py3-none-any.whl → 0.1.9__py3-none-any.whl

datapipelab/app/node/processor/bigquery_api_node.py

@@ -3,8 +3,6 @@ from datapipelab.logger import logger
 
 class BigQueryAPIProcessorNode(TNode):
     def __init__(self, spark, tnode_config):
-        from google.cloud import bigquery
-        from google.oauth2 import service_account
         super().__init__(spark=spark)
         self.sql_query = tnode_config['options']['query']
         self.node_name = tnode_config['name']
@@ -12,7 +10,9 @@ class BigQueryAPIProcessorNode(TNode):
         self.return_as_spark_df = tnode_config['options']['return_as_spark_df']
         self.project_name = tnode_config['options']['project_name']
 
-    def __sql_query(self, sql_query):
+    def __sql_biqquery(self, sql_query):
+        from google.cloud import bigquery
+        from google.oauth2 import service_account
         credentials = service_account.Credentials.from_service_account_file(self.credentials_path)
         client = bigquery.Client(credentials=credentials, project=self.project_name)
 
@@ -28,6 +28,6 @@ class BigQueryAPIProcessorNode(TNode):
         logger.info(rows)
 
     def _process(self):
-        self.__sql_query(self.sql_query)
+        self.__sql_biqquery(self.sql_query)
         self._createOrReplaceTempView()
         return self.node

datapipelab/app/node/processor/bigquery_spark_node.py

@@ -1,30 +1,22 @@
 from datapipelab.app.node.tnode import TNode
 from datapipelab.logger import logger
 
+
 class BigQuerySparkProcessorNode(TNode):
     def __init__(self, spark, tnode_config):
         super().__init__(spark=spark)
         self.sql_query = tnode_config['options']['query']
         self.node_name = tnode_config['name']
-        self.credentials_path = tnode_config['options']['materialization_dataset'] # materializationDataset
-        self.return_as_spark_df = tnode_config['options']['parent_project'] # parentProject
-
-    def __sql_query(self, sql_query):
-        credentials = service_account.Credentials.from_service_account_file(self.credentials_path)
-        client = bigquery.Client(credentials=credentials, project=self.project_name)
-
-        # run the job
-        query_job = client.query(sql_query)
+        self.materialization_dataset = tnode_config['options']['materialization_dataset']  # materializationDataset
+        self.parent_project = tnode_config['options']['parent_project']  # parentProject
 
-        results = query_job.result()
-        rows = [dict(row) for row in results]
-        if self.return_as_spark_df:
-            self.node = self.spark.createDataFrame(rows)
-        else:
-            self.node = None
-        logger.info(rows)
+    def __sql_query(self):
+        self.node = self.spark.read.format("bigquery").option("materializationDataset",
+                                                              self.materialization_dataset).option("query",
+                                                                                                   self.sql_query).option(
+            "parentProject", self.parent_project).load()
 
     def _process(self):
-        self.__sql_query(self.sql_query)
+        self.__sql_query()
         self._createOrReplaceTempView()
         return self.node

{datapipelab-0.1.7.dist-info → datapipelab-0.1.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datapipelab
-Version: 0.1.7
+Version: 0.1.9
 Summary: A data pipeline library with connectors, sources, processors, and sinks.
 Requires-Dist: json5
 Requires-Dist: loguru

{datapipelab-0.1.7.dist-info → datapipelab-0.1.9.dist-info}/RECORD

@@ -10,8 +10,8 @@ datapipelab/app/node/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hS
 datapipelab/app/node/custom_node.py,sha256=VvjwkECTobRhO_fYKUrJCd117B5MoR9P6UKYZfRLhV4,1017
 datapipelab/app/node/tnode.py,sha256=npHG4fFZty5JZ3F_okO9xml-BRhu4DkrZuNE6oaLbvw,446
 datapipelab/app/node/processor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datapipelab/app/node/processor/bigquery_api_node.py,sha256=2kF6hgYOUi7te-aXXr-cTZfkvitBDEPJAHT8KG-i8fE,1293
-datapipelab/app/node/processor/bigquery_spark_node.py,sha256=t8JJqMWTZwBuJUKV7-l72ZLdLVoHHSFJHFNovAY-2nc,1179
+datapipelab/app/node/processor/bigquery_api_node.py,sha256=uBq9Ruk791Sa4qUK7-dnGPklXH68xqWy0yvQsASwHP8,1299
+datapipelab/app/node/processor/bigquery_spark_node.py,sha256=S9kIYW0RE5b0RjniKFFBTzA3Tx4_plFdkFQXzhl1xTY,1039
 datapipelab/app/node/processor/custom_node.py,sha256=1nqbJEhNiMP1rmN9ufpUuKO1IkuI2BEM5auW4JceGMA,933
 datapipelab/app/node/processor/shell_node.py,sha256=s3dKgfEqbpUIEiwORERgvp7FNDE5JkFHBo7EnJYBPnA,669
 datapipelab/app/node/processor/spark_node.py,sha256=jzqdffIHUCgOfMFcoqjXdl8wFag-3gafxfNCdssKnwc,483
@@ -25,7 +25,7 @@ datapipelab/app/node/source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMp
 datapipelab/app/node/source/delta_node.py,sha256=gg7SfuKBAAfjk6OX2jNrot9XX61HoBe3us3D8O-dscE,529
 datapipelab/app/node/source/hive_node.py,sha256=h_AMCnnmon7uLRIGsaHAPWEReD3VaWZXnz9r0TpLGNM,478
 datapipelab/app/node/source/spark_node.py,sha256=S_x2atRFPDnXmhCUtcmaLc4BDFd2H4uQq6wnEJb7Uug,480
-datapipelab-0.1.7.dist-info/METADATA,sha256=oiRy9y4GTRDU-Yiz3EQJc831TRgaTiMRXOaNimFllTQ,220
-datapipelab-0.1.7.dist-info/WHEEL,sha256=Nw36Djuh_5VDukK0H78QzOX-_FQEo6V37m3nkm96gtU,91
-datapipelab-0.1.7.dist-info/top_level.txt,sha256=HgeBjHvXorKzvNqU5BNPutoI771HtiqVit9_-0Zyrb4,12
-datapipelab-0.1.7.dist-info/RECORD,,
+datapipelab-0.1.9.dist-info/METADATA,sha256=GPQ-3fWDVD8G5z4rv-fjaRZ6L61U_a_pmuagIJw66Us,220
+datapipelab-0.1.9.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
+datapipelab-0.1.9.dist-info/top_level.txt,sha256=HgeBjHvXorKzvNqU5BNPutoI771HtiqVit9_-0Zyrb4,12
+datapipelab-0.1.9.dist-info/RECORD,,

{datapipelab-0.1.7.dist-info → datapipelab-0.1.9.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.7.1)
+Generator: setuptools (80.8.0)
 Root-Is-Purelib: true
 Tag: py3-none-any