datapipelab 0.3.5__py3-none-any.whl → 0.3.7__py3-none-any.whl

datapipelab/app/node/processor/bigquery_api_node.py

@@ -6,16 +6,29 @@ class BigQueryAPIProcessorNode(TNode):
         super().__init__(spark=spark)
         self.sql_query = tnode_config['options']['query']
         self.node_name = tnode_config['name']
-        self.credentials_path = tnode_config['options']['credentials_path']
+        self.credentials_path = tnode_config['options'].get('credentials_path', None)
+        self.credentials_info_env_name = tnode_config['options'].get('credentials_info_env_name', None)
         self.return_as_spark_df = tnode_config['options']['return_as_spark_df']
         self.project_name = tnode_config['options']['project_name']
         self.return_as_python_list = tnode_config['options'].get('return_as_python_list', False)
         self.return_as_is = tnode_config['options'].get('return_as_is', False)
 
+    def __get_gcp_credentials(self):
+        from google.oauth2 import service_account
+        if self.credentials_path:
+            return service_account.Credentials.from_service_account_file(self.credentials_path)
+        elif self.credentials_info_env_name:
+            import os
+            import json
+            credentials_info = json.loads(os.environ[self.credentials_info_env_name])
+            return service_account.Credentials.from_service_account_info(credentials_info)
+        else:
+            logger.warning("No credentials provided. Please provide either 'credentials_path' or 'credentials_info'.")
+            return None
+
     def __sql_biqquery(self, sql_query):
         from google.cloud import bigquery
-        from google.oauth2 import service_account
-        credentials = service_account.Credentials.from_service_account_file(self.credentials_path)
+        credentials = self.__get_gcp_credentials()
         client = bigquery.Client(credentials=credentials, project=self.project_name)
 
         # run the job

datapipelab/app/node/sink/spark_api_node.py

@@ -22,6 +22,8 @@ class SparkApiSourceNode(TNode):
             self.options['table'] = self.spark_options.get('table')
         if 'write_method' in self.spark_options:
             self.options['writeMethod'] = self.spark_options.get('write_method')
+        if 'temporary_gcs_bucket' in self.spark_options:
+            self.options['temporaryGcsBucket'] = self.spark_options.get('temporary_gcs_bucket')
 
     def __write_df(self):
         writer = self.df.write

{datapipelab-0.3.5.dist-info → datapipelab-0.3.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datapipelab
-Version: 0.3.5
+Version: 0.3.7
 Summary: A data pipeline library with connectors, sources, processors, and sinks.
 Description-Content-Type: text/markdown
 Requires-Dist: json5

{datapipelab-0.3.5.dist-info → datapipelab-0.3.7.dist-info}/RECORD

@@ -10,7 +10,7 @@ datapipelab/app/node/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hS
 datapipelab/app/node/custom_node.py,sha256=3Se4DweMvm5VK4MTZ-pQSQ_lE_fOm6cGj-wzcqKEU0E,1019
 datapipelab/app/node/tnode.py,sha256=-2hnQkIuLwEy7xVTig54TByO7L2l7UujolXMQL0CQJA,484
 datapipelab/app/node/processor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datapipelab/app/node/processor/bigquery_api_node.py,sha256=IclDkGxo9ltGJVkBaHKFPFCSlEEyzefgalaAOLA17bE,1752
+datapipelab/app/node/processor/bigquery_api_node.py,sha256=wXmdRN9OJv5OvqRbxzIBx-Qhct3iR2TesQEjGFHIOBw,2411
 datapipelab/app/node/processor/bigquery_spark_node.py,sha256=pklpsqYqztidCIECkl3rpjfY6LiB0p4thvE7-PzBodE,1099
 datapipelab/app/node/processor/custom_node.py,sha256=1nqbJEhNiMP1rmN9ufpUuKO1IkuI2BEM5auW4JceGMA,933
 datapipelab/app/node/processor/gcp_bucket_node.py,sha256=bzV2c89-g5S0OH5bcKKQ-9yKOGwlmOR7h7_5uO6Gnq0,1904
@@ -21,7 +21,7 @@ datapipelab/app/node/sink/csv_node.py,sha256=ZcrMZXIwJ_ln4ZZbpCAT-iMDAZIDFI9eSHN
 datapipelab/app/node/sink/delta_node.py,sha256=4ajvMyz3cpXbd29_mZq0MW-gwpLJqdj6F9urmP8uHJw,2089
 datapipelab/app/node/sink/hive_node.py,sha256=ycknOPBBwZGH3oHram_6LjHy-ygFjhuFNvVoPaNGaCU,1220
 datapipelab/app/node/sink/pandas_csv_node.py,sha256=JsJFt2XRpwxGeJyt_PDUgqZafiQROf1Sk5TUhQPxh4c,870
-datapipelab/app/node/sink/spark_api_node.py,sha256=Uu25EtQEXJkqzm3eBEwvpuqSfVXpYUBbWRaybC_BoQQ,1406
+datapipelab/app/node/sink/spark_api_node.py,sha256=dkNg1xDbBbvZKcmpQM8r4TatqDlxJ3G0BnOSAEnRPJ0,1559
 datapipelab/app/node/sink/spark_node.py,sha256=tP3tZae2jzQtAtfIm8C-166WWSLdZs54mqoIyZOSy58,1221
 datapipelab/app/node/sink/teams_notification_node.py,sha256=6ZufdbhVvRXi3QTQafLo5uKl9kLyDnkYIE_VZFT0QNw,3581
 datapipelab/app/node/source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -31,7 +31,7 @@ datapipelab/app/node/source/spark_api_node.py,sha256=SSJW3PnuqdHPbC57pIvBSmJXoDv
 datapipelab/app/node/source/spark_node.py,sha256=TDfezmlk8Ts2YTGkB92-God_AyGVUslTUoevXolN7W8,532
 datapipelab/app/wrapper/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datapipelab/app/wrapper/source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datapipelab-0.3.5.dist-info/METADATA,sha256=0lChNDWJe-G2Hdih2jAKTPUyWMLuJSHQ8KLEGKvtmgI,7582
-datapipelab-0.3.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-datapipelab-0.3.5.dist-info/top_level.txt,sha256=HgeBjHvXorKzvNqU5BNPutoI771HtiqVit9_-0Zyrb4,12
-datapipelab-0.3.5.dist-info/RECORD,,
+datapipelab-0.3.7.dist-info/METADATA,sha256=-5JXg6lz23dHiFEpXZvZUbGoOpTJOMgnyekWQCUCw7k,7582
+datapipelab-0.3.7.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+datapipelab-0.3.7.dist-info/top_level.txt,sha256=HgeBjHvXorKzvNqU5BNPutoI771HtiqVit9_-0Zyrb4,12
+datapipelab-0.3.7.dist-info/RECORD,,