datapipelab 0.3.5__tar.gz → 0.3.7__tar.gz

{datapipelab-0.3.5 → datapipelab-0.3.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datapipelab
-Version: 0.3.5
+Version: 0.3.7
 Summary: A data pipeline library with connectors, sources, processors, and sinks.
 Description-Content-Type: text/markdown
 Requires-Dist: json5

{datapipelab-0.3.5 → datapipelab-0.3.7}/datapipelab/app/node/processor/bigquery_api_node.py

@@ -6,16 +6,29 @@ class BigQueryAPIProcessorNode(TNode):
         super().__init__(spark=spark)
         self.sql_query = tnode_config['options']['query']
         self.node_name = tnode_config['name']
-        self.credentials_path = tnode_config['options']['credentials_path']
+        self.credentials_path = tnode_config['options'].get('credentials_path', None)
+        self.credentials_info_env_name = tnode_config['options'].get('credentials_info_env_name', None)
         self.return_as_spark_df = tnode_config['options']['return_as_spark_df']
         self.project_name = tnode_config['options']['project_name']
         self.return_as_python_list = tnode_config['options'].get('return_as_python_list', False)
         self.return_as_is = tnode_config['options'].get('return_as_is', False)
 
+    def __get_gcp_credentials(self):
+        from google.oauth2 import service_account
+        if self.credentials_path:
+            return service_account.Credentials.from_service_account_file(self.credentials_path)
+        elif self.credentials_info_env_name:
+            import os
+            import json
+            credentials_info = json.loads(os.environ[self.credentials_info_env_name])
+            return service_account.Credentials.from_service_account_info(credentials_info)
+        else:
+            logger.warning("No credentials provided. Please provide either 'credentials_path' or 'credentials_info'.")
+            return None
+
     def __sql_biqquery(self, sql_query):
         from google.cloud import bigquery
-        from google.oauth2 import service_account
-        credentials = service_account.Credentials.from_service_account_file(self.credentials_path)
+        credentials = self.__get_gcp_credentials()
         client = bigquery.Client(credentials=credentials, project=self.project_name)
 
         # run the job

{datapipelab-0.3.5 → datapipelab-0.3.7}/datapipelab/app/node/sink/spark_api_node.py

@@ -22,6 +22,8 @@ class SparkApiSourceNode(TNode):
             self.options['table'] = self.spark_options.get('table')
         if 'write_method' in self.spark_options:
             self.options['writeMethod'] = self.spark_options.get('write_method')
+        if 'temporary_gcs_bucket' in self.spark_options:
+            self.options['temporaryGcsBucket'] = self.spark_options.get('temporary_gcs_bucket')
 
     def __write_df(self):
         writer = self.df.write

{datapipelab-0.3.5 → datapipelab-0.3.7}/datapipelab.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datapipelab
-Version: 0.3.5
+Version: 0.3.7
 Summary: A data pipeline library with connectors, sources, processors, and sinks.
 Description-Content-Type: text/markdown
 Requires-Dist: json5

{datapipelab-0.3.5 → datapipelab-0.3.7}/setup.py

@@ -6,7 +6,7 @@ long_description = (this_directory / "README.md").read_text()
 
 setup(
     name='datapipelab',
-    version='0.3.5',
+    version='0.3.7',
     description='A data pipeline library with connectors, sources, processors, and sinks.',
     packages=find_packages(),
     include_package_data=True,