logos-sdk 0.0.23__tar.gz

logos_sdk-0.0.23/LICENSE

@@ -0,0 +1,17 @@
+MIT License
+Copyright (c) 2018 YOUR NAME
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

logos_sdk-0.0.23/PKG-INFO

@@ -0,0 +1,244 @@
+Metadata-Version: 2.1
+Name: logos-sdk
+Version: 0.0.23
+Summary: SDK for Logos platform
+Home-page: https://bitbucket.org/databy/logos-sdk-pip/src/master/
+Author: Databy.io
+Author-email: admin@proficio.cz
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: requests
+Requires-Dist: google-auth
+Requires-Dist: google-cloud-logging
+Requires-Dist: google-cloud-bigquery
+Requires-Dist: python-dotenv
+Requires-Dist: google-api-python-client
+Requires-Dist: httplib2
+Requires-Dist: pandas
+Requires-Dist: db-dtypes
+Requires-Dist: numpy
+
+# Logos Software Development Kit  
+  
+Logos Software Development Kit (Logos SDK) is a private library encapsulating functionality shared by Logos Scripts - configurable scripts running as Cloud Functions, triggered by Cloud Scheduler, which are controlled and monitored via the Logos UI, as a part of the Logos Ecosystem.  
+  
+# Functionality modules  
+  
+## Logging  
+Cloud Function Scripts within the Logos ecosystem have special requirements for logging. Apart from simple JSON messages, the logs need to also carry information on:  
+  
+- trace (used for tracking runs of Cloud Functions)  
+- labels (fields used for internal Logos filtering)  
+- logger name  
+  
+The standard Python `logging` library does not offer the possibility of capturing a combination of these values in a format parsable by Google Cloud Logging - this is only possible through the `google-cloud-logging` library. On the other hand, `google-cloud-logging` transmits all logs to Google Cloud Logging even when a script is run on the local machine (with the exception of running it within the `functions-framework` environment), which is something to be avoided when developing locally or testing on Bitbucket, as we do not want to waste our resources. 
+
+This module serves as a wrapper over the standard Python `logging` and `google-cloud-logging` libraries, with the default being set to  `google-cloud-logging`. The switch is based on the presence of environment variables `DEVELOPMENT` or `TESTING`: if either of these is set within the venv environment,  `logging` library is used instead.
+If deploying to development instance in Cloud, we want the logs to be logged into Cloud, but not to clash with the production instance, therefore setting the environment variable `CLOUD_DEVELOPMENT` ensures that the logs are logged under name `logos-logging-development` instead of the standard `logos-logging`.
+
+```bash  
+export DEVELOPMENT=True  
+#or 
+export TESTING=True  
+#or
+export CLOUD_DEVELOPMENT=True
+```  
+  
+### Usage  
+  
+To set up the Logger, one can use the parsing function `setup_from_request`, which expects a Cloud Function Script trigger request in a usual format:
+  
+```python  
+import functions_framework  
+import logos_sdk.logging as logos_logging  
+  
+@functions_framework.http  
+def run(request):  
+	logger, labels, settings, secrets = logos_logging.setup_from_request(request, "logos-logging")
+```
+```
+EXPECTED_REQUEST_JSON_BODY = {  
+	"settings": {},  
+	"id": "",  
+	"author": "",  
+	"script": "",  
+	"client": "",  
+	"accesses": [  
+		{  
+			"account": {"id": "", "account_platform_id": ""},  
+			"secret": {"id": "", "name": ""},  
+			"platform": {"id": "", "short_name": ""}  
+		},  
+		...  
+	],  
+}  
+```  
+  
+The trigger request can be also processed separately, in which case the logger is instantiated from the class itself:  
+
+```python
+LogosLogger(name="logos-logging", labels={}, trace="")`
+```
+  
+### Output  
+When the standard Python `logging` library is used, the output logs have the following structure:  
+  
+```python  
+Lowest possible severity: INFO  
+  
+String output: SEVERITY:logger-name:message  
+  
+LogEntry output:  
+	LogEntry.name = "logos-logging"  
+	LogEntry.msg = {"message": ""}  
+	LogEntry.level = 40 | ...  
+	LogEntry.levelname = INFO | ERROR | ...  
+	LogEntry.json_fields = {  
+	"logging.googleapis.com/trace": trace,  
+	"logging.googleapis.com/labels": {  
+	    "log_type": RESULT | NOTIFICATION | DEBUG,  
+	    **labels,  
+	},  
+}  
+```  
+  
+With `google-cloud-logging`, the output logs should have the following structure: 
+  
+```python  
+Lowest possible severity: INFO  
+{  
+	"logName": "projects/logos-382010/logs/logos-logging",  
+	"trace": "projects/logos-382010/traces/some-trace",  
+	"severity": INFO | ERROR | ...,  
+	"jsonPayload": { "message": "", ... },  
+	"labels": {  
+		"log_type": RESULT | NOTIFICATION | DEBUG,  
+		"id": "Logos id of the job",  
+		"author": "Logos id of the job author",   
+		"client": "Logos id of the client",  
+		"script": "Logos id of the script",
+		**platform_accounts,  
+	}  
+}  
+```  
+   
+Depending on the `platform_accounts` the Cloud Function Script accesses during its run, `labels` might also contain Logos platform `short_name` field as a key and Logo `account_platform_id` as a value, for example:  
+  
+```python  
+{  
+	"merchant-center": "xxxxxxx",  
+	"google-ads": "xxx-xxx-xxxx",  
+}  
+```  
+  
+To sum it up, an example of the complete labels contents would be:  
+  
+```python  
+{  
+	"labels": {  
+		"log_type": "result",  
+		"id": "0",  
+		"author": "0",  
+		"client": "0",  
+		"script": "0",  
+		"merchant-center": "0000000",  
+		"google-ads": "000-000-0000"  
+	}  
+}  
+```  
+  
+## Services  
+This module serves as a wrapper over communicating with Logos Services. When `DEVELOPMENT`, `TESTING` or `CLOUD_DEVELOPMENT` environment variable is set  
+  
+```bash  
+export DEVELOPMENT=True 
+#or 
+export TESTING=True  
+#or
+export CLOUD_DEVELOPMENT=True
+```  
+  
+the URLs of development services (bound to the development branches in BitBucket) are used. On the other hand, if none of these is set, production URLs are called (used in the Cloud production environment).  
+  
+# Installation as a dependency to Logos Scripts  
+  
+## Local environment  
+  
+This library is hosted as a private package in Google Artifact Registry, only accessible to authorised users - to be able to include it as a pip dependency in a Logos Script requirements.txt file, one must ensure that `google-auth`, `keyring` and `keyrings.google-artifactregistry-auth` packages were pre-installed. These libraries are able to read the contents of `GOOGLE_APPLICATION_CREDENTIALS` environment variable and pull the information necessary for authenticating to Google Artifact Registry when fetching the `logos-sdk` package from the extra index URL.  
+  
+To sum it up, the required steps for installing `logos-sdk` as a dependency are:  
+  
+1. In a Logos Script requirements.txt, put the following:  
+```bash  
+--extra-index-url https://europe-west1-python.pkg.dev/logos-382010/logos-sdk/simple logos-sdk  
+  
+logos-sdk=[desired version]  
+```  
+2. Then in your terminal with the venv active, run following:  
+```bash
+export GOOGLE_APPLICATION_CREDENTIALS=logos-382010-ee0fd6995649.json  
+pip3 install -r requirements.txt  
+export DEVELOPMENT=True  
+```  
+  
+To verify that the library was successfully installed, you can view the installed package information via:  
+  
+```bash  
+pip3 show -f logos-sdk  
+```  
+  
+## BitBucket pipelines environment  
+  
+Firstly, read the local environment setting. If "been there, done that", the setting for BitBucket pipeline testing is very similar. At first, go to pipeline variables in BitBucket UI and create a variable `GOOGLE_CREDENTIALS`, pasting the contents of the `logos-accessor` service account credentials file into it. Then your bitbucket-pipelines.yml testing step should looks like this:  
+  
+```  
+pipelines:  
+	default:  
+		- step:  
+			name: Test  
+			caches:  
+				- pip  
+			script:  
+				- echo $GOOGLE_CREDENTIALS > logos-382010-ee0fd6995649.json  
+				- export GOOGLE_APPLICATION_CREDENTIALS=$PWD/logos-382010-ee0fd6995649.json  
+				- export TESTING=True  
+				- pip3 install pytest google-auth keyring keyrings.google-artifactregistry-auth  
+				- if [ -f requirements.txt ]; then pip3 install -r requirements.txt; fi  
+				- python3 -m pytest -v tests/* --junitxml=test-reports/report.xml  
+```  
+  
+## Google Cloud environment  
+Firstly, read the local environment setting. If "been there, done that", your Dockerfile should contain the following steps:  
+  
+```  
+RUN apt-get update && apt-get install -y git  
+RUN pip3 install google-auth keyring keyrings.google-artifactregistry-auth  
+RUN pip3 install --no-cache-dir -r requirements.txt  
+```  
+  
+In your cloudbuild.yaml file, the build step should contain the `--network=cloudbuild` parameter, as this ensures that the `keyring` auth libraries have access to the necessary credentials directly from the Cloud Build environment (we no longer need to set `GOOGLE_APPLICATION_CREDENTIALS` environment variable):  
+  
+```  
+- name: 'gcr.io/cloud-builders/docker'  
+args: [ 'build', '--network=cloudbuild', '-t', 'gcr.io/logos-382010/merchant-control', '.' ]  
+```  
+
+If you are deploying a development cloud instance, in Cloud Run settings, `CLOUD_DEVELOPMENT`needs to be set:
+
+```bash  
+export CLOUD_DEVELOPMENT=True
+``` 
+
+  
+# Development & versioning  
+This is a versioned package and a version number can only be pushed once into Google Artifact Registry. Development of new features/refactor/debug follow the naming convention of:  
+  
+```  
+"feature/[short name]"  
+"refactor/[short name]"  
+"debug/[short name]"  
+```  
+  
+Pull requests are directed do the current development branch, which always bears the number of the newest version, for example `development-0.0.2`.  After adding a major feature, or a number of less significant refactors and hot-fixes, the current development branch is merged into master and deployed into Google Artifact Registry. The old development branch is then deleted and a  new branch with the following version `development-0.0.3` will be created, and the first commit must include changing of  the version in the `setup.py` file.

logos_sdk-0.0.23/README.md

@@ -0,0 +1,222 @@
+# Logos Software Development Kit  
+  
+Logos Software Development Kit (Logos SDK) is a private library encapsulating functionality shared by Logos Scripts - configurable scripts running as Cloud Functions, triggered by Cloud Scheduler, which are controlled and monitored via the Logos UI, as a part of the Logos Ecosystem.  
+  
+# Functionality modules  
+  
+## Logging  
+Cloud Function Scripts within the Logos ecosystem have special requirements for logging. Apart from simple JSON messages, the logs need to also carry information on:  
+  
+- trace (used for tracking runs of Cloud Functions)  
+- labels (fields used for internal Logos filtering)  
+- logger name  
+  
+The standard Python `logging` library does not offer the possibility of capturing a combination of these values in a format parsable by Google Cloud Logging - this is only possible through the `google-cloud-logging` library. On the other hand, `google-cloud-logging` transmits all logs to Google Cloud Logging even when a script is run on the local machine (with the exception of running it within the `functions-framework` environment), which is something to be avoided when developing locally or testing on Bitbucket, as we do not want to waste our resources. 
+
+This module serves as a wrapper over the standard Python `logging` and `google-cloud-logging` libraries, with the default being set to  `google-cloud-logging`. The switch is based on the presence of environment variables `DEVELOPMENT` or `TESTING`: if either of these is set within the venv environment,  `logging` library is used instead.
+If deploying to development instance in Cloud, we want the logs to be logged into Cloud, but not to clash with the production instance, therefore setting the environment variable `CLOUD_DEVELOPMENT` ensures that the logs are logged under name `logos-logging-development` instead of the standard `logos-logging`.
+
+```bash  
+export DEVELOPMENT=True  
+#or 
+export TESTING=True  
+#or
+export CLOUD_DEVELOPMENT=True
+```  
+  
+### Usage  
+  
+To set up the Logger, one can use the parsing function `setup_from_request`, which expects a Cloud Function Script trigger request in a usual format:
+  
+```python  
+import functions_framework  
+import logos_sdk.logging as logos_logging  
+  
+@functions_framework.http  
+def run(request):  
+	logger, labels, settings, secrets = logos_logging.setup_from_request(request, "logos-logging")
+```
+```
+EXPECTED_REQUEST_JSON_BODY = {  
+	"settings": {},  
+	"id": "",  
+	"author": "",  
+	"script": "",  
+	"client": "",  
+	"accesses": [  
+		{  
+			"account": {"id": "", "account_platform_id": ""},  
+			"secret": {"id": "", "name": ""},  
+			"platform": {"id": "", "short_name": ""}  
+		},  
+		...  
+	],  
+}  
+```  
+  
+The trigger request can be also processed separately, in which case the logger is instantiated from the class itself:  
+
+```python
+LogosLogger(name="logos-logging", labels={}, trace="")`
+```
+  
+### Output  
+When the standard Python `logging` library is used, the output logs have the following structure:  
+  
+```python  
+Lowest possible severity: INFO  
+  
+String output: SEVERITY:logger-name:message  
+  
+LogEntry output:  
+	LogEntry.name = "logos-logging"  
+	LogEntry.msg = {"message": ""}  
+	LogEntry.level = 40 | ...  
+	LogEntry.levelname = INFO | ERROR | ...  
+	LogEntry.json_fields = {  
+	"logging.googleapis.com/trace": trace,  
+	"logging.googleapis.com/labels": {  
+	    "log_type": RESULT | NOTIFICATION | DEBUG,  
+	    **labels,  
+	},  
+}  
+```  
+  
+With `google-cloud-logging`, the output logs should have the following structure: 
+  
+```python  
+Lowest possible severity: INFO  
+{  
+	"logName": "projects/logos-382010/logs/logos-logging",  
+	"trace": "projects/logos-382010/traces/some-trace",  
+	"severity": INFO | ERROR | ...,  
+	"jsonPayload": { "message": "", ... },  
+	"labels": {  
+		"log_type": RESULT | NOTIFICATION | DEBUG,  
+		"id": "Logos id of the job",  
+		"author": "Logos id of the job author",   
+		"client": "Logos id of the client",  
+		"script": "Logos id of the script",
+		**platform_accounts,  
+	}  
+}  
+```  
+   
+Depending on the `platform_accounts` the Cloud Function Script accesses during its run, `labels` might also contain Logos platform `short_name` field as a key and Logo `account_platform_id` as a value, for example:  
+  
+```python  
+{  
+	"merchant-center": "xxxxxxx",  
+	"google-ads": "xxx-xxx-xxxx",  
+}  
+```  
+  
+To sum it up, an example of the complete labels contents would be:  
+  
+```python  
+{  
+	"labels": {  
+		"log_type": "result",  
+		"id": "0",  
+		"author": "0",  
+		"client": "0",  
+		"script": "0",  
+		"merchant-center": "0000000",  
+		"google-ads": "000-000-0000"  
+	}  
+}  
+```  
+  
+## Services  
+This module serves as a wrapper over communicating with Logos Services. When `DEVELOPMENT`, `TESTING` or `CLOUD_DEVELOPMENT` environment variable is set  
+  
+```bash  
+export DEVELOPMENT=True 
+#or 
+export TESTING=True  
+#or
+export CLOUD_DEVELOPMENT=True
+```  
+  
+the URLs of development services (bound to the development branches in BitBucket) are used. On the other hand, if none of these is set, production URLs are called (used in the Cloud production environment).  
+  
+# Installation as a dependency to Logos Scripts  
+  
+## Local environment  
+  
+This library is hosted as a private package in Google Artifact Registry, only accessible to authorised users - to be able to include it as a pip dependency in a Logos Script requirements.txt file, one must ensure that `google-auth`, `keyring` and `keyrings.google-artifactregistry-auth` packages were pre-installed. These libraries are able to read the contents of `GOOGLE_APPLICATION_CREDENTIALS` environment variable and pull the information necessary for authenticating to Google Artifact Registry when fetching the `logos-sdk` package from the extra index URL.  
+  
+To sum it up, the required steps for installing `logos-sdk` as a dependency are:  
+  
+1. In a Logos Script requirements.txt, put the following:  
+```bash  
+--extra-index-url https://europe-west1-python.pkg.dev/logos-382010/logos-sdk/simple logos-sdk  
+  
+logos-sdk=[desired version]  
+```  
+2. Then in your terminal with the venv active, run following:  
+```bash
+export GOOGLE_APPLICATION_CREDENTIALS=logos-382010-ee0fd6995649.json  
+pip3 install -r requirements.txt  
+export DEVELOPMENT=True  
+```  
+  
+To verify that the library was successfully installed, you can view the installed package information via:  
+  
+```bash  
+pip3 show -f logos-sdk  
+```  
+  
+## BitBucket pipelines environment  
+  
+Firstly, read the local environment setting. If "been there, done that", the setting for BitBucket pipeline testing is very similar. At first, go to pipeline variables in BitBucket UI and create a variable `GOOGLE_CREDENTIALS`, pasting the contents of the `logos-accessor` service account credentials file into it. Then your bitbucket-pipelines.yml testing step should looks like this:  
+  
+```  
+pipelines:  
+	default:  
+		- step:  
+			name: Test  
+			caches:  
+				- pip  
+			script:  
+				- echo $GOOGLE_CREDENTIALS > logos-382010-ee0fd6995649.json  
+				- export GOOGLE_APPLICATION_CREDENTIALS=$PWD/logos-382010-ee0fd6995649.json  
+				- export TESTING=True  
+				- pip3 install pytest google-auth keyring keyrings.google-artifactregistry-auth  
+				- if [ -f requirements.txt ]; then pip3 install -r requirements.txt; fi  
+				- python3 -m pytest -v tests/* --junitxml=test-reports/report.xml  
+```  
+  
+## Google Cloud environment  
+Firstly, read the local environment setting. If "been there, done that", your Dockerfile should contain the following steps:  
+  
+```  
+RUN apt-get update && apt-get install -y git  
+RUN pip3 install google-auth keyring keyrings.google-artifactregistry-auth  
+RUN pip3 install --no-cache-dir -r requirements.txt  
+```  
+  
+In your cloudbuild.yaml file, the build step should contain the `--network=cloudbuild` parameter, as this ensures that the `keyring` auth libraries have access to the necessary credentials directly from the Cloud Build environment (we no longer need to set `GOOGLE_APPLICATION_CREDENTIALS` environment variable):  
+  
+```  
+- name: 'gcr.io/cloud-builders/docker'  
+args: [ 'build', '--network=cloudbuild', '-t', 'gcr.io/logos-382010/merchant-control', '.' ]  
+```  
+
+If you are deploying a development cloud instance, in Cloud Run settings, `CLOUD_DEVELOPMENT`needs to be set:
+
+```bash  
+export CLOUD_DEVELOPMENT=True
+``` 
+
+  
+# Development & versioning  
+This is a versioned package and a version number can only be pushed once into Google Artifact Registry. Development of new features/refactor/debug follow the naming convention of:  
+  
+```  
+"feature/[short name]"  
+"refactor/[short name]"  
+"debug/[short name]"  
+```  
+  
+Pull requests are directed do the current development branch, which always bears the number of the newest version, for example `development-0.0.2`.  After adding a major feature, or a number of less significant refactors and hot-fixes, the current development branch is merged into master and deployed into Google Artifact Registry. The old development branch is then deleted and a  new branch with the following version `development-0.0.3` will be created, and the first commit must include changing of  the version in the `setup.py` file.

logos_sdk-0.0.23/logos_sdk/__init__.py

@@ -0,0 +1,4 @@
+PRODUCTION = "PRODUCTION"
+DEVELOPMENT = "DEVELOPMENT"
+TESTING = "TESTING"
+CLOUD_DEVELOPMENT = "CLOUD_DEVELOPMENT"

logos_sdk-0.0.23/logos_sdk/big_query/BigQuery.py

@@ -0,0 +1,152 @@
+import os
+from datetime import datetime
+
+import google.auth.exceptions
+from typing import List, Dict, Union, Optional
+
+from google.api_core.exceptions import NotFound
+from google.cloud.logging import Client as LoggerClient
+from google.auth.exceptions import DefaultCredentialsError
+from google.cloud.bigquery import Table
+from google.cloud import bigquery
+import numpy as np
+import time
+
+from google.api_core.retry import Retry
+
+from logos_sdk.big_query import retry_on_not_found
+from dotenv import load_dotenv
+
+
+class BigQueryException(Exception):
+    def __init__(self, messages):
+        self.messages = messages
+
+
+class BigQuery:
+    BQ_ROWS_LIMIT = 10000
+    _service = None
+
+    def __init__(self):
+        load_dotenv()
+        self.project_id = os.environ.get("PROJECT_ID")
+        self._service = bigquery.Client(project=self.project_id)
+        try:
+            self.logger = LoggerClient(_use_grpc=False).logger(name="logos-logging")
+        except DefaultCredentialsError:
+            self.logger = None
+
+    def get_dataset(self, dataset_id: str):
+        return self._service.get_dataset(dataset_id)
+
+    def create_dataset(self, dataset_id: str):
+        return self._service.create_dataset(dataset_id)
+
+    def delete_dataset(self, dataset_id: str):
+        self._service.delete_dataset(dataset_id, not_found_ok=True)
+
+    def _get_table_id_sql_format(self, dataset_id: str, table_id: str):
+        return f"{self.project_id}.{dataset_id}.{table_id}"
+
+    def run_query(self, query: str) -> List[Dict]:
+        df = self._service.query(query).result().to_dataframe().fillna(np.nan)
+        return df.replace([np.nan], [None]).to_dict("records")
+
+    def get_table(self, dataset_id: str, table_id: str) -> Table:
+        sql_format = self._get_table_id_sql_format(dataset_id, table_id)
+        return self._service.get_table(sql_format)
+
+    def insert_into_table(
+            self, dataset_id: str, table_id: str, records: List[Dict]
+    ) -> None:
+        bq_table = self.get_table(dataset_id, table_id)
+        self._insert_into_table(bq_table, records)
+
+    def insert_create_table(
+            self,
+            dataset_id: str,
+            table_id: str,
+            records: List[Dict],
+            schema_columns: List[Dict],
+    ) -> None:
+        bq_table = self.check_table_exists(dataset_id, table_id)
+        if bq_table is None:
+            bq_table = self.create_table(dataset_id, table_id, schema_columns)
+
+        self._insert_into_table(bq_table, records)
+
+    def delete_table(self, dataset_id: str, table_id: str) -> None:
+        if self.check_table_exists(dataset_id, table_id):
+            sql_format = self._get_table_id_sql_format(dataset_id, table_id)
+            self._service.delete_table(sql_format)
+
+    def check_table_exists(self, dataset_id: str, table_id: str) -> Optional[Table]:
+        try:
+            return self.get_table(dataset_id, table_id)
+        except google.cloud.exceptions.NotFound:
+            return None
+
+    def create_table(
+            self, dataset_id: str, table_id: str, schema_columns: List[Dict]
+    ) -> Union[bool, Table]:
+        table_schema = [
+            bigquery.schema.SchemaField(row["name"], row["col_type"], mode=row["mode"])
+            for row in schema_columns
+        ]
+        try:
+            sql_format = self._get_table_id_sql_format(dataset_id, table_id)
+            table_object = bigquery.Table(sql_format, schema=table_schema)
+            return self._service.create_table(table_object)
+        except google.cloud.exceptions.Conflict:
+            return False
+
+    def create_view(self, dataset_id, view_id, sql_string):
+        try:
+            sql_format = self._get_table_id_sql_format(dataset_id, view_id)
+            view = bigquery.Table(sql_format)
+            view.view_query = sql_string
+            return self._service.create_table(view)
+        except google.cloud.exceptions.Conflict:
+            return False
+
+    def get_table_last_modified_date(self, dataset_id: str, table_id: str):
+        last_modified_timestamp = (
+            self._service.query(
+                f"SELECT TIMESTAMP_MILLIS(last_modified_time) as time_stamp "
+                f"FROM `{self.project_id}.{dataset_id}.__TABLES__` "
+                f"WHERE table_id = '{table_id}'"
+            )
+            .result()
+            .to_dataframe()
+        )
+
+        if last_modified_timestamp.empty:
+            raise NotFound("Table does not exist!")
+
+        last_modified_timestamp = last_modified_timestamp["time_stamp"].iloc[0]
+        last_modified_date = datetime.strptime(
+            str(last_modified_timestamp), "%Y-%m-%d %H:%M:%S.%f+00:00"
+        )
+
+        return last_modified_date
+
+    @retry_on_not_found
+    def _insert_into_table(
+            self, bq_table: Table, records: List[Dict], attempts: int
+    ) -> None:
+        if len(records) > self.BQ_ROWS_LIMIT:
+            for index in range(0, len(records), self.BQ_ROWS_LIMIT):
+                errors = self._service.insert_rows(
+                    bq_table,
+                    records[index: (index + self.BQ_ROWS_LIMIT)],
+                    retry=Retry(
+                        total=2, connect=4, backoff_factor=2, allowed_methods=None
+                    ),
+                )
+                time.sleep(2)
+                if errors:
+                    raise BigQueryException(errors)
+        else:
+            errors = self._service.insert_rows(bq_table, records)
+            if errors:
+                raise BigQueryException(errors)

logos_sdk-0.0.23/logos_sdk/big_query/__init__.py

@@ -0,0 +1,22 @@
+from functools import wraps
+from google.api_core.exceptions import NotFound
+import time
+
+MAX_NUMBER_OF_ATTEMPTS = 2
+
+
+def retry_on_not_found(wrapped_function):
+    """This decorator retry call when table is not found. Insert into newly created table often fails with error
+    because API probably needs few seconds to see new created table"""
+
+    @wraps(wrapped_function)
+    def inner(*args, **kwargs):
+        for i in range(1, MAX_NUMBER_OF_ATTEMPTS + 1):
+            try:
+                kwargs["attempts"] = i
+                return wrapped_function(*args, **kwargs)
+                # this is because all request share same service
+            except NotFound as err:
+                time.sleep(2)
+
+    return inner