logos-sdk 0.0.23__py3-none-any.whl

logos_sdk/__init__.py

@@ -0,0 +1,4 @@
+PRODUCTION = "PRODUCTION"
+DEVELOPMENT = "DEVELOPMENT"
+TESTING = "TESTING"
+CLOUD_DEVELOPMENT = "CLOUD_DEVELOPMENT"

logos_sdk/big_query/BigQuery.py

@@ -0,0 +1,152 @@
+import os
+from datetime import datetime
+
+import google.auth.exceptions
+from typing import List, Dict, Union, Optional
+
+from google.api_core.exceptions import NotFound
+from google.cloud.logging import Client as LoggerClient
+from google.auth.exceptions import DefaultCredentialsError
+from google.cloud.bigquery import Table
+from google.cloud import bigquery
+import numpy as np
+import time
+
+from google.api_core.retry import Retry
+
+from logos_sdk.big_query import retry_on_not_found
+from dotenv import load_dotenv
+
+
+class BigQueryException(Exception):
+    def __init__(self, messages):
+        self.messages = messages
+
+
+class BigQuery:
+    BQ_ROWS_LIMIT = 10000
+    _service = None
+
+    def __init__(self):
+        load_dotenv()
+        self.project_id = os.environ.get("PROJECT_ID")
+        self._service = bigquery.Client(project=self.project_id)
+        try:
+            self.logger = LoggerClient(_use_grpc=False).logger(name="logos-logging")
+        except DefaultCredentialsError:
+            self.logger = None
+
+    def get_dataset(self, dataset_id: str):
+        return self._service.get_dataset(dataset_id)
+
+    def create_dataset(self, dataset_id: str):
+        return self._service.create_dataset(dataset_id)
+
+    def delete_dataset(self, dataset_id: str):
+        self._service.delete_dataset(dataset_id, not_found_ok=True)
+
+    def _get_table_id_sql_format(self, dataset_id: str, table_id: str):
+        return f"{self.project_id}.{dataset_id}.{table_id}"
+
+    def run_query(self, query: str) -> List[Dict]:
+        df = self._service.query(query).result().to_dataframe().fillna(np.nan)
+        return df.replace([np.nan], [None]).to_dict("records")
+
+    def get_table(self, dataset_id: str, table_id: str) -> Table:
+        sql_format = self._get_table_id_sql_format(dataset_id, table_id)
+        return self._service.get_table(sql_format)
+
+    def insert_into_table(
+            self, dataset_id: str, table_id: str, records: List[Dict]
+    ) -> None:
+        bq_table = self.get_table(dataset_id, table_id)
+        self._insert_into_table(bq_table, records)
+
+    def insert_create_table(
+            self,
+            dataset_id: str,
+            table_id: str,
+            records: List[Dict],
+            schema_columns: List[Dict],
+    ) -> None:
+        bq_table = self.check_table_exists(dataset_id, table_id)
+        if bq_table is None:
+            bq_table = self.create_table(dataset_id, table_id, schema_columns)
+
+        self._insert_into_table(bq_table, records)
+
+    def delete_table(self, dataset_id: str, table_id: str) -> None:
+        if self.check_table_exists(dataset_id, table_id):
+            sql_format = self._get_table_id_sql_format(dataset_id, table_id)
+            self._service.delete_table(sql_format)
+
+    def check_table_exists(self, dataset_id: str, table_id: str) -> Optional[Table]:
+        try:
+            return self.get_table(dataset_id, table_id)
+        except google.cloud.exceptions.NotFound:
+            return None
+
+    def create_table(
+            self, dataset_id: str, table_id: str, schema_columns: List[Dict]
+    ) -> Union[bool, Table]:
+        table_schema = [
+            bigquery.schema.SchemaField(row["name"], row["col_type"], mode=row["mode"])
+            for row in schema_columns
+        ]
+        try:
+            sql_format = self._get_table_id_sql_format(dataset_id, table_id)
+            table_object = bigquery.Table(sql_format, schema=table_schema)
+            return self._service.create_table(table_object)
+        except google.cloud.exceptions.Conflict:
+            return False
+
+    def create_view(self, dataset_id, view_id, sql_string):
+        try:
+            sql_format = self._get_table_id_sql_format(dataset_id, view_id)
+            view = bigquery.Table(sql_format)
+            view.view_query = sql_string
+            return self._service.create_table(view)
+        except google.cloud.exceptions.Conflict:
+            return False
+
+    def get_table_last_modified_date(self, dataset_id: str, table_id: str):
+        last_modified_timestamp = (
+            self._service.query(
+                f"SELECT TIMESTAMP_MILLIS(last_modified_time) as time_stamp "
+                f"FROM `{self.project_id}.{dataset_id}.__TABLES__` "
+                f"WHERE table_id = '{table_id}'"
+            )
+            .result()
+            .to_dataframe()
+        )
+
+        if last_modified_timestamp.empty:
+            raise NotFound("Table does not exist!")
+
+        last_modified_timestamp = last_modified_timestamp["time_stamp"].iloc[0]
+        last_modified_date = datetime.strptime(
+            str(last_modified_timestamp), "%Y-%m-%d %H:%M:%S.%f+00:00"
+        )
+
+        return last_modified_date
+
+    @retry_on_not_found
+    def _insert_into_table(
+            self, bq_table: Table, records: List[Dict], attempts: int
+    ) -> None:
+        if len(records) > self.BQ_ROWS_LIMIT:
+            for index in range(0, len(records), self.BQ_ROWS_LIMIT):
+                errors = self._service.insert_rows(
+                    bq_table,
+                    records[index: (index + self.BQ_ROWS_LIMIT)],
+                    retry=Retry(
+                        total=2, connect=4, backoff_factor=2, allowed_methods=None
+                    ),
+                )
+                time.sleep(2)
+                if errors:
+                    raise BigQueryException(errors)
+        else:
+            errors = self._service.insert_rows(bq_table, records)
+            if errors:
+                raise BigQueryException(errors)

logos_sdk/big_query/__init__.py

@@ -0,0 +1,22 @@
+from functools import wraps
+from google.api_core.exceptions import NotFound
+import time
+
+MAX_NUMBER_OF_ATTEMPTS = 2
+
+
+def retry_on_not_found(wrapped_function):
+    """This decorator retry call when table is not found. Insert into newly created table often fails with error
+    because API probably needs few seconds to see new created table"""
+
+    @wraps(wrapped_function)
+    def inner(*args, **kwargs):
+        for i in range(1, MAX_NUMBER_OF_ATTEMPTS + 1):
+            try:
+                kwargs["attempts"] = i
+                return wrapped_function(*args, **kwargs)
+                # this is because all request share same service
+            except NotFound as err:
+                time.sleep(2)
+
+    return inner

logos_sdk/logging/LogosLogger.py

@@ -0,0 +1,66 @@
+import os
+import logging
+from google.cloud.logging import Client as LoggerClient
+from dotenv import load_dotenv
+from logos_sdk import DEVELOPMENT, TESTING, CLOUD_DEVELOPMENT, PRODUCTION
+
+
+class LogosLogger:
+
+    def __init__(self, name="logos-logging", labels=None, trace=None):
+
+        load_dotenv()
+
+        self.labels = labels
+        self.trace = trace
+        self.name = name
+        self.settings = {}
+        self.accesses = []
+
+        # if current environment is local development or Bitbucket pipeline testing, we do not care
+        # we want the name of the logger to be 'logos-logging', because it is not being sent to Cloud
+        # and we want to be able to test it as a real thing. On the other hand, if we are in production or
+        # cloud development, we need to be logging the logs into Cloud, therefore these two environments need
+        # to be separated, so that the logs do not clash.
+
+        if os.environ.get(DEVELOPMENT):
+            self.env = DEVELOPMENT
+        elif os.environ.get(TESTING):
+            self.env = TESTING
+        elif os.environ.get(CLOUD_DEVELOPMENT):
+            self.env = CLOUD_DEVELOPMENT
+        else:
+            self.env = PRODUCTION
+
+        if self.env == DEVELOPMENT or self.env == TESTING:
+            self.stream_logger = logging.getLogger(name=self.name)
+        else:
+            if self.env == CLOUD_DEVELOPMENT and self.name == "logos-logging":
+                self.name = "logos-logging-development"
+
+            self.cloud_client = LoggerClient()
+            self.cloud_logger = self.cloud_client.logger(name=self.name)
+
+    def get_name(self):
+        return self.name
+
+    def log(self, message, severity, log_type=None):
+        labels = self.labels if log_type is None else {"log_type": log_type, **(self.labels or {})}
+        if self.env == DEVELOPMENT or self.env == TESTING:
+            self.stream_logger.log(
+                msg={"message": message, "settings": self.settings, "accesses": self.accesses},
+                level=severity,
+                extra={
+                    "json_fields": {
+                        "logging.googleapis.com/trace": self.trace,
+                        "logging.googleapis.com/labels": labels
+                    }
+                }
+            )
+        else:
+            self.cloud_logger.log_struct(
+                info={"message": message, "settings": self.settings, "accesses": self.accesses},
+                labels=labels,
+                severity=logging.getLevelName(severity),
+                trace=self.trace
+            )

logos_sdk/logging/__init__.py

@@ -0,0 +1,76 @@
+from logos_sdk.logging.LogosLogger import LogosLogger
+import logging
+
+# log types in labels
+NOTIFICATION = "notification"
+DEBUG = "debug"
+RESULT = "result"
+RESULT_UI_ONLY = "result-ui-only"
+
+def setup_from_request(request, logger_name="logos-logging"):
+    # We want to parse as much as possible without raising an exception.
+    # we want to be able to at least create a complete notification log, for this we need labels without accesses and trace.
+    try:
+        trace = request.headers["X-Cloud-Trace-Context"]
+    except Exception as err:
+        raise Exception(f"Unable to retrieve trace from request ({err})")
+
+    try:
+        logger = LogosLogger(logger_name, {}, trace)
+    except Exception as err:
+        raise Exception(f"Unable to ser up LogosLogger ({err})")
+
+    try:
+        body = request.get_json()
+    except Exception as err:
+        message = f"Unable to parse request body as a valid json ({err})"
+        logger.log(message=message, severity=logging.ERROR, log_type=NOTIFICATION)
+        raise Exception(message)
+
+    try:
+        labels = {
+            "id": str(body["id"]),
+            "client": str(body["client"]),
+            "script": str(body["script"]),
+            "author": str(body["author"]),
+        }
+        logger.labels = labels
+    except Exception as err:
+        message = f"Unable to create labels, missing key ({err})"
+        logger.log(message=message, severity=logging.ERROR, log_type=NOTIFICATION)
+        raise Exception(message)
+
+    # To run the script itself, we need the rest. If we are not able to run it,
+    # we want to be able to at least create a complete notification log.
+    try:
+        settings = body["settings"]
+        logger.settings = body["settings"]
+    except Exception as err:
+        message = f"Missing required settings ({err})"
+        logger.log(message=message, severity=logging.ERROR, log_type=NOTIFICATION)
+        raise Exception(message)
+
+    try:
+        accesses = {
+            str(access["platform"]["short_name"]): str(access["account"]["account_platform_id"]) for access in
+            body["accesses"]
+        }
+        labels = {**labels, **accesses}
+        logger.labels = labels
+        logger.accesses = body["accesses"]
+    except Exception as err:
+        message = f"Unable to parse accesses ({err})"
+        logger.log(message=message, severity=logging.ERROR, log_type=NOTIFICATION)
+        raise Exception(message)
+
+    try:
+        secrets = {
+            str(access["platform"]["short_name"]): str(access["secret"]["name"]) for access in body["accesses"]
+        }
+    except Exception as err:
+        message = f"Unable to parse secrets ({err})"
+        logger.log(message=message, severity=logging.ERROR, log_type=NOTIFICATION)
+        raise Exception(message)
+
+
+    return logger, labels, settings, secrets

logos_sdk/services/CampaignManager.py

@@ -0,0 +1,346 @@
+import io
+import json
+import httplib2
+import os
+import time
+
+from random import randint
+from logos_sdk.services import get_headers
+from requests import request
+from typing import Dict, List
+from http import HTTPStatus
+from googleapiclient.http import MediaIoBaseDownload, HttpRequest
+from pandas import read_csv
+from dotenv import load_dotenv
+
+
+class CampaignManagerServiceException(Exception):
+    pass
+
+
+class CampaignManagerService:
+    def __init__(self, url: str = None):
+        load_dotenv()
+        self._URL = url or os.environ.get("CM360_SERVICE_PATH")
+        self._CREATE_REPORT = self._URL + "/create-report"
+        self._GET_REPORT = self._URL + "/get-report"
+        self._RUN_REPORT = self._URL + "/run-report"
+        self._PATCH_REPORT = self._URL + "/patch-report"
+        self._DELETE_REPORT = self._URL + "/delete-report"
+        self._GET_FILE = self._URL + "/get-file"
+        self._GET_FILE_MEDIA_REQUEST = self._URL + "/get-file-media-request"
+        self._QUERY_DIMENSION_VALUES = self._URL + "/query-dimension-values"
+
+    def create_report(
+        self,
+        account_id: str,
+        name: str,
+        start_date: str,
+        end_date: str,
+        dimensions: list,
+        metrics_names: list,
+        dimension_filters: list,
+        secret_id: str,
+    ) -> Dict:
+        """
+        Method for creating a report in API and returning a dict containing its ID for further use
+        :param account_id: the Campaign Manager 360 account ID
+        :param name: the name for the report
+        :param start_date: start date for the report, in yyyy-mm-dd format
+        :param end_date: end date for the report, in yyyy-mm-dd format
+        :param dimensions: a list containing dimensions for the report
+        :param metrics_names: a list containing metrics for the report
+        :param dimension_filters: a list containing filters for the dimensions
+        :param secret_id: ID of the secret in Secret Manager to be used to access Campaign Manager 360
+        :return Dict
+        """
+        body = {
+            "account_id": account_id,
+            "name": name,
+            "date_range": {"startDate": start_date, "endDate": end_date},
+            "dimensions": dimensions,
+            "metrics_names": metrics_names,
+            "dimension_filters": dimension_filters,
+            "secret_id": secret_id,
+        }
+        header = get_headers(self._CREATE_REPORT)
+        response = request(
+            method="post", url=self._CREATE_REPORT, json=body, headers=header
+        )
+
+        if response.status_code == HTTPStatus.OK:
+            service_response = response.json()
+            return service_response["data"]
+        else:
+            raise CampaignManagerServiceException(response.content)
+
+    def get_report(self, account_id: str, report_id: str, secret_id: str) -> Dict:
+        """
+        Method to return report
+        :param account_id: The ID of the account to fetch the report from
+        :param report_id: The ID of the report
+        :param secret_id: ID of the secret in Secret Manager to be used to access Campaign Manager 360
+        :return Dict
+        """
+        body = {
+            "account_id": account_id,
+            "report_id": report_id,
+            "secret_id": secret_id,
+        }
+
+        header = get_headers(self._GET_REPORT)
+        response = request("post", url=self._GET_REPORT, json=body, headers=header)
+
+        if response.status_code == HTTPStatus.OK:
+            service_response = response.json()
+            return service_response["data"]
+        else:
+            raise CampaignManagerServiceException(response.content)
+
+    def run_report(self, account_id: str, report_id: str, secret_id: str) -> Dict:
+        """
+        Method to run report
+        :param account_id: The ID of the account to run the report for
+        :param report_id: The ID of the report
+        :param secret_id: ID of the secret in Secret Manager to be used to access Campaign Manager 360
+        :return Dict
+        """
+        body = {
+            "account_id": account_id,
+            "report_id": report_id,
+            "secret_id": secret_id,
+        }
+        header = get_headers(self._RUN_REPORT)
+        response = request("post", url=self._RUN_REPORT, json=body, headers=header)
+        service_response = response.json()
+
+        if response.status_code == HTTPStatus.OK:
+            return service_response["data"]
+        else:
+            raise CampaignManagerServiceException(service_response)
+
+    # def patch_report(
+    #     self,
+    #     report_id: str,
+    #     advertiser_name: str,
+    #     start_date: str,
+    #     end_date: str,
+    #     dimensions: list,
+    #     metrics_names: list,
+    #     dimension_filters: list,
+    #     secret_id: str,
+    # ) -> bool:
+    #     """
+    #     Method to patch report
+    #     :param report_id: The ID of the report.
+    #     :param advertiser_name: advertiser name of advertiser dimension filter.
+    #     :param start_date: The start date for which this report should be run.
+    #     :param end_date: The end date for which this report should be run.
+    #     :param dimensions: The list of standard dimensions. Valid dimensions: dv360InsertionOrder, dv360LineItem, dv360Site.
+    #     :param metrics_names: The list of names of metrics. Valid metric names: clickRate, impressions, totalConversions.
+    #     :param dimension_filters: The list of filters on which dimensions are filtered.
+    #     :param secret_id: ID of the secret in Secret Manager to be used to access Campaign Manager 360
+    #     :return Boolean
+    #     """
+    #     body = {
+    #         "report_id": report_id,
+    #         "advertiser_name": advertiser_name,
+    #         "date_range": {"startDate": start_date, "endDate": end_date},
+    #         "dimensions": dimensions,
+    #         "metrics_names": metrics_names,
+    #         "dimension_filters": dimension_filters,
+    #         "secret_id": secret_id,
+    #     }
+    #
+    #     header = get_headers(self._PATCH_REPORT)
+    #     response = request("patch", url=self._PATCH_REPORT, json=body, headers=header)
+    #     service_response = response.json()
+    #     if response.status_code == HTTPStatus.OK:
+    #         return True
+    #     else:
+    #         raise CampaignManagerServiceException(service_response)
+
+    def check_report_ready(self, report_id: str, file_id: str, secret_id: str) -> Dict:
+        """
+        Method to check weather the report is already available for download
+        :param report_id: The ID of the report
+        :param file_id: The ID of the report file to be downloaded
+        :param secret_id: ID of the secret in Secret Manager to be used to access Campaign Manager 360
+        :return Dict
+        """
+        body = {"report_id": report_id, "file_id": file_id, "secret_id": secret_id}
+        header = get_headers(self._GET_FILE)
+        response = request("post", url=self._GET_FILE, json=body, headers=header)
+
+        if response.status_code == HTTPStatus.OK:
+            service_response = response.json()
+            return service_response["data"]["status"] == "REPORT_AVAILABLE"
+        else:
+            raise CampaignManagerServiceException(response.content)
+
+    def check_report_ready_with_exponential_backoff(
+        self, report_id: str, file_id: str, secret_id: str, backoff_attempts: int = 10
+    ) -> bool:
+        """
+        Implements exponential backoff for pooling the API for readiness of the report, suggested in
+        algorithm suggested by https://developers.google.com/doubleclick-advertisers/upload#exp-backoff
+        :param report_id: The ID of the report
+        :param file_id: The ID of the report file to be downloaded
+        :param secret_id: ID of the secret in Secret Manager to be used to access Campaign Manager 360
+        :param backoff_attempts: number of attempts
+        :return Bool
+        """
+        for attempt in range(0, backoff_attempts):
+            if self.check_report_ready(report_id, file_id, secret_id):
+                return True
+            else:
+                time.sleep((2 ** attempt) + randint(1, 20))
+
+        return False
+
+    def get_report_results(self, report_id: str, file_id: str, secret_id: str) -> Dict:
+        """
+        Method to fetch report, based on https://developers.google.com/doubleclick-advertisers/guides/download_reports
+        and https://googleapis.github.io/google-api-python-client/docs/epy/googleapiclient.http.MediaIoBaseDownload-class.html
+        :param account_id: The ID of the account to check for the report
+        :param report_id: The ID of the report
+        :param file_id: The ID of the report file to be downloaded
+        :param secret_id: ID of the secret in Secret Manager to be used to access Campaign Manager 360
+        :return Dict
+        """
+        body = {"report_id": report_id, "file_id": file_id, "secret_id": secret_id}
+        header = get_headers(self._GET_FILE_MEDIA_REQUEST)
+
+        # fetch the authorized request from our service for downloading the report from API
+        response = request(
+            "post", url=self._GET_FILE_MEDIA_REQUEST, json=body, headers=header
+        )
+
+        if response.status_code == HTTPStatus.OK:
+            service_response = response.json()
+            authorised_request = HttpRequest.from_json(
+                json.dumps(service_response["data"]),
+                http=httplib2.Http(),
+                postproc=None,
+            )
+
+            # crete tmp directory if not exist
+            if not os.path.exists("tmp"):
+                os.makedirs("tmp")
+            result_temp_file_location = os.path.join("tmp", "report-result-temp.csv")
+            # prepare a local file to download the report contents to
+            result_temp_file = io.FileIO(result_temp_file_location, mode="wb")
+            # 1024 * 1024 * 10 = 10 MB, based on best practices described in CM360 API documentation, default would be 1 MB
+            chunk_size = 10485760
+            # create a media downloader instance
+            media_downloader = MediaIoBaseDownload(
+                result_temp_file, authorised_request, chunksize=chunk_size
+            )
+            # execute the get request and download the file.
+            download_finished = False
+            while download_finished is False:
+                _, download_finished = media_downloader.next_chunk()
+
+            result_temp_file.close()
+
+            skip = 0
+            with open(result_temp_file_location, "r") as file:
+                for line in file:
+                    skip += 1
+                    if "Report Fields" == line.strip():
+                        break
+
+            results = read_csv(
+                result_temp_file_location,
+                on_bad_lines="skip",
+                skiprows=skip,  # skip metadata
+                skipfooter=1,  # last row contains "grand total"
+                engine="python",  # otherwise warning that skipfooter arg does not exist
+            )
+            results = results.to_dict(orient="records")
+
+            # get rid of the tmp file
+            os.remove(result_temp_file_location)
+            return results
+
+        else:
+            raise CampaignManagerServiceException(response.content)
+
+    def delete_report(self, account_id: str, report_id: str, secret_id: str) -> bool:
+        """
+        Method to delete report
+        :param account_id: The ID of the account to check for the report
+        :param report_id: The ID of the report
+        :param secret_id: ID of the secret in Secret Manager to be used to access Campaign Manager 360
+        :return Boolean
+        """
+        body = {
+            "account_id": account_id,
+            "report_id": report_id,
+            "secret_id": secret_id,
+        }
+        header = get_headers(self._DELETE_REPORT)
+        response = request("post", url=self._DELETE_REPORT, json=body, headers=header)
+
+        if response.status_code == HTTPStatus.OK:
+            return True
+        else:
+            raise CampaignManagerServiceException(response.content)
+
+    def query_dimension_values(
+        self,
+        account_id: str,
+        secret_id: str,
+        dimension_name: str,
+        filters: List,
+        start_date: str,
+        end_date: int,
+        max_results: int = 100,
+    ) -> List[Dict]:
+        """
+        Retrieves values of selected dimensions for data filtered according to rules set in list of filters
+        :param account_id: Account ID
+        :param dimension_name: The name of the dimension for which values should be requested
+        :param secret_id: The ID of the secret in secret manager
+        :param filters: The list of filters by which to filter values
+        :param start_date: The start date of the date range for which to retrieve dimension values
+        :param end_date: The end date of the date range for which to retrieve dimension values
+        :param max_results: Maximum number of results to return
+
+
+        return {"next_page_token": token, items: values of selected dimensions}
+        """
+        body = {
+            "account_id": account_id,
+            "dimension_name": dimension_name,
+            "secret_id": secret_id,
+            "filters": filters,
+            "start_date": start_date,
+            "end_date": end_date,
+            "page_token": None,
+            "max_results": max_results,
+        }
+
+        header = get_headers(self._QUERY_DIMENSION_VALUES)
+        response = request(
+            "post", url=self._QUERY_DIMENSION_VALUES, json=body, headers=header
+        )
+
+        if response.status_code != HTTPStatus.OK:
+            raise CampaignManagerServiceException(response.content)
+
+        service_response = response.json()
+        yield service_response["data"]["items"]
+
+        # if there was a last page response is empty string
+        while service_response["data"]["nextPageToken"]:
+            body["page_token"] = service_response["data"]["nextPageToken"]
+            response = request(
+                "post", url=self._QUERY_DIMENSION_VALUES, json=body, headers=header
+            )
+
+            if response.status_code != HTTPStatus.OK:
+                raise CampaignManagerServiceException(response.content)
+
+            service_response = response.json()
+            yield service_response["data"]["items"]