digital-analytics-toolkit 0.1.0__py3-none-any.whl

digital_analytics_toolkit/__init__.py

@@ -0,0 +1,81 @@
+from . import (
+    bq_functions,
+    free_form_report_GA4,
+    funnel_report_GA4,
+    ga4_admin_functions,
+    ga4_audience,
+    gtm_functions,
+    utils,
+)
+from .bq_functions import (
+    get_from_bigquery,
+    get_ga4_data_from_bq,
+    list_bq_datasets,
+)
+from .free_form_report_GA4 import dimension_filter_generation, free_form_report_GA4
+from .funnel_report_GA4 import Funnel_Reports
+from .ga4_admin_functions import (
+    get_ga4_accounts_list,
+    get_ga4_properties_list,
+    get_ga4_property_bq_link,
+    get_ga4_property_keyEvents,
+)
+from .ga4_audience import (
+    GA4AudienceClient,
+    GA4AudienceError,
+    build_admin_client,
+    create_audience,
+)
+from .gtm_functions import (
+    get_accounts,
+    get_built_in_variables,
+    get_containers,
+    get_custom_templates,
+    get_tags,
+    get_triggers,
+    get_variables,
+    get_workspaces,
+    getLatestVersionHeader,
+)
+from .utils import get_dimension_metric
+
+__version__ = "0.1.0"
+
+__all__ = [
+    # Submodules
+    "bq_functions",
+    "ga4_admin_functions",
+    "ga4_audience",
+    "gtm_functions",
+    "utils",
+    # Reporting
+    "Funnel_Reports",
+    "free_form_report_GA4",
+    "dimension_filter_generation",
+    # GA4 Admin
+    "get_ga4_accounts_list",
+    "get_ga4_properties_list",
+    "get_ga4_property_keyEvents",
+    "get_ga4_property_bq_link",
+    # Audiences
+    "GA4AudienceClient",
+    "create_audience",
+    "build_admin_client",
+    "GA4AudienceError",
+    # GTM
+    "get_accounts",
+    "get_containers",
+    "get_workspaces",
+    "getLatestVersionHeader",
+    "get_tags",
+    "get_variables",
+    "get_triggers",
+    "get_built_in_variables",
+    "get_custom_templates",
+    # BigQuery
+    "list_bq_datasets",
+    "get_from_bigquery",
+    "get_ga4_data_from_bq",
+    # Utilities
+    "get_dimension_metric",
+]

digital_analytics_toolkit/bq_functions.py

@@ -0,0 +1,97 @@
+import pandas as pd
+from google.cloud import bigquery
+from typing import Optional
+from google.oauth2 import service_account
+
+
+def list_bq_datasets(credentials, gcp_project_id):
+    """
+    List all BigQuery datasets in a given Google Cloud project.
+
+    Args:
+        credentials: Authenticated Google credentials object used to initialize
+            the BigQuery client.
+        gcp_project_id (str): The Google Cloud project ID.
+    """
+    client = bigquery.Client(project=gcp_project_id, credentials=credentials)
+    dataset_objects = client.list_datasets(project=str(gcp_project_id))
+
+    # Extract just the string IDs from the objects
+    datasets = [dataset.dataset_id for dataset in dataset_objects]
+
+    return datasets
+
+
+def get_from_bigquery(project_id:str,dataset_id:str,table_id:str,service_account_file:Optional[str] = None, OAuth_creds:Optional[str] = None) -> pd.DataFrame:
+    '''
+     Args:
+    --------
+    project_id: str
+        Google Cloud Project ID where the BigQuery table is located.
+    
+    dataset_id: str
+        BigQuery dataset ID where the table is located.
+    
+    table_id: str
+        BigQuery table ID from where the data needs to be extracted.
+    service_account_file: str 
+        API service account JSON key file.
+    
+    --------
+    Returns:
+    pd.DataFrame
+    '''
+    if service_account_file != None:
+        credentials = service_account.Credentials.from_service_account_file(
+            service_account_file,
+            scopes=["https://www.googleapis.com/auth/cloud-platform"]
+        )
+    elif OAuth_creds != None:
+        credentials = OAuth_creds
+    client = bigquery.Client(credentials=credentials, project=project_id)
+    query = f"SELECT * FROM `{project_id}.{dataset_id}.{table_id}`"
+    query_job = client.query(query)
+    results = query_job.result().to_dataframe()
+    return results
+    
+def get_ga4_data_from_bq(project_id: str,query: str, service_account_file:Optional[str] = None, OAuth_creds:Optional[str] = None) -> pd.DataFrame:
+    '''
+    Args:
+    --------
+    project_id: str
+        Google Cloud Project ID where the BigQuery table is located.
+    query: str
+        SQL query to execute.
+    service_account: str
+        Path to service account json file
+    OAuth_creds:
+        OAuth creds
+    --------
+    Returns:
+    pd.DataFrame
+    '''
+    if service_account_file != None:
+        credentials = service_account.Credentials.from_service_account_file(
+            service_account_file,
+            scopes=["https://www.googleapis.com/auth/cloud-platform"]
+        )
+    elif OAuth_creds != None:
+        credentials = OAuth_creds
+    client = bigquery.Client(credentials=credentials, project=project_id)
+    query_job = client.query(query)
+    results = query_job.result().to_dataframe()
+    return results
+
+# def create_bq_table(project_id:str,dataset_id:str,table_id:str,schema_dict:dict,service_account_file:Optional[str] = None, OAuth_creds:Optional[str] = None) -> pd.DataFrame:
+#     table_path = f"{project_id}.{dataset_id}.{table_id}"
+#     schema = []
+#     for key, value in schema_dict.items():
+#         if type(value) == dict:
+#             temp_list = []
+#             for nested_key, nested_value in value.items():
+#                 temp_list.append(bigquery.SchemaField(nested_key,nested_value))
+#             schema.append(bigquery.SchemaField(key, "RECORD",fields=temp_list))
+#         else:
+#             schema.append(bigquery.SchemaField(key, value))
+#     table = bigquery.Table(table_path, schema=schema)
+# def write_to_bq(project_id:str,dataset_id:str,table_id:str,service_account_file:Optional[str] = None, OAuth_creds:Optional[str] = None) -> pd.DataFrame:

digital_analytics_toolkit/free_form_report_GA4.py

@@ -0,0 +1,155 @@
+from google.oauth2 import service_account
+from google.analytics.data_v1beta import BetaAnalyticsDataClient
+import pandas as pd
+from datetime import datetime
+from typing import Optional, Union
+
+class free_form_report_GA4():
+    '''
+    Generate Free Form GA4 reports.
+
+    Args:
+        service_account_file (str): API service account JSON key file.
+        property_id (str): Property ID of the Google Analytics Property from which report needs to be generated.
+    '''
+    def __init__(self, service_account_file: str, property_id: str):
+        self.service_account_file = service_account_file
+        self.property_id = property_id
+        self.credentials = service_account.Credentials.from_service_account_file(self.service_account_file)
+        self.client = BetaAnalyticsDataClient(credentials=self.credentials)
+        self.property_id = f'properties/{property_id}'
+
+    def free_form_report(self, start_date, end_date, metrics: list, dimensions:Optional[list] = None, dimension_filter:Optional[str] = None, metric_filter:Optional[str] = None):
+        '''
+        Collects date ranges, metrics and dimensions to generate report.
+        
+        Args:
+            start_date (str):
+                Report date range starting date in "YYYY-MM-DD" format or "YYYY-MM-DD HH:MM:SS" format or GA4 string format like "7daysAgo", "today" etc.
+            end_date (str):
+                Report date range ending date in "YYYY-MM-DD" format or "YYYY-MM-DD HH:MM:SS" format or GA4 string format like "7daysAgo", "today" etc.
+            metrics (list):
+                List of metrics required in the report pass only the api names.
+                example: ['totalUsers', 'eventCount']
+            dimensions (list, Optional):
+                List of dimensional breakdown required in the report pass only the api names.
+                example: ['browser','sessionDefaultChannelGroup']
+            dimension_filter (str, Optional):
+                Dimension Filter condition required in the report in JSON format
+            metric_filter (str, Optional):
+                Metrics Filter condition required in the report in JSON format
+            
+        Returns:
+            result_df (DataFrame): Report generated in pandas DataFrame format.
+        '''
+        self.dimensions = []
+        self.metrics = []
+        if "-" in str(start_date):
+            self.start_date = str(start_date).split(" ")[0]
+        else:
+            self.start_date = str(start_date)
+        if "-" in str(end_date):
+            self.end_date = str(end_date).split(" ")[0]
+        else:
+            self.end_date = str(end_date)
+        try:
+            for dim in dimensions:
+                self.dimensions.append({'name' : dim})
+        except TypeError as e:
+            print("dimensions received is not in list format")
+        try:
+            for met in metrics:
+                self.metrics.append({'name' : met})
+        except TypeError as e:
+            print("metrics received is not in list format")
+        self.request = {
+            "property": self.property_id,
+            "date_ranges" : [
+                {
+                    "start_date" : self.start_date,
+                    "end_date" : self.end_date,
+                }
+            ],
+            "dimensions" : self.dimensions,
+            "metrics" : self.metrics,
+            "dimension_filter" : dimension_filter,
+            "metric_filter" : metric_filter,
+            "limit" :  "250000"
+        }
+
+        try:
+            self.response = self.client.run_report(request=self.request)
+        except Exception as e:
+            print(e)
+            return None
+        self.data = []
+        for row in self.response.rows:
+            dimension_values = [value.value for value in row.dimension_values]
+            metric_values = [float(value.value) for value in row.metric_values]
+            self.data.append(dimension_values + metric_values)
+        self.columns = [dimension.name for dimension in self.response.dimension_headers] + [metric.name for metric in self.response.metric_headers]
+        self.result_df = pd.DataFrame(data=self.data, columns=self.columns)
+
+        return self.result_df
+
+def dimension_filter_generation(operator: str, value: Union[str, list], dimension_name: Optional[str] = None):
+    '''
+    Generates dimension filter in JSON format to be used in report generation.
+
+    Args:
+        operator (str): Operator for the filter condition. available options: "EXACT", "BEGINS_WITH", "ENDS_WITH", "CONTAINS", "FULL_REGEXP", "AND", "OR", "NOT", "IN_LIST".
+        value (Union[str, list]): Value for the filter condition. For operators "AND", "OR" and "NOT", pass the value in list format with each condition as a separate item in the list.
+        dimension_name (str): Name of the dimension for which filter needs to be generated. Required only when operator is not "AND", "OR" or "NOT".
+
+    Returns:
+        dimension_filter (str): Generated dimension filter in JSON format.
+    '''
+    secondary_opeartor = {
+        "AND" : "and_group",
+        "OR" : "or_group",
+        "NOT" : "not_expression"
+    }
+    if operator in ["AND", "OR"]:
+        # filter_expression = value.split(",")
+        # filter_expression = [item.strip() for item in filter_expression]
+        filter_expression = value
+        dimension_filter = {
+            secondary_opeartor[operator] : {
+                "expressions" : filter_expression
+            }
+        }
+    elif operator in ["NOT"]:
+        # filter_expression = value.split(",")
+        # filter_expression = [item.strip() for item in filter_expression]
+        filter_expression = value
+        dimension_filter = {
+            secondary_opeartor[operator] : filter_expression
+        }    
+    elif operator == "IN_LIST":
+        dimension_filter = {
+            "filter": {
+                "field_name": dimension_name,
+                "in_list_filter": {
+                    "values": value
+                }
+            }
+        }
+    else:
+        dimension_filter = {
+            "filter": {
+                "field_name": dimension_name,
+                "string_filter": {
+                    "match_type": operator,
+                    "value": value
+                }
+            }
+        }
+    return dimension_filter
+    # def anomaly_detection(self, report, comparison_report, report_date_range, comparison_date_range, dimensions_list):
+    #     self.data = report
+    #     self.comp_data = comparison_report
+    #     self.dimensions_list = dimensions_list
+    #     self.data_start_date, self.data_end_date = [str(date).split(" ")[0] for date in report_date_range]
+    #     self.comp_start_date, self.comp_end_date = [str(date).split(" ")[0] for date in comparison_date_range]
+
+    #     merged_df = pd.merge(self.data, self.comp_data, on=self.dimensions_list, how="outer")

digital_analytics_toolkit/funnel_report_GA4.py

@@ -0,0 +1,269 @@
+import gspread
+from google.oauth2 import service_account
+from google.analytics.data_v1alpha import AlphaAnalyticsDataClient
+from datetime import datetime
+from typing import Optional
+from google.analytics.data_v1alpha.types import (
+    DateRange,
+    Dimension,
+    Funnel,
+    FunnelBreakdown,
+    FunnelEventFilter,
+    FunnelFieldFilter,
+    FunnelFilterExpression,
+    FunnelFilterExpressionList,
+    FunnelStep,
+    RunFunnelReportRequest,
+    StringFilter,
+    Filter,
+    FilterExpressionList,
+    FilterExpression,
+    QuotaStatus,
+    InListFilter,
+)
+import pandas as pd
+
+class Funnel_Reports():
+
+    '''
+    
+    Initializing object to run funnel reports.
+    
+    Parameters:
+    -----------
+    reporting_client : str
+        Service Account JSON key for Google Analytics Data API
+    prop_id : str
+        Property ID of the Google Analytics Property from which report needs to be generated.
+    
+    Funnel functions covered:
+    -----------
+    1. OR Group
+    2. AND Group
+    3. Contains
+    4. Event Name
+    5. Equals To
+    6. Is One of
+    7. Begins with
+    8. Ends with
+    9. Matched Regex
+    10. Does not in combition with all the other mentioned functions
+    '''
+    
+    def __init__(self,reporting_client: str,prop_id: str):
+        self.property_id = prop_id
+        self.SCOPES = [
+            'https://www.googleapis.com/auth/analytics.readonly',
+            'https://www.googleapis.com/auth/spreadsheets',
+            'https://www.googleapis.com/auth/drive'
+        ]
+        # self.file_n = file_n
+        self.credentials = service_account.Credentials.from_service_account_file(reporting_client, scopes=self.SCOPES)
+        self.client = AlphaAnalyticsDataClient(credentials = self.credentials)
+
+
+    def condition_creator(self,condition_string):
+        if ' or ' in condition_string.lower():
+            conditions_li = condition_string.lower().split(' or ')
+            expressions = []
+            funnel_filter_expression = FunnelFilterExpression(
+                or_group = FunnelFilterExpressionList(
+                    expressions = [self.condition_creator(condition) for condition in conditions_li]
+                )
+            )
+        elif ' and ' in condition_string.lower():
+            conditions_li = condition_string.split(' and ')
+            expressions = []
+            funnel_filter_expression = FunnelFilterExpression(
+                and_group = FunnelFilterExpressionList(
+                    expressions = [self.condition_creator(condition) for condition in conditions_li]
+                )
+            )
+        elif ' does not ' in condition_string.lower():
+            new_condition = condition_string.split(' does not ')[0]+ ' '+condition_string.split(' does not ')[1]
+            funnel_filter_expression = FunnelFilterExpression(
+                not_expression = FunnelFilterExpression(
+                    self.condition_creator(new_condition)
+                )
+            )
+        else:
+            if '= ' in condition_string and 'eventName' in condition_string:
+                event_name = condition_string.split('"')[1]
+                funnel_event_filter = FunnelEventFilter(event_name=event_name)
+                funnel_filter_expression = FunnelFilterExpression(funnel_event_filter=funnel_event_filter)
+            elif 'contains' in condition_string:
+                field_name = condition_string.split(' contains ')[0]
+                event_name = condition_string.split('"')[1]
+                string_filter=StringFilter(match_type=StringFilter.MatchType.CONTAINS,case_sensitive=False,value=event_name)
+                funnel_field_filter = FunnelFieldFilter(field_name= field_name, string_filter=string_filter)
+                funnel_filter_expression = FunnelFilterExpression(funnel_field_filter=funnel_field_filter)
+
+            elif 'is one of' in condition_string:
+                field_name, event_name = condition_string.split(' is one of ')[0], condition_string.split(' is one of ')[1]
+                test_str = event_name.replace('"','').replace('[','').replace(']','').replace('\n','')
+                new_list = test_str.split(', ')
+                # print(event_name)
+                in_list_filter=InListFilter(values=new_list)
+
+                funnel_field_filter = FunnelFieldFilter(field_name= field_name, in_list_filter=in_list_filter)
+                funnel_filter_expression = FunnelFilterExpression(funnel_field_filter=funnel_field_filter)
+
+            elif '=' in condition_string:
+                field_name = condition_string.split(' = ')[0]
+                event_name = condition_string.split('"')[1]
+                string_filter=StringFilter(match_type=StringFilter.MatchType.EXACT,case_sensitive=False,value=event_name)
+                funnel_field_filter = FunnelFieldFilter(field_name= field_name, string_filter=string_filter)
+                funnel_filter_expression = FunnelFilterExpression(funnel_field_filter=funnel_field_filter)
+            elif 'begins with' in condition_string:
+                field_name = condition_string.split(' begins with ')[0]
+                event_name = condition_string.split('"')[1]
+                string_filter=StringFilter(match_type=StringFilter.MatchType.BEGINS_WITH,case_sensitive=False,value=event_name)
+                funnel_field_filter = FunnelFieldFilter(field_name= field_name, string_filter=string_filter)
+                funnel_filter_expression = FunnelFilterExpression(funnel_field_filter=funnel_field_filter)
+            elif 'ends with' in condition_string:
+                field_name = condition_string.split(' ends with ')[0]
+                event_name = condition_string.split('"')[1]
+                string_filter=StringFilter(match_type=StringFilter.MatchType.ENDS_WITH,case_sensitive=False,value=event_name)
+                funnel_field_filter = FunnelFieldFilter(field_name= field_name, string_filter=string_filter)
+                funnel_filter_expression = FunnelFilterExpression(funnel_field_filter=funnel_field_filter)
+            elif 'matches regex' in condition_string:
+                field_name = condition_string.split(' matches regex ')[0]
+                field_name = field_name.strip()
+                event_name = condition_string.split('"')[1]
+                string_filter=StringFilter(match_type=StringFilter.MatchType.FULL_REGEXP,case_sensitive=False,value=event_name)
+                funnel_field_filter = FunnelFieldFilter(field_name= field_name, string_filter=string_filter)
+                funnel_filter_expression = FunnelFilterExpression(funnel_field_filter=funnel_field_filter)
+        return funnel_filter_expression
+
+    def generate_dimension_filter(self, dim_filter):
+        '''
+        Pass filter in string format
+        
+        Example: 
+        ---
+        1. sessionDefaultChannelGroup contains "Organic"
+        2. sessionDefaultChannelGroup = "Direct"
+        '''
+        if 'contains' in dim_filter:
+            filter_field_name = dim_filter.split(' contains ')[0]
+            filter_event_name = dim_filter.split('"')[1]
+            str_filter = StringFilter(match_type=StringFilter.MatchType.CONTAINS, case_sensitive=False, value=filter_event_name)
+            filter_condition = Filter(field_name=filter_field_name, string_filter=str_filter)
+            dimension_filter = FilterExpression(filter=filter_condition)
+        elif ' = ' in dim_filter:
+            filter_field_name = dim_filter.split(' contains ')[0]
+            filter_event_name = dim_filter.split('"')[1]
+            str_filter = StringFilter(match_type=StringFilter.MatchType.EXACT, value=filter_event_name)
+            filter_condition = Filter(field_name=filter_field_name, string_filter=str_filter)
+            dimension_filter = FilterExpression(filter=filter_condition)
+        return dimension_filter
+
+
+    def get_funnel_steps(self):
+        self.steps = len(self.funnel_data.keys())
+        funnel_steps = []
+        for i in range(1,self.steps):
+            if self.funnel_data.iloc[0,i] == 'Single Condition':
+                fs = FunnelStep(
+                    name = self.funnel_data.keys()[i],
+                    filter_expression = FunnelFilterExpression(self.condition_creator(self.funnel_data.iloc[1,i]))
+                )
+                funnel_steps.append(fs)
+            elif self.funnel_data.iloc[0,i] == 'AND Group':
+                fs = FunnelStep(
+                    name = self.funnel_data.keys()[i],
+                    filter_expression = FunnelFilterExpression(
+                        and_group = FunnelFilterExpressionList(
+                            expressions = [self.condition_creator(self.funnel_data.iloc[j,i]) for j in range(1,len(self.funnel_data))]
+                        )
+                    )
+                )
+                funnel_steps.append(fs)
+            elif self.funnel_data.iloc[0,i] == 'OR Group':
+                fs = FunnelStep(
+                    name = self.funnel_data.keys()[i],
+                    filter_expression = FunnelFilterExpression(
+                        or_group = FunnelFilterExpressionList(
+                            expressions = [self.condition_creator(self.funnel_data.iloc[j,i]) for j in range(1,len(self.funnel_data))]
+                        )
+                    )
+                )
+                funnel_steps.append(fs)
+        return funnel_steps
+
+    def funnel_report(self,startDate: datetime,endDate: datetime,file_n: str,dimension_filter:Optional[str] = None, breakdown:Optional[str] = None,breakdown_limit:Optional[str] = None):
+        '''
+        Generate funnel report between two dates.
+
+        Parameters
+        ----------
+        startDate : datetime
+            Start date of the report.
+        endDate : datetime
+            End date of the report.
+        file_n : str
+            Google Sheets File containing Steps of the funnel.
+        dimension_filter : str, optional
+            Filter condition for the report (must be a string if provided).
+        breakdown : str, optional
+            Breakdown dimension to view funnels across dimension.
+        breakdown_limit : str, optional
+            Available limit 1 - 15.
+            Need to specify the number of breakdown rows to be displayed per step.
+            
+            Default limit = 15
+        '''
+        self.SCOPES = [
+            'https://www.googleapis.com/auth/analytics.readonly',
+            'https://www.googleapis.com/auth/spreadsheets',
+            'https://www.googleapis.com/auth/drive'
+        ]
+        self.breakdown = breakdown
+        if breakdown_limit == None:
+            self.breakdown_limit = '15'
+        else:
+            self.breakdown_limit = breakdown_limit
+        self.dimension_filter = dimension_filter
+        self.file_n = file_n
+        self.gc = gspread.authorize(self.credentials)
+        self.worksheet = self.gc.open(f'{self.file_n}').sheet1
+        self.rows = self.worksheet.get_all_values()
+        self.funnel_data = pd.DataFrame(self.rows[1:], columns=self.rows[0])
+        if dimension_filter == None:
+            if self.breakdown == None:
+                request = RunFunnelReportRequest(
+                    property=f"properties/{self.property_id}",
+                    date_ranges=[DateRange(start_date=str(startDate).split(" ")[0], end_date=str(endDate).split(" ")[0])],
+                    funnel=Funnel(steps = self.get_funnel_steps()),
+                )
+            else:
+                request = RunFunnelReportRequest(
+                    property=f"properties/{self.property_id}",
+                    date_ranges=[DateRange(start_date=str(startDate).split(" ")[0], end_date=str(endDate).split(" ")[0])],
+                    funnel_breakdown = FunnelBreakdown(
+                        breakdown_dimension = Dimension(name=self.breakdown),
+                        limit = self.breakdown_limit
+                    ),
+                    funnel=Funnel(steps = self.get_funnel_steps()),
+                )
+        else:
+            if self.breakdown == None:
+                request = RunFunnelReportRequest(
+                    property=f"properties/{self.property_id}",
+                    date_ranges=[DateRange(start_date=str(startDate).split(" ")[0], end_date=str(endDate).split(" ")[0])],
+                    funnel=Funnel(steps = self.get_funnel_steps()),
+                    dimension_filter = self.generate_dimension_filter(self.dimension_filter)
+                )
+            else:
+                request = RunFunnelReportRequest(
+                    property=f"properties/{self.property_id}",
+                    date_ranges=[DateRange(start_date=str(startDate).split(" ")[0], end_date=str(endDate).split(" ")[0])],
+                    funnel_breakdown = FunnelBreakdown(
+                        breakdown_dimension = Dimension(name=self.breakdown),
+                        limit = self.breakdown_limit
+                    ),
+                    funnel=Funnel(steps = self.get_funnel_steps()),
+                    dimension_filter = self.generate_dimension_filter(self.dimension_filter)
+                )
+        response = self.client.run_funnel_report(request)
+        return response