fullscript.spellbook 2.1__py3-none-any.whl

spellbook/drive_magic.py

@@ -0,0 +1,266 @@
+'''
+What this script does:
+
+https://www.thepythoncode.com/article/using-google-drive--api-in-python
+https://developers.google.com/drive/api/v3/reference/files
+'''
+
+# importing packages
+import os
+import io
+import pandas as pd
+import base64
+import pickle
+from googleapiclient.discovery import build
+from google_auth_oauthlib.flow import InstalledAppFlow
+from google.auth.transport.requests import Request
+from googleapiclient.http import MediaFileUpload
+from googleapiclient.http import MediaIoBaseDownload
+from googleapiclient.errors import HttpError
+from google.oauth2 import service_account
+from spellbook.utils import load_and_parse_config
+
+'''
+Authentication Process
+'''
+
+
+def get_service(scopes, service_account_name, auth_type='service_account', **kwargs):
+    """Gets Google Drive service
+    Parameters
+    ----------
+    scopes : list
+      List of strings representing the scopes one needs service for.
+      Example: ['https://www.googleapis.com/auth/drive.metadata.readonly',
+                'https://www.googleapis.com/auth/drive.file']
+    auth_type : str
+      Authorization type selected. It can be 'service_account' or 'oauth'
+
+    Returns
+    -------
+    resource
+      Google Drive service object
+      :param scopes: The scopes needed for the service
+      :param service_account_name: The name of the Google Sheets service account to be used for authorization
+  """
+    try:
+        if auth_type == 'service_account':
+            # option that uses the credential process we already have for sheets_magic
+            # this method requires a folder to be shared with the service account email (as we do with our sheets)
+
+            # Load the configuration
+            config = load_and_parse_config(configurations='google_accounts', **kwargs)
+            gs_service_account = None
+
+            # Find the Google Sheets service account details in the configuration
+            google_accounts = config.get('google_accounts', [])
+            for gs_account in google_accounts:
+                if gs_account.get('name') == service_account_name:
+                    gs_service_account = gs_account.get('service_account')
+                    break
+
+            if not gs_service_account:
+                raise ValueError(
+                    f"Google Sheets service account '{service_account_name}' not found in configuration file.")
+
+            # creates a credentials file using the info in the env var GS_SERVICE_ACCOUNT
+            with open('credentials.json', 'w') as credential_file:
+                print(base64.b64decode(os.environ['GS_SERVICE_ACCOUNT']).decode(), file=credential_file)
+
+            # uses the file to create the scoped credentials
+            credentials = service_account.Credentials.from_service_account_file('credentials.json')
+            scoped_credentials = credentials.with_scopes(scopes)
+            os.remove('credentials.json')
+
+        elif auth_type == 'oauth':
+            # This option will prompt the user to authorize the script to use their identity. This allows the script to read
+            # and write to any folder the user has access to.
+            # The file token.pickle stores the user's access and refresh tokens. It's automatically created at the first time
+            # the user completes the authorization.
+            # It's important to notice everytime a new scope is needed, the authorization needs to be run again.
+
+            # initializing scoped_credentials
+            scoped_credentials = None
+
+            # if the token.pickle already exists, we can get credentials from it
+            if os.path.exists('token.pickle'):
+                with open('token.pickle', 'rb') as token:
+                    scoped_credentials = pickle.load(token)
+
+            # If the file doesn't exist or is expired, we need to run the authorization flow
+            if not scoped_credentials or not scoped_credentials.valid:
+
+                if scoped_credentials and scoped_credentials.expired and scoped_credentials.refresh_token:
+                    # refresh the credentials
+                    scoped_credentials.refresh(Request())
+                else:
+                    # run the authorization flow from the client secrets stored as a env var
+                    with open('client_secrets.json', 'w') as credential_file:
+                        print(base64.b64decode(os.environ['GS_CLIENT_SECRETS']).decode(), file=credential_file)
+
+                    flow = InstalledAppFlow.from_client_secrets_file('client_secrets.json', scopes)
+                    scoped_credentials = flow.run_local_server(port=3000)
+
+                    os.remove('client_secrets.json')
+
+                # save the credentials for the next run
+                with open('token.pickle', 'wb') as token:
+                    pickle.dump(scoped_credentials, token)
+
+        # return the built service object.
+        return build('drive', 'v3', credentials=scoped_credentials)
+    except HttpError as error:
+        print(f'An error occurred: {error}')
+        return None
+
+
+'''
+Other functions
+'''
+
+
+def list_folder_contents(service, folder_url, max_results=1000, show_trashed=False):
+    """Gets files in a specific folder
+    Parameters
+    ----------
+    service : resource
+      Google Drive resource object
+    folder_url : str
+      Folder URL
+    max_results : int
+      Number of results to be returned
+    show_trashed : bool
+      Flag whether files in the trash should be returned
+
+    Returns
+    -------
+    pd.DataFrame
+      Dataframe with file data
+  """
+
+    # get the folder id
+    folder_id = folder_url.rsplit('/', 1)[-1]
+
+    try:
+        # get items from GDrive
+        results = service.files().list(q=f"'{folder_id}' in parents and trashed = {str(show_trashed).lower()}",
+                                       pageSize=max_results, orderBy='name',
+                                       includeItemsFromAllDrives=True, supportsAllDrives=True,
+                                       fields="nextPageToken, files(id, name, mimeType, size, parents, modifiedTime)").execute()
+        items = results.get('files', [])
+
+        # returns as a DF
+        df_temp = pd.DataFrame(items)
+        df_temp = df_temp.rename({'size': 'size_bytes'}, axis=1)
+        return df_temp
+    except HttpError as error:
+        print(f'An error occurred: {error}')
+        return None
+
+
+def download_file(service, file_id, filepath):
+    """Downloads a file from Google Drive
+    Parameters
+    ----------
+    service : resource
+      Google Drive resource object
+    file_id : str
+      ID of the file to be downloaded
+    filepath : str
+      File path where the downloaded file should be saved. (i.e.: 'samples/test_file.txt')
+
+    Returns
+    -------
+    bool
+      Flag whether the file was successfully downloaded or not
+  """
+
+    try:
+        # pylint: disable=maybe-no-member
+        request = service.files().get_media(fileId=file_id, supportsAllDrives=True)
+        file = io.BytesIO()
+        downloader = MediaIoBaseDownload(file, request)
+        done = False
+        while done is False:
+            status, done = downloader.next_chunk()
+            print(F'Download {int(status.progress() * 100)}.')
+
+        with open(filepath, "wb") as f:
+            f.write(file.getbuffer())
+
+        return True
+
+    except HttpError as error:
+        print(F'An error occurred: {error}')
+        return False
+
+
+def upload_file(service, folder_url, filepath):
+    """Gets files in a specific folder
+    Parameters
+    ----------
+    service : resource
+      Google Drive resource object
+    folder_url : str
+      Folder URL
+    filepath : str
+      Path of the file to be uploaded
+
+    Returns
+    -------
+    str
+      ID of the created file
+  """
+
+    # get the folder id
+    folder_id = folder_url.rsplit('/', 1)[-1]
+
+    # first, define file metadata, such as the name and the parent folder ID
+    filename = os.path.basename(filepath)
+    file_metadata = {
+        "name": filename,
+        "parents": [folder_id]
+    }
+    # upload
+    try:
+        media = MediaFileUpload(filepath, resumable=True)
+        file = service.files().create(body=file_metadata, media_body=media, fields='id',
+                                      supportsAllDrives=True).execute()
+        print(f'File created! - id: {file.get("id")}')
+        return file.get("id")
+    except HttpError as error:
+        print(f'An error occurred: {error}')
+        return None
+
+
+def delete_file(service, file_id):
+    """Permanently deletes a file owned by the user without moving it to the trash. If the file belongs to a shared drive
+  the user must be an organizer on the parent. If the target is a folder, all descendants owned by the user are also
+  deleted. In case the delete method is unsuccessful, tries to move the file to trash.
+    Parameters
+    ----------
+    service : resource
+      Google Drive resource object
+    file_id : str
+      ID of the file to be deleted
+
+    Returns
+    -------
+    bool
+      Flag whether the file was successfuly deleted or not
+  """
+    try:
+        try:
+            # delete method
+            service.files().delete(fileId=file_id, supportsAllDrives=True).execute()
+            print('File deleted successfully!')
+            return True
+        except:
+            # move to trash method
+            body = {'trashed': True}
+            service.files().update(fileId=file_id, body=body, supportsAllDrives=True).execute()
+            print('Not able to delete the file. Moved to trash instead.')
+            return True
+    except HttpError as error:
+        print(f'An error occurred: {error}')
+        return False

spellbook/sheet_magic.py

@@ -0,0 +1,181 @@
+import os
+import pandas as pd
+import pygsheets
+import base64
+from spellbook.utils import load_and_parse_config
+
+
+class GSheetsHandler:
+    """
+    :param gsheet_url: The URL of the target Google Sheet workbook
+    :param service_account_name: The name of the Google Sheets service account to be used for authorization
+    """
+
+    def __init__(self, gsheet_url, service_account_name):
+        self._gsheet_url = gsheet_url
+        self._service_account_name = service_account_name
+        if os.path.isfile('spellbook_config.yaml'): 
+            self._config_file = 'spellbook_config.yaml'
+        else: 
+            self._config_file = os.path.expanduser('~')+'/'+'spellbook_config.yaml'
+        self._df_results = pd.DataFrame()
+        self._gc = None
+        self._sh = None
+
+        # Initialize Google Sheets authorization
+        self._authorize_google_sheets()
+
+        # Open the Google Spreadsheet using the URL
+        if self._gc:
+            self._sh = self._gc.open_by_url(self._gsheet_url)
+
+    def _authorize_google_sheets(self):
+        """
+        Private method to handle Google Sheets authorization using credentials from the configuration file.
+        Loads the service account credentials, decodes them, and uses them for authorization.
+        """
+        try:
+            # Load the configuration
+            config = load_and_parse_config(config_file_path=self._config_file, configurations='google_accounts')
+            gs_service_account = None
+
+            # Find the Google Sheets service account details in the configuration
+            google_accounts = config.get('google_accounts', [])
+            for gs_account in google_accounts:
+                if gs_account.get('name') == self._service_account_name:
+                    gs_service_account = gs_account.get('service_account')
+                    break
+
+            if not gs_service_account:
+                raise ValueError(f"Google Sheets service account '{self._service_account_name}' not found in configuration file.")
+
+            # Decode and write credentials from configuration to a temporary file
+            with open("credentials.json", "w") as credential_file:
+                print(base64.b64decode(gs_service_account).decode(), file=credential_file)
+
+            # Use the JSON file to authenticate on Google Sheets
+            self._gc = pygsheets.authorize(service_file="credentials.json")
+            os.remove("credentials.json")
+        except FileNotFoundError:
+            print("Config file not found. Please ensure 'spelbook_config.yaml' exists.")
+        except ValueError as e:
+            print(f'Error: {e}')
+        except Exception as e:
+            print(f'Could not load Google Sheets service account. Error: {e}')
+
+
+    def delete_sheet(self, worksheet):
+        """
+        Deletes the specified worksheet from the Google Sheet.
+        If the worksheet does not exist, an error message is printed.
+        :param worksheet: The name or index of the worksheet to be deleted
+        """
+        try:
+            if str(worksheet).isdigit():
+                # Delete sheet by index
+                self._sh.del_worksheet(self._sh[int(worksheet)])
+            else:
+                # Delete sheet by title
+                self._sh.del_worksheet(self._sh.worksheet_by_title(worksheet))
+            print('Tab deleted')
+        except pygsheets.WorksheetNotFound as error:
+            print(f'Error while deleting sheet: {error}')
+
+    def create_sheet(self, worksheet):
+        """
+        Creates a new worksheet in the Google Sheet with the specified name.
+        If a worksheet with the given name already exists, it will not create a duplicate.
+        :param worksheet: The name of the new worksheet to create
+        """
+        try:
+            self._sh.add_worksheet(str(worksheet))
+        except Exception as error:
+            print(f'Error while creating sheet: {error}')
+
+    def get_sheet(self, worksheet):
+        """
+        Retrieves the specified worksheet object from the Google Sheet.
+        The worksheet can be accessed either by its name or its index.
+        If the worksheet does not exist, an error message is printed.
+        :param worksheet: The name or index of the worksheet to retrieve
+        :return: The worksheet object or None if not found
+        """
+        try:
+            if str(worksheet).isdigit():
+                return self._sh[int(worksheet)]
+            else:
+                return self._sh.worksheet_by_title(worksheet)
+        except pygsheets.WorksheetNotFound:
+            print(f'Sheet does not exist')
+            return None
+
+    def count_rows(self, worksheet, target_column=1, include_tailing_empty=False):
+        """
+        Counts the number of rows in the specified target column of a worksheet.
+        :param worksheet: The name of the worksheet to read from
+        :param target_column: The column number to count rows in
+        :param include_tailing_empty: Whether to include empty trailing cells in the count
+        :return: The count of rows in the specified column
+        """
+        wks = self.get_sheet(worksheet)
+        if wks:
+            return len(wks.get_col(col=target_column, returnas='matrix', include_tailing_empty=include_tailing_empty))
+        return 0
+
+    def clear_range(self, worksheet, clear_range_str=None):
+        """
+        Clears the specified range or the entire worksheet in the Google Sheet.
+        :param worksheet: The name of the worksheet to clear
+        :param clear_range_str: The range to be cleared (e.g., 'A1:B2'). If None, the entire worksheet will be cleared
+        """
+        wks = self.get_sheet(worksheet)
+        if wks:
+            if clear_range_str:
+                clear_range = clear_range_str.split(':')
+                wks.clear(clear_range[0], clear_range[1])
+            else:
+                wks.clear()
+
+    def write_to_sheet(self, input_dataframe, worksheet, starting_pos='A1', extend=True, fit=False, nan='', clear_ws=True, clear_range_str=None):
+        """
+        Writes a pandas DataFrame to the specified worksheet in the Google Sheet.
+        If the worksheet does not exist, it will be created.
+        Optionally clears the worksheet or a specific range before writing.
+        :param input_dataframe: The pandas DataFrame to be written to the worksheet
+        :param worksheet: The worksheet to write to
+        :param starting_pos: The starting cell position to write the DataFrame (e.g., 'A1')
+        :param extend: Whether to extend the worksheet to fit the DataFrame if necessary
+        :param fit: Whether to resize the worksheet to fit the entire DataFrame
+        :param nan: The value to replace NaN entries with
+        :param clear_ws: Whether to clear the worksheet before writing the DataFrame
+        :param clear_range_str: The range to be cleared before writing if clear_ws is True
+        """
+        wks = self.get_sheet(worksheet)
+        if not wks:
+            print('Creating new sheet')
+            self.create_sheet(worksheet)
+            wks = self.get_sheet(worksheet)
+
+        if clear_ws and wks:
+            self.clear_range(worksheet=worksheet, clear_range_str=clear_range_str)
+
+        if wks:
+            wks.set_dataframe(df=input_dataframe, start=starting_pos, extend=extend, fit=fit, nan=nan)
+            print('Write to sheet completed')
+
+    def read_sheet(self, worksheet, target_range_str=None):
+        """
+        Reads data from the specified worksheet and returns it as a pandas DataFrame.
+        If the target range is provided, only that specific range is read.
+        :param worksheet: The name of the worksheet to read from
+        :param target_range_str: The target range to read from (e.g., 'A1:B2'). If None, the entire worksheet will be read
+        :return: A pandas DataFrame containing the data from the specified range or worksheet
+        """
+        wks = self.get_sheet(worksheet)
+        if wks:
+            if target_range_str:
+                target_range = target_range_str.split(':')
+                return wks.get_as_df(start=target_range[0], end=target_range[1])
+            else:
+                return wks.get_as_df()
+        return pd.DataFrame()

spellbook/stats_magic.py

@@ -0,0 +1,211 @@
+import scipy.stats as stats
+import pandas as pd
+from statsmodels.stats.multicomp import pairwise_tukeyhsd
+import scikit_posthocs as sp
+
+'''
+Statistical tests
+'''
+
+
+def shapiro_wilk_normality_test(df, group_column, values_column, alpha_level = 0.05, verbose = False):
+    results = []
+
+    if verbose:
+        print('Shapiro-Wilk Normality Test - H0: Samples follow a normal distribution')
+
+    # getting list of groups
+    list_groups = list(set(df[group_column]))
+
+    for column in list_groups:
+        # p value of the test
+        p_shapiro = stats.shapiro(df[df[group_column] == column][values_column].dropna()).pvalue
+
+        # printing results
+        if p_shapiro < alpha_level:
+            result = 'H0 rejected - non-normal distribution'
+        else:
+            result = 'H0 cannot be rejected - normal distribution'
+
+        if verbose:
+            print('Shapiro-Wilk Normality Test for: {} - p = {:.3f} - Result: {}'.format(column, p_shapiro, result))
+
+        results.append([p_shapiro, result])
+
+    return pd.DataFrame(results, columns=['p_value', 'result'])
+
+
+def levene_variance_test(df, group_column, values_column, alpha_level = 0.05, center_method = 'mean', verbose = False):
+    '''
+  Three variations of Levene’s test are possible. The possibilities and their recommended usages are:
+    ‘median’ : Recommended for skewed (non-normal) distributions
+    ‘mean’ : Recommended for symmetric, moderate-tailed distributions.
+    ‘trimmed’ : Recommended for heavy-tailed distributions.
+  '''
+    # getting list of groups
+    list_groups = list(set(df[group_column]))
+    list_of_lists = [df[df[group_column] == column][values_column].dropna() for column in list_groups]
+
+    results = []
+
+    if verbose:
+        print("Levene's Variance Test - H0: Samples come from populations with equal variances")
+
+    stat_levene, p_levene = stats.levene(*list_of_lists, center=center_method)
+
+    if p_levene < alpha_level:
+        result = 'H0 rejected - different variances'
+    else:
+        result = 'H0 cannot be rejected - same variance'
+
+    if verbose:
+        print('Levene Variance Test: p = {:.3f} - Result: {}'.format(p_levene, result))
+
+    results.append([p_levene, result])
+
+    return pd.DataFrame(results, columns=['p_value', 'result'])
+
+
+def anova(df, group_column, values_column, alpha_level = 0.05, verbose = False):
+    # getting list of groups
+    list_groups = list(set(df[group_column]))
+    list_of_lists = [df[df[group_column] == column][values_column].dropna() for column in list_groups]
+
+    results = []
+
+    # for normal data, One Way ANOVA
+    f_anova, p_anova = stats.f_oneway(*list_of_lists)
+
+    if p_anova < alpha_level:
+        result = 'H0 rejected -  The average values have statistically significant differences.'
+    else:
+        result = 'H0 cannot be rejected - The average values have NOT statistically significant differences.'
+
+    if verbose:
+        print("ANOVA - H0: Average values are the same")
+        print('F statistic = {:5.3f} and probability p = {:5.3f}'.format(f_anova, p_anova))
+        print('Result: {}'.format(result))
+
+    results.append([f_anova, p_anova, result])
+
+    return pd.DataFrame(results, columns=['f_statistic', 'p_value', 'result'])
+
+
+def tukey_test(df, group_column, values_column, alpha_level = 0.05, verbose = False):
+    tukey_table = pairwise_tukeyhsd(endog=df[values_column], groups=df[group_column], alpha=alpha_level)
+    if verbose:
+        print("Tukey's Test - H0: No significant difference has been observed for the pair")
+        print(tukey_table.summary())
+
+    return tukey_table
+
+
+def mood_median_test(df, group_column, values_column, alpha_level = 0.05, verbose = False):
+    # getting list of groups
+    list_groups = list(set(df[group_column]))
+    list_of_lists = [df[df[group_column] == column][values_column].dropna() for column in list_groups]
+
+    results = []
+
+    # for non-normal data, Mood's Median Test
+    f_mood, p_mood, med_mood, tbl_mood = stats.median_test(*list_of_lists)
+
+    if p_mood < alpha_level:
+        result = 'H0 rejected - The median values have statistically significant differences.'
+    else:
+        result = 'H0 cannot be rejected - The median values have NOT statistically significant differences.'
+
+    if verbose:
+        print("Mood's Median Test - H0: Median values are the same")
+        print("F statistic = {:5.3f} and probability p = {:5.3f}".format(f_mood, p_mood))
+        print('Result: {}'.format(result))
+
+    results.append([f_mood, p_mood, result])
+
+    return pd.DataFrame(results, columns=['f_statistic', 'p_value', 'result'])
+
+
+def dunn_test(df, group_column, values_column, alpha_level = 0.05, verbose = False):
+    # Dunn's test to check where is the difference
+    dunn_results = sp.posthoc_dunn(
+        a=df
+        , val_col=values_column
+        , group_col=group_column
+        , p_adjust='bonferroni'
+    )
+
+    # clean retults
+    list_groups = list(set(df[group_column]))
+    results = [[a, b, dunn_results[a][b]] for idx, a in enumerate(list_groups) for b in list_groups[idx + 1:]]
+
+    results = pd.DataFrame(results, columns=['a', 'b', 'p_value'])
+
+    results['reject'] = (results['p_value'] < alpha_level)
+
+    if verbose:
+        print("Dunn's Test - H0: No significant difference has been observed for the pair")
+        print(results)
+
+    return results
+
+
+def t_welch_test(df, group_column, values_column, equal_variances = True, alpha_level = 0.05, verbose = False):
+    # getting list of groups
+    list_groups = list(set(df[group_column]))
+
+    if len(list_groups) == 2:
+
+        results = []
+
+        list_of_lists = [df[df[group_column] == column][values_column].dropna() for column in list_groups]
+
+        # T test for same variance / Welch’s t-test for different variances
+        f_ttest, p_ttest = stats.ttest_ind(*list_of_lists, equal_var=equal_variances)
+
+        if p_ttest < alpha_level:
+            result = 'H0 rejected - The average values have statistically significant differences.'
+        else:
+            result = 'H0 cannot be rejected - The average values have NOT statistically significant differences.'
+
+        if verbose:
+            print("T-Test / Welch's T-Test - H0: Averages values are the same")
+            print("F-statistic = {:5.3f} and p-value = {:5.3f}".format(f_ttest, p_ttest))
+            print('Result: {}'.format(result))
+
+        results.append([f_ttest, p_ttest, result])
+
+        return pd.DataFrame(results, columns=['f_statistic', 'p_value', 'result'])
+
+    else:
+        print("T-Test / Welch's T-Test: Error: This functions requires exactly 2 groups")
+
+
+def mann_whitney_u_test(df, group_column, values_column, alpha_level = 0.05, verbose = False):
+    # getting list of groups
+    list_groups = list(set(df[group_column]))
+
+    if len(list_groups) == 2:
+
+        results = []
+
+        list_of_lists = [df[df[group_column] == column][values_column].dropna() for column in list_groups]
+
+        # performs the test
+        stat_mann, p_mann = stats.mannwhitneyu(*list_of_lists)
+
+        if p_mann < alpha_level:
+            result = 'H0 rejected - Different distributions.'
+        else:
+            result = 'H0 cannot be rejected - Same distribution.'
+
+        if verbose:
+            print("Mann-Whitney U Test - H0: Same Distributions")
+            print("Statistic = {:5.3f} and p-value = {:5.3f}".format(stat_mann, p_mann))
+            print('Result: {}'.format(result))
+
+        results.append([stat_mann, p_mann, result])
+
+        return pd.DataFrame(results, columns=['statistic', 'p_value', 'result'])
+
+    else:
+        print("Mann-Whitney U Test: Error: This functions requires exactly 2 groups")