imsciences 0.6.2.1__py3-none-any.whl → 0.6.2.3__py3-none-any.whl

imsciences/datafunctions.py

@@ -109,8 +109,8 @@ class dataprocessing:
         
         print("\n17. pivot_table")
         print("    - Description: Dynamically pivots a DataFrame based on specified columns.")
-        print("    - Usage: pivot_table(df, filters_dict, index_col, columns, values_col, fill_value=0,aggfunc='sum',margins=False,margins_name='Total',datetime_trans_needed=True)")
-        print("    - Example: pivot_table(df, {'Master Include':' == 1','OBS':' >= datetime(2019,9,9)','Metric Short Names':' == 'spd''}, 'OBS', 'Channel Short Names', 'Value', fill_value=0,aggfunc='sum',margins=False,margins_name='Total',datetime_trans_needed=True)")
+        print("    - Usage: pivot_table(df, index_col, columns, values_col, filters_dict=None, fill_value=0,aggfunc='sum',margins=False,margins_name='Total',datetime_trans_needed=True,reverse_header_order = 'False')")
+        print("    - Example: pivot_table(df, 'OBS', 'Channel Short Names', 'Value',filters_dict={'Master Include':' == 1','OBS':' >= datetime(2019,9,9)','Metric Short Names':' == 'spd''}, fill_value=0,aggfunc='sum',margins=False,margins_name='Total',datetime_trans_needed=True,reverse_header_order = 'True')")
         
         print("\n18. apply_lookup_table_for_columns")
         print("    - Description: Equivalent of xlookup in excel. Allows you to map a dictionary of substrings within a column. If multiple columns are need for the LUT then a | seperator is needed.")
@@ -177,42 +177,42 @@ class dataprocessing:
         print("   - Usage: read_and_concatenate_files(folder_path, file_type='csv')")
         print("   - Example: read_and_concatenate_files(folder_path, file_type='csv')")  
 
-        print("\n31. remove zero values")
+        print("\n31. remove_zero_values")
         print("   - Description: Remove zero values in a specified column.")
         print("   - Usage: remove_zero_values(self, data_frame, column_to_filter)")
         print("   - Example: remove_zero_values(None, df, 'Funeral_Delivery')")       
 
-        print("\n32. upgrade all packages")
+        print("\n32. upgrade_outdated_packages")
         print("   - Description: Upgrades all packages.")
         print("   - Usage: upgrade_outdated_packages()")
         print("   - Example: upgrade_outdated_packages()")
         
-        print("\n33. Convert Mixed Formats Dates")
+        print("\n33. convert_mixed_formats_dates")
         print("   - Description: Convert a mix of US and UK dates to datetime.")
         print("   - Usage: convert_mixed_formats_dates(df, datecol)")
         print("   - Example: convert_mixed_formats_dates(df, 'OBS')")
         
-        print("\n34. Fill Weekly Missing Dates")
+        print("\n34. fill_weekly_date_range")
         print("   - Description: Fill in any missing weeks with 0.")
         print("   - Usage: fill_weekly_date_range(df, date_column, freq)")
         print("   - Example: fill_weekly_date_range(df, 'OBS', 'W-MON')")     
 
-        print("\n35. Add Prefix and/or Suffix to Column Headers")
+        print("\n35. add_prefix_and_suffix")
         print("   - Description: Add Prefix and/or Suffix to Column Headers.")
         print("   - Usage: add_prefix_and_suffix(df, prefix='', suffix='', date_col=None)")
         print("   - Example: add_prefix_and_suffix(df, prefix='media_', suffix='_spd', date_col='obs')") 
 
-        print("\n36. Change all data to dummies")
+        print("\n36. create_dummies")
         print("   - Description: Changes time series to 0s and 1s based off threshold")
         print("   - Usage: create_dummies(df, date_col=None, dummy_threshold=0, add_total_dummy_col='No', total_col_name='total')")
         print("   - Example: create_dummies(df, date_col='obs', dummy_threshold=100, add_total_dummy_col='Yes', total_col_name='med_total_dum')") 
 
-        print("\n37. Replace substrings in column of strings")
+        print("\n37. replace_substrings")
         print("   - Description: Replace substrings in column of strings based off dictionary, can also change column to lower")
         print("   - Usage: replace_substrings(df, column, replacements, to_lower=False, new_column=None)")
         print("   - Example: replace_substrings(df, 'Influencer Handle', replacement_dict, to_lower=True, new_column='Short Version')") 
 
-        print("\n38. Add totals column")
+        print("\n38. add_total_column")
         print("   - Description: Sums all columns with the option to exclude an date column to create a total column")
         print("   - Usage: add_total_column(df, exclude_col=None, total_col_name='Total')")
         print("   - Example: add_total_column(df, exclude_col='obs', total_col_name='total_media_spd')")   
@@ -221,6 +221,13 @@ class dataprocessing:
         print("    - Description: Equivalent of xlookup in excel, but only based on substrings. If a substring is found in a cell, than look it up in the dictionary. Otherwise use the other label")
         print("    - Usage: apply_lookup_table_based_on_substring(df, column_name, category_dict, new_col_name='Category', other_label='Other')")
         print("    - Example: apply_lookup_table_based_on_substring(df, 'Campaign Name', campaign_dict, new_col_name='Campaign Name Short', other_label='Full Funnel')")
+        
+        print("\n40. compare_overlap")
+        print("    - Description: With two matching dataset, it takes the common columns and rows and takes the difference between them, outputing a differences and total differences table")
+        print("    - Usage: compare_overlap(df1, df2, date_col)")
+        print("    - Example: compare_overlap(df_1, df_2, 'obs')")        
+        
+        
                 
     def get_wd_levels(self, levels):
         """
@@ -657,59 +664,68 @@ class dataprocessing:
 
         return combined_df
     
-    def pivot_table(self, df, filters_dict, index_col, columns, values_col, fill_value=0,aggfunc='sum',margins=False,margins_name="Total",datetime_trans_needed=True):
+    def pivot_table(self, df, index_col, columns, values_col, filters_dict=None, fill_value=0, aggfunc='sum', margins=False, margins_name="Total", datetime_trans_needed=True, reverse_header_order=False):
         """
         Provides the ability to create pivot tables, filtering the data to get to data you want and then pivoting on certain columns
 
         Args:
             df (pandas.DataFrame): The DataFrame containing the data.
-            filters_dict (dict): Dictionary of conditions for the boolean mask i.e. what to filter your df on to get to your chosen cell
             index_col (str): Name of Column for your pivot table to index on
             columns (str): Name of Columns for your pivot table.
             values_col (str): Name of Values Columns for your pivot table.
+            filters_dict (dict, optional): Dictionary of conditions for the boolean mask i.e. what to filter your df on to get to your chosen cell. Defaults to None
             fill_value (int, optional): The value to replace nan with. Defaults to 0.
             aggfunc (str, optional): The method on which to aggregate the values column. Defaults to sum.
             margins (bool, optional): Whether the pivot table needs a total rows and column. Defaults to False.
             margins_name (str, optional): The name of the Totals columns. Defaults to "Total".
             datetime_trans_needed (bool, optional): Whether the index column needs to be transformed into datetime format. Defaults to False.
+            reverse_header_order (bool, optional): Reverses the order of the column headers. Defaults to False.
 
         Returns:
             pandas.DataFrame: The pivot table specified
         """
         
         # Create the filtered df by applying the conditions
-        df_filtered = self.filter_df_on_multiple_conditions(df, filters_dict)
+        if filters_dict is None:
+            df_filtered = df
+        else:
+            df_filtered = self.filter_df_on_multiple_conditions(df, filters_dict)
         
-        # Ensure OBS is in datetime format for proper sorting
+        # Ensure index column is in datetime format for proper sorting
         df_filtered = df_filtered.copy()
         
         # If datetime transformation is needed
-        if datetime_trans_needed is True:
-            df_filtered.loc[:,index_col] = pd.to_datetime(df_filtered[index_col], dayfirst=True)
+        if datetime_trans_needed:
+            df_filtered[index_col] = pd.to_datetime(df_filtered[index_col], dayfirst=True)
         
         # Create the pivot table
-        pivoted_df = df_filtered.pivot_table(index=index_col, columns=columns, values=values_col, aggfunc=aggfunc,margins=margins,margins_name=margins_name)
+        pivoted_df = df_filtered.pivot_table(index=index_col, columns=columns, values=values_col, aggfunc=aggfunc, margins=margins, margins_name=margins_name)
             
         # Handling MultiIndex columns if present, making them a flat structure
-        if isinstance(pivoted_df.columns, pd.MultiIndex):
-            pivoted_df.columns = ['_'.join(map(str, col)).strip() for col in pivoted_df.columns.values]
+        if not reverse_header_order:
+            if isinstance(pivoted_df.columns, pd.MultiIndex):
+                pivoted_df.columns = ['_'.join(map(str, col)).strip() for col in pivoted_df.columns.values]
+            else:
+                pivoted_df.columns = pivoted_df.columns.map(str)
         else:
-            pivoted_df.columns = pivoted_df.columns.map(str)
+            if isinstance(pivoted_df.columns, pd.MultiIndex):
+                # Reorder the MultiIndex columns
+                pivoted_df.columns = ['_'.join(reversed(list(map(str, col)))).strip() for col in pivoted_df.columns.values]
+            else:
+                pivoted_df.columns = pivoted_df.columns.map(str)
+                # Reverse the order for single index columns
+                pivoted_df.columns = ['_'.join(reversed(col.split('_'))).strip() for col in pivoted_df.columns]
         
         # Reset the pivot before returning
         pivoted_df = pivoted_df.reset_index()
         
-        # Sort by OBS from oldest to newest
-        if datetime_trans_needed is True:
-            # pivoted_df = pivoted_df.reset_index()
+        # Sort by index column from oldest to newest
+        if datetime_trans_needed:
             pivoted_df[index_col] = pd.to_datetime(pivoted_df[index_col])  # Ensure sorting works correctly
             pivoted_df = pivoted_df.sort_values(by=index_col)
-        
-            # Convert OBS back to a string in YYYY-MM-DD format for display purposes
-            pivoted_df[index_col] = pivoted_df[index_col].dt.strftime('%Y-%m-%d')
             
-            # Set index back to date column
-            # pivoted_df.set_index(index_col,inplace=True)
+            # Convert index column back to a string in YYYY-MM-DD format for display purposes
+            pivoted_df[index_col] = pivoted_df[index_col].dt.strftime('%Y-%m-%d')
         
         # Fill in any NaNs
         pivoted_df = pivoted_df.fillna(fill_value)
@@ -1466,7 +1482,59 @@ class dataprocessing:
         df[new_col_name] = df[column_name].apply(categorize_text)
         return df
 
+    def compare_overlap(df1, df2, date_col):
+        """
+        Compare overlapping periods between two DataFrames and provide a summary of total differences.
+
+        Args:
+            df1 (pandas.DataFrame): First DataFrame containing date-based data.
+            df2 (pandas.DataFrame): Second DataFrame containing date-based data.
+            date_col (str): The name of the date column used for aligning data.
 
+        Returns:
+            tuple: A tuple containing the DataFrame of differences and a summary DataFrame with total differences by column.
+        """
+        # Ensure date columns are in datetime format
+        df1[date_col] = pd.to_datetime(df1[date_col])
+        df2[date_col] = pd.to_datetime(df2[date_col])
+        
+        # Determine the overlap period
+        start_date = max(df1[date_col].min(), df2[date_col].min())
+        end_date = min(df1[date_col].max(), df2[date_col].max())
+        
+        # Filter dataframes to the overlapping period
+        df1_overlap = df1[(df1[date_col] >= start_date) & (df1[date_col] <= end_date)]
+        df2_overlap = df2[(df2[date_col] >= start_date) & (df2[date_col] <= end_date)]
+        
+        # Merge the dataframes on the date column to align data for comparison
+        merged_df = pd.merge(df1_overlap, df2_overlap, on=date_col, suffixes=('_df1', '_df2'))
+        
+        # Initialize a list to collect total differences for each column
+        total_diff_list = []
+        
+        # Compare the values in each column (excluding the date column)
+        diff_df = pd.DataFrame({date_col: merged_df[date_col]})  # Initialize diff_df with the date column
+        
+        for col in df1.columns:
+            if col != date_col:
+                # Calculate the difference for each row
+                diff_col = f'diff_{col}'
+                diff_df[diff_col] = merged_df[f'{col}_df1'] - merged_df[f'{col}_df2']
+                
+                # Calculate the total difference for the column and add it to the list
+                total_diff = diff_df[diff_col].sum()
+                total_diff_list.append({'Column': col, 'Total Difference': total_diff})
+        
+        # Create a DataFrame for the summary of total differences
+        total_diff_df = pd.DataFrame(total_diff_list)
+        
+        # Apply formatting to the numerical columns
+        float_format = "{:,.2f}".format  # Format to 2 decimal places with comma as thousand separator
+        diff_df.iloc[:, 1:] = diff_df.iloc[:, 1:].applymap(float_format)
+        total_diff_df['Total Difference'] = total_diff_df['Total Difference'].apply(float_format)
+        
+        return diff_df, total_diff_df
+    
 ########################################################################################################################################
 ########################################################################################################################################
 

{imsciences-0.6.2.1.dist-info → imsciences-0.6.2.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: imsciences
-Version: 0.6.2.1
+Version: 0.6.2.3
 Summary: IMS Data Processing Package
 Author: IMS
 Author-email: cam@im-sciences.com

{imsciences-0.6.2.1.dist-info → imsciences-0.6.2.3.dist-info}/RECORD

@@ -2,13 +2,13 @@ dataprocessing/__init__.py,sha256=quSwsLs6IuLoA5Rzi0ZD40xZaQudwDteF7_ai9JfTPk,32
 dataprocessing/data-processing-functions.py,sha256=vE1vsZ8xOSbR9Bwlp9SWXwEHXQ0nFydwGkvzHXf2f1Y,41
 dataprocessing/datafunctions.py,sha256=vE1vsZ8xOSbR9Bwlp9SWXwEHXQ0nFydwGkvzHXf2f1Y,41
 imsciences/__init__.py,sha256=GIPbLmWc06sVcOySWwNvMNUr6XGOHqPLryFIWgtpHh8,78
-imsciences/datafunctions.py,sha256=zI_vhjBQfa4Lef2NucUViYAJFenEB2RlJ1rnXIIBG5Y,139645
+imsciences/datafunctions.py,sha256=IrcIfw80MQnnRc2gD6QfuKIlDgVQxkZX-bTj7LKOiEU,143441
 imsciences/datapull.py,sha256=TPY0LDgOkcKTBk8OekbD0Grg5x0SomAK2dZ7MuT6X1E,19000
 imsciencesdataprocessing/__init__.py,sha256=quSwsLs6IuLoA5Rzi0ZD40xZaQudwDteF7_ai9JfTPk,32
 imsciencesdataprocessing/datafunctions.py,sha256=vE1vsZ8xOSbR9Bwlp9SWXwEHXQ0nFydwGkvzHXf2f1Y,41
 imsdataprocessing/__init__.py,sha256=quSwsLs6IuLoA5Rzi0ZD40xZaQudwDteF7_ai9JfTPk,32
 imsdataprocessing/datafunctions.py,sha256=vE1vsZ8xOSbR9Bwlp9SWXwEHXQ0nFydwGkvzHXf2f1Y,41
-imsciences-0.6.2.1.dist-info/METADATA,sha256=4p9HLTYPZbsBAkr2dzC1dvvQL-GWZsjTrNXEKGb_5hc,854
-imsciences-0.6.2.1.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-imsciences-0.6.2.1.dist-info/top_level.txt,sha256=hsENS-AlDVRh8tQJ6-426iUQlla9bPcGc0-UlFF0_iU,11
-imsciences-0.6.2.1.dist-info/RECORD,,
+imsciences-0.6.2.3.dist-info/METADATA,sha256=diBYqgQ-3WJ9pcVQfeAmJkUyLzpy5tqMX1VWjD6zT7k,854
+imsciences-0.6.2.3.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+imsciences-0.6.2.3.dist-info/top_level.txt,sha256=hsENS-AlDVRh8tQJ6-426iUQlla9bPcGc0-UlFF0_iU,11
+imsciences-0.6.2.3.dist-info/RECORD,,