imsciences 0.6.0.1__py3-none-any.whl → 0.6.0.3__py3-none-any.whl

imsciences/datafunctions.py

@@ -1,6 +1,5 @@
 import pandas as pd
 import calendar
-import requests
 import os
 import plotly.express as px
 import plotly.graph_objs as go
@@ -18,7 +17,8 @@ import requests_cache
 import urllib.request
 import requests
 from geopy.geocoders import Nominatim
-
+import subprocess
+import json
 
 class dataprocessing:
     
@@ -180,7 +180,16 @@ class dataprocessing:
         print("   - Description: Remove zero values in a specified column.")
         print("   - Usage: remove_zero_values(self, data_frame, column_to_filter)")
         print("   - Example: remove_zero_values(None, df, 'Funeral_Delivery')")       
-    
+
+        print("\n32. upgrade all packages")
+        print("   - Description: Upgrades all packages.")
+        print("   - Usage: upgrade_outdated_packages()")
+        print("   - Example: upgrade_outdated_packages()")
+        
+        print("\n33. Convert Mixed Formats Dates")
+        print("   - Description: Convert a mix of US and UK dates to datetime.")
+        print("   - Usage: convert_mixed_formats_dates(df, datecol)")
+        print("   - Example: convert_mixed_formats_dates(df, 'OBS')")     
     
     def get_wd_levels(self, levels):
         """
@@ -598,10 +607,6 @@ class dataprocessing:
         )
         return df
 
-        # Apply the fix to the specified column
-        df[date_col] = df[date_col].apply(lambda x: fix_date(x) if not pd.isnull(x) else x)
-        return df
-
     def combine_sheets(self, all_sheets):
         """
         Combines multiple DataFrames from a dictionary into a single DataFrame.
@@ -1194,6 +1199,69 @@ class dataprocessing:
             
             return data_frame.loc[~(data_frame[column_to_filter] ==0)]
 
+    def upgrade_outdated_packages(self):
+        try:
+            # Get all installed packages
+            installed_packages_result = subprocess.run("pip list --format=json", shell=True, capture_output=True, text=True)
+            installed_packages = json.loads(installed_packages_result.stdout)
+
+            # Get the list of outdated packages
+            outdated_packages_result = subprocess.run("pip list --outdated --format=json", shell=True, capture_output=True, text=True)
+            outdated_packages = json.loads(outdated_packages_result.stdout)
+
+            # Create a set of outdated package names for quick lookup
+            outdated_package_names = {pkg['name'] for pkg in outdated_packages}
+
+            # Upgrade only outdated packages
+            for package in installed_packages:
+                package_name = package['name']
+                if package_name in outdated_package_names:
+                    try:
+                        print(f"Upgrading package: {package_name}")
+                        upgrade_result = subprocess.run(f"pip install --upgrade {package_name}", shell=True, capture_output=True, text=True)
+                        if upgrade_result.returncode == 0:
+                            print(f"Successfully upgraded {package_name}")
+                        else:
+                            print(f"Failed to upgrade {package_name}: {upgrade_result.stderr}")
+                    except Exception as e:
+                        print(f"An error occurred while upgrading {package_name}: {e}")
+                else:
+                    print(f"{package_name} is already up to date")
+        except Exception as e:
+            print(f"An error occurred during the upgrade process: {e}")
+
+    def convert_mixed_formats_dates(self, df, column_name):
+        # Convert initial dates to datetime with coercion to handle errors
+        df[column_name] = pd.to_datetime(df[column_name], errors='coerce')
+        df[column_name] = df[column_name].astype(str)
+        corrected_dates = []
+        
+        for date_str in df[column_name]:
+            date_str = date_str.replace('-', '').replace('/', '')
+            if len(date_str) == 8:
+                year = date_str[:4]
+                month = date_str[4:6]
+                day = date_str[6:8]
+                if int(day) <= 12:
+                    # Swap month and day
+                    corrected_date_str = f"{year}-{day}-{month}"
+                else:
+                    corrected_date_str = f"{year}-{month}-{day}"
+                # Convert to datetime
+                corrected_date = pd.to_datetime(corrected_date_str, errors='coerce')
+            else:
+                corrected_date = pd.to_datetime(date_str, errors='coerce')
+            
+            corrected_dates.append(corrected_date)
+        
+        # Check length of the corrected_dates list
+        if len(corrected_dates) != len(df):
+            raise ValueError("Length of corrected_dates does not match the original DataFrame")
+        
+        # Assign the corrected dates back to the DataFrame
+        df[column_name] = corrected_dates
+        return df
+
 
 
 
@@ -1483,14 +1551,13 @@ class datapull:
         
         return ons_df_final
     
-    def pull_macro(self, country : str = "GBR", week_commencing : str = "mon"):
-        
+    def pull_macro(self, country: str = "GBR", week_commencing: str = "mon"):
         # Change country input to list
         countries_list = [country]
-        
+
         # Check if the data wants to be inputted at any other week commencing date
-        day_dict = {"mon" : 0, "tue" : 1, "wed" : 2, "thur" : 3, "fri" : 4, "sat" : 5, "sun" : 6}
-        
+        day_dict = {"mon": 0, "tue": 1, "wed": 2, "thur": 3, "fri": 4, "sat": 5, "sun": 6}
+
         # Two useful functions for quarterly data
         # Define a function to get quarterly data
         def get_quarter(p_date: datetime.date) -> int:
@@ -1500,39 +1567,32 @@ class datapull:
         def get_last_day_of_the_quarter(p_date: datetime.date):
             quarter = get_quarter(p_date)
             return datetime(p_date.year + 3 * quarter // 12, 3 * quarter % 12 + 1, 1) + pd.Timedelta(days=-1)
-        
+
         # For the monthly data
-        data_M, subjects_M, measures_M = cif.createDataFrameFromOECD(countries = countries_list, dsname = 'MEI',subject = ['LCEAMN01',
-                                                                                                                    'LCEAPR',
-                                                                                                                    'CSCICP03',
-                                                                                                                    'CPALTT01',
-                                                                                                                    'LRHUTTTT',
-                                                                                                                    'LORSGPRT',
-                                                                                                                    'IR3TIB01',     
-                                                                                                                    'PRINTO01'],
-                                                                measure = ['IXOBSA','IXNSA','IXNB','STSA','ST','GPSA','GY'],
-                                                                    frequency = 'M', startDate = '2015-01')
-        data_M = data_M.stack(level=[0,-1,-2]).reset_index()
-
-        data_Q, subjects_Q, measures_Q = cif.createDataFrameFromOECD(countries = countries_list, dsname = 'MEI',subject = ['LCEAMN01',
-                                                                                                                    'LCEAPR',
-                                                                                                                    'CSCICP03',
-                                                                                                                    'CPALTT01',
-                                                                                                                    'LRHUTTTT',
-                                                                                                                    'LORSGPRT',
-                                                                                                                    'IR3TIB01',
-                                                                                                                    'PRINTO01'],
-                                                                measure = ['IXOBSA','IXNSA','IXNB','STSA','ST','GPSA','GY'], frequency = 'Q',startDate = '2015-01')
-
-        data_Q=data_Q.stack(level=[0,-1,-2]).reset_index()
-
-        # create a data frame dictionary to store your monthly data frames
-        DataFrameDict_M = {elem : pd.DataFrame() for elem in countries_list}
+        data_M, subjects_M, measures_M = cif.createDataFrameFromOECD(countries=countries_list, dsname='MEI',
+                                                                    subject=['LCEAMN01', 'LCEAPR', 'CSCICP03', 'CPALTT01',
+                                                                            'LRHUTTTT', 'LORSGPRT', 'IR3TIB01',
+                                                                            'PRINTO01'],
+                                                                    measure=['IXOBSA', 'IXNSA', 'IXNB', 'STSA', 'ST', 'GPSA', 'GY'],
+                                                                    frequency='M', startDate='2015-01')
+        data_M = data_M.stack(level=[0, -1, -2]).reset_index()
+
+        data_Q, subjects_Q, measures_Q = cif.createDataFrameFromOECD(countries=countries_list, dsname='MEI',
+                                                                    subject=['LCEAMN01', 'LCEAPR', 'CSCICP03', 'CPALTT01',
+                                                                            'LRHUTTTT', 'LORSGPRT', 'IR3TIB01',
+                                                                            'PRINTO01'],
+                                                                    measure=['IXOBSA', 'IXNSA', 'IXNB', 'STSA', 'ST', 'GPSA', 'GY'],
+                                                                    frequency='Q', startDate='2015-01')
+
+        data_Q = data_Q.stack(level=[0, -1, -2]).reset_index()
+
+        # Create a data frame dictionary to store your monthly data frames
+        DataFrameDict_M = {elem: pd.DataFrame() for elem in countries_list}
         for key in DataFrameDict_M.keys():
             DataFrameDict_M[key] = data_M[:][data_M.country == key]
 
-        # create a data frame dictionary to store your quarterly data frames
-        DataFrameDict_Q = {elem : pd.DataFrame() for elem in countries_list}
+        # Create a data frame dictionary to store your quarterly data frames
+        DataFrameDict_Q = {elem: pd.DataFrame() for elem in countries_list}
         for key in DataFrameDict_Q.keys():
             DataFrameDict_Q[key] = data_Q[:][data_Q.country == key]
 
@@ -1540,41 +1600,40 @@ class datapull:
         countries_df_list_M = []
         for i in countries_list:
             df = pd.DataFrame(DataFrameDict_M[i])
-            df.rename(columns={0:'Values'},inplace=True)
-            df = pd.pivot_table(data=df,index='time',values='Values',columns=['subject','measure'])
+            df.rename(columns={0: 'Values'}, inplace=True)
+            df = pd.pivot_table(data=df, index='time', values='Values', columns=['subject', 'measure'])
             countries_df_list_M.append(df)
 
         # Create a quarterly list of the dataframes to iterate through
         countries_df_list_Q = []
         for i in countries_list:
             df = pd.DataFrame(DataFrameDict_Q[i])
-            df.rename(columns={0:'Values'},inplace=True)
-            df = pd.pivot_table(data=df,index='time',values='Values',columns=['subject','measure'])
+            df.rename(columns={0: 'Values'}, inplace=True)
+            df = pd.pivot_table(data=df, index='time', values='Values', columns=['subject', 'measure'])
             countries_df_list_Q.append(df)
 
-        combined_countries_df_list = list(zip(countries_df_list_M,countries_df_list_Q))
+        combined_countries_df_list = list(zip(countries_df_list_M, countries_df_list_Q))
 
         # Loop through and create dataframes for every country
         for index, data in enumerate(combined_countries_df_list):
-
             # Find country being extracted
-            country = countries_list[index]  
+            country = countries_list[index]
             print(country)
 
             # For consumer confidence
             # For countries with no data
-            if country in ['CAN','IND','NOR']:
+            if country in ['CAN', 'IND', 'NOR']:
                 Consumer_Confidence_Index_df_M = pd.DataFrame()
                 Consumer_Confidence_Index_df_Q = pd.DataFrame()
-            # For countries with quarterly data   
+            # For countries with quarterly data
             elif country in []:
                 Consumer_Confidence_Index_df_Q = data[1]['CSCICP03']['IXNSA']
-                Consumer_Confidence_Index_df_Q.rename('consumer_confidence_index',inplace=True)
+                Consumer_Confidence_Index_df_Q.rename('consumer_confidence_index', inplace=True)
                 Consumer_Confidence_Index_df_M = pd.DataFrame()
             # For countries with monthly data
             else:
                 Consumer_Confidence_Index_df_M = data[0]['CSCICP03']['IXNSA']
-                Consumer_Confidence_Index_df_M.rename('consumer_confidence_index',inplace=True)
+                Consumer_Confidence_Index_df_M.rename('consumer_confidence_index', inplace=True)
                 Consumer_Confidence_Index_df_Q = pd.DataFrame()
 
             # For consumer prices for COST OF LIVING
@@ -1583,14 +1642,14 @@ class datapull:
                 Consumer_Price_Index_Cost_Of_Living_df_M = pd.DataFrame()
                 Consumer_Price_Index_Cost_Of_Living_df_Q = pd.DataFrame()
             # For countries with quarterly data
-            elif country in ['AUS','NZL']:
+            elif country in ['AUS', 'NZL']:
                 Consumer_Price_Index_Cost_Of_Living_df_Q = data[1]['CPALTT01']['IXNB']
-                Consumer_Price_Index_Cost_Of_Living_df_Q.rename('consumer_price_index_cost_of_living',inplace=True)
+                Consumer_Price_Index_Cost_Of_Living_df_Q.rename('consumer_price_index_cost_of_living', inplace=True)
                 Consumer_Price_Index_Cost_Of_Living_df_M = pd.DataFrame()
             # For countries with monthly data
             else:
                 Consumer_Price_Index_Cost_Of_Living_df_M = data[0]['CPALTT01']['IXNB']
-                Consumer_Price_Index_Cost_Of_Living_df_M.rename('consumer_price_index_cost_of_living',inplace=True)
+                Consumer_Price_Index_Cost_Of_Living_df_M.rename('consumer_price_index_cost_of_living', inplace=True)
                 Consumer_Price_Index_Cost_Of_Living_df_Q = pd.DataFrame()
 
             # For consumer prices FOR INFLATION
@@ -1599,125 +1658,113 @@ class datapull:
                 Consumer_Price_Index_Inflation_df_M = pd.DataFrame()
                 Consumer_Price_Index_Inflation_df_Q = pd.DataFrame()
             # For countries with quarterly data
-            elif country in ['AUS','NZL']:
+            elif country in ['AUS', 'NZL']:
                 Consumer_Price_Index_Inflation_df_Q = data[1]['CPALTT01']['GY']
-                Consumer_Price_Index_Inflation_df_Q.rename('consumer_price_index_inflation',inplace=True)
+                Consumer_Price_Index_Inflation_df_Q.rename('consumer_price_index_inflation', inplace=True)
                 Consumer_Price_Index_Inflation_df_M = pd.DataFrame()
             # For countries with monthly data
             else:
                 Consumer_Price_Index_Inflation_df_M = data[0]['CPALTT01']['GY']
-                Consumer_Price_Index_Inflation_df_M.rename('consumer_price_index_inflation',inplace=True)
+                Consumer_Price_Index_Inflation_df_M.rename('consumer_price_index_inflation', inplace=True)
                 Consumer_Price_Index_Inflation_df_Q = pd.DataFrame()
 
-            # For GDP Index Smoothed 
+            # For GDP Index Smoothed
             # For countries with no data
-            if country in ['NLD','CHE','NZL','SWE','NOR']:
+            if country in ['NLD', 'CHE', 'NZL', 'SWE', 'NOR']:
                 GDP_Index_Smoothed_df_M = pd.DataFrame()
                 GDP_Index_Smoothed_df_Q = pd.DataFrame()
             # For countries with quarterly data
             elif country in []:
                 GDP_Index_Smoothed_df_Q = data[1]['LORSGPRT']['STSA']
-                GDP_Index_Smoothed_df_Q.rename('gdp_index_smoothed',inplace=True)
+                GDP_Index_Smoothed_df_Q.rename('gdp_index_smoothed', inplace=True)
                 GDP_Index_Smoothed_df_M = pd.DataFrame()
             # For countries with monthly data
             else:
                 GDP_Index_Smoothed_df_M = data[0]['LORSGPRT']['STSA']
-                GDP_Index_Smoothed_df_M.rename('gdp_index_smoothed',inplace=True)
+                GDP_Index_Smoothed_df_M.rename('gdp_index_smoothed', inplace=True)
                 GDP_Index_Smoothed_df_Q = pd.DataFrame()
 
             # For Harmonised Unemployment Index
             # For countries with no data
-            if country in ['IND','CHE','ZAF','CHN']:
+            if country in ['IND', 'CHE', 'ZAF', 'CHN']:
                 Harmonised_Unemployment_Index_df_M = pd.DataFrame()
                 Harmonised_Unemployment_Index_df_Q = pd.DataFrame()
             # For countries with quarterly data
             elif country in ['NZL']:
                 Harmonised_Unemployment_Index_df_Q = data[1]['LRHUTTTT']['STSA']
-                Harmonised_Unemployment_Index_df_Q.rename('harmonised_unemployment_index',inplace=True)  
+                Harmonised_Unemployment_Index_df_Q.rename('harmonised_unemployment_index', inplace=True)
                 Harmonised_Unemployment_Index_df_M = pd.DataFrame()
             # For countries with monthly data
-            else:     
+            else:
                 Harmonised_Unemployment_Index_df_M = data[0]['LRHUTTTT']['STSA']
-                Harmonised_Unemployment_Index_df_M.rename('harmonised_unemployment_index',inplace=True)  
+                Harmonised_Unemployment_Index_df_M.rename('harmonised_unemployment_index', inplace=True)
                 Harmonised_Unemployment_Index_df_Q = pd.DataFrame()
 
-            # For hourly earnings index manufacturing 
+            # For hourly earnings index manufacturing
             # For countries with no data
-            if country in ['IND','CHE','ZAF','CHN']:
+            if country in ['IND', 'CHE', 'ZAF', 'CHN']:
                 Hourly_Earnings_Index_Manufacturing_df_M = pd.DataFrame()
                 Hourly_Earnings_Index_Manufacturing_df_Q = pd.DataFrame()
             # For countries with quarterly data
-            elif country in ['FRA','DEU','ESP','AUS','NZL','KOR','NOR']:
+            elif country in ['FRA', 'DEU', 'ESP', 'AUS', 'NZL', 'KOR', 'NOR']:
                 Hourly_Earnings_Index_Manufacturing_df_Q = data[1]['LCEAMN01']['IXOBSA']
-                Hourly_Earnings_Index_Manufacturing_df_Q.rename('hourly_earnings_index_manufacturing',inplace=True)
+                Hourly_Earnings_Index_Manufacturing_df_Q.rename('hourly_earnings_index_manufacturing', inplace=True)
                 Hourly_Earnings_Index_Manufacturing_df_M = pd.DataFrame()
             # For countries with monthly data
             else:
                 Hourly_Earnings_Index_Manufacturing_df_M = data[0]['LCEAMN01']['IXOBSA']
-                Hourly_Earnings_Index_Manufacturing_df_M.rename('hourly_earnings_index_manufacturing',inplace=True)
+                Hourly_Earnings_Index_Manufacturing_df_M.rename('hourly_earnings_index_manufacturing', inplace=True)
                 Hourly_Earnings_Index_Manufacturing_df_Q = pd.DataFrame()
 
-            # For hourly earnings index private
-            # Hourly_Earnings_Index_Private_df = data['GBR']['LCEAPR']['IXOBSA']
-            # Hourly_Earnings_Index_Private_df.rename('Hourly Earnings Index Private',inplace=True)
-
             # For Short Term Interest Rate
             # For countries with no data
             if country in []:
                 Short_Term_Interest_Rate_df_M = pd.DataFrame()
                 Short_Term_Interest_Rate_df_Q = pd.DataFrame()
-            # For countries with quarterly data    
+            # For countries with quarterly data
             elif country in []:
                 Short_Term_Interest_Rate_df_Q = data[1]['IR3TIB01']['ST']
-                Short_Term_Interest_Rate_df_Q.rename('short_term_interest_rate',inplace=True)
+                Short_Term_Interest_Rate_df_Q.rename('short_term_interest_rate', inplace=True)
                 Short_Term_Interest_Rate_df_M = pd.DataFrame()
             # For countries with monthly data
             else:
                 Short_Term_Interest_Rate_df_M = data[0]['IR3TIB01']['ST']
-                Short_Term_Interest_Rate_df_M.rename('short_term_interest_rate',inplace=True)
+                Short_Term_Interest_Rate_df_M.rename('short_term_interest_rate', inplace=True)
                 Short_Term_Interest_Rate_df_Q = pd.DataFrame()
 
             # For Industrial Product Growth on Previous Period
             # For countries with no data
-            if country in ['ZAF','CHN']:
+            if country in ['ZAF', 'CHN']:
                 Industrial_Product_Growth_on_Previous_Period_df_M = pd.DataFrame()
                 Industrial_Product_Growth_on_Previous_Period_df_Q = pd.DataFrame()
             # For countries with quarterly data
-            elif country in ['AUS','NZL']:
+            elif country in ['AUS', 'NZL']:
                 Industrial_Product_Growth_on_Previous_Period_df_Q = data[1]['PRINTO01']['GPSA']
-                Industrial_Product_Growth_on_Previous_Period_df_Q.rename('industrial_product_growth_on_previous_period',inplace=True)
+                Industrial_Product_Growth_on_Previous_Period_df_Q.rename('industrial_product_growth_on_previous_period', inplace=True)
                 Industrial_Product_Growth_on_Previous_Period_df_M = pd.DataFrame()
             # For countries with monthly data
             else:
                 Industrial_Product_Growth_on_Previous_Period_df_M = data[0]['PRINTO01']['GPSA']
-                Industrial_Product_Growth_on_Previous_Period_df_M.rename('industrial_product_growth_on_previous_period',inplace=True)
+                Industrial_Product_Growth_on_Previous_Period_df_M.rename('industrial_product_growth_on_previous_period', inplace=True)
                 Industrial_Product_Growth_on_Previous_Period_df_Q = pd.DataFrame()
 
             # For Industrial Production Index
             # For countries with no data
-            if country in ['ZAF','CHN']:
+            if country in ['ZAF', 'CHN']:
                 Industrial_Production_Index_df_M = pd.DataFrame()
                 Industrial_Production_Index_df_Q = pd.DataFrame()
             # For countries with quarterly data
-            elif country in ['AUS','NZL']:
+            elif country in ['AUS', 'NZL']:
                 Industrial_Production_Index_df_Q = data[1]['PRINTO01']['IXOBSA']
-                Industrial_Production_Index_df_Q.rename('industrial_production_index',inplace=True) 
+                Industrial_Production_Index_df_Q.rename('industrial_production_index', inplace=True)
                 Industrial_Production_Index_df_M = pd.DataFrame()
             # For countries with monthly data
             else:
                 Industrial_Production_Index_df_M = data[0]['PRINTO01']['IXOBSA']
-                Industrial_Production_Index_df_M.rename('industrial_production_index',inplace=True)
+                Industrial_Production_Index_df_M.rename('industrial_production_index', inplace=True)
                 Industrial_Production_Index_df_Q = pd.DataFrame()
 
-            # For USD GBP Exchange Rate
-            # USD_GBP_Exchange_Rate_df = data['GBR']['PRINTO01']['IXOBSA']
-            # USD_GBP_Exchange_Rate_df.rename('Industrial Production Index',inplace=True)
-            
-        
-
-        
-        
-            # Create monthly macroeconomic dataframe 
+            # Create monthly macroeconomic dataframe
             all_dfs_list_M = [Consumer_Confidence_Index_df_M,
                             Consumer_Price_Index_Cost_Of_Living_df_M,
                             Consumer_Price_Index_Inflation_df_M,
@@ -1728,11 +1775,11 @@ class datapull:
                             Industrial_Product_Growth_on_Previous_Period_df_M,
                             Industrial_Production_Index_df_M]
 
-            # Check if any dataframes are empty and if there are remove them 
-            all_dfs_list_M = [df for df in all_dfs_list_M if not df.empty]    
-            cif_Macroeconomic_df_M = pd.concat(all_dfs_list_M,axis=1)
+            # Check if any dataframes are empty and if there are remove them
+            all_dfs_list_M = [df for df in all_dfs_list_M if not df.empty]
+            cif_Macroeconomic_df_M = pd.concat(all_dfs_list_M, axis=1)
 
-            # Create quarterly macroeconomic dataframe 
+            # Create quarterly macroeconomic dataframe
             all_dfs_list_Q = [Consumer_Confidence_Index_df_Q,
                             Consumer_Price_Index_Cost_Of_Living_df_Q,
                             Consumer_Price_Index_Inflation_df_Q,
@@ -1743,38 +1790,38 @@ class datapull:
                             Industrial_Product_Growth_on_Previous_Period_df_Q,
                             Industrial_Production_Index_df_Q]
 
-            # Check if any dataframes are empty and if there are remove them 
-            all_dfs_list_Q = [df for df in all_dfs_list_Q if not df.empty]   
+            # Check if any dataframes are empty and if there are remove them
+            all_dfs_list_Q = [df for df in all_dfs_list_Q if not df.empty]
             if all_dfs_list_Q != []:
-                macroeconomic_monthly_df_Q = pd.concat(all_dfs_list_Q,axis=1)
-            else: 
-                macroeconomic_monthly_df_Q  = []
+                macroeconomic_monthly_df_Q = pd.concat(all_dfs_list_Q, axis=1)
+            else:
+                macroeconomic_monthly_df_Q = pd.DataFrame()
 
             # For USD GBP Exchange Rate
             # If it's the UK add this series else don't
             if countries_list[index] == 'GBR':
-                USD_GBP_Exchange_Rate_df = pd.read_csv('https://stats.oecd.org/SDMX-JSON/data/MEI_FIN/CCUS.' + countries_list[index] + '.M/OECD?contentType=csv')
+                USD_GBP_Exchange_Rate_df = pd.read_csv(
+                    'https://stats.oecd.org/SDMX-JSON/data/MEI_FIN/CCUS.' + countries_list[index] + '.M/OECD?contentType=csv')
                 USD_GBP_Exchange_Rate_df.head()
-                USD_GBP_Exchange_Rate_df_pivot = pd.pivot_table(USD_GBP_Exchange_Rate_df,values='Value',index='TIME',columns='Subject')
+                USD_GBP_Exchange_Rate_df_pivot = pd.pivot_table(USD_GBP_Exchange_Rate_df, values='Value', index='TIME',
+                                                                columns='Subject')
                 USD_GBP_Exchange_Rate_df_pivot_final = USD_GBP_Exchange_Rate_df_pivot.loc["2015-01":]
-                USD_GBP_Exchange_Rate_df_pivot_final.rename(columns={'Currency exchange rates, monthly average':'usd_gbp_exchange_rate'},inplace=True)
+                USD_GBP_Exchange_Rate_df_pivot_final.rename(
+                    columns={'Currency exchange rates, monthly average': 'usd_gbp_exchange_rate'}, inplace=True)
 
                 # Create final monthly dataframe
-                macroeconomic_monthly_df_M = pd.concat([cif_Macroeconomic_df_M,USD_GBP_Exchange_Rate_df_pivot_final],axis=1)
+                macroeconomic_monthly_df_M = pd.concat([cif_Macroeconomic_df_M, USD_GBP_Exchange_Rate_df_pivot_final], axis=1)
             else:
                 # Create final monthly dataframe
                 macroeconomic_monthly_df_M = cif_Macroeconomic_df_M
 
-                
-                
-
-            # Create the final W/C Sunday dataframe 
+            # Create the final W/C Sunday dataframe
             # For monthly data
-            macroeconomic_monthly_df_M['Date']=macroeconomic_monthly_df_M.index
+            macroeconomic_monthly_df_M['Date'] = macroeconomic_monthly_df_M.index
             df_M = macroeconomic_monthly_df_M.set_index(pd.to_datetime(macroeconomic_monthly_df_M['Date'])).drop(columns='Date')
-            df_M.fillna(method="ffill",inplace=True)
+            df_M.fillna(method="ffill", inplace=True)
             df_M.reset_index(inplace=True)
-            
+
             daily_records = []
             # Iterate over each row in the DataFrame
             for _, row in df_M.iterrows():
@@ -1788,37 +1835,32 @@ class datapull:
 
             # Convert the list of daily records into a DataFrame
             daily_df = pd.DataFrame(daily_records)
-            
-            # Extend dataframe to include the current data if needed        
-            datelist = pd.date_range(daily_df["Date"].iloc[-1]+pd.Timedelta(days=1),datetime.today()).tolist()
-            extended_data =  np.repeat([list(daily_df.iloc[-1,1:].values)],len(datelist),axis=0)
-            q = pd.Series(datelist,name="Date")
-            s = pd.DataFrame(extended_data,columns=list(df_M.columns[1:]))
-            extended_daily_df = pd.concat([q,s],axis=1)
-            extended_daily_df = daily_df.append(extended_daily_df, ignore_index=False)
-            
-            # Create a week commencing column
-            extended_daily_df["Date"] = pd.to_datetime(extended_daily_df["Date"], format='%d %b %Y') 
-            extended_daily_df['week_start'] = extended_daily_df["Date"].apply(lambda x: x - pd.Timedelta(days=(x.weekday() - day_dict[week_commencing]) % 7))
-            extended_daily_df.drop("Date",axis=1,inplace=True)
-            extended_daily_df.rename(columns={'week_start':"Date"},inplace=True)
-            
-            # Take a weekly average
-            macroeconomic_weekly_df_M = extended_daily_df.groupby('Date').mean()
-            
-
-
-
 
+            # Extend dataframe to include the current data if needed
+            datelist = pd.date_range(daily_df["Date"].iloc[-1] + pd.Timedelta(days=1), datetime.today()).tolist()
+            extended_data = np.repeat([list(daily_df.iloc[-1, 1:].values)], len(datelist), axis=0)
+            q = pd.Series(datelist, name="Date")
+            s = pd.DataFrame(extended_data, columns=list(df_M.columns[1:]))
+            extended_daily_df = pd.concat([q, s], axis=1)
+            extended_daily_df = pd.concat([daily_df, extended_daily_df], ignore_index=False)
 
+            # Create a week commencing column
+            extended_daily_df["Date"] = pd.to_datetime(extended_daily_df["Date"], format='%d %b %Y')
+            extended_daily_df['week_start'] = extended_daily_df["Date"].apply(
+                lambda x: x - pd.Timedelta(days=(x.weekday() - day_dict[week_commencing]) % 7))
+            extended_daily_df.drop("Date", axis=1, inplace=True)
+            extended_daily_df.rename(columns={'week_start': "Date"}, inplace=True)
 
+            # Take a weekly average
+            macroeconomic_weekly_df_M = extended_daily_df.groupby('Date').mean()
 
             # For quarterly data
             # If there are quarterly datasets
             if all_dfs_list_Q != []:
-                macroeconomic_monthly_df_Q['Date']=macroeconomic_monthly_df_Q.index
-                df_Q = macroeconomic_monthly_df_Q.set_index(pd.to_datetime(macroeconomic_monthly_df_Q['Date'])).drop(columns='Date')
-                df_Q.fillna(method="ffill",inplace=True)
+                macroeconomic_monthly_df_Q['Date'] = macroeconomic_monthly_df_Q.index
+                df_Q = macroeconomic_monthly_df_Q.set_index(pd.to_datetime(macroeconomic_monthly_df_Q['Date'])).drop(
+                    columns='Date')
+                df_Q.fillna(method="ffill", inplace=True)
                 df_Q.reset_index(inplace=True)
 
                 daily_records = []
@@ -1826,45 +1868,47 @@ class datapull:
                     year = row["Date"].year
                     month = row["Date"].month
                     day = row["Date"].day
-                    last_date = get_last_day_of_the_quarter(datetime(year,month,day).date())
-                    all_days = pd.date_range(row["Date"],last_date,freq="D")
+                    last_date = get_last_day_of_the_quarter(datetime(year, month, day).date())
+                    all_days = pd.date_range(row["Date"], last_date, freq="D")
 
                     # Create a new record for each day of the quarter
                     for day in all_days:
                         daily_row = row.copy()
-                        daily_row["Date"] = row["Date"].replace(day=day.day,month=day.month)
+                        daily_row["Date"] = row["Date"].replace(day=day.day, month=day.month)
                         daily_records.append(daily_row)
 
                 # Convert the list of daily records into a DataFrame
                 daily_df = pd.DataFrame(daily_records)
-                
+
                 # Extend dataframe to include data up to today
-                datelist = pd.date_range(daily_df["Date"].iloc[-1]+pd.Timedelta(days=1),datetime.today()).tolist()
-                extended_data =  np.repeat([list(daily_df.iloc[-1,1:].values)],len(datelist),axis=0)
-                q = pd.Series(datelist,name="Date")
-                s = pd.DataFrame(extended_data,columns=list(df_Q.columns[1:]))
-                extended_daily_df = pd.concat([q,s],axis=1)
-                extended_daily_df = daily_df.append(extended_daily_df, ignore_index=False)
-                
+                datelist = pd.date_range(daily_df["Date"].iloc[-1] + pd.Timedelta(days=1), datetime.today()).tolist()
+                extended_data = np.repeat([list(daily_df.iloc[-1, 1:].values)], len(datelist), axis=0)
+                q = pd.Series(datelist, name="Date")
+                s = pd.DataFrame(extended_data, columns=list(df_Q.columns[1:]))
+                extended_daily_df = pd.concat([q, s], axis=1)
+                extended_daily_df = pd.concat([daily_df, extended_daily_df], ignore_index=False)
+
                 # Create a week commencing column
-                extended_daily_df["Date"] = pd.to_datetime(extended_daily_df["Date"], format='%d %b %Y') 
-                extended_daily_df['week_start'] = extended_daily_df["Date"].apply(lambda x: x - pd.Timedelta(days=(x.weekday() - day_dict[week_commencing]) % 7))
-                extended_daily_df.drop("Date",axis=1,inplace=True)
-                extended_daily_df.rename(columns={'week_start':"Date"},inplace=True)
-            
+                extended_daily_df["Date"] = pd.to_datetime(extended_daily_df["Date"], format='%d %b %Y')
+                extended_daily_df['week_start'] = extended_daily_df["Date"].apply(
+                    lambda x: x - pd.Timedelta(days=(x.weekday() - day_dict[week_commencing]) % 7))
+                extended_daily_df.drop("Date", axis=1, inplace=True)
+                extended_daily_df.rename(columns={'week_start': "Date"}, inplace=True)
+
                 # Take a weekly average
                 macroeconomic_weekly_df_Q = extended_daily_df.groupby('Date').mean()
 
             # Merge the two datasets together
             if all_dfs_list_Q != []:
-                macroeconomic_weekly_df = macroeconomic_weekly_df_M.merge(macroeconomic_weekly_df_Q,left_index=True, right_index=True)
+                macroeconomic_weekly_df = macroeconomic_weekly_df_M.merge(macroeconomic_weekly_df_Q, left_index=True,
+                                                                        right_index=True)
             # If there are no quarterly datasets
             else:
                 macroeconomic_weekly_df = macroeconomic_weekly_df_M
-            
-            # Change datime format
-            macroeconomic_weekly_df.index = macroeconomic_weekly_df.index.strftime('%d/%m/%Y') 
-            
+
+            # Change datetime format
+            macroeconomic_weekly_df.index = macroeconomic_weekly_df.index.strftime('%d/%m/%Y')
+
         macroeconomic_weekly_df.reset_index()
         macroeconomic_weekly_df.reset_index(drop=False, inplace=True)
         macroeconomic_weekly_df.rename(columns={'Date': 'OBS'}, inplace=True)
@@ -2050,12 +2094,11 @@ class datapull:
         country = country_dict[country]
 
         # Choose start and end dates
-        import datetime
         start_day = 1
         start_month = 1
         start_year = 2014
-        formatted_date = datetime.datetime(start_year, start_month, start_day).strftime("%Y-%m-%d")
-        today = datetime.datetime.now()
+        formatted_date = datetime(start_year, start_month, start_day).strftime("%Y-%m-%d")
+        today = datetime.now()
         end_day = today.day
         end_month = today.month
         end_year = today.year
@@ -2150,7 +2193,8 @@ class datapull:
             weather['week_starting'] = weather["day"].apply(lambda x: x - pd.Timedelta(days=(x.weekday() - day_dict[week_commencing]) % 7))
 
             # Group by week_starting and summarize
-            weekly_avg_temp = weather.groupby("week_starting").mean()
+            numeric_columns = weather.select_dtypes(include='number').columns
+            weekly_avg_temp = weather.groupby("week_starting")[numeric_columns].mean()
             weekly_avg_temp.rename(columns={"max_temp_f": "avg_max_temp_f",
                                             "min_temp_f": "avg_min_temp_f",
                                             "mean_temp_f": "avg_mean_temp_f",
@@ -2209,7 +2253,8 @@ class datapull:
                 weather['week_starting'] = weather["day"].apply(lambda x: x - pd.Timedelta(days=(x.weekday() - day_dict[week_commencing]) % 7))
 
                 # Group by week_starting and summarize
-                weekly_avg_temp = weather.groupby("week_starting").mean()
+                numeric_columns = weather.select_dtypes(include='number').columns
+                weekly_avg_temp = weather.groupby("week_starting")[numeric_columns].mean()
                 weekly_avg_temp.rename(columns={"max_temp_f": "avg_max_temp_f",
                                                 "min_temp_f": "avg_min_temp_f",
                                                 "mean_temp_f": "avg_mean_temp_f",
@@ -2260,7 +2305,8 @@ class datapull:
             weather['week_starting'] = weather["day"].apply(lambda x: x - pd.Timedelta(days=(x.weekday() - day_dict[week_commencing]) % 7))
 
             # Group by week_starting and summarize
-            weekly_avg_temp = weather.groupby("week_starting").mean()
+            numeric_columns = weather.select_dtypes(include='number').columns
+            weekly_avg_temp = weather.groupby("week_starting")[numeric_columns].mean()
             weekly_avg_temp.rename(columns={"max_temp_f": "avg_max_temp_f",
                                             "min_temp_f": "avg_min_temp_f",
                                             "mean_temp_f": "avg_mean_temp_f",
@@ -2282,7 +2328,6 @@ class datapull:
 
             # Loop through each city and fetch weather data
             for city in cities:
-
                 # Initialize Nominatim API
                 geolocator = Nominatim(user_agent="MyApp")
                 location = geolocator.geocode(city)
@@ -2318,7 +2363,8 @@ class datapull:
             all_weather_data['week_starting'] = all_weather_data["date"].apply(lambda x: x - pd.Timedelta(days=(x.weekday() - day_dict[week_commencing]) % 7))
 
             # Group by week_starting and summarize
-            weekly_avg_rain = all_weather_data.groupby("week_starting").mean()
+            numeric_columns = all_weather_data.select_dtypes(include='number').columns
+            weekly_avg_rain = all_weather_data.groupby("week_starting")[numeric_columns].mean()
             weekly_avg_rain.rename(columns={"rainfall": "avg_rainfall"}, inplace=True)
 
             # Change index to datetime
@@ -2337,7 +2383,6 @@ class datapull:
 
             # Loop through each city and fetch weather data
             for city in cities:
-
                 # Initialize Nominatim API
                 geolocator = Nominatim(user_agent="MyApp")
                 location = geolocator.geocode(city)
@@ -2373,7 +2418,8 @@ class datapull:
             all_weather_data['week_starting'] = all_weather_data["date"].apply(lambda x: x - pd.Timedelta(days=(x.weekday() - day_dict[week_commencing]) % 7))
 
             # Group by week_starting and summarize
-            weekly_avg_rain = all_weather_data.groupby("week_starting").mean()
+            numeric_columns = all_weather_data.select_dtypes(include='number').columns
+            weekly_avg_rain = all_weather_data.groupby("week_starting")[numeric_columns].mean()
             weekly_avg_rain.rename(columns={"rainfall": "avg_rainfall"}, inplace=True)
 
             # Change index to datetime
@@ -2392,7 +2438,6 @@ class datapull:
 
             # Loop through each city and fetch weather data
             for city in cities:
-
                 # Initialize Nominatim API
                 geolocator = Nominatim(user_agent="MyApp")
                 location = geolocator.geocode(city)
@@ -2428,7 +2473,8 @@ class datapull:
             all_weather_data['week_starting'] = all_weather_data["date"].apply(lambda x: x - pd.Timedelta(days=(x.weekday() - day_dict[week_commencing]) % 7))
 
             # Group by week_starting and summarize
-            weekly_avg_rain = all_weather_data.groupby("week_starting").mean()
+            numeric_columns = all_weather_data.select_dtypes(include='number').columns
+            weekly_avg_rain = all_weather_data.groupby("week_starting")[numeric_columns].mean()
             weekly_avg_rain.rename(columns={"rainfall": "avg_rainfall"}, inplace=True)
 
             # Change index to datetime
@@ -2447,7 +2493,6 @@ class datapull:
 
             # Loop through each city and fetch weather data
             for city in cities:
-
                 # Initialize Nominatim API
                 geolocator = Nominatim(user_agent="MyApp")
                 location = geolocator.geocode(city)
@@ -2483,7 +2528,8 @@ class datapull:
             all_weather_data['week_starting'] = all_weather_data["date"].apply(lambda x: x - pd.Timedelta(days=(x.weekday() - day_dict[week_commencing]) % 7))
 
             # Group by week_starting and summarize
-            weekly_avg_rain = all_weather_data.groupby("week_starting").mean()
+            numeric_columns = all_weather_data.select_dtypes(include='number').columns
+            weekly_avg_rain = all_weather_data.groupby("week_starting")[numeric_columns].mean()
             weekly_avg_rain.rename(columns={"rainfall": "avg_rainfall"}, inplace=True)
 
             # Change index to datetime

{imsciences-0.6.0.1.dist-info → imsciences-0.6.0.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: imsciences
-Version: 0.6.0.1
+Version: 0.6.0.3
 Summary: IMS Data Processing Package
 Author: IMS
 Author-email: cam@im-sciences.com

{imsciences-0.6.0.1.dist-info → imsciences-0.6.0.3.dist-info}/RECORD

@@ -2,13 +2,13 @@ dataprocessing/__init__.py,sha256=quSwsLs6IuLoA5Rzi0ZD40xZaQudwDteF7_ai9JfTPk,32
 dataprocessing/data-processing-functions.py,sha256=vE1vsZ8xOSbR9Bwlp9SWXwEHXQ0nFydwGkvzHXf2f1Y,41
 dataprocessing/datafunctions.py,sha256=vE1vsZ8xOSbR9Bwlp9SWXwEHXQ0nFydwGkvzHXf2f1Y,41
 imsciences/__init__.py,sha256=GIPbLmWc06sVcOySWwNvMNUr6XGOHqPLryFIWgtpHh8,78
-imsciences/datafunctions.py,sha256=0zo2vqyYY-8AifRatiB5VJk-LcAOedMZsmJyHUMxZHY,131239
+imsciences/datafunctions.py,sha256=DKK03w47pFXl5xTPZ_0CdCmJP_IVwT1OLBVzK5-QjGU,133927
 imsciences/datapull.py,sha256=TPY0LDgOkcKTBk8OekbD0Grg5x0SomAK2dZ7MuT6X1E,19000
 imsciencesdataprocessing/__init__.py,sha256=quSwsLs6IuLoA5Rzi0ZD40xZaQudwDteF7_ai9JfTPk,32
 imsciencesdataprocessing/datafunctions.py,sha256=vE1vsZ8xOSbR9Bwlp9SWXwEHXQ0nFydwGkvzHXf2f1Y,41
 imsdataprocessing/__init__.py,sha256=quSwsLs6IuLoA5Rzi0ZD40xZaQudwDteF7_ai9JfTPk,32
 imsdataprocessing/datafunctions.py,sha256=vE1vsZ8xOSbR9Bwlp9SWXwEHXQ0nFydwGkvzHXf2f1Y,41
-imsciences-0.6.0.1.dist-info/METADATA,sha256=o_k6Ulx0n5Cm7tK4H3N92Hw4HpVaXwECbCC3K-4HQoQ,11571
-imsciences-0.6.0.1.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-imsciences-0.6.0.1.dist-info/top_level.txt,sha256=hsENS-AlDVRh8tQJ6-426iUQlla9bPcGc0-UlFF0_iU,11
-imsciences-0.6.0.1.dist-info/RECORD,,
+imsciences-0.6.0.3.dist-info/METADATA,sha256=8_UQYWlXnPmBctZ1w9wGltM-DN-PQmovqiQ9ERxe7Ng,11571
+imsciences-0.6.0.3.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+imsciences-0.6.0.3.dist-info/top_level.txt,sha256=hsENS-AlDVRh8tQJ6-426iUQlla9bPcGc0-UlFF0_iU,11
+imsciences-0.6.0.3.dist-info/RECORD,,