imsciences 1.0.2__py3-none-any.whl → 1.1.6__py3-none-any.whl

imsciences/pull-IMS-24Ltp-3.py

@@ -0,0 +1,3132 @@
+import importlib
+import re
+import time
+import urllib.request
+import xml.etree.ElementTree as ET
+from datetime import datetime, timedelta
+from io import StringIO
+
+import numpy as np
+import pandas as pd
+import requests
+import yfinance as yf
+from bs4 import BeautifulSoup
+from dateutil.easter import easter
+from fredapi import Fred
+from geopy.geocoders import Nominatim
+
+from imsciences.mmm import dataprocessing
+
+ims_proc = dataprocessing()
+
+
+class datapull:
+    def help(self):
+        print("This is the help section. The functions in the package are as follows:")
+
+        print("\n1. pull_fred_data")
+        print("   - Description: Get data from FRED by using series id tokens.")
+        print("   - Usage: pull_fred_data(week_commencing, series_id_list)")
+        print("   - Example: pull_fred_data('mon', ['GPDIC1'])")
+
+        print("\n2. pull_boe_data")
+        print("   - Description: Fetch and process Bank of England interest rate data.")
+        print("   - Usage: pull_boe_data(week_commencing)")
+        print("   - Example: pull_boe_data('mon')")
+
+        print("\n3. pull_oecd")
+        print(
+            "   - Description: Fetch macroeconomic data from OECD for a specified country.",
+        )
+        print(
+            "   - Usage: pull_oecd(country='GBR', week_commencing='mon', start_date: '2020-01-01')",
+        )
+        print("   - Example: pull_oecd('GBR', 'mon', '2000-01-01')")
+
+        print("\n4. get_google_mobility_data")
+        print("   - Description: Fetch Google Mobility data for the specified country.")
+        print("   - Usage: get_google_mobility_data(country, wc)")
+        print("   - Example: get_google_mobility_data('United Kingdom', 'mon')")
+
+        print("\n5. pull_seasonality")
+        print(
+            "   - Description: Generate combined dummy variables for seasonality, trends, and COVID lockdowns.",
+        )
+        print("   - Usage: pull_seasonality(week_commencing, start_date, countries)")
+        print("   - Example: pull_seasonality('mon', '2020-01-01', ['US', 'GB'])")
+
+        print("\n6. pull_weather")
+        print(
+            "   - Description: Fetch and process historical weather data for the specified country.",
+        )
+        print("   - Usage: pull_weather(week_commencing, start_date, country)")
+        print("   - Example: pull_weather('mon', '2020-01-01', ['GBR'])")
+
+        print("\n7. pull_macro_ons_uk")
+        print(
+            "   - Description: Fetch and process time series data from the Beta ONS API.",
+        )
+        print("   - Usage: pull_macro_ons_uk(aditional_list, week_commencing, sector)")
+        print("   - Example: pull_macro_ons_uk(['HBOI'], 'mon', 'fast_food')")
+
+        print("\n8. pull_yfinance")
+        print(
+            "   - Description: Fetch and process time series data from the Beta ONS API.",
+        )
+        print("   - Usage: pull_yfinance(tickers, week_start_day)")
+        print("   - Example: pull_yfinance(['^FTMC', '^IXIC'], 'mon')")
+
+        print("\n9. pull_sports_events")
+        print(
+            "   - Description: Pull a veriety of sports events primaraly football and rugby.",
+        )
+        print("   - Usage: pull_sports_events(start_date, week_commencing)")
+        print("   - Example: pull_sports_events('2020-01-01', 'mon')")
+
+    ###############################################################  MACRO ##########################################################################
+
+    def pull_fred_data(
+        self,
+        week_commencing: str = "mon",
+        series_id_list: list[str] = ["GPDIC1", "Y057RX1Q020SBEA", "GCEC1"],
+    ) -> pd.DataFrame:
+        """
+        Parameters
+        ----------
+        week_commencing : str
+            specify the day for the week commencing, the default is 'sun' (e.g., 'mon', 'tue', 'wed', 'thu', 'fri', 'sat', 'sun')
+
+        series_id_list : list[str]
+            provide a list with IDs to download data series from FRED (link: https://fred.stlouisfed.org/tags/series?t=id). Default list is
+            ["GPDIC1", "Y057RX1Q020SBEA", "GCEC1"]
+
+        Returns
+        -------
+        pd.DataFrame
+            Return a data frame with FRED data according to the series IDs provided
+
+        """
+        # Fred API
+        fred = Fred(api_key="76f5f8156145fdb8fbaf66f1eb944f8a")
+
+        # Fetch the metadata for each series to get the full names
+        series_names = {
+            series_id: fred.get_series_info(series_id).title
+            for series_id in series_id_list
+        }
+
+        # Download data from series id list
+        fred_series = {
+            series_id: fred.get_series(series_id) for series_id in series_id_list
+        }
+
+        # Data processing
+        date_range = {
+            "OBS": pd.date_range(
+                "1950-01-01",
+                datetime.today().strftime("%Y-%m-%d"),
+                freq="d",
+            ),
+        }
+        fred_series_df = pd.DataFrame(date_range)
+
+        for series_id, series_data in fred_series.items():
+            series_data = series_data.reset_index()
+            series_data.columns = [
+                "OBS",
+                series_names[series_id],
+            ]  # Use the series name as the column header
+            fred_series_df = pd.merge_asof(
+                fred_series_df,
+                series_data,
+                on="OBS",
+                direction="backward",
+            )
+
+        # Handle duplicate columns
+        for col in fred_series_df.columns:
+            if "_x" in col:
+                base_col = col.replace("_x", "")
+                fred_series_df[base_col] = fred_series_df[col].combine_first(
+                    fred_series_df[base_col + "_y"],
+                )
+                fred_series_df.drop([col, base_col + "_y"], axis=1, inplace=True)
+
+        # Ensure sum_columns are present in the DataFrame
+        sum_columns = [
+            series_names[series_id]
+            for series_id in series_id_list
+            if series_names[series_id] in fred_series_df.columns
+        ]
+
+        # Aggregate results by week
+        fred_df_final = ims_proc.aggregate_daily_to_wc_wide(
+            df=fred_series_df,
+            date_column="OBS",
+            group_columns=[],
+            sum_columns=sum_columns,
+            wc=week_commencing,
+            aggregation="average",
+        )
+
+        # Remove anything after the instance of any ':' in the column names and rename, except for 'OBS'
+        fred_df_final.columns = [
+            "OBS"
+            if col == "OBS"
+            else "macro_" + col.lower().split(":")[0].replace(" ", "_")
+            for col in fred_df_final.columns
+        ]
+
+        return fred_df_final
+
+    def pull_boe_data(self, week_commencing="mon", max_retries=5, delay=5):
+        """
+        Fetch and process Bank of England interest rate data.
+
+        Args:
+            week_commencing (str): The starting day of the week for aggregation.
+                                Options are "mon", "tue", "wed", "thu", "fri", "sat", "sun".
+                                Default is "mon".
+            max_retries (int): Maximum number of retries to fetch data in case of failure. Default is 5.
+            delay (int): Delay in seconds between retry attempts. Default is 5.
+
+        Returns:
+            pd.DataFrame: A DataFrame with weekly aggregated Bank of England interest rates.
+                        The 'OBS' column contains the week commencing dates in 'dd/mm/yyyy' format
+                        and 'macro_boe_intr_rate' contains the average interest rate for the week.
+
+        """
+        # Week commencing dictionary
+        day_dict = {
+            "mon": 0,
+            "tue": 1,
+            "wed": 2,
+            "thu": 3,
+            "fri": 4,
+            "sat": 5,
+            "sun": 6,
+        }
+
+        # URL of the Bank of England data page
+        url = "https://www.bankofengland.co.uk/boeapps/database/Bank-Rate.asp"
+
+        # Retry logic for HTTP request
+        for attempt in range(max_retries):
+            try:
+                # Set up headers to mimic a browser request
+                headers = {
+                    "User-Agent": (
+                        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
+                        "Chrome/91.0.4472.124 Safari/537.36"
+                    ),
+                }
+                response = requests.get(url, headers=headers)
+                response.raise_for_status()  # Raise an exception for HTTP errors
+                break
+            except requests.exceptions.RequestException as e:
+                print(f"Attempt {attempt + 1} failed: {e}")
+                if attempt < max_retries - 1:
+                    time.sleep(delay)
+                else:
+                    raise
+
+        # Parse the HTML page
+        soup = BeautifulSoup(response.content, "html.parser")
+
+        # Find the table on the page
+        table = soup.find("table")  # Locate the first table
+        table_html = str(table)  # Convert table to string
+        df = pd.read_html(StringIO(table_html))[
+            0
+        ]  # Use StringIO to wrap the table HTML
+
+        # Rename and clean up columns
+        df.rename(
+            columns={"Date Changed": "OBS", "Rate": "macro_boe_intr_rate"},
+            inplace=True,
+        )
+        df["OBS"] = pd.to_datetime(df["OBS"], format="%d %b %y")
+        df.sort_values("OBS", inplace=True)
+
+        # Create a daily date range
+        date_range = pd.date_range(df["OBS"].min(), datetime.today(), freq="D")
+        df_daily = pd.DataFrame(date_range, columns=["OBS"])
+
+        # Adjust each date to the specified week commencing day
+        df_daily["Week_Commencing"] = df_daily["OBS"].apply(
+            lambda x: x - timedelta(days=(x.weekday() - day_dict[week_commencing]) % 7),
+        )
+
+        # Merge and forward-fill missing rates
+        df_daily = df_daily.merge(df, on="OBS", how="left")
+        df_daily["macro_boe_intr_rate"] = df_daily["macro_boe_intr_rate"].ffill()
+
+        # Group by week commencing and calculate the average rate
+        df_final = (
+            df_daily.groupby("Week_Commencing")["macro_boe_intr_rate"]
+            .mean()
+            .reset_index()
+        )
+        df_final["Week_Commencing"] = df_final["Week_Commencing"].dt.strftime(
+            "%d/%m/%Y",
+        )
+        df_final.rename(columns={"Week_Commencing": "OBS"}, inplace=True)
+
+        return df_final
+
+    def pull_oecd(
+        self,
+        country: str = "GBR",
+        week_commencing: str = "mon",
+        start_date: str = "2020-01-01",
+    ) -> pd.DataFrame:
+        """
+        Fetch and process time series data from the OECD API.
+
+        Args:
+            country (list): A string containing a 3-letter code the of country of interest (E.g: "GBR", "FRA", "USA", "DEU")
+            week_commencing (str): The starting day of the week for aggregation.
+                                Options are "mon", "tue", "wed", "thu", "fri", "sat", "sun".
+            start_date (str): Dataset start date in the format "YYYY-MM-DD"
+
+        Returns:
+            pd.DataFrame: A DataFrame with weekly aggregated OECD data. The 'OBS' column contains the week
+                        commencing dates, and other columns contain the aggregated time series values.
+
+        """
+
+        def parse_quarter(date_str):
+            """Parses a string in 'YYYY-Q#' format into a datetime object."""
+            year, quarter = date_str.split("-")
+            quarter_number = int(quarter[1])
+            month = (quarter_number - 1) * 3 + 1
+            return pd.Timestamp(f"{year}-{month:02d}-01")
+
+        # Generate a date range from 1950-01-01 to today
+        date_range = pd.date_range(start=start_date, end=datetime.today(), freq="D")
+
+        url_details = [
+            [
+                "BCICP",
+                "SDD.STES,DSD_STES@DF_CLI,",
+                ".....",
+                "macro_business_confidence_index",
+            ],
+            [
+                "CCICP",
+                "SDD.STES,DSD_STES@DF_CLI,",
+                ".....",
+                "macro_consumer_confidence_index",
+            ],
+            [
+                "N.CPI",
+                "SDD.TPS,DSD_PRICES@DF_PRICES_ALL,",
+                "PA._T.N.GY",
+                "macro_cpi_total",
+            ],
+            [
+                "N.CPI",
+                "SDD.TPS,DSD_PRICES@DF_PRICES_ALL,",
+                "PA.CP041T043.N.GY",
+                "macro_cpi_housing",
+            ],
+            [
+                "N.CPI",
+                "SDD.TPS,DSD_PRICES@DF_PRICES_ALL,",
+                "PA.CP01.N.GY",
+                "macro_cpi_food",
+            ],
+            [
+                "N.CPI",
+                "SDD.TPS,DSD_PRICES@DF_PRICES_ALL,",
+                "PA.CP045_0722.N.GY",
+                "macro_cpi_energy",
+            ],
+            [
+                "UNE_LF_M",
+                "SDD.TPS,DSD_LFS@DF_IALFS_UNE_M,",
+                "._Z.Y._T.Y_GE15.",
+                "macro_unemployment_rate",
+            ],
+            [
+                "EAR",
+                "SDD.TPS,DSD_EAR@DF_HOU_EAR,",
+                ".Y..S1D",
+                "macro_private_hourly_earnings",
+            ],
+            [
+                "RHP",
+                "ECO.MPD,DSD_AN_HOUSE_PRICES@DF_HOUSE_PRICES,1.0",
+                "",
+                "macro_real_house_prices",
+            ],
+            [
+                "PRVM",
+                "SDD.STES,DSD_KEI@DF_KEI,4.0",
+                "IX.C..",
+                "macro_manufacturing_production_volume",
+            ],
+            [
+                "TOVM",
+                "SDD.STES,DSD_KEI@DF_KEI,4.0",
+                "IX...",
+                "macro_retail_trade_volume",
+            ],
+            ["IRSTCI", "SDD.STES,DSD_KEI@DF_KEI,4.0", "PA...", "macro_interbank_rate"],
+            [
+                "IRLT",
+                "SDD.STES,DSD_KEI@DF_KEI,4.0",
+                "PA...",
+                "macro_long_term_interest_rate",
+            ],
+            [
+                "B1GQ",
+                "SDD.NAD,DSD_NAMAIN1@DF_QNA,1.1",
+                "._Z....GY.T0102",
+                "macro_gdp_growth_yoy",
+            ],
+        ]
+
+        # Create empty final dataframe
+        oecd_df_final = pd.DataFrame()
+
+        daily_df = pd.DataFrame({"OBS": date_range})
+        value_columns = []
+
+        # Iterate for each variable of interest
+        for series_details in url_details:
+            series = series_details[0]
+            dataset_id = series_details[1]
+            filter = series_details[2]
+            col_name = series_details[3]
+
+            # check if request was successful and determine the most granular data available
+            for freq in ["M", "Q", "A"]:
+                if series in ["UNE_LF_M", "EAR"]:
+                    data_url = f"https://sdmx.oecd.org/public/rest/data/OECD.{dataset_id}/{country}.{series}.{filter}.{freq}?startPeriod=1950-01"
+                elif series in ["B1GQ"]:
+                    data_url = f"https://sdmx.oecd.org/public/rest/data/OECD.{dataset_id}/{freq}..{country}...{series}.{filter}?startPeriod=1950-01"
+                else:
+                    data_url = f"https://sdmx.oecd.org/public/rest/data/OECD.{dataset_id}/{country}.{freq}.{series}.{filter}?startPeriod=1950-01"
+
+                # Make the request to the OECD API for data
+                data_response = requests.get(data_url)
+
+                # Check if the request was successful
+                if data_response.status_code != 200:
+                    print(
+                        f"Failed to fetch data for series {series} with frequency '{freq}' for {country}: {data_response.status_code} {data_response.text}",
+                    )
+                    url_test = False
+                    continue
+                url_test = True
+                break
+
+            # get data for the next variable if url doesn't exist
+            if url_test is False:
+                continue
+
+            root = ET.fromstring(data_response.content)
+
+            # Define namespaces if necessary (the namespace is included in the tags)
+            namespaces = {
+                "generic": "http://www.sdmx.org/resources/sdmxml/schemas/v2_1/data/generic",
+            }
+
+            # Lists to store the data
+            dates = []
+            values = []
+
+            # Iterate over all <Obs> elements and extract date and value
+            for obs in root.findall(".//generic:Obs", namespaces):
+                # Extracting the time period (date)
+                time_period = obs.find(".//generic:ObsDimension", namespaces).get(
+                    "value",
+                )
+
+                # Extracting the observation value
+                value = obs.find(".//generic:ObsValue", namespaces).get("value")
+
+                # Storing the data
+                if time_period and value:
+                    dates.append(time_period)
+                    values.append(float(value))  # Convert value to float
+
+            # Add variable names that were found to a list
+            value_columns.append(col_name)
+
+            # Creating a DataFrame
+            data = pd.DataFrame({"OBS": dates, col_name: values})
+
+            # Convert date strings into datetime format
+            if freq == "Q":
+                data["OBS"] = data["OBS"].apply(parse_quarter)
+            else:
+                # Display the DataFrame
+                data["OBS"] = data["OBS"].apply(lambda x: datetime.strptime(x, "%Y-%m"))
+
+            # Sort data by chronological order
+            data.sort_values(by="OBS", inplace=True)
+
+            # Merge the data based on the observation date
+            daily_df = pd.merge_asof(
+                daily_df,
+                data[["OBS", col_name]],
+                on="OBS",
+                direction="backward",
+            )
+
+        # Ensure columns are numeric
+        for col in value_columns:
+            if col in daily_df.columns:
+                daily_df[col] = pd.to_numeric(daily_df[col], errors="coerce").fillna(0)
+            else:
+                print(f"Column {col} not found in daily_df")
+
+        # Aggregate results by week
+        country_df = ims_proc.aggregate_daily_to_wc_wide(
+            df=daily_df,
+            date_column="OBS",
+            group_columns=[],
+            sum_columns=value_columns,
+            wc=week_commencing,
+            aggregation="average",
+        )
+
+        oecd_df_final = pd.concat(
+            [oecd_df_final, country_df],
+            axis=0,
+            ignore_index=True,
+        )
+
+        return oecd_df_final
+
+    def get_google_mobility_data(
+        self,
+        country="United Kingdom",
+        wc="mon",
+    ) -> pd.DataFrame:
+        """
+        Fetch Google Mobility data for the specified country.
+
+        Parameters
+        ----------
+        - country (str): The name of the country for which to fetch data.
+
+        Returns
+        -------
+        - pd.DataFrame: A DataFrame containing the Google Mobility data.
+
+        """
+        # URL of the Google Mobility Reports CSV file
+        url = "https://www.gstatic.com/covid19/mobility/Global_Mobility_Report.csv"
+
+        # Fetch the CSV file
+        response = requests.get(url)
+        if response.status_code != 200:
+            raise Exception(f"Failed to fetch data: {response.status_code}")
+
+        # Load the CSV file into a pandas DataFrame
+        csv_data = StringIO(response.text)
+        df = pd.read_csv(csv_data, low_memory=False)
+
+        # Filter the DataFrame for the specified country
+        country_df = df[df["country_region"] == country]
+
+        final_covid = ims_proc.aggregate_daily_to_wc_wide(
+            country_df,
+            "date",
+            [],
+            [
+                "retail_and_recreation_percent_change_from_baseline",
+                "grocery_and_pharmacy_percent_change_from_baseline",
+                "parks_percent_change_from_baseline",
+                "transit_stations_percent_change_from_baseline",
+                "workplaces_percent_change_from_baseline",
+                "residential_percent_change_from_baseline",
+            ],
+            wc,
+            "average",
+        )
+
+        final_covid1 = ims_proc.rename_cols(final_covid, "covid_")
+        return final_covid1
+
+    ###############################################################  Seasonality  ##########################################################################
+
+    def pull_seasonality(self, week_commencing, start_date, countries):
+        """
+        Generates a DataFrame with weekly seasonality features.
+
+        Args:
+            week_commencing (str): The starting day of the week ('mon', 'tue', ..., 'sun').
+            start_date (str): The start date in 'YYYY-MM-DD' format.
+            countries (list): A list of country codes (e.g., ['GB', 'US']) for holidays.
+
+        Returns:
+            pd.DataFrame: A DataFrame indexed by week start date, containing various
+                        seasonal dummy variables, holidays, trend, and constant.
+                        The date column is named 'OBS'.
+
+        """
+        # ---------------------------------------------------------------------
+        # 0. Setup: dictionary for 'week_commencing' to Python weekday() integer
+        # ---------------------------------------------------------------------
+        day_dict = {
+            "mon": 0,
+            "tue": 1,
+            "wed": 2,
+            "thu": 3,
+            "fri": 4,
+            "sat": 5,
+            "sun": 6,
+        }
+        if week_commencing not in day_dict:
+            raise ValueError(
+                f"Invalid week_commencing value: {week_commencing}. Use one of {list(day_dict.keys())}",
+            )
+
+        # ---------------------------------------------------------------------
+        # 0.2 Setup: dictionary continents and countries
+        # ---------------------------------------------------------------------
+        COUNTRY_TO_CONTINENT = {
+            # Europe
+            "Austria": "europe",
+            "Belarus": "europe",
+            "Belgium": "europe",
+            "Bulgaria": "europe",
+            "Croatia": "europe",
+            "Cyprus": "europe",
+            "Czechia": "europe",
+            "CzechRepublic": "europe",
+            "Denmark": "europe",
+            "Estonia": "europe",
+            "EuropeanCentralBank": "europe",
+            "Finland": "europe",
+            "France": "europe",
+            "FranceAlsaceMoselle": "europe",
+            "Germany": "europe",
+            "GermanyBaden": "europe",
+            "GermanyBavaria": "europe",
+            "GermanyBerlin": "europe",
+            "GermanyBrandenburg": "europe",
+            "GermanyBremen": "europe",
+            "GermanyHamburg": "europe",
+            "GermanyHesse": "europe",
+            "GermanyLowerSaxony": "europe",
+            "GermanyMecklenburgVorpommern": "europe",
+            "GermanyNorthRhineWestphalia": "europe",
+            "GermanyRhinelandPalatinate": "europe",
+            "GermanySaarland": "europe",
+            "GermanySaxony": "europe",
+            "GermanySaxonyAnhalt": "europe",
+            "GermanySchleswigHolstein": "europe",
+            "GermanyThuringia": "europe",
+            "Greece": "europe",
+            "Hungary": "europe",
+            "Iceland": "europe",
+            "Ireland": "europe",
+            "Italy": "europe",
+            "Latvia": "europe",
+            "Lithuania": "europe",
+            "Luxembourg": "europe",
+            "Malta": "europe",
+            "Monaco": "europe",
+            "Netherlands": "europe",
+            "Norway": "europe",
+            "Poland": "europe",
+            "Portugal": "europe",
+            "Romania": "europe",
+            "Russia": "europe",
+            "Serbia": "europe",
+            "Slovakia": "europe",
+            "Slovenia": "europe",
+            "Spain": "europe",
+            "SpainAndalusia": "europe",
+            "SpainAragon": "europe",
+            "SpainAsturias": "europe",
+            "SpainBalearicIslands": "europe",
+            "SpainBasqueCountry": "europe",
+            "SpainCanaryIslands": "europe",
+            "SpainCantabria": "europe",
+            "SpainCastileAndLeon": "europe",
+            "SpainCastillaLaMancha": "europe",
+            "SpainCatalonia": "europe",
+            "SpainExtremadura": "europe",
+            "SpainGalicia": "europe",
+            "SpainLaRioja": "europe",
+            "SpainMadrid": "europe",
+            "SpainMurcia": "europe",
+            "SpainNavarre": "europe",
+            "SpainValencia": "europe",
+            "Sweden": "europe",
+            "Switzerland": "europe",
+            "Ukraine": "europe",
+            "UnitedKingdom": "europe",
+            # Americas
+            "Argentina": "america",
+            "Barbados": "america",
+            "Brazil": "america",
+            "Canada": "america",
+            "Chile": "america",
+            "Colombia": "america",
+            "Mexico": "america",
+            "Panama": "america",
+            "Paraguay": "america",
+            "Peru": "america",
+            "UnitedStates": "usa",
+            # US States
+            "Alabama": "usa.states",
+            "Alaska": "usa.states",
+            "Arizona": "usa.states",
+            "Arkansas": "usa.states",
+            "California": "usa.states",
+            "Colorado": "usa.states",
+            "Connecticut": "usa.states",
+            "Delaware": "usa.states",
+            "DistrictOfColumbia": "usa.states",
+            "Florida": "usa.states",
+            "Georgia": "usa.states",
+            "Hawaii": "usa.states",
+            "Idaho": "usa.states",
+            "Illinois": "usa.states",
+            "Indiana": "usa.states",
+            "Iowa": "usa.states",
+            "Kansas": "usa.states",
+            "Kentucky": "usa.states",
+            "Louisiana": "usa.states",
+            "Maine": "usa.states",
+            "Maryland": "usa.states",
+            "Massachusetts": "usa.states",
+            "Michigan": "usa.states",
+            "Minnesota": "usa.states",
+            "Mississippi": "usa.states",
+            "Missouri": "usa.states",
+            "Montana": "usa.states",
+            "Nebraska": "usa.states",
+            "Nevada": "usa.states",
+            "NewHampshire": "usa.states",
+            "NewJersey": "usa.states",
+            "NewMexico": "usa.states",
+            "NewYork": "usa.states",
+            "NorthCarolina": "usa.states",
+            "NorthDakota": "usa.states",
+            "Ohio": "usa.states",
+            "Oklahoma": "usa.states",
+            "Oregon": "usa.states",
+            "Pennsylvania": "usa.states",
+            "RhodeIsland": "usa.states",
+            "SouthCarolina": "usa.states",
+            "SouthDakota": "usa.states",
+            "Tennessee": "usa.states",
+            "Texas": "usa.states",
+            "Utah": "usa.states",
+            "Vermont": "usa.states",
+            "Virginia": "usa.states",
+            "Washington": "usa.states",
+            "WestVirginia": "usa.states",
+            "Wisconsin": "usa.states",
+            "Wyoming": "usa.states",
+            # Oceania
+            "Australia": "oceania",
+            "AustraliaCapitalTerritory": "oceania",
+            "AustraliaNewSouthWales": "oceania",
+            "AustraliaNorthernTerritory": "oceania",
+            "AustraliaQueensland": "oceania",
+            "AustraliaSouthAustralia": "oceania",
+            "AustraliaTasmania": "oceania",
+            "AustraliaVictoria": "oceania",
+            "AustraliaWesternAustralia": "oceania",
+            "MarshallIslands": "oceania",
+            "NewZealand": "oceania",
+            # Asia
+            "China": "asia",
+            "HongKong": "asia",
+            "India": "asia",
+            "Israel": "asia",
+            "Japan": "asia",
+            "Kazakhstan": "asia",
+            "Malaysia": "asia",
+            "Qatar": "asia",
+            "Singapore": "asia",
+            "SouthKorea": "asia",
+            "Taiwan": "asia",
+            "Turkey": "asia",
+            "Vietnam": "asia",
+            # Africa
+            "Algeria": "africa",
+            "Angola": "africa",
+            "Benin": "africa",
+            "IvoryCoast": "africa",
+            "Kenya": "africa",
+            "Madagascar": "africa",
+            "Nigeria": "africa",
+            "SaoTomeAndPrincipe": "africa",
+            "SouthAfrica": "africa",
+        }
+
+        # Dictionary mapping ISO country codes to their corresponding workalendar country names
+        holiday_country = {
+            # Major countries with required formats
+            "GB": "UnitedKingdom",
+            "US": "UnitedStates",
+            "USA": "UnitedStates",  # Alternative code for US
+            "CA": "Canada",
+            "ZA": "SouthAfrica",
+            "FR": "France",
+            "DE": "Germany",
+            "AU": "Australia",
+            "AUS": "Australia",  # Alternative code for Australia
+            # European countries
+            "AT": "Austria",
+            "BY": "Belarus",
+            "BE": "Belgium",
+            "BG": "Bulgaria",
+            "HR": "Croatia",
+            "CY": "Cyprus",
+            "CZ": "Czechia",
+            "DK": "Denmark",
+            "EE": "Estonia",
+            "FI": "Finland",
+            "GR": "Greece",
+            "HU": "Hungary",
+            "IS": "Iceland",
+            "IE": "Ireland",
+            "IT": "Italy",
+            "LV": "Latvia",
+            "LT": "Lithuania",
+            "LU": "Luxembourg",
+            "MT": "Malta",
+            "MC": "Monaco",
+            "NL": "Netherlands",
+            "NO": "Norway",
+            "PL": "Poland",
+            "PT": "Portugal",
+            "RO": "Romania",
+            "RU": "Russia",
+            "RS": "Serbia",
+            "SK": "Slovakia",
+            "SI": "Slovenia",
+            "ES": "Spain",
+            "SE": "Sweden",
+            "CH": "Switzerland",
+            "UA": "Ukraine",
+            # Americas
+            "AR": "Argentina",
+            "BB": "Barbados",
+            "BR": "Brazil",
+            "CL": "Chile",
+            "CO": "Colombia",
+            "MX": "Mexico",
+            "PA": "Panama",
+            "PY": "Paraguay",
+            "PE": "Peru",
+            # USA States (using common abbreviations)
+            "AL": "Alabama",
+            "AK": "Alaska",
+            "AZ": "Arizona",
+            "AR": "Arkansas",
+            "CA_US": "California",
+            "CO_US": "Colorado",
+            "CT": "Connecticut",
+            "DE_US": "Delaware",
+            "DC": "DistrictOfColumbia",
+            "FL": "Florida",
+            "GA": "Georgia",
+            "HI": "Hawaii",
+            "ID": "Idaho",
+            "IL": "Illinois",
+            "IN": "Indiana",
+            "IA": "Iowa",
+            "KS": "Kansas",
+            "KY": "Kentucky",
+            "LA": "Louisiana",
+            "ME": "Maine",
+            "MD": "Maryland",
+            "MA": "Massachusetts",
+            "MI": "Michigan",
+            "MN": "Minnesota",
+            "MS": "Mississippi",
+            "MO": "Missouri",
+            "MT": "Montana",
+            "NE": "Nebraska",
+            "NV": "Nevada",
+            "NH": "NewHampshire",
+            "NJ": "NewJersey",
+            "NM": "NewMexico",
+            "NY": "NewYork",
+            "NC": "NorthCarolina",
+            "ND": "NorthDakota",
+            "OH": "Ohio",
+            "OK": "Oklahoma",
+            "OR": "Oregon",
+            "PA_US": "Pennsylvania",
+            "RI": "RhodeIsland",
+            "SC": "SouthCarolina",
+            "SD": "SouthDakota",
+            "TN": "Tennessee",
+            "TX": "Texas",
+            "UT": "Utah",
+            "VT": "Vermont",
+            "VA": "Virginia",
+            "WA": "Washington",
+            "WV": "WestVirginia",
+            "WI": "Wisconsin",
+            "WY": "Wyoming",
+            # Australia territories
+            "ACT": "AustraliaCapitalTerritory",
+            "NSW": "AustraliaNewSouthWales",
+            "NT": "AustraliaNorthernTerritory",
+            "QLD": "AustraliaQueensland",
+            "SA": "AustraliaSouthAustralia",
+            "TAS": "AustraliaTasmania",
+            "VIC": "AustraliaVictoria",
+            "WA_AU": "AustraliaWesternAustralia",
+            # Asian countries
+            "CN": "China",
+            "HK": "HongKong",
+            "IN": "India",
+            "IL": "Israel",
+            "JP": "Japan",
+            "KZ": "Kazakhstan",
+            "MY": "Malaysia",
+            "QA": "Qatar",
+            "SG": "Singapore",
+            "KR": "SouthKorea",
+            "TW": "Taiwan",
+            "TR": "Turkey",
+            "VN": "Vietnam",
+            # Other Oceania countries
+            "MH": "MarshallIslands",
+            "NZ": "NewZealand",
+            # African countries
+            "DZ": "Algeria",
+            "AO": "Angola",
+            "BJ": "Benin",
+            "CI": "IvoryCoast",
+            "KE": "Kenya",
+            "MG": "Madagascar",
+            "NG": "Nigeria",
+            "ST": "SaoTomeAndPrincipe",
+        }
+
+        # ---------------------------------------------------------------------
+        # 1. Create daily date range from start_date to today
+        # ---------------------------------------------------------------------
+        try:
+            start_dt = pd.to_datetime(start_date)
+        except ValueError:
+            raise ValueError(
+                f"Invalid start_date format: {start_date}. Use 'YYYY-MM-DD'",
+            )
+
+        end_dt = datetime.today()
+        # Ensure end date is not before start date
+        if end_dt < start_dt:
+            end_dt = start_dt + timedelta(days=1)  # Or handle as error if preferred
+
+        date_range = pd.date_range(start=start_dt, end=end_dt, freq="D")
+        df_daily = pd.DataFrame(date_range, columns=["Date"])
+
+        # ---------------------------------------------------------------------
+        # 1.1 Identify "week_start" for each daily row, based on week_commencing
+        # ---------------------------------------------------------------------
+        start_day_int = day_dict[week_commencing]
+        df_daily["week_start"] = df_daily["Date"].apply(
+            lambda x: x - pd.Timedelta(days=(x.weekday() - start_day_int) % 7),
+        )
+
+        # ---------------------------------------------------------------------
+        # 1.2 Calculate ISO week number for each DAY (for later aggregation)
+        #     Also calculate Year for each DAY to handle year transitions correctly
+        # ---------------------------------------------------------------------
+        df_daily["iso_week_daily"] = df_daily["Date"].dt.isocalendar().week.astype(int)
+        df_daily["iso_year_daily"] = df_daily["Date"].dt.isocalendar().year.astype(int)
+
+        # ---------------------------------------------------------------------
+        # 2. Build a weekly index (df_weekly_start) based on unique week_start dates
+        # ---------------------------------------------------------------------
+        df_weekly_start = (
+            df_daily[["week_start"]]
+            .drop_duplicates()
+            .sort_values("week_start")
+            .reset_index(drop=True)
+        )
+        df_weekly_start.rename(columns={"week_start": "Date"}, inplace=True)
+        df_weekly_start.set_index("Date", inplace=True)
+
+        # Create individual weekly dummies (optional, uncomment if needed)
+        dummy_columns = {}
+        for i, date_index in enumerate(df_weekly_start.index):
+            col_name = f"dum_{date_index.strftime('%Y_%m_%d')}"
+            dummy_columns[col_name] = [0] * len(df_weekly_start)
+            dummy_columns[col_name][i] = 1
+        df_dummies = pd.DataFrame(dummy_columns, index=df_weekly_start.index)
+        df_weekly_start = pd.concat([df_weekly_start, df_dummies], axis=1)
+
+        # ---------------------------------------------------------------------
+        # 3. Public holidays (daily) using WorkCalendar
+        # ---------------------------------------------------------------------
+        start_year = start_dt.year
+        end_year = end_dt.year
+        years_range = range(start_year, end_year + 1)
+
+        # Dictionary to store holiday dummies for each country
+        country_holiday_dummies = {}
+
+        for country_code in countries:
+            # Skip if country code not found in holiday_country dictionary
+            if country_code not in holiday_country:
+                print(
+                    f"Warning: Country code '{country_code}' not found in country code dictionary. Skipping.",
+                )
+                continue
+
+            country = holiday_country[country_code]
+
+            # Skip if country not found in continent lookup dictionary
+            if country not in COUNTRY_TO_CONTINENT:
+                print(
+                    f"Warning: Country '{country}' not found in continent lookup dictionary. Skipping.",
+                )
+                continue
+
+            continent = COUNTRY_TO_CONTINENT[country]
+            module_path = f"workalendar.{continent}"
+            try:
+                module = importlib.import_module(module_path)
+                calendar_class = getattr(module, country)
+                cal = calendar_class()
+            except (ImportError, AttributeError) as e:
+                print(f"Error importing calendar for {country}: {e}. Skipping.")
+                continue
+
+            # Collect holidays
+            holidays_list = []
+            for year in years_range:
+                holidays_list.extend(cal.holidays(year))
+
+            holidays_df = pd.DataFrame(holidays_list, columns=["Date", "Holiday"])
+            holidays_df["Date"] = pd.to_datetime(holidays_df["Date"])
+
+            # Filter out any holidays with "shift" or "substitute" in their name
+            holidays_df = holidays_df[
+                ~(
+                    holidays_df["Holiday"].str.lower().str.contains("shift")
+                    | holidays_df["Holiday"].str.lower().str.contains("substitute")
+                )
+            ]
+
+            # Filter by date range
+            holidays_df = holidays_df[
+                (holidays_df["Date"] >= start_dt) & (holidays_df["Date"] <= end_dt)
+            ]
+            # ---------------------------------------------------------------------
+            # 3.1 Additional Public Holidays for Canada due to poor API data
+            # ---------------------------------------------------------------------
+            if country_code == "CA":
+                # Add Canada Day (July 1st) if not already in the list
+                for year in years_range:
+                    canada_day = pd.Timestamp(f"{year}-07-01")
+                    if canada_day >= start_dt and canada_day <= end_dt:
+                        if not (
+                            (holidays_df["Date"] == canada_day)
+                            & (
+                                holidays_df["Holiday"]
+                                .str.lower()
+                                .str.contains("canada day")
+                            )
+                        ).any():
+                            holidays_df = pd.concat(
+                                [
+                                    holidays_df,
+                                    pd.DataFrame(
+                                        {
+                                            "Date": [canada_day],
+                                            "Holiday": ["Canada Day"],
+                                        },
+                                    ),
+                                ],
+                                ignore_index=True,
+                            )
+
+                # Add Labour Day (first Monday in September)
+                for year in years_range:
+                    # Get first day of September
+                    first_day = pd.Timestamp(f"{year}-09-01")
+                    # Calculate days until first Monday (Monday is weekday 0)
+                    days_until_monday = (7 - first_day.weekday()) % 7
+                    if days_until_monday == 0:  # If first day is already Monday
+                        labour_day = first_day
+                    else:
+                        labour_day = first_day + pd.Timedelta(days=days_until_monday)
+
+                    if labour_day >= start_dt and labour_day <= end_dt:
+                        if not (
+                            (holidays_df["Date"] == labour_day)
+                            & (
+                                holidays_df["Holiday"]
+                                .str.lower()
+                                .str.contains("labour day")
+                            )
+                        ).any():
+                            holidays_df = pd.concat(
+                                [
+                                    holidays_df,
+                                    pd.DataFrame(
+                                        {
+                                            "Date": [labour_day],
+                                            "Holiday": ["Labour Day"],
+                                        },
+                                    ),
+                                ],
+                                ignore_index=True,
+                            )
+
+                # Add Thanksgiving (second Monday in October)
+                for year in years_range:
+                    # Get first day of October
+                    first_day = pd.Timestamp(f"{year}-10-01")
+                    # Calculate days until first Monday
+                    days_until_monday = (7 - first_day.weekday()) % 7
+                    if days_until_monday == 0:  # If first day is already Monday
+                        first_monday = first_day
+                    else:
+                        first_monday = first_day + pd.Timedelta(days=days_until_monday)
+
+                    # Second Monday is 7 days after first Monday
+                    thanksgiving = first_monday + pd.Timedelta(days=7)
+
+                    if thanksgiving >= start_dt and thanksgiving <= end_dt:
+                        if not (
+                            (holidays_df["Date"] == thanksgiving)
+                            & (
+                                holidays_df["Holiday"]
+                                .str.lower()
+                                .str.contains("thanksgiving")
+                            )
+                        ).any():
+                            holidays_df = pd.concat(
+                                [
+                                    holidays_df,
+                                    pd.DataFrame(
+                                        {
+                                            "Date": [thanksgiving],
+                                            "Holiday": ["Thanksgiving"],
+                                        },
+                                    ),
+                                ],
+                                ignore_index=True,
+                            )
+
+            # Now process the collected holidays and add to df_daily
+            for _, row in holidays_df.iterrows():
+                holiday_date = row["Date"]
+                # Create column name without modifying original holiday names
+                holiday_name = row["Holiday"].lower().replace(" ", "_")
+
+                # Remove "_shift" or "_substitute" if they appear as standalone suffixes
+                if holiday_name.endswith("_shift"):
+                    holiday_name = holiday_name[:-6]
+                elif holiday_name.endswith("_substitute"):
+                    holiday_name = holiday_name[:-11]
+
+                column_name = f"seas_{holiday_name}_{country_code.lower()}"
+
+                if column_name not in df_daily.columns:
+                    df_daily[column_name] = 0
+
+                # Mark the specific holiday date
+                df_daily.loc[df_daily["Date"] == holiday_date, column_name] = 1
+
+                # Also mark a general holiday indicator for each country
+                holiday_indicator = f"seas_holiday_{country_code.lower()}"
+                if holiday_indicator not in df_daily.columns:
+                    df_daily[holiday_indicator] = 0
+                df_daily.loc[df_daily["Date"] == holiday_date, holiday_indicator] = 1
+
+        # ---------------------------------------------------------------------
+        # 3.1 Additional Special Days (Father's Day, Mother's Day, etc.)
+        # ---------------------------------------------------------------------
+        extra_cols = [
+            "seas_valentines_day",
+            "seas_halloween",
+            "seas_fathers_day_us_uk",  # Note: UK/US is 3rd Sun Jun, others vary
+            "seas_mothers_day_us",  # Note: US is 2nd Sun May
+            "seas_mothers_day_uk",  # Note: UK Mothering Sunday varies with Easter
+            "seas_good_friday",
+            "seas_easter_monday",
+            "seas_black_friday",  # US-centric, but globally adopted
+            "seas_cyber_monday",  # US-centric, but globally adopted
+        ]
+        for c in extra_cols:
+            if (
+                c not in df_daily.columns
+            ):  # Avoid overwriting if already created by holidays pkg
+                df_daily[c] = 0
+
+        # Helper: nth_weekday_of_month(year, month, weekday, nth)
+        def nth_weekday_of_month(year, month, weekday, nth):
+            d = datetime(year, month, 1)
+            w = d.weekday()
+            delta = (weekday - w + 7) % 7  # Ensure positive delta
+            first_weekday = d + timedelta(days=delta)
+            target_date = first_weekday + timedelta(days=7 * (nth - 1))
+            # Check if the calculated date is still in the same month
+            if target_date.month == month:
+                return target_date
+            # This can happen if nth is too large (e.g., 5th Friday)
+            # Return the last occurrence of that weekday in the month instead
+            return target_date - timedelta(days=7)
+
+        def get_good_friday(year):
+            return easter(year) - timedelta(days=2)
+
+        def get_easter_monday(year):
+            return easter(year) + timedelta(days=1)
+
+        def get_black_friday(year):
+            # US Thanksgiving is 4th Thursday in November (weekday=3)
+            thanksgiving = nth_weekday_of_month(year, 11, 3, 4)
+            return thanksgiving + timedelta(days=1)
+
+        def get_cyber_monday(year):
+            # Monday after US Thanksgiving
+            thanksgiving = nth_weekday_of_month(year, 11, 3, 4)
+            return thanksgiving + timedelta(days=4)
+
+        def get_mothering_sunday_uk(year):
+            # Fourth Sunday in Lent (3 weeks before Easter Sunday)
+            # Lent starts on Ash Wednesday, 46 days before Easter.
+            # Easter Sunday is day 0. Sunday before is -7, etc.
+            # 4th Sunday in Lent is 3 weeks before Easter.
+            return easter(year) - timedelta(days=21)
+
+        # Loop over each year in range
+        for yr in range(start_year, end_year + 1):
+            try:  # Wrap calculations in try-except for robustness
+                # Valentines = Feb 14
+                valentines_day = datetime(yr, 2, 14)
+                # Halloween = Oct 31
+                halloween_day = datetime(yr, 10, 31)
+                # Father's Day (US & UK) = 3rd Sunday (6) in June
+                fathers_day = nth_weekday_of_month(yr, 6, 6, 3)
+                # Mother's Day US = 2nd Sunday (6) in May
+                mothers_day_us = nth_weekday_of_month(yr, 5, 6, 2)
+                # Mother's Day UK (Mothering Sunday)
+                mothering_sunday = get_mothering_sunday_uk(yr)
+
+                # Good Friday, Easter Monday
+                gf = get_good_friday(yr)
+                em = get_easter_monday(yr)
+
+                # Black Friday, Cyber Monday
+                bf = get_black_friday(yr)
+                cm = get_cyber_monday(yr)
+
+                # Mark them in df_daily if in range
+                special_days_map = [
+                    (valentines_day, "seas_valentines_day"),
+                    (halloween_day, "seas_halloween"),
+                    (fathers_day, "seas_fathers_day_us_uk"),
+                    (mothers_day_us, "seas_mothers_day_us"),
+                    (mothering_sunday, "seas_mothers_day_uk"),
+                    (gf, "seas_good_friday"),
+                    (em, "seas_easter_monday"),
+                    (bf, "seas_black_friday"),
+                    (cm, "seas_cyber_monday"),
+                ]
+
+                for special_date, col in special_days_map:
+                    if (
+                        special_date is not None
+                    ):  # nth_weekday_of_month can return None edge cases
+                        special_ts = pd.Timestamp(special_date)
+                        # Only set if it's within the daily range AND column exists
+                        if (
+                            (special_ts >= df_daily["Date"].min())
+                            and (special_ts <= df_daily["Date"].max())
+                            and (col in df_daily.columns)
+                        ):
+                            df_daily.loc[df_daily["Date"] == special_ts, col] = 1
+            except Exception as e:
+                print(f"Warning: Could not calculate special days for year {yr}: {e}")
+
+        # ---------------------------------------------------------------------
+        # 4. Add daily indicators for last day & last Friday of month & payday
+        # ---------------------------------------------------------------------
+        df_daily["is_last_day_of_month"] = df_daily["Date"].dt.is_month_end
+
+        def is_last_friday(date):
+            # Check if it's a Friday first
+            if date.weekday() != 4:  # Friday is 4
+                return 0
+            # Check if next Friday is in the next month
+            next_friday = date + timedelta(days=7)
+            return 1 if next_friday.month != date.month else 0
+
+        def is_payday(date):
+            return 1 if date.day >= 25 else 0
+
+        df_daily["is_last_friday_of_month"] = df_daily["Date"].apply(is_last_friday)
+
+        df_daily["is_payday"] = df_daily["Date"].apply(is_payday)
+
+        # Rename for clarity prefix
+        df_daily.rename(
+            columns={
+                "is_last_day_of_month": "seas_last_day_of_month",
+                "is_last_friday_of_month": "seas_last_friday_of_month",
+                "is_payday": "seas_payday",
+            },
+            inplace=True,
+        )
+
+        # ---------------------------------------------------------------------
+        # 5. Weekly aggregation
+        # ---------------------------------------------------------------------
+
+        # Select only columns that are indicators/flags (intended for max aggregation)
+        flag_cols = [
+            col
+            for col in df_daily.columns
+            if (col.startswith("seas_") or col.startswith("is_"))
+            and col != "seas_payday"
+        ]
+        # Ensure 'week_start' is present for grouping
+        df_to_agg = df_daily[["week_start"] + flag_cols]
+
+        df_weekly_flags = (
+            df_to_agg.groupby("week_start")
+            .max()  # if any day=1 in that week, entire week=1
+            .reset_index()
+            .rename(columns={"week_start": "Date"})
+            .set_index("Date")
+        )
+
+        # Do specific aggregation for payday
+        # Make sure 'date' column exists in df_daily
+        df_daily["month"] = df_daily["Date"].dt.month
+        df_daily["year"] = df_daily["Date"].dt.year
+
+        # Sum of seas_payday flags per week
+        week_payday_sum = df_daily.groupby("week_start")["seas_payday"].sum()
+
+        # Divide the number of payday flags by number of paydays per month
+        payday_days_in_month = df_daily.groupby(["year", "month"])["seas_payday"].sum()
+        week_month = df_daily.groupby("week_start").first()[["month", "year"]]
+        week_days_in_month = week_month.apply(
+            lambda row: payday_days_in_month.loc[(row["year"], row["month"])],
+            axis=1,
+        )
+        df_weekly_flags["seas_payday"] = (
+            (week_payday_sum / week_days_in_month).fillna(0).values
+        )
+
+        # # Drop intermediate columns
+        # df_weekly_flags = df_weekly_flags.drop(columns=["month", "year"])
+
+        # --- Aggregate Week Number using MODE ---
+        # Define aggregation function for mode (handling potential multi-modal cases by taking the first)
+        def get_mode(x):
+            modes = pd.Series.mode(x)
+            return modes[0] if not modes.empty else np.nan  # Return first mode or NaN
+
+        df_weekly_iso_week_year = (
+            df_daily[["week_start", "iso_week_daily", "iso_year_daily"]]
+            .groupby("week_start")
+            .agg(
+                # Find the most frequent week number and year within the group
+                Week=("iso_week_daily", get_mode),
+                Year=("iso_year_daily", get_mode),
+            )
+            .reset_index()
+            .rename(columns={"week_start": "Date"})
+            .set_index("Date")
+        )
+        # Convert Week/Year back to integer type after aggregation
+        df_weekly_iso_week_year["Week"] = df_weekly_iso_week_year["Week"].astype(int)
+        df_weekly_iso_week_year["Year"] = df_weekly_iso_week_year["Year"].astype(int)
+
+        # --- Monthly dummies (spread evenly across week) ---
+        df_daily["Month"] = df_daily["Date"].dt.month_name().str.lower()
+        df_monthly_dummies_daily = pd.get_dummies(
+            df_daily[["week_start", "Month"]],  # Only need these columns
+            prefix="seas_month",
+            columns=["Month"],
+            dtype=float,  # Use float for division
+        )
+        # Sum daily dummies within the week
+        df_monthly_dummies_summed = df_monthly_dummies_daily.groupby("week_start").sum()
+        # Divide by number of days in that specific week group (usually 7, except potentially start/end)
+        days_in_week = df_daily.groupby("week_start").size()
+        df_weekly_monthly_dummies = df_monthly_dummies_summed.div(days_in_week, axis=0)
+
+        # Reset index to merge
+        df_weekly_monthly_dummies.reset_index(inplace=True)
+        df_weekly_monthly_dummies.rename(columns={"week_start": "Date"}, inplace=True)
+        df_weekly_monthly_dummies.set_index("Date", inplace=True)
+
+        # ---------------------------------------------------------------------
+        # 6. Combine all weekly components
+        # ---------------------------------------------------------------------
+        # Start with the basic weekly index
+        df_combined = df_weekly_start.copy()
+
+        # Join the other aggregated DataFrames
+        df_combined = df_combined.join(df_weekly_flags, how="left")
+        df_combined = df_combined.join(df_weekly_iso_week_year, how="left")
+        df_combined = df_combined.join(df_weekly_monthly_dummies, how="left")
+
+        # Fill potential NaNs created by joins (e.g., if a flag column didn't exist) with 0
+        # Exclude 'Week' and 'Year' which should always be present
+        cols_to_fill = df_combined.columns.difference(["Week", "Year"])
+        df_combined[cols_to_fill] = df_combined[cols_to_fill].fillna(0)
+
+        # Ensure correct types for flag columns (int)
+        for col in df_weekly_flags.columns:
+            if col != "seas_payday":
+                if col in df_combined.columns:
+                    df_combined[col] = df_combined[col].astype(int)
+
+        # Ensure correct types for month columns (float)
+        for col in df_weekly_monthly_dummies.columns:
+            if col in df_combined.columns:
+                df_combined[col] = df_combined[col].astype(float)
+
+        # ---------------------------------------------------------------------
+        # 7. Create weekly dummies for Week of Year & yearly dummies from aggregated cols
+        # ---------------------------------------------------------------------
+        df_combined.reset_index(inplace=True)  # 'Date', 'Week', 'Year' become columns
+
+        # Create dummies from the aggregated 'Week' column
+        df_combined = pd.get_dummies(
+            df_combined,
+            prefix="seas",
+            columns=["Week"],
+            dtype=int,
+            prefix_sep="_",
+        )
+
+        # Create dummies from the aggregated 'Year' column
+        df_combined = pd.get_dummies(
+            df_combined,
+            prefix="seas",
+            columns=["Year"],
+            dtype=int,
+            prefix_sep="_",
+        )
+
+        # ---------------------------------------------------------------------
+        # 8. Add constant & trend
+        # ---------------------------------------------------------------------
+        df_combined["Constant"] = 1
+        df_combined.reset_index(
+            drop=True,
+            inplace=True,
+        )  # Ensure index is 0, 1, 2... for trend
+        df_combined["Trend"] = df_combined.index + 1
+
+        # ---------------------------------------------------------------------
+        # 9. Rename Date -> OBS and select final columns
+        # ---------------------------------------------------------------------
+        df_combined.rename(columns={"Date": "OBS"}, inplace=True)
+
+        # Reorder columns - OBS first, then Constant, Trend, then seasonal features
+        cols_order = (
+            ["OBS", "Constant", "Trend"]
+            + sorted([col for col in df_combined.columns if col.startswith("seas_")])
+            + sorted([col for col in df_combined.columns if col.startswith("dum_")])
+        )  # If individual week dummies were enabled
+
+        # Filter out columns not in the desired order list (handles case where dum_ cols are off)
+        final_cols = [col for col in cols_order if col in df_combined.columns]
+        df_combined = df_combined[final_cols]
+
+        return df_combined
+
+    def pull_weather(self, week_commencing, start_date, country_codes) -> pd.DataFrame:
+        """
+        Pull weather data for a given week-commencing day and one or more country codes.
+        Tester
+        LOGIC:
+        1) For non-US countries (AU, GB, DE, CA, ZA):
+            - Mesonet => max_temp_f, min_temp_f -> compute mean_temp_f -> weekly average => 'avg_max_temp_f', etc.
+            - Open-Meteo => precipitation_sum => 'avg_rain_sum', snowfall_sum => 'avg_snow_sum'.
+            - Merge, then rename columns with prefix 'seas_{country}_'.
+
+        2) For the US:
+            - We have multiple <STATE>_ASOS networks (e.g. CA_ASOS, TX_ASOS).
+            - For each state, fetch from Mesonet => max_temp_f, min_temp_f, precip_in, snow_in -> compute mean_temp_f -> weekly average => 'avg_max_temp_f', 'avg_rain_sum', 'avg_snow_sum', etc.
+            - Rename columns for each state with prefix 'seas_us_{state}_'.
+            - Merge all states (and countries) into a single DataFrame.
+
+        :param week_commencing: A string in {"mon","tue","wed","thur","fri","sat","sun"}.
+        :param country_codes: A list of 2-letter country codes or a single string, e.g. ["GB","US"].
+        :return: A single Pandas DataFrame with weekly-aggregated data for all requested countries.
+        """
+        # ------------------------------------------------------------------ #
+        # 0) Handle either a single code or list of codes
+        # ------------------------------------------------------------------ #
+        if isinstance(country_codes, str):
+            country_codes = [country_codes]
+        elif not isinstance(country_codes, (list, tuple)):
+            raise ValueError("country_codes must be a list/tuple or a single string.")
+
+        # --- Setup / Constants --- #
+        day_dict = {
+            "mon": 0,
+            "tue": 1,
+            "wed": 2,
+            "thu": 3,
+            "fri": 4,
+            "sat": 5,
+            "sun": 6,
+        }
+        # Map each 2-letter code to a key
+        country_dict = {
+            "US": "US_STATES",
+            "CA": "Canada",
+            "AU": "AU__ASOS",
+            "GB": "GB__ASOS",
+            "DE": "DE__ASOS",
+            "ZA": "ZA__ASOS",
+        }
+
+        # Station-based countries for Mesonet
+        station_map = {
+            "GB__ASOS": [
+                "&stations=EGCC",
+                "&stations=EGNM",
+                "&stations=EGBB",
+                "&stations=EGSH",
+                "&stations=EGFF",
+                "&stations=EGHI",
+                "&stations=EGLC",
+                "&stations=EGHQ",
+                "&stations=EGAC",
+                "&stations=EGPF",
+                "&stations=EGGD",
+                "&stations=EGPE",
+                "&stations=EGNT",
+            ],
+            "AU__ASOS": [
+                "&stations=YPDN",
+                "&stations=YBCS",
+                "&stations=YBBN",
+                "&stations=YSSY",
+                "&stations=YSSY",
+                "&stations=YMEN",
+                "&stations=YPAD",
+                "&stations=YPPH",
+            ],
+            "DE__ASOS": [
+                "&stations=EDDL",
+                "&stations=EDDH",
+                "&stations=EDDB",
+                "&stations=EDDN",
+                "&stations=EDDF",
+                "&stations=EDDK",
+                "&stations=EDLW",
+                "&stations=EDDM",
+            ],
+            # Example: if ZA is also station-based, add it here.
+            "ZA__ASOS": [
+                # If you know the station codes, add them here:
+                # e.g. "&stations=FACT", "&stations=FAJS", ...
+            ],
+            # "FR__ASOS" if you need France, etc.
+        }
+
+        # Non-US countries that also fetch RAIN & SNOW from Open-Meteo
+        rainfall_city_map = {
+            "GB__ASOS": [
+                "Manchester",
+                "Leeds",
+                "Birmingham",
+                "London",
+                "Glasgow",
+            ],
+            "AU__ASOS": [
+                "Darwin",
+                "Cairns",
+                "Brisbane",
+                "Sydney",
+                "Melbourne",
+                "Adelaide",
+                "Perth",
+            ],
+            "DE__ASOS": [
+                "Dortmund",
+                "Düsseldorf",
+                "Frankfurt",
+                "Munich",
+                "Cologne",
+                "Berlin",
+                "Hamburg",
+                "Nuernberg",
+            ],
+            "ZA__ASOS": ["Johannesburg", "Cape Town", "Durban", "Pretoria"],
+        }
+
+        # Canada sub-networks
+        institute_vector = [
+            "CA_NB_ASOS",
+            "CA_NF_ASOS",
+            "CA_NT_ASOS",
+            "CA_NS_ASOS",
+            "CA_NU_ASOS",
+        ]
+        stations_list_canada = [
+            [
+                "&stations=CYQM",
+                "&stations=CERM",
+                "&stations=CZCR",
+                "&stations=CZBF",
+                "&stations=CYFC",
+                "&stations=CYCX",
+            ],
+            [
+                "&stations=CWZZ",
+                "&stations=CYDP",
+                "&stations=CYMH",
+                "&stations=CYAY",
+                "&stations=CWDO",
+                "&stations=CXTP",
+                "&stations=CYJT",
+                "&stations=CYYR",
+                "&stations=CZUM",
+                "&stations=CYWK",
+                "&stations=CYWK",
+            ],
+            [
+                "&stations=CYHI",
+                "&stations=CZCP",
+                "&stations=CWLI",
+                "&stations=CWND",
+                "&stations=CXTV",
+                "&stations=CYVL",
+                "&stations=CYCO",
+                "&stations=CXDE",
+                "&stations=CYWE",
+                "&stations=CYLK",
+                "&stations=CWID",
+                "&stations=CYRF",
+                "&stations=CXYH",
+                "&stations=CYWY",
+                "&stations=CWMT",
+            ],
+            [
+                "&stations=CWEF",
+                "&stations=CXIB",
+                "&stations=CYQY",
+                "&stations=CYPD",
+                "&stations=CXNP",
+                "&stations=CXMY",
+                "&stations=CYAW",
+                "&stations=CWKG",
+                "&stations=CWVU",
+                "&stations=CXLB",
+                "&stations=CWSA",
+                "&stations=CWRN",
+            ],
+            [
+                "&stations=CYLT",
+                "&stations=CWEU",
+                "&stations=CWGZ",
+                "&stations=CYIO",
+                "&stations=CXSE",
+                "&stations=CYCB",
+                "&stations=CWIL",
+                "&stations=CXWB",
+                "&stations=CYZS",
+                "&stations=CWJC",
+                "&stations=CYFB",
+                "&stations=CWUW",
+            ],
+        ]
+
+        # US states and stations - each sub-network
+        us_state_networks = {
+            state: f"{state}_ASOS"
+            for state in [
+                "AL",
+                "AR",
+                "AZ",
+                "CA",
+                "CO",
+                "CT",
+                "DE",
+                "FL",
+                "GA",
+                "IA",
+                "ID",
+                "IL",
+                "IN",
+                "KS",
+                "KY",
+                "LA",
+                "MA",
+                "MD",
+                "ME",
+                "MI",
+                "MN",
+                "MO",
+                "MS",
+                "MT",
+                "NC",
+                "ND",
+                "NE",
+                "NH",
+                "NJ",
+                "NM",
+                "NV",
+                "NY",
+                "OH",
+                "OK",
+                "OR",
+                "PA",
+                "RI",
+                "SC",
+                "SD",
+                "TN",
+                "TX",
+                "UT",
+                "VA",
+                "VT",
+                "WA",
+                "WI",
+                "WV",
+                "WY",
+            ]
+        }
+
+        us_stations_map = {
+            "AL_ASOS": [
+                "&stations=BHM",
+                "&stations=HSV",
+                "&stations=MGM",
+                "&stations=MOB",
+                "&stations=TCL",
+            ],
+            "AR_ASOS": [
+                "&stations=LIT",
+                "&stations=FSM",
+                "&stations=TXK",
+                "&stations=HOT",
+                "&stations=FYV",
+            ],
+            "AZ_ASOS": [
+                "&stations=PHX",
+                "&stations=TUS",
+                "&stations=FLG",
+                "&stations=YUM",
+                "&stations=PRC",
+            ],
+            "CA_ASOS": [
+                "&stations=LAX",
+                "&stations=SAN",
+                "&stations=SJC",
+                "&stations=SFO",
+                "&stations=FAT",
+            ],
+            "CO_ASOS": [
+                "&stations=DEN",
+                "&stations=COS",
+                "&stations=GJT",
+                "&stations=PUB",
+                "&stations=ASE",
+            ],
+            "CT_ASOS": [
+                "&stations=BDL",
+                "&stations=HVN",
+                "&stations=BDR",
+                "&stations=GON",
+                "&stations=HFD",
+            ],
+            "DE_ASOS": ["&stations=ILG", "&stations=GED", "&stations=DOV"],
+            "FL_ASOS": [
+                "&stations=MIA",
+                "&stations=TPA",
+                "&stations=ORL",
+                "&stations=JAX",
+                "&stations=TLH",
+            ],
+            "GA_ASOS": [
+                "&stations=ATL",
+                "&stations=SAV",
+                "&stations=CSG",
+                "&stations=MCN",
+                "&stations=AGS",
+            ],
+            "IA_ASOS": [
+                "&stations=DSM",
+                "&stations=CID",
+                "&stations=DBQ",
+                "&stations=ALO",
+                "&stations=SUX",
+            ],
+            "ID_ASOS": [
+                "&stations=BOI",
+                "&stations=IDA",
+                "&stations=PIH",
+                "&stations=SUN",
+                "&stations=COE",
+            ],
+            "IL_ASOS": [
+                "&stations=ORD",
+                "&stations=MDW",
+                "&stations=PIA",
+                "&stations=SPI",
+                "&stations=MLI",
+            ],
+            "IN_ASOS": [
+                "&stations=IND",
+                "&stations=FWA",
+                "&stations=SBN",
+                "&stations=EVV",
+                "&stations=HUF",
+            ],
+            "KS_ASOS": [
+                "&stations=ICT",
+                "&stations=FOE",
+                "&stations=GCK",
+                "&stations=HYS",
+                "&stations=SLN",
+            ],
+            "KY_ASOS": [
+                "&stations=SDF",
+                "&stations=LEX",
+                "&stations=CVG",
+                "&stations=PAH",
+                "&stations=BWG",
+            ],
+            "LA_ASOS": [
+                "&stations=MSY",
+                "&stations=SHV",
+                "&stations=LFT",
+                "&stations=BTR",
+                "&stations=MLU",
+            ],
+            "MA_ASOS": [
+                "&stations=BOS",
+                "&stations=ORH",
+                "&stations=HYA",
+                "&stations=ACK",
+                "&stations=BED",
+            ],
+            "MD_ASOS": [
+                "&stations=BWI",
+                "&stations=MTN",
+                "&stations=SBY",
+                "&stations=HGR",
+                "&stations=ADW",
+            ],
+            "ME_ASOS": [
+                "&stations=PWM",
+                "&stations=BGR",
+                "&stations=CAR",
+                "&stations=PQI",
+                "&stations=RKD",
+            ],
+            "MI_ASOS": [
+                "&stations=DTW",
+                "&stations=GRR",
+                "&stations=FNT",
+                "&stations=LAN",
+                "&stations=MKG",
+            ],
+            "MN_ASOS": [
+                "&stations=MSP",
+                "&stations=DLH",
+                "&stations=RST",
+                "&stations=STC",
+                "&stations=INL",
+            ],
+            "MO_ASOS": [
+                "&stations=STL",
+                "&stations=MCI",
+                "&stations=SGF",
+                "&stations=COU",
+                "&stations=JLN",
+            ],
+            "MS_ASOS": [
+                "&stations=JAN",
+                "&stations=GPT",
+                "&stations=MEI",
+                "&stations=PIB",
+                "&stations=GLH",
+            ],
+            "MT_ASOS": [
+                "&stations=BIL",
+                "&stations=MSO",
+                "&stations=GTF",
+                "&stations=HLN",
+                "&stations=BZN",
+            ],
+            "NC_ASOS": [
+                "&stations=CLT",
+                "&stations=RDU",
+                "&stations=GSO",
+                "&stations=ILM",
+                "&stations=AVL",
+            ],
+            "ND_ASOS": [
+                "&stations=BIS",
+                "&stations=FAR",
+                "&stations=GFK",
+                "&stations=ISN",
+                "&stations=JMS",
+            ],
+            "NE_ASOS": ["&stations=OMA"],
+            "NH_ASOS": [
+                "&stations=MHT",
+                "&stations=PSM",
+                "&stations=CON",
+                "&stations=LEB",
+                "&stations=ASH",
+            ],
+            "NJ_ASOS": [
+                "&stations=EWR",
+                "&stations=ACY",
+                "&stations=TTN",
+                "&stations=MMU",
+                "&stations=TEB",
+            ],
+            "NM_ASOS": [
+                "&stations=ABQ",
+                "&stations=SAF",
+                "&stations=ROW",
+                "&stations=HOB",
+                "&stations=FMN",
+            ],
+            "NV_ASOS": ["&stations=LAS"],
+            "NY_ASOS": [
+                "&stations=JFK",
+                "&stations=LGA",
+                "&stations=BUF",
+                "&stations=ALB",
+                "&stations=SYR",
+            ],
+            "OH_ASOS": ["&stations=CMH"],
+            "OK_ASOS": [
+                "&stations=OKC",
+                "&stations=TUL",
+                "&stations=LAW",
+                "&stations=SWO",
+                "&stations=PNC",
+            ],
+            "OR_ASOS": ["&stations=PDX"],
+            "PA_ASOS": [
+                "&stations=PHL",
+                "&stations=PIT",
+                "&stations=ERI",
+                "&stations=MDT",
+                "&stations=AVP",
+            ],
+            "RI_ASOS": ["&stations=PVD", "&stations=WST", "&stations=UUU"],
+            "SC_ASOS": [
+                "&stations=CHS",
+                "&stations=CAE",
+                "&stations=GSP",
+                "&stations=MYR",
+                "&stations=FLO",
+            ],
+            "SD_ASOS": [
+                "&stations=FSD",
+                "&stations=RAP",
+                "&stations=PIR",
+                "&stations=ABR",
+                "&stations=YKN",
+            ],
+            "TN_ASOS": [
+                "&stations=BNA",
+                "&stations=MEM",
+                "&stations=TYS",
+                "&stations=CHA",
+                "&stations=TRI",
+            ],
+            "TX_ASOS": [
+                "&stations=DFW",
+                "&stations=IAH",
+                "&stations=AUS",
+                "&stations=SAT",
+                "&stations=ELP",
+            ],
+            "UT_ASOS": [
+                "&stations=SLC",
+                "&stations=OGD",
+                "&stations=PVU",
+                "&stations=SGU",
+                "&stations=CNY",
+            ],
+            "VA_ASOS": [
+                "&stations=DCA",
+                "&stations=RIC",
+                "&stations=ROA",
+                "&stations=ORF",
+                "&stations=SHD",
+            ],
+            "VT_ASOS": [
+                "&stations=BTV",
+                "&stations=MPV",
+                "&stations=RUT",
+                "&stations=VSF",
+                "&stations=MVL",
+            ],
+            "WA_ASOS": [
+                "&stations=SEA",
+                "&stations=GEG",
+                "&stations=TIW",
+                "&stations=VUO",
+                "&stations=BFI",
+            ],
+            "WI_ASOS": [
+                "&stations=MKE",
+                "&stations=MSN",
+                "&stations=GRB",
+                "&stations=EAU",
+                "&stations=LSE",
+            ],
+            "WV_ASOS": [
+                "&stations=CRW",
+                "&stations=CKB",
+                "&stations=HTS",
+                "&stations=MGW",
+                "&stations=BKW",
+            ],
+            "WY_ASOS": [
+                "&stations=CPR",
+                "&stations=JAC",
+                "&stations=SHR",
+                "&stations=COD",
+                "&stations=RKS",
+            ],
+        }
+        # --- Date setup --- #
+        date_object = datetime.strptime(start_date, "%Y-%m-%d")
+        start_day = date_object.day
+        start_month = date_object.month
+        start_year = date_object.year
+        formatted_date = f"{start_year:04d}-01-01"  # "2000-01-01"
+        today = datetime.now()
+        end_day, end_month, end_year = today.day, today.month, today.year
+
+        # ------------------------------------------------------------------ #
+        # Utility functions
+        # ------------------------------------------------------------------ #
+        def convert_f_to_c(series_f: pd.Series) -> pd.Series:
+            """Convert Fahrenheit to Celsius."""
+            return (series_f - 32) * 5.0 / 9.0
+
+        def fetch_mesonet_data(network: str, stations: list) -> pd.DataFrame:
+            """Fetch station-based data (daily) from Iowa Mesonet."""
+            import csv
+
+            station_query = "".join(stations)
+            url = (
+                "https://mesonet.agron.iastate.edu/cgi-bin/request/daily.py?"
+                f"network={network}{station_query}"
+                f"&year1={start_year}&month1={start_month}&day1={start_day}"
+                f"&year2={end_year}&month2={end_month}&day2={end_day}"
+            )
+            with urllib.request.urlopen(url) as f:
+                df = pd.read_csv(f, dtype=str, quoting=csv.QUOTE_ALL)
+            return df
+
+        def fetch_canada_data() -> pd.DataFrame:
+            """Canada uses multiple sub-networks. Combine them all."""
+            import csv
+
+            final_df = pd.DataFrame()
+            for i, institute_temp in enumerate(institute_vector):
+                station_query_temp = "".join(stations_list_canada[i])
+                mesonet_url = (
+                    "https://mesonet.agron.iastate.edu/cgi-bin/request/daily.py?"
+                    f"network={institute_temp}{station_query_temp}"
+                    f"&year1={start_year}&month1={start_month}&day1={start_day}"
+                    f"&year2={end_year}&month2={end_month}&day2={end_day}"
+                )
+                with urllib.request.urlopen(mesonet_url) as f:
+                    temp_df = pd.read_csv(f, dtype=str, quoting=csv.QUOTE_ALL)
+
+                if not temp_df.empty:
+                    final_df = pd.concat([final_df, temp_df], ignore_index=True)
+            return final_df
+
+        def fetch_openmeteo_rain_snow(cities: list) -> pd.DataFrame:
+            """
+            Fetch daily precipitation_sum (rain) and snowfall_sum (snow) from Open-Meteo.
+            Returns columns: ["date", "rain_sum", "snow_sum", "city"] for each day.
+            We'll then do a weekly aggregator that yields avg_rain_sum, avg_snow_sum.
+            """
+            weather_data_list = []
+            geolocator = Nominatim(user_agent="MyApp")
+
+            for city in cities:
+                loc = geolocator.geocode(city)
+                if not loc:
+                    print(f"Could not find location for {city}, skipping.")
+                    continue
+
+                url = "https://archive-api.open-meteo.com/v1/archive"
+                params = {
+                    "latitude": loc.latitude,
+                    "longitude": loc.longitude,
+                    "start_date": formatted_date,
+                    "end_date": today.strftime("%Y-%m-%d"),
+                    "daily": "precipitation_sum,snowfall_sum",
+                    "timezone": "auto",
+                }
+                resp = requests.get(url, params=params)
+                if resp.status_code != 200:
+                    print(
+                        f"[ERROR] open-meteo returned status {resp.status_code} for city={city}",
+                    )
+                    continue
+                try:
+                    data_json = resp.json()
+                except ValueError:
+                    print(f"[ERROR] invalid JSON from open-meteo for city={city}")
+                    continue
+
+                daily_block = data_json.get("daily", {})
+                if not {"time", "precipitation_sum", "snowfall_sum"}.issubset(
+                    daily_block.keys(),
+                ):
+                    print(
+                        f"[ERROR] missing required keys in open-meteo for city={city}",
+                    )
+                    continue
+
+                df_temp = pd.DataFrame(
+                    {
+                        "date": daily_block["time"],
+                        "rain_sum": daily_block["precipitation_sum"],
+                        "snow_sum": daily_block["snowfall_sum"],
+                    },
+                )
+                df_temp["city"] = city
+                weather_data_list.append(df_temp)
+
+            if weather_data_list:
+                return pd.concat(weather_data_list, ignore_index=True)
+            return pd.DataFrame()
+
+        def weekly_aggregate_temp_mesonet(df: pd.DataFrame) -> pd.DataFrame:
+            """
+            For NON-US mesonet data, we only keep max_temp_f, min_temp_f,
+            then compute mean_temp_f, plus Celsius, and do weekly average.
+            """
+            import pandas as pd
+
+            # Convert day col
+            if "day" not in df.columns:
+                return pd.DataFrame()
+
+            # Only keep relevant columns
+            keep_cols = []
+            for c in ["day", "max_temp_f", "min_temp_f"]:
+                if c in df.columns:
+                    keep_cols.append(c)
+            df = df[keep_cols].copy()
+
+            # Convert "None" => numeric
+            for c in ["max_temp_f", "min_temp_f"]:
+                if c in df.columns:
+                    df[c] = df[c].replace("None", pd.NA)
+                    df[c] = pd.to_numeric(df[c], errors="coerce")
+
+            df["day"] = pd.to_datetime(df["day"], errors="coerce")
+            df["mean_temp_f"] = (df["max_temp_f"] + df["min_temp_f"]) / 2
+            df["max_temp_c"] = convert_f_to_c(df["max_temp_f"])
+            df["min_temp_c"] = convert_f_to_c(df["min_temp_f"])
+            df["mean_temp_c"] = convert_f_to_c(df["mean_temp_f"])
+
+            # Group by "week_starting"
+            df["week_starting"] = df["day"].apply(
+                lambda x: x
+                - pd.Timedelta(days=(x.weekday() - day_dict[week_commencing]) % 7)
+                if pd.notnull(x)
+                else pd.NaT,
+            )
+            numeric_cols = df.select_dtypes(include="number").columns
+            weekly = df.groupby("week_starting")[numeric_cols].mean()
+
+            # Rename columns
+            rename_map = {
+                "max_temp_f": "avg_max_temp_f",
+                "min_temp_f": "avg_min_temp_f",
+                "mean_temp_f": "avg_mean_temp_f",
+                "max_temp_c": "avg_max_temp_c",
+                "min_temp_c": "avg_min_temp_c",
+                "mean_temp_c": "avg_mean_temp_c",
+            }
+            weekly.rename(columns=rename_map, inplace=True)
+
+            # Return as a DataFrame w/ index = week_starting
+            return weekly
+
+        def weekly_aggregate_rain_snow_openmeteo(df: pd.DataFrame) -> pd.DataFrame:
+            """
+            For NON-US, from open-meteo, we have daily columns 'date','rain_sum','snow_sum'.
+            We'll do weekly average of each. -> 'avg_rain_sum', 'avg_snow_sum'.
+            """
+            import pandas as pd
+
+            if "date" not in df.columns:
+                return pd.DataFrame()
+
+            df["date"] = pd.to_datetime(df["date"], errors="coerce")
+            df["week_starting"] = df["date"].apply(
+                lambda x: x
+                - pd.Timedelta(days=(x.weekday() - day_dict[week_commencing]) % 7)
+                if pd.notnull(x)
+                else pd.NaT,
+            )
+
+            # Convert to numeric
+            for c in ["rain_sum", "snow_sum"]:
+                if c in df.columns:
+                    df[c] = pd.to_numeric(df[c], errors="coerce")
+
+            numeric_cols = df.select_dtypes(include="number").columns
+            weekly = df.groupby("week_starting")[numeric_cols].mean()
+
+            rename_map = {"rain_sum": "avg_rain_sum", "snow_sum": "avg_snow_sum"}
+            weekly.rename(columns=rename_map, inplace=True)
+            return weekly
+
+        def weekly_aggregate_us(df: pd.DataFrame) -> pd.DataFrame:
+            """
+            For US Mesonet data (per state), we keep max_temp_f, min_temp_f, precip_in, snow_in,
+            then compute mean_temp_f & convert to celsius, group weekly.
+            We'll rename:
+            max_temp_f -> avg_max_temp_f
+            min_temp_f -> avg_min_temp_f
+            mean_temp_f -> avg_mean_temp_f
+            precip_in -> avg_rain_sum
+            snow_in -> avg_snow_sum
+            """
+            import pandas as pd
+
+            if "day" not in df.columns:
+                return pd.DataFrame()
+
+            # Convert day
+            df["day"] = pd.to_datetime(df["day"], errors="coerce")
+
+            # Convert "None" => numeric
+            for c in ["max_temp_f", "min_temp_f", "precip_in", "snow_in"]:
+                if c in df.columns:
+                    df[c] = df[c].replace("None", pd.NA)
+                    df[c] = pd.to_numeric(df[c], errors="coerce")
+
+            # Compute mean_temp_f, celsius
+            df["mean_temp_f"] = (df["max_temp_f"] + df["min_temp_f"]) / 2
+            df["max_temp_c"] = convert_f_to_c(df["max_temp_f"])
+            df["min_temp_c"] = convert_f_to_c(df["min_temp_f"])
+            df["mean_temp_c"] = convert_f_to_c(df["mean_temp_f"])
+
+            # Weekly grouping
+            df["week_starting"] = df["day"].apply(
+                lambda x: x
+                - pd.Timedelta(days=(x.weekday() - day_dict[week_commencing]) % 7)
+                if pd.notnull(x)
+                else pd.NaT,
+            )
+            numeric_cols = df.select_dtypes(include="number").columns
+            weekly = df.groupby("week_starting")[numeric_cols].mean()
+
+            rename_map = {
+                "max_temp_f": "avg_max_temp_f",
+                "min_temp_f": "avg_min_temp_f",
+                "mean_temp_f": "avg_mean_temp_f",
+                "max_temp_c": "avg_max_temp_c",
+                "min_temp_c": "avg_min_temp_c",
+                "mean_temp_c": "avg_mean_temp_c",
+                "precip_in": "avg_rain_sum",
+                "snow_in": "avg_snow_sum",
+            }
+            weekly.rename(columns=rename_map, inplace=True)
+            return weekly
+
+        def rename_with_prefix(df: pd.DataFrame, prefix: str) -> pd.DataFrame:
+            """Rename all columns except 'week_starting' or 'OBS' with the given prefix."""
+            df2 = df.copy()
+            new_cols = {}
+            for col in df2.columns:
+                if col not in ["week_starting", "OBS"]:
+                    new_cols[col] = prefix + col
+            df2.rename(columns=new_cols, inplace=True)
+            return df2
+
+        # ------------------------------------------------------------------ #
+        # The final combined DataFrame
+        # ------------------------------------------------------------------ #
+        combined_df = pd.DataFrame()
+
+        # ------------------------------------------------------------------ #
+        # 1) Loop over each requested country
+        # ------------------------------------------------------------------ #
+        for country_code in country_codes:
+            net = country_dict.get(country_code)
+            if net is None:
+                print(f"Warning: Invalid country_code '{country_code}' – skipping.")
+                continue
+
+            # =========================
+            # 2) Special Logic for US
+            # =========================
+            if net == "US_STATES":
+                for state_code, network_code in us_state_networks.items():
+                    stations = us_stations_map.get(network_code, [])
+                    if not stations:
+                        print(f"[DEBUG] No stations for {network_code}, skipping.")
+                        continue
+
+                    raw_df = fetch_mesonet_data(network_code, stations)
+                    if raw_df.empty:
+                        print(f"[DEBUG] DataFrame empty for {network_code}, skipping.")
+                        continue
+
+                    weekly_state = weekly_aggregate_us(raw_df)
+                    if weekly_state.empty:
+                        print(
+                            f"[DEBUG] Aggregated weekly DataFrame empty for {network_code}, skipping.",
+                        )
+                        continue
+
+                    weekly_state.reset_index(inplace=True)
+                    weekly_state.rename(columns={"week_starting": "OBS"}, inplace=True)
+
+                    # Now rename columns with prefix: seas_us_{statecode}_
+                    prefix = f"seas_us_{state_code.lower()}_"
+                    weekly_state = rename_with_prefix(weekly_state, prefix)
+
+                    # Merge into combined
+                    if combined_df.empty:
+                        combined_df = weekly_state
+                    else:
+                        combined_df = pd.merge(
+                            combined_df,
+                            weekly_state,
+                            on="OBS",
+                            how="outer",
+                        )
+
+                # Done with the US. Move on to the next country in the loop
+                continue
+
+            # =======================================
+            # 3) Logic for Non-US (AU, GB, DE, CA, ZA)
+            # =======================================
+            # A) Fetch temperature data from Mesonet
+            if net == "Canada":
+                raw_temp = fetch_canada_data()
+            else:
+                # e.g. "GB__ASOS", "AU__ASOS", "DE__ASOS", "ZA__ASOS" (if added)
+                stations = station_map.get(net, [])
+                if not stations and net != "ZA__ASOS":
+                    # If we have no stations for net and it's not ZA,
+                    # there's no data. (If ZA has stations, add them above.)
+                    raw_temp = pd.DataFrame()
+                else:
+                    raw_temp = fetch_mesonet_data(net, stations)
+
+            weekly_temp = pd.DataFrame()
+            if not raw_temp.empty:
+                # For these countries, we only keep max_temp_f, min_temp_f, mean_temp_f
+                weekly_temp = weekly_aggregate_temp_mesonet(raw_temp)
+
+            # B) Fetch rain+snow from Open-Meteo (only if we have an entry in rainfall_city_map)
+            weekly_precip = pd.DataFrame()
+            if net in rainfall_city_map:
+                city_list = rainfall_city_map[net]
+                df_rain_snow = fetch_openmeteo_rain_snow(city_list)
+                if not df_rain_snow.empty:
+                    weekly_precip = weekly_aggregate_rain_snow_openmeteo(df_rain_snow)
+
+            # C) Merge the temperature data + precip/snow data on the weekly index
+            if not weekly_temp.empty and not weekly_precip.empty:
+                merged_df = pd.merge(
+                    weekly_temp,
+                    weekly_precip,
+                    left_index=True,
+                    right_index=True,
+                    how="outer",
+                )
+            elif not weekly_temp.empty:
+                merged_df = weekly_temp
+            else:
+                merged_df = weekly_precip
+
+            if merged_df.empty:
+                print(f"No data retrieved for country: {country_code}")
+                continue
+
+            # D) Convert index -> a column OBS
+            merged_df.reset_index(inplace=True)
+            merged_df.rename(columns={"week_starting": "OBS"}, inplace=True)
+
+            # E) Rename with prefix = "seas_{country_code}_"
+            prefix = f"seas_{country_code.lower()}_"
+            merged_df = rename_with_prefix(merged_df, prefix)
+
+            # F) Merge into combined_df
+            if combined_df.empty:
+                combined_df = merged_df
+            else:
+                combined_df = pd.merge(combined_df, merged_df, on="OBS", how="outer")
+
+        # ------------------------------------------------------------------ #
+        # 4) Sort final by OBS (optional)
+        # ------------------------------------------------------------------ #
+        if not combined_df.empty:
+            combined_df.sort_values(by="OBS", inplace=True)
+
+        return combined_df
+
+    def pull_macro_ons_uk(self, cdid_list=None, week_start_day="mon", sector=None):
+        """
+        Fetches time series data for multiple CDIDs from the ONS API, converts it to daily frequency,
+        aggregates it to weekly averages, and renames variables based on specified rules.
+
+        Parameters
+        ----------
+            cdid_list (list, optional): A list of additional CDIDs to fetch (e.g., ['JP9Z', 'UKPOP']). Defaults to None.
+            week_start_day (str, optional): The day the week starts on ('mon', 'tue', 'wed', 'thu', 'fri', 'sat', 'sun'). Defaults to 'mon'.
+            sector (str or list, optional): The sector(s) for which the standard CDIDs are fetched
+                                                (e.g., 'fast_food', ['fast_food', 'retail']). Defaults to None (only default CDIDs).
+
+        Returns
+        -------
+            pd.DataFrame: A DataFrame with weekly frequency, containing an 'OBS' column (week commencing date)
+                            and all series as renamed columns (e.g., 'macro_retail_sales_uk').
+                            Returns an empty DataFrame if no data is fetched or processed.
+
+        """
+        # Define CDIDs for sectors and defaults
+        sector_cdids_map = {
+            "fast_food": ["L7TD", "L78Q", "DOAD"],
+            "clothing_footwear": ["D7BW", "D7GO", "CHBJ"],
+            "fuel": ["A9FS", "L7FP", "CHOL"],
+            "cars": ["D7E8", "D7E9", "D7CO"],
+            "default": ["D7G7", "MGSX", "UKPOP", "IHYQ", "YBEZ", "MS77"],
+        }
+
+        default_cdids = sector_cdids_map["default"]
+        sector_specific_cdids = []  # Initialize empty list for sector CDIDs
+
+        if sector:  # Check if sector is not None or empty
+            if isinstance(sector, str):
+                # If it's a single string, wrap it in a list
+                sector_list = [sector]
+            elif isinstance(sector, list):
+                # If it's already a list, use it directly
+                sector_list = sector
+            else:
+                raise TypeError(
+                    "`sector` parameter must be a string or a list of strings.",
+                )
+
+            # Iterate through the list of sectors and collect their CDIDs
+            for sec in sector_list:
+                sector_specific_cdids.extend(
+                    sector_cdids_map.get(sec, []),
+                )  # Use extend to add items from the list
+
+        # Combine standard CDIDs and any additional user-provided CDIDs
+        standard_cdids = list(dict.fromkeys(default_cdids + sector_specific_cdids))
+        if cdid_list is None:
+            cdid_list = []
+        final_cdid_list = list(dict.fromkeys(standard_cdids + cdid_list))
+
+        base_search_url = (
+            "https://api.beta.ons.gov.uk/v1/search?content_type=timeseries&cdids="
+        )
+        base_data_url = "https://api.beta.ons.gov.uk/v1/data?uri="
+        combined_df = pd.DataFrame()
+
+        # Map week start day to pandas weekday convention
+        days_map = {
+            "mon": 0,
+            "tue": 1,
+            "wed": 2,
+            "thu": 3,
+            "fri": 4,
+            "sat": 5,
+            "sun": 6,
+        }
+        if week_start_day.lower() not in days_map:
+            raise ValueError(
+                "Invalid week start day. Choose from: " + ", ".join(days_map.keys()),
+            )
+        week_start = days_map[
+            week_start_day.lower()
+        ]  # Use lower() for case-insensitivity
+
+        for cdid in final_cdid_list:  # Use the final combined list
+            try:
+                # Search for the series
+                search_url = f"{base_search_url}{cdid}"
+                search_response = requests.get(search_url, timeout=30)  # Add timeout
+                search_response.raise_for_status()
+                search_data = search_response.json()
+
+                items = search_data.get("items", [])
+                if not items:
+                    print(f"Warning: No data found for CDID: {cdid}")
+                    continue
+
+                # Extract series name and latest release URI
+                # Find the item with the most recent release_date
+                latest_item = None
+                latest_date = None
+                for item in items:
+                    if "release_date" in item:
+                        try:
+                            # Ensure timezone awareness for comparison
+                            current_date = datetime.fromisoformat(
+                                item["release_date"].replace("Z", "+00:00"),
+                            )
+                            if latest_date is None or current_date > latest_date:
+                                latest_date = current_date
+                                latest_item = item
+                        except ValueError:
+                            print(
+                                f"Warning: Could not parse release_date '{item['release_date']}' for CDID {cdid}",
+                            )
+                            continue  # Skip this item if date is invalid
+
+                if latest_item is None:
+                    print(f"Warning: No valid release date found for CDID: {cdid}")
+                    continue
+
+                series_name = latest_item.get(
+                    "title",
+                    f"Series_{cdid}",
+                )  # Use title from the latest item
+                latest_uri = latest_item.get("uri")
+                if not latest_uri:
+                    print(
+                        f"Warning: No URI found for the latest release of CDID: {cdid}",
+                    )
+                    continue
+
+                # Fetch the dataset
+                data_url = f"{base_data_url}{latest_uri}"
+                data_response = requests.get(data_url, timeout=30)  # Add timeout
+                data_response.raise_for_status()
+                data_json = data_response.json()
+
+                # Detect the frequency and process accordingly
+                frequency_key = None
+                if data_json.get("months"):
+                    frequency_key = "months"
+                elif data_json.get("quarters"):
+                    frequency_key = "quarters"
+                elif data_json.get("years"):
+                    frequency_key = "years"
+                else:
+                    print(
+                        f"Warning: Unsupported frequency or no data values found for CDID: {cdid} at URI {latest_uri}",
+                    )
+                    continue
+
+                # Prepare the DataFrame
+                if not data_json[frequency_key]:  # Check if the list of values is empty
+                    print(
+                        f"Warning: Empty data list for frequency '{frequency_key}' for CDID: {cdid}",
+                    )
+                    continue
+
+                df = pd.DataFrame(data_json[frequency_key])
+
+                # Check if essential columns exist
+                if "date" not in df.columns or "value" not in df.columns:
+                    print(f"Warning: Missing 'date' or 'value' column for CDID: {cdid}")
+                    continue
+
+                # Parse the 'date' field based on frequency
+                try:
+                    if frequency_key == "months":
+                        # Handles "YYYY Mon" format (e.g., "2023 FEB") - adjust if format differs
+                        df["date"] = pd.to_datetime(
+                            df["date"],
+                            format="%Y %b",
+                            errors="coerce",
+                        )
+                    elif frequency_key == "quarters":
+
+                        def parse_quarter(quarter_str):
+                            try:
+                                year, qtr = quarter_str.split(" Q")
+                                month = {"1": 1, "2": 4, "3": 7, "4": 10}[qtr]
+                                return datetime(int(year), month, 1)
+                            except (ValueError, KeyError):
+                                return pd.NaT  # Return Not a Time for parsing errors
+
+                        df["date"] = df["date"].apply(parse_quarter)
+                    elif frequency_key == "years":
+                        df["date"] = pd.to_datetime(
+                            df["date"],
+                            format="%Y",
+                            errors="coerce",
+                        )
+                except Exception as e:
+                    print(
+                        f"Error parsing date for CDID {cdid} with frequency {frequency_key}: {e}",
+                    )
+                    continue  # Skip this series if date parsing fails
+
+                # Coerce value to numeric, handle potential errors
+                df["value"] = pd.to_numeric(df["value"], errors="coerce")
+
+                # Drop rows where date or value parsing failed
+                df.dropna(subset=["date", "value"], inplace=True)
+
+                if df.empty:
+                    print(
+                        f"Warning: No valid data points after processing for CDID: {cdid}",
+                    )
+                    continue
+
+                df.rename(columns={"value": series_name}, inplace=True)
+
+                # Combine data
+                df_subset = df.loc[:, ["date", series_name]].reset_index(
+                    drop=True,
+                )  # Explicitly select columns
+                if combined_df.empty:
+                    combined_df = df_subset
+                else:
+                    # Use outer merge to keep all dates, sort afterwards
+                    combined_df = pd.merge(
+                        combined_df,
+                        df_subset,
+                        on="date",
+                        how="outer",
+                    )
+
+            except requests.exceptions.RequestException as e:
+                print(f"Error fetching data for CDID {cdid}: {e}")
+            except (KeyError, ValueError, TypeError) as e:  # Added TypeError
+                print(f"Error processing data for CDID {cdid}: {e}")
+            except Exception as e:  # Catch unexpected errors
+                print(f"An unexpected error occurred for CDID {cdid}: {e}")
+
+        if not combined_df.empty:
+            # Sort by date after merging to ensure correct forward fill
+            combined_df.sort_values(by="date", inplace=True)
+            combined_df.reset_index(drop=True, inplace=True)
+
+            # Create a complete daily date range
+            min_date = combined_df["date"].min()
+            # Ensure max_date is timezone-naive if min_date is, or consistent otherwise
+            max_date = pd.Timestamp(
+                datetime.today().date(),
+            )  # Use today's date, timezone-naive
+
+            if pd.isna(min_date):
+                print("Error: Minimum date is NaT, cannot create date range.")
+                return pd.DataFrame()
+
+            # Make sure min_date is not NaT before creating the range
+            date_range = pd.date_range(start=min_date, end=max_date, freq="D")
+            daily_df = pd.DataFrame(date_range, columns=["date"])
+
+            # Merge with original data and forward fill
+            daily_df = pd.merge(daily_df, combined_df, on="date", how="left")
+            daily_df = daily_df.ffill()
+
+            # Drop rows before the first valid data point after ffill
+            first_valid_index = daily_df.dropna(
+                subset=daily_df.columns.difference(["date"]),
+            ).index.min()
+            if pd.notna(first_valid_index):
+                daily_df = daily_df.loc[first_valid_index:]
+            else:
+                print("Warning: No valid data points found after forward filling.")
+                return pd.DataFrame()  # Return empty if ffill results in no data
+
+            # Aggregate to weekly frequency
+            # Ensure 'date' column is datetime type before dt accessor
+            daily_df["date"] = pd.to_datetime(daily_df["date"])
+            daily_df["week_commencing"] = daily_df["date"] - pd.to_timedelta(
+                (daily_df["date"].dt.weekday - week_start + 7) % 7,
+                unit="D",
+            )  # Corrected logic for week start
+            # Group by week_commencing and calculate mean for numeric columns only
+            weekly_df = (
+                daily_df.groupby("week_commencing")
+                .mean(numeric_only=True)
+                .reset_index()
+            )
+
+            def clean_column_name(name):
+                # Remove content within parentheses
+                name = re.sub(r"\(.*?\)", "", name)
+                
+                # Special handling for ANY CPI items (not just CPI INDEX)
+                if "CPI" in name.upper():
+                    # Extract the description part after the colon for CPI items
+                    if ":" in name:
+                        parts = name.split(":")
+                        if len(parts) >= 2:
+                            # Take the description part (usually the second part)
+                            description = parts[1].strip()
+                            # Remove any remaining colons and everything after
+                            description = description.split(":")[0].strip()
+                            name = f"CPI {description}"
+                    
+                    # Remove numbers and dots for ALL CPI items (like 00, 06.2.2, 12.5.3/5)
+                    name = re.sub(r"\d+\.?\d*/?\.?\d*", "", name)
+                    
+                else:
+                    # For non-CPI items, take only the part before the first colon
+                    name = re.split(r":", name)[0]
+                    # Remove all digits for non-CPI items too
+                    name = re.sub(r"\d+", "", name)
+                
+                # Remove year references like "2015=100"
+                name = re.sub(r"\d{4}=\d+", "", name)
+                
+                # Remove specific words case-insensitively
+                name = re.sub(r"\b(annual|rate|index|seasonally|adjusted|sa|cvm)\b", "", name, flags=re.IGNORECASE)
+                
+                # Remove percentage symbols and "%" 
+                name = re.sub(r"%", "percent", name)
+                
+                # Remove non-alphanumeric characters (except underscore and space)
+                name = re.sub(r"[^\w\s]", "", name)
+                
+                # Replace spaces with underscores
+                name = name.strip().replace(" ", "_")
+                
+                # Replace multiple underscores with a single one
+                name = re.sub(r"_+", "_", name)
+                
+                # Remove leading/trailing underscores
+                name = name.strip("_")
+                
+                # Truncate very long names (optional)
+                if len(name) > 50:
+                    words = name.split("_")
+                    # Keep first few meaningful words
+                    name = "_".join(words[:4])
+                
+                return f"macro_{name.lower()}_uk"
+            
+            # Apply cleaning function to relevant columns
+            weekly_df.columns = [
+                clean_column_name(col) if col != "week_commencing" else col
+                for col in weekly_df.columns
+            ]
+            weekly_df.rename(
+                columns={"week_commencing": "OBS"},
+                inplace=True,
+            )  # Rename week commencing col
+
+            # Optional: Fill remaining NaNs (e.g., at the beginning if ffill didn't cover) with 0
+            # Consider if 0 is the appropriate fill value for your use case
+            # weekly_df = weekly_df.fillna(0)
+
+            # Get only the data columns (excluding OBS)
+            data_columns = [col for col in weekly_df.columns if col != "OBS"]
+
+            new_columns = ["OBS"]
+            for i, col in enumerate(data_columns):
+                if i < len(final_cdid_list):
+                    new_columns.append(f"{col}_{final_cdid_list[i]}")
+                else:
+                    new_columns.append(col)  # Keep original if no matching CDID
+
+            # Apply the new column names to the DataFrame
+            weekly_df.columns = new_columns
+            
+            return weekly_df
+        print("No data successfully fetched or processed.")
+        return pd.DataFrame()
+
+    def pull_yfinance(self, tickers=None, week_start_day="mon"):
+        """
+        Fetches stock data for multiple tickers from Yahoo Finance, converts it to daily frequency,
+        aggregates it to weekly averages, and renames variables.
+
+        Parameters
+        ----------
+            tickers (list): A list of additional stock tickers to fetch (e.g., ['AAPL', 'MSFT']). Defaults to None.
+            week_start_day (str): The day the week starts on (e.g., 'Monday', 'Sunday').
+
+        Returns
+        -------
+            pd.DataFrame: A DataFrame with weekly frequency, containing an 'OBS' column
+                        and aggregated stock data for the specified tickers, with NaN values filled with 0.
+
+        """
+        # Define default tickers
+        default_tickers = ["^FTSE", "GBPUSD=X", "GBPEUR=X", "^GSPC"]
+
+        # Combine default tickers with additional ones
+        if tickers is None:
+            tickers = []
+        tickers = list(set(default_tickers + tickers))  # Ensure no duplicates
+
+        # Automatically set end_date to today
+        end_date = datetime.today().strftime("%Y-%m-%d")
+
+        # Mapping week start day to pandas weekday convention
+        days_map = {
+            "mon": 0,
+            "tue": 1,
+            "wed": 2,
+            "thu": 3,
+            "fri": 4,
+            "sat": 5,
+            "sun": 6,
+        }
+        if week_start_day not in days_map:
+            raise ValueError(
+                "Invalid week start day. Choose from: " + ", ".join(days_map.keys()),
+            )
+        week_start = days_map[week_start_day]
+
+        # Fetch data for all tickers without specifying a start date to get all available data
+        data = yf.download(tickers, end=end_date, group_by="ticker", auto_adjust=True)
+
+        # Process the data
+        combined_df = pd.DataFrame()
+        for ticker in tickers:
+            try:
+                # Extract the ticker's data
+                ticker_data = data[ticker] if len(tickers) > 1 else data
+                ticker_data = ticker_data.reset_index()
+
+                # Ensure necessary columns are present
+                if "Close" not in ticker_data.columns:
+                    raise ValueError(
+                        f"Ticker {ticker} does not have 'Close' price data.",
+                    )
+
+                # Keep only relevant columns
+                ticker_data = ticker_data[["Date", "Close"]]
+                ticker_data.rename(columns={"Close": ticker}, inplace=True)
+
+                # Merge data
+                if combined_df.empty:
+                    combined_df = ticker_data
+                else:
+                    combined_df = pd.merge(
+                        combined_df,
+                        ticker_data,
+                        on="Date",
+                        how="outer",
+                    )
+
+            except KeyError:
+                print(f"Data for ticker {ticker} not available.")
+            except Exception as e:
+                print(f"Error processing ticker {ticker}: {e}")
+
+        if not combined_df.empty:
+            # Convert to daily frequency
+            combined_df["Date"] = pd.to_datetime(combined_df["Date"])
+            combined_df.set_index("Date", inplace=True)
+
+            # Fill missing dates
+            min_date = combined_df.index.min()
+            max_date = combined_df.index.max()
+            daily_index = pd.date_range(start=min_date, end=max_date, freq="D")
+            combined_df = combined_df.reindex(daily_index)
+            combined_df.index.name = "Date"
+            combined_df = combined_df.ffill()
+
+            # Aggregate to weekly frequency
+            combined_df["OBS"] = combined_df.index - pd.to_timedelta(
+                (combined_df.index.weekday - week_start) % 7,
+                unit="D",
+            )
+            weekly_df = combined_df.groupby("OBS").mean(numeric_only=True).reset_index()
+
+            # Fill NaN values with 0
+            weekly_df = weekly_df.fillna(0)
+
+            # Clean column names
+            def clean_column_name(name):
+                name = re.sub(r"[^\w\s]", "", name)
+                return f"macro_{name.lower()}"
+
+            weekly_df.columns = [
+                clean_column_name(col) if col != "OBS" else col
+                for col in weekly_df.columns
+            ]
+
+            return weekly_df
+
+        print("No data available to process.")
+        return pd.DataFrame()
+
+    def pull_sports_events(self, start_date="2020-01-01", week_commencing="mon"):
+        """
+        Combines scraping logic for:
+        - UEFA Champions League and NFL from TheSportsDB (website-scraping approach)
+        - FIFA World Cup, UEFA Euro, Rugby World Cup, Six Nations (via TheSportsDB API)
+
+        Returns a single merged DataFrame with all event dummy variables.
+        """
+
+        ############################################################
+        # 1) SCRAPE UEFA CHAMPIONS LEAGUE & NFL (YOUR FIRST FUNCTION)
+        ############################################################
+        def scrape_sports_events(
+            start_date=start_date,
+            week_commencing=week_commencing,
+        ):
+            sports = {
+                "uefa_champions_league": {
+                    "league_id": "4480",
+                    "seasons_url": "https://www.thesportsdb.com/league/4480-UEFA-Champions-League?a=1#allseasons",
+                    "season_url_template": "https://www.thesportsdb.com/season/4480-UEFA-Champions-League/{season}&all=1&view=",
+                    "round_filters": ["quarter", "semi", "final"],
+                },
+                "nfl": {
+                    "league_id": "4391",
+                    "seasons_url": "https://www.thesportsdb.com/league/4391-NFL?a=1#allseasons",
+                    "season_url_template": "https://www.thesportsdb.com/season/4391-NFL/{season}&all=1&view=",
+                    "round_filters": ["quarter", "semi", "final"],
+                },
+            }
+
+            headers = {"User-Agent": "Mozilla/5.0"}
+            start_date_dt = datetime.strptime(start_date, "%Y-%m-%d")
+
+            # Create a full date range DataFrame
+            full_date_range = pd.date_range(
+                start=start_date,
+                end=pd.to_datetime("today"),
+            )
+            time_series_df = pd.DataFrame({"date": full_date_range})
+            time_series_df["seas_uefa_champions_league"] = 0
+            time_series_df["seas_nfl"] = 0
+
+            for sport, details in sports.items():
+                # Get available seasons
+                response = requests.get(details["seasons_url"], headers=headers)
+                if response.status_code != 200:
+                    continue  # Skip this sport if the request fails
+
+                soup = BeautifulSoup(response.text, "html.parser")
+
+                # Extract season names
+                seasons = []
+                for link in soup.find_all("a", href=True):
+                    href = link["href"]
+                    if "season" in href and sport.replace("_", "-") in href.lower():
+                        season_name = href.split("/")[-1]  # e.g. "2023-2024"
+                        try:
+                            season_start_year = int(season_name.split("-")[0])
+                            season_start_date = datetime(season_start_year, 1, 1)
+                            if season_start_date >= start_date_dt:
+                                seasons.append(season_name)
+                        except ValueError:
+                            continue
+
+                # Scrape matches for filtered seasons
+                filtered_matches = []
+                for season in seasons:
+                    season_url = details["season_url_template"].format(season=season)
+                    season_response = requests.get(season_url, headers=headers)
+                    if season_response.status_code != 200:
+                        continue
+
+                    season_soup = BeautifulSoup(season_response.text, "html.parser")
+                    for row in season_soup.find_all("tr"):
+                        cols = row.find_all("td")
+                        if len(cols) >= 5:
+                            match_date = cols[0].text.strip()
+                            round_name = cols[1].text.strip().lower()
+                            try:
+                                match_date_dt = datetime.strptime(
+                                    match_date,
+                                    "%d %b %y",
+                                )
+                                if match_date_dt >= start_date_dt and any(
+                                    r in round_name for r in details["round_filters"]
+                                ):
+                                    filtered_matches.append(match_date_dt)
+                            except ValueError:
+                                continue
+
+                # Convert matches into time series format
+                df_sport = pd.DataFrame({"date": filtered_matches})
+                if df_sport.empty:
+                    continue
+
+                col_name = (
+                    "seas_nfl" if sport == "nfl" else "seas_uefa_champions_league"
+                )
+                time_series_df.loc[
+                    time_series_df["date"].isin(df_sport["date"]),
+                    col_name,
+                ] = 1
+
+            # Aggregate by week commencing
+            day_offsets = {
+                "mon": "W-MON",
+                "tue": "W-TUE",
+                "wed": "W-WED",
+                "thu": "W-THU",
+                "fri": "W-FRI",
+                "sat": "W-SAT",
+                "sun": "W-SUN",
+            }
+            if week_commencing.lower() not in day_offsets:
+                raise ValueError(
+                    f"Invalid week_commencing value: {week_commencing}. Must be one of {list(day_offsets.keys())}.",
+                )
+
+            time_series_df = (
+                time_series_df.set_index("date")
+                .resample(day_offsets[week_commencing.lower()])
+                .max()
+                .reset_index()
+            )
+
+            time_series_df.rename(columns={"date": "OBS"}, inplace=True)
+            time_series_df.fillna(0, inplace=True)
+
+            return time_series_df
+
+        ############################################################
+        # 2) FETCH FIFA WC, UEFA EURO, RUGBY, SIX NATIONS (2ND FUNC)
+        ############################################################
+        def fetch_events(start_date=start_date, week_commencing=week_commencing):
+            # Initialize date range
+            start_date_obj = datetime.strptime(start_date, "%Y-%m-%d")
+            end_date_obj = datetime.today()
+            date_range = pd.date_range(start=start_date_obj, end=end_date_obj)
+            df = pd.DataFrame({"OBS": date_range}).set_index("OBS")
+
+            # Define columns for sports
+            event_columns = {
+                "seas_fifa_world_cup": {
+                    "league_id": 4429,
+                    "start_year": 1950,
+                    "interval": 4,
+                },
+                "seas_uefa_european_championship": {
+                    "league_id": 4502,
+                    "start_year": 1960,
+                    "interval": 4,
+                    "extra_years": [2021],
+                },
+                "seas_rugby_world_cup": {
+                    "league_id": 4574,
+                    "start_year": 1987,
+                    "interval": 4,
+                },
+                "seas_six_nations": {
+                    "league_id": 4714,
+                    "start_year": 2000,
+                    "interval": 1,
+                },
+            }
+
+            # Initialize columns
+            for col in event_columns:
+                df[col] = 0
+
+            def fetch_league_events(
+                league_id,
+                column_name,
+                start_year,
+                interval,
+                extra_years=None,
+            ):
+                extra_years = extra_years or []
+                # Fetch seasons
+                seasons_url = f"https://www.thesportsdb.com/api/v1/json/3/search_all_seasons.php?id={league_id}"
+                seasons_response = requests.get(seasons_url)
+                if seasons_response.status_code != 200:
+                    return  # Skip on failure
+
+                seasons_data = seasons_response.json().get("seasons", [])
+                for season in seasons_data:
+                    season_name = season.get("strSeason", "")
+                    if not season_name.isdigit():
+                        continue
+
+                    year = int(season_name)
+                    # Check if the year is valid for this competition
+                    if year in extra_years or (
+                        year >= start_year and (year - start_year) % interval == 0
+                    ):
+                        # Fetch events
+                        events_url = f"https://www.thesportsdb.com/api/v1/json/3/eventsseason.php?id={league_id}&s={season_name}"
+                        events_response = requests.get(events_url)
+                        if events_response.status_code != 200:
+                            continue
+
+                        events_data = events_response.json().get("events", [])
+                        for event in events_data:
+                            event_date_str = event.get("dateEvent")
+                            if event_date_str:
+                                event_date = datetime.strptime(
+                                    event_date_str,
+                                    "%Y-%m-%d",
+                                )
+                                if event_date in df.index:
+                                    df.loc[event_date, column_name] = 1
+
+            # Fetch events for all defined leagues
+            for column_name, params in event_columns.items():
+                fetch_league_events(
+                    league_id=params["league_id"],
+                    column_name=column_name,
+                    start_year=params["start_year"],
+                    interval=params["interval"],
+                    extra_years=params.get("extra_years", []),
+                )
+
+            # Resample by week
+            day_offsets = {
+                "mon": "W-MON",
+                "tue": "W-TUE",
+                "wed": "W-WED",
+                "thu": "W-THU",
+                "fri": "W-FRI",
+                "sat": "W-SAT",
+                "sun": "W-SUN",
+            }
+
+            if week_commencing.lower() not in day_offsets:
+                raise ValueError(
+                    f"Invalid week_commencing value: {week_commencing}. "
+                    f"Must be one of {list(day_offsets.keys())}.",
+                )
+
+            df = df.resample(day_offsets[week_commencing.lower()]).max()
+            df = df.reset_index()
+            return df
+
+        ###################################################
+        # 3) CALL BOTH, THEN MERGE ON "OBS" & FILL WITH 0s
+        ###################################################
+        df_uefa_nfl = scrape_sports_events(start_date, week_commencing)
+        df_other_events = fetch_events(start_date, week_commencing)
+
+        # Merge on "OBS" column (outer join to preserve all dates in range)
+        final_df = pd.merge(df_uefa_nfl, df_other_events, on="OBS", how="outer")
+
+        # Fill any NaNs with 0 for event columns
+        # (Only fill numeric columns or everything except 'OBS')
+        for col in final_df.columns:
+            if col != "OBS":
+                final_df[col] = final_df[col].fillna(0)
+
+        # Sort by date just in case
+        final_df.sort_values(by="OBS", inplace=True)
+        final_df.reset_index(drop=True, inplace=True)
+
+        return final_df