imsciences 0.9.6.2__py3-none-any.whl → 0.9.6.4__py3-none-any.whl

imsciences/pull.py

@@ -380,265 +380,368 @@ class datapull:
     ###############################################################  Seasonality  ##########################################################################
 
     def pull_seasonality(self, week_commencing, start_date, countries):
-        # ---------------------------------------------------------------------
-        # 0. Setup: dictionary for 'week_commencing' to Python weekday() integer
-        # ---------------------------------------------------------------------
-        day_dict = {"mon": 0, "tue": 1, "wed": 2, "thu": 3, "fri": 4, "sat": 5, "sun": 6}
-
-        # ---------------------------------------------------------------------
-        # 1. Create daily date range from start_date to today
-        # ---------------------------------------------------------------------
-        date_range = pd.date_range(
-            start=pd.to_datetime(start_date), 
-            end=datetime.today(), 
-            freq="D"
-        )
-        df_daily = pd.DataFrame(date_range, columns=["Date"])
+            """
+            Generates a DataFrame with weekly seasonality features.
 
-        # ---------------------------------------------------------------------
-        # 1.1 Identify "week_start" for each daily row, based on week_commencing
-        # ---------------------------------------------------------------------
-        df_daily['week_start'] = df_daily["Date"].apply(
-            lambda x: x - pd.Timedelta(days=(x.weekday() - day_dict[week_commencing]) % 7)
-        )
+            Args:
+                week_commencing (str): The starting day of the week ('mon', 'tue', ..., 'sun').
+                start_date (str): The start date in 'YYYY-MM-DD' format.
+                countries (list): A list of country codes (e.g., ['GB', 'US']) for holidays.
 
-        # ---------------------------------------------------------------------
-        # 2. Build a weekly index (df_weekly_start) with dummy columns
-        # ---------------------------------------------------------------------
-        df_weekly_start = df_daily[['week_start']].drop_duplicates().reset_index(drop=True)
-        df_weekly_start.rename(columns={'week_start': "Date"}, inplace=True)
-        
-        # Set index to weekly "start of week"
-        df_weekly_start.index = np.arange(1, len(df_weekly_start) + 1)
-        df_weekly_start.set_index("Date", inplace=True)
-
-        # Create individual weekly dummies
-        dummy_columns = {}
-        for i in range(len(df_weekly_start)):
-            col_name = f"dum_{df_weekly_start.index[i].strftime('%Y_%m_%d')}"
-            dummy_columns[col_name] = [0] * len(df_weekly_start)
-            dummy_columns[col_name][i] = 1
-
-        df_dummies = pd.DataFrame(dummy_columns, index=df_weekly_start.index)
-        df_weekly_start = pd.concat([df_weekly_start, df_dummies], axis=1)
-
-        # ---------------------------------------------------------------------
-        # 3. Public holidays (daily) from 'holidays' package + each holiday name
-        # ---------------------------------------------------------------------
-        for country in countries:
-            country_holidays = holidays.CountryHoliday(
-                country, 
-                years=range(int(start_date[:4]), datetime.today().year + 1)
+            Returns:
+                pd.DataFrame: A DataFrame indexed by week start date, containing various
+                            seasonal dummy variables, holidays, trend, and constant.
+                            The date column is named 'OBS'.
+            """
+            # ---------------------------------------------------------------------
+            # 0. Setup: dictionary for 'week_commencing' to Python weekday() integer
+            # ---------------------------------------------------------------------
+            day_dict = {"mon": 0, "tue": 1, "wed": 2, "thu": 3, "fri": 4, "sat": 5, "sun": 6}
+            if week_commencing not in day_dict:
+                raise ValueError(f"Invalid week_commencing value: {week_commencing}. Use one of {list(day_dict.keys())}")
+
+            # ---------------------------------------------------------------------
+            # 1. Create daily date range from start_date to today
+            # ---------------------------------------------------------------------
+            try:
+                start_dt = pd.to_datetime(start_date)
+            except ValueError:
+                raise ValueError(f"Invalid start_date format: {start_date}. Use 'YYYY-MM-DD'")
+
+            end_dt = datetime.today()
+            # Ensure end date is not before start date
+            if end_dt < start_dt:
+                end_dt = start_dt + timedelta(days=1) # Or handle as error if preferred
+
+            date_range = pd.date_range(
+                start=start_dt,
+                end=end_dt,
+                freq="D"
             )
-            # Daily indicator: 1 if that date is a holiday
-            df_daily[f"seas_holiday_{country.lower()}"] = df_daily["Date"].apply(
-                lambda x: 1 if x in country_holidays else 0
+            df_daily = pd.DataFrame(date_range, columns=["Date"])
+
+            # ---------------------------------------------------------------------
+            # 1.1 Identify "week_start" for each daily row, based on week_commencing
+            # ---------------------------------------------------------------------
+            start_day_int = day_dict[week_commencing]
+            df_daily['week_start'] = df_daily["Date"].apply(
+                lambda x: x - pd.Timedelta(days=(x.weekday() - start_day_int) % 7)
             )
-            # Create columns for specific holiday names
-            for date_hol, name in country_holidays.items():
-                col_name = f"seas_{name.replace(' ', '_').lower()}_{country.lower()}"
-                if col_name not in df_daily.columns:
-                    df_daily[col_name] = 0
-                df_daily.loc[df_daily["Date"] == pd.Timestamp(date_hol), col_name] = 1
-
-        # ---------------------------------------------------------------------
-        # 3.1 Additional Special Days (Father's Day, Mother's Day, etc.)
-        #     We'll add daily columns for each. 
-        # ---------------------------------------------------------------------
-        # Initialize columns
-        extra_cols = [
-            "seas_valentines_day", 
-            "seas_halloween", 
-            "seas_fathers_day_us_uk",
-            "seas_mothers_day_us",
-            "seas_mothers_day_uk",
-            "seas_good_friday",
-            "seas_easter_monday",
-            "seas_black_friday",
-            "seas_cyber_monday",
-        ]
-        for c in extra_cols:
-            df_daily[c] = 0  # default zero
-
-        # Helper: nth_weekday_of_month(year, month, weekday, nth=1 => first, 2 => second, etc.)
-        # weekday: Monday=0, Tuesday=1, ... Sunday=6
-        def nth_weekday_of_month(year, month, weekday, nth):
-            """
-            Returns date of the nth <weekday> in <month> of <year>.
-            E.g. nth_weekday_of_month(2023, 6, 6, 3) => 3rd Sunday of June 2023.
-            """
-            # 1st day of the month
-            d = datetime(year, month, 1)
-            # What is the weekday of day #1?
-            w = d.weekday()  # Monday=0, Tuesday=1, ... Sunday=6
-            # If we want, e.g. Sunday=6, we see how many days to add
-            delta = (weekday - w) % 7
-            # This is the first <weekday> in that month
-            first_weekday = d + timedelta(days=delta)
-            # Now add 7*(nth-1) days
-            return first_weekday + timedelta(days=7 * (nth-1))
-
-        def get_good_friday(year):
-            """Good Friday is 2 days before Easter Sunday."""
-            return easter(year) - timedelta(days=2)
-
-        def get_easter_monday(year):
-            """Easter Monday is 1 day after Easter Sunday."""
-            return easter(year) + timedelta(days=1)
-
-        def get_black_friday(year):
-            """
-            Black Friday = day after US Thanksgiving, 
-            and US Thanksgiving is the 4th Thursday in November.
-            """
-            # 4th Thursday in November
-            fourth_thursday = nth_weekday_of_month(year, 11, 3, 4)  # weekday=3 => Thursday
-            return fourth_thursday + timedelta(days=1)
-
-        def get_cyber_monday(year):
-            """Cyber Monday = Monday after US Thanksgiving, i.e. 4 days after 4th Thursday in Nov."""
-            # 4th Thursday in November
-            fourth_thursday = nth_weekday_of_month(year, 11, 3, 4)
-            return fourth_thursday + timedelta(days=4)  # Monday after Thanksgiving
-
-        # Loop over each year in range
-        start_yr = int(start_date[:4])
-        end_yr = datetime.today().year
-
-        for yr in range(start_yr, end_yr + 1):
-            # Valentines = Feb 14
-            valentines_day = datetime(yr, 2, 14)
-            # Halloween = Oct 31
-            halloween_day  = datetime(yr, 10, 31)
-            # Father's Day (US & UK) = 3rd Sunday in June
-            fathers_day    = nth_weekday_of_month(yr, 6, 6, 3)  # Sunday=6
-            # Mother's Day US = 2nd Sunday in May
-            mothers_day_us = nth_weekday_of_month(yr, 5, 6, 2)
-            mothering_sunday = easter(yr) - timedelta(days=21)
-            # If for some reason that's not a Sunday (rare corner cases), shift to Sunday:
-            while mothering_sunday.weekday() != 6:  # Sunday=6
-                mothering_sunday -= timedelta(days=1)
-
-            # Good Friday, Easter Monday
-            gf = get_good_friday(yr)
-            em = get_easter_monday(yr)
-
-            # Black Friday, Cyber Monday
-            bf = get_black_friday(yr)
-            cm = get_cyber_monday(yr)
-
-            # Mark them in df_daily if in range
-            for special_date, col in [
-                (valentines_day, "seas_valentines_day"),
-                (halloween_day,  "seas_halloween"),
-                (fathers_day,    "seas_fathers_day_us_uk"),
-                (mothers_day_us, "seas_mothers_day_us"),
-                (mothering_sunday, "seas_mothers_day_uk"),
-                (gf, "seas_good_friday"),
-                (em, "seas_easter_monday"),
-                (bf, "seas_black_friday"),
-                (cm, "seas_cyber_monday"),
-            ]:
-                # Convert to pd.Timestamp:
-                special_ts = pd.Timestamp(special_date)
-
-                # Only set if it's within your daily range
-                if (special_ts >= df_daily["Date"].min()) and (special_ts <= df_daily["Date"].max()):
-                    df_daily.loc[df_daily["Date"] == special_ts, col] = 1
-
-        # ---------------------------------------------------------------------
-        # 4. Add daily indicators for last day & last Friday of month
-        #    Then aggregate them to weekly level using .max()
-        # ---------------------------------------------------------------------
-        # Last day of month (daily)
-        df_daily["seas_last_day_of_month"] = df_daily["Date"].apply(
-            lambda d: 1 if d == d.to_period("M").to_timestamp("M") else 0
-        )
-
-        # Last Friday of month (daily)
-        def is_last_friday(date):
-            # last day of the month
-            last_day_of_month = date.to_period("M").to_timestamp("M")
-            last_day_weekday = last_day_of_month.weekday()  # Monday=0,...Sunday=6
-            # Determine how many days we go back from the last day to get Friday (weekday=4)
-            if last_day_weekday >= 4:
-                days_to_subtract = last_day_weekday - 4
-            else:
-                days_to_subtract = last_day_weekday + 3
-            last_friday = last_day_of_month - pd.Timedelta(days=days_to_subtract)
-            return 1 if date == last_friday else 0
-
-        df_daily["seas_last_friday_of_month"] = df_daily["Date"].apply(is_last_friday)
-
-        # ---------------------------------------------------------------------
-        # 5. Weekly aggregation for holiday columns & monthly dummies
-        # ---------------------------------------------------------------------
-        # For monthly dummies, create a daily col "Month", then get_dummies
-        df_daily["Month"] = df_daily["Date"].dt.month_name().str.lower()
-        df_monthly_dummies = pd.get_dummies(
-            df_daily, 
-            prefix="seas", 
-            columns=["Month"], 
-            dtype=int
-        )
-        # Recalculate 'week_start' (already in df_daily, but just to be sure)
-        df_monthly_dummies['week_start'] = df_daily['week_start']
-
-        # Group monthly dummies by .sum() or .mean()—we often spread them across the week
-        df_monthly_dummies = (
-            df_monthly_dummies
-            .groupby('week_start')
-            .sum(numeric_only=True)    # sum the daily flags
-            .reset_index()
-            .rename(columns={'week_start': "Date"})
-            .set_index("Date")
-        )
-        # Spread monthly dummies by 7 to distribute across that week
-        monthly_cols = [c for c in df_monthly_dummies.columns if c.startswith("seas_month_")]
-        df_monthly_dummies[monthly_cols] = df_monthly_dummies[monthly_cols] / 7
-
-        # Group holiday & special-day columns by .max() => binary at weekly level
-        df_holidays = (
-            df_daily
-            .groupby('week_start')
-            .max(numeric_only=True)   # if any day=1 in that week, entire week=1
-            .reset_index()
-            .rename(columns={'week_start': "Date"})
-            .set_index("Date")
-        )
 
-        # ---------------------------------------------------------------------
-        # 6. Combine weekly start, monthly dummies, holiday flags
-        # ---------------------------------------------------------------------
-        df_combined = pd.concat([df_weekly_start, df_monthly_dummies], axis=1)
-        df_combined = pd.concat([df_combined, df_holidays], axis=1)
-        df_combined = df_combined.loc[:, ~df_combined.columns.duplicated()]
-
-        # ---------------------------------------------------------------------
-        # 7. Create weekly dummies for Week of Year & yearly dummies
-        # ---------------------------------------------------------------------
-        df_combined.reset_index(inplace=True)
-        df_combined.rename(columns={"index": "old_index"}, inplace=True)  # just in case
-        
-        df_combined["Week"] = df_combined["Date"].dt.isocalendar().week
-        df_combined = pd.get_dummies(df_combined, prefix="seas", columns=["Week"], dtype=int)
-        
-        df_combined["Year"] = df_combined["Date"].dt.year
-        df_combined = pd.get_dummies(df_combined, prefix="seas", columns=["Year"], dtype=int)
-        
-        # ---------------------------------------------------------------------
-        # 8. Add constant & trend
-        # ---------------------------------------------------------------------
-        df_combined["Constant"] = 1
-        df_combined["Trend"] = df_combined.index + 1
-        
-        # ---------------------------------------------------------------------
-        # 9. Rename Date -> OBS and return
-        # ---------------------------------------------------------------------
-        df_combined.rename(columns={"Date": "OBS"}, inplace=True)
-        
-        return df_combined
+            # ---------------------------------------------------------------------
+            # 1.2 Calculate ISO week number for each DAY (for later aggregation)
+            #     Also calculate Year for each DAY to handle year transitions correctly
+            # ---------------------------------------------------------------------
+            df_daily['iso_week_daily'] = df_daily['Date'].dt.isocalendar().week.astype(int)
+            df_daily['iso_year_daily'] = df_daily['Date'].dt.isocalendar().year.astype(int)
+
+
+            # ---------------------------------------------------------------------
+            # 2. Build a weekly index (df_weekly_start) based on unique week_start dates
+            # ---------------------------------------------------------------------
+            df_weekly_start = df_daily[['week_start']].drop_duplicates().sort_values('week_start').reset_index(drop=True)
+            df_weekly_start.rename(columns={'week_start': "Date"}, inplace=True)
+            df_weekly_start.set_index("Date", inplace=True)
+
+            # Create individual weekly dummies (optional, uncomment if needed)
+            # dummy_columns = {}
+            # for i, date_index in enumerate(df_weekly_start.index):
+            #     col_name = f"dum_{date_index.strftime('%Y_%m_%d')}"
+            #     dummy_columns[col_name] = [0] * len(df_weekly_start)
+            #     dummy_columns[col_name][i] = 1
+            # df_dummies = pd.DataFrame(dummy_columns, index=df_weekly_start.index)
+            # df_weekly_start = pd.concat([df_weekly_start, df_dummies], axis=1)
+
+
+            # ---------------------------------------------------------------------
+            # 3. Public holidays (daily) from 'holidays' package + each holiday name
+            # ---------------------------------------------------------------------
+            start_year = start_dt.year
+            end_year = end_dt.year
+            years_range = range(start_year, end_year + 1)
+
+            for country in countries:
+                try:
+                    country_holidays = holidays.CountryHoliday(
+                        country,
+                        years=years_range,
+                        observed=False # Typically you want the actual date, not observed substitute
+                    )
+                    # Handle cases like UK where specific subdivisions might be needed for some holidays
+                    # Example: if country == 'GB': country_holidays.observed = True # If observed are needed
+                except KeyError:
+                    print(f"Warning: Country code '{country}' not found in holidays library. Skipping.")
+                    continue # Skip to next country
+
+                # Daily indicator: 1 if that date is a holiday
+                df_daily[f"seas_holiday_{country.lower()}"] = df_daily["Date"].apply(
+                    lambda x: 1 if x in country_holidays else 0
+                )
+                # Create columns for specific holiday names
+                for date_hol, name in sorted(country_holidays.items()): # Sort for consistent column order
+                    # Clean name: lower, replace space with underscore, remove non-alphanumeric (except underscore)
+                    clean_name = ''.join(c for c in name if c.isalnum() or c == ' ').strip().replace(' ', '_').lower()
+                    clean_name = clean_name.replace('_(observed)', '').replace("'", "") # specific cleaning
+                    col_name = f"seas_{clean_name}_{country.lower()}"
+
+                    # Only create column if the holiday occurs within the df_daily date range
+                    if pd.Timestamp(date_hol).year in years_range:
+                        if col_name not in df_daily.columns:
+                            df_daily[col_name] = 0
+                        # Ensure date_hol is within the actual daily range before assigning
+                        if (pd.Timestamp(date_hol) >= df_daily["Date"].min()) and (pd.Timestamp(date_hol) <= df_daily["Date"].max()):
+                            df_daily.loc[df_daily["Date"] == pd.Timestamp(date_hol), col_name] = 1
+
+            # ---------------------------------------------------------------------
+            # 3.1 Additional Special Days (Father's Day, Mother's Day, etc.)
+            # ---------------------------------------------------------------------
+            extra_cols = [
+                "seas_valentines_day",
+                "seas_halloween",
+                "seas_fathers_day_us_uk", # Note: UK/US is 3rd Sun Jun, others vary
+                "seas_mothers_day_us",    # Note: US is 2nd Sun May
+                "seas_mothers_day_uk",    # Note: UK Mothering Sunday varies with Easter
+                "seas_good_friday",
+                "seas_easter_monday",
+                "seas_black_friday",      # US-centric, but globally adopted
+                "seas_cyber_monday",      # US-centric, but globally adopted
+            ]
+            for c in extra_cols:
+                if c not in df_daily.columns: # Avoid overwriting if already created by holidays pkg
+                    df_daily[c] = 0
+
+            # Helper: nth_weekday_of_month(year, month, weekday, nth)
+            def nth_weekday_of_month(year, month, weekday, nth):
+                d = datetime(year, month, 1)
+                w = d.weekday()
+                delta = (weekday - w + 7) % 7 # Ensure positive delta
+                first_weekday = d + timedelta(days=delta)
+                target_date = first_weekday + timedelta(days=7 * (nth - 1))
+                # Check if the calculated date is still in the same month
+                if target_date.month == month:
+                    return target_date
+                else:
+                    # This can happen if nth is too large (e.g., 5th Friday)
+                    # Return the last occurrence of that weekday in the month instead
+                    return target_date - timedelta(days=7)
+
+
+            def get_good_friday(year):
+                return easter(year) - timedelta(days=2)
+
+            def get_easter_monday(year):
+                return easter(year) + timedelta(days=1)
+
+            def get_black_friday(year):
+                # US Thanksgiving is 4th Thursday in November (weekday=3)
+                thanksgiving = nth_weekday_of_month(year, 11, 3, 4)
+                return thanksgiving + timedelta(days=1)
+
+            def get_cyber_monday(year):
+                # Monday after US Thanksgiving
+                thanksgiving = nth_weekday_of_month(year, 11, 3, 4)
+                return thanksgiving + timedelta(days=4)
+
+            def get_mothering_sunday_uk(year):
+                # Fourth Sunday in Lent (3 weeks before Easter Sunday)
+                # Lent starts on Ash Wednesday, 46 days before Easter.
+                # Easter Sunday is day 0. Sunday before is -7, etc.
+                # 4th Sunday in Lent is 3 weeks before Easter.
+                return easter(year) - timedelta(days=21)
+
+
+            # Loop over each year in range
+            for yr in range(start_year, end_year + 1):
+                try: # Wrap calculations in try-except for robustness
+                    # Valentines = Feb 14
+                    valentines_day = datetime(yr, 2, 14)
+                    # Halloween = Oct 31
+                    halloween_day  = datetime(yr, 10, 31)
+                    # Father's Day (US & UK) = 3rd Sunday (6) in June
+                    fathers_day    = nth_weekday_of_month(yr, 6, 6, 3)
+                    # Mother's Day US = 2nd Sunday (6) in May
+                    mothers_day_us = nth_weekday_of_month(yr, 5, 6, 2)
+                    # Mother's Day UK (Mothering Sunday)
+                    mothering_sunday = get_mothering_sunday_uk(yr)
+
+                    # Good Friday, Easter Monday
+                    gf = get_good_friday(yr)
+                    em = get_easter_monday(yr)
+
+                    # Black Friday, Cyber Monday
+                    bf = get_black_friday(yr)
+                    cm = get_cyber_monday(yr)
+
+                    # Mark them in df_daily if in range
+                    special_days_map = [
+                        (valentines_day, "seas_valentines_day"),
+                        (halloween_day,  "seas_halloween"),
+                        (fathers_day,    "seas_fathers_day_us_uk"),
+                        (mothers_day_us, "seas_mothers_day_us"),
+                        (mothering_sunday,"seas_mothers_day_uk"),
+                        (gf, "seas_good_friday"),
+                        (em, "seas_easter_monday"),
+                        (bf, "seas_black_friday"),
+                        (cm, "seas_cyber_monday"),
+                    ]
+
+                    for special_date, col in special_days_map:
+                        if special_date is not None: # nth_weekday_of_month can return None edge cases
+                            special_ts = pd.Timestamp(special_date)
+                            # Only set if it's within the daily range AND column exists
+                            if (special_ts >= df_daily["Date"].min()) and \
+                            (special_ts <= df_daily["Date"].max()) and \
+                            (col in df_daily.columns):
+                                df_daily.loc[df_daily["Date"] == special_ts, col] = 1
+                except Exception as e:
+                    print(f"Warning: Could not calculate special days for year {yr}: {e}")
+
+
+            # ---------------------------------------------------------------------
+            # 4. Add daily indicators for last day & last Friday of month
+            # ---------------------------------------------------------------------
+            df_daily["is_last_day_of_month"] = df_daily["Date"].dt.is_month_end
+
+            def is_last_friday(date):
+                # Check if it's a Friday first
+                if date.weekday() != 4: # Friday is 4
+                    return 0
+                # Check if next Friday is in the next month
+                next_friday = date + timedelta(days=7)
+                return 1 if next_friday.month != date.month else 0
+
+            df_daily["is_last_friday_of_month"] = df_daily["Date"].apply(is_last_friday)
+
+            # Rename for clarity prefix
+            df_daily.rename(columns={
+                "is_last_day_of_month": "seas_last_day_of_month",
+                "is_last_friday_of_month": "seas_last_friday_of_month"
+            }, inplace=True)
+
+
+            # ---------------------------------------------------------------------
+            # 5. Weekly aggregation
+            # ---------------------------------------------------------------------
+
+            # --- Aggregate flags using MAX (1 if any day in week is flagged) ---
+            # Select only columns that are indicators/flags (intended for max aggregation)
+            flag_cols = [col for col in df_daily.columns if col.startswith('seas_') or col.startswith('is_')]
+            # Ensure 'week_start' is present for grouping
+            df_to_agg = df_daily[['week_start'] + flag_cols]
+
+            df_weekly_flags = (
+                df_to_agg
+                .groupby('week_start')
+                .max() # if any day=1 in that week, entire week=1
+                .reset_index()
+                .rename(columns={'week_start': "Date"})
+                .set_index("Date")
+            )
 
+            # --- Aggregate Week Number using MODE ---
+            # Define aggregation function for mode (handling potential multi-modal cases by taking the first)
+            def get_mode(x):
+                modes = pd.Series.mode(x)
+                return modes[0] if not modes.empty else np.nan # Return first mode or NaN
+
+            df_weekly_iso_week_year = (
+                df_daily[['week_start', 'iso_week_daily', 'iso_year_daily']]
+                .groupby('week_start')
+                .agg(
+                    # Find the most frequent week number and year within the group
+                    Week=('iso_week_daily', get_mode),
+                    Year=('iso_year_daily', get_mode)
+                )
+                .reset_index()
+                .rename(columns={'week_start': 'Date'})
+                .set_index('Date')
+            )
+            # Convert Week/Year back to integer type after aggregation
+            df_weekly_iso_week_year['Week'] = df_weekly_iso_week_year['Week'].astype(int)
+            df_weekly_iso_week_year['Year'] = df_weekly_iso_week_year['Year'].astype(int)
+
+
+            # --- Monthly dummies (spread evenly across week) ---
+            df_daily["Month"] = df_daily["Date"].dt.month_name().str.lower()
+            df_monthly_dummies_daily = pd.get_dummies(
+                df_daily[["week_start", "Month"]], # Only need these columns
+                prefix="seas_month",
+                columns=["Month"],
+                dtype=float # Use float for division
+            )
+            # Sum daily dummies within the week
+            df_monthly_dummies_summed = df_monthly_dummies_daily.groupby('week_start').sum()
+            # Divide by number of days in that specific week group (usually 7, except potentially start/end)
+            days_in_week = df_daily.groupby('week_start').size()
+            df_weekly_monthly_dummies = df_monthly_dummies_summed.div(days_in_week, axis=0)
+
+            # Reset index to merge
+            df_weekly_monthly_dummies.reset_index(inplace=True)
+            df_weekly_monthly_dummies.rename(columns={'week_start': 'Date'}, inplace=True)
+            df_weekly_monthly_dummies.set_index('Date', inplace=True)
+
+
+            # ---------------------------------------------------------------------
+            # 6. Combine all weekly components
+            # ---------------------------------------------------------------------
+            # Start with the basic weekly index
+            df_combined = df_weekly_start.copy()
+
+            # Join the other aggregated DataFrames
+            df_combined = df_combined.join(df_weekly_flags, how='left')
+            df_combined = df_combined.join(df_weekly_iso_week_year, how='left')
+            df_combined = df_combined.join(df_weekly_monthly_dummies, how='left')
+
+            # Fill potential NaNs created by joins (e.g., if a flag column didn't exist) with 0
+            # Exclude 'Week' and 'Year' which should always be present
+            cols_to_fill = df_combined.columns.difference(['Week', 'Year'])
+            df_combined[cols_to_fill] = df_combined[cols_to_fill].fillna(0)
+
+            # Ensure correct types for flag columns (int)
+            for col in df_weekly_flags.columns:
+                if col in df_combined.columns:
+                    df_combined[col] = df_combined[col].astype(int)
+
+            # Ensure correct types for month columns (float)
+            for col in df_weekly_monthly_dummies.columns:
+                if col in df_combined.columns:
+                    df_combined[col] = df_combined[col].astype(float)
+
+
+            # ---------------------------------------------------------------------
+            # 7. Create weekly dummies for Week of Year & yearly dummies from aggregated cols
+            # ---------------------------------------------------------------------
+            df_combined.reset_index(inplace=True) # 'Date', 'Week', 'Year' become columns
+
+            # Create dummies from the aggregated 'Week' column
+            df_combined = pd.get_dummies(df_combined, prefix="seas", columns=["Week"], dtype=int, prefix_sep='_')
+
+            # Create dummies from the aggregated 'Year' column
+            df_combined = pd.get_dummies(df_combined, prefix="seas", columns=["Year"], dtype=int, prefix_sep='_')
+
+            # ---------------------------------------------------------------------
+            # 8. Add constant & trend
+            # ---------------------------------------------------------------------
+            df_combined["Constant"] = 1
+            df_combined.reset_index(drop=True, inplace=True) # Ensure index is 0, 1, 2... for trend
+            df_combined["Trend"] = df_combined.index + 1
+
+            # ---------------------------------------------------------------------
+            # 9. Rename Date -> OBS and select final columns
+            # ---------------------------------------------------------------------
+            df_combined.rename(columns={"Date": "OBS"}, inplace=True)
+
+            # Reorder columns - OBS first, then Constant, Trend, then seasonal features
+            cols_order = ['OBS', 'Constant', 'Trend'] + \
+                        sorted([col for col in df_combined.columns if col.startswith('seas_')]) + \
+                        sorted([col for col in df_combined.columns if col.startswith('dum_')]) # If individual week dummies were enabled
+
+            # Filter out columns not in the desired order list (handles case where dum_ cols are off)
+            final_cols = [col for col in cols_order if col in df_combined.columns]
+            df_combined = df_combined[final_cols]
+
+            return df_combined
+    
     def pull_weather(self, week_commencing, start_date, country_codes) -> pd.DataFrame:
         """
         Pull weather data for a given week-commencing day and one or more country codes.
@@ -1171,20 +1274,22 @@ class datapull:
         
     def pull_macro_ons_uk(self, cdid_list=None, week_start_day="mon", sector=None):
         """
-        Fetches time series data for multiple CDIDs from the ONS API, converts it to daily frequency, 
+        Fetches time series data for multiple CDIDs from the ONS API, converts it to daily frequency,
         aggregates it to weekly averages, and renames variables based on specified rules.
 
         Parameters:
-            cdid_list (list): A list of additional CDIDs to fetch (e.g., ['JP9Z', 'UKPOP']). Defaults to None.
-            week_start_day (str): The day the week starts on (e.g., 'Monday', 'Sunday').
-            sector (str): The sector for which the standard CDIDs are fetched (e.g., 'fast_food', 'retail').
+            cdid_list (list, optional): A list of additional CDIDs to fetch (e.g., ['JP9Z', 'UKPOP']). Defaults to None.
+            week_start_day (str, optional): The day the week starts on ('mon', 'tue', 'wed', 'thu', 'fri', 'sat', 'sun'). Defaults to 'mon'.
+            sector (str or list, optional): The sector(s) for which the standard CDIDs are fetched
+                                             (e.g., 'fast_food', ['fast_food', 'retail']). Defaults to None (only default CDIDs).
 
         Returns:
-            pd.DataFrame: A DataFrame with weekly frequency, containing a 'week_commencing' column 
-                        and all series as renamed columns.
+            pd.DataFrame: A DataFrame with weekly frequency, containing an 'OBS' column (week commencing date)
+                          and all series as renamed columns (e.g., 'macro_retail_sales_uk').
+                          Returns an empty DataFrame if no data is fetched or processed.
         """
         # Define CDIDs for sectors and defaults
-        sector_cdids = {
+        sector_cdids_map = {
             "fast_food": ["L7TD", "L78Q", "DOAD"],
             "clothing_footwear": ["D7BW","D7GO","CHBJ"],
             "fuel": ["A9FS","L7FP","CHOL"],
@@ -1192,14 +1297,29 @@ class datapull:
             "default": ["D7G7", "MGSX", "UKPOP", "IHYQ", "YBEZ", "MS77"],
         }
 
-        default_cdids = sector_cdids["default"]
-        sector_specific_cdids = sector_cdids.get(sector, [])
-        standard_cdids = list(set(default_cdids + sector_specific_cdids))  # Avoid duplicates
+        default_cdids = sector_cdids_map["default"]
+        sector_specific_cdids = [] # Initialize empty list for sector CDIDs
 
-        # Combine standard CDIDs and additional CDIDs
+        if sector: # Check if sector is not None or empty
+            if isinstance(sector, str):
+                # If it's a single string, wrap it in a list
+                sector_list = [sector]
+            elif isinstance(sector, list):
+                # If it's already a list, use it directly
+                sector_list = sector
+            else:
+                raise TypeError("`sector` parameter must be a string or a list of strings.")
+
+            # Iterate through the list of sectors and collect their CDIDs
+            for sec in sector_list:
+                sector_specific_cdids.extend(sector_cdids_map.get(sec, [])) # Use extend to add items from the list
+
+        standard_cdids = list(set(default_cdids + sector_specific_cdids))  # Combine default and selected sector CDIDs, ensure uniqueness
+
+        # Combine standard CDIDs and any additional user-provided CDIDs
         if cdid_list is None:
             cdid_list = []
-        cdid_list = list(set(standard_cdids + cdid_list))  # Avoid duplicates
+        final_cdid_list = list(set(standard_cdids + cdid_list))  # Ensure uniqueness in the final list
 
         base_search_url = "https://api.beta.ons.gov.uk/v1/search?content_type=timeseries&cdids="
         base_data_url = "https://api.beta.ons.gov.uk/v1/data?uri="
@@ -1207,41 +1327,57 @@ class datapull:
 
         # Map week start day to pandas weekday convention
         days_map = {"mon": 0, "tue": 1, "wed": 2, "thu": 3, "fri": 4, "sat": 5, "sun": 6}
-        if week_start_day not in days_map:
+        if week_start_day.lower() not in days_map:
             raise ValueError("Invalid week start day. Choose from: " + ", ".join(days_map.keys()))
-        week_start = days_map[week_start_day]
+        week_start = days_map[week_start_day.lower()] # Use lower() for case-insensitivity
 
-        for cdid in cdid_list:
+        for cdid in final_cdid_list: # Use the final combined list
             try:
                 # Search for the series
                 search_url = f"{base_search_url}{cdid}"
-                search_response = requests.get(search_url)
+                search_response = requests.get(search_url, timeout=30) # Add timeout
                 search_response.raise_for_status()
                 search_data = search_response.json()
 
                 items = search_data.get("items", [])
                 if not items:
-                    print(f"No data found for CDID: {cdid}")
+                    print(f"Warning: No data found for CDID: {cdid}")
                     continue
 
                 # Extract series name and latest release URI
-                series_name = items[0].get("title", f"Series_{cdid}")
-                latest_date = max(
-                    datetime.fromisoformat(item["release_date"].replace("Z", "+00:00"))
-                    for item in items if "release_date" in item
-                )
-                latest_uri = next(
-                    item["uri"] for item in items
-                    if "release_date" in item and datetime.fromisoformat(item["release_date"].replace("Z", "+00:00")) == latest_date
-                )
+                # Find the item with the most recent release_date
+                latest_item = None
+                latest_date = None
+                for item in items:
+                    if "release_date" in item:
+                        try:
+                            # Ensure timezone awareness for comparison
+                            current_date = datetime.fromisoformat(item["release_date"].replace("Z", "+00:00"))
+                            if latest_date is None or current_date > latest_date:
+                                latest_date = current_date
+                                latest_item = item
+                        except ValueError:
+                            print(f"Warning: Could not parse release_date '{item['release_date']}' for CDID {cdid}")
+                            continue # Skip this item if date is invalid
+
+                if latest_item is None:
+                     print(f"Warning: No valid release date found for CDID: {cdid}")
+                     continue
+
+                series_name = latest_item.get("title", f"Series_{cdid}") # Use title from the latest item
+                latest_uri = latest_item.get("uri")
+                if not latest_uri:
+                     print(f"Warning: No URI found for the latest release of CDID: {cdid}")
+                     continue
 
                 # Fetch the dataset
                 data_url = f"{base_data_url}{latest_uri}"
-                data_response = requests.get(data_url)
+                data_response = requests.get(data_url, timeout=30) # Add timeout
                 data_response.raise_for_status()
                 data_json = data_response.json()
 
                 # Detect the frequency and process accordingly
+                frequency_key = None
                 if "months" in data_json and data_json["months"]:
                     frequency_key = "months"
                 elif "quarters" in data_json and data_json["quarters"]:
@@ -1249,72 +1385,142 @@ class datapull:
                 elif "years" in data_json and data_json["years"]:
                     frequency_key = "years"
                 else:
-                    print(f"Unsupported frequency or no data for CDID: {cdid}")
+                    print(f"Warning: Unsupported frequency or no data values found for CDID: {cdid} at URI {latest_uri}")
                     continue
 
                 # Prepare the DataFrame
+                if not data_json[frequency_key]: # Check if the list of values is empty
+                     print(f"Warning: Empty data list for frequency '{frequency_key}' for CDID: {cdid}")
+                     continue
+
                 df = pd.DataFrame(data_json[frequency_key])
 
-                # Parse the 'date' field based on frequency
-                if frequency_key == "months":
-                    df["date"] = pd.to_datetime(df["date"], format="%Y %b", errors="coerce")
-                elif frequency_key == "quarters":
-                    def parse_quarter(quarter_str):
-                        year, qtr = quarter_str.split(" Q")
-                        month = {"1": 1, "2": 4, "3": 7, "4": 10}[qtr]
-                        return datetime(int(year), month, 1)
-                    df["date"] = df["date"].apply(parse_quarter)
-                elif frequency_key == "years":
-                    df["date"] = pd.to_datetime(df["date"], format="%Y", errors="coerce")
+                # Check if essential columns exist
+                if "date" not in df.columns or "value" not in df.columns:
+                    print(f"Warning: Missing 'date' or 'value' column for CDID: {cdid}")
+                    continue
 
+                # Parse the 'date' field based on frequency
+                try:
+                    if frequency_key == "months":
+                        # Handles "YYYY Mon" format (e.g., "2023 FEB") - adjust if format differs
+                        df["date"] = pd.to_datetime(df["date"], format="%Y %b", errors="coerce")
+                    elif frequency_key == "quarters":
+                        def parse_quarter(quarter_str):
+                            try:
+                                year, qtr = quarter_str.split(" Q")
+                                month = {"1": 1, "2": 4, "3": 7, "4": 10}[qtr]
+                                return datetime(int(year), month, 1)
+                            except (ValueError, KeyError):
+                                return pd.NaT # Return Not a Time for parsing errors
+                        df["date"] = df["date"].apply(parse_quarter)
+                    elif frequency_key == "years":
+                        df["date"] = pd.to_datetime(df["date"], format="%Y", errors="coerce")
+                except Exception as e:
+                    print(f"Error parsing date for CDID {cdid} with frequency {frequency_key}: {e}")
+                    continue # Skip this series if date parsing fails
+
+                # Coerce value to numeric, handle potential errors
                 df["value"] = pd.to_numeric(df["value"], errors="coerce")
+
+                # Drop rows where date or value parsing failed
+                df.dropna(subset=["date", "value"], inplace=True)
+
+                if df.empty:
+                    print(f"Warning: No valid data points after processing for CDID: {cdid}")
+                    continue
+
                 df.rename(columns={"value": series_name}, inplace=True)
 
                 # Combine data
-                df = df.loc[:, ["date", series_name]].dropna().reset_index(drop=True)
+                df_subset = df.loc[:, ["date", series_name]].reset_index(drop=True) # Explicitly select columns
                 if combined_df.empty:
-                    combined_df = df
+                    combined_df = df_subset
                 else:
-                    combined_df = pd.merge(combined_df, df, on="date", how="outer")
+                    # Use outer merge to keep all dates, sort afterwards
+                    combined_df = pd.merge(combined_df, df_subset, on="date", how="outer")
 
             except requests.exceptions.RequestException as e:
                 print(f"Error fetching data for CDID {cdid}: {e}")
-            except (KeyError, ValueError) as e:
+            except (KeyError, ValueError, TypeError) as e: # Added TypeError
                 print(f"Error processing data for CDID {cdid}: {e}")
+            except Exception as e: # Catch unexpected errors
+                 print(f"An unexpected error occurred for CDID {cdid}: {e}")
+
 
         if not combined_df.empty:
+            # Sort by date after merging to ensure correct forward fill
+            combined_df.sort_values(by="date", inplace=True)
+            combined_df.reset_index(drop=True, inplace=True)
+
+            # Create a complete daily date range
             min_date = combined_df["date"].min()
-            max_date = datetime.today()
+            # Ensure max_date is timezone-naive if min_date is, or consistent otherwise
+            max_date = pd.Timestamp(datetime.today().date()) # Use today's date, timezone-naive
+
+            if pd.isna(min_date):
+                 print("Error: Minimum date is NaT, cannot create date range.")
+                 return pd.DataFrame()
+
+            # Make sure min_date is not NaT before creating the range
             date_range = pd.date_range(start=min_date, end=max_date, freq='D')
             daily_df = pd.DataFrame(date_range, columns=['date'])
+
+            # Merge with original data and forward fill
             daily_df = pd.merge(daily_df, combined_df, on="date", how="left")
             daily_df = daily_df.ffill()
 
+            # Drop rows before the first valid data point after ffill
+            first_valid_index = daily_df.dropna(subset=daily_df.columns.difference(['date'])).index.min()
+            if pd.notna(first_valid_index):
+                 daily_df = daily_df.loc[first_valid_index:]
+            else:
+                 print("Warning: No valid data points found after forward filling.")
+                 return pd.DataFrame() # Return empty if ffill results in no data
+
+
             # Aggregate to weekly frequency
-            daily_df["week_commencing"] = daily_df["date"] - pd.to_timedelta((daily_df["date"].dt.weekday - week_start) % 7, unit='D')
+            # Ensure 'date' column is datetime type before dt accessor
+            daily_df['date'] = pd.to_datetime(daily_df['date'])
+            daily_df["week_commencing"] = daily_df["date"] - pd.to_timedelta((daily_df["date"].dt.weekday - week_start + 7) % 7, unit='D') # Corrected logic for week start
+            # Group by week_commencing and calculate mean for numeric columns only
             weekly_df = daily_df.groupby("week_commencing").mean(numeric_only=True).reset_index()
 
+
             def clean_column_name(name):
+                # Remove content within parentheses (e.g., CPI INDEX 00: ALL ITEMS 2015=100)
                 name = re.sub(r"\(.*?\)", "", name)
+                # Take only the part before the first colon if present
                 name = re.split(r":", name)[0]
-                name = re.sub(r"\d+", "", name)
+                # Remove digits
+                #name = re.sub(r"\d+", "", name) # Reconsider removing all digits, might be needed for some series
+                # Remove specific words like 'annual', 'rate' case-insensitively
                 name = re.sub(r"\b(annual|rate)\b", "", name, flags=re.IGNORECASE)
+                # Remove non-alphanumeric characters (except underscore and space)
                 name = re.sub(r"[^\w\s]", "", name)
+                # Replace spaces with underscores
+                name = name.strip() # Remove leading/trailing whitespace
                 name = name.replace(" ", "_")
+                # Replace multiple underscores with a single one
                 name = re.sub(r"_+", "_", name)
+                # Remove trailing underscores
                 name = name.rstrip("_")
+                # Add prefix and suffix
                 return f"macro_{name.lower()}_uk"
 
+            # Apply cleaning function to relevant columns
             weekly_df.columns = [clean_column_name(col) if col != "week_commencing" else col for col in weekly_df.columns]
-            weekly_df.rename(columns={"week_commencing": "OBS"}, inplace=True)
+            weekly_df.rename(columns={"week_commencing": "OBS"}, inplace=True) # Rename week commencing col
 
-            weekly_df = weekly_df.fillna(0)
+            # Optional: Fill remaining NaNs (e.g., at the beginning if ffill didn't cover) with 0
+            # Consider if 0 is the appropriate fill value for your use case
+            # weekly_df = weekly_df.fillna(0)
 
             return weekly_df
         else:
-            print("No data available to process.")
+            print("No data successfully fetched or processed.")
             return pd.DataFrame()
-
+        
     def pull_yfinance(self, tickers=None, week_start_day="mon"):
         """
         Fetches stock data for multiple tickers from Yahoo Finance, converts it to daily frequency,